From a667db0320871d4619ccf4dad9586793c1eb5891 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 26 Apr 2023 20:57:11 +0100 Subject: [PATCH 001/740] ci: Add timeouts and fail handling on some hanging actions (#26996) Signed-off-by: Ryan Northey --- .azure-pipelines/pipelines.yml | 2 ++ .azure-pipelines/stage/publish.yml | 9 ++++++++- .azure-pipelines/stages.yml | 5 +++++ ci/docker_ci.sh | 14 +++++++++++++- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 70ceb234e249..e7bbb8c78070 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -84,6 +84,8 @@ stages: - ${{ if eq(variables.pipelinePostsubmit, true) }}: - template: stages.yml parameters: + # Build/publish can take longer in postsubmit, esp during release + timeoutDockerBuild: 720 timeoutDockerPublish: 30 buildStageDeps: - env diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index b1fbe46d7bc2..e4f6102d285d 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -6,10 +6,16 @@ parameters: default: "" ## Timeouts +# Workaround for https://github.com/envoyproxy/envoy/issues/26634 - name: timeoutDockerPublish displayName: "Timout Docker publish" type: number - default: 10 + # in seconds + default: 15 +- name: timeoutDockerBuild + displayName: "Timout Docker build" + type: number + default: 400 # Auth - name: authGCP @@ -121,6 +127,7 @@ jobs: AZP_SHA1: $(Build.SourceVersion) DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} + DOCKER_BUILD_TIMEOUT: ${{ parameters.timeoutDockerBuild }} - bash: | echo "disk space at end of build:" df -h diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index 3195bc08384a..c6d5348f6aa0 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -40,6 +40,10 @@ parameters: displayName: "Timout Docker publish" type: number default: 10 +- name: timeoutDockerBuild + displayName: "Timout Docker build" + type: number + default: 400 ## Build settings - name: cacheTestResults @@ -146,6 +150,7 @@ stages: authSSHDocsKey: $(DocsPrivateKey) authSSHKeyPassphrase: $(SshDeployKeyPassphrase) bucketGCP: $(GcsArtifactBucket) + timeoutDockerBuild: ${{ parameters.timeoutDockerBuild }} timeoutDockerPublish: ${{ parameters.timeoutDockerPublish }} runDocker: variables['RUN_DOCKER'] runPackaging: variables['RUN_PACKAGING'] diff --git a/ci/docker_ci.sh b/ci/docker_ci.sh index c07c895f57c8..136676cf0955 100755 --- a/ci/docker_ci.sh +++ b/ci/docker_ci.sh @@ -20,6 +20,9 @@ set -e # AZP_BRANCH=refs/tags/v1.77.3 ## +# Workaround for https://github.com/envoyproxy/envoy/issues/26634 +DOCKER_BUILD_TIMEOUT="${DOCKER_BUILD_TIMEOUT:-400}" + function is_windows() { [[ -n "$DOCKER_FAKE_WIN" ]] || [[ "$(uname -s)" == *NT* ]] @@ -232,7 +235,16 @@ build_and_maybe_push_image () { if [[ -z "$DOCKER_CI_DRYRUN" ]]; then echo "..." - docker "${docker_build_args[@]}" + timeout "$DOCKER_BUILD_TIMEOUT" docker "${docker_build_args[@]}" || { + if [[ "$?" == 124 ]]; then + echo "Docker build timed out ..." >&2 + else + echo "Docker build errored ..." >&2 + fi + sleep 5 + echo "trying again ..." >&2 + docker "${docker_build_args[@]}" + } fi if [[ -z "$PUSH_IMAGES_TO_REGISTRY" ]]; then return From 6d6661caa18007375dea0cfae65c041df40726fd Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 26 Apr 2023 19:21:33 -0400 Subject: [PATCH 002/740] runtime: remove yaml from the critical path (#26451) Avoiding a yaml translation step when loading runtime . This also allows turning up e2e !yaml testing for Envoy Mobile (with limited API calls) so adding in regression testing via CI as well. Risk Level: high Testing: existing tests, new unit tests Docs Changes: n/a Release Notes: n/a part of #26450 Signed-off-by: Alyssa Wilk --- .../workflows/mobile-compile_time_options.yml | 12 +- mobile/library/cc/BUILD | 2 +- mobile/library/common/jni/BUILD | 2 +- source/common/runtime/runtime_impl.cc | 159 +++++++++++------- source/common/runtime/runtime_impl.h | 24 +-- test/common/runtime/runtime_impl_test.cc | 24 ++- test/integration/utility.cc | 19 ++- 7 files changed, 152 insertions(+), 90 deletions(-) diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index 14f42ea48350..cdf3c7178221 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -112,4 +112,14 @@ jobs: --define=google_grpc=disabled \ --define=envoy_yaml=disabled \ --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //:android_dist + //:android_dist && ./bazelw test \ + --fat_apk_cpu=x86_64 \ + --define=admin_html=enabled \ + --define=envoy_mobile_request_compression=disabled \ + --define=envoy_enable_http_datagrams=disabled \ + --define=google_grpc=disabled \ + --define=envoy_yaml=disabled \ + --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ + //test/java/integration:android_engine_socket_tag_test \ + //test/java/integration:android_engine_start_test \ + //test/java/io/envoyproxy/envoymobile/utilities:certificate_verification_tests diff --git a/mobile/library/cc/BUILD b/mobile/library/cc/BUILD index 7ab6e92eaaf1..193e6b40857b 100644 --- a/mobile/library/cc/BUILD +++ b/mobile/library/cc/BUILD @@ -22,7 +22,6 @@ envoy_cc_library( deps = [ ":envoy_engine_cc_lib_no_stamp", "@envoy//source/common/common:assert_lib", - "@envoy_mobile//library/common/types:matcher_data_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/compression/brotli/decompressor/v3:pkg_cc_proto", @@ -41,6 +40,7 @@ envoy_cc_library( "@envoy_mobile//library/common/extensions/filters/http/local_error:filter_cc_proto", "@envoy_mobile//library/common/extensions/filters/http/network_configuration:filter_cc_proto", "@envoy_mobile//library/common/extensions/filters/http/socket_tag:filter_cc_proto", + "@envoy_mobile//library/common/types:matcher_data_lib", ] + envoy_select_envoy_mobile_request_compression( [ "@envoy_api//envoy/extensions/compression/brotli/compressor/v3:pkg_cc_proto", diff --git a/mobile/library/common/jni/BUILD b/mobile/library/common/jni/BUILD index 0c2e65535003..9377e59a6f8b 100644 --- a/mobile/library/common/jni/BUILD +++ b/mobile/library/common/jni/BUILD @@ -171,8 +171,8 @@ cc_library( "android_jni_utility.h", ], deps = [ - "@envoy//source/common/common:assert_lib", "//library/common/types:c_types_lib", + "@envoy//source/common/common:assert_lib", ] + select({ "@envoy//bazel:android": [ ":jni_support_lib", diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 9ef8730bbc46..0b634d96bee2 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -222,88 +222,126 @@ SnapshotImpl::SnapshotImpl(Random::RandomGenerator& generator, RuntimeStats& sta stats.num_keys_.set(values_.size()); } -SnapshotImpl::Entry SnapshotImpl::createEntry(const std::string& value) { - Entry entry; - entry.raw_string_value_ = value; - - // As a perf optimization, attempt to parse the entry's string and store it inside the struct. If - // we don't succeed that's fine. - resolveEntryType(entry); - - return entry; -} +void parseFractionValue(SnapshotImpl::Entry& entry, const ProtobufWkt::Struct& value) { + envoy::type::v3::FractionalPercent percent; + static_assert(envoy::type::v3::FractionalPercent::MILLION == + envoy::type::v3::FractionalPercent::DenominatorType_MAX); + percent.set_denominator(envoy::type::v3::FractionalPercent::HUNDRED); + for (const auto& f : value.fields()) { + if (f.first == "numerator") { + if (f.second.has_number_value()) { + percent.set_numerator(f.second.number_value()); + } + } else if (f.first == "denominator" && f.second.has_string_value()) { + if (f.second.string_value() == "HUNDRED") { + percent.set_denominator(envoy::type::v3::FractionalPercent::HUNDRED); + } else if (f.second.string_value() == "TEN_THOUSAND") { + percent.set_denominator(envoy::type::v3::FractionalPercent::TEN_THOUSAND); + } else if (f.second.string_value() == "MILLION") { + percent.set_denominator(envoy::type::v3::FractionalPercent::MILLION); + } else { + return; + } + } else { + return; + } + } -SnapshotImpl::Entry SnapshotImpl::createEntry(const ProtobufWkt::Value& value) { - // This isn't the smartest way to do it; we're round-tripping via YAML, this should be optimized - // if runtime parsing becomes performance sensitive. -#ifdef ENVOY_ENABLE_YAML - return createEntry(MessageUtil::getYamlStringFromMessage(value, false, false)); -#else - IS_ENVOY_BUG("Runtime loading requires YAML support"); - UNREFERENCED_PARAMETER(value); - return Entry(); -#endif + entry.fractional_percent_value_ = percent; } -bool SnapshotImpl::parseEntryBooleanValue(Entry& entry) { - absl::string_view stripped = entry.raw_string_value_; - stripped = absl::StripAsciiWhitespace(stripped); - - uint64_t parse_int; - if (absl::SimpleAtoi(stripped, &parse_int)) { - entry.bool_value_ = (parse_int != 0); - // This is really an integer, so return false here not because of failure, but so we continue to - // parse doubles/int. - return false; - } else if (absl::EqualsIgnoreCase(stripped, "true")) { - entry.bool_value_ = true; - return true; - } else if (absl::EqualsIgnoreCase(stripped, "false")) { - entry.bool_value_ = false; - return true; +void setNumberValue(Envoy::Runtime::Snapshot::Entry& entry, double value) { + entry.double_value_ = value; + if (value < std::numeric_limits::max() && value == static_cast(value)) { + entry.bool_value_ = value != 0; + } + if (entry.double_value_ >= 0 && entry.double_value_ <= std::numeric_limits::max()) { + // Valid uint values will always be parseable as doubles, so we assign the value to both the + // uint and double fields. In cases where the value is something like "3.1", we will floor the + // number by casting it to a uint and assigning the uint value. + entry.uint_value_ = entry.double_value_; } - return false; } -bool SnapshotImpl::parseEntryDoubleValue(Entry& entry) { +// Handle corner cases in parsing: negatives and decimals aren't always parsed as doubles. +void parseEntryDoubleValue(Envoy::Runtime::Snapshot::Entry& entry) { double converted_double; if (absl::SimpleAtod(entry.raw_string_value_, &converted_double)) { - entry.double_value_ = converted_double; - return true; + setNumberValue(entry, converted_double); } - return false; } -void SnapshotImpl::parseEntryFractionalPercentValue(Entry& entry) { - envoy::type::v3::FractionalPercent converted_fractional_percent; - TRY_ASSERT_MAIN_THREAD { -#ifdef ENVOY_ENABLE_YAML - MessageUtil::loadFromYamlAndValidate(entry.raw_string_value_, converted_fractional_percent, - ProtobufMessage::getStrictValidationVisitor()); -#endif +// Handle an absolutely awful corner case where we explicitly shove a yaml percent in a proto string +// value. +void parseEntryFractionalPercentValue(Envoy::Runtime::Snapshot::Entry& entry) { + if (!absl::StrContains(entry.raw_string_value_, "numerator:")) { + return; } + envoy::type::v3::FractionalPercent converted_fractional_percent; + TRY_ASSERT_MAIN_THREAD { entry.fractional_percent_value_ = converted_fractional_percent; } END_TRY catch (const ProtoValidationException& ex) { - ENVOY_LOG(error, "unable to validate fraction percent runtime proto: {}", ex.what()); return; } - catch (const EnvoyException& ex) { - // An EnvoyException is thrown when we try to parse a bogus string as a protobuf. This is fine, - // since there was no expectation that the raw string was a valid proto. - return; +} + +// Handle corner cases in non-yaml parsing: mixed case strings aren't parsed as booleans. +void parseEntryBooleanValue(Envoy::Runtime::Snapshot::Entry& entry) { + absl::string_view stripped = entry.raw_string_value_; + stripped = absl::StripAsciiWhitespace(stripped); + + if (absl::EqualsIgnoreCase(stripped, "true")) { + entry.bool_value_ = true; + } else if (absl::EqualsIgnoreCase(stripped, "false")) { + entry.bool_value_ = false; + } +} + +SnapshotImpl::Entry SnapshotImpl::createEntry(const ProtobufWkt::Value& value, + absl::string_view raw_string) { + Entry entry; + switch (value.kind_case()) { + case ProtobufWkt::Value::kNumberValue: + setNumberValue(entry, value.number_value()); + break; + case ProtobufWkt::Value::kBoolValue: + entry.bool_value_ = value.bool_value(); + break; + case ProtobufWkt::Value::kStructValue: + parseFractionValue(entry, value.struct_value()); + break; + case ProtobufWkt::Value::kStringValue: + entry.raw_string_value_ = value.string_value(); + parseEntryDoubleValue(entry); + // TODO(alyssawilk) after this PR lands and sticks, ENVOY_BUG these + // functions and see if we can remove the special casing. + parseEntryBooleanValue(entry); + parseEntryFractionalPercentValue(entry); + if (!raw_string.empty()) { + entry.raw_string_value_ = raw_string; + } + default: + break; } - entry.fractional_percent_value_ = converted_fractional_percent; + return entry; } void AdminLayer::mergeValues(const absl::node_hash_map& values) { +#ifdef ENVOY_ENABLE_YAML for (const auto& kv : values) { values_.erase(kv.first); if (!kv.second.empty()) { - values_.emplace(kv.first, SnapshotImpl::createEntry(kv.second)); + values_.emplace(kv.first, + SnapshotImpl::createEntry(ValueUtil::loadFromYaml(kv.second), kv.second)); } } stats_.admin_overrides_active_.set(values_.empty() ? 0 : 1); +#else + IS_ENVOY_BUG("Runtime admin reload requires YAML support"); + UNREFERENCED_PARAMETER(values); + return; +#endif } DiskLayer::DiskLayer(absl::string_view name, const std::string& path, Api::Api& api) @@ -363,7 +401,14 @@ void DiskLayer::walkDirectory(const std::string& path, const std::string& prefix // Separate erase/insert calls required due to the value type being constant; this prevents // the use of the [] operator. Can leverage insert_or_assign in C++17 in the future. values_.erase(full_prefix); - values_.insert({full_prefix, SnapshotImpl::createEntry(value)}); +#ifdef ENVOY_ENABLE_YAML + values_.insert( + {full_prefix, SnapshotImpl::createEntry(ValueUtil::loadFromYaml(value), value)}); +#else + IS_ENVOY_BUG("Runtime admin reload requires YAML support"); + UNREFERENCED_PARAMETER(value); + return; +#endif } } } @@ -383,7 +428,7 @@ void ProtoLayer::walkProtoValue(const ProtobufWkt::Value& v, const std::string& throw EnvoyException(absl::StrCat("Invalid runtime entry value for ", prefix)); break; case ProtobufWkt::Value::kStringValue: - values_.emplace(prefix, SnapshotImpl::createEntry(v.string_value())); + values_.emplace(prefix, SnapshotImpl::createEntry(v)); break; case ProtobufWkt::Value::kNumberValue: case ProtobufWkt::Value::kBoolValue: diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index 7c1f36385d8a..f8e98ec72aec 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -87,31 +87,9 @@ class SnapshotImpl : public Snapshot, Logger::Loggable { const EntryMap& values() const; - static Entry createEntry(const std::string& value); - static Entry createEntry(const ProtobufWkt::Value& value); + static Entry createEntry(const ProtobufWkt::Value& value, absl::string_view raw_string = ""); private: - static void resolveEntryType(Entry& entry) { - if (parseEntryBooleanValue(entry)) { - return; - } - - if (parseEntryDoubleValue(entry) && entry.double_value_ >= 0 && - entry.double_value_ <= std::numeric_limits::max()) { - // Valid uint values will always be parseable as doubles, so we assign the value to both the - // uint and double fields. In cases where the value is something like "3.1", we will floor the - // number by casting it to a uint and assigning the uint value. - entry.uint_value_ = entry.double_value_; - return; - } - - parseEntryFractionalPercentValue(entry); - } - - static bool parseEntryBooleanValue(Entry& entry); - static bool parseEntryDoubleValue(Entry& entry); - static void parseEntryFractionalPercentValue(Entry& entry); - const std::vector layers_; EntryMap values_; Random::RandomGenerator& generator_; diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index cec763fe582f..cfc9cebadd89 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -578,6 +578,11 @@ TEST_F(StaticLoaderImplTest, RemovedFlags) { EXPECT_ENVOY_BUG(setup(), "envoy.reloadable_features.removed_foo"); } +TEST_F(StaticLoaderImplTest, ProtoParsingInvalidField) { + base_ = TestUtility::parseYaml("file0:"); + EXPECT_THROW_WITH_MESSAGE(setup(), EnvoyException, "Invalid runtime entry value for file0"); +} + // Validate proto parsing sanity. TEST_F(StaticLoaderImplTest, ProtoParsing) { base_ = TestUtility::parseYaml(R"EOF( @@ -588,6 +593,12 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) { file8: numerator: 52 denominator: HUNDRED + file80: + numerator: 52 + denominator: TEN_THOUSAND + file800: + numerator: 52 + denominator: MILLION file9: numerator: 100 denominator: NONSENSE @@ -666,10 +677,14 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) { // Fractional percent feature enablement envoy::type::v3::FractionalPercent fractional_percent; fractional_percent.set_numerator(5); - fractional_percent.set_denominator(envoy::type::v3::FractionalPercent::TEN_THOUSAND); + fractional_percent.set_denominator(envoy::type::v3::FractionalPercent::MILLION); EXPECT_CALL(generator_, random()).WillOnce(Return(50)); EXPECT_TRUE(loader_->snapshot().featureEnabled("file8", fractional_percent)); // valid data + EXPECT_CALL(generator_, random()).WillOnce(Return(50)); + EXPECT_TRUE(loader_->snapshot().featureEnabled("file80", fractional_percent)); // valid data + EXPECT_CALL(generator_, random()).WillOnce(Return(50)); + EXPECT_TRUE(loader_->snapshot().featureEnabled("file800", fractional_percent)); // valid data EXPECT_CALL(generator_, random()).WillOnce(Return(60)); EXPECT_FALSE(loader_->snapshot().featureEnabled("file8", fractional_percent)); // valid data @@ -718,8 +733,13 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) { EXPECT_EQ(0, store_.counter("runtime.load_error").value()); EXPECT_EQ(1, store_.counter("runtime.load_success").value()); - EXPECT_EQ(21, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(23, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); + + // While null values are generally filtered out by walkProtoValue, test manually. + ProtobufWkt::Value empty_value; + const_cast(dynamic_cast(loader_->snapshot())) + .createEntry(empty_value); } TEST_F(StaticLoaderImplTest, InvalidNumerator) { diff --git a/test/integration/utility.cc b/test/integration/utility.cc index b620d12522b7..56a8105d582c 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -128,7 +128,6 @@ class TestConnectionCallbacks : public Network::ConnectionCallbacks { bool connected_{false}; }; -#ifdef ENVOY_ENABLE_YAML Network::UpstreamTransportSocketFactoryPtr IntegrationUtil::createQuicUpstreamTransportSocketFactory(Api::Api& api, Stats::Store& store, Ssl::ContextManager& context_manager, @@ -140,9 +139,15 @@ IntegrationUtil::createQuicUpstreamTransportSocketFactory(Api::Api& api, Stats:: envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport quic_transport_socket_config; auto* tls_context = quic_transport_socket_config.mutable_upstream_tls_context(); +#ifdef ENVOY_ENABLE_YAML initializeUpstreamTlsContextConfig( Ssl::ClientSslTransportOptions().setAlpn(true).setSan(san_to_match).setSni("lyft.com"), *tls_context); +#else + UNREFERENCED_PARAMETER(tls_context); + UNREFERENCED_PARAMETER(san_to_match); + RELEASE_ASSERT(0, "unsupported"); +#endif // ENVOY_ENABLE_YAML envoy::config::core::v3::TransportSocket message; message.mutable_typed_config()->PackFrom(quic_transport_socket_config); @@ -200,9 +205,14 @@ IntegrationUtil::makeSingleRequest(const Network::Address::InstanceConstSharedPt Network::TransportSocketOptionsConstSharedPtr options; std::shared_ptr cluster{new NiceMock()}; - Upstream::HostDescriptionConstSharedPtr host_description{Upstream::makeTestHostDescription( - cluster, fmt::format("{}://127.0.0.1:80", (type == Http::CodecType::HTTP3 ? "udp" : "tcp")), - time_system)}; + Upstream::HostDescriptionConstSharedPtr host_description = + std::make_shared( + cluster, "", + Network::Utility::resolveUrl( + fmt::format("{}://127.0.0.1:80", (type == Http::CodecType::HTTP3 ? "udp" : "tcp"))), + nullptr, envoy::config::core::v3::Locality().default_instance(), + envoy::config::endpoint::v3::Endpoint::HealthCheckConfig::default_instance(), 0, + time_system); if (type <= Http::CodecType::HTTP2) { Http::CodecClientProd client(type, @@ -259,7 +269,6 @@ IntegrationUtil::makeSingleRequest(uint32_t port, const std::string& method, con fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(ip_version), port)); return makeSingleRequest(addr, method, url, body, type, host, content_type); } -#endif // ENVOY_ENABLE_YAML RawConnectionDriver::RawConnectionDriver(uint32_t port, Buffer::Instance& request_data, ReadCallback response_data_callback, From 584688b096efcbbd55f13f1baa1be107ae239241 Mon Sep 17 00:00:00 2001 From: Chris Schoener Date: Wed, 26 Apr 2023 20:37:19 -0400 Subject: [PATCH 003/740] Mobile: one-liner deps cleanup (#26998) Signed-off-by: Chris Schoener --- mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD b/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD index 47df86080a61..60e89e41cc84 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD @@ -41,13 +41,13 @@ java_library( "HeaderMatchConfig.java", "JniBridgeUtility.java", "JniLibrary.java", - "JvmBridgeUtility.java", "JvmCallbackContext.java", "JvmFilterContext.java", "JvmFilterFactoryContext.java", "JvmKeyValueStoreContext.java", "JvmStringAccessorContext.java", "VirtualClusterConfig.java", + "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib_srcs", ], visibility = ["//visibility:public"], deps = [ From 011c4c107e14025bd0408d882236578eaf8b943c Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Wed, 26 Apr 2023 21:05:18 -0400 Subject: [PATCH 004/740] TCP Health Check: log parity if TCP HC succeed with no payload. (#26857) log parity if TCP HC succeed with no payload. Signed-off-by: Kevin Baichoo --- source/extensions/health_checkers/tcp/health_checker_impl.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/source/extensions/health_checkers/tcp/health_checker_impl.cc b/source/extensions/health_checkers/tcp/health_checker_impl.cc index f3eadd06fcc2..08a3fdde67fc 100644 --- a/source/extensions/health_checkers/tcp/health_checker_impl.cc +++ b/source/extensions/health_checkers/tcp/health_checker_impl.cc @@ -108,6 +108,7 @@ void TcpHealthCheckerImpl::TcpActiveHealthCheckSession::onEvent(Network::Connect // TODO(mattklein123): In the case that a user configured bytes to write, they will not be // be written, since we currently have no way to know if the bytes actually get written via // the connection interface. We might want to figure out how to handle this better later. + ENVOY_CONN_LOG(trace, "healthcheck passed", *client_); expect_close_ = true; client_->close(Network::ConnectionCloseType::Abort); handleSuccess(false); From acced063747dfd3a0f92c1ab7667794a9875c734 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 26 Apr 2023 23:27:31 -0400 Subject: [PATCH 005/740] build: moving subset lb into extensions (#26572) Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: inline Signed-off-by: Alyssa Wilk --- CODEOWNERS | 1 + changelogs/current.yaml | 4 ++ .../upstream/load_balancing/subsets.rst | 1 + envoy/upstream/load_balancer.h | 27 +++++++++++ source/common/upstream/BUILD | 23 +-------- .../common/upstream/cluster_manager_impl.cc | 12 ++--- source/extensions/extensions_build_config.bzl | 1 + source/extensions/extensions_metadata.yaml | 5 ++ .../load_balancing_policies/subset/BUILD | 47 +++++++++++++++++++ .../load_balancing_policies/subset/config.cc | 30 ++++++++++++ .../load_balancing_policies/subset/config.h | 28 +++++++++++ .../subset}/subset_lb.cc | 11 ++--- .../subset}/subset_lb.h | 7 +-- test/common/upstream/BUILD | 6 +-- .../upstream/load_balancer_benchmark.cc | 2 +- test/common/upstream/subset_lb_test.cc | 2 +- test/common/upstream/test_cluster_manager.h | 2 +- test/integration/BUILD | 3 ++ test/per_file_coverage.sh | 2 + test/server/BUILD | 1 + tools/code_format/config.yaml | 1 + 21 files changed, 172 insertions(+), 44 deletions(-) create mode 100644 source/extensions/load_balancing_policies/subset/BUILD create mode 100644 source/extensions/load_balancing_policies/subset/config.cc create mode 100644 source/extensions/load_balancing_policies/subset/config.h rename source/{common/upstream => extensions/load_balancing_policies/subset}/subset_lb.cc (99%) rename source/{common/upstream => extensions/load_balancing_policies/subset}/subset_lb.h (98%) diff --git a/CODEOWNERS b/CODEOWNERS index 76d493092703..801f9378f652 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -294,6 +294,7 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/load_balancing_policies/round_robin @wbpcode @UNOWNED /*/extensions/load_balancing_policies/ring_hash @wbpcode @UNOWNED /*/extensions/load_balancing_policies/maglev @wbpcode @UNOWNED +/*/extensions/load_balancing_policies/subset @wbpcode @zuercher # Early header mutation /*/extensions/http/early_header_mutation/header_mutation @wbpcode @UNOWNED # Network matching extensions diff --git a/changelogs/current.yaml b/changelogs/current.yaml index d4f0ab3825ce..01fe7b2dd9a9 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -2,6 +2,10 @@ date: Pending behavior_changes: # *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* +- area: build + change: | + Moved the subset LB code into extensions. If you use the subset LB and override extensions_build_config.bzl you will + need to include it explicitly. minor_behavior_changes: # *Changes that may cause incompatibilities for some users, but should not for most* diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst b/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst index fdef94da57d6..f30ebf03c051 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst @@ -1,4 +1,5 @@ .. _arch_overview_load_balancer_subsets: +.. _extension_envoy.load_balancing_policies.subset: Load Balancer Subsets --------------------- diff --git a/envoy/upstream/load_balancer.h b/envoy/upstream/load_balancer.h index 62d3624e472f..0d18614c8e46 100644 --- a/envoy/upstream/load_balancer.h +++ b/envoy/upstream/load_balancer.h @@ -254,5 +254,32 @@ class TypedLoadBalancerFactory : public Config::TypedFactory { std::string category() const override { return "envoy.load_balancing_policies"; } }; +/** + * Factory config for non-thread-aware load balancers. To support a load balancing policy of + * LOAD_BALANCING_POLICY_CONFIG, at least one load balancer factory corresponding to a policy in + * load_balancing_policy must be registered with Envoy. Envoy will use the first policy for which + * it has a registered factory. + */ +class NonThreadAwareLoadBalancerFactory : public Config::UntypedFactory { +public: + ~NonThreadAwareLoadBalancerFactory() override = default; + + /** + * @return LoadBalancerPtr a new non-thread-aware load balancer. + * + * @param cluster_info supplies the cluster info. + * @param priority_set supplies the priority set. + * @param local_priority_set supplies the local priority set. + * @param runtime supplies the runtime loader. + * @param random supplies the random generator. + * @param time_source supplies the time source. + */ + virtual LoadBalancerPtr create(const ClusterInfo& cluster_info, const PrioritySet& priority_set, + const PrioritySet* local_priority_set, Runtime::Loader& runtime, + Random::RandomGenerator& random, TimeSource& time_source) PURE; + + std::string category() const override { return "envoy.load_balancing_policies"; } +}; + } // namespace Upstream } // namespace Envoy diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index afd5e174c881..49b06978e780 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -85,8 +85,8 @@ envoy_cc_library( ":load_balancer_lib", ":load_stats_reporter_lib", ":od_cds_api_lib", + ":maglev_lb_lib", ":ring_hash_lb_lib", - ":subset_lb_lib", ":host_utility_lib", "//envoy/api:api_interface", "//envoy/config:xds_resources_delegate_interface", @@ -412,27 +412,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "subset_lb_lib", - srcs = ["subset_lb.cc"], - hdrs = ["subset_lb.h"], - deps = [ - ":load_balancer_lib", - ":maglev_lb_lib", - ":ring_hash_lb_lib", - ":upstream_lib", - "//envoy/runtime:runtime_interface", - "//envoy/upstream:load_balancer_interface", - "//source/common/common:assert_lib", - "//source/common/common:minimal_logger_lib", - "//source/common/config:metadata_lib", - "//source/common/protobuf", - "//source/common/protobuf:utility_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - ], -) - envoy_cc_library( name = "upstream_lib", srcs = ["upstream_impl.cc"], diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 664a0f998b58..a5019aadc602 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -47,7 +47,6 @@ #include "source/common/upstream/maglev_lb.h" #include "source/common/upstream/priority_conn_pool_map_impl.h" #include "source/common/upstream/ring_hash_lb.h" -#include "source/common/upstream/subset_lb.h" #ifdef ENVOY_ENABLE_QUIC #include "source/common/http/conn_pool_grid.h" @@ -1549,12 +1548,11 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::ClusterEntry( // TODO(mattklein123): Consider converting other LBs over to thread local. All of them could // benefit given the healthy panic, locality, and priority calculations that take place. if (cluster->lbSubsetInfo().isEnabled()) { - lb_ = std::make_unique( - cluster->lbType(), priority_set_, parent_.local_priority_set_, cluster->lbStats(), - cluster->statsScope(), parent.parent_.runtime_, parent.parent_.random_, - cluster->lbSubsetInfo(), cluster->lbRingHashConfig(), cluster->lbMaglevConfig(), - cluster->lbRoundRobinConfig(), cluster->lbLeastRequestConfig(), cluster->lbConfig(), - parent_.thread_local_dispatcher_.timeSource()); + auto& factory = Config::Utility::getAndCheckFactoryByName( + "envoy.load_balancing_policies.subset"); + lb_ = factory.create(*cluster, priority_set_, parent_.local_priority_set_, + parent.parent_.runtime_, parent.parent_.random_, + parent_.thread_local_dispatcher_.timeSource()); } else { switch (cluster->lbType()) { case LoadBalancerType::LeastRequest: { diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 9feafa4d2206..a39fe32ecb9b 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -425,6 +425,7 @@ EXTENSIONS = { "envoy.load_balancing_policies.round_robin": "//source/extensions/load_balancing_policies/round_robin:config", "envoy.load_balancing_policies.maglev": "//source/extensions/load_balancing_policies/maglev:config", "envoy.load_balancing_policies.ring_hash": "//source/extensions/load_balancing_policies/ring_hash:config", + "envoy.load_balancing_policies.subset": "//source/extensions/load_balancing_policies/subset:config", # # HTTP Early Header Mutation diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 857dc3dbcb67..f71bce09dc56 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -1471,6 +1471,11 @@ envoy.load_balancing_policies.maglev: status: stable type_urls: - envoy.extensions.load_balancing_policies.maglev.v3.Maglev +envoy.load_balancing_policies.subset: + categories: + - envoy.load_balancing_policies + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable envoy.http.early_header_mutation.header_mutation: categories: - envoy.http.early_header_mutation diff --git a/source/extensions/load_balancing_policies/subset/BUILD b/source/extensions/load_balancing_policies/subset/BUILD new file mode 100644 index 000000000000..951b7f4e09b9 --- /dev/null +++ b/source/extensions/load_balancing_policies/subset/BUILD @@ -0,0 +1,47 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "subset_lb_lib", + srcs = ["subset_lb.cc"], + hdrs = ["subset_lb.h"], + deps = [ + "//envoy/runtime:runtime_interface", + "//envoy/upstream:load_balancer_interface", + "//source/common/common:assert_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/config:metadata_lib", + "//source/common/protobuf", + "//source/common/protobuf:utility_lib", + "//source/common/upstream:load_balancer_lib", + "//source/common/upstream:maglev_lb_lib", + "//source/common/upstream:ring_hash_lb_lib", + "//source/common/upstream:upstream_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + extra_visibility = [ + # previously considered core code. + "//test:__subpackages__", + ], + deps = [ + ":subset_lb_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/upstream:load_balancer_factory_base_lib", + "//source/common/upstream:load_balancer_lib", + ], +) diff --git a/source/extensions/load_balancing_policies/subset/config.cc b/source/extensions/load_balancing_policies/subset/config.cc new file mode 100644 index 000000000000..8944814252d8 --- /dev/null +++ b/source/extensions/load_balancing_policies/subset/config.cc @@ -0,0 +1,30 @@ +#include "source/extensions/load_balancing_policies/subset/config.h" + +#include "source/extensions/load_balancing_policies/subset/subset_lb.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Subset { + +Upstream::LoadBalancerPtr Factory::create(const Upstream::ClusterInfo& cluster, + const Upstream::PrioritySet& priority_set, + const Upstream::PrioritySet* local_priority_set, + Runtime::Loader& runtime, Random::RandomGenerator& random, + TimeSource& time_source) { + return std::make_unique( + cluster.lbType(), priority_set, local_priority_set, cluster.lbStats(), cluster.statsScope(), + runtime, random, cluster.lbSubsetInfo(), cluster.lbRingHashConfig(), cluster.lbMaglevConfig(), + cluster.lbRoundRobinConfig(), cluster.lbLeastRequestConfig(), cluster.lbConfig(), + time_source); +} + +/** + * Static registration for the Factory. @see RegisterFactory. + */ +REGISTER_FACTORY(Factory, Upstream::NonThreadAwareLoadBalancerFactory); + +} // namespace Subset +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/load_balancing_policies/subset/config.h b/source/extensions/load_balancing_policies/subset/config.h new file mode 100644 index 000000000000..d3ed76b688d5 --- /dev/null +++ b/source/extensions/load_balancing_policies/subset/config.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "envoy/upstream/load_balancer.h" + +#include "source/common/upstream/load_balancer_factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Subset { + +class Factory : public Upstream::NonThreadAwareLoadBalancerFactory { +public: + std::string name() const override { return "envoy.load_balancing_policies.subset"; } + + Upstream::LoadBalancerPtr create(const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& priority_set, + const Upstream::PrioritySet* local_priority_set, + Runtime::Loader& runtime, Random::RandomGenerator& random, + TimeSource& time_source) override; +}; + +} // namespace Subset +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/source/common/upstream/subset_lb.cc b/source/extensions/load_balancing_policies/subset/subset_lb.cc similarity index 99% rename from source/common/upstream/subset_lb.cc rename to source/extensions/load_balancing_policies/subset/subset_lb.cc index 94f61706d332..20dedf08e161 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/extensions/load_balancing_policies/subset/subset_lb.cc @@ -1,4 +1,4 @@ -#include "source/common/upstream/subset_lb.h" +#include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include @@ -23,9 +23,10 @@ namespace Upstream { using HostPredicate = std::function; SubsetLoadBalancer::SubsetLoadBalancer( - LoadBalancerType lb_type, PrioritySet& priority_set, const PrioritySet* local_priority_set, - ClusterLbStats& stats, Stats::Scope& scope, Runtime::Loader& runtime, - Random::RandomGenerator& random, const LoadBalancerSubsetInfo& subsets, + LoadBalancerType lb_type, const PrioritySet& priority_set, + const PrioritySet* local_priority_set, ClusterLbStats& stats, Stats::Scope& scope, + Runtime::Loader& runtime, Random::RandomGenerator& random, + const LoadBalancerSubsetInfo& subsets, OptRef lb_ring_hash_config, OptRef lb_maglev_config, OptRef round_robin_config, @@ -582,7 +583,6 @@ std::string SubsetLoadBalancer::describeMetadata(const SubsetLoadBalancer::Subse } std::ostringstream buf; -#ifdef ENVOY_ENABLE_YAML bool first = true; for (const auto& it : kvs) { if (!first) { @@ -594,7 +594,6 @@ std::string SubsetLoadBalancer::describeMetadata(const SubsetLoadBalancer::Subse const ProtobufWkt::Value& value = it.second; buf << it.first << "=" << MessageUtil::getJsonStringFromMessageOrError(value); } -#endif return buf.str(); } diff --git a/source/common/upstream/subset_lb.h b/source/extensions/load_balancing_policies/subset/subset_lb.h similarity index 98% rename from source/common/upstream/subset_lb.h rename to source/extensions/load_balancing_policies/subset/subset_lb.h index 567f65542c93..0f5ec5ccada6 100644 --- a/source/common/upstream/subset_lb.h +++ b/source/extensions/load_balancing_policies/subset/subset_lb.h @@ -30,9 +30,10 @@ using HostHashSet = absl::flat_hash_set; class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable { public: SubsetLoadBalancer( - LoadBalancerType lb_type, PrioritySet& priority_set, const PrioritySet* local_priority_set, - ClusterLbStats& stats, Stats::Scope& scope, Runtime::Loader& runtime, - Random::RandomGenerator& random, const LoadBalancerSubsetInfo& subsets, + LoadBalancerType lb_type, const PrioritySet& priority_set, + const PrioritySet* local_priority_set, ClusterLbStats& stats, Stats::Scope& scope, + Runtime::Loader& runtime, Random::RandomGenerator& random, + const LoadBalancerSubsetInfo& subsets, OptRef lb_ring_hash_config, OptRef lb_maglev_config, OptRef round_robin_config, diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 4bde37077df0..8fdc80087cd8 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -486,8 +486,8 @@ envoy_cc_benchmark_binary( "//source/common/memory:stats_lib", "//source/common/upstream:maglev_lb_lib", "//source/common/upstream:ring_hash_lb_lib", - "//source/common/upstream:subset_lb_lib", "//source/common/upstream:upstream_lib", + "//source/extensions/load_balancing_policies/subset:config", "//test/common/upstream:utility_lib", "//test/mocks/upstream:cluster_info_mocks", "//test/test_common:printers_lib", @@ -510,9 +510,9 @@ envoy_cc_test( "//source/common/common:minimal_logger_lib", "//source/common/network:utility_lib", "//source/common/upstream:load_balancer_lib", - "//source/common/upstream:subset_lb_lib", "//source/common/upstream:upstream_includes", "//source/common/upstream:upstream_lib", + "//source/extensions/load_balancing_policies/subset:config", "//test/mocks:common_lib", "//test/mocks/access_log:access_log_mocks", "//test/mocks/filesystem:filesystem_mocks", @@ -629,7 +629,7 @@ envoy_cc_test_library( "//source/common/stats:stats_lib", "//source/common/upstream:cluster_factory_lib", "//source/common/upstream:cluster_manager_lib", - "//source/common/upstream:subset_lb_lib", + "//source/extensions/load_balancing_policies/subset:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tls:context_lib", "//test/common/stats:stat_test_utility_lib", diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc index 2c0e9233fa74..95cc869af6fc 100644 --- a/test/common/upstream/load_balancer_benchmark.cc +++ b/test/common/upstream/load_balancer_benchmark.cc @@ -8,8 +8,8 @@ #include "source/common/memory/stats.h" #include "source/common/upstream/maglev_lb.h" #include "source/common/upstream/ring_hash_lb.h" -#include "source/common/upstream/subset_lb.h" #include "source/common/upstream/upstream_impl.h" +#include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include "test/benchmark/main.h" #include "test/common/upstream/utility.h" diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index 9cb86caf042c..f030eaf2b53d 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -10,8 +10,8 @@ #include "source/common/common/logger.h" #include "source/common/config/metadata.h" -#include "source/common/upstream/subset_lb.h" #include "source/common/upstream/upstream_impl.h" +#include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include "test/common/upstream/utility.h" #include "test/mocks/access_log/mocks.h" diff --git a/test/common/upstream/test_cluster_manager.h b/test/common/upstream/test_cluster_manager.h index b7548411825a..db02fc526e7d 100644 --- a/test/common/upstream/test_cluster_manager.h +++ b/test/common/upstream/test_cluster_manager.h @@ -21,7 +21,7 @@ #include "source/common/singleton/manager_impl.h" #include "source/common/upstream/cluster_factory_impl.h" #include "source/common/upstream/cluster_manager_impl.h" -#include "source/common/upstream/subset_lb.h" +#include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include "source/extensions/transport_sockets/tls/context_manager_impl.h" #include "test/common/stats/stat_test_utility.h" diff --git a/test/integration/BUILD b/test/integration/BUILD index 453cc163cf2d..a753f1d41ddd 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -546,6 +546,7 @@ envoy_cc_test( ], deps = [ ":http_integration_lib", + "//source/extensions/load_balancing_policies/subset:config", "//test/common/upstream:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", @@ -565,6 +566,7 @@ envoy_cc_test( ], deps = [ ":http_integration_lib", + "//source/extensions/load_balancing_policies/subset:config", "//test/common/upstream:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", @@ -1534,6 +1536,7 @@ envoy_cc_test( "//source/extensions/access_loggers/file:config", "//source/extensions/filters/network/common:factory_base_lib", "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/load_balancing_policies/subset:config", "//source/extensions/transport_sockets/tls:config", "//source/extensions/transport_sockets/tls:context_config_lib", "//source/extensions/transport_sockets/tls:context_lib", diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 8d0ab14da6d1..62ecae6eab39 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -80,6 +80,8 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/health_checkers:95.5" "source/extensions/health_checkers/http:93.8" "source/extensions/health_checkers/grpc:92.0" +"source/extensions/load_balancing_policies:94.7" +"source/extensions/load_balancing_policies/subset:94.3" ) [[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" diff --git a/test/server/BUILD b/test/server/BUILD index feed30e6fc5f..ed3858549df0 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -56,6 +56,7 @@ envoy_cc_test( "//source/common/upstream:cluster_manager_lib", "//source/extensions/access_loggers/file:config", "//source/extensions/clusters/static:static_cluster_lib", + "//source/extensions/load_balancing_policies/subset:config", "//source/extensions/stat_sinks/statsd:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:configuration_lib", diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index 6d87353d8858..39a31a4a42fe 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -318,3 +318,4 @@ visibility_excludes: - source/extensions/health_checkers/BUILD - source/extensions/config_subscription/rest/BUILD - source/extensions/config_subscription/filesystem/BUILD +- source/extensions/load_balancing_policies/subset/BUILD From 4c8ca4ada52d59d7758376c1c4ee7f8cbc1800b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Apr 2023 10:54:14 +0000 Subject: [PATCH 006/740] build(deps): bump requests from 2.28.2 to 2.29.0 in /examples/grpc-bridge/client (#27013) build(deps): bump requests in /examples/grpc-bridge/client Bumps [requests](https://github.com/psf/requests) from 2.28.2 to 2.29.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.28.2...v2.29.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/client/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index 281cab3aecf4..fc107979ec9d 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -129,9 +129,9 @@ protobuf==4.22.3 \ # via # -r requirements.in # grpcio-tools -requests==2.28.2 \ - --hash=sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa \ - --hash=sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf +requests==2.29.0 \ + --hash=sha256:e8f3c9be120d3333921d213eef078af392fba3933ab7ed2d1cba3b56f2568c3b \ + --hash=sha256:f2e34a75f4749019bb0e3effb66683630e4ffeaf75819fb51bebef1bf5aef059 # via -r requirements.in urllib3==1.26.7 \ --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \ From bb619eaeb2a1ee3f30cdd2aa087b7c9bee604ff8 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 27 Apr 2023 13:56:19 +0100 Subject: [PATCH 007/740] repo: Ignore websocket example certs dir (#26968) Signed-off-by: Ryan Northey --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 73f1c5317fd6..d5c8c6cadf2a 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,4 @@ bazel.output.txt **/*.iml tools/dev/src distribution/custom +examples/websocket/certs From e0461906790832aa6596738bfaecf82c9244776a Mon Sep 17 00:00:00 2001 From: Loong Dai Date: Thu, 27 Apr 2023 21:21:42 +0800 Subject: [PATCH 008/740] dlb: optimize retry logic (#27009) Signed-off-by: Loong --- .../dlb/source/connection_balancer_impl.cc | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc b/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc index 09344d5bbd5d..6d8cec6c3c04 100644 --- a/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc +++ b/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc @@ -249,22 +249,33 @@ void DlbBalancedConnectionHandlerImpl::post(Network::ConnectionSocketPtr&& socke events[0].adv_send.udata64 = reinterpret_cast(s); int ret = dlb_send(DlbConnectionBalanceFactorySingleton::get().tx_ports[index_], 1, &events[0]); if (ret != 1) { - uint i = 0; - while (i < DlbConnectionBalanceFactorySingleton::get().max_retries) { - ENVOY_LOG(debug, "{} dlb_send fail, start retry, errono: {}", name_, errno); - ret = dlb_send(DlbConnectionBalanceFactorySingleton::get().tx_ports[index_], 1, &events[0]); - if (ret == 1) { - ENVOY_LOG(warn, "{} dlb_send retry {} times and succeed", name_, i + 1); - break; + if (DlbConnectionBalanceFactorySingleton::get().max_retries > 0) { + uint i = 0; + while (i < DlbConnectionBalanceFactorySingleton::get().max_retries) { + ENVOY_LOG(debug, "{} dlb_send fail, start retry, errono: {}", name_, errno); + ret = dlb_send(DlbConnectionBalanceFactorySingleton::get().tx_ports[index_], 1, &events[0]); + if (ret == 1) { + ENVOY_LOG(warn, "{} dlb_send retry {} times and succeed", name_, i + 1); + break; + } + i++; } - i++; - } - if (ret != 1) { - ENVOY_LOG(error, "{} dlb_send fail with {} times retry, errono: {}, message: {}", name_, - DlbConnectionBalanceFactorySingleton::get().max_retries, errno, + if (ret != 1) { + ENVOY_LOG(error, + "{} dlb_send fail with {} times retry, errono: {}, message: {}, increase " + "max_retries may help", + name_, DlbConnectionBalanceFactorySingleton::get().max_retries, errno, + errorDetails(errno)); + } + } else { + ENVOY_LOG(error, + "{} dlb_send fail without retry, errono: {}, message: {}, set " + "max_retries may help", + name_, DlbConnectionBalanceFactorySingleton::get().max_retries, errno, errorDetails(errno)); } + } else { ENVOY_LOG(debug, "{} dlb send fd {}", name_, s->ioHandle().fdDoNotUse()); } From c30a953e720e90bb6dc9df789153b017d5c2f0d2 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 27 Apr 2023 09:28:13 -0400 Subject: [PATCH 009/740] mobile: cleaning up unused validate imports (#27004) Signed-off-by: Alyssa Wilk --- .../extensions/filters/http/test_event_tracker/filter.proto | 2 -- .../common/extensions/filters/http/test_logger/filter.proto | 2 -- .../common/extensions/filters/http/test_read/filter.proto | 2 -- .../retry/options/network_configuration/predicate.proto | 2 -- 4 files changed, 8 deletions(-) diff --git a/mobile/library/common/extensions/filters/http/test_event_tracker/filter.proto b/mobile/library/common/extensions/filters/http/test_event_tracker/filter.proto index 7d9715356e17..444153d0d191 100644 --- a/mobile/library/common/extensions/filters/http/test_event_tracker/filter.proto +++ b/mobile/library/common/extensions/filters/http/test_event_tracker/filter.proto @@ -2,8 +2,6 @@ syntax = "proto3"; package envoymobile.extensions.filters.http.test_event_tracker; -import "validate/validate.proto"; - message TestEventTracker { // The attributes to report as part of envoy event emitted by `TestEventTrackerFilter` filter. map attributes = 1; diff --git a/mobile/library/common/extensions/filters/http/test_logger/filter.proto b/mobile/library/common/extensions/filters/http/test_logger/filter.proto index 14db04a93dad..ea139eccbf5c 100644 --- a/mobile/library/common/extensions/filters/http/test_logger/filter.proto +++ b/mobile/library/common/extensions/filters/http/test_logger/filter.proto @@ -2,7 +2,5 @@ syntax = "proto3"; package envoymobile.extensions.filters.http.test_logger; -import "validate/validate.proto"; - message TestLogger { } diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.proto b/mobile/library/common/extensions/filters/http/test_read/filter.proto index 05fd68d94009..de164df7d450 100644 --- a/mobile/library/common/extensions/filters/http/test_read/filter.proto +++ b/mobile/library/common/extensions/filters/http/test_read/filter.proto @@ -2,7 +2,5 @@ syntax = "proto3"; package envoymobile.test.integration.filters.http.test_read; -import "validate/validate.proto"; - message TestRead { } diff --git a/mobile/library/common/extensions/retry/options/network_configuration/predicate.proto b/mobile/library/common/extensions/retry/options/network_configuration/predicate.proto index 1ef0576d159d..ce8e2cb35953 100644 --- a/mobile/library/common/extensions/retry/options/network_configuration/predicate.proto +++ b/mobile/library/common/extensions/retry/options/network_configuration/predicate.proto @@ -2,7 +2,5 @@ syntax = "proto3"; package envoymobile.extensions.retry.options.network_configuration; -import "validate/validate.proto"; - message NetworkConfigurationOptionsPredicate { } From dca0337bcf44ca8df90a86aa6b1c5fb830414dcb Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Thu, 27 Apr 2023 17:05:06 +0300 Subject: [PATCH 010/740] access_log: add ACCESS_LOG_TYPE substitution string (#26755) Signed-off-by: ohadvano --- api/envoy/data/accesslog/v3/BUILD | 1 + api/envoy/data/accesslog/v3/accesslog.proto | 32 +- changelogs/current.yaml | 8 + .../filters/http/source/golang_filter.cc | 3 +- .../filters/http/source/golang_filter.h | 4 +- .../observability/access_log/usage.rst | 16 + envoy/access_log/BUILD | 1 + envoy/access_log/access_log.h | 7 +- envoy/formatter/BUILD | 1 + envoy/formatter/substitution_formatter.h | 34 ++- .../formatter/substitution_formatter.cc | 248 +++++++-------- .../common/formatter/substitution_formatter.h | 283 +++++++++++------- source/common/http/conn_manager_impl.cc | 6 +- source/common/http/filter_manager.h | 5 +- source/common/router/upstream_request.cc | 10 +- source/common/router/upstream_request.h | 2 +- source/common/tcp_proxy/tcp_proxy.cc | 9 +- .../access_loggers/common/access_log_base.cc | 6 +- .../access_loggers/common/access_log_base.h | 13 +- .../common/file_access_log_impl.cc | 5 +- .../common/file_access_log_impl.h | 10 +- .../grpc/grpc_access_log_utils.cc | 7 +- .../grpc/grpc_access_log_utils.h | 4 +- .../grpc/http_grpc_access_log_impl.cc | 5 +- .../grpc/http_grpc_access_log_impl.h | 10 +- .../grpc/tcp_grpc_access_log_impl.cc | 5 +- .../grpc/tcp_grpc_access_log_impl.h | 10 +- .../open_telemetry/access_log_impl.cc | 13 +- .../open_telemetry/access_log_impl.h | 4 +- .../open_telemetry/substitution_formatter.cc | 8 +- .../open_telemetry/substitution_formatter.h | 17 +- .../wasm/wasm_access_log_impl.h | 2 +- source/extensions/common/wasm/context.cc | 2 +- source/extensions/common/wasm/context.h | 3 +- .../filters/http/composite/filter.h | 5 +- .../extensions/filters/http/tap/tap_filter.cc | 3 +- .../extensions/filters/http/tap/tap_filter.h | 3 +- .../req_without_query/req_without_query.cc | 20 +- .../req_without_query/req_without_query.h | 4 +- test/common/formatter/command_extension.cc | 26 +- test/common/formatter/command_extension.h | 28 +- test/common/http/conn_manager_impl_test.cc | 57 ++-- test/common/http/conn_manager_impl_test_2.cc | 16 +- .../quic/envoy_quic_server_stream_test.cc | 2 +- test/common/router/router_test.cc | 3 +- .../common/router/router_upstream_log_test.cc | 17 +- test/common/tcp_proxy/tcp_proxy_test.cc | 19 +- .../common/access_log_base_test.cc | 3 +- .../grpc/http_grpc_access_log_impl_test.cc | 16 + .../http_grpc_access_log_integration_test.cc | 3 + .../tcp_grpc_access_log_integration_test.cc | 8 + .../active_internal_listener_test.cc | 2 +- .../filters/http/composite/filter_test.cc | 4 +- test/integration/protocol_integration_test.cc | 58 ++++ .../integration/tcp_proxy_integration_test.cc | 62 +++- .../typed_metadata_integration_test.cc | 3 +- .../upstream_access_log_integration_test.cc | 64 +++- test/mocks/access_log/mocks.h | 2 +- test/server/connection_handler_test.cc | 100 +++---- 59 files changed, 850 insertions(+), 472 deletions(-) diff --git a/api/envoy/data/accesslog/v3/BUILD b/api/envoy/data/accesslog/v3/BUILD index 1c1a6f6b4423..a1775bbe6f51 100644 --- a/api/envoy/data/accesslog/v3/BUILD +++ b/api/envoy/data/accesslog/v3/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], diff --git a/api/envoy/data/accesslog/v3/accesslog.proto b/api/envoy/data/accesslog/v3/accesslog.proto index f74391c79b5c..d069f4b771cf 100644 --- a/api/envoy/data/accesslog/v3/accesslog.proto +++ b/api/envoy/data/accesslog/v3/accesslog.proto @@ -10,6 +10,7 @@ import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "google/protobuf/wrappers.proto"; +import "envoy/annotations/deprecation.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -31,6 +32,19 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Fields describing *upstream* interaction will explicitly include ``upstream`` // in their name. +enum AccessLogType { + NotSet = 0; + TcpUpstreamConnected = 1; + TcpPeriodic = 2; + TcpConnectionEnd = 3; + DownstreamStart = 4; + DownstreamPeriodic = 5; + DownstreamEnd = 6; + UpstreamPoolReady = 7; + UpstreamPeriodic = 8; + UpstreamEnd = 9; +} + message TCPAccessLogEntry { option (udpa.annotations.versioning).previous_message_type = "envoy.data.accesslog.v2.TCPAccessLogEntry"; @@ -80,7 +94,7 @@ message ConnectionProperties { } // Defines fields that are shared by all Envoy access logs. -// [#next-free-field: 33] +// [#next-free-field: 34] message AccessLogCommon { option (udpa.annotations.versioning).previous_message_type = "envoy.data.accesslog.v2.AccessLogCommon"; @@ -214,7 +228,13 @@ message AccessLogCommon { // And if it is necessary, unique ID or identifier can be added to the log entry // :ref:`stream_id ` to // correlate all these intermediate log entries and final log entry. - bool intermediate_log_entry = 27; + // + // .. attention:: + // + // This field is deprecated in favor of ``access_log_type`` for better indication of the + // type of the access log record. + bool intermediate_log_entry = 27 + [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; // If downstream connection in listener failed due to transport socket (e.g. TLS handshake), provides the // failure reason from the transport socket. The format of this field depends on the configured downstream @@ -236,6 +256,14 @@ message AccessLogCommon { // For HTTP: Total number of bytes received from the upstream by the http stream. // For TCP: Total number of bytes sent to the upstream by the tcp proxy. uint64 upstream_wire_bytes_received = 32; + + // The type of the access log, which indicates when the log was recorded. + // See :ref:`ACCESS_LOG_TYPE ` for the available values. + // In case the access log was recorded by a flow which does not correspond to one of the supported + // values, then the default value will be ``NotSet``. + // For more information about how access log behaves and when it is being recorded, + // please refer to :ref:`access logging `. + AccessLogType access_log_type = 33; } // Flags indicating occurrences during request/response processing. diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 01fe7b2dd9a9..b44dda5460e6 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -49,6 +49,10 @@ removed_config_or_runtime: removed runtime key ``envoy.reloadable_features.http_response_half_close`` and legacy code paths. new_features: +- area: access_log + change: | + added %ACCESS_LOG_TYPE% substitution string, to help distinguishing between access log records and when they are being + recorded. Please refer to the access log configuration documentation for more information. - area: http change: | added Runtime feature ``envoy.reloadable_features.max_request_headers_size_kb`` to override the default value of @@ -69,3 +73,7 @@ new_features: ` to allow for setting rate limit domains on a per-route basis. deprecated: +- area: access_log + change: | + deprecated (1.25.0) :ref:`intermediate_log_entry ` + in favour of :ref:`access_log_type `. diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc index d2dbddff3efb..05eebfceb2e9 100644 --- a/contrib/golang/filters/http/source/golang_filter.cc +++ b/contrib/golang/filters/http/source/golang_filter.cc @@ -201,7 +201,8 @@ void Filter::onDestroy() { // access_log is executed before the log of the stream filter void Filter::log(const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&) { + const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&, + Envoy::AccessLog::AccessLogType) { // Todo log phase of stream filter } diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h index c91398aa6a6f..3edc149d64d6 100644 --- a/contrib/golang/filters/http/source/golang_filter.h +++ b/contrib/golang/filters/http/source/golang_filter.h @@ -153,7 +153,9 @@ class Filter : public Http::StreamFilter, void log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) override; + const StreamInfo::StreamInfo& stream_info, + Envoy::AccessLog::AccessLogType access_log_type = + Envoy::AccessLog::AccessLogType::NotSet) override; void onStreamComplete() override {} diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index 9bd9073e4973..491aecf0aa24 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -1051,6 +1051,22 @@ The following command operators are supported: %FILTER_CHAIN_NAME% The :ref:`network filter chain name ` of the downstream connection. +.. _config_access_log_format_access_log_type: + +%ACCESS_LOG_TYPE% + The type of the access log, which indicates when the access log was recorded. If a non-supported log (from the list below), + uses this substitution string, then the value will be an empty string. + + * TcpUpstreamConnected - When TCP Proxy filter has successfully established an upstream connection. + * TcpPeriodic - On any TCP Proxy filter periodic log record. + * TcpConnectionEnd - When a TCP connection is ended on TCP Proxy filter. + * DownstreamStart - When HTTP Connection Manager filter receives a new HTTP request. + * DownstreamPeriodic - On any HTTP Connection Manager periodic log record. + * DownstreamEnd - When an HTTP stream is ended on HTTP Connection Manager filter. + * UpstreamPoolReady - When a new HTTP request is received by the HTTP Router filter. + * UpstreamPeriodic - On any HTTP Router filter periodic log record. + * UpstreamEnd - When an HTTP request is finished on the HTTP Router filter. + %ENVIRONMENT(X):Z% Environment value of environment variable X. If no valid environment variable X, '-' symbol will be used. Z is an optional parameter denoting string truncation up to Z characters long. diff --git a/envoy/access_log/BUILD b/envoy/access_log/BUILD index c9578cb50311..2efbbd196f75 100644 --- a/envoy/access_log/BUILD +++ b/envoy/access_log/BUILD @@ -17,5 +17,6 @@ envoy_cc_library( "//envoy/http:header_map_interface", "//envoy/stream_info:stream_info_interface", "//source/common/protobuf", + "@envoy_api//envoy/data/accesslog/v3:pkg_cc_proto", ], ) diff --git a/envoy/access_log/access_log.h b/envoy/access_log/access_log.h index 5c8d952877d4..a024091a524d 100644 --- a/envoy/access_log/access_log.h +++ b/envoy/access_log/access_log.h @@ -4,6 +4,7 @@ #include #include "envoy/common/pure.h" +#include "envoy/data/accesslog/v3/accesslog.pb.h" #include "envoy/filesystem/filesystem.h" #include "envoy/http/header_map.h" #include "envoy/stream_info/stream_info.h" @@ -54,6 +55,7 @@ class AccessLogManager { }; using AccessLogManagerPtr = std::unique_ptr; +using AccessLogType = envoy::data::accesslog::v3::AccessLogType; /** * Interface for access log filters. @@ -88,11 +90,14 @@ class Instance { * @param response_trailers supplies response trailers. * @param stream_info supplies additional information about the request not * contained in the request headers. + * @param access_log_type supplies additional information about the type of the + * log record, i.e the location in the code which recorded the log. */ virtual void log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) PURE; + const StreamInfo::StreamInfo& stream_info, + AccessLogType access_log_type = AccessLogType::NotSet) PURE; }; using InstanceSharedPtr = std::shared_ptr; diff --git a/envoy/formatter/BUILD b/envoy/formatter/BUILD index f630c6607644..8c691b773bad 100644 --- a/envoy/formatter/BUILD +++ b/envoy/formatter/BUILD @@ -12,6 +12,7 @@ envoy_cc_library( name = "substitution_formatter_interface", hdrs = ["substitution_formatter.h"], deps = [ + "//envoy/access_log:access_log_interface", "//envoy/config:typed_config_interface", "//envoy/http:header_map_interface", "//envoy/stream_info:stream_info_interface", diff --git a/envoy/formatter/substitution_formatter.h b/envoy/formatter/substitution_formatter.h index c0d1502c19b2..7be401d102ae 100644 --- a/envoy/formatter/substitution_formatter.h +++ b/envoy/formatter/substitution_formatter.h @@ -3,6 +3,7 @@ #include #include +#include "envoy/access_log/access_log.h" #include "envoy/common/pure.h" #include "envoy/config/typed_config.h" #include "envoy/http/header_map.h" @@ -28,11 +29,12 @@ class Formatter { * @param local_reply_body supplies the local reply body. * @return std::string string containing the complete formatted substitution line. */ - virtual std::string format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const PURE; + virtual std::string + format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const PURE; }; using FormatterPtr = std::unique_ptr; @@ -56,11 +58,12 @@ class FormatterProvider { * @return absl::optional optional string containing a single value extracted from * the given headers/trailers/stream. */ - virtual absl::optional format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const PURE; + virtual absl::optional + format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const PURE; /** * Extract a value from the provided headers/trailers/stream, preserving the value's type. * @param request_headers supplies the request headers. @@ -71,11 +74,12 @@ class FormatterProvider { * @return ProtobufWkt::Value containing a single value extracted from the given * headers/trailers/stream. */ - virtual ProtobufWkt::Value formatValue(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const PURE; + virtual ProtobufWkt::Value formatValue( + const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const PURE; }; using FormatterProviderPtr = std::unique_ptr; diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index 81b6244d5a47..a337e70de86e 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -129,13 +129,14 @@ std::string FormatterImpl::format(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const { + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const { std::string log_line; log_line.reserve(256); for (const FormatterProviderPtr& provider : providers_) { const auto bit = provider->format(request_headers, response_headers, response_trailers, - stream_info, local_reply_body); + stream_info, local_reply_body, access_log_type); log_line += bit.value_or(empty_value_string_); } @@ -146,9 +147,11 @@ std::string JsonFormatterImpl::format(const Http::RequestHeaderMap& request_head const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const { - const ProtobufWkt::Struct output_struct = struct_formatter_.format( - request_headers, response_headers, response_trailers, stream_info, local_reply_body); + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const { + const ProtobufWkt::Struct output_struct = + struct_formatter_.format(request_headers, response_headers, response_trailers, stream_info, + local_reply_body, access_log_type); #ifdef ENVOY_ENABLE_YAML const std::string log_line = @@ -248,29 +251,30 @@ ProtobufWkt::Value StructFormatter::providersCallback( const std::vector& providers, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const { + absl::string_view local_reply_body, AccessLog::AccessLogType access_log_type) const { ASSERT(!providers.empty()); if (providers.size() == 1) { const auto& provider = providers.front(); if (preserve_types_) { return provider->formatValue(request_headers, response_headers, response_trailers, - stream_info, local_reply_body); + stream_info, local_reply_body, access_log_type); } if (omit_empty_values_) { - return ValueUtil::optionalStringValue(provider->format( - request_headers, response_headers, response_trailers, stream_info, local_reply_body)); + return ValueUtil::optionalStringValue(provider->format(request_headers, response_headers, + response_trailers, stream_info, + local_reply_body, access_log_type)); } const auto str = provider->format(request_headers, response_headers, response_trailers, - stream_info, local_reply_body); + stream_info, local_reply_body, access_log_type); return ValueUtil::stringValue(str.value_or(DefaultUnspecifiedValueString)); } // Multiple providers forces string output. std::string str; for (const auto& provider : providers) { const auto bit = provider->format(request_headers, response_headers, response_trailers, - stream_info, local_reply_body); + stream_info, local_reply_body, access_log_type); str += bit.value_or(empty_value_); } return ValueUtil::stringValue(str); @@ -312,11 +316,12 @@ ProtobufWkt::Struct StructFormatter::format(const Http::RequestHeaderMap& reques const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const { + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const { StructFormatMapVisitor visitor{ [&](const std::vector& providers) { return providersCallback(providers, request_headers, response_headers, response_trailers, - stream_info, local_reply_body); + stream_info, local_reply_body, access_log_type); }, [&, this](const StructFormatter::StructFormatMapWrapper& format_map) { return structFormatMapCallback(format_map, visitor); @@ -392,6 +397,11 @@ SubstitutionFormatParser::getKnownFormatters() { [](const std::string&, absl::optional&) { return std::make_unique(); }}}, + {"ACCESS_LOG_TYPE", + {CommandSyntaxChecker::COMMAND_ONLY, + [](const std::string&, absl::optional&) { + return std::make_unique(); + }}}, {"GRPC_STATUS", {CommandSyntaxChecker::PARAMS_OPTIONAL, [](const std::string& format, const absl::optional&) { @@ -1602,63 +1612,57 @@ StreamInfoFormatter::StreamInfoFormatter(const std::string& command, const std:: field_extractor_ = (*it).second.second(subcommand, length); } -absl::optional StreamInfoFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const { +absl::optional StreamInfoFormatter::format( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view, AccessLog::AccessLogType) const { return field_extractor_->extract(stream_info); } -ProtobufWkt::Value StreamInfoFormatter::formatValue(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const { +ProtobufWkt::Value StreamInfoFormatter::formatValue( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view, AccessLog::AccessLogType) const { return field_extractor_->extractValue(stream_info); } PlainStringFormatter::PlainStringFormatter(const std::string& str) { str_.set_string_value(str); } -absl::optional PlainStringFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +absl::optional +PlainStringFormatter::format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return str_.string_value(); } -ProtobufWkt::Value PlainStringFormatter::formatValue(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +ProtobufWkt::Value +PlainStringFormatter::formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return str_; } PlainNumberFormatter::PlainNumberFormatter(double num) { num_.set_number_value(num); } -absl::optional PlainNumberFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +absl::optional +PlainNumberFormatter::format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { std::string str = absl::StrFormat("%g", num_.number_value()); return str; } -ProtobufWkt::Value PlainNumberFormatter::formatValue(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +ProtobufWkt::Value +PlainNumberFormatter::formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return num_; } -absl::optional -LocalReplyBodyFormatter::format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view local_reply_body) const { +absl::optional LocalReplyBodyFormatter::format(const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo&, + absl::string_view local_reply_body, + AccessLog::AccessLogType) const { return std::string(local_reply_body); } @@ -1666,10 +1670,26 @@ ProtobufWkt::Value LocalReplyBodyFormatter::formatValue(const Http::RequestHeade const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view local_reply_body) const { + absl::string_view local_reply_body, + AccessLog::AccessLogType) const { return ValueUtil::stringValue(std::string(local_reply_body)); } +absl::optional +AccessLogTypeFormatter::format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType access_log_type) const { + return AccessLogType_Name(access_log_type); +} + +ProtobufWkt::Value +AccessLogTypeFormatter::formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, + AccessLog::AccessLogType access_log_type) const { + return ValueUtil::stringValue(AccessLogType_Name(access_log_type)); +} + HeaderFormatter::HeaderFormatter(const std::string& main_header, const std::string& alternative_header, absl::optional max_length) @@ -1714,15 +1734,19 @@ ResponseHeaderFormatter::ResponseHeaderFormatter(const std::string& main_header, absl::optional max_length) : HeaderFormatter(main_header, alternative_header, max_length) {} -absl::optional ResponseHeaderFormatter::format( - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view) const { +absl::optional +ResponseHeaderFormatter::format(const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return HeaderFormatter::format(response_headers); } -ProtobufWkt::Value ResponseHeaderFormatter::formatValue( - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view) const { +ProtobufWkt::Value +ResponseHeaderFormatter::formatValue(const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return HeaderFormatter::formatValue(response_headers); } @@ -1734,14 +1758,16 @@ RequestHeaderFormatter::RequestHeaderFormatter(const std::string& main_header, absl::optional RequestHeaderFormatter::format(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, absl::string_view) const { + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const { return HeaderFormatter::format(request_headers); } ProtobufWkt::Value RequestHeaderFormatter::formatValue(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, absl::string_view) const { + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const { return HeaderFormatter::formatValue(request_headers); } @@ -1753,14 +1779,16 @@ ResponseTrailerFormatter::ResponseTrailerFormatter(const std::string& main_heade absl::optional ResponseTrailerFormatter::format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo&, absl::string_view) const { + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const { return HeaderFormatter::format(response_trailers); } ProtobufWkt::Value ResponseTrailerFormatter::formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo&, absl::string_view) const { + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const { return HeaderFormatter::formatValue(response_trailers); } @@ -1781,19 +1809,17 @@ uint64_t HeadersByteSizeFormatter::extractHeadersByteSize( PANIC_DUE_TO_CORRUPT_ENUM; } -absl::optional -HeadersByteSizeFormatter::format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo&, absl::string_view) const { +absl::optional HeadersByteSizeFormatter::format( + const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return absl::StrCat(extractHeadersByteSize(request_headers, response_headers, response_trailers)); } -ProtobufWkt::Value -HeadersByteSizeFormatter::formatValue(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo&, absl::string_view) const { +ProtobufWkt::Value HeadersByteSizeFormatter::formatValue( + const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return ValueUtil::numberValue( extractHeadersByteSize(request_headers, response_headers, response_trailers)); } @@ -1818,11 +1844,10 @@ GrpcStatusFormatter::GrpcStatusFormatter(const std::string& main_header, absl::optional max_length, Format format) : HeaderFormatter(main_header, alternative_header, max_length), format_(format) {} -absl::optional -GrpcStatusFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& info, absl::string_view) const { +absl::optional GrpcStatusFormatter::format( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& info, + absl::string_view, AccessLog::AccessLogType) const { const auto grpc_status = Grpc::Common::getGrpcStatus(response_trailers, response_headers, info, true); if (!grpc_status.has_value()) { @@ -1851,11 +1876,10 @@ GrpcStatusFormatter::format(const Http::RequestHeaderMap&, PANIC_DUE_TO_CORRUPT_ENUM; } -ProtobufWkt::Value -GrpcStatusFormatter::formatValue(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& info, absl::string_view) const { +ProtobufWkt::Value GrpcStatusFormatter::formatValue( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& info, + absl::string_view, AccessLog::AccessLogType) const { const auto grpc_status = Grpc::Common::getGrpcStatus(response_trailers, response_headers, info, true); if (!grpc_status.has_value()) { @@ -1937,20 +1961,16 @@ MetadataFormatter::formatMetadataValue(const envoy::config::core::v3::Metadata& return val; } -absl::optional MetadataFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const { +absl::optional MetadataFormatter::format( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view, AccessLog::AccessLogType) const { auto metadata = get_func_(stream_info); return (metadata != nullptr) ? formatMetadata(*metadata) : absl::nullopt; } -ProtobufWkt::Value MetadataFormatter::formatValue(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const { +ProtobufWkt::Value MetadataFormatter::formatValue( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view, AccessLog::AccessLogType) const { auto metadata = get_func_(stream_info); return formatMetadataValue((metadata != nullptr) ? *metadata : envoy::config::core::v3::Metadata()); @@ -2046,11 +2066,9 @@ FilterStateFormatter::filterState(const StreamInfo::StreamInfo& stream_info) con return nullptr; } -absl::optional FilterStateFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const { +absl::optional FilterStateFormatter::format( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view, AccessLog::AccessLogType) const { const Envoy::StreamInfo::FilterState::Object* state = filterState(stream_info); if (!state) { return absl::nullopt; @@ -2082,11 +2100,9 @@ absl::optional FilterStateFormatter::format(const Http::RequestHead return value; } -ProtobufWkt::Value FilterStateFormatter::formatValue(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const { +ProtobufWkt::Value FilterStateFormatter::formatValue( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view, AccessLog::AccessLogType) const { const Envoy::StreamInfo::FilterState::Object* state = filterState(stream_info); if (!state) { return unspecifiedValue(); @@ -2178,11 +2194,9 @@ SystemTimeFormatter::SystemTimeFormatter(const std::string& format, TimeFieldExt } } -absl::optional SystemTimeFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const { +absl::optional SystemTimeFormatter::format( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view, AccessLog::AccessLogType) const { const auto time_field = (*time_field_extractor_)(stream_info); if (!time_field.has_value()) { return absl::nullopt; @@ -2196,9 +2210,9 @@ absl::optional SystemTimeFormatter::format(const Http::RequestHeade ProtobufWkt::Value SystemTimeFormatter::formatValue( const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const { - return ValueUtil::optionalStringValue( - format(request_headers, response_headers, response_trailers, stream_info, local_reply_body)); + absl::string_view local_reply_body, AccessLog::AccessLogType access_log_type) const { + return ValueUtil::optionalStringValue(format(request_headers, response_headers, response_trailers, + stream_info, local_reply_body, access_log_type)); } void CommandSyntaxChecker::verifySyntax(CommandSyntaxFlags flags, const std::string& command, @@ -2231,18 +2245,16 @@ EnvironmentFormatter::EnvironmentFormatter(const std::string& key, str_.set_string_value(DefaultUnspecifiedValueString); } -absl::optional EnvironmentFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +absl::optional +EnvironmentFormatter::format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return str_.string_value(); } -ProtobufWkt::Value EnvironmentFormatter::formatValue(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +ProtobufWkt::Value +EnvironmentFormatter::formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return str_; } @@ -2253,13 +2265,13 @@ StreamInfoRequestHeaderFormatter::StreamInfoRequestHeaderFormatter( absl::optional StreamInfoRequestHeaderFormatter::format( const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, absl::string_view) const { + const StreamInfo::StreamInfo& stream_info, absl::string_view, AccessLog::AccessLogType) const { return HeaderFormatter::format(*stream_info.getRequestHeaders()); } ProtobufWkt::Value StreamInfoRequestHeaderFormatter::formatValue( const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, absl::string_view) const { + const StreamInfo::StreamInfo& stream_info, absl::string_view, AccessLog::AccessLogType) const { return HeaderFormatter::formatValue(*stream_info.getRequestHeaders()); } diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index 9f635f8ac444..5d8d5bb2c58d 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -145,11 +145,12 @@ class FormatterImpl : public Formatter { const std::vector& command_parsers); // Formatter::format - std::string format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const override; + std::string format( + const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const override; private: const std::string& empty_value_string_; @@ -171,11 +172,12 @@ class StructFormatter { StructFormatter(const ProtobufWkt::Struct& format_mapping, bool preserve_types, bool omit_empty_values, const std::vector& commands); - ProtobufWkt::Struct format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const; + ProtobufWkt::Struct + format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const; private: struct StructFormatMapWrapper; @@ -219,12 +221,13 @@ class StructFormatter { }; // Methods for doing the actual formatting. - ProtobufWkt::Value providersCallback(const std::vector& providers, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const; + ProtobufWkt::Value providersCallback( + const std::vector& providers, + const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const; ProtobufWkt::Value structFormatMapCallback(const StructFormatter::StructFormatMapWrapper& format_map, const StructFormatMapVisitor& visitor) const; @@ -251,11 +254,12 @@ class JsonFormatterImpl : public Formatter { : struct_formatter_(format_mapping, preserve_types, omit_empty_values, commands) {} // Formatter::format - std::string format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const override; + std::string format( + const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const override; private: const StructFormatter struct_formatter_; @@ -270,12 +274,14 @@ class PlainStringFormatter : public FormatterProvider { PlainStringFormatter(const std::string& str); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; private: ProtobufWkt::Value str_; @@ -289,12 +295,14 @@ class PlainNumberFormatter : public FormatterProvider { PlainNumberFormatter(double num); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; private: ProtobufWkt::Value num_; @@ -308,12 +316,36 @@ class LocalReplyBodyFormatter : public FormatterProvider { LocalReplyBodyFormatter() = default; // Formatter::format - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view local_reply_body) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view local_reply_body) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view local_reply_body, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view local_reply_body, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; +}; + +/** + * FormatterProvider for access log type. It returns the string from `access_log_type` argument. + */ +class AccessLogTypeFormatter : public FormatterProvider { +public: + AccessLogTypeFormatter() = default; + + // Formatter::format + absl::optional format( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value formatValue( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const override; }; class HeaderFormatter { @@ -343,15 +375,18 @@ class HeadersByteSizeFormatter : public FormatterProvider { HeadersByteSizeFormatter(const HeaderType header_type); - absl::optional format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo&, absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, + absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, + absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; private: uint64_t extractHeadersByteSize(const Http::RequestHeaderMap& request_headers, @@ -369,13 +404,14 @@ class RequestHeaderFormatter : public FormatterProvider, HeaderFormatter { absl::optional max_length); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; }; /** @@ -387,13 +423,14 @@ class ResponseHeaderFormatter : public FormatterProvider, HeaderFormatter { absl::optional max_length); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; }; /** @@ -405,13 +442,15 @@ class ResponseTrailerFormatter : public FormatterProvider, HeaderFormatter { absl::optional max_length); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, + absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; }; class GrpcStatusFormatter : public FormatterProvider, HeaderFormatter { @@ -426,14 +465,15 @@ class GrpcStatusFormatter : public FormatterProvider, HeaderFormatter { absl::optional max_length, Format format); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, + absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; static Format parseFormat(absl::string_view format); @@ -450,12 +490,14 @@ class StreamInfoFormatter : public FormatterProvider { const absl::optional& = absl::nullopt); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; class FieldExtractor { public: @@ -494,15 +536,17 @@ class MetadataFormatter : public FormatterProvider { MetadataFormatter(const std::string& filter_namespace, const std::vector& path, absl::optional max_length, GetMetadataFunction get); - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo& stream_info, + absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo& stream_info, + absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; protected: absl::optional @@ -556,12 +600,14 @@ class FilterStateFormatter : public FormatterProvider { bool serialize_as_string, bool is_upstream = false); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; private: const Envoy::StreamInfo::FilterState::Object* @@ -586,12 +632,14 @@ class SystemTimeFormatter : public FormatterProvider { SystemTimeFormatter(const std::string& format, TimeFieldExtractorPtr f); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; private: const Envoy::DateFormatter date_formatter_; @@ -650,12 +698,14 @@ class EnvironmentFormatter : public FormatterProvider { EnvironmentFormatter(const std::string& key, absl::optional max_length); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; private: ProtobufWkt::Value str_; @@ -672,13 +722,14 @@ class StreamInfoRequestHeaderFormatter : public FormatterProvider, HeaderFormatt absl::optional max_length); // FormatterProvider - absl::optional format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; }; } // namespace Formatter diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index fa4b126419c3..528654411984 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -320,7 +320,7 @@ void ConnectionManagerImpl::doDeferredStreamDestroy(ActiveStream& stream) { stream.deferHeadersAndTrailers(); } else { // For HTTP/1 and HTTP/2, log here as usual. - stream.filter_manager_.log(); + stream.filter_manager_.log(AccessLog::AccessLogType::DownstreamEnd); } stream.filter_manager_.destroyFilters(); @@ -815,7 +815,7 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect // If the request is complete, we've already done the stream-end access-log, and shouldn't // do the periodic log. if (!streamInfo().requestComplete().has_value()) { - filter_manager_.log(); + filter_manager_.log(AccessLog::AccessLogType::DownstreamPeriodic); refreshAccessLogFlushTimer(); } }); @@ -1244,7 +1244,7 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt const bool upgrade_rejected = filter_manager_.createFilterChain() == false; if (connection_manager_.config_.flushAccessLogOnNewRequest()) { - filter_manager_.log(); + filter_manager_.log(AccessLog::AccessLogType::DownstreamStart); } // TODO if there are no filters when starting a filter iteration, the connection manager diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index b7621c85a7fb..09c131d673d5 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -654,7 +654,7 @@ class FilterManager : public ScopeTrackedObject, // FilterChainManager void applyFilterFactoryCb(FilterContext context, FilterFactoryCb& factory) override; - void log() { + void log(AccessLog::AccessLogType access_log_type) { RequestHeaderMap* request_headers = nullptr; if (filter_manager_callbacks_.requestHeaders()) { request_headers = filter_manager_callbacks_.requestHeaders().ptr(); @@ -669,7 +669,8 @@ class FilterManager : public ScopeTrackedObject, } for (const auto& log_handler : access_log_handlers_) { - log_handler->log(request_headers, response_headers, response_trailers, streamInfo()); + log_handler->log(request_headers, response_headers, response_trailers, streamInfo(), + access_log_type); } } diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index f68ce1a9a249..6766a5cb5987 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -206,7 +206,7 @@ void UpstreamRequest::cleanUp() { } stream_info_.onRequestComplete(); - upstreamLog(); + upstreamLog(AccessLog::AccessLogType::UpstreamEnd); while (downstream_data_disabled_ != 0) { parent_.callbacks()->onDecoderFilterBelowWriteBufferLowWatermark(); @@ -219,10 +219,10 @@ void UpstreamRequest::cleanUp() { parent_.callbacks()->dispatcher().deferredDelete(std::move(filter_manager_callbacks_)); } -void UpstreamRequest::upstreamLog() { +void UpstreamRequest::upstreamLog(AccessLog::AccessLogType access_log_type) { for (const auto& upstream_log : parent_.config().upstream_logs_) { upstream_log->log(parent_.downstreamHeaders(), upstream_headers_.get(), - upstream_trailers_.get(), stream_info_); + upstream_trailers_.get(), stream_info_, access_log_type); } } @@ -374,7 +374,7 @@ void UpstreamRequest::acceptHeadersFromRouter(bool end_stream) { // If the request is complete, we've already done the stream-end upstream log, and shouldn't // do the periodic log. if (!streamInfo().requestComplete().has_value()) { - upstreamLog(); + upstreamLog(AccessLog::AccessLogType::UpstreamPeriodic); resetUpstreamLogFlushTimer(); } }); @@ -646,7 +646,7 @@ void UpstreamRequest::onPoolReady(std::unique_ptr&& upstream, stream_info_.setRequestHeaders(*parent_.downstreamHeaders()); if (parent_.config().flush_upstream_log_on_upstream_stream_) { - upstreamLog(); + upstreamLog(AccessLog::AccessLogType::UpstreamPoolReady); } for (auto* callback : upstream_callbacks_) { diff --git a/source/common/router/upstream_request.h b/source/common/router/upstream_request.h index 378995ad09ff..a91b75c833f2 100644 --- a/source/common/router/upstream_request.h +++ b/source/common/router/upstream_request.h @@ -184,7 +184,7 @@ class UpstreamRequest : public Logger::Loggable, void resetPerTryIdleTimer(); void onPerTryTimeout(); void onPerTryIdleTimeout(); - void upstreamLog(); + void upstreamLog(AccessLog::AccessLogType access_log_type); void resetUpstreamLogFlushTimer(); RouterFilterInterface& parent_; diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index d81621702167..421cf41eb9a1 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -213,7 +213,8 @@ Filter::~Filter() { // Flush the final end stream access log entry. for (const auto& access_log : config_->accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, getStreamInfo()); + access_log->log(nullptr, nullptr, nullptr, getStreamInfo(), + AccessLog::AccessLogType::TcpConnectionEnd); } ASSERT(generic_conn_pool_ == nullptr); @@ -813,7 +814,8 @@ void Filter::onUpstreamConnection() { if (config_->flushAccessLogOnConnected()) { for (const auto& access_log : config_->accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, getStreamInfo()); + access_log->log(nullptr, nullptr, nullptr, getStreamInfo(), + AccessLog::AccessLogType::TcpUpstreamConnected); } } } @@ -838,7 +840,8 @@ void Filter::onMaxDownstreamConnectionDuration() { void Filter::onAccessLogFlushInterval() { for (const auto& access_log : config_->accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, getStreamInfo()); + access_log->log(nullptr, nullptr, nullptr, getStreamInfo(), + AccessLog::AccessLogType::TcpPeriodic); } resetAccessLogFlushTimer(); } diff --git a/source/extensions/access_loggers/common/access_log_base.cc b/source/extensions/access_loggers/common/access_log_base.cc index f5ed7675f85f..73efca022219 100644 --- a/source/extensions/access_loggers/common/access_log_base.cc +++ b/source/extensions/access_loggers/common/access_log_base.cc @@ -11,7 +11,8 @@ namespace Common { void ImplBase::log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { if (!request_headers) { request_headers = Http::StaticEmptyHeaders::get().request_headers.get(); } @@ -25,7 +26,8 @@ void ImplBase::log(const Http::RequestHeaderMap* request_headers, !filter_->evaluate(stream_info, *request_headers, *response_headers, *response_trailers)) { return; } - return emitLog(*request_headers, *response_headers, *response_trailers, stream_info); + return emitLog(*request_headers, *response_headers, *response_trailers, stream_info, + access_log_type); } } // namespace Common diff --git a/source/extensions/access_loggers/common/access_log_base.h b/source/extensions/access_loggers/common/access_log_base.h index 5ae041c72951..648d73226eb7 100644 --- a/source/extensions/access_loggers/common/access_log_base.h +++ b/source/extensions/access_loggers/common/access_log_base.h @@ -29,7 +29,8 @@ class ImplBase : public AccessLog::Instance { void log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) override; + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; private: /** @@ -40,10 +41,12 @@ class ImplBase : public AccessLog::Instance { * @param stream_info supplies additional information about the request not * contained in the request headers. */ - virtual void emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) PURE; + virtual void + emitLog(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) PURE; AccessLog::FilterPtr filter_; }; diff --git a/source/extensions/access_loggers/common/file_access_log_impl.cc b/source/extensions/access_loggers/common/file_access_log_impl.cc index 9b074362e55a..2190a300f0e2 100644 --- a/source/extensions/access_loggers/common/file_access_log_impl.cc +++ b/source/extensions/access_loggers/common/file_access_log_impl.cc @@ -15,9 +15,10 @@ FileAccessLog::FileAccessLog(const Filesystem::FilePathAndType& access_log_file_ void FileAccessLog::emitLog(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { log_file_->write(formatter_->format(request_headers, response_headers, response_trailers, - stream_info, absl::string_view())); + stream_info, absl::string_view(), access_log_type)); } } // namespace File diff --git a/source/extensions/access_loggers/common/file_access_log_impl.h b/source/extensions/access_loggers/common/file_access_log_impl.h index 6a4f0daf7df9..45a8be54ab2d 100644 --- a/source/extensions/access_loggers/common/file_access_log_impl.h +++ b/source/extensions/access_loggers/common/file_access_log_impl.h @@ -19,10 +19,12 @@ class FileAccessLog : public Common::ImplBase { private: // Common::ImplBase - void emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) override; + void + emitLog(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; AccessLog::AccessLogFileSharedPtr log_file_; Formatter::FormatterPtr formatter_; diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc index b16dd6fbff87..7155a13ee5fe 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc @@ -155,7 +155,8 @@ void Utility::responseFlagsToAccessLogResponseFlags( void Utility::extractCommonAccessLogProperties( envoy::data::accesslog::v3::AccessLogCommon& common_access_log, const Http::RequestHeaderMap& request_header, const StreamInfo::StreamInfo& stream_info, - const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config) { + const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config, + AccessLog::AccessLogType access_log_type) { // TODO(mattklein123): Populate sample_rate field. if (stream_info.downstreamAddressProvider().remoteAddress() != nullptr) { Network::Utility::addressToProtobufAddress( @@ -321,7 +322,7 @@ void Utility::extractCommonAccessLogProperties( // If the stream is not complete, then this log entry is intermediate log entry. if (!stream_info.requestComplete().has_value()) { - common_access_log.set_intermediate_log_entry(true); + common_access_log.set_intermediate_log_entry(true); // Deprecated field } // Set stream unique id from the stream info. @@ -337,6 +338,8 @@ void Utility::extractCommonAccessLogProperties( common_access_log.set_upstream_wire_bytes_sent(bytes_meter->wireBytesSent()); common_access_log.set_upstream_wire_bytes_received(bytes_meter->wireBytesReceived()); } + + common_access_log.set_access_log_type(access_log_type); } } // namespace GrpcCommon diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.h b/source/extensions/access_loggers/grpc/grpc_access_log_utils.h index f355d1f116c7..9f4f1e07fbd5 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_utils.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/access_log/access_log.h" #include "envoy/data/accesslog/v3/accesslog.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" #include "envoy/stream_info/stream_info.h" @@ -15,7 +16,8 @@ class Utility { envoy::data::accesslog::v3::AccessLogCommon& common_access_log, const Http::RequestHeaderMap& request_header, const StreamInfo::StreamInfo& stream_info, const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& - filter_states_to_log); + filter_states_to_log, + AccessLog::AccessLogType access_log_type); static void responseFlagsToAccessLogResponseFlags( envoy::data::accesslog::v3::AccessLogCommon& common_access_log, diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc index c619ed479fbf..1fe5c36634d1 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc @@ -53,13 +53,14 @@ HttpGrpcAccessLog::HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, void HttpGrpcAccessLog::emitLog(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { // Common log properties. // TODO(mattklein123): Populate sample_rate field. envoy::data::accesslog::v3::HTTPAccessLogEntry log_entry; GrpcCommon::Utility::extractCommonAccessLogProperties(*log_entry.mutable_common_properties(), request_headers, stream_info, - config_->common_config()); + config_->common_config(), access_log_type); if (stream_info.protocol()) { switch (stream_info.protocol().value()) { diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h index 6cfaf97d5617..47411915f38a 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h @@ -44,10 +44,12 @@ class HttpGrpcAccessLog : public Common::ImplBase { }; // Common::ImplBase - void emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) override; + void + emitLog(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; const HttpGrpcAccessLogConfigConstSharedPtr config_; const ThreadLocal::SlotPtr tls_slot_; diff --git a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc index 822d24bceb37..4213549465ca 100644 --- a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc @@ -34,12 +34,13 @@ TcpGrpcAccessLog::TcpGrpcAccessLog(AccessLog::FilterPtr&& filter, void TcpGrpcAccessLog::emitLog(const Http::RequestHeaderMap& request_header, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { // Common log properties. envoy::data::accesslog::v3::TCPAccessLogEntry log_entry; GrpcCommon::Utility::extractCommonAccessLogProperties(*log_entry.mutable_common_properties(), request_header, stream_info, - config_->common_config()); + config_->common_config(), access_log_type); envoy::data::accesslog::v3::ConnectionProperties& connection_properties = *log_entry.mutable_connection_properties(); diff --git a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h index 897091d0367a..acd552e78fab 100644 --- a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h @@ -43,10 +43,12 @@ class TcpGrpcAccessLog : public Common::ImplBase { }; // Common::ImplBase - void emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) override; + void + emitLog(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; const TcpGrpcAccessLogConfigConstSharedPtr config_; const ThreadLocal::SlotPtr tls_slot_; diff --git a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc index bcea4208784f..293374edda86 100644 --- a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc @@ -79,7 +79,8 @@ AccessLog::AccessLog( void AccessLog::emitLog(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, + Envoy::AccessLog::AccessLogType access_log_type) { opentelemetry::proto::logs::v1::LogRecord log_entry; log_entry.set_time_unix_nano(std::chrono::duration_cast( stream_info.startTime().time_since_epoch()) @@ -87,12 +88,14 @@ void AccessLog::emitLog(const Http::RequestHeaderMap& request_headers, // Unpacking the body "KeyValueList" to "AnyValue". if (body_formatter_) { - const auto formatted_body = unpackBody(body_formatter_->format( - request_headers, response_headers, response_trailers, stream_info, absl::string_view())); + const auto formatted_body = + unpackBody(body_formatter_->format(request_headers, response_headers, response_trailers, + stream_info, absl::string_view(), access_log_type)); *log_entry.mutable_body() = formatted_body; } - const auto formatted_attributes = attributes_formatter_->format( - request_headers, response_headers, response_trailers, stream_info, absl::string_view()); + const auto formatted_attributes = + attributes_formatter_->format(request_headers, response_headers, response_trailers, + stream_info, absl::string_view(), access_log_type); *log_entry.mutable_attributes() = formatted_attributes.values(); tls_slot_->getTyped().logger_->log(std::move(log_entry)); diff --git a/source/extensions/access_loggers/open_telemetry/access_log_impl.h b/source/extensions/access_loggers/open_telemetry/access_log_impl.h index b35db3bd32d9..fa029aa9b1d2 100644 --- a/source/extensions/access_loggers/open_telemetry/access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/access_log_impl.h @@ -52,7 +52,9 @@ class AccessLog : public Common::ImplBase { void emitLog(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) override; + const StreamInfo::StreamInfo& stream_info, + Envoy::AccessLog::AccessLogType access_log_type = + Envoy::AccessLog::AccessLogType::NotSet) override; const ThreadLocal::SlotPtr tls_slot_; const GrpcAccessLoggerCacheSharedPtr access_logger_cache_; diff --git a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc index d590ca2d5ca2..bf1fb4acddab 100644 --- a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc +++ b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc @@ -84,7 +84,7 @@ ::opentelemetry::proto::common::v1::AnyValue OpenTelemetryFormatter::providersCa const std::vector& providers, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const { + absl::string_view local_reply_body, AccessLog::AccessLogType access_log_type) const { ASSERT(!providers.empty()); ::opentelemetry::proto::common::v1::AnyValue output; std::vector bits(providers.size()); @@ -92,7 +92,7 @@ ::opentelemetry::proto::common::v1::AnyValue OpenTelemetryFormatter::providersCa [&](const Formatter::FormatterProviderPtr& provider) { return provider ->format(request_headers, response_headers, response_trailers, stream_info, - local_reply_body) + local_reply_body, access_log_type) .value_or(DefaultUnspecifiedValueString); }); output.set_string_value(absl::StrJoin(bits, "")); @@ -129,11 +129,11 @@ OpenTelemetryFormatter::openTelemetryFormatListCallback( ::opentelemetry::proto::common::v1::KeyValueList OpenTelemetryFormatter::format( const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const { + absl::string_view local_reply_body, AccessLog::AccessLogType access_log_type) const { OpenTelemetryFormatMapVisitor visitor{ [&](const std::vector& providers) { return providersCallback(providers, request_headers, response_headers, response_trailers, - stream_info, local_reply_body); + stream_info, local_reply_body, access_log_type); }, [&, this](const OpenTelemetryFormatter::OpenTelemetryFormatMapWrapper& format_map) { return openTelemetryFormatMapCallback(format_map, visitor); diff --git a/source/extensions/access_loggers/open_telemetry/substitution_formatter.h b/source/extensions/access_loggers/open_telemetry/substitution_formatter.h index dcbe0c050c54..89a050c584b0 100644 --- a/source/extensions/access_loggers/open_telemetry/substitution_formatter.h +++ b/source/extensions/access_loggers/open_telemetry/substitution_formatter.h @@ -32,7 +32,8 @@ class OpenTelemetryFormatter { format(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body) const; + const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const; private: struct OpenTelemetryFormatMapWrapper; @@ -74,13 +75,13 @@ class OpenTelemetryFormatter { }; // Methods for doing the actual formatting. - ::opentelemetry::proto::common::v1::AnyValue - providersCallback(const std::vector& providers, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const; + ::opentelemetry::proto::common::v1::AnyValue providersCallback( + const std::vector& providers, + const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const; ::opentelemetry::proto::common::v1::AnyValue openTelemetryFormatMapCallback( const OpenTelemetryFormatter::OpenTelemetryFormatMapWrapper& format_map, const OpenTelemetryFormatMapVisitor& visitor) const; diff --git a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h index 6faf2c2fc621..824df76b0644 100644 --- a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h +++ b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h @@ -23,7 +23,7 @@ class WasmAccessLog : public AccessLog::Instance { void log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) override { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) override { if (filter_ && request_headers && response_headers && response_trailers) { if (!filter_->evaluate(stream_info, *request_headers, *response_headers, *response_trailers)) { diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index 5972963dcd5f..a2215c2ede4d 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -1456,7 +1456,7 @@ void Context::initializeWriteFilterCallbacks(Network::WriteFilterCallbacks& call void Context::log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { // `log` may be called multiple times due to mid-request logging -- we only want to run on the // last call. if (!stream_info.requestComplete().has_value()) { diff --git a/source/extensions/common/wasm/context.h b/source/extensions/common/wasm/context.h index 2bbe45ebe247..e093df09c2a8 100644 --- a/source/extensions/common/wasm/context.h +++ b/source/extensions/common/wasm/context.h @@ -151,7 +151,8 @@ class Context : public proxy_wasm::ContextBase, void log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) override; + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; uint32_t getLogLevel() override; diff --git a/source/extensions/filters/http/composite/filter.h b/source/extensions/filters/http/composite/filter.h index 796b5d06b5e7..ea6bfc35c7f7 100644 --- a/source/extensions/filters/http/composite/filter.h +++ b/source/extensions/filters/http/composite/filter.h @@ -70,9 +70,10 @@ class Filter : public Http::StreamFilter, void log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) override { + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override { for (const auto& log : access_loggers_) { - log->log(request_headers, response_headers, response_trailers, stream_info); + log->log(request_headers, response_headers, response_trailers, stream_info, access_log_type); } } diff --git a/source/extensions/filters/http/tap/tap_filter.cc b/source/extensions/filters/http/tap/tap_filter.cc index 63c50d01a246..de7b4350411b 100644 --- a/source/extensions/filters/http/tap/tap_filter.cc +++ b/source/extensions/filters/http/tap/tap_filter.cc @@ -70,7 +70,8 @@ Http::FilterTrailersStatus Filter::encodeTrailers(Http::ResponseTrailerMap& trai } void Filter::log(const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&) { + const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&, + AccessLog::AccessLogType) { if (tapper_ != nullptr && tapper_->onDestroyLog()) { config_->stats().rq_tapped_.inc(); } diff --git a/source/extensions/filters/http/tap/tap_filter.h b/source/extensions/filters/http/tap/tap_filter.h index 3840fe7d02bc..e18334fcdfee 100644 --- a/source/extensions/filters/http/tap/tap_filter.h +++ b/source/extensions/filters/http/tap/tap_filter.h @@ -110,7 +110,8 @@ class Filter : public Http::StreamFilter, public AccessLog::Instance { void log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) override; + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; private: FilterConfigSharedPtr config_; diff --git a/source/extensions/formatter/req_without_query/req_without_query.cc b/source/extensions/formatter/req_without_query/req_without_query.cc index b985cd906d76..284e546724e1 100644 --- a/source/extensions/formatter/req_without_query/req_without_query.cc +++ b/source/extensions/formatter/req_without_query/req_without_query.cc @@ -26,11 +26,11 @@ ReqWithoutQuery::ReqWithoutQuery(const std::string& main_header, absl::optional max_length) : main_header_(main_header), alternative_header_(alternative_header), max_length_(max_length) {} -absl::optional ReqWithoutQuery::format(const Http::RequestHeaderMap& request, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +absl::optional +ReqWithoutQuery::format(const Http::RequestHeaderMap& request, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const { const Http::HeaderEntry* header = findHeader(request); if (!header) { return absl::nullopt; @@ -42,11 +42,11 @@ absl::optional ReqWithoutQuery::format(const Http::RequestHeaderMap return val; } -ProtobufWkt::Value ReqWithoutQuery::formatValue(const Http::RequestHeaderMap& request, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +ProtobufWkt::Value +ReqWithoutQuery::formatValue(const Http::RequestHeaderMap& request, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const { const Http::HeaderEntry* header = findHeader(request); if (!header) { return ValueUtil::nullValue(); diff --git a/source/extensions/formatter/req_without_query/req_without_query.h b/source/extensions/formatter/req_without_query/req_without_query.h index 07916f782239..a937d3b7dbd1 100644 --- a/source/extensions/formatter/req_without_query/req_without_query.h +++ b/source/extensions/formatter/req_without_query/req_without_query.h @@ -18,10 +18,10 @@ class ReqWithoutQuery : public ::Envoy::Formatter::FormatterProvider { absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::string_view, AccessLog::AccessLogType) const override; ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::string_view, AccessLog::AccessLogType) const override; private: const Http::HeaderEntry* findHeader(const Http::HeaderMap& headers) const; diff --git a/test/common/formatter/command_extension.cc b/test/common/formatter/command_extension.cc index 22b1bebc963a..99dd2f7f6cfa 100644 --- a/test/common/formatter/command_extension.cc +++ b/test/common/formatter/command_extension.cc @@ -8,16 +8,16 @@ namespace Formatter { absl::optional TestFormatter::format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const { return "TestFormatter"; } ProtobufWkt::Value TestFormatter::formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const { return ValueUtil::stringValue(""); } @@ -45,19 +45,17 @@ ProtobufTypes::MessagePtr TestCommandFactory::createEmptyConfigProto() { std::string TestCommandFactory::name() const { return "envoy.formatter.TestFormatter"; } -absl::optional AdditionalFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +absl::optional +AdditionalFormatter::format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return "AdditionalFormatter"; } -ProtobufWkt::Value AdditionalFormatter::formatValue(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo&, - absl::string_view) const { +ProtobufWkt::Value +AdditionalFormatter::formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const { return ValueUtil::stringValue(""); } diff --git a/test/common/formatter/command_extension.h b/test/common/formatter/command_extension.h index baf9d41db05c..e0c049b506a3 100644 --- a/test/common/formatter/command_extension.h +++ b/test/common/formatter/command_extension.h @@ -13,12 +13,14 @@ namespace Formatter { class TestFormatter : public FormatterProvider { public: // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; }; class TestCommandParser : public CommandParser { @@ -38,12 +40,14 @@ class TestCommandFactory : public CommandParserFactory { class AdditionalFormatter : public FormatterProvider { public: // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + absl::optional + format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value + formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; }; class AdditionalCommandParser : public CommandParser { diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index e828c00d0489..c83cbac73bda 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -2130,9 +2130,9 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLog) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([&](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_EQ(&decoder_->streamInfo(), &stream_info); EXPECT_TRUE(stream_info.responseCode()); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); @@ -2196,9 +2196,9 @@ TEST_F(HttpConnectionManagerImplTest, TestFilterCanEnrichAccessLogs) { filter->callbacks_->streamInfo().setDynamicMetadata("metadata_key", metadata); })); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { auto dynamic_meta = stream_info.dynamicMetadata().filter_metadata().at("metadata_key"); EXPECT_EQ("value", dynamic_meta.fields().at("field").string_value()); })); @@ -2240,9 +2240,9 @@ TEST_F(HttpConnectionManagerImplTest, TestRemoteDownstreamDisconnectAccessLog) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_FALSE(stream_info.responseCode()); EXPECT_TRUE(stream_info.hasAnyResponseFlag()); EXPECT_TRUE( @@ -2284,9 +2284,9 @@ TEST_F(HttpConnectionManagerImplTest, TestLocalDownstreamDisconnectAccessLog) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_EQ("downstream_local_disconnect(reason_for_local_close)", stream_info.responseCodeDetails().value()); })); @@ -2326,9 +2326,9 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogWithTrailers) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_TRUE(stream_info.responseCode()); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); @@ -2379,9 +2379,9 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogWithInvalidRequest) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_TRUE(stream_info.responseCode()); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(400)); EXPECT_EQ("missing_host_header", stream_info.responseCodeDetails().value()); @@ -2425,17 +2425,21 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogOnNewRequest) { flush_access_log_on_new_request_ = true; - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { // First call to log() is made when a new HTTP request has been received // On the first call it is expected that there is no response code. + EXPECT_EQ(AccessLog::AccessLogType::DownstreamStart, access_log_type); EXPECT_FALSE(stream_info.responseCode()); })) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { // Second call to log() is made when filter is destroyed, so it is expected // that the response code is available and matches the response headers. + EXPECT_EQ(AccessLog::AccessLogType::DownstreamEnd, access_log_type); EXPECT_TRUE(stream_info.responseCode()); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); @@ -2507,11 +2511,12 @@ TEST_F(HttpConnectionManagerImplTest, TestPeriodicAccessLogging) { Buffer::OwnedImpl fake_input("1234"); conn_manager_->onData(fake_input, false); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .Times(2) - .WillRepeatedly( - Invoke([&](const HeaderMap* request_headers, const HeaderMap* response_headers, - const HeaderMap*, const StreamInfo::StreamInfo& stream_info) { + .WillRepeatedly(Invoke( + [&](const HeaderMap* request_headers, const HeaderMap* response_headers, const HeaderMap*, + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType access_log_type) { + EXPECT_EQ(AccessLog::AccessLogType::DownstreamPeriodic, access_log_type); EXPECT_EQ(&decoder_->streamInfo(), &stream_info); EXPECT_THAT(request_headers, testing::NotNull()); EXPECT_THAT(response_headers, testing::IsNull()); @@ -2521,9 +2526,11 @@ TEST_F(HttpConnectionManagerImplTest, TestPeriodicAccessLogging) { EXPECT_CALL(*periodic_log_timer, enableTimer(*access_log_flush_interval_, _)).Times(2); periodic_log_timer->invokeCallback(); periodic_log_timer->invokeCallback(); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([&](const HeaderMap* request_headers, const HeaderMap* response_headers, - const HeaderMap*, const StreamInfo::StreamInfo& stream_info) { + const HeaderMap*, const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { + EXPECT_EQ(AccessLog::AccessLogType::DownstreamEnd, access_log_type); EXPECT_EQ(&decoder_->streamInfo(), &stream_info); EXPECT_THAT(request_headers, testing::NotNull()); EXPECT_THAT(response_headers, testing::NotNull()); @@ -2634,9 +2641,9 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogSsl) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_TRUE(stream_info.responseCode()); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); @@ -2877,9 +2884,9 @@ TEST_F(HttpConnectionManagerImplTest, TestStreamIdleAccessLog) { std::string response_body; EXPECT_CALL(response_encoder_, encodeData(_, true)).WillOnce(AddBufferToString(&response_body)); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_TRUE(stream_info.responseCode()); EXPECT_TRUE(stream_info.hasAnyResponseFlag()); EXPECT_TRUE(stream_info.hasResponseFlag(StreamInfo::ResponseFlag::StreamIdleTimeout)); diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index ce870258687a..8f1139e44839 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -173,9 +173,9 @@ TEST_F(HttpConnectionManagerImplTest, TestDownstreamProtocolErrorAccessLog) { access_logs_ = {handler}; setup(false, ""); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_FALSE(stream_info.responseCode()); EXPECT_TRUE(stream_info.hasAnyResponseFlag()); EXPECT_TRUE(stream_info.hasResponseFlag(StreamInfo::ResponseFlag::DownstreamProtocolError)); @@ -206,9 +206,9 @@ TEST_F(HttpConnectionManagerImplTest, TestDownstreamProtocolErrorAfterHeadersAcc return true; })); - EXPECT_CALL(*handler, log(_, _, _, _)) + EXPECT_CALL(*handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_FALSE(stream_info.responseCode()); EXPECT_TRUE(stream_info.hasAnyResponseFlag()); EXPECT_TRUE(stream_info.hasResponseFlag(StreamInfo::ResponseFlag::DownstreamProtocolError)); @@ -247,9 +247,9 @@ TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWriteAndDelay, _)); - EXPECT_CALL(*log_handler, log(_, _, _, _)) + EXPECT_CALL(*log_handler, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { ASSERT_TRUE(stream_info.responseCodeDetails().has_value()); EXPECT_EQ("codec_error:too_many_outbound_frames", stream_info.responseCodeDetails().value()); @@ -1519,9 +1519,9 @@ TEST_F(HttpConnectionManagerImplTest, HitFilterWatermarkLimits) { EXPECT_CALL(callbacks2, onBelowWriteBufferLowWatermark()).Times(0); encoder_filters_[1]->callbacks_->setEncoderBufferLimit((buffer_len + 1) * 2); - EXPECT_CALL(*log_handler_, log(_, _, _, _)) + EXPECT_CALL(*log_handler_, log(_, _, _, _, _)) .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { EXPECT_FALSE(stream_info.hasAnyResponseFlag()); })); diff --git a/test/common/quic/envoy_quic_server_stream_test.cc b/test/common/quic/envoy_quic_server_stream_test.cc index 6b1a2737836c..f2a79a6c6b67 100644 --- a/test/common/quic/envoy_quic_server_stream_test.cc +++ b/test/common/quic/envoy_quic_server_stream_test.cc @@ -845,7 +845,7 @@ TEST_F(EnvoyQuicServerStreamTest, StatsGathererLogsOnStreamDestruction) { EXPECT_GT(quic_stream_->statsGatherer()->bytesOutstanding(), 0); // Close the stream; incoming acks will no longer invoke the stats gatherer but // the stats gatherer should log on stream close despite not receiving final downstream ack. - EXPECT_CALL(*mock_logger, log(_, _, _, _)); + EXPECT_CALL(*mock_logger, log(_, _, _, _, _)); quic_stream_->resetStream(Http::StreamResetReason::LocalRefusedStreamReset); } diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index cc9bbce231bb..8291e357148e 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -72,7 +72,8 @@ class TestAccessLog : public AccessLog::Instance { explicit TestAccessLog(std::function func) : func_(func) {} void log(const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo& info) override { + const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo& info, + AccessLog::AccessLogType) override { func_(info); } diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc index cffe9e1c89c8..62d9fd8dafc2 100644 --- a/test/common/router/router_upstream_log_test.cc +++ b/test/common/router/router_upstream_log_test.cc @@ -429,7 +429,7 @@ name: accesslog "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog log_format: text_format_source: - inline_string: "%UPSTREAM_CLUSTER%" + inline_string: "%UPSTREAM_CLUSTER% %ACCESS_LOG_TYPE%" path: "/dev/null" )EOF"; @@ -442,8 +442,11 @@ name: accesslog // It is expected that there will be two log records, one when a new request is received // and one when the request is finished, due to 'flush_upstream_log_on_upstream_stream' enabled EXPECT_EQ(output_.size(), 2U); - EXPECT_EQ(output_.front(), "cluster_0"); - EXPECT_EQ(output_.back(), "cluster_0"); + EXPECT_EQ( + output_.front(), + absl::StrCat("cluster_0 ", AccessLogType_Name(AccessLog::AccessLogType::UpstreamPoolReady))); + EXPECT_EQ(output_.back(), + absl::StrCat("cluster_0 ", AccessLogType_Name(AccessLog::AccessLogType::UpstreamEnd))); } TEST_F(RouterUpstreamLogTest, PeriodicLog) { @@ -453,7 +456,7 @@ name: accesslog "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog log_format: text_format_source: - inline_string: "%UPSTREAM_CLUSTER%" + inline_string: "%ACCESS_LOG_TYPE%" path: "/dev/null" )EOF"; @@ -494,13 +497,13 @@ name: accesslog EXPECT_CALL(*periodic_log_flush_, enableTimer(_, _)); periodic_log_flush_->invokeCallback(); EXPECT_EQ(output_.size(), 1U); - EXPECT_EQ(output_.front(), "cluster_0"); + EXPECT_EQ(output_.front(), AccessLogType_Name(AccessLog::AccessLogType::UpstreamPeriodic)); EXPECT_CALL(*periodic_log_flush_, enableTimer(_, _)); periodic_log_flush_->invokeCallback(); EXPECT_EQ(output_.size(), 2U); - EXPECT_EQ(output_.front(), "cluster_0"); - EXPECT_EQ(output_.back(), "cluster_0"); + EXPECT_EQ(output_.front(), AccessLogType_Name(AccessLog::AccessLogType::UpstreamPeriodic)); + EXPECT_EQ(output_.back(), AccessLogType_Name(AccessLog::AccessLogType::UpstreamPeriodic)); Http::ResponseHeaderMapPtr response_headers(new Http::TestResponseHeaderMapImpl()); response_headers->setStatus(200); diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index 3ac254c69460..735d9ba6ae41 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -956,9 +956,9 @@ TEST_F(TcpProxyTest, AccessLogDownstreamAddress) { EXPECT_EQ(access_log_data_, "1.1.1.1 1.1.1.2:20000"); } -// Test that intermediate log entry by field %DURATION%. +// Test that intermediate log entry by field %ACCESS_LOG_TYPE%. TEST_F(TcpProxyTest, IntermediateLogEntry) { - auto config = accessLogConfig("%DURATION%"); + auto config = accessLogConfig("%ACCESS_LOG_TYPE%"); config.mutable_access_log_options()->mutable_access_log_flush_interval()->set_seconds(1); config.mutable_idle_timeout()->set_seconds(0); @@ -973,14 +973,17 @@ TEST_F(TcpProxyTest, IntermediateLogEntry) { flush_timer->invokeCallback(); // No valid duration until the connection is closed. - EXPECT_EQ(access_log_data_.value(), fmt::format("-")); + EXPECT_EQ(access_log_data_.value(), AccessLogType_Name(AccessLog::AccessLogType::TcpPeriodic)); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); filter_.reset(); + + EXPECT_EQ(access_log_data_.value(), + AccessLogType_Name(AccessLog::AccessLogType::TcpConnectionEnd)); } TEST_F(TcpProxyTest, TestAccessLogOnUpstreamConnected) { - auto config = accessLogConfig("%UPSTREAM_HOST%"); + auto config = accessLogConfig("%UPSTREAM_HOST% %ACCESS_LOG_TYPE%"); config.mutable_access_log_options()->set_flush_access_log_on_connected(true); setup(1, config); @@ -989,10 +992,16 @@ TEST_F(TcpProxyTest, TestAccessLogOnUpstreamConnected) { // Default access log will only be flushed after the stream is closed. // Passing the following check makes sure that the access log was flushed // before the stream was closed. - EXPECT_EQ(access_log_data_, "127.0.0.1:80"); + EXPECT_EQ(access_log_data_.value(), + absl::StrCat("127.0.0.1:80 ", + AccessLogType_Name(AccessLog::AccessLogType::TcpUpstreamConnected))); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); filter_.reset(); + + EXPECT_EQ(access_log_data_.value(), + absl::StrCat("127.0.0.1:80 ", + AccessLogType_Name(AccessLog::AccessLogType::TcpConnectionEnd))); } TEST_F(TcpProxyTest, AccessLogUpstreamSSLConnection) { diff --git a/test/extensions/access_loggers/common/access_log_base_test.cc b/test/extensions/access_loggers/common/access_log_base_test.cc index 021459669616..fbcd559b4754 100644 --- a/test/extensions/access_loggers/common/access_log_base_test.cc +++ b/test/extensions/access_loggers/common/access_log_base_test.cc @@ -24,7 +24,8 @@ class TestImpl : public ImplBase { private: void emitLog(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&) override { + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) override { count_++; } diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index f8ef1ea55dd3..1b9c45708f6c 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -142,6 +142,7 @@ class HttpGrpcAccessLogTest : public testing::Test { socket_address: address: "127.0.0.2" port_value: 0 + access_log_type: NotSet upstream_local_address: socket_address: address: "127.1.2.3" @@ -214,6 +215,7 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -263,6 +265,7 @@ response: {} socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -340,6 +343,7 @@ response: {} socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -420,6 +424,7 @@ protocol_version: HTTP10 socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -477,6 +482,7 @@ response: {} socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -542,6 +548,7 @@ response: {} socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -597,6 +604,7 @@ response: {} socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -652,6 +660,7 @@ response: {} socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -707,6 +716,7 @@ response: {} socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -767,6 +777,7 @@ response: {} socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -859,6 +870,7 @@ TEST_F(HttpGrpcAccessLogTest, MarshallingAdditionalHeaders) { socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -945,6 +957,7 @@ TEST_F(HttpGrpcAccessLogTest, SanitizeUTF8) { socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: "127.0.0.3" @@ -1012,6 +1025,7 @@ tag: ltag socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet downstream_local_address: socket_address: address: "127.0.0.2" @@ -1070,6 +1084,7 @@ tag: mtag socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet upstream_remote_address: socket_address: address: "10.0.0.1" @@ -1125,6 +1140,7 @@ tag: mtag socket_address: address: "127.0.0.1" port_value: 0 + access_log_type: NotSet upstream_remote_address: socket_address: address: "10.0.0.1" diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 1f2af19defc0..d871b2eed496 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -143,6 +143,7 @@ TEST_P(AccessLogIntegrationTest, BasicAccessLogFlow) { no_route_found: true downstream_wire_bytes_sent: 178 downstream_wire_bytes_received: 38 + access_log_type: DownstreamEnd protocol_version: HTTP11 request: scheme: http @@ -171,6 +172,7 @@ TEST_P(AccessLogIntegrationTest, BasicAccessLogFlow) { no_route_found: true downstream_wire_bytes_sent: 178 downstream_wire_bytes_received: 38 + access_log_type: DownstreamEnd protocol_version: HTTP11 request: downstream_header_bytes_received: 11 @@ -224,6 +226,7 @@ TEST_P(AccessLogIntegrationTest, BasicAccessLogFlow) { no_route_found: true downstream_wire_bytes_sent: 178 downstream_wire_bytes_received: 38 + access_log_type: DownstreamEnd protocol_version: HTTP11 request: downstream_header_bytes_received: 11 diff --git a/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc index 4371b3fbec2d..f84a0549a5c8 100644 --- a/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc @@ -318,6 +318,7 @@ TEST_P(TcpGrpcAccessLogIntegrationTest, BasicAccessLogFlow) { downstream_direct_remote_address: socket_address: address: {} + access_log_type: NotSet connection_properties: received_bytes: 3 sent_bytes: 5 @@ -377,6 +378,7 @@ TEST_P(TcpGrpcAccessLogIntegrationTest, BasicAccessLogFlowWithIntermediateLog) { downstream_wire_bytes_received: 3 upstream_wire_bytes_sent: 3 upstream_wire_bytes_received: 5 + access_log_type: TcpPeriodic downstream_direct_remote_address: socket_address: address: {} @@ -471,6 +473,7 @@ name: envoy.filters.network.rbac downstream_direct_remote_address: socket_address: address: {} + access_log_type: NotSet connection_properties: received_bytes: 3 sent_bytes: 5 @@ -537,6 +540,7 @@ TEST_P(TcpGrpcAccessLogIntegrationTest, SslTerminatedNoJA3) { downstream_direct_remote_address: socket_address: address: {} + access_log_type: NotSet connection_properties: )EOF", Network::Test::getLoopbackAddressString(ipVersion()), @@ -600,6 +604,7 @@ TEST_P(TcpGrpcAccessLogIntegrationTest, SslTerminatedWithJA3) { downstream_direct_remote_address: socket_address: address: {} + access_log_type: NotSet connection_properties: )EOF", Network::Test::getLoopbackAddressString(ipVersion()), @@ -646,6 +651,7 @@ TEST_P(TcpGrpcAccessLogIntegrationTest, SslNotTerminated) { socket_address: upstream_local_address: socket_address: + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: {} @@ -699,6 +705,7 @@ TEST_P(TcpGrpcAccessLogIntegrationTest, SslNotTerminatedWithJA3) { socket_address: upstream_local_address: socket_address: + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: {} @@ -752,6 +759,7 @@ TEST_P(TcpGrpcAccessLogIntegrationTest, SslNotTerminatedWithJA3NoSNI) { socket_address: upstream_local_address: socket_address: + access_log_type: NotSet downstream_direct_remote_address: socket_address: address: {} diff --git a/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc b/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc index 3980b72b9e79..a63d2d5e1cb9 100644 --- a/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc +++ b/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc @@ -453,7 +453,7 @@ TEST_F(ConnectionHandlerTest, InternalListenerInplaceUpdate) { EXPECT_CALL(manager_, findFilterChain(_, _)).Times(0); EXPECT_CALL(*overridden_filter_chain_manager, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); internal_listener_cb.value().get().onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); diff --git a/test/extensions/filters/http/composite/filter_test.cc b/test/extensions/filters/http/composite/filter_test.cc index 4f15d5c086dd..7ce2e7ffd641 100644 --- a/test/extensions/filters/http/composite/filter_test.cc +++ b/test/extensions/filters/http/composite/filter_test.cc @@ -224,8 +224,8 @@ TEST_F(FilterTest, StreamFilterDelegationMultipleAccessLoggers) { EXPECT_CALL(*encode_filter, onDestroy()); filter_.onDestroy(); - EXPECT_CALL(*access_log_1, log(_, _, _, _)); - EXPECT_CALL(*access_log_2, log(_, _, _, _)); + EXPECT_CALL(*access_log_1, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_2, log(_, _, _, _, _)); filter_.log(nullptr, nullptr, nullptr, StreamInfo::MockStreamInfo()); } diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 42d7ea9fb092..d8227d3178ab 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -406,6 +406,64 @@ name: add-trailers-filter } } +TEST_P(ProtocolIntegrationTest, AccessLogTest) { + if (upstreamProtocol() == Http::CodecType::HTTP3) { + return; + } + + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { + hcm.mutable_access_log_options()->set_flush_access_log_on_new_request(true); + }); + + useAccessLog("%RESPONSE_CODE% %ACCESS_LOG_TYPE%"); + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, {":path", "/test"}, {":scheme", "http"}, {":authority", "host.com"}}); + waitForNextUpstreamRequest(); + EXPECT_EQ(absl::StrCat("0 ", AccessLogType_Name(AccessLog::AccessLogType::DownstreamStart)), + waitForAccessLog(access_log_name_, 0, true)); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + + if (downstream_protocol_ != Http::CodecType::HTTP3) { + EXPECT_EQ(absl::StrCat("200 ", AccessLogType_Name(AccessLog::AccessLogType::DownstreamEnd)), + waitForAccessLog(access_log_name_, 1, true)); + } +} + +TEST_P(ProtocolIntegrationTest, PeriodicAccessLog) { + if (upstreamProtocol() == Http::CodecType::HTTP3) { + return; + } + + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { + hcm.mutable_access_log_options()->mutable_access_log_flush_interval()->set_nanos( + 100000000); // 0.1 seconds + }); + + useAccessLog("%ACCESS_LOG_TYPE%"); + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, {":path", "/test"}, {":scheme", "http"}, {":authority", "host.com"}}); + waitForNextUpstreamRequest(); + EXPECT_EQ(AccessLogType_Name(AccessLog::AccessLogType::DownstreamPeriodic), + waitForAccessLog(access_log_name_)); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + // Regression test for https://github.com/envoyproxy/envoy/issues/9873 TEST_P(ProtocolIntegrationTest, ResponseWithHostHeader) { initialize(); diff --git a/test/integration/tcp_proxy_integration_test.cc b/test/integration/tcp_proxy_integration_test.cc index 23ec227d1d0f..20837ec3f68d 100644 --- a/test/integration/tcp_proxy_integration_test.cc +++ b/test/integration/tcp_proxy_integration_test.cc @@ -569,13 +569,64 @@ TEST_P(TcpProxyIntegrationTest, AccessLogOnUpstreamConnect) { envoy::extensions::access_loggers::file::v3::FileAccessLog access_log_config; access_log_config.set_path(access_log_path); access_log_config.mutable_log_format()->mutable_text_format_source()->set_inline_string( - "DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT=%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%"); + "ACCESS_LOG_TYPE=%ACCESS_LOG_TYPE%"); access_log->mutable_typed_config()->PackFrom(access_log_config); config_blob->PackFrom(tcp_proxy_config); }); - const std::string ip_regex = - (version_ == Network::Address::IpVersion::v4) ? R"EOF(127\.0\.0\.1)EOF" : R"EOF(::1)EOF"; + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + ASSERT_TRUE(tcp_client->write("hello")); + FakeRawConnectionPtr fake_upstream_connection; + + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + auto log_result = waitForAccessLog(access_log_path); + EXPECT_EQ(absl::StrCat("ACCESS_LOG_TYPE=", + AccessLogType_Name(AccessLog::AccessLogType::TcpUpstreamConnected)), + log_result); + + ASSERT_TRUE(fake_upstream_connection->waitForData(5)); + ASSERT_TRUE(tcp_client->write("", true)); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->write("", true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + tcp_client->waitForDisconnect(); + test_server_.reset(); + log_result = waitForAccessLog(access_log_path); + EXPECT_EQ( + absl::StrCat( + "ACCESS_LOG_TYPE=", AccessLogType_Name(AccessLog::AccessLogType::TcpUpstreamConnected), + "ACCESS_LOG_TYPE=", AccessLogType_Name(AccessLog::AccessLogType::TcpConnectionEnd)), + log_result); +} + +TEST_P(TcpProxyIntegrationTest, PeriodicAccessLog) { + std::string access_log_path = TestEnvironment::temporaryPath( + fmt::format("access_log{}{}.txt", version_ == Network::Address::IpVersion::v4 ? "v4" : "v6", + TestUtility::uniqueFilename())); + + setupByteMeterAccessLog(); + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter_chain = listener->mutable_filter_chains(0); + auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); + + ASSERT_TRUE(config_blob->Is()); + auto tcp_proxy_config = + MessageUtil::anyConvert( + *config_blob); + + tcp_proxy_config.mutable_access_log_options()->mutable_access_log_flush_interval()->set_nanos( + 100000000); // 0.1 seconds + auto* access_log = tcp_proxy_config.add_access_log(); + access_log->set_name("accesslog"); + envoy::extensions::access_loggers::file::v3::FileAccessLog access_log_config; + access_log_config.set_path(access_log_path); + access_log_config.mutable_log_format()->mutable_text_format_source()->set_inline_string( + "ACCESS_LOG_TYPE=%ACCESS_LOG_TYPE%"); + access_log->mutable_typed_config()->PackFrom(access_log_config); + config_blob->PackFrom(tcp_proxy_config); + }); initialize(); IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); @@ -584,8 +635,9 @@ TEST_P(TcpProxyIntegrationTest, AccessLogOnUpstreamConnect) { ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); auto log_result = waitForAccessLog(access_log_path); - EXPECT_THAT(log_result, - MatchesRegex(fmt::format("DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT={}", ip_regex))); + EXPECT_EQ( + absl::StrCat("ACCESS_LOG_TYPE=", AccessLogType_Name(AccessLog::AccessLogType::TcpPeriodic)), + log_result); ASSERT_TRUE(fake_upstream_connection->waitForData(5)); ASSERT_TRUE(tcp_client->write("", true)); diff --git a/test/integration/typed_metadata_integration_test.cc b/test/integration/typed_metadata_integration_test.cc index 52caa8b08474..e5c8ce2dc2dc 100644 --- a/test/integration/typed_metadata_integration_test.cc +++ b/test/integration/typed_metadata_integration_test.cc @@ -47,7 +47,8 @@ class MockAccessLog : public AccessLog::Instance { public: MOCK_METHOD(void, log, (const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&)); + const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&, + AccessLog::AccessLogType)); }; class TestAccessLogFactory : public Server::Configuration::AccessLogInstanceFactory { diff --git a/test/integration/upstream_access_log_integration_test.cc b/test/integration/upstream_access_log_integration_test.cc index 4148320aa5d3..c58f481fdfb5 100644 --- a/test/integration/upstream_access_log_integration_test.cc +++ b/test/integration/upstream_access_log_integration_test.cc @@ -198,7 +198,7 @@ TEST_P(UpstreamAccessLogTest, Retry) { envoy::extensions::access_loggers::file::v3::FileAccessLog access_log_config; access_log_config.set_path(log_file); access_log_config.mutable_log_format()->mutable_text_format_source()->set_inline_string( - "%RESPONSE_CODE%\n"); + "%RESPONSE_CODE% %ACCESS_LOG_TYPE%\n"); upstream_log_config->mutable_typed_config()->PackFrom(access_log_config); typed_config->PackFrom(router_config); }); @@ -218,7 +218,8 @@ TEST_P(UpstreamAccessLogTest, Retry) { waitForNextUpstreamRequest({}, std::chrono::milliseconds(300000)); // Start of first stream access log - no response status code yet - EXPECT_THAT(waitForAccessLog(log_file, 0, true), testing::HasSubstr("0")); + EXPECT_EQ(absl::StrCat("0 ", AccessLogType_Name(AccessLog::AccessLogType::UpstreamPoolReady)), + waitForAccessLog(log_file, 0, true)); upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "503"}}, false); @@ -231,12 +232,14 @@ TEST_P(UpstreamAccessLogTest, Retry) { } // End of first request access log - EXPECT_THAT(waitForAccessLog(log_file, 1, true), testing::HasSubstr("503")); + EXPECT_EQ(absl::StrCat("503 ", AccessLogType_Name(AccessLog::AccessLogType::UpstreamEnd)), + waitForAccessLog(log_file, 1, true)); waitForNextUpstreamRequest(); // Start of second stream access log - no response status code yet - EXPECT_THAT(waitForAccessLog(log_file, 2, true), testing::HasSubstr("0")); + EXPECT_EQ(absl::StrCat("0 ", AccessLogType_Name(AccessLog::AccessLogType::UpstreamPoolReady)), + waitForAccessLog(log_file, 2, true)); upstream_request_->encodeHeaders(default_response_headers_, false); upstream_request_->encodeData(512, true); @@ -250,7 +253,58 @@ TEST_P(UpstreamAccessLogTest, Retry) { EXPECT_EQ(512U, response->body().size()); // End of second request access log - EXPECT_THAT(waitForAccessLog(log_file, 3, true), testing::HasSubstr("200")); + EXPECT_EQ(absl::StrCat("200 ", AccessLogType_Name(AccessLog::AccessLogType::UpstreamEnd)), + waitForAccessLog(log_file, 3, true)); +} + +TEST_P(UpstreamAccessLogTest, Periodic) { + auto log_file = TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); + + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { + auto* typed_config = + hcm.mutable_http_filters(hcm.http_filters_size() - 1)->mutable_typed_config(); + + envoy::extensions::filters::http::router::v3::Router router_config; + router_config.mutable_upstream_log_options() + ->mutable_upstream_log_flush_interval() + ->set_nanos(100000000); // 0.1 seconds + + auto* upstream_log_config = router_config.add_upstream_log(); + upstream_log_config->set_name("accesslog"); + envoy::extensions::access_loggers::file::v3::FileAccessLog access_log_config; + access_log_config.set_path(log_file); + access_log_config.mutable_log_format()->mutable_text_format_source()->set_inline_string( + "%ACCESS_LOG_TYPE%\n"); + upstream_log_config->mutable_typed_config()->PackFrom(access_log_config); + typed_config->PackFrom(router_config); + }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + Http::TestRequestHeaderMapImpl headers{ + {":method", "POST"}, {":path", "/api"}, {":authority", "host"}, {":scheme", "http"}}; + auto response = codec_client_->makeRequestWithBody(headers, "hello!"); + + waitForNextUpstreamRequest({}, std::chrono::milliseconds(300000)); + + EXPECT_EQ(AccessLogType_Name(AccessLog::AccessLogType::UpstreamPeriodic), + waitForAccessLog(log_file)); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ("hello!", upstream_request_->body().toString()); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + Buffer::OwnedImpl response_data{"greetings"}; + upstream_request_->encodeData(response_data, true); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + EXPECT_EQ("greetings", response->body()); } } // namespace Envoy diff --git a/test/mocks/access_log/mocks.h b/test/mocks/access_log/mocks.h index bdd1dccaaed9..db6809530c59 100644 --- a/test/mocks/access_log/mocks.h +++ b/test/mocks/access_log/mocks.h @@ -57,7 +57,7 @@ class MockInstance : public Instance { (const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info)); + const StreamInfo::StreamInfo& stream_info, AccessLogType access_log_type)); }; } // namespace AccessLog diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 52f25f229087..376ed11245fa 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -457,7 +457,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::LoggablenumConnections()); EXPECT_CALL(*listener, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } Stats::TestUtil::TestStore stats_store_; @@ -506,7 +506,7 @@ TEST_F(ConnectionHandlerTest, RemoveListenerDuringRebalance) { Network::MockConnectionSocket* connection = new NiceMock(); current_handler->incNumConnections(); #ifndef NDEBUG - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); #endif current_handler->post(Network::ConnectionSocketPtr{connection}); @@ -569,7 +569,7 @@ TEST_F(ConnectionHandlerTest, ListenerConnectionLimitEnforced) { // We expect that listener 2 accepts the connection, so there will be a call to // createServerConnection and active cx should increase, while cx overflow remains the same. - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks2->onAccept( Network::ConnectionSocketPtr{new NiceMock()}); EXPECT_EQ(0, handler_->numConnections()); @@ -621,7 +621,7 @@ TEST_F(ConnectionHandlerTest, RemoveListener) { handler_->addListener(absl::nullopt, *test_listener, runtime_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -667,7 +667,7 @@ TEST_F(ConnectionHandlerTest, RemoveListenerWithMultiAddrs) { handler_->addListener(absl::nullopt, *test_listener, runtime_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks1->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -777,7 +777,7 @@ TEST_F(ConnectionHandlerTest, RebalanceWithMultiAddressListener) { // then mock_connection_balancer1 will balance the connection to the same listener. EXPECT_CALL(*mock_connection_balancer1, pickTargetHandler(_)) .WillOnce(ReturnRef(*current_handler1)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); current_handler1->incNumConnections(); @@ -787,7 +787,7 @@ TEST_F(ConnectionHandlerTest, RebalanceWithMultiAddressListener) { // then mock_connection_balancer2 will balance the connection to the same listener. EXPECT_CALL(*mock_connection_balancer2, pickTargetHandler(_)) .WillOnce(ReturnRef(*current_handler2)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); current_handler2->incNumConnections(); @@ -909,7 +909,7 @@ TEST_F(ConnectionHandlerTest, SetsTransportSocketConnectTimeout) { .WillOnce(Return(std::chrono::seconds(5))); EXPECT_CALL(*server_connection, setTransportSocketConnectTimeout(std::chrono::milliseconds(5 * 1000), _)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(std::make_unique>()); @@ -926,7 +926,7 @@ TEST_F(ConnectionHandlerTest, DestroyCloseConnections) { handler_->addListener(absl::nullopt, *test_listener, runtime_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -950,7 +950,7 @@ TEST_F(ConnectionHandlerTest, CloseDuringFilterChainCreate) { EXPECT_CALL(factory_, createNetworkFilterChain(_, _)).WillOnce(Return(true)); EXPECT_CALL(*connection, state()).WillOnce(Return(Network::Connection::State::Closed)); EXPECT_CALL(*connection, addConnectionCallbacks(_)).Times(0); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -973,7 +973,7 @@ TEST_F(ConnectionHandlerTest, CloseConnectionOnEmptyFilterChain) { EXPECT_CALL(factory_, createNetworkFilterChain(_, _)).WillOnce(Return(false)); EXPECT_CALL(*connection, close(Network::ConnectionCloseType::NoFlush, _)); EXPECT_CALL(*connection, addConnectionCallbacks(_)).Times(0); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -1029,12 +1029,12 @@ TEST_F(ConnectionHandlerTest, NormalRedirect) { EXPECT_EQ(1UL, TestUtility::findCounter(stats_store_, "test.downstream_cx_total")->value()); EXPECT_EQ(1UL, TestUtility::findGauge(stats_store_, "test.downstream_cx_active")->value()); - EXPECT_CALL(*access_log_, log(_, _, _, _)) - .WillOnce( - Invoke([&](const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo& stream_info) { - EXPECT_EQ(alt_address, stream_info.downstreamAddressProvider().localAddress()); - })); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)) + .WillOnce(Invoke([&](const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, + const Http::ResponseTrailerMap*, + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_EQ(alt_address, stream_info.downstreamAddressProvider().localAddress()); + })); connection->close(Network::ConnectionCloseType::NoFlush); dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0UL, TestUtility::findGauge(stats_store_, "downstream_cx_active")->value()); @@ -1099,12 +1099,12 @@ TEST_F(ConnectionHandlerTest, NormalRedirectWithMultiAddrs) { EXPECT_EQ(1UL, TestUtility::findCounter(stats_store_, "test.downstream_cx_total")->value()); EXPECT_EQ(1UL, TestUtility::findGauge(stats_store_, "test.downstream_cx_active")->value()); - EXPECT_CALL(*access_log_, log(_, _, _, _)) - .WillOnce( - Invoke([&](const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo& stream_info) { - EXPECT_EQ(alt_address, stream_info.downstreamAddressProvider().localAddress()); - })); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)) + .WillOnce(Invoke([&](const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, + const Http::ResponseTrailerMap*, + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_EQ(alt_address, stream_info.downstreamAddressProvider().localAddress()); + })); connection->close(Network::ConnectionCloseType::NoFlush); dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0UL, TestUtility::findGauge(stats_store_, "downstream_cx_active")->value()); @@ -1192,7 +1192,7 @@ TEST_F(ConnectionHandlerTest, MatchLatestListener) { EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedListener) { @@ -1252,7 +1252,7 @@ TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedListener) { EXPECT_EQ(1UL, handler_->numConnections()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedAnyAddressListener) { @@ -1312,7 +1312,7 @@ TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedAnyAddressListener) { EXPECT_EQ(1UL, handler_->numConnections()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } TEST_F(ConnectionHandlerTest, FallbackToWildcardListener) { @@ -1362,7 +1362,7 @@ TEST_F(ConnectionHandlerTest, FallbackToWildcardListener) { EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListener) { @@ -1433,7 +1433,7 @@ TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListener) { EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } // This tests the ConnectionHandler's `getBalancedHandlerByAddress` will match @@ -1498,7 +1498,7 @@ TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListenerWithAnyAddressAndIpv4Comp EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } // This tests the ConnectionHandler's `getBalancedHandlerByAddress` will match @@ -1563,7 +1563,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4CompatiableIPv6ListenerWithIpv4CompatFla EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } // This tests the ConnectionHandler's `getBalancedHandlerByAddress` won't match @@ -1627,7 +1627,7 @@ TEST_F(ConnectionHandlerTest, NotMatchIPv6WildcardListenerWithoutIpv4CompatFlag) EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } // This tests the case both "0.0.0.0" and "::" with ipv4_compat are added. The @@ -1707,7 +1707,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag) { EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } // This test is same as above except the Ipv4 listener is added first, then Ipv6 @@ -1787,7 +1787,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag2) EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } // This test the case of an update for listener which listening @@ -1867,7 +1867,7 @@ TEST_F(ConnectionHandlerTest, UpdateIpv4MappedListener) { EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } TEST_F(ConnectionHandlerTest, WildcardListenerWithOriginalDstInbound) { @@ -1917,7 +1917,7 @@ TEST_F(ConnectionHandlerTest, WildcardListenerWithNoOriginalDst) { EXPECT_EQ(1UL, handler_->numConnections()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); } TEST_F(ConnectionHandlerTest, TransportProtocolDefault) { @@ -1932,7 +1932,7 @@ TEST_F(ConnectionHandlerTest, TransportProtocolDefault) { .WillOnce(Return(absl::string_view(""))); EXPECT_CALL(*accepted_socket, setDetectedTransportProtocol(absl::string_view("raw_buffer"))); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}); EXPECT_CALL(*listener, onDestroy()); @@ -1962,7 +1962,7 @@ TEST_F(ConnectionHandlerTest, TransportProtocolCustom) { EXPECT_CALL(*accepted_socket, setDetectedTransportProtocol(dummy)); EXPECT_CALL(*accepted_socket, detectedTransportProtocol()).WillOnce(Return(dummy)); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}); EXPECT_CALL(*listener, onDestroy()); @@ -2005,7 +2005,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeout) { EXPECT_EQ(1UL, downstream_pre_cx_active.value()); EXPECT_CALL(*timeout, disableTimer()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); timeout->invokeCallback(); EXPECT_CALL(*test_filter, destroy_()); dispatcher_.clearDeferredDeleteList(); @@ -2058,7 +2058,7 @@ TEST_F(ConnectionHandlerTest, ContinueOnListenerFilterTimeout) { EXPECT_CALL(*test_filter, destroy_()); // Barrier: test_filter must be destructed before findFilterChain EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); EXPECT_CALL(*timeout, disableTimer()); timeout->invokeCallback(); dispatcher_.clearDeferredDeleteList(); @@ -2127,7 +2127,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeoutResetOnSuccess) { EXPECT_CALL(io_handle, resetFileEvents()); EXPECT_CALL(*test_filter, destroy_()); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); EXPECT_CALL(*timeout, disableTimer()); file_event_callback(Event::FileReadyType::Read); @@ -2203,7 +2203,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterReportError) { cb.socket().close(); return Network::FilterStatus::StopIteration; })); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); // The last filter won't be invoked EXPECT_CALL(*last_filter, onAccept(_)).Times(0); EXPECT_CALL(*first_filter, destroy_()); @@ -2321,7 +2321,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerInplaceUpdate) { .WillOnce(ReturnRef(*current_handler)); EXPECT_CALL(manager_, findFilterChain(_, _)).Times(0); EXPECT_CALL(*overridden_filter_chain_manager, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); EXPECT_CALL(*mock_connection_balancer, unregisterHandler(_)); old_listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2342,7 +2342,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveFilterChain) { auto* server_connection = new NiceMock(); EXPECT_CALL(dispatcher_, createServerConnection_()).WillOnce(Return(server_connection)); EXPECT_CALL(factory_, createNetworkFilterChain(_, _)).WillOnce(Return(true)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); @@ -2402,7 +2402,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveFilterChainCalledAfterListenerIsR handler_->stopListeners(listener_tag); EXPECT_CALL(dispatcher_, clearDeferredDeleteList()); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); { // Filter chain removal in the same poll cycle but earlier. @@ -2446,7 +2446,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveListener) { handler_->addListener(absl::nullopt, *test_listener, runtime_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2478,7 +2478,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveIpv6AnyAddressWithIpv4CompatListe handler_->addListener(absl::nullopt, *test_listener, runtime_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2507,7 +2507,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveIpv4CompatAddressListener) { handler_->addListener(absl::nullopt, *test_listener, runtime_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2544,12 +2544,12 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveWithBothIpv4AnyAndIpv6Any) { handler_->addListener(absl::nullopt, *test_listener2, runtime_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); Network::MockConnectionSocket* connection2 = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks2->onAccept(Network::ConnectionSocketPtr{connection2}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2630,7 +2630,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterWorks) { EXPECT_CALL(*disabled_listener_filter, destroy_()); EXPECT_CALL(*enabled_filter, destroy_()); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _, _, _, _)); listener_callbacks->onAccept(std::make_unique>()); EXPECT_CALL(*listener, onDestroy()); } From 1932f299edd18d9514cba8ac246e5bc69a9a38e6 Mon Sep 17 00:00:00 2001 From: code Date: Thu, 27 Apr 2023 22:41:31 +0800 Subject: [PATCH 011/740] minor code optimization: use string view rather than string for virtual host finding (#26964) Commit Message: minor code optimization: use string view rather than string for virtual host finding Additional Description: Minor opt to reduce unnecessary string creation/memory allocation when searing wildcard virtual hosts. Risk Level: low. Testing: n/a. Docs Changes: n/a. Release Notes: n/a. Platform Specific Features: n/a. --------- Signed-off-by: wbpcode --- source/common/router/config_impl.cc | 13 +++++++------ source/common/router/config_impl.h | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 7becceaabae5..c7bda0dc134e 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -1884,7 +1884,7 @@ RouteConstSharedPtr VirtualHostImpl::getRouteFromEntries(const RouteCallback& cb } const VirtualHostImpl* RouteMatcher::findWildcardVirtualHost( - const std::string& host, const RouteMatcher::WildcardVirtualHosts& wildcard_virtual_hosts, + absl::string_view host, const RouteMatcher::WildcardVirtualHosts& wildcard_virtual_hosts, RouteMatcher::SubstringFunction substring_function) const { // We do a longest wildcard match against the host that's passed in // (e.g. "foo-bar.baz.com" should match "*-bar.baz.com" before matching "*.baz.com" for suffix @@ -1897,7 +1897,7 @@ const VirtualHostImpl* RouteMatcher::findWildcardVirtualHost( if (wildcard_length >= host.size()) { continue; } - const auto& match = wildcard_map.find(substring_function(host, wildcard_length)); + const auto match = wildcard_map.find(substring_function(host, wildcard_length)); if (match != wildcard_map.end()) { return match->second.get(); } @@ -1922,7 +1922,8 @@ RouteMatcher::RouteMatcher(const envoy::config::route::v3::RouteConfiguration& r virtual_host_config, optional_http_filters, global_route_config, factory_context, *vhost_scope_, validator, validation_clusters); for (const std::string& domain_name : virtual_host_config.domains()) { - const std::string domain = Http::LowerCaseString(domain_name).get(); + const Http::LowerCaseString lower_case_domain_name(domain_name); + absl::string_view domain = lower_case_domain_name; bool duplicate_found = false; if ("*" == domain) { if (default_virtual_host_) { @@ -1975,14 +1976,14 @@ const VirtualHostImpl* RouteMatcher::findVirtualHost(const Http::RequestHeaderMa // request with VHost, using wildcard match // Lower-case the value of the host header, as hostnames are case insensitive. const std::string host = absl::AsciiStrToLower(host_header_value); - const auto& iter = virtual_hosts_.find(host); + const auto iter = virtual_hosts_.find(host); if (iter != virtual_hosts_.end()) { return iter->second.get(); } if (!wildcard_virtual_host_suffixes_.empty()) { const VirtualHostImpl* vhost = findWildcardVirtualHost( host, wildcard_virtual_host_suffixes_, - [](const std::string& h, int l) -> std::string { return h.substr(h.size() - l); }); + [](absl::string_view h, int l) -> absl::string_view { return h.substr(h.size() - l); }); if (vhost != nullptr) { return vhost; } @@ -1990,7 +1991,7 @@ const VirtualHostImpl* RouteMatcher::findVirtualHost(const Http::RequestHeaderMa if (!wildcard_virtual_host_prefixes_.empty()) { const VirtualHostImpl* vhost = findWildcardVirtualHost( host, wildcard_virtual_host_prefixes_, - [](const std::string& h, int l) -> std::string { return h.substr(0, l); }); + [](absl::string_view h, int l) -> absl::string_view { return h.substr(0, l); }); if (vhost != nullptr) { return vhost; } diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index ac7743969d46..3694640f491e 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -1487,8 +1487,8 @@ class RouteMatcher { private: using WildcardVirtualHosts = std::map, std::greater<>>; - using SubstringFunction = std::function; - const VirtualHostImpl* findWildcardVirtualHost(const std::string& host, + using SubstringFunction = std::function; + const VirtualHostImpl* findWildcardVirtualHost(absl::string_view host, const WildcardVirtualHosts& wildcard_virtual_hosts, SubstringFunction substring_function) const; bool ignorePortInHostMatching() const { return ignore_port_in_host_matching_; } From a0106917f323efd7f4cfad2f67d67e14dc696e5d Mon Sep 17 00:00:00 2001 From: Xie Zhihao Date: Thu, 27 Apr 2023 23:01:33 +0800 Subject: [PATCH 012/740] udp: remove runtime udp_proxy_connect (#26966) Commit Message: udp: remove runtime udp_proxy_connect Additional Description: Risk Level: Low Testing: N/A Docs Changes: N/A Release Notes: Added Platform Specific Features: N/A Fixes #26909 Signed-off-by: Xie Zhihao --- changelogs/current.yaml | 3 +++ source/common/runtime/runtime_features.cc | 1 - .../filters/udp/udp_proxy/udp_proxy_filter.cc | 16 +++++++--------- .../filters/udp/udp_proxy/udp_proxy_filter.h | 6 ++---- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index b44dda5460e6..eb6ea891f43e 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -47,6 +47,9 @@ removed_config_or_runtime: - area: http change: | removed runtime key ``envoy.reloadable_features.http_response_half_close`` and legacy code paths. +- area: udp + change: | + removed runtime key ``envoy.reloadable_features.udp_proxy_connect`` and legacy code paths. new_features: - area: access_log diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 625a3118052b..e7cc369a4b05 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -66,7 +66,6 @@ RUNTIME_GUARD(envoy_reloadable_features_test_feature_true); RUNTIME_GUARD(envoy_reloadable_features_thrift_allow_negative_field_ids); RUNTIME_GUARD(envoy_reloadable_features_thrift_connection_draining); RUNTIME_GUARD(envoy_reloadable_features_tls_async_cert_validation); -RUNTIME_GUARD(envoy_reloadable_features_udp_proxy_connect); RUNTIME_GUARD(envoy_reloadable_features_uhv_allow_malformed_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_uhv_preserve_url_encoded_case); RUNTIME_GUARD(envoy_reloadable_features_uhv_translate_backslash_to_slash); diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc index cb0480018b10..32e52135853e 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc @@ -389,17 +389,15 @@ void UdpProxyFilter::ActiveSession::write(const Buffer::Instance& buffer) { // set. We allow the OS to select the right IP based on outbound routing rules if // use_original_src_ip_ is not set, else use downstream peer IP as local IP. const Network::Address::Ip* local_ip = use_original_src_ip_ ? addresses_.peer_->ip() : nullptr; - if (!use_original_src_ip_ && !skip_connect_) { - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.udp_proxy_connect")) { - Api::SysCallIntResult rc = socket_->ioHandle().connect(host_->address()); - if (SOCKET_FAILURE(rc.return_value_)) { - ENVOY_LOG(debug, "cannot connect: ({}) {}", rc.errno_, errorDetails(rc.errno_)); - cluster_.cluster_stats_.sess_tx_errors_.inc(); - return; - } + if (!use_original_src_ip_ && !connected_) { + Api::SysCallIntResult rc = socket_->ioHandle().connect(host_->address()); + if (SOCKET_FAILURE(rc.return_value_)) { + ENVOY_LOG(debug, "cannot connect: ({}) {}", rc.errno_, errorDetails(rc.errno_)); + cluster_.cluster_stats_.sess_tx_errors_.inc(); + return; } - skip_connect_ = true; + connected_ = true; } Api::IoCallUint64Result rc = Network::Utility::writeToSocket(socket_->ioHandle(), buffer, local_ip, *host_->address()); diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h index e44edf9c5492..925dc8453e17 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h @@ -247,10 +247,8 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, // packets from the upstream host. Note that a a local ephemeral port is bound on the first // write to the upstream host. const Network::SocketPtr socket_; - // The socket should be connected to avoid port exhaustion unless runtime guard - // envoy.reloadable_features.udp_proxy_connect is unset or use_original_src_ip_ is set. If it - // is true, there will be no calling `connect()` on the socket. - bool skip_connect_{}; + // The socket has been connected to avoid port exhaustion. + bool connected_{}; UdpProxySessionStats session_stats_{}; absl::optional udp_session_stats_; From 90ed175ac5e819a5693c56c9f0babc1c28a5aef1 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 27 Apr 2023 11:24:42 -0400 Subject: [PATCH 013/740] header formatter: handling out of bounds protos consistently (#26954) Signed-off-by: Alyssa Wilk --- .../preserve_case/preserve_case_formatter.cc | 4 +--- .../preserve_case/preserve_case_formatter_test.cc | 8 -------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.cc b/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.cc index 048a82ac8683..0fb1c42e6c0e 100644 --- a/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.cc +++ b/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.cc @@ -13,6 +13,7 @@ PreserveCaseHeaderFormatter::PreserveCaseHeaderFormatter( : forward_reason_phrase_(forward_reason_phrase), formatter_type_on_envoy_headers_(formatter_type_on_envoy_headers) { switch (formatter_type_on_envoy_headers_) { + PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; case envoy::extensions::http::header_formatters::preserve_case::v3::PreserveCaseFormatterConfig:: DEFAULT: header_key_formatter_on_enovy_headers_ = Envoy::Http::HeaderKeyFormatterConstPtr(); @@ -22,9 +23,6 @@ PreserveCaseHeaderFormatter::PreserveCaseHeaderFormatter( header_key_formatter_on_enovy_headers_ = std::make_unique(); break; - default: - throw EnvoyException(fmt::format("Not supported FormatterTypeOnEnvoyHeaders: {}.", - formatter_type_on_envoy_headers_)); } } diff --git a/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_test.cc b/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_test.cc index 4ee9de8750a7..fbd6f0544a54 100644 --- a/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_test.cc +++ b/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_test.cc @@ -86,14 +86,6 @@ TEST(PreserveCaseFormatterTest, DefaultFormatterOnEnvoyHeadersEnabled) { EXPECT_EQ(false, formatter.formatterOnEnvoyHeaders().has_value()); } -TEST(PreserveCaseFormatterTest, InvalidFormatterOnEnvoyHeaders) { - EXPECT_THROW_WITH_REGEX( - PreserveCaseHeaderFormatter formatter( - false, static_cast(-1)), - EnvoyException, "Not supported FormatterTypeOnEnvoyHeaders:.*"); -} - } // namespace PreserveCase } // namespace HeaderFormatters } // namespace Http From 2418a2d9b0280dbaa1979309a8e744565d62b950 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 27 Apr 2023 12:06:05 -0400 Subject: [PATCH 014/740] mobile: fixing a tsan flake (#27025) Signed-off-by: Alyssa Wilk --- mobile/library/cc/engine.cc | 5 +++-- mobile/library/cc/engine.h | 2 +- mobile/library/common/engine_handle.cc | 5 +++-- mobile/library/common/engine_handle.h | 4 ++-- mobile/library/common/main_interface.cc | 4 ++-- mobile/library/common/main_interface.h | 3 ++- mobile/test/common/engine_test.cc | 10 +++++----- 7 files changed, 18 insertions(+), 15 deletions(-) diff --git a/mobile/library/cc/engine.cc b/mobile/library/cc/engine.cc index 3bb00bdb6e99..b807119637e8 100644 --- a/mobile/library/cc/engine.cc +++ b/mobile/library/cc/engine.cc @@ -24,12 +24,13 @@ StreamClientSharedPtr Engine::streamClient() { PulseClientSharedPtr Engine::pulseClient() { return std::make_shared(); } -void Engine::terminate() { +envoy_status_t Engine::terminate() { if (terminated_) { throw std::runtime_error("attempting to double terminate Engine"); } - terminate_engine(engine_, /* release */ false); + envoy_status_t ret = terminate_engine(engine_, /* release */ false); terminated_ = true; + return ret; } } // namespace Platform diff --git a/mobile/library/cc/engine.h b/mobile/library/cc/engine.h index a95db95d7095..61261332e4e8 100644 --- a/mobile/library/cc/engine.h +++ b/mobile/library/cc/engine.h @@ -22,7 +22,7 @@ class Engine : public std::enable_shared_from_this { StreamClientSharedPtr streamClient(); PulseClientSharedPtr pulseClient(); - void terminate(); + envoy_status_t terminate(); private: Engine(envoy_engine_t engine); diff --git a/mobile/library/common/engine_handle.cc b/mobile/library/common/engine_handle.cc index e4e7c63807cb..975386b7af23 100644 --- a/mobile/library/common/engine_handle.cc +++ b/mobile/library/common/engine_handle.cc @@ -25,13 +25,14 @@ envoy_status_t EngineHandle::runEngine(envoy_engine_t handle, const char* config return ENVOY_FAILURE; } -void EngineHandle::terminateEngine(envoy_engine_t handle, bool release) { +envoy_status_t EngineHandle::terminateEngine(envoy_engine_t handle, bool release) { auto engine = reinterpret_cast(handle); - engine->terminate(); + envoy_status_t ret = engine->terminate(); if (release) { // TODO(jpsim): Always delete engine to avoid leaking it delete engine; } + return ret; } } // namespace Envoy diff --git a/mobile/library/common/engine_handle.h b/mobile/library/common/engine_handle.h index 7c7a453d9445..1c97d8e5bfad 100644 --- a/mobile/library/common/engine_handle.h +++ b/mobile/library/common/engine_handle.h @@ -27,14 +27,14 @@ class EngineHandle { envoy_event_tracker event_tracker); static envoy_status_t runEngine(envoy_engine_t, const char* config, const char* log_level, const char* admin_address_path); - static void terminateEngine(envoy_engine_t handle, bool release); + static envoy_status_t terminateEngine(envoy_engine_t handle, bool release); // Allow a specific list of functions to access the internal setup/teardown functionality. friend envoy_engine_t(::init_engine)(envoy_engine_callbacks callbacks, envoy_logger logger, envoy_event_tracker event_tracker); friend envoy_status_t(::run_engine)(envoy_engine_t, const char* config, const char* log_level, const char* admin_address_path); - friend void ::terminate_engine(envoy_engine_t engine, bool release); + friend envoy_status_t(::terminate_engine)(envoy_engine_t engine, bool release); }; } // namespace Envoy diff --git a/mobile/library/common/main_interface.cc b/mobile/library/common/main_interface.cc index e59ee645120b..3324db60efce 100644 --- a/mobile/library/common/main_interface.cc +++ b/mobile/library/common/main_interface.cc @@ -163,8 +163,8 @@ envoy_status_t run_engine(envoy_engine_t engine, const char* config, const char* return Envoy::EngineHandle::runEngine(engine, config, log_level, admin_path); } -void terminate_engine(envoy_engine_t engine, bool release) { - Envoy::EngineHandle::terminateEngine(engine, release); +envoy_status_t terminate_engine(envoy_engine_t engine, bool release) { + return Envoy::EngineHandle::terminateEngine(engine, release); } envoy_status_t reset_connectivity_state(envoy_engine_t e) { diff --git a/mobile/library/common/main_interface.h b/mobile/library/common/main_interface.h index 4edf019cd408..6a7ac30c7cdd 100644 --- a/mobile/library/common/main_interface.h +++ b/mobile/library/common/main_interface.h @@ -211,8 +211,9 @@ envoy_status_t run_engine(envoy_engine_t engine, const char* config, const char* * terminated engine is illegal. * @param engine, handle to the engine to terminate. * @param release, set to true to release the engine from memory. + * @return envoy_status_t, the resulting status of the operation. */ -void terminate_engine(envoy_engine_t engine, bool release); +envoy_status_t terminate_engine(envoy_engine_t engine, bool release); /** * Refresh DNS, and drain connections associated with an engine. diff --git a/mobile/test/common/engine_test.cc b/mobile/test/common/engine_test.cc index a828633c77e2..dc1f51ab3b1b 100644 --- a/mobile/test/common/engine_test.cc +++ b/mobile/test/common/engine_test.cc @@ -21,7 +21,7 @@ struct TestEngineHandle { run_engine(handle_, yaml.c_str(), level.c_str(), ""); } - void terminate() { terminate_engine(handle_, /* release */ false); } + envoy_status_t terminate() { return terminate_engine(handle_, /* release */ false); } ~TestEngineHandle() { terminate_engine(handle_, /* release */ true); } }; @@ -53,10 +53,10 @@ TEST_F(EngineTest, EarlyExit) { engine_ = std::make_unique(callbacks, level); envoy_engine_t handle = engine_->handle_; - ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(3))); + ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(10))); - engine_->terminate(); - ASSERT_TRUE(test_context.on_exit.WaitForNotificationWithTimeout(absl::Seconds(3))); + ASSERT_EQ(engine_->terminate(), ENVOY_SUCCESS); + ASSERT_TRUE(test_context.on_exit.WaitForNotificationWithTimeout(absl::Seconds(10))); start_stream(handle, 0, {}, false); @@ -76,7 +76,7 @@ TEST_F(EngineTest, AccessEngineAfterInitialization) { engine_ = std::make_unique(callbacks, level); envoy_engine_t handle = engine_->handle_; - ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(3))); + ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(10))); absl::Notification getClusterManagerInvoked; // Scheduling on the dispatcher should work, the engine is running. From d3caca0e6a3deba79b0150a01511777e48649da8 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 27 Apr 2023 09:54:04 -0700 Subject: [PATCH 015/740] fips: fix Wasm extensions to comply with FIPS (#26986) Signed-off-by: Kuat Yessenov --- bazel/proxy_wasm_cpp_host.patch | 13 +++++++++++++ bazel/repositories.bzl | 8 +++++++- changelogs/current.yaml | 3 +++ source/common/version/BUILD | 1 + source/common/version/version.cc | 5 +++++ 5 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 bazel/proxy_wasm_cpp_host.patch diff --git a/bazel/proxy_wasm_cpp_host.patch b/bazel/proxy_wasm_cpp_host.patch new file mode 100644 index 000000000000..ce159af5ed42 --- /dev/null +++ b/bazel/proxy_wasm_cpp_host.patch @@ -0,0 +1,13 @@ +diff --git a/BUILD b/BUILD +index 69c9bda..d293092 100644 +--- a/BUILD ++++ b/BUILD +@@ -88,7 +88,7 @@ cc_library( + ":headers", + ] + select({ + "//bazel:crypto_system": [], +- "//conditions:default": ["@boringssl//:crypto"], ++ "//conditions:default": ["@envoy//bazel:boringcrypto"], + }), + alwayslink = 1, + ) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 2edf67ce610b..6d2cf2014cba 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1130,7 +1130,13 @@ def _proxy_wasm_cpp_sdk(): external_http_archive(name = "proxy_wasm_cpp_sdk") def _proxy_wasm_cpp_host(): - external_http_archive(name = "proxy_wasm_cpp_host") + external_http_archive( + name = "proxy_wasm_cpp_host", + patch_args = ["-p1"], + patches = [ + "@envoy//bazel:proxy_wasm_cpp_host.patch", + ], + ) def _emsdk(): external_http_archive( diff --git a/changelogs/current.yaml b/changelogs/current.yaml index eb6ea891f43e..9544f31da376 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -29,6 +29,9 @@ minor_behavior_changes: bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* +- area: tls + change: | + Fix build FIPS compliance when using both FIPS mode and Wasm extensions (``--define boringssl=fips`` and ``--define wasm=v8``). removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/source/common/version/BUILD b/source/common/version/BUILD index 6178b696bd40..7f88244cb351 100644 --- a/source/common/version/BUILD +++ b/source/common/version/BUILD @@ -65,6 +65,7 @@ envoy_cc_library( ], ["-DENVOY_SSL_VERSION=\\\"BoringSSL\\\""], ), + external_deps = ["ssl"], deps = [ ":version_includes", "//source/common/common:macros", diff --git a/source/common/version/version.cc b/source/common/version/version.cc index 1d0b3ff7d05b..1afb04dbdc9c 100644 --- a/source/common/version/version.cc +++ b/source/common/version/version.cc @@ -12,6 +12,10 @@ #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#ifdef ENVOY_SSL_FIPS +#include "openssl/crypto.h" +#endif + extern const char build_scm_revision[]; extern const char build_scm_status[]; @@ -38,6 +42,7 @@ const envoy::config::core::v3::BuildVersion& VersionInfo::buildVersion() { bool VersionInfo::sslFipsCompliant() { #ifdef ENVOY_SSL_FIPS + RELEASE_ASSERT(FIPS_mode() == 1, "FIPS mode must be enabled in Envoy FIPS configuration."); return true; #else return false; From bb518040baac159a1d39cd10245de2a626cbf2bf Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 27 Apr 2023 18:15:44 +0100 Subject: [PATCH 016/740] dependabot: Run earlier (#27017) Signed-off-by: Ryan Northey --- .github/dependabot.yml | 84 +++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 12ab4c9bb135..fa90aaa0a12e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,251 +13,251 @@ updates: directory: "/.github/actions/pr_notifier" schedule: interval: "daily" - time: "09:00" + time: "06:00" - package-ecosystem: "pip" directory: "/examples/grpc-bridge/client" schedule: interval: "daily" - time: "09:00" + time: "06:00" - package-ecosystem: "pip" directory: "/examples/cache" schedule: interval: "daily" - time: "09:00" + time: "06:00" - package-ecosystem: "pip" directory: "/examples/shared/python/aiohttp" schedule: interval: "daily" - time: "09:00" + time: "06:00" - package-ecosystem: "pip" directory: "/examples/shared/python/postgres" schedule: interval: "daily" - time: "09:00" + time: "06:00" - package-ecosystem: "pip" directory: "/mobile/docs" schedule: interval: "daily" - time: "09:00" + time: "06:00" - package-ecosystem: "pip" directory: "/tools/base" schedule: interval: "daily" - time: "09:00" + time: "06:00" - package-ecosystem: "pip" directory: "/tools/code_format" schedule: interval: "daily" - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/.devcontainer" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/ci" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/ext_authz" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/fault-injection" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/grpc-bridge" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/kafka" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/local_ratelimit" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/mysql" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/opentelemetry" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/redis" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/shared/build" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/shared/echo" schedule: interval: daily - time: "09:00" + time: "06:00" # TODO(phlax): just use above - package-ecosystem: "docker" directory: "/examples/shared/echo2" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/shared/golang" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/shared/jaeger" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/shared/node" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/shared/postgres" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/shared/python" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/shared/websocket" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/skywalking" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/udp" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "docker" directory: "/examples/zipkin" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "github-actions" directory: "/" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/basic" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/dummy" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/echo" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/passthrough" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/routeconfig" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/router/cluster_specifier/test/test_data/simple" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/examples/ext_authz/auth/grpc-service" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/examples/load-reporting-service" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/examples/grpc-bridge/server" schedule: interval: daily - time: "09:00" + time: "06:00" - package-ecosystem: "gomod" directory: "/examples/golang/simple" schedule: interval: daily - time: "09:00" + time: "06:00" From b3563e5945c05af1ab180373a6b15091b3f47640 Mon Sep 17 00:00:00 2001 From: StarryNight Date: Fri, 28 Apr 2023 01:17:01 +0800 Subject: [PATCH 017/740] fix wasm connection close without reason (#27012) Signed-off-by: wangkai19 --- source/extensions/common/wasm/context.cc | 4 ++-- .../filters/network/wasm/wasm_filter_test.cc | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index a2215c2ede4d..adcc4be4c8b3 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -1556,7 +1556,7 @@ WasmResult Context::closeStream(WasmStreamType stream_type) { // We are in a reentrant call, so defer. wasm()->addAfterVmCallAction([this] { network_read_filter_callbacks_->connection().close( - Envoy::Network::ConnectionCloseType::FlushWrite); + Envoy::Network::ConnectionCloseType::FlushWrite, "wasm_downstream_close"); }); } return WasmResult::Ok; @@ -1565,7 +1565,7 @@ WasmResult Context::closeStream(WasmStreamType stream_type) { // We are in a reentrant call, so defer. wasm()->addAfterVmCallAction([this] { network_write_filter_callbacks_->connection().close( - Envoy::Network::ConnectionCloseType::FlushWrite); + Envoy::Network::ConnectionCloseType::FlushWrite, "wasm_upstream_close"); }); } return WasmResult::Ok; diff --git a/test/extensions/filters/network/wasm/wasm_filter_test.cc b/test/extensions/filters/network/wasm/wasm_filter_test.cc index a04cff29793e..c13975701fdf 100644 --- a/test/extensions/filters/network/wasm/wasm_filter_test.cc +++ b/test/extensions/filters/network/wasm/wasm_filter_test.cc @@ -434,11 +434,13 @@ TEST_P(WasmNetworkFilterTest, CloseDownstream) { EXPECT_EQ(write_filter_callbacks_.connection().state(), Network::Connection::State::Open); Buffer::OwnedImpl fake_downstream_data("Fake"); filter().onCreate(); // Create context without calling OnNewConnection. - EXPECT_EQ(Network::FilterStatus::Continue, filter().onWrite(fake_downstream_data, false)); + EXPECT_CALL(read_filter_callbacks_.connection_, + close(Network::ConnectionCloseType::FlushWrite, "wasm_downstream_close")); + EXPECT_EQ(Network::FilterStatus::Continue, filter().onData(fake_downstream_data, false)); // Should close downstream. - EXPECT_EQ(read_filter_callbacks_.connection().state(), Network::Connection::State::Open); - EXPECT_EQ(write_filter_callbacks_.connection().state(), Network::Connection::State::Closed); + EXPECT_EQ(read_filter_callbacks_.connection().state(), Network::Connection::State::Closed); + EXPECT_EQ(write_filter_callbacks_.connection().state(), Network::Connection::State::Open); } TEST_P(WasmNetworkFilterTest, CloseUpstream) { @@ -448,11 +450,13 @@ TEST_P(WasmNetworkFilterTest, CloseUpstream) { EXPECT_EQ(write_filter_callbacks_.connection().state(), Network::Connection::State::Open); Buffer::OwnedImpl fake_upstream_data("Fake"); filter().onCreate(); // Create context without calling OnNewConnection. - EXPECT_EQ(Network::FilterStatus::Continue, filter().onData(fake_upstream_data, false)); + EXPECT_CALL(write_filter_callbacks_.connection_, + close(Network::ConnectionCloseType::FlushWrite, "wasm_upstream_close")); + EXPECT_EQ(Network::FilterStatus::Continue, filter().onWrite(fake_upstream_data, false)); - // Should close downstream. - EXPECT_EQ(read_filter_callbacks_.connection().state(), Network::Connection::State::Closed); - EXPECT_EQ(write_filter_callbacks_.connection().state(), Network::Connection::State::Open); + // Should close upstream. + EXPECT_EQ(read_filter_callbacks_.connection().state(), Network::Connection::State::Open); + EXPECT_EQ(write_filter_callbacks_.connection().state(), Network::Connection::State::Closed); } } // namespace Wasm From b168dc2b7cfa3564b3c76c72802b836f7408fd9e Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 27 Apr 2023 18:24:56 +0100 Subject: [PATCH 018/740] do/ci: Switch switch to switch (#27010) Signed-off-by: Ryan Northey --- ci/do_ci.sh | 760 ++++++++++++++++++++++++++-------------------------- 1 file changed, 383 insertions(+), 377 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 5c8642dbfab7..10d072243830 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -206,383 +206,389 @@ else TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "@com_github_google_quiche//:ci_tests") fi -if [[ "$CI_TARGET" == "bazel.release" ]]; then - # When testing memory consumption, we want to test against exact byte-counts - # where possible. As these differ between platforms and compile options, we - # define the 'release' builds as canonical and test them only in CI, so the - # toolchain is kept consistent. This ifdef is checked in - # test/common/stats/stat_test_utility.cc when computing - # Stats::TestUtil::MemoryTest::mode(). - [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]] && BAZEL_BUILD_OPTIONS+=("--test_env=ENVOY_MEMORY_TEST_EXACT=true") - - setup_clang_toolchain - - ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" - mkdir -p "$ENVOY_BINARY_DIR" - - # Run release tests - echo "Testing ${TEST_TARGETS[*]} with options: ${BAZEL_BUILD_OPTIONS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_minimal -c opt "${TEST_TARGETS[@]}" - - # Build release binaries - # As the binary build package enforces `-c opt`, adding here to ensure the distribution build reuses - # any already compiled artefacts, the bundle itself will always be compiled `-c opt` - bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c opt //distribution/binary:release - - # Copy release binaries to binary export directory - cp -a "bazel-bin/distribution/binary/release.tar.zst" "${ENVOY_BINARY_DIR}/release.tar.zst" - - # Grab the schema_validator_tool - # TODO(phlax): bundle this with the release when #26390 is resolved - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel --stripopt=--strip-all -c opt //test/tools/schema_validator:schema_validator_tool.stripped - - # Copy schema_validator_tool to binary export directory - cp -a bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped "${ENVOY_BINARY_DIR}/schema_validator_tool" - - exit 0 -elif [[ "$CI_TARGET" == "bazel.distribution" ]]; then - echo "Building distro packages..." - - setup_clang_toolchain - - # Extract the Envoy binary from the tarball - mkdir -p distribution/custom - if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - ENVOY_RELEASE_TARBALL="/build/bazel.release/bin/release.tar.zst" - else - ENVOY_RELEASE_TARBALL="/build/bazel.release.arm64/bin/release.tar.zst" - fi - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/zstd -- --stdout -d "$ENVOY_RELEASE_TARBALL" | tar xfO - envoy > distribution/custom/envoy - - # By default the packages will be signed by the first available key. - # If there is no key available, a throwaway key is created - # and the packages signed with it, for the purpose of testing only. - if ! gpg --list-secret-keys "*"; then - export PACKAGES_MAINTAINER_NAME="Envoy CI" - export PACKAGES_MAINTAINER_EMAIL="envoy-ci@for.testing.only" - BAZEL_BUILD_OPTIONS+=( - "--action_env=PACKAGES_GEN_KEY=1" - "--action_env=PACKAGES_MAINTAINER_NAME" - "--action_env=PACKAGES_MAINTAINER_EMAIL") - fi +if [[ "$CI_TARGET" =~ bazel.* ]]; then + CI_TARGET="$(echo "${CI_TARGET}" | cut -d. -f2-)" +fi - # Build the packages - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c opt --//distribution:envoy-binary=//distribution:custom/envoy //distribution:packages.tar.gz - if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.x64.tar.gz" - else - cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.arm64.tar.gz" - fi - exit 0 -elif [[ "$CI_TARGET" == "bazel.release.server_only" ]]; then - setup_clang_toolchain - echo "bazel release build..." - bazel_envoy_binary_build release - exit 0 -elif [[ "$CI_TARGET" == "bazel.sizeopt.server_only" ]]; then - setup_clang_toolchain - echo "bazel size optimized build..." - bazel_envoy_binary_build sizeopt - exit 0 -elif [[ "$CI_TARGET" == "bazel.sizeopt" ]]; then - setup_clang_toolchain - echo "Testing ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --config=sizeopt "${TEST_TARGETS[@]}" - - echo "bazel size optimized build with tests..." - bazel_envoy_binary_build sizeopt - exit 0 -elif [[ "$CI_TARGET" == "bazel.gcc" ]]; then - BAZEL_BUILD_OPTIONS+=("--test_env=HEAPCHECK=") - setup_gcc_toolchain - - echo "Testing ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild --remote_download_minimal -- "${TEST_TARGETS[@]}" - - echo "bazel release build with gcc..." - bazel_envoy_binary_build fastbuild - exit 0 -elif [[ "$CI_TARGET" == "bazel.debug" ]]; then - setup_clang_toolchain - echo "Testing ${TEST_TARGETS[*]}" - # Make sure that there are no regressions to building Envoy with autolink disabled. - EXTRA_OPTIONS=( - "--define" "library_autolink=disabled") - bazel test "${BAZEL_BUILD_OPTIONS[@]}" "${EXTRA_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" - - echo "bazel debug build with tests..." - bazel_envoy_binary_build debug - exit 0 -elif [[ "$CI_TARGET" == "bazel.debug.server_only" ]]; then - setup_clang_toolchain - echo "bazel debug build..." - bazel_envoy_binary_build debug - exit 0 -elif [[ "$CI_TARGET" == "bazel.asan" ]]; then - setup_clang_toolchain - BAZEL_BUILD_OPTIONS+=(-c dbg "--config=clang-asan" "--build_tests_only" "--remote_download_minimal") - echo "bazel ASAN/UBSAN debug build with tests" - echo "Building and testing envoy tests ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" "${TEST_TARGETS[@]}" - if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then - echo "Building and testing envoy-filter-example tests..." - pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" - popd - fi +case $CI_TARGET in + release) + # When testing memory consumption, we want to test against exact byte-counts + # where possible. As these differ between platforms and compile options, we + # define the 'release' builds as canonical and test them only in CI, so the + # toolchain is kept consistent. This ifdef is checked in + # test/common/stats/stat_test_utility.cc when computing + # Stats::TestUtil::MemoryTest::mode(). + [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]] && BAZEL_BUILD_OPTIONS+=("--test_env=ENVOY_MEMORY_TEST_EXACT=true") - # TODO(mattklein123): This part of the test is now flaky in CI and it's unclear why, possibly - # due to sandboxing issue. Debug and enable it again. - # if [ "${CI_SKIP_INTEGRATION_TEST_TRAFFIC_TAPPING}" != "1" ] ; then - # Also validate that integration test traffic tapping (useful when debugging etc.) - # works. This requires that we set TAP_PATH. We do this under bazel.asan to - # ensure a debug build in CI. - # echo "Validating integration test traffic tapping..." - # bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" \ - # --run_under=@envoy//bazel/test:verify_tap_test.sh \ - # //test/extensions/transport_sockets/tls/integration:ssl_integration_test - # fi - exit 0 -elif [[ "$CI_TARGET" == "bazel.tsan" ]]; then - setup_clang_toolchain - echo "bazel TSAN debug build with tests" - echo "Building and testing envoy tests ${TEST_TARGETS[*]}" - bazel_with_collection test --config=rbe-toolchain-tsan "${BAZEL_BUILD_OPTIONS[@]}" -c dbg --build_tests_only --remote_download_minimal "${TEST_TARGETS[@]}" - if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then - echo "Building and testing envoy-filter-example tests..." - pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c dbg --config=clang-tsan "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" - popd - fi - exit 0 -elif [[ "$CI_TARGET" == "bazel.msan" ]]; then - ENVOY_STDLIB=libc++ - setup_clang_toolchain - # rbe-toolchain-msan must comes as first to win library link order. - BAZEL_BUILD_OPTIONS=("--config=rbe-toolchain-msan" "${BAZEL_BUILD_OPTIONS[@]}" "-c" "dbg" "--build_tests_only" "--remote_download_minimal") - echo "bazel MSAN debug build with tests" - echo "Building and testing envoy tests ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -- "${TEST_TARGETS[@]}" - exit 0 -elif [[ "$CI_TARGET" == "bazel.dev" ]]; then - setup_clang_toolchain - # This doesn't go into CI but is available for developer convenience. - echo "bazel fastbuild build with tests..." - echo "Building..." - bazel_envoy_binary_build fastbuild - - echo "Testing ${TEST_TARGETS[*]}" - bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" - exit 0 -elif [[ "$CI_TARGET" == "bazel.dev.contrib" ]]; then - setup_clang_toolchain - # This doesn't go into CI but is available for developer convenience. - echo "bazel fastbuild build with contrib extensions and tests..." - echo "Building..." - bazel_contrib_binary_build fastbuild - - echo "Testing ${TEST_TARGETS[*]}" - bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" - exit 0 -elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then - # Right now, none of the available compile-time options conflict with each other. If this - # changes, this build type may need to be broken up. - - COMPILE_TIME_OPTIONS=( - "--define" "admin_html=disabled" - "--define" "signal_trace=disabled" - "--define" "hot_restart=disabled" - "--define" "google_grpc=disabled" - "--define" "boringssl=fips" - "--define" "log_debug_assert_in_release=enabled" - "--define" "path_normalization_by_default=true" - "--define" "deprecated_features=disabled" - "--define" "tcmalloc=gperftools" - "--define" "zlib=ng" - "--define" "uhv=enabled" - "--@envoy//bazel:http3=False" - "--@envoy//source/extensions/filters/http/kill_request:enabled" - "--test_env=ENVOY_HAS_EXTRA_EXTENSIONS=true" - "--remote_download_minimal") - - ENVOY_STDLIB="${ENVOY_STDLIB:-libstdc++}" - setup_clang_toolchain - # This doesn't go into CI but is available for developer convenience. - echo "bazel with different compiletime options build with tests..." - - cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" - TEST_TARGETS=("${TEST_TARGETS[@]/#\/\//@envoy\/\/}") - - # Building all the dependencies from scratch to link them against libc++. - echo "Building and testing with wasm=wamr: ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wamr "${COMPILE_TIME_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only - - echo "Building and testing with wasm=wasmtime: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wasmtime --define admin_html=disabled --define admin_functionality=disabled "${COMPILE_TIME_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only - - echo "Building and testing with wasm=wavm: ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only - - # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only - # these tests under "-c opt" to save time in CI. - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c opt @envoy//test/common/common:assert_test @envoy//test/server:server_test - - # "--define log_fast_debug_assert_in_release=enabled" must be tested with a release build, so run only these tests under "-c opt" to save time in CI. This option will test only ASSERT()s without SLOW_ASSERT()s, so additionally disable "--define log_debug_assert_in_release" which compiles in both. - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c opt @envoy//test/common/common:assert_test --define log_fast_debug_assert_in_release=enabled --define log_debug_assert_in_release=disabled - - echo "Building binary with wasm=wavm... and logging disabled" - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm --define enable_logging=disabled "${COMPILE_TIME_OPTIONS[@]}" -c dbg @envoy//source/exe:envoy-static --build_tag_filters=-nofips - collect_build_profile build - exit 0 -elif [[ "$CI_TARGET" == "bazel.api" ]]; then - # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib, - # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild. - ENVOY_STDLIB="libstdc++" - setup_clang_toolchain - export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config - echo "Run protoxform test" - bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ - --//tools/api_proto_plugin:default_type_db_target=//tools/testdata/protoxform:fix_protos \ - --//tools/api_proto_plugin:extra_args=api_version:3.7 \ - //tools/protoprint:protoprint_test - echo "Validating API structure..." - "${ENVOY_SRCDIR}"/tools/api/validate_structure.py - echo "Validate Golang protobuf generation..." - "${ENVOY_SRCDIR}"/tools/api/generate_go_protobuf.py - echo "Testing API..." - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_minimal -c fastbuild @envoy_api//test/... @envoy_api//tools/... \ - @envoy_api//tools:tap2pcap_test - echo "Building API..." - bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api//envoy/... - exit 0 -elif [[ "$CI_TARGET" == "bazel.api_compat" ]]; then - echo "Checking API for breaking changes to protobuf backwards compatibility..." - BASE_BRANCH_REF=$("${ENVOY_SRCDIR}"/tools/git/last_github_commit.sh) - COMMIT_TITLE=$(git log -n 1 --pretty='format:%C(auto)%h (%s, %ad)' "${BASE_BRANCH_REF}") - echo -e "\tUsing base commit ${COMMIT_TITLE}" - # BAZEL_BUILD_OPTIONS needed for setting the repository_cache param. - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/api_proto_breaking_change_detector:detector_ci "${BASE_BRANCH_REF}" - exit 0 -elif [[ "$CI_TARGET" == "bazel.coverage" || "$CI_TARGET" == "bazel.fuzz_coverage" ]]; then - setup_clang_toolchain - - echo "${CI_TARGET} build with tests ${COVERAGE_TEST_TARGETS[*]}" - - [[ "$CI_TARGET" == "bazel.fuzz_coverage" ]] && export FUZZ_COVERAGE=true - - # We use custom BAZEL_BUILD_OPTIONS here to cover profiler's code. - BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]} --define tcmalloc=gperftools" "${ENVOY_SRCDIR}"/test/run_envoy_bazel_coverage.sh "${COVERAGE_TEST_TARGETS[@]}" - collect_build_profile coverage - exit 0 -elif [[ "$CI_TARGET" == "bazel.clang_tidy" ]]; then - # clang-tidy will warn on standard library issues with libc++ - ENVOY_STDLIB="libstdc++" - setup_clang_toolchain - - export CLANG_TIDY_FIX_DIFF="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixed.diff" - export FIX_YAML="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixes.yaml" - export CLANG_TIDY_APPLY_FIXES=1 - mkdir -p "${ENVOY_TEST_TMPDIR}/lint-fixes" - BAZEL_BUILD_OPTIONS+=(--remote_download_minimal) - NUM_CPUS=$NUM_CPUS "${ENVOY_SRCDIR}"/ci/run_clang_tidy.sh "$@" || { - if [[ -s "$FIX_YAML" ]]; then - echo >&2 - echo "Diff/yaml files with (some) fixes will be uploaded. Please check the artefacts for this PR run in the azure pipeline." >&2 - echo >&2 - else - echo "Clang-tidy failed." >&2 - fi - exit 1 - } - exit 0 -elif [[ "$CI_TARGET" == "bazel.fuzz" ]]; then - setup_clang_toolchain - FUZZ_TEST_TARGETS=("$(bazel query "attr('tags','fuzzer',${TEST_TARGETS[*]})")") - echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS[*]}" - echo "Building envoy fuzzers and executing 100 fuzz iterations..." - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --config=asan-fuzzer "${FUZZ_TEST_TARGETS[@]}" --test_arg="-runs=10" - exit 0 -elif [[ "$CI_TARGET" == "format" ]]; then - setup_clang_toolchain - "${ENVOY_SRCDIR}"/ci/format_pre.sh -elif [[ "$CI_TARGET" == "fix_proto_format" ]]; then - # proto_format.sh needs to build protobuf. - setup_clang_toolchain - "${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" fix - exit 0 -elif [[ "$CI_TARGET" == "check_proto_format" ]]; then - setup_clang_toolchain - echo "Check proto format ..." - "${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" check - exit 0 -elif [[ "$CI_TARGET" == "check_and_fix_proto_format" ]]; then - setup_clang_toolchain - echo "Check and fix proto format ..." - "${ENVOY_SRCDIR}/ci/check_and_fix_format.sh" - exit 0 -elif [[ "$CI_TARGET" == "docs" ]]; then - setup_clang_toolchain - echo "generating docs..." - # Build docs. - "${ENVOY_SRCDIR}"/docs/build.sh - exit 0 -elif [[ "$CI_TARGET" == "deps" ]]; then - setup_clang_toolchain - - echo "dependency validate_test..." - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:validate_test - - echo "verifying dependencies..." - # Validate dependency relationships between core/extensions and external deps. - time bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:validate - - # Validate repository metadata. - echo "check repositories..." - "${ENVOY_SRCDIR}"/tools/check_repositories.sh - - echo "check dependencies..." - # Using todays date as an action_env expires the NIST cache daily, which is the update frequency - TODAY_DATE=$(date -u -I"date") - export TODAY_DATE - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:check \ - --action_env=TODAY_DATE \ - -- -v warn \ - -c cves release_dates releases - - # Run dependabot tests - echo "Check dependabot ..." - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:dependatool - - exit 0 -elif [[ "$CI_TARGET" == "verify_examples" ]]; then - run_ci_verify "*" "win32-front-proxy|shared" - exit 0 -elif [[ "$CI_TARGET" == "verify_distro" ]]; then - if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - PACKAGE_BUILD=/build/bazel.distribution/packages.x64.tar.gz - else - PACKAGE_BUILD=/build/bazel.distribution.arm64/packages.arm64.tar.gz - fi - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //distribution:verify_packages "$PACKAGE_BUILD" - exit 0 -elif [[ "$CI_TARGET" == "publish" ]]; then - # If we are on a non-main release branch but the patch version is 0 - then this branch - # has just been created, and the tag/release was cut from `main` - there is no need to - # create the tag/release from here - version="$(cat VERSION.txt)" - patch_version="$(echo "$version" | rev | cut -d. -f1)" - if [[ "$AZP_BRANCH" == "main" || "$AZP_BRANCH" == "refs/heads/main" ]]; then - if [[ "$patch_version" -eq 0 ]]; then - # It can take some time to get here in CI so the branch may have changed - create the release - # from the current commit (as this only happens on non-PRs we are safe from merges) - BUILD_SHA="$(git rev-parse HEAD)" - bazel run "${BAZEL_BUILD_OPTIONS[@]}" @envoy_repo//:publish -- --publish-commitish="$BUILD_SHA" - exit 0 + setup_clang_toolchain + + ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" + mkdir -p "$ENVOY_BINARY_DIR" + + # Run release tests + echo "Testing ${TEST_TARGETS[*]} with options: ${BAZEL_BUILD_OPTIONS[*]}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_minimal -c opt "${TEST_TARGETS[@]}" + + # Build release binaries + # As the binary build package enforces `-c opt`, adding here to ensure the distribution build reuses + # any already compiled artefacts, the bundle itself will always be compiled `-c opt` + bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c opt //distribution/binary:release + + # Copy release binaries to binary export directory + cp -a "bazel-bin/distribution/binary/release.tar.zst" "${ENVOY_BINARY_DIR}/release.tar.zst" + + # Grab the schema_validator_tool + # TODO(phlax): bundle this with the release when #26390 is resolved + bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel --stripopt=--strip-all -c opt //test/tools/schema_validator:schema_validator_tool.stripped + + # Copy schema_validator_tool to binary export directory + cp -a bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped "${ENVOY_BINARY_DIR}/schema_validator_tool" + + ;; + distribution) + echo "Building distro packages..." + + setup_clang_toolchain + + # Extract the Envoy binary from the tarball + mkdir -p distribution/custom + if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + ENVOY_RELEASE_TARBALL="/build/bazel.release/bin/release.tar.zst" + else + ENVOY_RELEASE_TARBALL="/build/bazel.release.arm64/bin/release.tar.zst" fi - fi - echo "Not creating a tag/release for ${version} from ${AZP_BRANCH}" - exit 0 -else - echo "Invalid do_ci.sh target, see ci/README.md for valid targets." - exit 1 -fi + bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/zstd -- --stdout -d "$ENVOY_RELEASE_TARBALL" | tar xfO - envoy > distribution/custom/envoy + + # By default the packages will be signed by the first available key. + # If there is no key available, a throwaway key is created + # and the packages signed with it, for the purpose of testing only. + if ! gpg --list-secret-keys "*"; then + export PACKAGES_MAINTAINER_NAME="Envoy CI" + export PACKAGES_MAINTAINER_EMAIL="envoy-ci@for.testing.only" + BAZEL_BUILD_OPTIONS+=( + "--action_env=PACKAGES_GEN_KEY=1" + "--action_env=PACKAGES_MAINTAINER_NAME" + "--action_env=PACKAGES_MAINTAINER_EMAIL") + fi + + # Build the packages + bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c opt --//distribution:envoy-binary=//distribution:custom/envoy //distribution:packages.tar.gz + if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.x64.tar.gz" + else + cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.arm64.tar.gz" + fi + ;; + release.server_only) + setup_clang_toolchain + echo "bazel release build..." + bazel_envoy_binary_build release + ;; + sizeopt.server_only) + setup_clang_toolchain + echo "bazel size optimized build..." + bazel_envoy_binary_build sizeopt + ;; + sizeopt) + setup_clang_toolchain + echo "Testing ${TEST_TARGETS[*]}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --config=sizeopt "${TEST_TARGETS[@]}" + + echo "bazel size optimized build with tests..." + bazel_envoy_binary_build sizeopt + ;; + gcc) + BAZEL_BUILD_OPTIONS+=("--test_env=HEAPCHECK=") + setup_gcc_toolchain + + echo "Testing ${TEST_TARGETS[*]}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild --remote_download_minimal -- "${TEST_TARGETS[@]}" + + echo "bazel release build with gcc..." + bazel_envoy_binary_build fastbuild + ;; + debug) + setup_clang_toolchain + echo "Testing ${TEST_TARGETS[*]}" + # Make sure that there are no regressions to building Envoy with autolink disabled. + EXTRA_OPTIONS=( + "--define" "library_autolink=disabled") + bazel test "${BAZEL_BUILD_OPTIONS[@]}" "${EXTRA_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" + + echo "bazel debug build with tests..." + bazel_envoy_binary_build debug + ;; + debug.server_only) + setup_clang_toolchain + echo "bazel debug build..." + bazel_envoy_binary_build debug + ;; + asan) + setup_clang_toolchain + BAZEL_BUILD_OPTIONS+=(-c dbg "--config=clang-asan" "--build_tests_only" "--remote_download_minimal") + echo "bazel ASAN/UBSAN debug build with tests" + echo "Building and testing envoy tests ${TEST_TARGETS[*]}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" "${TEST_TARGETS[@]}" + if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then + echo "Building and testing envoy-filter-example tests..." + pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" + popd + fi + + # TODO(mattklein123): This part of the test is now flaky in CI and it's unclear why, possibly + # due to sandboxing issue. Debug and enable it again. + # if [ "${CI_SKIP_INTEGRATION_TEST_TRAFFIC_TAPPING}" != "1" ] ; then + # Also validate that integration test traffic tapping (useful when debugging etc.) + # works. This requires that we set TAP_PATH. We do this under bazel.asan to + # ensure a debug build in CI. + # echo "Validating integration test traffic tapping..." + # bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" \ + # --run_under=@envoy//bazel/test:verify_tap_test.sh \ + # //test/extensions/transport_sockets/tls/integration:ssl_integration_test + # fi + ;; + tsan) + setup_clang_toolchain + echo "bazel TSAN debug build with tests" + echo "Building and testing envoy tests ${TEST_TARGETS[*]}" + bazel_with_collection test --config=rbe-toolchain-tsan "${BAZEL_BUILD_OPTIONS[@]}" -c dbg --build_tests_only --remote_download_minimal "${TEST_TARGETS[@]}" + if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then + echo "Building and testing envoy-filter-example tests..." + pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c dbg --config=clang-tsan "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" + popd + fi + ;; + msan) + ENVOY_STDLIB=libc++ + setup_clang_toolchain + # rbe-toolchain-msan must comes as first to win library link order. + BAZEL_BUILD_OPTIONS=("--config=rbe-toolchain-msan" "${BAZEL_BUILD_OPTIONS[@]}" "-c" "dbg" "--build_tests_only" "--remote_download_minimal") + echo "bazel MSAN debug build with tests" + echo "Building and testing envoy tests ${TEST_TARGETS[*]}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -- "${TEST_TARGETS[@]}" + ;; + dev) + setup_clang_toolchain + # This doesn't go into CI but is available for developer convenience. + echo "bazel fastbuild build with tests..." + echo "Building..." + bazel_envoy_binary_build fastbuild + + echo "Testing ${TEST_TARGETS[*]}" + bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" + ;; + dev.contrib) + setup_clang_toolchain + # This doesn't go into CI but is available for developer convenience. + echo "bazel fastbuild build with contrib extensions and tests..." + echo "Building..." + bazel_contrib_binary_build fastbuild + + echo "Testing ${TEST_TARGETS[*]}" + bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" + ;; + compile_time_options) + # Right now, none of the available compile-time options conflict with each other. If this + # changes, this build type may need to be broken up. + + COMPILE_TIME_OPTIONS=( + "--define" "admin_html=disabled" + "--define" "signal_trace=disabled" + "--define" "hot_restart=disabled" + "--define" "google_grpc=disabled" + "--define" "boringssl=fips" + "--define" "log_debug_assert_in_release=enabled" + "--define" "path_normalization_by_default=true" + "--define" "deprecated_features=disabled" + "--define" "tcmalloc=gperftools" + "--define" "zlib=ng" + "--define" "uhv=enabled" + "--@envoy//bazel:http3=False" + "--@envoy//source/extensions/filters/http/kill_request:enabled" + "--test_env=ENVOY_HAS_EXTRA_EXTENSIONS=true" + "--remote_download_minimal") + + ENVOY_STDLIB="${ENVOY_STDLIB:-libstdc++}" + setup_clang_toolchain + # This doesn't go into CI but is available for developer convenience. + echo "bazel with different compiletime options build with tests..." + + cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" + TEST_TARGETS=("${TEST_TARGETS[@]/#\/\//@envoy\/\/}") + + # Building all the dependencies from scratch to link them against libc++. + echo "Building and testing with wasm=wamr: ${TEST_TARGETS[*]}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wamr "${COMPILE_TIME_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only + + echo "Building and testing with wasm=wasmtime: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wasmtime --define admin_html=disabled --define admin_functionality=disabled "${COMPILE_TIME_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only + + echo "Building and testing with wasm=wavm: ${TEST_TARGETS[*]}" + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only + + # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only + # these tests under "-c opt" to save time in CI. + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c opt @envoy//test/common/common:assert_test @envoy//test/server:server_test + + # "--define log_fast_debug_assert_in_release=enabled" must be tested with a release build, so run only these tests under "-c opt" to save time in CI. This option will test only ASSERT()s without SLOW_ASSERT()s, so additionally disable "--define log_debug_assert_in_release" which compiles in both. + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c opt @envoy//test/common/common:assert_test --define log_fast_debug_assert_in_release=enabled --define log_debug_assert_in_release=disabled + + echo "Building binary with wasm=wavm... and logging disabled" + bazel build "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm --define enable_logging=disabled "${COMPILE_TIME_OPTIONS[@]}" -c dbg @envoy//source/exe:envoy-static --build_tag_filters=-nofips + collect_build_profile build + ;; + api) + # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib, + # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild. + ENVOY_STDLIB="libstdc++" + setup_clang_toolchain + export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config + echo "Run protoxform test" + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + --//tools/api_proto_plugin:default_type_db_target=//tools/testdata/protoxform:fix_protos \ + --//tools/api_proto_plugin:extra_args=api_version:3.7 \ + //tools/protoprint:protoprint_test + echo "Validating API structure..." + "${ENVOY_SRCDIR}"/tools/api/validate_structure.py + echo "Validate Golang protobuf generation..." + "${ENVOY_SRCDIR}"/tools/api/generate_go_protobuf.py + echo "Testing API..." + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_minimal -c fastbuild @envoy_api//test/... @envoy_api//tools/... \ + @envoy_api//tools:tap2pcap_test + echo "Building API..." + bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api//envoy/... + ;; + api_compat) + echo "Checking API for breaking changes to protobuf backwards compatibility..." + BASE_BRANCH_REF=$("${ENVOY_SRCDIR}"/tools/git/last_github_commit.sh) + COMMIT_TITLE=$(git log -n 1 --pretty='format:%C(auto)%h (%s, %ad)' "${BASE_BRANCH_REF}") + echo -e "\tUsing base commit ${COMMIT_TITLE}" + # BAZEL_BUILD_OPTIONS needed for setting the repository_cache param. + bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/api_proto_breaking_change_detector:detector_ci "${BASE_BRANCH_REF}" + ;; + coverage|fuzz_coverage) + setup_clang_toolchain + + echo "${CI_TARGET} build with tests ${COVERAGE_TEST_TARGETS[*]}" + + [[ "$CI_TARGET" == "fuzz_coverage" ]] && export FUZZ_COVERAGE=true + + # We use custom BAZEL_BUILD_OPTIONS here to cover profiler's code. + BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]} --define tcmalloc=gperftools" "${ENVOY_SRCDIR}"/test/run_envoy_bazel_coverage.sh "${COVERAGE_TEST_TARGETS[@]}" + collect_build_profile coverage + ;; + clang_tidy) + # clang-tidy will warn on standard library issues with libc++ + ENVOY_STDLIB="libstdc++" + setup_clang_toolchain + + export CLANG_TIDY_FIX_DIFF="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixed.diff" + export FIX_YAML="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixes.yaml" + export CLANG_TIDY_APPLY_FIXES=1 + mkdir -p "${ENVOY_TEST_TMPDIR}/lint-fixes" + BAZEL_BUILD_OPTIONS+=(--remote_download_minimal) + NUM_CPUS=$NUM_CPUS "${ENVOY_SRCDIR}"/ci/run_clang_tidy.sh "$@" || { + if [[ -s "$FIX_YAML" ]]; then + echo >&2 + echo "Diff/yaml files with (some) fixes will be uploaded. Please check the artefacts for this PR run in the azure pipeline." >&2 + echo >&2 + else + echo "Clang-tidy failed." >&2 + fi + exit 1 + } + ;; + fuzz) + setup_clang_toolchain + FUZZ_TEST_TARGETS=("$(bazel query "attr('tags','fuzzer',${TEST_TARGETS[*]})")") + echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS[*]}" + echo "Building envoy fuzzers and executing 100 fuzz iterations..." + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --config=asan-fuzzer "${FUZZ_TEST_TARGETS[@]}" --test_arg="-runs=10" + ;; + format) + setup_clang_toolchain + "${ENVOY_SRCDIR}"/ci/format_pre.sh + ;; + fix_proto_format) + # proto_format.sh needs to build protobuf. + setup_clang_toolchain + "${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" fix + ;; + check_proto_format) + setup_clang_toolchain + echo "Check proto format ..." + "${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" check + ;; + check_and_fix_proto_format) + setup_clang_toolchain + echo "Check and fix proto format ..." + "${ENVOY_SRCDIR}/ci/check_and_fix_format.sh" + ;; + docs) + setup_clang_toolchain + echo "generating docs..." + # Build docs. + "${ENVOY_SRCDIR}"/docs/build.sh + ;; + deps) + setup_clang_toolchain + + echo "dependency validate_test..." + bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:validate_test + + echo "verifying dependencies..." + # Validate dependency relationships between core/extensions and external deps. + time bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:validate + + # Validate repository metadata. + echo "check repositories..." + "${ENVOY_SRCDIR}"/tools/check_repositories.sh + + echo "check dependencies..." + # Using todays date as an action_env expires the NIST cache daily, which is the update frequency + TODAY_DATE=$(date -u -I"date") + export TODAY_DATE + bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:check \ + --action_env=TODAY_DATE \ + -- -v warn \ + -c cves release_dates releases + + # Run dependabot tests + echo "Check dependabot ..." + bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:dependatool + ;; + verify_examples) + run_ci_verify "*" "win32-front-proxy|shared" + ;; + verify_distro) + if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + PACKAGE_BUILD=/build/bazel.distribution/packages.x64.tar.gz + else + PACKAGE_BUILD=/build/bazel.distribution.arm64/packages.arm64.tar.gz + fi + bazel run "${BAZEL_BUILD_OPTIONS[@]}" //distribution:verify_packages "$PACKAGE_BUILD" + ;; + publish) + # If we are on a non-main release branch but the patch version is 0 - then this branch + # has just been created, and the tag/release was cut from `main` - there is no need to + # create the tag/release from here + version="$(cat VERSION.txt)" + patch_version="$(echo "$version" | rev | cut -d. -f1)" + if [[ "$AZP_BRANCH" == "main" || "$AZP_BRANCH" == "refs/heads/main" ]]; then + if [[ "$patch_version" -eq 0 ]]; then + # It can take some time to get here in CI so the branch may have changed - create the release + # from the current commit (as this only happens on non-PRs we are safe from merges) + BUILD_SHA="$(git rev-parse HEAD)" + bazel run "${BAZEL_BUILD_OPTIONS[@]}" @envoy_repo//:publish -- --publish-commitish="$BUILD_SHA" + exit 0 + fi + fi + echo "Not creating a tag/release for ${version} from ${AZP_BRANCH}" + ;; + *) + echo "Invalid do_ci.sh target (${CI_TARGET}), see ci/README.md for valid targets." + exit 1 + ;; +esac From a380096537d1439206c89f413e44d44a8d8a01cb Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Thu, 27 Apr 2023 10:27:46 -0700 Subject: [PATCH 019/740] matcher: Add RuntimeFraction input matcher (#26958) * matcher: Add RuntimeFraction input matcher Signed-off-by: Ruslan Nigmatullin --- CODEOWNERS | 2 + api/BUILD | 1 + .../input_matchers/runtime_fraction/v3/BUILD | 12 ++ .../v3/runtime_fraction.proto | 35 +++++ api/versioning/BUILD | 1 + changelogs/current.yaml | 4 + .../common_messages/common_messages.rst | 1 + source/extensions/extensions_build_config.bzl | 1 + source/extensions/extensions_metadata.yaml | 7 + .../input_matchers/runtime_fraction/BUILD | 34 +++++ .../input_matchers/runtime_fraction/config.cc | 33 +++++ .../input_matchers/runtime_fraction/config.h | 34 +++++ .../input_matchers/runtime_fraction/matcher.h | 41 ++++++ .../input_matchers/runtime_fraction/BUILD | 34 +++++ .../runtime_fraction/config_test.cc | 59 +++++++++ .../runtime_fraction/matcher_test.cc | 123 ++++++++++++++++++ 16 files changed, 422 insertions(+) create mode 100644 api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/BUILD create mode 100644 api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/runtime_fraction.proto create mode 100644 source/extensions/matching/input_matchers/runtime_fraction/BUILD create mode 100644 source/extensions/matching/input_matchers/runtime_fraction/config.cc create mode 100644 source/extensions/matching/input_matchers/runtime_fraction/config.h create mode 100644 source/extensions/matching/input_matchers/runtime_fraction/matcher.h create mode 100644 test/extensions/matching/input_matchers/runtime_fraction/BUILD create mode 100644 test/extensions/matching/input_matchers/runtime_fraction/config_test.cc create mode 100644 test/extensions/matching/input_matchers/runtime_fraction/matcher_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index 801f9378f652..195ef7302781 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -165,6 +165,8 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/rate_limit_descriptors/expr @kyessenov @lizan # hash input matcher /*/extensions/matching/input_matchers/consistent_hashing @snowp @donyu +# runtime fraction input matcher +/*/extensions/matching/input_matchers/runtime_fraction @ravenblackx @UNOWNED # environment generic input /*/extensions/matching/common_inputs/environment @snowp @donyu # format string matching diff --git a/api/BUILD b/api/BUILD index 6129177ef46b..26ac05ccad40 100644 --- a/api/BUILD +++ b/api/BUILD @@ -263,6 +263,7 @@ proto_library( "//envoy/extensions/matching/common_inputs/ssl/v3:pkg", "//envoy/extensions/matching/input_matchers/consistent_hashing/v3:pkg", "//envoy/extensions/matching/input_matchers/ip/v3:pkg", + "//envoy/extensions/matching/input_matchers/runtime_fraction/v3:pkg", "//envoy/extensions/network/dns_resolver/apple/v3:pkg", "//envoy/extensions/network/dns_resolver/cares/v3:pkg", "//envoy/extensions/network/dns_resolver/getaddrinfo/v3:pkg", diff --git a/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/BUILD b/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/BUILD new file mode 100644 index 000000000000..1c1a6f6b4423 --- /dev/null +++ b/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/runtime_fraction.proto b/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/runtime_fraction.proto new file mode 100644 index 000000000000..33a559e10f46 --- /dev/null +++ b/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/runtime_fraction.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package envoy.extensions.matching.input_matchers.runtime_fraction.v3; + +import "envoy/config/core/v3/base.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.matching.input_matchers.runtime_fraction.v3"; +option java_outer_classname = "RuntimeFractionProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/matching/input_matchers/runtime_fraction/v3;runtime_fractionv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Runtime matcher] +// [#extension: envoy.matching.matchers.runtime_fraction] + +// The runtime fraction matchers computes a hash from the input and matches if runtime feature is enabled +// for the the resulting hash. Every time the input is considered for a match, its hash must fall within +// the percentage of matches indicated by this field. For a fraction N/D, a number is computed as a hash +// of the input on a field in the range [0,D). If the number is less than or equal to the value of the +// numerator N, the matcher evaluates to true. A runtime_fraction input matcher can be used to gradually +// roll out matcher changes without requiring full code or configuration deployments. +// Note that distribution of matching results is only as good as one of the input. +message RuntimeFraction { + // Match the input against the given runtime key. The specified default value is used if key is not + // present in the runtime configuration. + config.core.v3.RuntimeFractionalPercent runtime_fraction = 1 + [(validate.rules).message = {required: true}]; + + // Optional seed passed through the hash function. This allows using additional information when computing + // the hash value: by changing the seed value, a potentially different outcome can be achieved for the same input. + uint64 seed = 2; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 3ccb627ee867..5d0548a4bbc3 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -202,6 +202,7 @@ proto_library( "//envoy/extensions/matching/common_inputs/ssl/v3:pkg", "//envoy/extensions/matching/input_matchers/consistent_hashing/v3:pkg", "//envoy/extensions/matching/input_matchers/ip/v3:pkg", + "//envoy/extensions/matching/input_matchers/runtime_fraction/v3:pkg", "//envoy/extensions/network/dns_resolver/apple/v3:pkg", "//envoy/extensions/network/dns_resolver/cares/v3:pkg", "//envoy/extensions/network/dns_resolver/getaddrinfo/v3:pkg", diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 9544f31da376..83dfbe937311 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -64,6 +64,10 @@ new_features: added Runtime feature ``envoy.reloadable_features.max_request_headers_size_kb`` to override the default value of :ref:`max request headers size `. +- area: matchers + change: | + Added :ref:`RuntimeFraction ` input + matcher. It allows matching hash of the input on a runtime key. - area: stat_sinks change: | Added ``envoy.stat_sinks.open_telemetry`` stats_sink, that supports flushing metrics by the OTLP protocol, diff --git a/docs/root/api-v3/common_messages/common_messages.rst b/docs/root/api-v3/common_messages/common_messages.rst index fbe9cc97f275..66d3389f738a 100644 --- a/docs/root/api-v3/common_messages/common_messages.rst +++ b/docs/root/api-v3/common_messages/common_messages.rst @@ -30,6 +30,7 @@ Common messages ../extensions/early_data/v3/default_early_data_policy.proto ../config/core/v3/http_uri.proto ../extensions/matching/input_matchers/ip/v3/ip.proto + ../extensions/matching/input_matchers/runtime_fraction/v3/runtime_fraction.proto ../config/core/v3/address.proto ../config/core/v3/protocol.proto ../config/core/v3/proxy_protocol.proto diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index a39fe32ecb9b..c8b5431bc579 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -72,6 +72,7 @@ EXTENSIONS = { "envoy.matching.matchers.consistent_hashing": "//source/extensions/matching/input_matchers/consistent_hashing:config", "envoy.matching.matchers.ip": "//source/extensions/matching/input_matchers/ip:config", + "envoy.matching.matchers.runtime_fraction": "//source/extensions/matching/input_matchers/runtime_fraction:config", # # Network Matchers diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index f71bce09dc56..15fbfcab0466 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -863,6 +863,13 @@ envoy.matching.matchers.ip: status: stable type_urls: - envoy.extensions.matching.input_matchers.ip.v3.Ip +envoy.matching.matchers.runtime_fraction: + categories: + - envoy.matching.input_matchers + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable + type_urls: + - envoy.extensions.matching.input_matchers.runtime_fraction.v3.RuntimeFraction envoy.path.match.uri_template.uri_template_matcher: categories: - envoy.path.match diff --git a/source/extensions/matching/input_matchers/runtime_fraction/BUILD b/source/extensions/matching/input_matchers/runtime_fraction/BUILD new file mode 100644 index 000000000000..54d7f48e135b --- /dev/null +++ b/source/extensions/matching/input_matchers/runtime_fraction/BUILD @@ -0,0 +1,34 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "runtime_fraction_lib", + hdrs = ["matcher.h"], + deps = [ + "//envoy/matcher:matcher_interface", + "//envoy/upstream:retry_interface", + "//source/common/common:hash_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":runtime_fraction_lib", + "//envoy/matcher:matcher_interface", + "//envoy/registry", + "//envoy/server:factory_context_interface", + "@envoy_api//envoy/extensions/matching/input_matchers/runtime_fraction/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/matching/input_matchers/runtime_fraction/config.cc b/source/extensions/matching/input_matchers/runtime_fraction/config.cc new file mode 100644 index 000000000000..7752f51862a7 --- /dev/null +++ b/source/extensions/matching/input_matchers/runtime_fraction/config.cc @@ -0,0 +1,33 @@ +#include "source/extensions/matching/input_matchers/runtime_fraction/config.h" + +#include "envoy/extensions/matching/input_matchers/runtime_fraction/v3/runtime_fraction.pb.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace RuntimeFraction { + +Envoy::Matcher::InputMatcherFactoryCb +Config::createInputMatcherFactoryCb(const Protobuf::Message& config, + Server::Configuration::ServerFactoryContext& factory_context) { + const auto& runtime_fraction_config = MessageUtil::downcastAndValidate< + const envoy::extensions::matching::input_matchers::runtime_fraction::v3::RuntimeFraction&>( + config, factory_context.messageValidationVisitor()); + + auto& runtime = factory_context.runtime(); + return [runtime_fraction_config, &runtime]() { + return std::make_unique(runtime, runtime_fraction_config.runtime_fraction(), + runtime_fraction_config.seed()); + }; +} +/** + * Static registration for the runtime fraction matcher. @see RegisterFactory. + */ +REGISTER_FACTORY(Config, Envoy::Matcher::InputMatcherFactory); + +} // namespace RuntimeFraction +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/matching/input_matchers/runtime_fraction/config.h b/source/extensions/matching/input_matchers/runtime_fraction/config.h new file mode 100644 index 000000000000..d247cd2bb9a6 --- /dev/null +++ b/source/extensions/matching/input_matchers/runtime_fraction/config.h @@ -0,0 +1,34 @@ +#pragma once + +#include "envoy/extensions/matching/input_matchers/runtime_fraction/v3/runtime_fraction.pb.h" +#include "envoy/extensions/matching/input_matchers/runtime_fraction/v3/runtime_fraction.pb.validate.h" +#include "envoy/matcher/matcher.h" +#include "envoy/server/factory_context.h" + +#include "source/common/protobuf/utility.h" +#include "source/extensions/matching/input_matchers/runtime_fraction/matcher.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace RuntimeFraction { + +class Config : public Envoy::Matcher::InputMatcherFactory { +public: + Envoy::Matcher::InputMatcherFactoryCb createInputMatcherFactoryCb( + const Protobuf::Message& config, + Server::Configuration::ServerFactoryContext& factory_context) override; + + std::string name() const override { return "envoy.matching.matchers.runtime_fraction"; } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique< + envoy::extensions::matching::input_matchers::runtime_fraction::v3::RuntimeFraction>(); + } +}; +} // namespace RuntimeFraction +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/matching/input_matchers/runtime_fraction/matcher.h b/source/extensions/matching/input_matchers/runtime_fraction/matcher.h new file mode 100644 index 000000000000..e805b1e478ce --- /dev/null +++ b/source/extensions/matching/input_matchers/runtime_fraction/matcher.h @@ -0,0 +1,41 @@ +#pragma once + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/matcher/matcher.h" +#include "envoy/runtime/runtime.h" + +#include "source/common/common/hash.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace RuntimeFraction { + +class Matcher : public Envoy::Matcher::InputMatcher { +public: + Matcher(Runtime::Loader& runtime, + envoy::config::core::v3::RuntimeFractionalPercent runtime_fraction, uint64_t seed) + : runtime_(runtime), runtime_fraction_(runtime_fraction), seed_(seed) {} + bool match(absl::optional input) override { + // Only match if the value is present. + if (!input) { + return false; + } + + // Otherwise, match if feature is enabled for hash(input). + const auto hash_value = HashUtil::xxHash64(*input, seed_); + return runtime_.snapshot().featureEnabled(runtime_fraction_.runtime_key(), + runtime_fraction_.default_value(), hash_value); + } + +private: + Runtime::Loader& runtime_; + const envoy::config::core::v3::RuntimeFractionalPercent runtime_fraction_; + const uint64_t seed_; +}; +} // namespace RuntimeFraction +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/matching/input_matchers/runtime_fraction/BUILD b/test/extensions/matching/input_matchers/runtime_fraction/BUILD new file mode 100644 index 000000000000..96767f502d3d --- /dev/null +++ b/test/extensions/matching/input_matchers/runtime_fraction/BUILD @@ -0,0 +1,34 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.matching.matchers.runtime_fraction"], + deps = [ + "//source/extensions/matching/input_matchers/runtime_fraction:config", + "//test/mocks/server:factory_context_mocks", + ], +) + +envoy_extension_cc_test( + name = "matcher_test", + srcs = ["matcher_test.cc"], + extension_names = ["envoy.matching.matchers.runtime_fraction"], + deps = [ + "//source/extensions/matching/input_matchers/runtime_fraction:runtime_fraction_lib", + "//test/mocks/runtime:runtime_mocks", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/type/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/matching/input_matchers/runtime_fraction/config_test.cc b/test/extensions/matching/input_matchers/runtime_fraction/config_test.cc new file mode 100644 index 000000000000..d00b487b61a5 --- /dev/null +++ b/test/extensions/matching/input_matchers/runtime_fraction/config_test.cc @@ -0,0 +1,59 @@ +#include "source/extensions/matching/input_matchers/runtime_fraction/config.h" + +#include "test/mocks/server/factory_context.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace RuntimeFraction { + +TEST(ConfigTest, TestConfig) { + NiceMock factory_context; + const std::string yaml_string = R"EOF( + name: hashing + typed_config: + "@type": type.googleapis.com/envoy.extensions.matching.input_matchers.runtime_fraction.v3.RuntimeFraction + runtime_fraction: + default_value: + numerator: 50 + denominator: MILLION + runtime_key: "some_key" +)EOF"; + + envoy::config::core::v3::TypedExtensionConfig config; + TestUtility::loadFromYaml(yaml_string, config); + + Config factory; + auto message = Envoy::Config::Utility::translateAnyToFactoryConfig( + config.typed_config(), ProtobufMessage::getStrictValidationVisitor(), factory); + auto matcher = factory.createInputMatcherFactoryCb(*message, factory_context); + ASSERT_NE(nullptr, matcher); + matcher(); +} + +TEST(ConfigTest, InvalidConfig) { + NiceMock factory_context; + + const std::string yaml_string = R"EOF( + name: hashing + typed_config: + "@type": type.googleapis.com/envoy.extensions.matching.input_matchers.runtime_fraction.v3.RuntimeFraction +)EOF"; + + envoy::config::core::v3::TypedExtensionConfig config; + TestUtility::loadFromYaml(yaml_string, config); + + Config factory; + auto message = Envoy::Config::Utility::translateAnyToFactoryConfig( + config.typed_config(), ProtobufMessage::getStrictValidationVisitor(), factory); + EXPECT_THROW_WITH_REGEX(factory.createInputMatcherFactoryCb(*message, factory_context), + EnvoyException, "RuntimeFraction: value is required"); +} +} // namespace RuntimeFraction +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/matching/input_matchers/runtime_fraction/matcher_test.cc b/test/extensions/matching/input_matchers/runtime_fraction/matcher_test.cc new file mode 100644 index 000000000000..72b74f429a03 --- /dev/null +++ b/test/extensions/matching/input_matchers/runtime_fraction/matcher_test.cc @@ -0,0 +1,123 @@ +#include + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/type/v3/percent.pb.h" + +#include "source/extensions/matching/input_matchers/runtime_fraction/matcher.h" + +#include "test/mocks/runtime/mocks.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace RuntimeFraction { + +using testing::_; + +namespace { +class TestMatcher { +public: + TestMatcher(std::string key, uint32_t numerator, + envoy::type::v3::FractionalPercent_DenominatorType denominator, uint64_t seed) + : key_(key) { + default_value_.set_numerator(numerator); + default_value_.set_denominator(denominator); + + envoy::config::core::v3::RuntimeFractionalPercent runtime_fraction; + runtime_fraction.set_runtime_key(key); + runtime_fraction.mutable_default_value()->CopyFrom(default_value_); + + matcher_ = std::make_unique(runtime_, runtime_fraction, seed); + } + + uint64_t matchAndReturnHash(absl::optional value, bool result) { + uint64_t called_random_value = 0; + EXPECT_CALL( + runtime_.snapshot_, + featureEnabled(key_, testing::Matcher(_), _)) + .WillOnce([&](absl::string_view, const envoy::type::v3::FractionalPercent& default_value, + uint64_t random_value) -> bool { + EXPECT_THAT(default_value, ProtoEq(default_value_)); + called_random_value = random_value; + return result; + }); + EXPECT_EQ(matcher_->match(value), result); + return called_random_value; + } + + void matchWithoutValue() { + EXPECT_CALL( + runtime_.snapshot_, + featureEnabled(key_, testing::Matcher(_), _)) + .Times(0); + EXPECT_FALSE(matcher_->match(absl::nullopt)); + } + +private: + testing::NiceMock runtime_; + std::string key_; + envoy::type::v3::FractionalPercent default_value_; + std::unique_ptr matcher_; +}; + +} // namespace + +class MatcherTest : public testing::Test { +protected: + void SetUp() override { + matcher1_ = + std::make_unique("key1", 42, envoy::type::v3::FractionalPercent::HUNDRED, 0); + matcher2_ = std::make_unique("key2", 21, + envoy::type::v3::FractionalPercent::TEN_THOUSAND, 0); + matcher3_ = + std::make_unique("key3", 42, envoy::type::v3::FractionalPercent::HUNDRED, 1); + } + + std::unique_ptr matcher1_; + std::unique_ptr matcher2_; + std::unique_ptr matcher3_; +}; + +TEST_F(MatcherTest, NoMatchOnNoInput) { + // If there is no input, fallback to no match. + matcher1_->matchWithoutValue(); + matcher2_->matchWithoutValue(); + matcher3_->matchWithoutValue(); +} + +TEST_F(MatcherTest, SameInput) { + // Same input provides the same hash iff seed is the same. + const auto hash1 = matcher1_->matchAndReturnHash("value1", true); + const auto hash2 = matcher2_->matchAndReturnHash("value1", true); + const auto hash3 = matcher3_->matchAndReturnHash("value1", true); + + EXPECT_EQ(hash1, hash2); + EXPECT_NE(hash1, hash3); +} + +TEST_F(MatcherTest, DifferentInput) { + // Different input provides different hash. + const auto hash1 = matcher1_->matchAndReturnHash("value1", true); + const auto hash2 = matcher2_->matchAndReturnHash("value2", true); + const auto hash3 = matcher3_->matchAndReturnHash("value3", true); + + EXPECT_NE(hash1, hash2); + EXPECT_NE(hash1, hash3); + EXPECT_NE(hash2, hash3); +} + +TEST_F(MatcherTest, HonorsRuntimeValue) { + // Matcher propagates result of runtime's `featureEnabled`. + matcher1_->matchAndReturnHash("value1", true); + matcher2_->matchAndReturnHash("value2", false); +} + +} // namespace RuntimeFraction +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy From 7f2605312e2ce8fef5bae698ab3452618703eb38 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 27 Apr 2023 20:21:13 +0100 Subject: [PATCH 020/740] mobile/ci: Add timeout and err handling on hanging jobs (#27011) Signed-off-by: Ryan Northey --- .github/workflows/mobile-android_build.yml | 37 ++++++++++++++-- .../workflows/mobile-compile_time_options.yml | 44 ++++++++++--------- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml index f60c5b024271..cf889b38ac2d 100644 --- a/.github/workflows/mobile-android_build.yml +++ b/.github/workflows/mobile-android_build.yml @@ -86,7 +86,14 @@ jobs: adb shell am start -n io.envoyproxy.envoymobile.helloenvoy/.MainActivity - name: 'Check connectivity' if: steps.should_run.outputs.run_ci_job == 'true' - run: adb logcat -e "received headers with status 301" -m 1 + run: | + timeout 30 adb logcat -e "received headers with status 301" -m 1 || { + echo "Failed checking for headers in adb logcat" >&2 + timeout 30 adb logcat || { + echo "Failed dumping adb logcat" >&2 + } + exit 1 + } kotlinhelloworld: if: github.repository == 'envoyproxy/envoy' name: kotlin_helloworld @@ -133,7 +140,15 @@ jobs: adb shell am start -n io.envoyproxy.envoymobile.helloenvoykotlin/.MainActivity - name: 'Check connectivity' if: steps.should_run.outputs.run_ci_job == 'true' - run: adb logcat -e "received headers with status 200" -m 1 + run: | + timeout 30 adb logcat -e "received headers with status 200" -m 1 || { + echo "Failed checking for headers in adb logcat" >&2 + timeout 30 adb logcat || { + echo "Failed dumping adb logcat" >&2 + } + exit 1 + } + kotlinbaselineapp: if: github.repository == 'envoyproxy/envoy' name: kotlin_baseline_app @@ -180,7 +195,14 @@ jobs: adb shell am start -n io.envoyproxy.envoymobile.helloenvoybaselinetest/.MainActivity - name: 'Check connectivity' if: steps.should_run.outputs.run_ci_job == 'true' - run: adb logcat -e "received headers with status 301" -m 1 + run: | + timeout 30 adb logcat -e "received headers with status 301" -m 1 || { + echo "Failed checking for headers in adb logcat" >&2 + timeout 30 adb logcat || { + echo "Failed dumping adb logcat" >&2 + } + exit 1 + } kotlinexperimentalapp: if: github.repository == 'envoyproxy/envoy' name: kotlin_experimental_app @@ -229,4 +251,11 @@ jobs: adb shell am start -n io.envoyproxy.envoymobile.helloenvoyexperimentaltest/.MainActivity - name: 'Check connectivity' if: steps.should_run.outputs.run_ci_job == 'true' - run: adb logcat -e "received headers with status 200" -m 1 + run: | + timeout 30 adb logcat -e "received headers with status 200" -m 1 || { + echo "Failed checking for headers in adb logcat" >&2 + timeout 30 adb logcat || { + echo "Failed dumping adb logcat" >&2 + } + exit 1 + } diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index cdf3c7178221..f085ab5dce04 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -80,7 +80,7 @@ jobs: if: github.repository == 'envoyproxy/envoy' name: kotlin_build runs-on: macos-12 - timeout-minutes: 90 + timeout-minutes: 120 steps: - uses: actions/checkout@v3 with: @@ -103,23 +103,25 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - cd mobile && ./bazelw build \ - $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ - --fat_apk_cpu=x86_64 \ - --define=admin_html=enabled \ - --define=envoy_mobile_request_compression=disabled \ - --define=envoy_enable_http_datagrams=disabled \ - --define=google_grpc=disabled \ - --define=envoy_yaml=disabled \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //:android_dist && ./bazelw test \ - --fat_apk_cpu=x86_64 \ - --define=admin_html=enabled \ - --define=envoy_mobile_request_compression=disabled \ - --define=envoy_enable_http_datagrams=disabled \ - --define=google_grpc=disabled \ - --define=envoy_yaml=disabled \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //test/java/integration:android_engine_socket_tag_test \ - //test/java/integration:android_engine_start_test \ - //test/java/io/envoyproxy/envoymobile/utilities:certificate_verification_tests + cd mobile \ + && ./bazelw build \ + $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ + --fat_apk_cpu=x86_64 \ + --define=admin_html=enabled \ + --define=envoy_mobile_request_compression=disabled \ + --define=envoy_enable_http_datagrams=disabled \ + --define=google_grpc=disabled \ + --define=envoy_yaml=disabled \ + --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ + //:android_dist \ + && ./bazelw test \ + --fat_apk_cpu=x86_64 \ + --define=admin_html=enabled \ + --define=envoy_mobile_request_compression=disabled \ + --define=envoy_enable_http_datagrams=disabled \ + --define=google_grpc=disabled \ + --define=envoy_yaml=disabled \ + --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ + //test/java/integration:android_engine_socket_tag_test \ + //test/java/integration:android_engine_start_test \ + //test/java/io/envoyproxy/envoymobile/utilities:certificate_verification_tests From 9bae4457dac6054db14cd42d0e03e746cb35f72b Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 27 Apr 2023 15:34:58 -0400 Subject: [PATCH 021/740] datadog: Config (final) portion of new tracing library (#26284) This is the final part of the work proposed in #23958. This PR additionally contains all of the changes from #26148. This diff will shrink when that PR is merged. This revision alters the DatadogTracerFactory in source/extensions/tracers/datadog/config.h to produce instances of the new dd-trace-cpp based implementation of the Datadog tracer, instead of the old dd-opentracing-cpp based one. This revision actually alters the implementation of the Datadog tracer. Previous pull requests have only added code for use here. This revision also moves some unit tests around: Two tests that were part of datadog_tracer_impl_test.cc have been moved into agent_http_client_test.cc, because they're about HTTP requests. One test that was part of datadog_tracer_impl_test.cc has been removed because its behavior is already covered by tests added in other PRs in this series. The remaining tests in datadog_tracer_impl_test_.cc were moved into config_test.cc, since they all now have to do with the configuration of the tracer and the resulting behavior. Finally, all references to dd-opentracing-cpp and msgpack have been removed from the repo, and the documentation (READMEs) and demo/ directory from #23958 have been added. Signed-off-by: David Goffredo --- .bazelrc | 4 - bazel/external/BUILD | 1 - bazel/repositories.bzl | 12 - bazel/repository_locations.bzl | 36 +- source/extensions/tracers/datadog/.gitignore | 2 + source/extensions/tracers/datadog/BUILD | 7 +- source/extensions/tracers/datadog/README.md | 181 +++++++ source/extensions/tracers/datadog/config.cc | 40 +- source/extensions/tracers/datadog/config.h | 16 +- .../tracers/datadog/datadog_tracer_impl.cc | 159 ------ .../tracers/datadog/datadog_tracer_impl.h | 147 ------ .../extensions/tracers/datadog/demo/README.md | 3 + .../tracers/datadog/demo/docker-compose.yaml | 33 ++ source/extensions/tracers/datadog/demo/envoy | 4 + .../tracers/datadog/demo/envoy.yaml | 95 ++++ .../tracers/datadog/demo/http.dockerfile | 14 + .../extensions/tracers/datadog/demo/http.js | 33 ++ source/extensions/tracers/datadog/diagram.dot | 42 ++ source/extensions/tracers/datadog/diagram.svg | 149 ++++++ .../common/ot/opentracing_driver_impl_test.cc | 142 +++++- test/extensions/tracers/datadog/BUILD | 19 +- .../tracers/datadog/agent_http_client_test.cc | 114 +++++ .../extensions/tracers/datadog/config_test.cc | 161 +++++- .../datadog/datadog_tracer_impl_test.cc | 466 ------------------ test/extensions/tracers/datadog/span_test.cc | 11 + .../extensions/tracers/datadog/tracer_test.cc | 3 + 26 files changed, 1034 insertions(+), 860 deletions(-) create mode 100644 source/extensions/tracers/datadog/.gitignore create mode 100644 source/extensions/tracers/datadog/README.md delete mode 100644 source/extensions/tracers/datadog/datadog_tracer_impl.cc delete mode 100644 source/extensions/tracers/datadog/datadog_tracer_impl.h create mode 100644 source/extensions/tracers/datadog/demo/README.md create mode 100644 source/extensions/tracers/datadog/demo/docker-compose.yaml create mode 100755 source/extensions/tracers/datadog/demo/envoy create mode 100644 source/extensions/tracers/datadog/demo/envoy.yaml create mode 100644 source/extensions/tracers/datadog/demo/http.dockerfile create mode 100644 source/extensions/tracers/datadog/demo/http.js create mode 100644 source/extensions/tracers/datadog/diagram.dot create mode 100644 source/extensions/tracers/datadog/diagram.svg delete mode 100644 test/extensions/tracers/datadog/datadog_tracer_impl_test.cc diff --git a/.bazelrc b/.bazelrc index 695c1bd2eb00..637a9a65bea2 100644 --- a/.bazelrc +++ b/.bazelrc @@ -58,10 +58,6 @@ build:linux --features=per_object_debug_info build:linux --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a build:linux --action_env=BAZEL_LINKOPTS=-lm -# TODO(keith): remove once https://github.com/DataDog/dd-opentracing-cpp/pull/252 is integrated -# this avoids warnings/errors on arm64 Linux builds -build:linux --per_file_copt=external/com_github_datadog_dd_opentracing_cpp/.*.cpp@-Wno-type-limits - # We already have absl in the build, define absl=1 to tell googletest to use absl for backtrace. build --define absl=1 diff --git a/bazel/external/BUILD b/bazel/external/BUILD index 4558a1cee2cd..ac145dcdfd17 100644 --- a/bazel/external/BUILD +++ b/bazel/external/BUILD @@ -12,7 +12,6 @@ cc_library( # @io_opentracing_cpp//:generate_version_h failed - needs porting tags = ["skip_on_windows"], deps = [ - "@com_github_datadog_dd_opentracing_cpp//:dd_opentracing_cpp", "@com_github_datadog_dd_trace_cpp//:dd_trace_cpp", "@com_google_googletest//:gtest", "@io_opentracing_cpp//:opentracing", diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 6d2cf2014cba..daa63ea13f27 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -244,7 +244,6 @@ def envoy_dependencies(skip_targets = []): _com_github_c_ares_c_ares() _com_github_circonus_labs_libcircllhist() _com_github_cyan4973_xxhash() - _com_github_datadog_dd_opentracing_cpp() _com_github_datadog_dd_trace_cpp() _com_github_mirror_tclap() _com_github_envoyproxy_sqlparser() @@ -650,17 +649,6 @@ def _io_opentracing_cpp(): actual = "@io_opentracing_cpp//:opentracing", ) -def _com_github_datadog_dd_opentracing_cpp(): - external_http_archive("com_github_datadog_dd_opentracing_cpp") - external_http_archive( - name = "com_github_msgpack_msgpack_c", - build_file = "@com_github_datadog_dd_opentracing_cpp//:bazel/external/msgpack.BUILD", - ) - native.bind( - name = "dd_opentracing_cpp", - actual = "@com_github_datadog_dd_opentracing_cpp//:dd_opentracing_cpp", - ) - def _com_github_datadog_dd_trace_cpp(): external_http_archive("com_github_datadog_dd_trace_cpp") native.bind( diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index dea81ec7dc50..6191b6905e94 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -520,32 +520,17 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "Apache-2.0", license_url = "https://github.com/SkyAPM/cpp2sky/blob/v{version}/LICENSE", ), - com_github_datadog_dd_opentracing_cpp = dict( - project_name = "Datadog OpenTracing C++ Client", - project_desc = "Datadog OpenTracing C++ Client", - project_url = "https://github.com/DataDog/dd-opentracing-cpp", - version = "1.2.1", - sha256 = "ae44699e4aa2d21b70ed897a6c0cf3ed7dfb411e1aae4e686e39af75cec7c9bf", - strip_prefix = "dd-opentracing-cpp-{version}", - urls = ["https://github.com/DataDog/dd-opentracing-cpp/archive/v{version}.tar.gz"], - use_category = ["observability_ext"], - extensions = ["envoy.tracers.datadog"], - release_date = "2021-01-27", - cpe = "N/A", - license = "Apache-2.0", - license_url = "https://github.com/DataDog/dd-opentracing-cpp/blob/v{version}/LICENSE", - ), com_github_datadog_dd_trace_cpp = dict( project_name = "Datadog C++ Tracing Library", project_desc = "Datadog distributed tracing for C++", project_url = "https://github.com/DataDog/dd-trace-cpp", - version = "0.1.7", - sha256 = "1dd304885e1d66d78dbeb43a6dc6d08b077802a7a604bb41ed6a02303abac276", + version = "0.1.8", + sha256 = "77162c26ba976b8b18e6daf50beaec39389286840733b08e1627d4e5572d4b51", strip_prefix = "dd-trace-cpp-{version}", urls = ["https://github.com/DataDog/dd-trace-cpp/archive/v{version}.tar.gz"], use_category = ["observability_ext"], extensions = ["envoy.tracers.datadog"], - release_date = "2023-03-17", + release_date = "2023-03-31", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/DataDog/dd-trace-cpp/blob/v{version}/LICENSE", @@ -733,21 +718,6 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "MIT", license_url = "https://github.com/jbeder/yaml-cpp/blob/{version}/LICENSE", ), - com_github_msgpack_msgpack_c = dict( - project_name = "msgpack for C/C++", - project_desc = "MessagePack is an efficient binary serialization format", - project_url = "https://github.com/msgpack/msgpack-c", - version = "3.3.0", - sha256 = "6e114d12a5ddb8cb11f669f83f32246e484a8addd0ce93f274996f1941c1f07b", - strip_prefix = "msgpack-{version}", - urls = ["https://github.com/msgpack/msgpack-c/releases/download/cpp-{version}/msgpack-{version}.tar.gz"], - use_category = ["observability_ext"], - extensions = ["envoy.tracers.datadog"], - release_date = "2020-06-05", - cpe = "N/A", - license = "Boost", - license_url = "https://github.com/msgpack/msgpack-c/blob/cpp-{version}/LICENSE_1_0.txt", - ), com_github_google_jwt_verify = dict( project_name = "jwt_verify_lib", project_desc = "JWT verification library for C++", diff --git a/source/extensions/tracers/datadog/.gitignore b/source/extensions/tracers/datadog/.gitignore new file mode 100644 index 000000000000..501d004e6c00 --- /dev/null +++ b/source/extensions/tracers/datadog/.gitignore @@ -0,0 +1,2 @@ +# Prevent the exclusion of the diagram used in the readme. +!diagram.svg diff --git a/source/extensions/tracers/datadog/BUILD b/source/extensions/tracers/datadog/BUILD index bb8d6c162d9e..43c0f6f2c0e7 100644 --- a/source/extensions/tracers/datadog/BUILD +++ b/source/extensions/tracers/datadog/BUILD @@ -15,7 +15,6 @@ envoy_cc_library( name = "datadog_tracer_lib", srcs = [ "agent_http_client.cc", - "datadog_tracer_impl.cc", "dict_util.cc", "event_scheduler.cc", "logger.cc", @@ -25,7 +24,6 @@ envoy_cc_library( ], hdrs = [ "agent_http_client.h", - "datadog_tracer_impl.h", "dict_util.h", "event_scheduler.h", "logger.h", @@ -41,15 +39,14 @@ envoy_cc_library( ], external_deps = [ "dd_trace_cpp", - "dd_opentracing_cpp", ], deps = [ "//source/common/config:utility_lib", "//source/common/http:async_client_utility_lib", + "//source/common/tracing:null_span_lib", "//source/common/upstream:cluster_update_tracker_lib", "//source/common/version:version_lib", - "//source/extensions/tracers/common/ot:opentracing_driver_lib", - "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + "//source/extensions/tracers/common:factory_base_lib", ], ) diff --git a/source/extensions/tracers/datadog/README.md b/source/extensions/tracers/datadog/README.md new file mode 100644 index 000000000000..69922a7bf8f5 --- /dev/null +++ b/source/extensions/tracers/datadog/README.md @@ -0,0 +1,181 @@ +Datadog Tracer +============== +This directory contains the source code for distributed tracing using Datadog. +It is a wrapper around [dd-trace-cpp][2], Datadog's core tracing C++ library. + +The unit tests are under [../../../../test/extensions/tracers/datadog/][1]. + +Design +------ +The following diagram illustrates the relationships among classes relevant to +this library. + +Each node denotes one of the following: + +- Hexagonal nodes are interfaces defined by Envoy. +- Nodes that mention the 🐶 namespace (`::datadog::tracing`) are + defined within the Datadog core C++ tracing library, [dd-trace-cpp][2]. +- Rectangular nodes are classes defined in this library. + +Each edge denotes one of the following relationships: + +- Edges labeled "has" indicate that the predecessor node contains an instance of (or + a pointer to) the successor node. +- Edges labeled "implemented by" indicate that the successor node is an + implementation of the predecessor node. + +![diagram of components of this library](diagram.svg) + +The design of [dd-trace-cpp][2] was informed by the anticipated integration with +Envoy, Nginx, and arbitrary C++ code. The integration-specific operations are: + +- Scheduling an event to happen periodically: sending batched traces to the + Datadog Agent. +- Sending an HTTP POST request to the Datadog Agent and handling the resulting + response. + +For event scheduling there is the [🐶::EventScheduler][4] interface, whose +Envoy-specific implementation is [EventScheduler](./event_scheduler.h). + +For HTTP requests there is the [🐶::HTTPClient][3] interface, whose +Envoy-specific implementation is [AgentHTTPClient](./agent_http_client.h). + +Directory Contents +------------------ +#### `agent_http_client.{h,cc}` +Contains `class AgentHTTPClient`, which is an implementation of +[🐶::HTTPClient][3] in terms of `Http::AsyncClient::Callbacks`. + +The HTTP client is passed to the configuration of +[🐶::DatadogAgent][10], a [🐶::Collector][11] +implementation that uses the HTTP client to send traces to the Datadog Agent. + +#### `BUILD` +The Bazel build configuration for this library ("datadog_tracer_lib"). + +#### `config.{h,cc}` +Contains `class DatadogTracerFactory`, a type that is registered to produce +`Tracer` instances (see `tracer.{h,cc}` below). + +#### `dict_util.{h,cc}` +The Datadog core tracing library uses two interfaces, `DictReader` and +`DictWriter`, for reading and writing, respectively, string mappings for trace +context (e.g. HTTP headers). + +These files contain implementations of those interfaces used by this library. + +#### `event_scheduler.{h,cc}` +Contains `class EventScheduler`, which is an implementation of +[🐶::EventScheduler][4] in terms of `Event::Dispatcher`. + +The event scheduler is passed to the configuration of +[🐶::DatadogAgent][10], a [🐶::Collector][11] +implementation that uses the event scheduler to periodically send batched traces +to the Datadog Agent. + +#### `logger.{h,cc}` +Contains `class Logger`, which is an implementation of +[🐶::Logger][5] in terms of `spdlog::logger`. + +The logger is used by the Datadog core C++ tracing library to print error +diagnostics in cases where an error cannot otherwise be reported, and to print +a banner of the library's configuration on startup. + +#### `span.{h,cc}` +Contains `class Span`, which is an implementation of `Tracing::Span` in terms +of [🐶::Span][6]. + +#### `time_util.{h,cc}` +Contains a conversion function that estimates the value of a [datadog::tracing::TimePoint][7] given a `SystemTime`. + +`🐶::TimePoint` contains both a system time point and a steady +(monotonic) time point. The system time is used to determine when a span +begins, while the steady time is used to determine a span's duration once it +has finished. + +Envoy's tracing interface exposes only the system time. However, the Datadog +core tracing library uses the steady (monotonic) component of the start time +to later calculate a span's duration. The conversion function estimates the +steady time when a span starts by examining the difference between the current +system time and the provided system time, and uses that difference as the offset +to the current steady time. + +#### `tracer_stats.h` +Contains `struct TracerStats`, which is used by this library to keep track of +certain events, such as when sending traces to the Datadog Agent fails. + +#### `tracer.{h,cc}` +Contains `class Tracer`, which is an implementation of `Tracing::Driver` in +terms of [🐶::Tracer][8]. + +Each instance of `Tracer` is local to a worker thread. + +`Tracer` is what is returned by `DatadogTracerFactory::createTracerDriverTyped`. + +Demo +---- +[demo/](demo/) contains a [docker compose][9] setup and supporting +configuration that can be used to manually test Datadog tracing in a local +build of Envoy. + +The local build of Envoy runs on the host, while the Datadog Agent and an +upstream HTTP server are services within the [docker compose setup](demo/docker-compose.yaml). + +See [demo/envoy](demo/envoy) and +[demo/docker-compose.yaml](demo/docker-compose.yaml). + +### Example Usage +In a command shell: +```console +$ cd demo +$ DD_API_KEY=your_datadog_api_key_here docker compose up --build +[...] +demo-http-1 | http node.js web server is running +demo-dd-agent-1 | system-probe exited with code 0, disabling +``` + +In another command shell: +```console +$ cd demo +$ ./envoy +[...] +[2022-11-10 20:06:10.217][264507][info][main] [source/server/server.cc:907] starting main dispatch loop +``` + +In a third command shell: +```console +$ curl 'http://localhost:1337' +{ + "service": "http", + "headers": { + "host": "localhost:1337", + "user-agent": "curl/7.81.0", + "accept": "*/*", + "x-forwarded-for": "172.16.14.142", + "x-forwarded-proto": "http", + "x-envoy-internal": "true", + "x-request-id": "cfa52b85-8660-9532-b347-bd484da76166", + "x-envoy-expected-rq-timeout-ms": "15000", + "x-foobar-banana": "", + "x-datadog-tags": "_dd.p.dm=-0", + "x-datadog-trace-id": "6487770461011569198", + "x-datadog-parent-id": "6487770461011569198", + "x-datadog-sampling-priority": "1" + } +} +``` + +If all went well, then the resulting trace will be visible in Datadog's UI. The +service name is `envoy-demo`. See the [Envoy config](demo/envoy.yaml). + +[1]: ../../../../test/extensions/tracers/datadog/ +[2]: https://github.com/DataDog/dd-trace-cpp +[3]: https://github.com/DataDog/dd-trace-cpp/blob/main/src/datadog/http_client.h +[4]: https://github.com/DataDog/dd-trace-cpp/blob/main/src/datadog/event_scheduler.h +[5]: https://github.com/DataDog/dd-trace-cpp/blob/main/src/datadog/logger.h +[6]: https://github.com/DataDog/dd-trace-cpp/blob/main/src/datadog/span.h +[7]: https://github.com/DataDog/dd-trace-cpp/blob/main/src/datadog/clock.h +[8]: https://github.com/DataDog/dd-trace-cpp/blob/main/src/datadog/tracer.h +[9]: https://docs.docker.com/compose/ +[10]: https://github.com/DataDog/dd-trace-cpp/blob/main/src/datadog/datadog_agent.h +[11]: https://github.com/DataDog/dd-trace-cpp/blob/main/src/datadog/collector.h diff --git a/source/extensions/tracers/datadog/config.cc b/source/extensions/tracers/datadog/config.cc index 562df5f645b1..56d49e7e3094 100644 --- a/source/extensions/tracers/datadog/config.cc +++ b/source/extensions/tracers/datadog/config.cc @@ -1,13 +1,15 @@ #include "source/extensions/tracers/datadog/config.h" +#include + +#include + #include "envoy/config/trace/v3/datadog.pb.h" #include "envoy/config/trace/v3/datadog.pb.validate.h" #include "envoy/registry/registry.h" -#include "source/common/common/utility.h" -#include "source/extensions/tracers/datadog/datadog_tracer_impl.h" - -#include "datadog/opentracing.h" +#include "source/common/version/version.h" +#include "source/extensions/tracers/datadog/tracer.h" namespace Envoy { namespace Extensions { @@ -16,13 +18,35 @@ namespace Datadog { DatadogTracerFactory::DatadogTracerFactory() : FactoryBase("envoy.tracers.datadog") {} +datadog::tracing::TracerConfig +DatadogTracerFactory::makeConfig(const envoy::config::trace::v3::DatadogConfig& proto_config) { + datadog::tracing::TracerConfig config; + config.defaults.version = "envoy " + Envoy::VersionInfo::version(); + config.defaults.name = "envoy.proxy"; + if (proto_config.service_name().empty()) { + config.defaults.service = "envoy"; + } else { + config.defaults.service = proto_config.service_name(); + } + return config; +} + +std::string DatadogTracerFactory::makeCollectorReferenceHost( + const envoy::config::trace::v3::DatadogConfig& proto_config) { + std::string collector_reference_host = proto_config.collector_hostname(); + if (collector_reference_host.empty()) { + collector_reference_host = proto_config.collector_cluster(); + } + return collector_reference_host; +} + Tracing::DriverSharedPtr DatadogTracerFactory::createTracerDriverTyped( const envoy::config::trace::v3::DatadogConfig& proto_config, Server::Configuration::TracerFactoryContext& context) { - return std::make_shared(proto_config, context.serverFactoryContext().clusterManager(), - context.serverFactoryContext().scope(), - context.serverFactoryContext().threadLocal(), - context.serverFactoryContext().runtime()); + return std::make_shared( + proto_config.collector_cluster(), makeCollectorReferenceHost(proto_config), + makeConfig(proto_config), context.serverFactoryContext().clusterManager(), + context.serverFactoryContext().scope(), context.serverFactoryContext().threadLocal()); } /** diff --git a/source/extensions/tracers/datadog/config.h b/source/extensions/tracers/datadog/config.h index f836916fca63..bffa90feadfa 100644 --- a/source/extensions/tracers/datadog/config.h +++ b/source/extensions/tracers/datadog/config.h @@ -7,6 +7,14 @@ #include "source/extensions/tracers/common/factory_base.h" +namespace datadog { +namespace tracing { + +struct TracerConfig; + +} // namespace tracing +} // namespace datadog + namespace Envoy { namespace Extensions { namespace Tracers { @@ -19,8 +27,14 @@ class DatadogTracerFactory : public Common::FactoryBase& tracer, - TraceReporterPtr&& reporter, Driver& driver) - : tracer_(tracer), reporter_(std::move(reporter)), driver_(driver) {} - -Driver::Driver(const envoy::config::trace::v3::DatadogConfig& datadog_config, - Upstream::ClusterManager& cluster_manager, Stats::Scope& scope, - ThreadLocal::SlotAllocator& tls, Runtime::Loader&) - : OpenTracingDriver{scope}, - cm_(cluster_manager), tracer_stats_{DATADOG_TRACER_STATS( - POOL_COUNTER_PREFIX(scope, "tracing.datadog."))}, - tls_(tls.allocateSlot()) { - - Config::Utility::checkCluster("envoy.tracers.datadog", datadog_config.collector_cluster(), cm_, - /* allow_added_via_api */ true); - cluster_ = datadog_config.collector_cluster(); - hostname_ = !datadog_config.collector_hostname().empty() ? datadog_config.collector_hostname() - : datadog_config.collector_cluster(); - - // Default tracer options. - tracer_options_.version = absl::StrCat("envoy ", Envoy::VersionInfo::version()); - tracer_options_.operation_name_override = "envoy.proxy"; - tracer_options_.service = "envoy"; - tracer_options_.inject = std::set{ - datadog::opentracing::PropagationStyle::Datadog}; - tracer_options_.extract = std::set{ - datadog::opentracing::PropagationStyle::Datadog}; - tracer_options_.log_func = [](datadog::opentracing::LogLevel level, - opentracing::string_view message) { - switch (level) { - case datadog::opentracing::LogLevel::debug: - ENVOY_LOG(debug, "{}", message); - break; - case datadog::opentracing::LogLevel::info: - ENVOY_LOG(info, "{}", message); - break; - case datadog::opentracing::LogLevel::error: - ENVOY_LOG(error, "{}", message); - break; - } - }; - - // Configuration overrides for tracer options. - if (!datadog_config.service_name().empty()) { - tracer_options_.service = datadog_config.service_name(); - } - - tls_->set([this](Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { - auto tp = datadog::opentracing::makeTracerAndEncoder(tracer_options_); - auto tracer = std::get<0>(tp); - auto encoder = std::get<1>(tp); - TraceReporterPtr reporter(new TraceReporter(encoder, *this, dispatcher)); - return ThreadLocal::ThreadLocalObjectSharedPtr{ - new TlsTracer(tracer, std::move(reporter), *this)}; - }); -} - -opentracing::Tracer& Driver::tracer() { return *tls_->getTyped().tracer_; } - -TraceReporter::TraceReporter(TraceEncoderSharedPtr encoder, Driver& driver, - Event::Dispatcher& dispatcher) - : driver_(driver), encoder_(encoder), - collector_cluster_(driver_.clusterManager(), driver_.cluster()) { - flush_timer_ = dispatcher.createTimer([this]() -> void { - for (auto& h : encoder_->headers()) { - lower_case_headers_.emplace(h.first, Http::LowerCaseString{h.first}); - } - driver_.tracerStats().timer_flushed_.inc(); - flushTraces(); - enableTimer(); - }); - - enableTimer(); -} - -void TraceReporter::enableTimer() { - // The duration for this timer should not be a factor of the - // datadog-agent's read timer of 5000ms. - // Further details in https://github.com/envoyproxy/envoy/pull/5358 - flush_timer_->enableTimer(std::chrono::milliseconds(900U)); -} - -void TraceReporter::flushTraces() { - auto pendingTraces = encoder_->pendingTraces(); - if (pendingTraces) { - ENVOY_LOG(debug, "flushing traces: {} traces", pendingTraces); - driver_.tracerStats().traces_sent_.add(pendingTraces); - - Http::RequestMessagePtr message(new Http::RequestMessageImpl()); - message->headers().setReferenceMethod(Http::Headers::get().MethodValues.Post); - message->headers().setReferencePath(encoder_->path()); - message->headers().setReferenceHost(driver_.hostname()); - for (auto& h : encoder_->headers()) { - message->headers().setReferenceKey(lower_case_headers_.at(h.first), h.second); - } - - message->body().add(encoder_->payload()); - ENVOY_LOG(debug, "submitting {} trace(s) to {} with payload size {}", pendingTraces, - encoder_->path(), encoder_->payload().size()); - - if (collector_cluster_.threadLocalCluster().has_value()) { - Http::AsyncClient::Request* request = - collector_cluster_.threadLocalCluster()->get().httpAsyncClient().send( - std::move(message), *this, - Http::AsyncClient::RequestOptions().setTimeout(std::chrono::milliseconds(1000U))); - if (request) { - active_requests_.add(*request); - } - } else { - ENVOY_LOG(debug, "collector cluster '{}' does not exist", driver_.cluster()); - driver_.tracerStats().reports_skipped_no_cluster_.inc(); - } - - encoder_->clearTraces(); - } -} - -void TraceReporter::onFailure(const Http::AsyncClient::Request& request, - Http::AsyncClient::FailureReason) { - active_requests_.remove(request); - ENVOY_LOG(debug, "failure submitting traces to datadog agent"); - driver_.tracerStats().reports_failed_.inc(); -} - -void TraceReporter::onSuccess(const Http::AsyncClient::Request& request, - Http::ResponseMessagePtr&& http_response) { - active_requests_.remove(request); - uint64_t responseStatus = Http::Utility::getResponseStatus(http_response->headers()); - if (responseStatus != enumToInt(Http::Code::OK)) { - // TODO: Consider adding retries for failed submissions. - ENVOY_LOG(debug, "unexpected HTTP response code from datadog agent: {}", responseStatus); - driver_.tracerStats().reports_dropped_.inc(); - } else { - ENVOY_LOG(debug, "traces successfully submitted to datadog agent"); - driver_.tracerStats().reports_sent_.inc(); - encoder_->handleResponse(http_response->bodyAsString()); - } -} - -} // namespace Datadog -} // namespace Tracers -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/tracers/datadog/datadog_tracer_impl.h b/source/extensions/tracers/datadog/datadog_tracer_impl.h deleted file mode 100644 index bcdb6ba442c6..000000000000 --- a/source/extensions/tracers/datadog/datadog_tracer_impl.h +++ /dev/null @@ -1,147 +0,0 @@ -#pragma once - -#include "envoy/config/trace/v3/datadog.pb.h" -#include "envoy/local_info/local_info.h" -#include "envoy/runtime/runtime.h" -#include "envoy/thread_local/thread_local.h" -#include "envoy/upstream/cluster_manager.h" - -#include "source/common/http/async_client_utility.h" -#include "source/common/http/header_map_impl.h" -#include "source/common/json/json_loader.h" -#include "source/common/upstream/cluster_update_tracker.h" -#include "source/extensions/tracers/common/ot/opentracing_driver_impl.h" - -#include "datadog/opentracing.h" -#include "fmt/ostream.h" - -namespace Envoy { -namespace Extensions { -namespace Tracers { -namespace Datadog { - -#define DATADOG_TRACER_STATS(COUNTER) \ - COUNTER(traces_sent) \ - COUNTER(timer_flushed) \ - COUNTER(reports_skipped_no_cluster) \ - COUNTER(reports_sent) \ - COUNTER(reports_dropped) \ - COUNTER(reports_failed) - -struct DatadogTracerStats { - DATADOG_TRACER_STATS(GENERATE_COUNTER_STRUCT) -}; - -class TraceReporter; -using TraceReporterPtr = std::unique_ptr; -using TraceEncoderSharedPtr = std::shared_ptr; - -/** - * Class for a Datadog-specific Driver. - */ -class Driver : public Common::Ot::OpenTracingDriver { -public: - /** - * Constructor. It adds itself and a newly-created Datadog::Tracer object to a thread-local store. - */ - Driver(const envoy::config::trace::v3::DatadogConfig& datadog_config, - Upstream::ClusterManager& cluster_manager, Stats::Scope& scope, - ThreadLocal::SlotAllocator& tls, Runtime::Loader& runtime); - - // Getters to return the DatadogDriver's key members. - Upstream::ClusterManager& clusterManager() { return cm_; } - const std::string& cluster() { return cluster_; } - const std::string& hostname() { return hostname_; } - DatadogTracerStats& tracerStats() { return tracer_stats_; } - const datadog::opentracing::TracerOptions& tracerOptions() { return tracer_options_; } - - // Tracer::OpenTracingDriver - opentracing::Tracer& tracer() override; - PropagationMode propagationMode() const override { - return Common::Ot::OpenTracingDriver::PropagationMode::TracerNative; - } - -private: - /** - * Thread-local store containing DatadogDriver and Datadog::Tracer objects. - */ - struct TlsTracer : ThreadLocal::ThreadLocalObject { - TlsTracer(const std::shared_ptr& tracer, TraceReporterPtr&& reporter, - Driver& driver); - - std::shared_ptr tracer_; - TraceReporterPtr reporter_; - Driver& driver_; - }; - - Upstream::ClusterManager& cm_; - std::string cluster_; - std::string hostname_; - DatadogTracerStats tracer_stats_; - datadog::opentracing::TracerOptions tracer_options_; - ThreadLocal::SlotPtr tls_; -}; - -/** - * This class wraps the encoder provided with the tracer at initialization - * and uses Http::AsyncClient to send completed traces to the Datadog Agent. - * - * The cluster to use for submitting traces to the agent is controlled with - * the setting tracing.datadog.collector_cluster, which is mandatory and must - * refer to a cluster in the active configuration. - * - * An internal timer is used to control how often traces are submitted. - * If zero traces have completed in the interval between timer events, - * no action is taken. - * The timer interval can be controlled with the setting - * tracing.datadog.flush_interval_ms, and defaults to 2000ms. - */ -class TraceReporter : public Http::AsyncClient::Callbacks, - protected Logger::Loggable { -public: - /** - * Constructor. - * - * @param encoder Provides methods to retrieve data for publishing traces. - * @param driver The driver to be associated with the reporter. - * @param dispatcher Controls the timer used to flush buffered traces. - */ - TraceReporter(TraceEncoderSharedPtr encoder, Driver& driver, Event::Dispatcher& dispatcher); - - // Http::AsyncClient::Callbacks. - void onSuccess(const Http::AsyncClient::Request&, Http::ResponseMessagePtr&&) override; - void onFailure(const Http::AsyncClient::Request&, Http::AsyncClient::FailureReason) override; - void onBeforeFinalizeUpstreamSpan(Tracing::Span&, const Http::ResponseHeaderMap*) override {} - -private: - /** - * Enables the trace-flushing timer. - */ - void enableTimer(); - - /** - * Removes all traces from the trace buffer and sends them to a Datadog Agent using - * Http::AsyncClient. - */ - void flushTraces(); - - Driver& driver_; - Event::TimerPtr flush_timer_; - TraceEncoderSharedPtr encoder_; - - std::map lower_case_headers_; - - Upstream::ClusterUpdateTracker collector_cluster_; - // Track active HTTP requests to be able to cancel them on destruction. - Http::AsyncClientRequestTracker active_requests_; -}; -} // namespace Datadog -} // namespace Tracers -} // namespace Extensions -} // namespace Envoy - -// NOLINT(namespace-envoy) -namespace fmt { -// Allow fmtlib to format opentracing::string_view -template <> struct formatter : ostream_formatter {}; -} // namespace fmt diff --git a/source/extensions/tracers/datadog/demo/README.md b/source/extensions/tracers/datadog/demo/README.md new file mode 100644 index 000000000000..0c5756263e13 --- /dev/null +++ b/source/extensions/tracers/datadog/demo/README.md @@ -0,0 +1,3 @@ +See the "Demo" section of the [parent readme][1]. + +[1]: ../README.md diff --git a/source/extensions/tracers/datadog/demo/docker-compose.yaml b/source/extensions/tracers/datadog/demo/docker-compose.yaml new file mode 100644 index 000000000000..40b81aa75c33 --- /dev/null +++ b/source/extensions/tracers/datadog/demo/docker-compose.yaml @@ -0,0 +1,33 @@ +version: "3.2" + +services: + + # `agent` is the Datadog Agent to which traces are sent. + # `agent` needs an API key set in the environment as the + # `DD_API_KEY` environment variable. + dd-agent: + volumes: + - '/var/run/docker.sock:/var/run/docker.sock:ro' + - '/proc/:/host/proc/:ro' + - '/sys/fs/cgroup/:/host/sys/fs/cgroup:ro' + environment: + - DD_API_KEY + - DD_APM_ENABLED=true + - DD_LOG_LEVEL=ERROR + ports: + - 8126:8126 + image: 'datadog/agent' + + # `http` is an HTTP server that is reverse proxied by `nginx`. + http: + build: + context: . + dockerfile: ./http.dockerfile + environment: + - DD_ENV=prod + - DD_AGENT_HOST=dd-agent + - DD_SERVICE=envoy-demo-http + ports: + - 8080:8080 + depends_on: + - dd-agent diff --git a/source/extensions/tracers/datadog/demo/envoy b/source/extensions/tracers/datadog/demo/envoy new file mode 100755 index 000000000000..2a29dc40f391 --- /dev/null +++ b/source/extensions/tracers/datadog/demo/envoy @@ -0,0 +1,4 @@ +#!/bin/sh + +here=$(dirname "$0") +"$(bazelisk info bazel-genfiles)"/source/exe/envoy-static --config-path "$here"/envoy.yaml "$@" diff --git a/source/extensions/tracers/datadog/demo/envoy.yaml b/source/extensions/tracers/datadog/demo/envoy.yaml new file mode 100644 index 000000000000..839ba18aac44 --- /dev/null +++ b/source/extensions/tracers/datadog/demo/envoy.yaml @@ -0,0 +1,95 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 1337 + traffic_direction: OUTBOUND + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: true + tracing: + provider: + name: envoy.tracers.datadog + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.DatadogConfig + collector_cluster: datadog_agent + service_name: envoy-demo + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + log_format: + text_format_source: + inline_string: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% \ + %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% \ + %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \ + \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \ + \"%RESP(X-AMZN-RequestId)%\" \"%REQ(X-DATADOG-TRACE-ID)%\" \"%REQ(X-DATADOG-PARENT-ID)%\"\n" + codec_type: auto + stat_prefix: ingress_http + route_config: + request_headers_to_add: + - header: + key: X-Foobar-Banana + value: " " + name: local_route + virtual_hosts: + - name: backend + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: service1 + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - exact_match: /healthcheck + name: :path + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + use_remote_address: true + clusters: + - name: service1 + connect_timeout: 0.250s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: service1 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: localhost + port_value: 8080 + - name: datadog_agent + connect_timeout: 1s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: datadog_agent + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: localhost + port_value: 8126 + +admin: + access_log_path: "/dev/null" + address: + socket_address: + address: 0.0.0.0 + port_value: 8001 diff --git a/source/extensions/tracers/datadog/demo/http.dockerfile b/source/extensions/tracers/datadog/demo/http.dockerfile new file mode 100644 index 000000000000..17c7875670aa --- /dev/null +++ b/source/extensions/tracers/datadog/demo/http.dockerfile @@ -0,0 +1,14 @@ +FROM alpine:3.16 + +RUN mkdir /opt/app +WORKDIR /opt/app + +# Note: WORKDIR must already be set (as it is above) before installing npm. +# If WORKDIR is not set, then npm is installed at the container root, +# which then causes `npm install` to fail later. +RUN apk update && apk add nodejs npm +RUN npm install dd-trace + +COPY ./http.js /opt/app/http.js + +CMD ["node", "--require", "dd-trace/init", "http.js"] diff --git a/source/extensions/tracers/datadog/demo/http.js b/source/extensions/tracers/datadog/demo/http.js new file mode 100644 index 000000000000..1858c89d8e8d --- /dev/null +++ b/source/extensions/tracers/datadog/demo/http.js @@ -0,0 +1,33 @@ +// This is an HTTP server that listens on port 8080 and responds to all +// requests with some text, including the request headers as JSON. + +const http = require('http'); +const process = require('process'); + +// In order for the span(s) associated with an HTTP request to be considered +// finished, the body of the response corresponding to the request must have +// ended. +function ignoreRequestBody(request) { + function ignore() {} + request.on('data', ignore); + request.on('end', ignore); +} + +function requestListener (request, response) { + ignoreRequestBody(request); + const responseBody = JSON.stringify({ + "service": "http", + "headers": request.headers + }, null, 2); + console.log(responseBody); + response.end(responseBody); +} + +console.log('http node.js web server is running'); +const server = http.createServer(requestListener); +server.listen(8080); + +process.on('SIGTERM', function () { + console.log('Received SIGTERM'); + server.close(function () { process.exit(0); }); +}); diff --git a/source/extensions/tracers/datadog/diagram.dot b/source/extensions/tracers/datadog/diagram.dot new file mode 100644 index 000000000000..3a21c358c6e1 --- /dev/null +++ b/source/extensions/tracers/datadog/diagram.dot @@ -0,0 +1,42 @@ +// To generate diagram.svg from this file, +// +// $ dot -Tsvg -o diagram.svg diagram.dot +// +// It requires the graphviz package, e.g. +// +// $ sudo apt install graphviz + +digraph { + rankdir="TB"; + + tracing_driver [label="Tracing::Driver", shape="hexagon", fixedsize="shape", width=2]; + tracer [label="Tracer", shape="box"]; + dd_tracer [label="🐶::Tracer"]; + dd_collector [label="🐶::Collector"]; + dd_datadog_agent [label="🐶::DatadogAgent"]; + dd_event_scheduler [label="🐶::EventScheduler"]; + dd_http_client [label="🐶::HTTPClient"]; + event_scheduler [label="EventScheduler", shape="box"]; + agent_http_client [label="AgentHTTPClient", shape="box"]; + http_async_client_callbacks [label="Http::AsyncClient::Callbacks", shape="hexagon", fixedsize="shape", width=3]; + event_dispatcher [label="Event::Dispatcher", shape="hexagon", fixedsize="shape", width=2]; + + tracing_driver -> tracer [label="implemented by", fontsize=9]; + + tracer -> dd_tracer [label="has", fontsize=9]; + dd_tracer -> dd_collector [label="has", fontsize=9]; + + // dd_datadog_agent -> dd_collector [label="implements", fontsize=9]; + dd_collector -> dd_datadog_agent [label="implemented by", fontsize=9]; + dd_datadog_agent -> dd_event_scheduler [label="has", fontsize=9]; + dd_datadog_agent -> dd_http_client [label="has", fontsize=9]; + + // event_scheduler -> dd_event_scheduler [label="implements", fontsize=9]; + dd_event_scheduler -> event_scheduler [label="implemented by", fontsize=9]; + // agent_http_client -> dd_http_client [label="implements", fontsize=9]; + dd_http_client -> agent_http_client [label="implemented by", fontsize=9]; + + http_async_client_callbacks -> agent_http_client [label="implemented by", fontsize=9]; + + event_scheduler -> event_dispatcher [label="contains", fontsize=9]; +} diff --git a/source/extensions/tracers/datadog/diagram.svg b/source/extensions/tracers/datadog/diagram.svg new file mode 100644 index 000000000000..33bb717f6d6d --- /dev/null +++ b/source/extensions/tracers/datadog/diagram.svg @@ -0,0 +1,149 @@ + + + + + + +%3 + + + +tracing_driver + +Tracing::Driver + + + +tracer + +Tracer + + + +tracing_driver->tracer + + +implemented by + + + +dd_tracer + +🐶::Tracer + + + +tracer->dd_tracer + + +has + + + +dd_collector + +🐶::Collector + + + +dd_tracer->dd_collector + + +has + + + +dd_datadog_agent + +🐶::DatadogAgent + + + +dd_collector->dd_datadog_agent + + +implemented by + + + +dd_event_scheduler + +🐶::EventScheduler + + + +dd_datadog_agent->dd_event_scheduler + + +has + + + +dd_http_client + +🐶::HTTPClient + + + +dd_datadog_agent->dd_http_client + + +has + + + +event_scheduler + +EventScheduler + + + +dd_event_scheduler->event_scheduler + + +implemented by + + + +agent_http_client + +AgentHTTPClient + + + +dd_http_client->agent_http_client + + +implemented by + + + +event_dispatcher + +Event::Dispatcher + + + +event_scheduler->event_dispatcher + + +contains + + + +http_async_client_callbacks + +Http::AsyncClient::Callbacks + + + +http_async_client_callbacks->agent_http_client + + +implemented by + + + diff --git a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc index a945db148b4c..7e43c3fff9b9 100644 --- a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc +++ b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc @@ -1,4 +1,6 @@ #include +#include +#include #include "source/extensions/tracers/common/ot/opentracing_driver_impl.h" @@ -18,6 +20,121 @@ namespace Common { namespace Ot { namespace { +class NullSpanContext : public opentracing::SpanContext { +public: + void ForeachBaggageItem( + std::function) const override { + // not implemented + } + + virtual std::unique_ptr clone() const noexcept { + return std::make_unique(*this); + } +}; + +class NullSpan : public opentracing::Span { +public: + explicit NullSpan(const opentracing::Tracer& tracer) : tracer_(tracer) {} + + void FinishWithOptions(const opentracing::FinishSpanOptions&) noexcept override { + // not implemented + } + + void SetOperationName(opentracing::string_view /*name*/) noexcept override { + // not implemented + } + + void SetTag(opentracing::string_view /*key*/, + const opentracing::Value& /*value*/) noexcept override { + // not implemented + } + + void SetBaggageItem(opentracing::string_view /*key*/, + opentracing::string_view /*value*/) noexcept override { + // not implemented + } + + std::string BaggageItem(opentracing::string_view /*key*/) const noexcept override { + return ""; // not implemented + } + + void Log(std::initializer_list< + std::pair> /*fields*/) noexcept override { + // not implemented + } + + const opentracing::SpanContext& context() const noexcept override { + static NullSpanContext context; + return context; + } + + const opentracing::Tracer& tracer() const noexcept override { return tracer_; } + + const opentracing::Tracer& tracer_; +}; + +class ContextIteratingTracer : public opentracing::Tracer { +public: + explicit ContextIteratingTracer( + std::vector>& extracted_headers_destination) + : extracted_headers_(&extracted_headers_destination) {} + + std::unique_ptr + StartSpanWithOptions(opentracing::string_view /*operation_name*/, + const opentracing::StartSpanOptions&) const noexcept override { + return std::make_unique(*this); + } + + opentracing::expected Inject(const opentracing::SpanContext&, + std::ostream& /*writer*/) const override { + return {}; // not implemented + } + + opentracing::expected Inject(const opentracing::SpanContext&, + const opentracing::TextMapWriter&) const override { + return {}; // not implemented + } + + opentracing::expected Inject(const opentracing::SpanContext&, + const opentracing::HTTPHeadersWriter&) const override { + return {}; // not implemented + } + + opentracing::expected + Inject(const opentracing::SpanContext& sc, + const opentracing::CustomCarrierWriter& writer) const override { + return opentracing::Tracer::Inject(sc, writer); + } + + opentracing::expected> + Extract(std::istream& /*reader*/) const override { + return std::unique_ptr(); // not implemented + } + + opentracing::expected> + Extract(const opentracing::TextMapReader&) const override { + return std::unique_ptr(); // not implemented + } + + opentracing::expected> + Extract(const opentracing::HTTPHeadersReader& reader) const override { + reader.ForeachKey([this](opentracing::string_view key, + opentracing::string_view value) -> opentracing::expected { + extracted_headers_->emplace_back(key, value); + return {}; + }); + + return std::unique_ptr(); + } + + opentracing::expected> + Extract(const opentracing::CustomCarrierReader& reader) const override { + return opentracing::Tracer::Extract(reader); + } + + std::vector>* extracted_headers_; +}; + class TestDriver : public OpenTracingDriver { public: TestDriver(OpenTracingDriver::PropagationMode propagation_mode, @@ -32,6 +149,10 @@ class TestDriver : public OpenTracingDriver { tracer_ = std::make_shared(std::move(options)); } + TestDriver(const std::shared_ptr& tracer, Stats::Scope& scope) + : OpenTracingDriver{scope}, + propagation_mode_{PropagationMode::TracerNative}, recorder_{nullptr}, tracer_{tracer} {} + const opentracing::mocktracer::InMemoryRecorder& recorder() const { return *recorder_; } // OpenTracingDriver @@ -42,7 +163,7 @@ class TestDriver : public OpenTracingDriver { private: const OpenTracingDriver::PropagationMode propagation_mode_; const opentracing::mocktracer::InMemoryRecorder* recorder_; - std::shared_ptr tracer_; + std::shared_ptr tracer_; }; class OpenTracingDriverTest : public testing::Test { @@ -55,6 +176,11 @@ class OpenTracingDriverTest : public testing::Test { std::make_unique(propagation_mode, propagation_options, *stats_.rootScope()); } + void setupValidDriver(std::vector>& headers_destination) { + auto tracer = std::make_shared(headers_destination); + driver_ = std::make_unique(tracer, *stats_.rootScope()); + } + const std::string operation_name_{"test"}; Http::TestRequestHeaderMapImpl request_headers_{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; @@ -228,6 +354,20 @@ TEST_F(OpenTracingDriverTest, GetTraceId) { ASSERT_EQ(first_span->getTraceIdAsHex(), ""); } +TEST_F(OpenTracingDriverTest, ExtractUsingForeach) { + std::vector> extracted_headers; + setupValidDriver(extracted_headers); + + // Starting a new span, given the `request_headers_`, will visit the headers + // using "for each." We can immediately discard the span. + driver_->startSpan(config_, request_headers_, operation_name_, start_time_, + {Tracing::Reason::Sampling, true}); + + for (const auto& [key, value] : extracted_headers) { + EXPECT_EQ(value, request_headers_.getByKey(key)); + } +} + } // namespace } // namespace Ot } // namespace Common diff --git a/test/extensions/tracers/datadog/BUILD b/test/extensions/tracers/datadog/BUILD index cc213e5661ea..bbd5b05870ac 100644 --- a/test/extensions/tracers/datadog/BUILD +++ b/test/extensions/tracers/datadog/BUILD @@ -15,7 +15,7 @@ envoy_extension_cc_test( name = "datadog_tracer_impl_test", srcs = [ "agent_http_client_test.cc", - "datadog_tracer_impl_test.cc", + "config_test.cc", "dict_util_test.cc", "event_scheduler_test.cc", "logger_test.cc", @@ -41,10 +41,13 @@ envoy_extension_cc_test( "//source/common/http:headers_lib", "//source/common/http:message_lib", "//source/common/runtime:runtime_lib", + "//source/extensions/tracers/datadog:config", "//source/extensions/tracers/datadog:datadog_tracer_lib", "//test/mocks/http:http_mocks", "//test/mocks/local_info:local_info_mocks", "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/mocks/server:tracer_factory_mocks", "//test/mocks/stats:stats_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/tracing:tracing_mocks", @@ -54,17 +57,3 @@ envoy_extension_cc_test( "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", ], ) - -envoy_extension_cc_test( - name = "config_test", - srcs = ["config_test.cc"], - extension_names = ["envoy.tracers.datadog"], - # TODO(wrowe): envoy_extension_ rules don't currently exclude windows extensions - tags = ["skip_on_windows"], - deps = [ - "//source/extensions/tracers/datadog:config", - "//test/mocks/server:server_mocks", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", - ], -) diff --git a/test/extensions/tracers/datadog/agent_http_client_test.cc b/test/extensions/tracers/datadog/agent_http_client_test.cc index 3791d89b9cc9..f4d59eb8b0dc 100644 --- a/test/extensions/tracers/datadog/agent_http_client_test.cc +++ b/test/extensions/tracers/datadog/agent_http_client_test.cc @@ -26,14 +26,22 @@ namespace Tracers { namespace Datadog { namespace { +using testing::DoAll; +using testing::Return; +using testing::WithArg; + struct InitializedMockClusterManager { InitializedMockClusterManager() { + EXPECT_CALL(instance_, addThreadLocalClusterUpdateCallbacks_(_)) + .WillOnce(DoAll(SaveArgAddress(&cluster_update_callbacks_), Return(nullptr))); + instance_.initializeClusters({"fake_cluster"}, {}); instance_.thread_local_cluster_.cluster_.info_->name_ = "fake_cluster"; instance_.initializeThreadLocalClusters({"fake_cluster"}); } NiceMock instance_; + Upstream::ClusterUpdateCallbacks* cluster_update_callbacks_; }; class DatadogAgentHttpClientTest : public testing::Test { @@ -460,6 +468,112 @@ TEST_F(DatadogAgentHttpClientTest, OnBeforeFinalizeUpstreamSpanIsANoOp) { client_.onBeforeFinalizeUpstreamSpan(null_span, nullptr); } +TEST_F(DatadogAgentHttpClientTest, SkipReportIfCollectorClusterHasBeenRemoved) { + // Verify the effect of onClusterAddOrUpdate()/onClusterRemoval() on reporting logic, + // keeping in mind that they will be called both for relevant and irrelevant clusters. + NiceMock& cm = cluster_manager_.instance_; + Upstream::ClusterUpdateCallbacks* cluster_update_callbacks = + cluster_manager_.cluster_update_callbacks_; + + { + // Simulate removal of the relevant cluster. + cluster_update_callbacks->onClusterRemoval("fake_cluster"); + + // Verify that no report will be sent. + EXPECT_CALL(cm.thread_local_cluster_, httpAsyncClient()).Times(0); + EXPECT_CALL(cm.thread_local_cluster_.async_client_, send_(_, _, _)).Times(0); + + // Attempt to send a request. + const auto ignore = [](auto&&...) {}; + datadog::tracing::Expected result = client_.post(url_, ignore, "", ignore, ignore); + EXPECT_TRUE(result); + + // Verify observability. + EXPECT_EQ(1U, stats_.reports_skipped_no_cluster_.value()); + EXPECT_EQ(0U, stats_.reports_sent_.value()); + EXPECT_EQ(0U, stats_.reports_dropped_.value()); + EXPECT_EQ(0U, stats_.reports_failed_.value()); + } + + { + // Simulate addition of an irrelevant cluster. + NiceMock unrelated_cluster; + unrelated_cluster.cluster_.info_->name_ = "unrelated_cluster"; + cluster_update_callbacks->onClusterAddOrUpdate(unrelated_cluster); + + // Verify that no report will be sent. + EXPECT_CALL(cm.thread_local_cluster_, httpAsyncClient()).Times(0); + EXPECT_CALL(cm.thread_local_cluster_.async_client_, send_(_, _, _)).Times(0); + + // Attempt to send a request. + const auto ignore = [](auto&&...) {}; + datadog::tracing::Expected result = client_.post(url_, ignore, "", ignore, ignore); + EXPECT_TRUE(result); + + // Verify observability. + EXPECT_EQ(2U, stats_.reports_skipped_no_cluster_.value()); + EXPECT_EQ(0U, stats_.reports_sent_.value()); + EXPECT_EQ(0U, stats_.reports_dropped_.value()); + EXPECT_EQ(0U, stats_.reports_failed_.value()); + } + + { + // Simulate addition of the relevant cluster. + cluster_update_callbacks->onClusterAddOrUpdate(cm.thread_local_cluster_); + + // Verify that report will be sent. + EXPECT_CALL(cm.thread_local_cluster_, httpAsyncClient()) + .WillOnce(ReturnRef(cm.thread_local_cluster_.async_client_)); + Http::MockAsyncClientRequest request(&cm.thread_local_cluster_.async_client_); + Http::AsyncClient::Callbacks* callback{}; + EXPECT_CALL(cm.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&callback)), Return(&request))); + + // Attempt to send a request. + const auto ignore = [](auto&&...) {}; + datadog::tracing::Expected result = client_.post(url_, ignore, "", ignore, ignore); + EXPECT_TRUE(result); + + // Complete in-flight request. + callback->onFailure(request, Http::AsyncClient::FailureReason::Reset); + + // Verify observability. + EXPECT_EQ(2U, stats_.reports_skipped_no_cluster_.value()); + EXPECT_EQ(0U, stats_.reports_sent_.value()); + EXPECT_EQ(0U, stats_.reports_dropped_.value()); + EXPECT_EQ(1U, stats_.reports_failed_.value()); + } + + { + // Simulate removal of an irrelevant cluster. + cluster_update_callbacks->onClusterRemoval("unrelated_cluster"); + + // Verify that report will be sent. + EXPECT_CALL(cm.thread_local_cluster_, httpAsyncClient()) + .WillOnce(ReturnRef(cm.thread_local_cluster_.async_client_)); + Http::MockAsyncClientRequest request(&cm.thread_local_cluster_.async_client_); + Http::AsyncClient::Callbacks* callback{}; + EXPECT_CALL(cm.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&callback)), Return(&request))); + + // Attempt to send a request. + const auto ignore = [](auto&&...) {}; + datadog::tracing::Expected result = client_.post(url_, ignore, "", ignore, ignore); + EXPECT_TRUE(result); + + // Complete in-flight request. + Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl( + Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "404"}}})); + callback->onSuccess(request, std::move(msg)); + + // Verify observability. + EXPECT_EQ(2U, stats_.reports_skipped_no_cluster_.value()); + EXPECT_EQ(0U, stats_.reports_sent_.value()); + EXPECT_EQ(1U, stats_.reports_dropped_.value()); + EXPECT_EQ(1U, stats_.reports_failed_.value()); + } +} + } // namespace } // namespace Datadog } // namespace Tracers diff --git a/test/extensions/tracers/datadog/config_test.cc b/test/extensions/tracers/datadog/config_test.cc index 38b39d11439e..1ac54fa8e5dc 100644 --- a/test/extensions/tracers/datadog/config_test.cc +++ b/test/extensions/tracers/datadog/config_test.cc @@ -1,16 +1,42 @@ +#include +#include +#include +#include + +#include "envoy/common/time.h" #include "envoy/config/trace/v3/datadog.pb.h" -#include "envoy/config/trace/v3/datadog.pb.validate.h" -#include "envoy/config/trace/v3/http_tracer.pb.h" +#include "source/common/common/base64.h" +#include "source/common/http/header_map_impl.h" +#include "source/common/http/headers.h" +#include "source/common/http/message_impl.h" +#include "source/common/runtime/runtime_impl.h" +#include "source/common/tracing/http_tracer_impl.h" +#include "source/common/tracing/null_span_impl.h" #include "source/extensions/tracers/datadog/config.h" +#include "source/extensions/tracers/datadog/span.h" +#include "source/extensions/tracers/datadog/tracer.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/runtime/mocks.h" #include "test/mocks/server/tracer_factory.h" #include "test/mocks/server/tracer_factory_context.h" +#include "test/mocks/stats/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" +#include "test/mocks/upstream/thread_local_cluster.h" +#include "test/test_common/environment.h" +#include "test/test_common/printers.h" +#include "test/test_common/utility.h" +#include "absl/strings/str_format.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::Eq; +using testing::_; +using testing::Invoke; using testing::NiceMock; using testing::Return; @@ -20,20 +46,89 @@ namespace Tracers { namespace Datadog { namespace { -TEST(DatadogTracerConfigTest, DatadogHttpTracer) { +template Config makeConfig(const std::string& yaml) { + Config result; + TestUtility::loadFromYaml(yaml, result); + return result; +} + +const std::chrono::milliseconds flush_interval(2000); + +class DatadogConfigTest : public testing::Test { +public: + void setup(envoy::config::trace::v3::DatadogConfig& datadog_config, bool init_timer) { + cm_.thread_local_cluster_.cluster_.info_->name_ = "fake_cluster"; + cm_.initializeThreadLocalClusters({"fake_cluster"}); + + if (init_timer) { + timer_ = new NiceMock(&tls_.dispatcher_); + EXPECT_CALL(*timer_, enableTimer(flush_interval, _)); + } + + tracer_ = std::make_unique( + datadog_config.collector_cluster(), + DatadogTracerFactory::makeCollectorReferenceHost(datadog_config), + DatadogTracerFactory::makeConfig(datadog_config), cm_, *stats_.rootScope(), tls_); + } + + void setupValidDriver() { + auto datadog_config = + makeConfig("collector_cluster: fake_cluster"); + + cm_.initializeClusters({"fake_cluster"}, {}); + setup(datadog_config, true); + } + + const std::string operation_name_{"test"}; + Http::TestRequestHeaderMapImpl request_headers_{ + {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; + SystemTime start_time_; + + NiceMock tls_; + NiceMock* timer_; + Stats::TestUtil::TestStore stats_; + NiceMock cm_; + + NiceMock config_; + std::unique_ptr tracer_; +}; + +TEST_F(DatadogConfigTest, ConfigureTracer) { + { + envoy::config::trace::v3::DatadogConfig datadog_config; + + EXPECT_THROW(setup(datadog_config, false), EnvoyException); + } + + { + // Valid config but not valid cluster. + auto datadog_config = + makeConfig("collector_cluster: fake_cluster"); + + EXPECT_THROW(setup(datadog_config, false), EnvoyException); + } + + { + auto datadog_config = + makeConfig("collector_cluster: fake_cluster"); + + cm_.initializeClusters({"fake_cluster"}, {}); + setup(datadog_config, true); + } +} + +TEST_F(DatadogConfigTest, ConfigureViaFactory) { NiceMock context; context.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); - const std::string yaml_string = R"EOF( + auto configuration = makeConfig(R"EOF( http: name: datadog typed_config: "@type": type.googleapis.com/envoy.config.trace.v3.DatadogConfig collector_cluster: fake_cluster service_name: fake_file - )EOF"; - envoy::config::trace::v3::Tracing configuration; - TestUtility::loadFromYaml(yaml_string, configuration); + )EOF"); DatadogTracerFactory factory; auto message = Config::Utility::translateToFactoryConfig( @@ -42,6 +137,56 @@ TEST(DatadogTracerConfigTest, DatadogHttpTracer) { EXPECT_NE(nullptr, datadog_tracer); } +TEST_F(DatadogConfigTest, AllowCollectorClusterToBeAddedViaApi) { + cm_.initializeClusters({"fake_cluster"}, {}); + ON_CALL(*cm_.active_clusters_["fake_cluster"]->info_, addedViaApi()).WillByDefault(Return(true)); + + auto datadog_config = + makeConfig("collector_cluster: fake_cluster"); + + setup(datadog_config, true); +} + +TEST_F(DatadogConfigTest, CollectorHostname) { + // We expect "fake_host" to be the Host header value, instead of the default + // "fake_cluster". + auto datadog_config = makeConfig(R"EOF( + collector_cluster: fake_cluster + collector_hostname: fake_host + )EOF"); + cm_.initializeClusters({"fake_cluster"}, {}); + setup(datadog_config, true); + + Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); + Http::AsyncClient::Callbacks* callback; + const absl::optional timeout(std::chrono::seconds(1)); + EXPECT_CALL(cm_.thread_local_cluster_.async_client_, + send_(_, _, Http::AsyncClient::RequestOptions().setTimeout(timeout))) + .WillOnce( + Invoke([&](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callback = &callbacks; + + // This is the crux of this test. + EXPECT_EQ("fake_host", message->headers().getHostValue()); + + return &request; + })); + + Tracing::SpanPtr span = tracer_->startSpan(config_, request_headers_, operation_name_, + start_time_, {Tracing::Reason::Sampling, true}); + span->finishSpan(); + + // Timer should be re-enabled. + EXPECT_CALL(*timer_, enableTimer(flush_interval, _)); + + timer_->invokeCallback(); + + Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl(Http::ResponseHeaderMapPtr{ + new Http::TestResponseHeaderMapImpl{{":status", "200"}, {"content-length", "0"}}})); + callback->onSuccess(request, std::move(msg)); +} + } // namespace } // namespace Datadog } // namespace Tracers diff --git a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc b/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc deleted file mode 100644 index 4f520507fcf1..000000000000 --- a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc +++ /dev/null @@ -1,466 +0,0 @@ -#include -#include -#include -#include - -#include "envoy/config/trace/v3/datadog.pb.h" - -#include "source/common/common/base64.h" -#include "source/common/http/header_map_impl.h" -#include "source/common/http/headers.h" -#include "source/common/http/message_impl.h" -#include "source/common/runtime/runtime_impl.h" -#include "source/common/tracing/http_tracer_impl.h" -#include "source/extensions/tracers/datadog/datadog_tracer_impl.h" - -#include "test/mocks/http/mocks.h" -#include "test/mocks/local_info/mocks.h" -#include "test/mocks/runtime/mocks.h" -#include "test/mocks/stats/mocks.h" -#include "test/mocks/thread_local/mocks.h" -#include "test/mocks/tracing/mocks.h" -#include "test/mocks/upstream/cluster_manager.h" -#include "test/mocks/upstream/thread_local_cluster.h" -#include "test/test_common/printers.h" -#include "test/test_common/utility.h" - -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -using testing::_; -using testing::AnyNumber; -using testing::DoAll; -using testing::Eq; -using testing::Invoke; -using testing::NiceMock; -using testing::Return; -using testing::ReturnRef; -using testing::StrictMock; -using testing::WithArg; - -namespace Envoy { -namespace Extensions { -namespace Tracers { -namespace Datadog { -namespace { - -class DatadogDriverTest : public testing::Test { -public: - void setup(envoy::config::trace::v3::DatadogConfig& datadog_config, bool init_timer) { - cm_.thread_local_cluster_.cluster_.info_->name_ = "fake_cluster"; - cm_.initializeThreadLocalClusters({"fake_cluster"}); - - if (init_timer) { - timer_ = new NiceMock(&tls_.dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900), _)); - } - - driver_ = std::make_unique(datadog_config, cm_, *stats_.rootScope(), tls_, runtime_); - } - - void setupValidDriver() { - const std::string yaml_string = R"EOF( - collector_cluster: fake_cluster - )EOF"; - envoy::config::trace::v3::DatadogConfig datadog_config; - TestUtility::loadFromYaml(yaml_string, datadog_config); - - cm_.initializeClusters({"fake_cluster"}, {}); - setup(datadog_config, true); - } - - const std::string operation_name_{"test"}; - Http::TestRequestHeaderMapImpl request_headers_{ - {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; - const Http::TestResponseHeaderMapImpl response_headers_{{":status", "500"}}; - SystemTime start_time_; - - NiceMock tls_; - std::unique_ptr driver_; - NiceMock* timer_; - Stats::TestUtil::TestStore stats_; - NiceMock cm_; - NiceMock random_; - NiceMock runtime_; - NiceMock local_info_; - - NiceMock config_; -}; - -TEST_F(DatadogDriverTest, InitializeDriver) { - { - envoy::config::trace::v3::DatadogConfig datadog_config; - - EXPECT_THROW(setup(datadog_config, false), EnvoyException); - } - - { - // Valid config but not valid cluster. - const std::string yaml_string = R"EOF( - collector_cluster: fake_cluster - )EOF"; - envoy::config::trace::v3::DatadogConfig datadog_config; - TestUtility::loadFromYaml(yaml_string, datadog_config); - - EXPECT_THROW(setup(datadog_config, false), EnvoyException); - } - - { - const std::string yaml_string = R"EOF( - collector_cluster: fake_cluster - )EOF"; - envoy::config::trace::v3::DatadogConfig datadog_config; - TestUtility::loadFromYaml(yaml_string, datadog_config); - - cm_.initializeClusters({"fake_cluster"}, {}); - setup(datadog_config, true); - } -} - -TEST_F(DatadogDriverTest, AllowCollectorClusterToBeAddedViaApi) { - cm_.initializeClusters({"fake_cluster"}, {}); - ON_CALL(*cm_.active_clusters_["fake_cluster"]->info_, addedViaApi()).WillByDefault(Return(true)); - - const std::string yaml_string = R"EOF( - collector_cluster: fake_cluster - )EOF"; - envoy::config::trace::v3::DatadogConfig datadog_config; - TestUtility::loadFromYaml(yaml_string, datadog_config); - - setup(datadog_config, true); -} - -TEST_F(DatadogDriverTest, FlushSpansTimer) { - setupValidDriver(); - - Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); - Http::AsyncClient::Callbacks* callback; - const absl::optional timeout(std::chrono::seconds(1)); - EXPECT_CALL(cm_.thread_local_cluster_.async_client_, - send_(_, _, Http::AsyncClient::RequestOptions().setTimeout(timeout))) - .WillOnce( - Invoke([&](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks& callbacks, - const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { - callback = &callbacks; - - EXPECT_EQ("fake_cluster", message->headers().getHostValue()); - EXPECT_EQ("application/msgpack", message->headers().getContentTypeValue()); - - return &request; - })); - - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); - span->finishSpan(); - - // Timer should be re-enabled. - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900), _)); - - timer_->invokeCallback(); - - EXPECT_EQ(1U, stats_.counter("tracing.datadog.timer_flushed").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.traces_sent").value()); - - Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl( - Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "200"}}})); - - callback->onSuccess(request, std::move(msg)); - - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_skipped_no_cluster").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.reports_sent").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_dropped").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_failed").value()); -} - -TEST_F(DatadogDriverTest, NoBody) { - setupValidDriver(); - - Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); - Http::AsyncClient::Callbacks* callback; - const absl::optional timeout(std::chrono::seconds(1)); - EXPECT_CALL(cm_.thread_local_cluster_.async_client_, - send_(_, _, Http::AsyncClient::RequestOptions().setTimeout(timeout))) - .WillOnce( - Invoke([&](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks& callbacks, - const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { - callback = &callbacks; - - EXPECT_EQ("fake_cluster", message->headers().getHostValue()); - EXPECT_EQ("application/msgpack", message->headers().getContentTypeValue()); - - return &request; - })); - - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); - span->finishSpan(); - - // Timer should be re-enabled. - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900), _)); - - timer_->invokeCallback(); - - EXPECT_EQ(1U, stats_.counter("tracing.datadog.timer_flushed").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.traces_sent").value()); - - Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl(Http::ResponseHeaderMapPtr{ - new Http::TestResponseHeaderMapImpl{{":status", "200"}, {"content-length", "0"}}})); - callback->onSuccess(request, std::move(msg)); - - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_skipped_no_cluster").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.reports_sent").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_dropped").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_failed").value()); -} - -TEST_F(DatadogDriverTest, CollectorHostname) { - // Valid config but not valid cluster. - const std::string yaml_string = R"EOF( - collector_cluster: fake_cluster - collector_hostname: fake_host - )EOF"; - envoy::config::trace::v3::DatadogConfig datadog_config; - TestUtility::loadFromYaml(yaml_string, datadog_config); - cm_.initializeClusters({"fake_cluster"}, {}); - setup(datadog_config, true); - - Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); - Http::AsyncClient::Callbacks* callback; - const absl::optional timeout(std::chrono::seconds(1)); - EXPECT_CALL(cm_.thread_local_cluster_.async_client_, - send_(_, _, Http::AsyncClient::RequestOptions().setTimeout(timeout))) - .WillOnce( - Invoke([&](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks& callbacks, - const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { - callback = &callbacks; - - EXPECT_EQ("fake_host", message->headers().getHostValue()); - EXPECT_EQ("application/msgpack", message->headers().getContentTypeValue()); - - return &request; - })); - - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); - span->finishSpan(); - - // Timer should be re-enabled. - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900), _)); - - timer_->invokeCallback(); - - EXPECT_EQ(1U, stats_.counter("tracing.datadog.timer_flushed").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.traces_sent").value()); - - Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl(Http::ResponseHeaderMapPtr{ - new Http::TestResponseHeaderMapImpl{{":status", "200"}, {"content-length", "0"}}})); - callback->onSuccess(request, std::move(msg)); - - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_skipped_no_cluster").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.reports_sent").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_dropped").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_failed").value()); -} - -TEST_F(DatadogDriverTest, SkipReportIfCollectorClusterHasBeenRemoved) { - Upstream::ClusterUpdateCallbacks* cluster_update_callbacks; - EXPECT_CALL(cm_, addThreadLocalClusterUpdateCallbacks_(_)) - .WillOnce(DoAll(SaveArgAddress(&cluster_update_callbacks), Return(nullptr))); - - setupValidDriver(); - - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900), _)).Times(AnyNumber()); - - // Verify the effect of onClusterAddOrUpdate()/onClusterRemoval() on reporting logic, - // keeping in mind that they will be called both for relevant and irrelevant clusters. - - { - // Simulate removal of the relevant cluster. - cluster_update_callbacks->onClusterRemoval("fake_cluster"); - - // Verify that no report will be sent. - EXPECT_CALL(cm_.thread_local_cluster_, httpAsyncClient()).Times(0); - EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)).Times(0); - - // Trigger flush of a span. - driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, - {Tracing::Reason::Sampling, true}) - ->finishSpan(); - timer_->invokeCallback(); - - // Verify observability. - EXPECT_EQ(1U, stats_.counter("tracing.datadog.timer_flushed").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.traces_sent").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.reports_skipped_no_cluster").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_sent").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_dropped").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_failed").value()); - } - - { - // Simulate addition of an irrelevant cluster. - NiceMock unrelated_cluster; - unrelated_cluster.cluster_.info_->name_ = "unrelated_cluster"; - cluster_update_callbacks->onClusterAddOrUpdate(unrelated_cluster); - - // Verify that no report will be sent. - EXPECT_CALL(cm_.thread_local_cluster_, httpAsyncClient()).Times(0); - EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)).Times(0); - - // Trigger flush of a span. - driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, - {Tracing::Reason::Sampling, true}) - ->finishSpan(); - timer_->invokeCallback(); - - // Verify observability. - EXPECT_EQ(2U, stats_.counter("tracing.datadog.timer_flushed").value()); - EXPECT_EQ(2U, stats_.counter("tracing.datadog.traces_sent").value()); - EXPECT_EQ(2U, stats_.counter("tracing.datadog.reports_skipped_no_cluster").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_sent").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_dropped").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_failed").value()); - } - - { - // Simulate addition of the relevant cluster. - cluster_update_callbacks->onClusterAddOrUpdate(cm_.thread_local_cluster_); - - // Verify that report will be sent. - EXPECT_CALL(cm_.thread_local_cluster_, httpAsyncClient()) - .WillOnce(ReturnRef(cm_.thread_local_cluster_.async_client_)); - Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); - Http::AsyncClient::Callbacks* callback{}; - EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)) - .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&callback)), Return(&request))); - - // Trigger flush of a span. - driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, - {Tracing::Reason::Sampling, true}) - ->finishSpan(); - timer_->invokeCallback(); - - // Complete in-flight request. - callback->onFailure(request, Http::AsyncClient::FailureReason::Reset); - - // Verify observability. - EXPECT_EQ(3U, stats_.counter("tracing.datadog.timer_flushed").value()); - EXPECT_EQ(3U, stats_.counter("tracing.datadog.traces_sent").value()); - EXPECT_EQ(2U, stats_.counter("tracing.datadog.reports_skipped_no_cluster").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_sent").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_dropped").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.reports_failed").value()); - } - - { - // Simulate removal of an irrelevant cluster. - cluster_update_callbacks->onClusterRemoval("unrelated_cluster"); - - // Verify that report will be sent. - EXPECT_CALL(cm_.thread_local_cluster_, httpAsyncClient()) - .WillOnce(ReturnRef(cm_.thread_local_cluster_.async_client_)); - Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); - Http::AsyncClient::Callbacks* callback{}; - EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)) - .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&callback)), Return(&request))); - - // Trigger flush of a span. - driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, - {Tracing::Reason::Sampling, true}) - ->finishSpan(); - timer_->invokeCallback(); - - // Complete in-flight request. - Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl( - Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "404"}}})); - callback->onSuccess(request, std::move(msg)); - - // Verify observability. - EXPECT_EQ(4U, stats_.counter("tracing.datadog.timer_flushed").value()); - EXPECT_EQ(4U, stats_.counter("tracing.datadog.traces_sent").value()); - EXPECT_EQ(2U, stats_.counter("tracing.datadog.reports_skipped_no_cluster").value()); - EXPECT_EQ(0U, stats_.counter("tracing.datadog.reports_sent").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.reports_dropped").value()); - EXPECT_EQ(1U, stats_.counter("tracing.datadog.reports_failed").value()); - } -} - -TEST_F(DatadogDriverTest, CancelInflightRequestsOnDestruction) { - setupValidDriver(); - - StrictMock request1(&cm_.thread_local_cluster_.async_client_), - request2(&cm_.thread_local_cluster_.async_client_), - request3(&cm_.thread_local_cluster_.async_client_), - request4(&cm_.thread_local_cluster_.async_client_); - Http::AsyncClient::Callbacks* callback{}; - const absl::optional timeout(std::chrono::seconds(1)); - - // Expect 4 separate report requests to be made. - EXPECT_CALL(cm_.thread_local_cluster_.async_client_, - send_(_, _, Http::AsyncClient::RequestOptions().setTimeout(timeout))) - .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&callback)), Return(&request1))) - .WillOnce(Return(&request2)) - .WillOnce(Return(&request3)) - .WillOnce(Return(&request4)); - // Expect timer to be re-enabled on each tick. - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900), _)).Times(4); - - // Trigger 1st report request. - driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, - {Tracing::Reason::Sampling, true}) - ->finishSpan(); - timer_->invokeCallback(); - // Trigger 2nd report request. - driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, - {Tracing::Reason::Sampling, true}) - ->finishSpan(); - timer_->invokeCallback(); - // Trigger 3rd report request. - driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, - {Tracing::Reason::Sampling, true}) - ->finishSpan(); - timer_->invokeCallback(); - // Trigger 4th report request. - driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, - {Tracing::Reason::Sampling, true}) - ->finishSpan(); - timer_->invokeCallback(); - - Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl( - Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "404"}}})); - // Simulate completion of the 2nd report request. - callback->onSuccess(request2, std::move(msg)); - - // Simulate failure of the 3rd report request. - callback->onFailure(request3, Http::AsyncClient::FailureReason::Reset); - - // Expect 1st and 4th requests to be cancelled on destruction. - EXPECT_CALL(request1, cancel()); - EXPECT_CALL(request4, cancel()); - - // Trigger destruction. - driver_.reset(); -} - -TEST_F(DatadogDriverTest, Logging) { - setupValidDriver(); - - // Ensure calls to the logger function work without errors or crashes. - driver_->tracerOptions().log_func(datadog::opentracing::LogLevel::debug, "debug"); - driver_->tracerOptions().log_func(datadog::opentracing::LogLevel::info, "info"); - driver_->tracerOptions().log_func(datadog::opentracing::LogLevel::error, "error"); -} - -} // namespace -} // namespace Datadog -} // namespace Tracers -} // namespace Extensions -} // namespace Envoy diff --git a/test/extensions/tracers/datadog/span_test.cc b/test/extensions/tracers/datadog/span_test.cc index 0e40766a1bc3..70b4fafd9352 100644 --- a/test/extensions/tracers/datadog/span_test.cc +++ b/test/extensions/tracers/datadog/span_test.cc @@ -22,6 +22,7 @@ #include "datadog/sampling_priority.h" #include "datadog/span_data.h" #include "datadog/tags.h" +#include "datadog/trace_segment.h" #include "datadog/tracer.h" #include "gtest/gtest.h" @@ -209,6 +210,11 @@ TEST_F(DatadogTracerSpanTest, SetSampledTrue) { // `datadog::tracing::tags::internal::sampling_priority` tag set to either -1 // (hard drop) or 2 (hard keep). { + // First ensure that the trace will be dropped (until we override it by + // calling `setSampled`, below). + span_.trace_segment().override_sampling_priority( + static_cast(datadog::tracing::SamplingPriority::USER_DROP)); + Span local_root{std::move(span_)}; auto child = local_root.spawnChild(Tracing::MockConfig{}, "child", time_.timeSystem().systemTime()); @@ -235,6 +241,11 @@ TEST_F(DatadogTracerSpanTest, SetSampledFalse) { // `datadog::tracing::tags::internal::sampling_priority` tag set to either -1 // (hard drop) or 2 (hard keep). { + // First ensure that the trace will be kept (until we override it by calling + // `setSampled`, below). + span_.trace_segment().override_sampling_priority( + static_cast(datadog::tracing::SamplingPriority::USER_KEEP)); + Span local_root{std::move(span_)}; auto child = local_root.spawnChild(Tracing::MockConfig{}, "child", time_.timeSystem().systemTime()); diff --git a/test/extensions/tracers/datadog/tracer_test.cc b/test/extensions/tracers/datadog/tracer_test.cc index a87ad438b9da..a1e0d3e7cde7 100644 --- a/test/extensions/tracers/datadog/tracer_test.cc +++ b/test/extensions/tracers/datadog/tracer_test.cc @@ -84,6 +84,9 @@ TEST_F(DatadogTracerTest, SpanProperties) { // resulting span. datadog::tracing::TracerConfig config; config.defaults.service = "envoy"; + // Configure the tracer to keep all spans. We then override that + // configuration in the `Tracing::Decision`, below. + config.trace_sampler.sample_rate = 1.0; // 100% Tracer tracer("fake_cluster", "test_host", config, cluster_manager_, *store_.rootScope(), thread_local_slot_allocator_); From 6a5954b9495dffe19cc48fc0a67e07a28f2d4990 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 27 Apr 2023 17:10:52 -0400 Subject: [PATCH 022/740] mobile: removing addVirtualCluster APIs (#27001) Risk Level: low Testing: n/a Docs Changes: inline Release Notes: inline for mobile Signed-off-by: Alyssa Wilk --- mobile/docs/root/api/starting_envoy.rst | 21 ----------- mobile/docs/root/intro/version_history.rst | 2 +- mobile/library/cc/engine_builder.cc | 37 ------------------- mobile/library/cc/engine_builder.h | 5 --- mobile/library/common/jni/jni_interface.cc | 28 ++++---------- .../io/envoyproxy/envoymobile/engine/BUILD | 1 - .../engine/EnvoyConfiguration.java | 25 ++++--------- .../envoymobile/engine/JniBridgeUtility.java | 28 -------------- .../envoymobile/engine/JniLibrary.java | 12 +++--- .../engine/VirtualClusterConfig.java | 21 ----------- .../impl/NativeCronetEngineBuilderImpl.java | 14 +++---- .../envoyproxy/envoymobile/EngineBuilder.kt | 29 --------------- .../library/objective-c/EnvoyConfiguration.h | 2 - .../library/objective-c/EnvoyConfiguration.mm | 5 --- mobile/library/swift/EngineBuilder.swift | 27 -------------- mobile/test/cc/unit/envoy_config_test.cc | 29 --------------- .../engine/EnvoyConfigurationTest.kt | 12 ------ .../envoymobile/EngineBuilderTest.kt | 10 ----- mobile/test/swift/EngineBuilderTests.swift | 22 ----------- 19 files changed, 27 insertions(+), 303 deletions(-) delete mode 100644 mobile/library/java/io/envoyproxy/envoymobile/engine/VirtualClusterConfig.java diff --git a/mobile/docs/root/api/starting_envoy.rst b/mobile/docs/root/api/starting_envoy.rst index c2bcdfd7a521..5ea1379891e7 100644 --- a/mobile/docs/root/api/starting_envoy.rst +++ b/mobile/docs/root/api/starting_envoy.rst @@ -225,27 +225,6 @@ This information is sent as metadata when flushing stats. // Swift builder.addAppId("com.mydomain.myapp) -~~~~~~~~~~~~~~~~~~~~~ -``addVirtualCluster`` -~~~~~~~~~~~~~~~~~~~~~ - -Add a virtual cluster config for Envoy Mobile's configuration. -The configuration is expected as a JSON object. -This functionality is used for stat segmentation. - -.. attention:: - - This API is non-ideal as it exposes lower-level internals of Envoy than desired by this project. - :issue:`#770 <770>` tracks enhancing this API. - -**Example**:: - - // Kotlin - builder.addVirtualCluster("{\"name\":\"vcluster\",\"headers\":[{\"name\":\":path\",\"exact_match\":\"/v1/vcluster\"}]}") - - // Swift - builder.addVirtualCluster("{\"name\":\"vcluster\",\"headers\":[{\"name\":\":path\",\"exact_match\":\"/v1/vcluster\"}]}") - ~~~~~~~~~~~~~~~~~~~~~~~~~ ``enableAdminInterface`` ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/mobile/docs/root/intro/version_history.rst b/mobile/docs/root/intro/version_history.rst index 0f3fbdf1772b..5d26891dc0de 100644 --- a/mobile/docs/root/intro/version_history.rst +++ b/mobile/docs/root/intro/version_history.rst @@ -17,7 +17,7 @@ Breaking changes: - all: enable HTTP/3 by default in Engine builders. - api: remove ``extendKeepaliveTimeout`` method from engine builders. - java: moved the Java builder to use the C++ builder's generated bootstrap, rather than doing YAML string manipulation (:issue: `#25392 <25392>`) -- api: move ``addVirtualClusters`` APIs taking concatenated cluster YAML to ``addVirtualCluster`` with one cluster config at a time (:issue: `#25297 <25297>`, :issue: `#25259 <25259>`, :issue: `#25457 <25457>`) +- api: remove ``addVirtualClusters`` APIs. - api: move ``dnsPreresolveHostnames`` APIs from taking concatenated cluster YAML to taking a list of String hostnames (:issue: `#25297 <25297>`, :issue: `#25259 <25259>`, :issue: `#25457 <25457>`) - api: added ``setRuntimeGuard`` APIs for all languages (:issue: `#25434 <25434>`) - api: added ``setRtdsLayer``, ``addAggregatedDiscoveryService``, ``setNodeId``, ``setNodeLocality`` APIs for all languages diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc index d7c4d847f83a..5bc4ba46a585 100644 --- a/mobile/library/cc/engine_builder.cc +++ b/mobile/library/cc/engine_builder.cc @@ -128,18 +128,6 @@ EngineBuilder::addH2ConnectionKeepaliveTimeoutSeconds(int h2_connection_keepaliv return *this; } -EngineBuilder& EngineBuilder::addVirtualCluster(std::string virtual_cluster) { - virtual_clusters_.push_back(std::move(virtual_cluster)); - return *this; -} - -EngineBuilder& EngineBuilder::addVirtualCluster(std::string name, - std::vector matchers) { - std::pair> matcher(name, std::move(matchers)); - virtual_cluster_data_.emplace_back(std::move(matcher)); - return *this; -} - EngineBuilder& EngineBuilder::addKeyValueStore(std::string name, KeyValueStoreSharedPtr key_value_store) { key_value_stores_[std::move(name)] = std::move(key_value_store); @@ -391,31 +379,6 @@ std::unique_ptr EngineBuilder::generate api_service->set_name("api"); api_service->set_include_attempt_count_in_response(true); api_service->add_domains("*"); - // Virtual clusters - for (auto& cluster : virtual_clusters_) { -#ifdef ENVOY_ENABLE_YAML - MessageUtil::loadFromYaml(cluster, *api_service->add_virtual_clusters(), - ProtobufMessage::getStrictValidationVisitor()); -#else - UNREFERENCED_PARAMETER(cluster); - IS_ENVOY_BUG("virtual clusters can not be added when YAML is compiled out."); -#endif - } - - for (auto& name_and_data : virtual_cluster_data_) { - auto* virtual_cluster = api_service->add_virtual_clusters(); - virtual_cluster->set_name(name_and_data.first); - for (auto& matcher : name_and_data.second) { - auto* match = virtual_cluster->add_headers(); - match->set_name(matcher.name); - if (matcher.type == MatcherData::EXACT) { - match->set_exact_match(matcher.value); - } else { - ASSERT(matcher.type == MatcherData::SAFE_REGEX); - match->mutable_safe_regex_match()->set_regex(matcher.value); - } - } - } for (auto& direct_response_in : direct_responses_) { auto* this_route = api_service->add_routes(); diff --git a/mobile/library/cc/engine_builder.h b/mobile/library/cc/engine_builder.h index 0ad68fe71616..51b5f92ac5ef 100644 --- a/mobile/library/cc/engine_builder.h +++ b/mobile/library/cc/engine_builder.h @@ -103,10 +103,7 @@ class EngineBuilder { EngineBuilder& addStatsFlushSeconds(int stats_flush_seconds); #endif EngineBuilder& addPlatformFilter(std::string name); - // TODO(alyssawilk) remove the legacy APIs and update docs once Lyft is moved over. - EngineBuilder& addVirtualCluster(std::string virtual_cluster); - EngineBuilder& addVirtualCluster(std::string name, std::vector matchers); EngineBuilder& setRuntimeGuard(std::string guard, bool value); // Add a direct response. For testing purposes only. @@ -195,8 +192,6 @@ class EngineBuilder { std::vector native_filter_chain_; std::vector dns_preresolve_hostnames_; - std::vector virtual_clusters_; - std::vector>> virtual_cluster_data_; std::vector direct_responses_; std::vector> runtime_guards_; diff --git a/mobile/library/common/jni/jni_interface.cc b/mobile/library/common/jni/jni_interface.cc index 80dc633bb7f8..5ac75a5f790c 100644 --- a/mobile/library/common/jni/jni_interface.cc +++ b/mobile/library/common/jni/jni_interface.cc @@ -1207,8 +1207,7 @@ void configureBuilder( jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host, jlong stats_flush_seconds, jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id, - jboolean trust_chain_verification, jobjectArray virtual_clusters_legacy, - jobjectArray virtual_clusters, jobjectArray filter_chain, jobjectArray stat_sinks, + jboolean trust_chain_verification, jobjectArray filter_chain, jobjectArray stat_sinks, jboolean enable_platform_certificates_validation, jboolean enable_skip_dns_lookup_for_proxied_requests, jobjectArray runtime_guards, jstring rtds_layer_name, jlong rtds_timeout_seconds, jstring ads_address, jlong ads_port, @@ -1260,16 +1259,6 @@ void configureBuilder( for (std::pair& filter : filters) { builder.addNativeFilter(filter.first, filter.second); } - std::vector clusters = javaObjectArrayToStringVector(env, virtual_clusters_legacy); - for (std::string& cluster : clusters) { - builder.addVirtualCluster(cluster); - } - std::string cluster_name; - std::vector matcher_data = - javaObjectArrayToMatcherData(env, virtual_clusters, cluster_name); - if (!cluster_name.empty()) { - builder.addVirtualCluster(cluster_name, matcher_data); - } std::vector sinks = javaObjectArrayToStringVector(env, stat_sinks); @@ -1320,8 +1309,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host, jlong stats_flush_seconds, jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id, - jboolean trust_chain_verification, jobjectArray virtual_clusters_legacy, - jobjectArray virtual_clusters, jobjectArray filter_chain, jobjectArray stat_sinks, + jboolean trust_chain_verification, jobjectArray filter_chain, jobjectArray stat_sinks, jboolean enable_platform_certificates_validation, jboolean enable_skip_dns_lookup_for_proxied_requests, jobjectArray runtime_guards, jstring rtds_layer_name, jlong rtds_timeout_seconds, jstring ads_address, jlong ads_port, @@ -1339,12 +1327,12 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr enable_happy_eyeballs, enable_interface_binding, h2_connection_keepalive_idle_interval_milliseconds, h2_connection_keepalive_timeout_seconds, max_connections_per_host, stats_flush_seconds, stream_idle_timeout_seconds, - per_try_idle_timeout_seconds, app_version, app_id, trust_chain_verification, - virtual_clusters_legacy, virtual_clusters, filter_chain, stat_sinks, - enable_platform_certificates_validation, enable_skip_dns_lookup_for_proxied_requests, - runtime_guards, rtds_layer_name, rtds_timeout_seconds, ads_address, ads_port, ads_token, - ads_token_lifetime, ads_root_certs, node_id, node_region, node_zone, node_sub_zone, - cds_resources_locator, cds_timeout_seconds, enable_cds, builder); + per_try_idle_timeout_seconds, app_version, app_id, trust_chain_verification, filter_chain, + stat_sinks, enable_platform_certificates_validation, + enable_skip_dns_lookup_for_proxied_requests, runtime_guards, rtds_layer_name, + rtds_timeout_seconds, ads_address, ads_port, ads_token, ads_token_lifetime, ads_root_certs, + node_id, node_region, node_zone, node_sub_zone, cds_resources_locator, cds_timeout_seconds, + enable_cds, builder); return reinterpret_cast(builder.generateBootstrap().release()); } diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD b/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD index 60e89e41cc84..c23318d4c075 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD @@ -46,7 +46,6 @@ java_library( "JvmFilterFactoryContext.java", "JvmKeyValueStoreContext.java", "JvmStringAccessorContext.java", - "VirtualClusterConfig.java", "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib_srcs", ], visibility = ["//visibility:public"], diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java index e89e08835903..d3fae5440e1a 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java @@ -10,7 +10,6 @@ import java.lang.StringBuilder; import javax.annotation.Nullable; -import io.envoyproxy.envoymobile.engine.VirtualClusterConfig; import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterFactory; import io.envoyproxy.envoymobile.engine.types.EnvoyStringAccessor; import io.envoyproxy.envoymobile.engine.types.EnvoyKeyValueStore; @@ -57,8 +56,6 @@ public enum TrustChainVerification { public final String appVersion; public final String appId; public final TrustChainVerification trustChainVerification; - public final List legacyVirtualClusters; - public final List virtualClusters; public final List nativeFilterChain; public final Map stringAccessors; public final Map keyValueStores; @@ -134,10 +131,6 @@ public enum TrustChainVerification { * Client. * @param trustChainVerification whether to mute TLS Cert verification - * for tests. - * @param legacyVirtualClusters the JSON list of virtual cluster - * configs. - * @param virtualClusters the structured list of virtual cluster - * configs. * @param nativeFilterChain the configuration for native filters. * @param httpPlatformFilterFactories the configuration for platform filters. * @param stringAccessors platform string accessors to register. @@ -174,8 +167,8 @@ public EnvoyConfiguration( int h2ConnectionKeepaliveIdleIntervalMilliseconds, int h2ConnectionKeepaliveTimeoutSeconds, int maxConnectionsPerHost, int statsFlushSeconds, int streamIdleTimeoutSeconds, int perTryIdleTimeoutSeconds, String appVersion, String appId, - TrustChainVerification trustChainVerification, List legacyVirtualClusters, - List virtualClusters, List nativeFilterChain, + TrustChainVerification trustChainVerification, + List nativeFilterChain, List httpPlatformFilterFactories, Map stringAccessors, Map keyValueStores, List statSinks, @@ -214,8 +207,6 @@ public EnvoyConfiguration( this.appVersion = appVersion; this.appId = appId; this.trustChainVerification = trustChainVerification; - this.legacyVirtualClusters = legacyVirtualClusters; - this.virtualClusters = virtualClusters; int index = 0; // Insert in this order to preserve prior ordering constraints. for (EnvoyHTTPFilterFactory filterFactory : httpPlatformFilterFactories) { @@ -262,11 +253,9 @@ public long createBootstrap() { Collections.reverse(reverseFilterChain); byte[][] filter_chain = JniBridgeUtility.toJniBytes(reverseFilterChain); - byte[][] clusters_legacy = JniBridgeUtility.stringsToJniBytes(legacyVirtualClusters); byte[][] stats_sinks = JniBridgeUtility.stringsToJniBytes(statSinks); byte[][] dns_preresolve = JniBridgeUtility.stringsToJniBytes(dnsPreresolveHostnames); byte[][] runtime_guards = JniBridgeUtility.mapToJniBytes(runtimeGuards); - byte[][] clusters = JniBridgeUtility.clusterConfigToJniBytes(virtualClusters); return JniLibrary.createBootstrap( grpcStatsDomain, adminInterfaceEnabled, connectTimeoutSeconds, dnsRefreshSeconds, @@ -276,11 +265,11 @@ public long createBootstrap() { enableSocketTagging, enableHappyEyeballs, enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds, h2ConnectionKeepaliveTimeoutSeconds, maxConnectionsPerHost, statsFlushSeconds, streamIdleTimeoutSeconds, - perTryIdleTimeoutSeconds, appVersion, appId, enforceTrustChainVerification, clusters_legacy, - clusters, filter_chain, stats_sinks, enablePlatformCertificatesValidation, - enableSkipDNSLookupForProxiedRequests, runtime_guards, rtdsLayerName, rtdsTimeoutSeconds, - adsAddress, adsPort, adsToken, adsTokenLifetime, adsRootCerts, nodeId, nodeRegion, nodeZone, - nodeSubZone, cdsResourcesLocator, cdsTimeoutSeconds, enableCds); + perTryIdleTimeoutSeconds, appVersion, appId, enforceTrustChainVerification, filter_chain, + stats_sinks, enablePlatformCertificatesValidation, enableSkipDNSLookupForProxiedRequests, + runtime_guards, rtdsLayerName, rtdsTimeoutSeconds, adsAddress, adsPort, adsToken, + adsTokenLifetime, adsRootCerts, nodeId, nodeRegion, nodeZone, nodeSubZone, + cdsResourcesLocator, cdsTimeoutSeconds, enableCds); } static class ConfigurationException extends RuntimeException { diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniBridgeUtility.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniBridgeUtility.java index 359ea4ca6a0b..6adb49b36b0a 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniBridgeUtility.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniBridgeUtility.java @@ -49,34 +49,6 @@ public static byte[][] stringsToJniBytes(List stringList) { return convertedBytes.toArray(new byte[0][0]); } - public static byte[][] clusterConfigToJniBytes(List configList) { - int size = 0; - for (VirtualClusterConfig config : configList) { - size += 1 + config.matches.size(); - } - final List convertedBytes = new ArrayList(size); - for (VirtualClusterConfig config : configList) { - convertedBytes.add(config.name.getBytes(StandardCharsets.UTF_8)); - for (HeaderMatchConfig match : config.matches) { - convertedBytes.add(match.name.getBytes(StandardCharsets.UTF_8)); - ByteBuffer type = ByteBuffer.allocate(4); - int int_type = 0; - switch (match.type) { - case EXACT: - int_type = 0; - break; - case SAFE_REGEX: - int_type = 1; - break; - } - type.putInt(int_type); - convertedBytes.add(type.array()); - convertedBytes.add(match.value.getBytes(StandardCharsets.UTF_8)); - } - } - return convertedBytes.toArray(new byte[0][0]); - } - public static byte[][] mapToJniBytes(Map stringMap) { final List convertedBytes = new ArrayList(stringMap.size() * 2); for (Map.Entry entry : stringMap.entrySet()) { diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java index 2cc46ec6809f..f016241b8558 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java @@ -306,10 +306,10 @@ public static native long createBootstrap( long h2ConnectionKeepaliveIdleIntervalMilliseconds, long h2ConnectionKeepaliveTimeoutSeconds, long maxConnectionsPerHost, long statsFlushSeconds, long streamIdleTimeoutSeconds, long perTryIdleTimeoutSeconds, String appVersion, String appId, - boolean trustChainVerification, byte[][] virtualClustersLegacy, byte[][] virtualClusters, - byte[][] filterChain, byte[][] statSinks, boolean enablePlatformCertificatesValidation, - boolean enableSkipDNSLookupForProxiedRequests, byte[][] runtimeGuards, String rtdsLayerName, - long rtdsTimeoutSeconds, String adsAddress, long adsPort, String adsToken, - long adsTokenLifetime, String adsRootCerts, String nodeId, String nodeRegion, String nodeZone, - String nodeSubZone, String cdsResourcesLocator, long cdsTimeoutSeconds, boolean enableCds); + boolean trustChainVerification, byte[][] filterChain, byte[][] statSinks, + boolean enablePlatformCertificatesValidation, boolean enableSkipDNSLookupForProxiedRequests, + byte[][] runtimeGuards, String rtdsLayerName, long rtdsTimeoutSeconds, String adsAddress, + long adsPort, String adsToken, long adsTokenLifetime, String adsRootCerts, String nodeId, + String nodeRegion, String nodeZone, String nodeSubZone, String cdsResourcesLocator, + long cdsTimeoutSeconds, boolean enableCds); } diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/VirtualClusterConfig.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/VirtualClusterConfig.java deleted file mode 100644 index 38432ad4ec91..000000000000 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/VirtualClusterConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -package io.envoyproxy.envoymobile.engine; - -import io.envoyproxy.envoymobile.engine.HeaderMatchConfig; -import java.util.List; - -/* Datatype used by the EnvoyConfiguration to configure virtual clusters. - * - * The fields map to their respective fields in - * https://github.com/envoyproxy/envoy/blob/main/api/envoy/config/route/v3/route_components.proto - * */ -public class VirtualClusterConfig { - // The name of the virtual cluster. - public final String name; - // The various rules to match requests to the virtual cluster. - public final List matches; - - public VirtualClusterConfig(String name, List matches) { - this.name = name; - this.matches = matches; - } -} diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderImpl.java index 6280e0683c34..32dfba0c769d 100644 --- a/mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderImpl.java @@ -11,7 +11,6 @@ import io.envoyproxy.envoymobile.engine.EnvoyConfiguration.TrustChainVerification; import io.envoyproxy.envoymobile.engine.EnvoyEngine; import io.envoyproxy.envoymobile.engine.EnvoyNativeFilterConfig; -import io.envoyproxy.envoymobile.engine.VirtualClusterConfig; import io.envoyproxy.envoymobile.engine.types.EnvoyEventTracker; import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterFactory; import io.envoyproxy.envoymobile.engine.types.EnvoyLogger; @@ -64,8 +63,6 @@ public class NativeCronetEngineBuilderImpl extends CronetEngineBuilderImpl { private String mAppVersion = "unspecified"; private String mAppId = "unspecified"; private TrustChainVerification mTrustChainVerification = VERIFY_TRUST_CHAIN; - private List mVirtualClusters = Collections.emptyList(); - private List mVirtualClusterConfig = Collections.emptyList(); private boolean mEnablePlatformCertificatesValidation = true; private String mRtdsLayerName = ""; private int mRtdsTimeoutSeconds = 0; @@ -146,11 +143,10 @@ mEnableGzipDecompression, brotliEnabled(), mEnableSocketTag, mEnableHappyEyeball mEnableInterfaceBinding, mH2ConnectionKeepaliveIdleIntervalMilliseconds, mH2ConnectionKeepaliveTimeoutSeconds, mMaxConnectionsPerHost, mStatsFlushSeconds, mStreamIdleTimeoutSeconds, mPerTryIdleTimeoutSeconds, mAppVersion, mAppId, - mTrustChainVerification, mVirtualClusters, mVirtualClusterConfig, nativeFilterChain, - platformFilterChain, stringAccessors, keyValueStores, statSinks, runtimeGuards, - mEnableSkipDNSLookupForProxiedRequests, mEnablePlatformCertificatesValidation, - mRtdsLayerName, mRtdsTimeoutSeconds, mAdsAddress, mAdsPort, mAdsToken, mAdsTokenLifetime, - mAdsRootCerts, mNodeId, mNodeRegion, mNodeZone, mNodeSubZone, mCdsResourcesLocator, - mCdsTimeoutSeconds, mEnableCds); + mTrustChainVerification, nativeFilterChain, platformFilterChain, stringAccessors, + keyValueStores, statSinks, runtimeGuards, mEnableSkipDNSLookupForProxiedRequests, + mEnablePlatformCertificatesValidation, mRtdsLayerName, mRtdsTimeoutSeconds, mAdsAddress, + mAdsPort, mAdsToken, mAdsTokenLifetime, mAdsRootCerts, mNodeId, mNodeRegion, mNodeZone, + mNodeSubZone, mCdsResourcesLocator, mCdsTimeoutSeconds, mEnableCds); } } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt index 912f722c0f9c..fa92aecea67c 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt @@ -5,7 +5,6 @@ import io.envoyproxy.envoymobile.engine.EnvoyConfiguration.TrustChainVerificatio import io.envoyproxy.envoymobile.engine.EnvoyEngine import io.envoyproxy.envoymobile.engine.EnvoyEngineImpl import io.envoyproxy.envoymobile.engine.EnvoyNativeFilterConfig -import io.envoyproxy.envoymobile.engine.VirtualClusterConfig import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterFactory import io.envoyproxy.envoymobile.engine.types.EnvoyKeyValueStore import io.envoyproxy.envoymobile.engine.types.EnvoyStringAccessor @@ -71,8 +70,6 @@ open class EngineBuilder( private var appVersion = "unspecified" private var appId = "unspecified" private var trustChainVerification = TrustChainVerification.VERIFY_TRUST_CHAIN - private var virtualClustersLegacy = mutableListOf() - private var virtualClusters = mutableListOf() private var platformFilterChain = mutableListOf() private var nativeFilterChain = mutableListOf() private var stringAccessors = mutableMapOf() @@ -552,30 +549,6 @@ open class EngineBuilder( return this } - /** - * Add virtual cluster configuration. - * - * @param cluster the JSON configuration string for a virtual cluster. - * - * @return this builder. - */ - fun addVirtualCluster(cluster: String): EngineBuilder { - this.virtualClustersLegacy.add(cluster) - return this - } - - /** - * Add virtual cluster configurations. - * - * @param configs structured configurations of virtual clusters. - * - * @return this builder. - */ - fun addVirtualClusters(configs: List): EngineBuilder { - this.virtualClusters + configs; - return this - } - /** * Sets the node.id field in the Bootstrap configuration. * @@ -722,8 +695,6 @@ open class EngineBuilder( appVersion, appId, trustChainVerification, - virtualClustersLegacy, - virtualClusters, nativeFilterChain, platformFilterChain, stringAccessors, diff --git a/mobile/library/objective-c/EnvoyConfiguration.h b/mobile/library/objective-c/EnvoyConfiguration.h index d5c785948c8e..6e0a057804e7 100644 --- a/mobile/library/objective-c/EnvoyConfiguration.h +++ b/mobile/library/objective-c/EnvoyConfiguration.h @@ -39,7 +39,6 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic, assign) UInt32 perTryIdleTimeoutSeconds; @property (nonatomic, strong) NSString *appVersion; @property (nonatomic, strong) NSString *appId; -@property (nonatomic, strong) NSArray *virtualClusters; @property (nonatomic, strong) NSDictionary *runtimeGuards; @property (nonatomic, strong) NSArray *typedDirectResponses; @property (nonatomic, strong) NSArray *nativeFilterChain; @@ -95,7 +94,6 @@ NS_ASSUME_NONNULL_BEGIN perTryIdleTimeoutSeconds:(UInt32)perTryIdleTimeoutSeconds appVersion:(NSString *)appVersion appId:(NSString *)appId - virtualClusters:(NSArray *)virtualClusters runtimeGuards: (NSDictionary *)runtimeGuards typedDirectResponses: diff --git a/mobile/library/objective-c/EnvoyConfiguration.mm b/mobile/library/objective-c/EnvoyConfiguration.mm index 2a351e7168a9..2a3b095ebe21 100644 --- a/mobile/library/objective-c/EnvoyConfiguration.mm +++ b/mobile/library/objective-c/EnvoyConfiguration.mm @@ -96,7 +96,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled perTryIdleTimeoutSeconds:(UInt32)perTryIdleTimeoutSeconds appVersion:(NSString *)appVersion appId:(NSString *)appId - virtualClusters:(NSArray *)virtualClusters runtimeGuards: (NSDictionary *)runtimeGuards typedDirectResponses: @@ -162,7 +161,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled self.perTryIdleTimeoutSeconds = perTryIdleTimeoutSeconds; self.appVersion = appVersion; self.appId = appId; - self.virtualClusters = virtualClusters; self.runtimeGuards = runtimeGuards; self.typedDirectResponses = typedDirectResponses; self.nativeFilterChain = nativeFilterChain; @@ -249,9 +247,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled builder.setAppVersion([self.appVersion toCXXString]); builder.setAppId([self.appId toCXXString]); builder.setDeviceOs("iOS"); - for (NSString *cluster in self.virtualClusters) { - builder.addVirtualCluster([cluster toCXXString]); - } builder.enablePlatformCertificatesValidation(self.enablePlatformCertificateValidation); builder.enableDnsCache(self.enableDNSCache, self.dnsCacheSaveIntervalSeconds); diff --git a/mobile/library/swift/EngineBuilder.swift b/mobile/library/swift/EngineBuilder.swift index 489b29aaf34b..68ad83ca3535 100644 --- a/mobile/library/swift/EngineBuilder.swift +++ b/mobile/library/swift/EngineBuilder.swift @@ -51,7 +51,6 @@ open class EngineBuilder: NSObject { private var perTryIdleTimeoutSeconds: UInt32 = 15 private var appVersion: String = "unspecified" private var appId: String = "unspecified" - private var virtualClusters: [String] = [] private var onEngineRunning: (() -> Void)? private var logger: ((String) -> Void)? private var eventTracker: (([String: String]) -> Void)? @@ -546,27 +545,6 @@ open class EngineBuilder: NSObject { return self } - /// Add virtual cluster configuration. - /// - /// - parameter virtualCluster: The JSON configuration string for a virtual cluster. - /// - /// - returns: This builder. - @discardableResult - public func addVirtualCluster(_ virtualCluster: String) -> Self { - self.virtualClusters.append(virtualCluster) - return self - } - - /// Add virtual cluster configurations. - /// - /// - parameter virtualClusters: The JSON configuration strings for virtual clusters. - /// - /// - returns: This builder. - @discardableResult - public func addVirtualClusters(_ virtualClusters: [String]) -> Self { - self.virtualClusters.append(contentsOf: virtualClusters) - return self - } #if ENVOY_GOOGLE_GRPC /// Sets the node.id field in the Bootstrap configuration. @@ -768,7 +746,6 @@ open class EngineBuilder: NSObject { perTryIdleTimeoutSeconds: self.perTryIdleTimeoutSeconds, appVersion: self.appVersion, appId: self.appId, - virtualClusters: self.virtualClusters, runtimeGuards: self.runtimeGuards.mapValues({ "\($0)" }), typedDirectResponses: self.directResponses.map({ $0.toObjC() }), nativeFilterChain: self.nativeFilterChain, @@ -805,7 +782,6 @@ open class EngineBuilder: NSObject { } } -// swiftlint:disable cyclomatic_complexity #if canImport(EnvoyCxxSwiftInterop) private extension EngineBuilder { func generateBootstrap() -> Bootstrap { @@ -850,9 +826,6 @@ private extension EngineBuilder { cxxBuilder.setAppVersion(self.appVersion.toCXX()) cxxBuilder.setAppId(self.appId.toCXX()) cxxBuilder.setDeviceOs("iOS".toCXX()) - for cluster in self.virtualClusters { - cxxBuilder.addVirtualCluster(cluster.toCXX()) - } for (runtimeGuard, value) in self.runtimeGuards { cxxBuilder.setRuntimeGuard(runtimeGuard.toCXX(), value) diff --git a/mobile/test/cc/unit/envoy_config_test.cc b/mobile/test/cc/unit/envoy_config_test.cc index ceed8336c9e5..f51fa2102e5b 100644 --- a/mobile/test/cc/unit/envoy_config_test.cc +++ b/mobile/test/cc/unit/envoy_config_test.cc @@ -446,35 +446,6 @@ TEST(TestConfig, DISABLED_StringAccessors) { release_envoy_data(data); } -TEST(TestConfig, AddVirtualClusterLegacy) { - EngineBuilder engine_builder; - - engine_builder.addVirtualCluster( - "{headers: [{name: ':method', string_match: {exact: POST}}], name: cluster1}"); - std::unique_ptr bootstrap = engine_builder.generateBootstrap(); - EXPECT_THAT(bootstrap->ShortDebugString(), HasSubstr("cluster1")); - - engine_builder.addVirtualCluster( - "{headers: [{name: ':method', string_match: {exact: GET}}], name: cluster2}"); - bootstrap = engine_builder.generateBootstrap(); - EXPECT_THAT(bootstrap->ShortDebugString(), HasSubstr("cluster2")); -} - -TEST(TestConfig, AddVirtualCluster) { - EngineBuilder engine_builder; - - std::vector matchers = {{":method", MatcherData::EXACT, "POST"}, - {":method", MatcherData::SAFE_REGEX, ".*E.*"}}; - engine_builder.addVirtualCluster("cluster1", matchers); - std::unique_ptr bootstrap = engine_builder.generateBootstrap(); - EXPECT_THAT(bootstrap->ShortDebugString(), HasSubstr("cluster1")); - - engine_builder.addVirtualCluster("cluster2", matchers); - bootstrap = engine_builder.generateBootstrap(); - EXPECT_THAT(bootstrap->ShortDebugString(), HasSubstr("cluster1")); - EXPECT_THAT(bootstrap->ShortDebugString(), HasSubstr("cluster2")); -} - #ifdef ENVOY_GOOGLE_GRPC TEST(TestConfig, SetNodeId) { EngineBuilder engine_builder; diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt index 38d5a2720787..53ca21f4fd67 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt @@ -4,7 +4,6 @@ import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilter import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterFactory import io.envoyproxy.envoymobile.engine.EnvoyConfiguration.TrustChainVerification import io.envoyproxy.envoymobile.engine.JniLibrary -import io.envoyproxy.envoymobile.engine.VirtualClusterConfig import io.envoyproxy.envoymobile.engine.HeaderMatchConfig import io.envoyproxy.envoymobile.engine.HeaderMatchConfig.Type import io.envoyproxy.envoymobile.engine.types.EnvoyStreamIntel @@ -96,8 +95,6 @@ class EnvoyConfigurationTest { appVersion: String = "v1.2.3", appId: String = "com.example.myapp", trustChainVerification: TrustChainVerification = TrustChainVerification.VERIFY_TRUST_CHAIN, - legacyVirtualClusters: MutableList = mutableListOf("{name: test1}", "{name: test2}"), - virtualClusters: List = emptyList(), filterChain: MutableList = mutableListOf(EnvoyNativeFilterConfig("buffer_filter_1", "{'@type': 'type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer'}"), EnvoyNativeFilterConfig("buffer_filter_2", "{'@type': 'type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer'}")), platformFilterFactories: MutableList = mutableListOf(TestEnvoyHTTPFilterFactory("name1"), TestEnvoyHTTPFilterFactory("name2")), runtimeGuards: Map = emptyMap(), @@ -148,8 +145,6 @@ class EnvoyConfigurationTest { appVersion, appId, trustChainVerification, - legacyVirtualClusters, - virtualClusters, filterChain, platformFilterFactories, emptyMap(), @@ -221,8 +216,6 @@ class EnvoyConfigurationTest { assertThat(resolvedTemplate).contains("app_version: v1.2.3") assertThat(resolvedTemplate).contains("app_id: com.example.myapp") - assertThat(resolvedTemplate).matches(Pattern.compile(".*virtual_clusters.*name: test1.*name: test2.*", Pattern.DOTALL)); - // Stats assertThat(resolvedTemplate).contains("stats_flush_interval: 567s") assertThat(resolvedTemplate).contains("stats.example.com"); @@ -271,7 +264,6 @@ class EnvoyConfigurationTest { enableSkipDNSLookupForProxiedRequests = true, enablePlatformCertificatesValidation = true, dnsPreresolveHostnames = mutableListOf(), - legacyVirtualClusters = mutableListOf(), filterChain = mutableListOf(), runtimeGuards = mapOf("test_feature_false" to true), statSinks = mutableListOf("{ name: envoy.stat_sinks.statsd, typed_config: { '@type': type.googleapis.com/envoy.config.metrics.v3.StatsdSink, address: { socket_address: { address: 127.0.0.1, port_value: 123 } } } }"), @@ -330,16 +322,12 @@ class EnvoyConfigurationTest { JniLibrary.loadTestLibrary() val envoyConfiguration = buildTestEnvoyConfiguration( runtimeGuards = mapOf("test_feature_false" to true, "test_feature_true" to false), - virtualClusters = listOf(VirtualClusterConfig("cluster1", listOf(HeaderMatchConfig(":method", Type.EXACT, "POST"), - HeaderMatchConfig(":authority", Type.SAFE_REGEX, "foo")))), ) val resolvedTemplate = TestJni.createYaml(envoyConfiguration) assertThat(resolvedTemplate).contains("test_feature_false"); assertThat(resolvedTemplate).contains("test_feature_true"); - assertThat(resolvedTemplate).matches(Pattern.compile(".*name: :method\n *exact_match: POST.*", Pattern.DOTALL)); - assertThat(resolvedTemplate).matches(Pattern.compile(".*name: :authority\n *safe_regex_match:\n *regex: foo.*", Pattern.DOTALL)); } @Test diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt index 53b99da3a971..6da0ac4c1796 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt @@ -204,16 +204,6 @@ class EngineBuilderTest { assertThat(engine.envoyConfiguration.appId).isEqualTo("com.envoymobile.android") } - @Test - fun `specifying virtual clusters overrides default`() { - engineBuilder = EngineBuilder(Standard()) - engineBuilder.addEngineType { envoyEngine } - engineBuilder.addVirtualCluster("[test]") - - val engine = engineBuilder.build() as EngineImpl - assertThat(engine.envoyConfiguration.legacyVirtualClusters.size).isEqualTo(1) - } - @Test fun `specifying native filters overrides default`() { engineBuilder = EngineBuilder(Standard()) diff --git a/mobile/test/swift/EngineBuilderTests.swift b/mobile/test/swift/EngineBuilderTests.swift index 39222f37d084..989def5bcc37 100644 --- a/mobile/test/swift/EngineBuilderTests.swift +++ b/mobile/test/swift/EngineBuilderTests.swift @@ -363,28 +363,6 @@ final class EngineBuilderTests: XCTestCase { self.waitForExpectations(timeout: 0.01) } - func testAddingVirtualClustersAddsToConfigurationWhenRunningEnvoy() { - let expectation = self.expectation(description: "Run called with expected data") - MockEnvoyEngine.onRunWithConfig = { config, _ in - XCTAssertEqual([ - """ - {"name":"test","headers":[{"name":":authority","string_match":{"exact":"envoymobile.io"}}]} - """, - ], config.virtualClusters) - expectation.fulfill() - } - - _ = EngineBuilder() - .addEngineType(MockEnvoyEngine.self) - .addVirtualClusters([ - """ - {"name":"test","headers":[{"name":":authority","string_match":{"exact":"envoymobile.io"}}]} - """, - ]) - .build() - self.waitForExpectations(timeout: 0.01) - } - func testAddingNativeFiltersToConfigurationWhenRunningEnvoy() { let expectation = self.expectation(description: "Run called with expected data") MockEnvoyEngine.onRunWithConfig = { config, _ in From 3dbb2bd6cac36a4e9bbab67733634da5f46ec19b Mon Sep 17 00:00:00 2001 From: danzh Date: Thu, 27 Apr 2023 17:22:22 -0400 Subject: [PATCH 023/740] mobile: skip SNI matching for empty hostname (#27028) stop matching empty hostname to SAN list in the cert. Risk Level: Low Testing: New unit tests Docs Changes: N/A Release Notes: N/A Platform Specific Features: mobile-only Signed-off-by: Dan Zhang --- .../platform_bridge_cert_validator.cc | 8 +++-- .../platform_bridge_cert_validator_test.cc | 34 ++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc index 88dd385eddad..0aa61cc41f5e 100644 --- a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc +++ b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc @@ -78,7 +78,7 @@ ValidationResults PlatformBridgeCertValidator::doVerifyCertChain( if (transport_socket_options != nullptr && !transport_socket_options->verifySubjectAltNameListOverride().empty()) { subject_alt_names = transport_socket_options->verifySubjectAltNameListOverride(); - } else { + } else if (!hostname.empty()) { subject_alt_names = {std::string(hostname)}; } @@ -115,8 +115,10 @@ void PlatformBridgeCertValidator::verifyCertChainByPlatform( } absl::string_view error_details; - // Verify that host name matches leaf cert. - success = DefaultCertValidator::verifySubjectAltName(leaf_cert.get(), subject_alt_names); + if (!subject_alt_names.empty()) { + // Verify that host name matches leaf cert. + success = DefaultCertValidator::verifySubjectAltName(leaf_cert.get(), subject_alt_names); + } if (!success) { error_details = "PlatformBridgeCertValidator_verifySubjectAltName failed: SNI mismatch."; ENVOY_LOG(debug, error_details); diff --git a/mobile/test/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator_test.cc b/mobile/test/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator_test.cc index af8cb85917cc..41d559fdcecb 100644 --- a/mobile/test/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator_test.cc +++ b/mobile/test/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator_test.cc @@ -224,7 +224,7 @@ TEST_P(PlatformBridgeCertValidatorTest, ValidCertificate) { EXPECT_FALSE(waitForDispatcherToExit()); } -TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateEmptySocketOptions) { +TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateEmptySanOverrides) { initializeConfig(); PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); @@ -256,6 +256,38 @@ TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateEmptySocketOptions) { EXPECT_FALSE(waitForDispatcherToExit()); } +TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateEmptyHostNoOverrides) { + initializeConfig(); + PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); + + std::string hostname = ""; + bssl::UniquePtr cert_chain = readCertChainFromFile(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns2_cert.pem")); + envoy_cert_validation_result result = {ENVOY_SUCCESS, 0, NULL}; + EXPECT_CALL(*mock_validator_, validate(_, _, _)).WillOnce(Return(result)); + EXPECT_CALL(*mock_validator_, cleanup()); + auto& callback_ref = *callback_; + EXPECT_CALL(callback_ref, dispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); + + // Set up transport socket options with an empty SAN list. + std::vector subject_alt_names; + transport_socket_options_ = + std::make_shared("", std::move(subject_alt_names)); + + ValidationResults results = + validator.doVerifyCertChain(*cert_chain, std::move(callback_), transport_socket_options_, + *ssl_ctx_, validation_context_, is_server_, hostname); + EXPECT_EQ(ValidationResults::ValidationStatus::Pending, results.status); + + EXPECT_CALL(callback_ref, + onCertValidationResult(true, Envoy::Ssl::ClientValidationStatus::Validated, "", 46)) + .WillOnce(Invoke([this]() { + EXPECT_EQ(main_thread_id_, std::this_thread::get_id()); + dispatcher_->exit(); + })); + EXPECT_FALSE(waitForDispatcherToExit()); +} + TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateButInvalidSni) { initializeConfig(); PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); From 330d001b5ad85a0a82fce10946ae30df53cda478 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Thu, 27 Apr 2023 20:13:59 -0400 Subject: [PATCH 024/740] UHV: Transform H/2 extended CONNECT to H/1 upgrade (#26526) * UHV: Transform H/2 extended CONNECT to H/1 upgrade Additionally: * split header validator interface for server and client validation as the behavior is different * Plumb UHV into test client and fake upstream Signed-off-by: Yan Avlasov --- envoy/http/header_validator.h | 111 ++++++-- envoy/upstream/upstream.h | 2 +- source/common/http/codec_client.cc | 28 +- source/common/http/codec_client.h | 5 +- source/common/http/conn_manager_impl.cc | 17 +- source/common/http/http2/codec_impl.cc | 16 ++ .../common/quic/envoy_quic_client_stream.cc | 11 + .../common/quic/envoy_quic_server_stream.cc | 7 +- source/common/upstream/upstream_impl.cc | 4 +- source/common/upstream/upstream_impl.h | 2 +- .../network/http_connection_manager/config.h | 6 +- .../header_validators/envoy_default/BUILD | 2 + .../envoy_default/header_validator.cc | 6 +- .../envoy_default/header_validator.h | 11 +- .../envoy_default/header_validator_factory.cc | 22 +- .../envoy_default/header_validator_factory.h | 9 +- .../envoy_default/http1_header_validator.cc | 24 +- .../envoy_default/http1_header_validator.h | 126 +++++++-- .../envoy_default/http2_header_validator.cc | 114 ++++++-- .../envoy_default/http2_header_validator.h | 99 ++++++- .../envoy_default/path_normalizer.h | 2 +- test/common/http/BUILD | 1 + test/common/http/codec_client_test.cc | 162 +++++++++++ test/common/http/conn_manager_impl_test_2.cc | 28 +- .../http/conn_manager_impl_test_base.cc | 24 +- .../common/http/conn_manager_impl_test_base.h | 10 +- test/common/http/http1/codec_impl_test.cc | 6 +- test/common/http/http2/codec_impl_test.cc | 16 +- .../compressor_integration_tests.cc | 5 - .../http_connection_manager/config_test.cc | 4 +- .../header_validator_factory_test.cc | 36 ++- .../envoy_default/header_validator_test.cc | 44 +-- .../http1_header_validator_test.cc | 123 +++++++-- .../http2_header_validator_test.cc | 256 +++++++++++++----- .../http_common_validation_test.cc | 4 +- test/extensions/upstreams/http/config_test.cc | 38 +-- test/integration/BUILD | 3 + test/integration/fake_upstream.cc | 52 +++- test/integration/fake_upstream.h | 6 + test/integration/http_integration.cc | 10 + .../tcp_tunneling_integration_test.cc | 10 - test/integration/utility.cc | 19 ++ test/integration/utility.h | 2 + .../integration/websocket_integration_test.cc | 30 -- test/mocks/http/header_validator.h | 28 +- test/mocks/upstream/cluster_info.cc | 18 ++ test/mocks/upstream/cluster_info.h | 4 +- 47 files changed, 1193 insertions(+), 370 deletions(-) diff --git a/envoy/http/header_validator.h b/envoy/http/header_validator.h index fa0f9e620fbd..c89b7e2a3c59 100644 --- a/envoy/http/header_validator.h +++ b/envoy/http/header_validator.h @@ -12,11 +12,12 @@ namespace Envoy { namespace Http { /** - * Interface for header validators. + * Common interface for server and client header validators. + * TODO(yanavlasov): rename interfaces in the next PR to `HeaderValidator` */ -class HeaderValidator { +class HeaderValidatorBase { public: - virtual ~HeaderValidator() = default; + virtual ~HeaderValidatorBase() = default; // A class that holds either success condition or an error condition with tuple of // action and error details. @@ -45,6 +46,7 @@ class HeaderValidator { enum class RejectOrRedirectAction { Accept, Reject, Redirect }; using RejectResult = Result; using RejectOrRedirectResult = Result; + using TransformationResult = RejectResult; /** * Validate the entire request header map. @@ -54,6 +56,36 @@ class HeaderValidator { using ValidationResult = RejectResult; virtual ValidationResult validateRequestHeaders(const RequestHeaderMap& header_map) PURE; + /** + * Validate the entire response header map. + * Returning the Reject value causes the HTTP request to be rejected with the 502 status, + * and the gRPC request with the UNAVAILABLE (14) error code. + */ + virtual ValidationResult validateResponseHeaders(const ResponseHeaderMap& header_map) PURE; + + /** + * Validate the entire request trailer map. + * Returning the Reject value causes the HTTP request to be rejected with the 502 status, + * and the gRPC request with the UNAVAILABLE (14) error code. + * If response headers have already been sent the request is reset. + */ + virtual ValidationResult validateRequestTrailers(const RequestTrailerMap& trailer_map) PURE; + + /** + * Validate the entire response trailer map. + * Returning the Reject value causes the HTTP request to be reset. + */ + virtual ValidationResult validateResponseTrailers(const ResponseTrailerMap& trailer_map) PURE; +}; + +/** + * Interface for server header validators. + * TODO(yanavlasov): rename interfaces in the next PR to `ServerHeaderValidator` + */ +class HeaderValidator : public HeaderValidatorBase { +public: + ~HeaderValidator() override = default; + /** * Transform the entire request header map. * This method transforms the header map, for example by normalizing URI path, before processing @@ -63,41 +95,76 @@ class HeaderValidator { * value causes the HTTP request to be redirected to the :path presudo header in the request map. * The gRPC request will still be rejected with the INTERNAL (13) error code. */ - using HeadersTransformationResult = RejectOrRedirectResult; - virtual HeadersTransformationResult transformRequestHeaders(RequestHeaderMap& header_map) PURE; + using RequestHeadersTransformationResult = RejectOrRedirectResult; + virtual RequestHeadersTransformationResult + transformRequestHeaders(RequestHeaderMap& header_map) PURE; /** - * Validate the entire response header map. + * Transform the entire request trailer map. * Returning the Reject value causes the HTTP request to be rejected with the 502 status, * and the gRPC request with the UNAVAILABLE (14) error code. + * If response headers have already been sent the request is reset. */ - virtual ValidationResult validateResponseHeaders(const ResponseHeaderMap& header_map) PURE; + virtual TransformationResult transformRequestTrailers(RequestTrailerMap& header_map) PURE; /** - * Validate the entire request trailer map. + * Transform the entire response header map. + * HTTP/2 and HTTP/3 server header validator may transform the HTTP/1 upgrade response + * to HTTP/2 extended CONNECT response, iff it transformed extended CONNECT to upgrade request + * during request validation. * Returning the Reject value causes the HTTP request to be rejected with the 502 status, * and the gRPC request with the UNAVAILABLE (14) error code. - * If response headers have already been sent the request is reset. */ - virtual ValidationResult validateRequestTrailers(const RequestTrailerMap& trailer_map) PURE; + struct ResponseHeadersTransformationResult { + static ResponseHeadersTransformationResult success() { + return ResponseHeadersTransformationResult{RejectResult::success(), nullptr}; + } + RejectResult status; + ResponseHeaderMapPtr new_headers; + }; + virtual ResponseHeadersTransformationResult + transformResponseHeaders(const ResponseHeaderMap& header_map) PURE; +}; + +/** + * Interface for server header validators. + */ +class ClientHeaderValidator : public HeaderValidatorBase { +public: + ~ClientHeaderValidator() override = default; /** - * Transform the entire request trailer map. - * Returning the Reject value causes the HTTP request to be rejected with the 502 status, - * and the gRPC request with the UNAVAILABLE (14) error code. - * If response headers have already been sent the request is reset. + * Transform the entire request header map. + * This method can not mutate the header map as it is immutable after the terminal decoder filter. + * However HTTP/2 and HTTP/3 header validators may need to change the request from the HTTP/1 + * upgrade to to the extended CONNECT. In this case the new header map is returned in the + * `new_headers` member of the returned structure. Returning the Reject value form this method + * causes the HTTP request to be rejected with 400 status, and the gRPC request with the INTERNAL + * (13) error code. */ - using TrailersTransformationResult = RejectResult; - virtual TrailersTransformationResult transformRequestTrailers(RequestTrailerMap& header_map) PURE; + struct RequestHeadersTransformationResult { + static RequestHeadersTransformationResult success() { + return RequestHeadersTransformationResult{RejectResult::success(), nullptr}; + } + RejectResult status; + RequestHeaderMapPtr new_headers; + }; + virtual RequestHeadersTransformationResult + transformRequestHeaders(const RequestHeaderMap& header_map) PURE; /** - * Validate the entire response trailer map. - * Returning the Reject value causes the HTTP request to be reset. + * Transform the entire response header map. + * HTTP/2 and HTTP/3 client header validator may transform the extended CONNECT response + * to HTTP/1 upgrade response, iff it transformed upgrade request to extended CONNECT + * during request validation. + * Returning the Reject value causes the HTTP request to be rejected with the 502 status, + * and the gRPC request with the UNAVAILABLE (14) error code. */ - virtual ValidationResult validateResponseTrailers(const ResponseTrailerMap& trailer_map) PURE; + virtual TransformationResult transformResponseHeaders(ResponseHeaderMap& header_map) PURE; }; using HeaderValidatorPtr = std::unique_ptr; +using ClientHeaderValidatorPtr = std::unique_ptr; /** * Interface for stats. @@ -113,6 +180,7 @@ class HeaderValidatorStats { /** * Interface for creating header validators. + * TODO(yanavlasov): split into factories dedicated to server and client header validators. */ class HeaderValidatorFactory { public: @@ -121,7 +189,10 @@ class HeaderValidatorFactory { /** * Create a new header validator for the specified protocol. */ - virtual HeaderValidatorPtr create(Protocol protocol, HeaderValidatorStats& stats) PURE; + virtual HeaderValidatorPtr createServerHeaderValidator(Protocol protocol, + HeaderValidatorStats& stats) PURE; + virtual ClientHeaderValidatorPtr createClientHeaderValidator(Protocol protocol, + HeaderValidatorStats& stats) PURE; }; using HeaderValidatorFactoryPtr = std::unique_ptr; diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h index f82850e62544..18460c6b6083 100644 --- a/envoy/upstream/upstream.h +++ b/envoy/upstream/upstream.h @@ -1191,7 +1191,7 @@ class ClusterInfo : public Http::FilterChainFactory { * @return create header validator based on cluster configuration. Returns nullptr if * ENVOY_ENABLE_UHV is undefined. */ - virtual Http::HeaderValidatorPtr makeHeaderValidator(Http::Protocol protocol) const PURE; + virtual Http::ClientHeaderValidatorPtr makeHeaderValidator(Http::Protocol protocol) const PURE; protected: /** diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index d11349594744..a6affd33780b 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -184,11 +184,34 @@ void CodecClient::onData(Buffer::Instance& data) { absl::StrCat("extraneous bytes after response complete: ", data.length())); } +Status CodecClient::ActiveRequest::encodeHeaders(const RequestHeaderMap& headers, bool end_stream) { +#ifdef ENVOY_ENABLE_UHV + if (header_validator_) { + auto result = header_validator_->transformRequestHeaders(headers); + if (!result.status.ok()) { + return absl::InvalidArgumentError( + absl::StrCat("header validation failed: ", result.status.details())); + } + if (result.new_headers) { + return RequestEncoderWrapper::encodeHeaders(*result.new_headers, end_stream); + } + } +#endif + return RequestEncoderWrapper::encodeHeaders(headers, end_stream); +} + void CodecClient::ActiveRequest::decodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream) { +#ifdef ENVOY_ENABLE_UHV if (header_validator_) { - const ::Envoy::Http::HeaderValidator::ValidationResult result = + const ::Envoy::Http::HeaderValidator::ValidationResult validation_result = header_validator_->validateResponseHeaders(*headers); - if (!result.ok()) { + bool failure = !validation_result.ok(); + if (!failure) { + const ::Envoy::Http::ClientHeaderValidator::TransformationResult transformation_result = + header_validator_->transformResponseHeaders(*headers); + failure = !transformation_result.ok(); + } + if (failure) { ENVOY_CONN_LOG(debug, "Response header validation failed\n{}", *parent_.connection_, *headers); if ((parent_.codec_->protocol() == Protocol::Http2 && @@ -210,6 +233,7 @@ void CodecClient::ActiveRequest::decodeHeaders(ResponseHeaderMapPtr&& headers, b return; } } +#endif ResponseDecoderWrapper::decodeHeaders(std::move(headers), end_stream); } diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index 8ba8e63a324d..56cb24dbe656 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -262,6 +262,9 @@ class CodecClient : protected Logger::Loggable, // RequestEncoderWrapper void onEncodeComplete() override { parent_.requestEncodeComplete(*this); } + // RequestEncoder + Status encodeHeaders(const RequestHeaderMap& headers, bool end_stream) override; + void setEncoder(RequestEncoder& encoder) { inner_encoder_ = &encoder; inner_encoder_->getStream().addCallbacks(*this); @@ -270,7 +273,7 @@ class CodecClient : protected Logger::Loggable, void removeEncoderCallbacks() { inner_encoder_->getStream().removeCallbacks(*this); } CodecClient& parent_; - Http::HeaderValidatorPtr header_validator_; + Http::ClientHeaderValidatorPtr header_validator_; bool wait_encode_complete_{true}; bool encode_complete_{false}; bool decode_complete_{false}; diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 528654411984..006ac3f93471 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -946,7 +946,7 @@ bool ConnectionManagerImpl::ActiveStream::validateHeaders() { auto transformation_result = header_validator_->transformRequestHeaders(*request_headers_); failure = !transformation_result.ok(); redirect = transformation_result.action() == - Http::HeaderValidator::HeadersTransformationResult::Action::Redirect; + Http::HeaderValidator::RequestHeadersTransformationResult::Action::Redirect; failure_details = transformation_result.details(); } if (failure) { @@ -1692,9 +1692,22 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& heade ENVOY_STREAM_LOG(debug, "encoding headers via codec (end_stream={}):\n{}", *this, end_stream, headers); - // Now actually encode via the codec. filter_manager_.streamInfo().downstreamTiming().onFirstDownstreamTxByteSent( connection_manager_.time_source_); + + if (header_validator_) { + auto result = header_validator_->transformResponseHeaders(headers); + if (!result.status.ok()) { + // It is possible that the header map is invalid if an encoder filter makes invalid + // modifications + // TODO(yanavlasov): add handling for this case. + } else if (result.new_headers) { + response_encoder_->encodeHeaders(*result.new_headers, end_stream); + return; + } + } + + // Now actually encode via the codec. response_encoder_->encodeHeaders(headers, end_stream); } diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 4be1206db910..3a3a8e464231 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -265,6 +265,8 @@ Status ConnectionImpl::ClientStreamImpl::encodeHeaders(const RequestHeaderMap& h RETURN_IF_ERROR(HeaderUtility::checkRequiredRequestHeaders(headers)); // Verify that a filter hasn't added an invalid header key or value. RETURN_IF_ERROR(HeaderUtility::checkValidRequestHeaders(headers)); +#ifndef ENVOY_ENABLE_UHV + // Extended CONNECT to H/1 upgrade transformation has moved to UHV // This must exist outside of the scope of isUpgrade as the underlying memory is // needed until encodeHeadersBase has been called. Http::RequestHeaderMapPtr modified_headers; @@ -282,6 +284,9 @@ Status ConnectionImpl::ClientStreamImpl::encodeHeaders(const RequestHeaderMap& h } else { encodeHeadersBase(headers, end_stream); } +#else + encodeHeadersBase(headers, end_stream); +#endif return okStatus(); } @@ -291,6 +296,8 @@ void ConnectionImpl::ServerStreamImpl::encodeHeaders(const ResponseHeaderMap& he // The contract is that client codecs must ensure that :status is present. ASSERT(headers.Status() != nullptr); +#ifndef ENVOY_ENABLE_UHV + // Extended CONNECT to H/1 upgrade transformation has moved to UHV // This must exist outside of the scope of isUpgrade as the underlying memory is // needed until encodeHeadersBase has been called. Http::ResponseHeaderMapPtr modified_headers; @@ -301,6 +308,9 @@ void ConnectionImpl::ServerStreamImpl::encodeHeaders(const ResponseHeaderMap& he } else { encodeHeadersBase(headers, end_stream); } +#else + encodeHeadersBase(headers, end_stream); +#endif } void ConnectionImpl::StreamImpl::encodeTrailersBase(const HeaderMap& trailers) { @@ -516,9 +526,12 @@ void ConnectionImpl::ClientStreamImpl::decodeHeaders() { auto& headers = absl::get(headers_or_trailers_); const uint64_t status = Http::Utility::getResponseStatus(*headers); +#ifndef ENVOY_ENABLE_UHV + // Extended CONNECT to H/1 upgrade transformation has moved to UHV if (!upgrade_type_.empty() && headers->Status()) { Http::Utility::transformUpgradeResponseFromH2toH1(*headers, upgrade_type_); } +#endif // Non-informational headers are non-1xx OR 101-SwitchingProtocols, since 101 implies that further // proxying is on an upgrade path. @@ -564,9 +577,12 @@ void ConnectionImpl::ClientStreamImpl::decodeTrailers() { void ConnectionImpl::ServerStreamImpl::decodeHeaders() { auto& headers = absl::get(headers_or_trailers_); +#ifndef ENVOY_ENABLE_UHV + // Extended CONNECT to H/1 upgrade transformation has moved to UHV if (Http::Utility::isH2UpgradeRequest(*headers)) { Http::Utility::transformUpgradeRequestFromH2toH1(*headers); } +#endif request_decoder_->decodeHeaders(std::move(headers), sendEndStream()); } diff --git a/source/common/quic/envoy_quic_client_stream.cc b/source/common/quic/envoy_quic_client_stream.cc index 621b4fa16b20..8d2cf965e37a 100644 --- a/source/common/quic/envoy_quic_client_stream.cc +++ b/source/common/quic/envoy_quic_client_stream.cc @@ -48,6 +48,8 @@ Http::Status EnvoyQuicClientStream::encodeHeaders(const Http::RequestHeaderMap& local_end_stream_ = end_stream; SendBufferMonitor::ScopedWatermarkBufferUpdater updater(this, this); spdy::Http2HeaderBlock spdy_headers; +#ifndef ENVOY_ENABLE_UHV + // Extended CONNECT to H/1 upgrade transformation has moved to UHV if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.use_http3_header_normalisation") && Http::Utility::isUpgrade(headers)) { // In Envoy, both upgrade requests and extended CONNECT requests are @@ -75,6 +77,12 @@ Http::Status EnvoyQuicClientStream::encodeHeaders(const Http::RequestHeaderMap& if (spdy_headers.empty()) { spdy_headers = envoyHeadersToHttp2HeaderBlock(headers); } +#else + spdy_headers = envoyHeadersToHttp2HeaderBlock(headers); + if (headers.Method()->value() == "HEAD") { + sent_head_request_ = true; + } +#endif { IncrementalBytesSentTracker tracker(*this, *mutableBytesMeter(), true); size_t bytes_sent = WriteHeaders(std::move(spdy_headers), end_stream, nullptr); @@ -236,6 +244,8 @@ void EnvoyQuicClientStream::OnInitialHeadersComplete(bool fin, size_t frame_len, set_headers_decompressed(false); } +#ifndef ENVOY_ENABLE_UHV + // Extended CONNECT to H/1 upgrade transformation has moved to UHV // In Envoy, both upgrade requests and extended CONNECT requests are // represented as their HTTP/1 forms, regardless of the HTTP version used. // Therefore, these need to be transformed into their HTTP/1 form. @@ -243,6 +253,7 @@ void EnvoyQuicClientStream::OnInitialHeadersComplete(bool fin, size_t frame_len, !upgrade_protocol_.empty()) { Http::Utility::transformUpgradeResponseFromH3toH1(*headers, upgrade_protocol_); } +#endif const bool is_special_1xx = Http::HeaderUtility::isSpecial1xx(*headers); if (is_special_1xx && !decoded_1xx_) { diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc index ffbe05b5b100..463e2412fa83 100644 --- a/source/common/quic/envoy_quic_server_stream.cc +++ b/source/common/quic/envoy_quic_server_stream.cc @@ -61,13 +61,15 @@ void EnvoyQuicServerStream::encodeHeaders(const Http::ResponseHeaderMap& headers // sending them. const Http::ResponseHeaderMap* header_map = &headers; std::unique_ptr modified_headers; +#ifndef ENVOY_ENABLE_UHV + // Extended CONNECT to H/1 upgrade transformation has moved to UHV if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.use_http3_header_normalisation") && Http::Utility::isUpgrade(headers)) { modified_headers = Http::createHeaderMap(headers); Http::Utility::transformUpgradeResponseFromH1toH3(*modified_headers); header_map = modified_headers.get(); } - +#endif // This is counting not serialized bytes in the send buffer. local_end_stream_ = end_stream; SendBufferMonitor::ScopedWatermarkBufferUpdater updater(this, this); @@ -242,11 +244,14 @@ void EnvoyQuicServerStream::OnInitialHeadersComplete(bool fin, size_t frame_len, return; } +#ifndef ENVOY_ENABLE_UHV + // Extended CONNECT to H/1 upgrade transformation has moved to UHV if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.use_http3_header_normalisation") && Http::Utility::isH3UpgradeRequest(*headers)) { // Transform Request from H3 to H1 Http::Utility::transformUpgradeRequestFromH3toH1(*headers); } +#endif request_decoder_->decodeHeaders(std::move(headers), /*end_stream=*/fin); ConsumeHeaderList(); diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 0d341d9ef166..130404364e3a 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1721,11 +1721,11 @@ ClusterInfoImpl::getHeaderValidatorStats(Http::Protocol protocol) const { } #endif -Http::HeaderValidatorPtr +Http::ClientHeaderValidatorPtr ClusterInfoImpl::makeHeaderValidator([[maybe_unused]] Http::Protocol protocol) const { #ifdef ENVOY_ENABLE_UHV return http_protocol_options_->header_validator_factory_ - ? http_protocol_options_->header_validator_factory_->create( + ? http_protocol_options_->header_validator_factory_->createClientHeaderValidator( protocol, getHeaderValidatorStats(protocol)) : nullptr; #else diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 88d0eaf793dc..6a9795e0e141 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -991,7 +991,7 @@ class ClusterInfoImpl : public ClusterInfo, Http::Http1::CodecStats& http1CodecStats() const override; Http::Http2::CodecStats& http2CodecStats() const override; Http::Http3::CodecStats& http3CodecStats() const override; - Http::HeaderValidatorPtr makeHeaderValidator(Http::Protocol protocol) const override; + Http::ClientHeaderValidatorPtr makeHeaderValidator(Http::Protocol protocol) const override; protected: // Gets the retry budget percent/concurrency from the circuit breaker thresholds. If the retry diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index fcd8ed1116d0..ec5e43cd0279 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -245,9 +245,9 @@ class HttpConnectionManagerConfig : Logger::Loggable, } Http::HeaderValidatorPtr makeHeaderValidator([[maybe_unused]] Http::Protocol protocol) override { #ifdef ENVOY_ENABLE_UHV - return header_validator_factory_ - ? header_validator_factory_->create(protocol, getHeaderValidatorStats(protocol)) - : nullptr; + return header_validator_factory_ ? header_validator_factory_->createServerHeaderValidator( + protocol, getHeaderValidatorStats(protocol)) + : nullptr; #else return nullptr; #endif diff --git a/source/extensions/http/header_validators/envoy_default/BUILD b/source/extensions/http/header_validators/envoy_default/BUILD index b6bf66aa8d7c..18252f0f142a 100644 --- a/source/extensions/http/header_validators/envoy_default/BUILD +++ b/source/extensions/http/header_validators/envoy_default/BUILD @@ -109,8 +109,10 @@ envoy_cc_library( "//envoy/http:header_validator_interface", "//external:abseil_node_hash_map", "//external:abseil_node_hash_set", + "//source/common/http:header_map_lib", "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", + "//source/common/http:utility_lib", "@com_google_absl//absl/functional:bind_front", ], ) diff --git a/source/extensions/http/header_validators/envoy_default/header_validator.cc b/source/extensions/http/header_validators/envoy_default/header_validator.cc index 65b8baa7c103..e9a43ae62f40 100644 --- a/source/extensions/http/header_validators/envoy_default/header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/header_validator.cc @@ -518,7 +518,7 @@ HeaderValidator::HeaderEntryValidationResult HeaderValidator::validateGenericReq // For H/1 the codec will never produce H/2 pseudo headers and per // https://www.rfc-editor.org/rfc/rfc9110#section-6.5 there are no other prohibitions. // As a result this common function can cover trailer validation for all protocols. -HeaderValidator::ValidationResult +::Envoy::Http::HeaderValidatorBase::ValidationResult HeaderValidator::validateTrailers(const ::Envoy::Http::HeaderMap& trailers) { std::string reject_details; trailers.iterate([this, &reject_details](const ::Envoy::Http::HeaderEntry& header_entry) @@ -541,10 +541,10 @@ HeaderValidator::validateTrailers(const ::Envoy::Http::HeaderMap& trailers) { }); if (!reject_details.empty()) { - return {ValidationResult::Action::Reject, reject_details}; + return {::Envoy::Http::HeaderValidatorBase::ValidationResult::Action::Reject, reject_details}; } - return ValidationResult::success(); + return ::Envoy::Http::HeaderValidatorBase::ValidationResult::success(); } void HeaderValidator::sanitizeHeadersWithUnderscores(::Envoy::Http::HeaderMap& header_map) { diff --git a/source/extensions/http/header_validators/envoy_default/header_validator.h b/source/extensions/http/header_validators/envoy_default/header_validator.h index bdde6b78ec35..0861574a3c48 100644 --- a/source/extensions/http/header_validators/envoy_default/header_validator.h +++ b/source/extensions/http/header_validators/envoy_default/header_validator.h @@ -20,15 +20,16 @@ namespace EnvoyDefault { * Base class for all HTTP codec header validations. This class has several methods to validate * headers that are shared across multiple codec versions where the RFC guidance did not change. */ -class HeaderValidator : public ::Envoy::Http::HeaderValidator { +class HeaderValidator { public: HeaderValidator( const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& config, ::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats); + virtual ~HeaderValidator() = default; - using HeaderEntryValidationResult = RejectResult; - using HeaderValueValidationResult = RejectResult; + using HeaderEntryValidationResult = ::Envoy::Http::HeaderValidator::RejectResult; + using HeaderValueValidationResult = ::Envoy::Http::HeaderValidator::RejectResult; /* * Validate the :method pseudo header, honoring the restrict_http_methods configuration option. */ @@ -88,6 +89,7 @@ class HeaderValidator : public ::Envoy::Http::HeaderValidator { */ class HostHeaderValidationResult { public: + using RejectAction = ::Envoy::Http::HeaderValidatorBase::RejectAction; HostHeaderValidationResult(RejectAction action, absl::string_view details, absl::string_view address, absl::string_view port) : result_(action, details, address, port) { @@ -150,7 +152,8 @@ class HeaderValidator : public ::Envoy::Http::HeaderValidator { /* * Common method for validating request or response trailers. */ - ValidationResult validateTrailers(const ::Envoy::Http::HeaderMap& trailers); + ::Envoy::Http::HeaderValidator::ValidationResult + validateTrailers(const ::Envoy::Http::HeaderMap& trailers); /** * Removes headers with underscores in their names iff the headers_with_underscores_action diff --git a/source/extensions/http/header_validators/envoy_default/header_validator_factory.cc b/source/extensions/http/header_validators/envoy_default/header_validator_factory.cc index 707d7dc7d966..6ab000b85358 100644 --- a/source/extensions/http/header_validators/envoy_default/header_validator_factory.cc +++ b/source/extensions/http/header_validators/envoy_default/header_validator_factory.cc @@ -16,17 +16,31 @@ HeaderValidatorFactory::HeaderValidatorFactory(const HeaderValidatorConfig& conf : config_(config) {} ::Envoy::Http::HeaderValidatorPtr -HeaderValidatorFactory::create(Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats) { +HeaderValidatorFactory::createServerHeaderValidator(Protocol protocol, + ::Envoy::Http::HeaderValidatorStats& stats) { switch (protocol) { case Protocol::Http3: case Protocol::Http2: - return std::make_unique(config_, protocol, stats); + return std::make_unique(config_, protocol, stats); case Protocol::Http11: case Protocol::Http10: - return std::make_unique(config_, protocol, stats); + return std::make_unique(config_, protocol, stats); } + PANIC_DUE_TO_CORRUPT_ENUM; +} - RELEASE_ASSERT(false, fmt::format("Unexpected protocol: {}", static_cast(protocol))); +::Envoy::Http::ClientHeaderValidatorPtr +HeaderValidatorFactory::createClientHeaderValidator(Protocol protocol, + ::Envoy::Http::HeaderValidatorStats& stats) { + switch (protocol) { + case Protocol::Http3: + case Protocol::Http2: + return std::make_unique(config_, protocol, stats); + case Protocol::Http11: + case Protocol::Http10: + return std::make_unique(config_, protocol, stats); + } + PANIC_DUE_TO_CORRUPT_ENUM; } } // namespace EnvoyDefault diff --git a/source/extensions/http/header_validators/envoy_default/header_validator_factory.h b/source/extensions/http/header_validators/envoy_default/header_validator_factory.h index 829502c44433..2fc2ebf72349 100644 --- a/source/extensions/http/header_validators/envoy_default/header_validator_factory.h +++ b/source/extensions/http/header_validators/envoy_default/header_validator_factory.h @@ -15,8 +15,13 @@ class HeaderValidatorFactory : public ::Envoy::Http::HeaderValidatorFactory { const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& config); - ::Envoy::Http::HeaderValidatorPtr create(::Envoy::Http::Protocol protocol, - ::Envoy::Http::HeaderValidatorStats& stats) override; + ::Envoy::Http::HeaderValidatorPtr + createServerHeaderValidator(::Envoy::Http::Protocol protocol, + ::Envoy::Http::HeaderValidatorStats& stats) override; + + ::Envoy::Http::ClientHeaderValidatorPtr + createClientHeaderValidator(::Envoy::Http::Protocol protocol, + ::Envoy::Http::HeaderValidatorStats& stats) override; private: const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig diff --git a/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc b/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc index e096bbae80b4..c8d821926994 100644 --- a/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc @@ -22,6 +22,7 @@ using ::Envoy::Http::LowerCaseString; using ::Envoy::Http::Protocol; using ::Envoy::Http::RequestHeaderMap; using ::Envoy::Http::UhvResponseCodeDetail; +using ValidationResult = ::Envoy::Http::HeaderValidatorBase::ValidationResult; struct Http1ResponseCodeDetailValues { const std::string InvalidTransferEncoding = "uhv.http1.invalid_transfer_encoding"; @@ -59,7 +60,6 @@ Http1HeaderValidator::validateRequestHeaderEntry(const HeaderString& key, const HeaderString& value) { // Pseudo headers in HTTP/1.1 are synthesized by the codec from the request line prior to // submitting the header map for validation in UHV. - return validateGenericRequestHeaderEntry(key, value, request_header_validator_map_); } @@ -99,8 +99,7 @@ Http1HeaderValidator::validateResponseHeaderEntry(const HeaderString& key, return validateGenericHeaderValue(value); } -HeaderValidator::ValidationResult -Http1HeaderValidator::validateRequestHeaders(const RequestHeaderMap& header_map) { +ValidationResult Http1HeaderValidator::validateRequestHeaders(const RequestHeaderMap& header_map) { absl::string_view path = header_map.getPathValue(); absl::string_view host = header_map.getHostValue(); // Step 1: verify that required pseudo headers are present. HTTP/1.1 requests requires the @@ -296,8 +295,8 @@ void Http1HeaderValidator::sanitizeContentLength(::Envoy::Http::RequestHeaderMap } } -HeaderValidator::HeadersTransformationResult -Http1HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) { +::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult +ServerHttp1HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) { sanitizeContentLength(header_map); sanitizeHeadersWithUnderscores(header_map); if (!config_.uri_path_normalization_options().skip_path_normalization()) { @@ -306,10 +305,10 @@ Http1HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeaderMap& h return path_result; } } - return HeadersTransformationResult::success(); + return ::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult::success(); } -HeaderValidator::ValidationResult +ValidationResult Http1HeaderValidator::validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map) { // Step 1: verify that required pseudo headers are present // @@ -378,18 +377,19 @@ Http1HeaderValidator::validateTransferEncodingHeader(const HeaderString& value) return HeaderValueValidationResult::success(); } -HeaderValidator::ValidationResult +ValidationResult Http1HeaderValidator::validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map) { return validateTrailers(trailer_map); } -HeaderValidator::TrailersTransformationResult -Http1HeaderValidator::transformRequestTrailers(::Envoy::Http::RequestTrailerMap& trailer_map) { +::Envoy::Http::HeaderValidator::TransformationResult +ServerHttp1HeaderValidator::transformRequestTrailers( + ::Envoy::Http::RequestTrailerMap& trailer_map) { sanitizeHeadersWithUnderscores(trailer_map); - return TrailersTransformationResult::success(); + return ::Envoy::Http::HeaderValidator::TransformationResult::success(); } -HeaderValidator::ValidationResult Http1HeaderValidator::validateResponseTrailers( +ValidationResult Http1HeaderValidator::validateResponseTrailers( const ::Envoy::Http::ResponseTrailerMap& trailer_map) { return validateTrailers(trailer_map); } diff --git a/source/extensions/http/header_validators/envoy_default/http1_header_validator.h b/source/extensions/http/header_validators/envoy_default/http1_header_validator.h index cdc826003169..183666f0d108 100644 --- a/source/extensions/http/header_validators/envoy_default/http1_header_validator.h +++ b/source/extensions/http/header_validators/envoy_default/http1_header_validator.h @@ -15,23 +15,29 @@ class Http1HeaderValidator : public HeaderValidator { config, ::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats); - ValidationResult - validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map) override; - - HeadersTransformationResult - transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) override; + ::Envoy::Http::HeaderValidatorBase::ValidationResult + validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map); - ValidationResult - validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map) override; + ::Envoy::Http::HeaderValidatorBase::ValidationResult + validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map); - ValidationResult - validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map) override; + ::Envoy::Http::HeaderValidatorBase::ValidationResult + validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map); - TrailersTransformationResult - transformRequestTrailers(::Envoy::Http::RequestTrailerMap& header_map) override; + ::Envoy::Http::HeaderValidatorBase::ValidationResult + validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map); - ValidationResult - validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map) override; +protected: + /** + * Checks for presence of both Transfer-Encoding and Content-Length headers and + * removes the Content-Length header iff the http1_protocol_options.allow_chunked_length + * config options is true. + * If the http1_protocol_options.allow_chunked_length is false a request with both + * Transfer-Encoding and Content-Length headers is rejected in the validateRequestHeaders method. + * Additionally if request is CONNECT and Content-Length is 0, the Content-Length header is + * removed. + */ + void sanitizeContentLength(::Envoy::Http::RequestHeaderMap& header_map); private: /* @@ -46,21 +52,91 @@ class Http1HeaderValidator : public HeaderValidator { HeaderEntryValidationResult validateResponseHeaderEntry(const ::Envoy::Http::HeaderString& key, const ::Envoy::Http::HeaderString& value); - /** - * Checks for presence of both Transfer-Encoding and Content-Length headers and - * removes the Content-Length header iff the http1_protocol_options.allow_chunked_length - * config options is true. - * If the http1_protocol_options.allow_chunked_length is false a request with both - * Transfer-Encoding and Content-Length headers is rejected in the validateRequestHeaders method. - * Additionally if request is CONNECT and Content-Length is 0, the Content-Length header is - * removed. - */ - void sanitizeContentLength(::Envoy::Http::RequestHeaderMap& header_map); - const HeaderValidatorMap request_header_validator_map_; }; -using Http1HeaderValidatorPtr = std::unique_ptr; +class ServerHttp1HeaderValidator : public Http1HeaderValidator, + public ::Envoy::Http::HeaderValidator { +public: + ServerHttp1HeaderValidator( + const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& + config, + ::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats) + : Http1HeaderValidator(config, protocol, stats) {} + + ValidationResult + validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map) override { + return Http1HeaderValidator::validateRequestHeaders(header_map); + } + + ValidationResult + validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map) override { + return Http1HeaderValidator::validateResponseHeaders(header_map); + } + + ValidationResult + validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map) override { + return Http1HeaderValidator::validateRequestTrailers(trailer_map); + } + + ValidationResult + validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map) override { + return Http1HeaderValidator::validateResponseTrailers(trailer_map); + } + + RequestHeadersTransformationResult + transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) override; + + TransformationResult + transformRequestTrailers(::Envoy::Http::RequestTrailerMap& header_map) override; + + ResponseHeadersTransformationResult + transformResponseHeaders(const ::Envoy::Http::ResponseHeaderMap&) override { + return ResponseHeadersTransformationResult::success(); + } +}; + +class ClientHttp1HeaderValidator : public Http1HeaderValidator, + public ::Envoy::Http::ClientHeaderValidator { +public: + ClientHttp1HeaderValidator( + const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& + config, + ::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats) + : Http1HeaderValidator(config, protocol, stats) {} + + ValidationResult + validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map) override { + return Http1HeaderValidator::validateRequestHeaders(header_map); + } + + ValidationResult + validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map) override { + return Http1HeaderValidator::validateResponseHeaders(header_map); + } + + ValidationResult + validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map) override { + return Http1HeaderValidator::validateRequestTrailers(trailer_map); + } + + ValidationResult + validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map) override { + return Http1HeaderValidator::validateResponseTrailers(trailer_map); + } + + RequestHeadersTransformationResult + transformRequestHeaders(const ::Envoy::Http::RequestHeaderMap&) override { + return RequestHeadersTransformationResult::success(); + } + + TransformationResult transformResponseHeaders(::Envoy::Http::ResponseHeaderMap&) override { + return TransformationResult::success(); + } +}; + +using ServerHttp1HeaderValidatorPtr = std::unique_ptr; +using ClientHttp1HeaderValidatorPtr = std::unique_ptr; } // namespace EnvoyDefault } // namespace HeaderValidators diff --git a/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc b/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc index 062c4fd2d556..49922360ff83 100644 --- a/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc @@ -1,11 +1,14 @@ #include "source/extensions/http/header_validators/envoy_default/http2_header_validator.h" +#include + #include "envoy/http/header_validator_errors.h" +#include "source/common/http/header_map_impl.h" #include "source/common/http/header_utility.h" +#include "source/common/http/utility.h" #include "source/extensions/http/header_validators/envoy_default/character_tables.h" -#include "absl/container/node_hash_map.h" #include "absl/container/node_hash_set.h" #include "absl/functional/bind_front.h" #include "absl/strings/match.h" @@ -23,8 +26,7 @@ using ::Envoy::Http::HeaderUtility; using ::Envoy::Http::Protocol; using ::Envoy::Http::testCharInTable; using ::Envoy::Http::UhvResponseCodeDetail; -using HeaderValidatorFunction1 = - HeaderValidator::HeaderValueValidationResult (Http2HeaderValidator::*)(const HeaderString&); +using ValidationResult = ::Envoy::Http::HeaderValidatorBase::ValidationResult; struct Http2ResponseCodeDetailValues { const std::string InvalidTE = "uhv.http2.invalid_te"; @@ -95,7 +97,7 @@ Http2HeaderValidator::validateResponseHeaderEntry(const HeaderString& key, return validateGenericHeaderValue(value); } -HeaderValidator::ValidationResult +ValidationResult Http2HeaderValidator::validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map) { static const absl::node_hash_set kAllowedPseudoHeadersForConnect = { ":method", ":authority"}; @@ -227,19 +229,7 @@ Http2HeaderValidator::validateRequestHeaders(const ::Envoy::Http::RequestHeaderM return ValidationResult::success(); } -HeaderValidator::HeadersTransformationResult -Http2HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) { - sanitizeHeadersWithUnderscores(header_map); - if (!config_.uri_path_normalization_options().skip_path_normalization()) { - auto path_result = path_normalizer_.normalizePathUri(header_map); - if (!path_result.ok()) { - return path_result; - } - } - return HeadersTransformationResult::success(); -} - -HeaderValidator::ValidationResult +ValidationResult Http2HeaderValidator::validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map) { // Step 1: verify that required pseudo headers are present // @@ -410,30 +400,106 @@ Http2HeaderValidator::validateGenericHeaderName(const HeaderString& name) { return HeaderEntryValidationResult::success(); } -HeaderValidator::ValidationResult +ValidationResult Http2HeaderValidator::validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map) { - HeaderValidator::ValidationResult result = validateTrailers(trailer_map); + ValidationResult result = validateTrailers(trailer_map); if (!result.ok()) { stats_.incMessagingError(); } return result; } -HeaderValidator::TrailersTransformationResult -Http2HeaderValidator::transformRequestTrailers(::Envoy::Http::RequestTrailerMap& trailer_map) { +::Envoy::Http::HeaderValidator::TransformationResult +ServerHttp2HeaderValidator::transformRequestTrailers( + ::Envoy::Http::RequestTrailerMap& trailer_map) { sanitizeHeadersWithUnderscores(trailer_map); - return TrailersTransformationResult::success(); + return ::Envoy::Http::HeaderValidator::TransformationResult::success(); } -HeaderValidator::ValidationResult Http2HeaderValidator::validateResponseTrailers( +ValidationResult Http2HeaderValidator::validateResponseTrailers( const ::Envoy::Http::ResponseTrailerMap& trailer_map) { - HeaderValidator::ValidationResult result = validateTrailers(trailer_map); + ValidationResult result = validateTrailers(trailer_map); if (!result.ok()) { stats_.incMessagingError(); } return result; } +::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult +ServerHttp2HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) { + sanitizeHeadersWithUnderscores(header_map); + if (!config_.uri_path_normalization_options().skip_path_normalization()) { + auto path_result = path_normalizer_.normalizePathUri(header_map); + if (!path_result.ok()) { + return path_result; + } + } + + // Transform H/2 extended CONNECT to H/1 UPGRADE, so that request processing always observes H/1 + // UPGRADE requests + if (::Envoy::Http::Utility::isH2UpgradeRequest(header_map)) { + ::Envoy::Http::Utility::transformUpgradeRequestFromH2toH1(header_map); + } + return ::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult::success(); +} + +::Envoy::Http::HeaderValidator::ResponseHeadersTransformationResult +ServerHttp2HeaderValidator::transformResponseHeaders( + const ::Envoy::Http::ResponseHeaderMap& header_map) { + // Check if the response is for the the H/1 UPGRADE and transform it to the H/2 extended CONNECT + // response. + // Note that at this point the header map may not be valid if a buggy encoder filter + // removed the :status header, so we check for this case as well. + + if (header_map.Status() != nullptr && ::Envoy::Http::Utility::isUpgrade(header_map)) { + ::Envoy::Http::ResponseHeaderMapPtr modified_headers = + ::Envoy::Http::createHeaderMap<::Envoy::Http::ResponseHeaderMapImpl>(header_map); + ::Envoy::Http::Utility::transformUpgradeResponseFromH1toH2(*modified_headers); + // Return new header map along with the success result + return {RejectResult::success(), std::move(modified_headers)}; + } + + return {RejectResult::success(), nullptr}; +} + +::Envoy::Http::ClientHeaderValidator::RequestHeadersTransformationResult +ClientHttp2HeaderValidator::transformRequestHeaders( + const ::Envoy::Http::RequestHeaderMap& header_map) { + ::Envoy::Http::RequestHeaderMapPtr modified_headers; + if (::Envoy::Http::Utility::isUpgrade(header_map)) { + // Remember the fact that H/1 upgrade was transformed into H/2 extended CONNECT, so that + // response can be transformed from extended CONNECT to H/1 upgrade. + upgrade_type_ = std::string(header_map.getUpgradeValue()); + modified_headers = + ::Envoy::Http::createHeaderMap<::Envoy::Http::RequestHeaderMapImpl>(header_map); + ::Envoy::Http::Utility::transformUpgradeRequestFromH1toH2(*modified_headers); + } else if (::Envoy::Http::HeaderUtility::isConnect(header_map)) { + // Sanitize the standard CONNECT request, as filters (and HCM) may add prohibited headers + // like :scheme, or :path (i.e. by a path rewrite rule) + modified_headers = + ::Envoy::Http::createHeaderMap<::Envoy::Http::RequestHeaderMapImpl>(header_map); + modified_headers->removeScheme(); + modified_headers->removePath(); + // Note that extended CONNECT is transformed to H/1 upgrade and handled above. + // The only case where the :protocol header would be present here is if an HTTP + // filter adds it. But this case is unsupported at this point. + modified_headers->removeProtocol(); + } + + return {RejectResult::success(), std::move(modified_headers)}; +} + +::Envoy::Http::ClientHeaderValidator::TransformationResult +ClientHttp2HeaderValidator::transformResponseHeaders(::Envoy::Http::ResponseHeaderMap& header_map) { + // Check if the request was the extended CONNECT and transform response from extended CONNECT + // to the H/1 upgrade response. + if (!upgrade_type_.empty() && header_map.Status() != nullptr) { + ::Envoy::Http::Utility::transformUpgradeResponseFromH2toH1(header_map, upgrade_type_); + } + + return TransformationResult::success(); +} + } // namespace EnvoyDefault } // namespace HeaderValidators } // namespace Http diff --git a/source/extensions/http/header_validators/envoy_default/http2_header_validator.h b/source/extensions/http/header_validators/envoy_default/http2_header_validator.h index c259023bbfd0..8f32f10c13e2 100644 --- a/source/extensions/http/header_validators/envoy_default/http2_header_validator.h +++ b/source/extensions/http/header_validators/envoy_default/http2_header_validator.h @@ -15,23 +15,17 @@ class Http2HeaderValidator : public HeaderValidator { config, ::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats); - ValidationResult - validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map) override; - - HeadersTransformationResult - transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) override; + ::Envoy::Http::HeaderValidatorBase::ValidationResult + validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map); - ValidationResult - validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map) override; + ::Envoy::Http::HeaderValidatorBase::ValidationResult + validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map); - ValidationResult - validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map) override; + ::Envoy::Http::HeaderValidatorBase::ValidationResult + validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map); - TrailersTransformationResult - transformRequestTrailers(::Envoy::Http::RequestTrailerMap& header_map) override; - - ValidationResult - validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map) override; + ::Envoy::Http::HeaderValidatorBase::ValidationResult + validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map); private: /* @@ -61,7 +55,82 @@ class Http2HeaderValidator : public HeaderValidator { const HeaderValidatorMap request_header_validator_map_; }; -using Http2HeaderValidatorPtr = std::unique_ptr; +class ServerHttp2HeaderValidator : public Http2HeaderValidator, + public ::Envoy::Http::HeaderValidator { +public: + ServerHttp2HeaderValidator( + const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& + config, + ::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats) + : Http2HeaderValidator(config, protocol, stats) {} + + ValidationResult + validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map) override { + return Http2HeaderValidator::validateRequestHeaders(header_map); + } + + ValidationResult + validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map) override { + return Http2HeaderValidator::validateResponseHeaders(header_map); + } + + ValidationResult + validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map) override { + return Http2HeaderValidator::validateRequestTrailers(trailer_map); + } + + ValidationResult + validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map) override { + return Http2HeaderValidator::validateResponseTrailers(trailer_map); + } + + RequestHeadersTransformationResult + transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) override; + + TransformationResult + transformRequestTrailers(::Envoy::Http::RequestTrailerMap& header_map) override; + + ResponseHeadersTransformationResult + transformResponseHeaders(const ::Envoy::Http::ResponseHeaderMap&) override; +}; + +class ClientHttp2HeaderValidator : public Http2HeaderValidator, + public ::Envoy::Http::ClientHeaderValidator { +public: + ClientHttp2HeaderValidator( + const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& + config, + ::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats) + : Http2HeaderValidator(config, protocol, stats) {} + + ValidationResult + validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map) override { + return Http2HeaderValidator::validateRequestHeaders(header_map); + } + + ValidationResult + validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map) override { + return Http2HeaderValidator::validateResponseHeaders(header_map); + } + + ValidationResult + validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map) override { + return Http2HeaderValidator::validateRequestTrailers(trailer_map); + } + + ValidationResult + validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map) override { + return Http2HeaderValidator::validateResponseTrailers(trailer_map); + } + + RequestHeadersTransformationResult + transformRequestHeaders(const ::Envoy::Http::RequestHeaderMap&) override; + + TransformationResult transformResponseHeaders(::Envoy::Http::ResponseHeaderMap&) override; + +private: + std::string upgrade_type_; +}; } // namespace EnvoyDefault } // namespace HeaderValidators diff --git a/source/extensions/http/header_validators/envoy_default/path_normalizer.h b/source/extensions/http/header_validators/envoy_default/path_normalizer.h index 3c5bb1166c3f..8901f2bd2e60 100644 --- a/source/extensions/http/header_validators/envoy_default/path_normalizer.h +++ b/source/extensions/http/header_validators/envoy_default/path_normalizer.h @@ -15,7 +15,7 @@ class PathNormalizer { const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& config); - using PathNormalizationResult = ::Envoy::Http::HeaderValidator::HeadersTransformationResult; + using PathNormalizationResult = ::Envoy::Http::HeaderValidator::RejectOrRedirectResult; /* * Normalize the path component of the :path header and update the header value. This method does diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 124df0269f9b..f19c28c0f951 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -74,6 +74,7 @@ envoy_cc_test( "//test/common/upstream:utility_lib", "//test/mocks:common_lib", "//test/mocks/event:event_mocks", + "//test/mocks/http:header_validator_mocks", "//test/mocks/http:http_mocks", "//test/mocks/network:network_mocks", "//test/mocks/ssl:ssl_mocks", diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 4c46a755c8d0..ffb60f2a19e1 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -13,6 +13,7 @@ #include "test/common/upstream/utility.h" #include "test/mocks/common.h" #include "test/mocks/event/mocks.h" +#include "test/mocks/http/header_validator.h" #include "test/mocks/http/mocks.h" #include "test/mocks/network/mocks.h" #include "test/mocks/runtime/mocks.h" @@ -29,6 +30,7 @@ using testing::_; using testing::AtMost; +using testing::ByMove; using testing::Invoke; using testing::InvokeWithoutArgs; using testing::NiceMock; @@ -62,6 +64,17 @@ class CodecClientTest : public Event::TestUsingSimulatedTime, public testing::Te client_ = std::make_unique(CodecType::HTTP1, std::move(connection), codec_, nullptr, host_, dispatcher_); ON_CALL(*connection_, streamInfo()).WillByDefault(ReturnRef(stream_info_)); +#ifdef ENVOY_ENABLE_UHV + ON_CALL(*header_validator_, transformRequestHeaders(_)) + .WillByDefault( + Return(ByMove(ClientHeaderValidator::RequestHeadersTransformationResult::success()))); + ON_CALL(*header_validator_, validateResponseHeaders(_)) + .WillByDefault(Return(HeaderValidator::ValidationResult::success())); + ON_CALL(*header_validator_, transformResponseHeaders(_)) + .WillByDefault(Return(HeaderValidator::TransformationResult::success())); + ON_CALL(*cluster_, makeHeaderValidator(_)) + .WillByDefault(Return(ByMove(std::unique_ptr(header_validator_)))); +#endif } ~CodecClientTest() override { EXPECT_EQ(0U, client_->numActiveRequests()); } @@ -78,6 +91,9 @@ class CodecClientTest : public Event::TestUsingSimulatedTime, public testing::Te Upstream::HostDescriptionConstSharedPtr host_{ Upstream::makeTestHostDescription(cluster_, "tcp://127.0.0.1:80", simTime())}; NiceMock stream_info_; +#ifdef ENVOY_ENABLE_UHV + NiceMock* header_validator_{new NiceMock}; +#endif }; TEST_F(CodecClientTest, NotCallDetectEarlyCloseWhenReadDiabledUsingHttp3) { @@ -293,6 +309,152 @@ TEST_F(CodecClientTest, WatermarkPassthrough) { connection_cb_->onBelowWriteBufferLowWatermark(); } +#ifdef ENVOY_ENABLE_UHV +TEST_F(CodecClientTest, RequestHeaderTransformationFails) { + initialize(); + EXPECT_CALL(*header_validator_, transformRequestHeaders(_)) + .WillOnce(Return(ByMove(ClientHeaderValidator::RequestHeadersTransformationResult{ + {HeaderValidator::RejectResult::Action::Reject, "some error"}, nullptr}))); + + ResponseDecoder* inner_decoder; + NiceMock inner_encoder; + EXPECT_CALL(*codec_, newStream(_)) + .WillOnce(Invoke([&](ResponseDecoder& decoder) -> RequestEncoder& { + inner_decoder = &decoder; + return inner_encoder; + })); + + Http::MockResponseDecoder outer_decoder; + Http::RequestEncoder& request_encoder = client_->newStream(outer_decoder); + + TestRequestHeaderMapImpl request_headers{ + {":authority", "host"}, {":path", "/"}, {":method", "GET"}}; + auto status = request_encoder.encodeHeaders(request_headers, true); + EXPECT_THAT(status, StatusHelpers::HasStatus(absl::StatusCode::kInvalidArgument, + testing::HasSubstr("some error"))); + // Router will reset upstream request when encodeHeaders returns failure status + inner_encoder.stream_.callbacks_.front()->onResetStream(StreamResetReason::LocalReset, + "some error"); +} + +TEST_F(CodecClientTest, RequestHeaderTransformationUpdatesHeaders) { + initialize(); + TestRequestHeaderMapImpl expected_new_headers{ + {":authority", "new_hosthost"}, {":path", "/new_path"}, {":method", "PUT"}}; + auto new_headers = std::make_unique(expected_new_headers); + EXPECT_CALL(*header_validator_, transformRequestHeaders(_)) + .WillOnce(Return(ByMove(ClientHeaderValidator::RequestHeadersTransformationResult{ + HeaderValidator::RejectResult::success(), std::move(new_headers)}))); + + ResponseDecoder* inner_decoder; + NiceMock inner_encoder; + EXPECT_CALL(*codec_, newStream(_)) + .WillOnce(Invoke([&](ResponseDecoder& decoder) -> RequestEncoder& { + inner_decoder = &decoder; + return inner_encoder; + })); + + Http::MockResponseDecoder outer_decoder; + Http::RequestEncoder& request_encoder = client_->newStream(outer_decoder); + + TestRequestHeaderMapImpl request_headers{ + {":authority", "host"}, {":path", "/"}, {":method", "GET"}}; + // Codec's encodeHeaders should observe header map modified by the transformRequestHeaders() + // method + EXPECT_CALL(inner_encoder, encodeHeaders(HeaderMapEqualRef(&expected_new_headers), _)); + + EXPECT_OK(request_encoder.encodeHeaders(request_headers, true)); + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + EXPECT_CALL(outer_decoder, decodeHeaders_(Pointee(Ref(*response_headers)), true)); + inner_decoder->decodeHeaders(std::move(response_headers), true); +} + +TEST_F(CodecClientTest, ResponseHeaderValidationFails) { + initialize(); + + ResponseDecoder* inner_decoder; + NiceMock inner_encoder; + EXPECT_CALL(*codec_, newStream(_)) + .WillOnce(Invoke([&](ResponseDecoder& decoder) -> RequestEncoder& { + inner_decoder = &decoder; + return inner_encoder; + })); + + Http::MockResponseDecoder outer_decoder; + Http::RequestEncoder& request_encoder = client_->newStream(outer_decoder); + + TestRequestHeaderMapImpl request_headers{ + {":authority", "host"}, {":path", "/"}, {":method", "GET"}}; + + EXPECT_OK(request_encoder.encodeHeaders(request_headers, true)); + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + EXPECT_CALL(*header_validator_, validateResponseHeaders(_)) + .WillOnce(Return(HeaderValidatorBase::ValidationResult{ + HeaderValidatorBase::ValidationResult::Action::Reject, "some error"})); + // Invalid response should cause stream reset + EXPECT_CALL(inner_encoder.stream_, resetStream(StreamResetReason::ProtocolError)); + inner_decoder->decodeHeaders(std::move(response_headers), true); +} + +TEST_F(CodecClientTest, ResponseHeaderTransformationFails) { + initialize(); + + ResponseDecoder* inner_decoder; + NiceMock inner_encoder; + EXPECT_CALL(*codec_, newStream(_)) + .WillOnce(Invoke([&](ResponseDecoder& decoder) -> RequestEncoder& { + inner_decoder = &decoder; + return inner_encoder; + })); + + Http::MockResponseDecoder outer_decoder; + Http::RequestEncoder& request_encoder = client_->newStream(outer_decoder); + + TestRequestHeaderMapImpl request_headers{ + {":authority", "host"}, {":path", "/"}, {":method", "GET"}}; + + EXPECT_OK(request_encoder.encodeHeaders(request_headers, true)); + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + EXPECT_CALL(*header_validator_, transformResponseHeaders(_)) + .WillOnce(Return(ClientHeaderValidator::TransformationResult{ + ClientHeaderValidator::TransformationResult::Action::Reject, "some error"})); + // Invalid transformation should cause stream reset + EXPECT_CALL(inner_encoder.stream_, resetStream(StreamResetReason::ProtocolError)); + inner_decoder->decodeHeaders(std::move(response_headers), true); +} + +TEST_F(CodecClientTest, ResponseHeaderValidationFailsWithConnectionClosure) { + initialize(); + EXPECT_CALL(*codec_, protocol()).WillRepeatedly(Return(Protocol::Http2)); + + ResponseDecoder* inner_decoder; + NiceMock inner_encoder; + EXPECT_CALL(*codec_, newStream(_)) + .WillOnce(Invoke([&](ResponseDecoder& decoder) -> RequestEncoder& { + inner_decoder = &decoder; + return inner_encoder; + })); + + Http::MockResponseDecoder outer_decoder; + Http::RequestEncoder& request_encoder = client_->newStream(outer_decoder); + + TestRequestHeaderMapImpl request_headers{ + {":authority", "host"}, {":path", "/"}, {":method", "GET"}}; + + EXPECT_OK(request_encoder.encodeHeaders(request_headers, true)); + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + EXPECT_CALL(*header_validator_, validateResponseHeaders(_)) + .WillOnce(Return(HeaderValidatorBase::ValidationResult{ + HeaderValidatorBase::ValidationResult::Action::Reject, "some error"})); + // By default H/2 and H/3 connections are disconnected on protocol errors + EXPECT_CALL(*connection_, close(_)); + inner_decoder->decodeHeaders(std::move(response_headers), true); + // Connection closure will cause stream to be reset + inner_encoder.stream_.callbacks_.front()->onResetStream(StreamResetReason::LocalReset, + "some error"); +} +#endif // ENVOY_ENABLE_UHV + // Test the codec getting input from a real TCP connection. class CodecNetworkTest : public Event::TestUsingSimulatedTime, public testing::TestWithParam { diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index 8f1139e44839..0b99d09ae01d 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -3317,7 +3317,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectHttp1) { setup(false, ""); expectUhvHeaderCheck(HeaderValidator::ValidationResult( HeaderValidator::ValidationResult::Action::Reject, "bad_header_map"), - HeaderValidator::HeadersTransformationResult::success()); + HeaderValidator::RequestHeadersTransformationResult::success()); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3363,7 +3363,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectHttp2) { setup(false, ""); expectUhvHeaderCheck(HeaderValidator::ValidationResult( HeaderValidator::ValidationResult::Action::Reject, "bad_header_map"), - HeaderValidator::HeadersTransformationResult::success()); + HeaderValidator::RequestHeadersTransformationResult::success()); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3392,7 +3392,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectGrpcRequest) { setup(false, ""); expectUhvHeaderCheck(HeaderValidator::ValidationResult( HeaderValidator::ValidationResult::Action::Reject, "bad_header_map"), - HeaderValidator::HeadersTransformationResult::success()); + HeaderValidator::RequestHeadersTransformationResult::success()); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3422,8 +3422,8 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRedirect) { setup(false, ""); expectUhvHeaderCheck( HeaderValidator::ValidationResult::success(), - HeaderValidator::HeadersTransformationResult( - HeaderValidator::HeadersTransformationResult::Action::Redirect, "bad_header_map")); + HeaderValidator::RequestHeadersTransformationResult( + HeaderValidator::RequestHeadersTransformationResult::Action::Redirect, "bad_header_map")); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3451,8 +3451,8 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRedirectGrpcRequest) { setup(false, ""); expectUhvHeaderCheck( HeaderValidator::ValidationResult::success(), - HeaderValidator::HeadersTransformationResult( - HeaderValidator::HeadersTransformationResult::Action::Redirect, "bad_header_map")); + HeaderValidator::RequestHeadersTransformationResult( + HeaderValidator::RequestHeadersTransformationResult::Action::Redirect, "bad_header_map")); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3483,7 +3483,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectTrailersBeforeRespons setup(false, ""); expectUhvTrailerCheck(HeaderValidator::ValidationResult( HeaderValidator::ValidationResult::Action::Reject, "bad_trailer_map"), - HeaderValidator::TrailersTransformationResult::success()); + HeaderValidator::TransformationResult::success()); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3512,7 +3512,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectTrailersBeforeRespons setup(false, ""); expectUhvTrailerCheck(HeaderValidator::ValidationResult( HeaderValidator::ValidationResult::Action::Reject, "bad_trailer_map"), - HeaderValidator::TrailersTransformationResult::success()); + HeaderValidator::TransformationResult::success(), false); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3535,9 +3535,9 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectTrailersBeforeRespons TEST_F(HttpConnectionManagerImplTest, HeaderValidatorFailTrailersTransformationBeforeResponse) { codec_->protocol_ = Protocol::Http11; setup(false, ""); - expectUhvTrailerCheck(HeaderValidator::TrailersTransformationResult::success(), - HeaderValidator::ValidationResult( - HeaderValidator::ValidationResult::Action::Reject, "bad_trailer_map")); + expectUhvTrailerCheck(HeaderValidator::ValidationResult( + HeaderValidator::ValidationResult::Action::Reject, "bad_trailer_map"), + HeaderValidator::TransformationResult::success()); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3568,7 +3568,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectTrailersAfterResponse setupFilterChain(1, 0, 1); expectUhvTrailerCheck(HeaderValidator::ValidationResult( HeaderValidator::ValidationResult::Action::Reject, "bad_trailer_map"), - HeaderValidator::TrailersTransformationResult::success()); + HeaderValidator::TransformationResult::success()); EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, false)) .WillRepeatedly(Invoke([&](RequestHeaderMap&, bool) -> FilterHeadersStatus { return FilterHeadersStatus::StopIteration; @@ -3606,7 +3606,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectTrailersAfterResponse TEST_F(HttpConnectionManagerImplTest, HeaderValidatorAccept) { setup(false, ""); expectUhvHeaderCheck(HeaderValidator::ValidationResult::success(), - HeaderValidator::HeadersTransformationResult::success()); + HeaderValidator::RequestHeadersTransformationResult::success()); // Store the basic request encoder during filter chain setup. std::shared_ptr filter(new NiceMock()); diff --git a/test/common/http/conn_manager_impl_test_base.cc b/test/common/http/conn_manager_impl_test_base.cc index 85d64bc679fe..bbbcd836b855 100644 --- a/test/common/http/conn_manager_impl_test_base.cc +++ b/test/common/http/conn_manager_impl_test_base.cc @@ -311,8 +311,8 @@ void HttpConnectionManagerImplMixin::testPathNormalization( void HttpConnectionManagerImplMixin::expectUhvHeaderCheck( HeaderValidator::ValidationResult validation_result, - HeaderValidator::HeadersTransformationResult transformation_result) { - EXPECT_CALL(header_validator_factory_, create(codec_->protocol_, _)) + HeaderValidator::RequestHeadersTransformationResult transformation_result) { + EXPECT_CALL(header_validator_factory_, createServerHeaderValidator(codec_->protocol_, _)) .WillOnce(InvokeWithoutArgs([validation_result, transformation_result]() { auto header_validator = std::make_unique>(); EXPECT_CALL(*header_validator, validateRequestHeaders(_)) @@ -322,29 +322,33 @@ void HttpConnectionManagerImplMixin::expectUhvHeaderCheck( EXPECT_CALL(*header_validator, transformRequestHeaders(_)) .WillOnce(Invoke([transformation_result](RequestHeaderMap& headers) { if (transformation_result.action() == - HeaderValidator::HeadersTransformationResult::Action::Redirect) { + HeaderValidator::RequestHeadersTransformationResult::Action::Redirect) { headers.setPath("/some/new/path"); } return transformation_result; })); } + EXPECT_CALL(*header_validator, transformResponseHeaders(_)) + .WillOnce(InvokeWithoutArgs( + []() { return HeaderValidator::ResponseHeadersTransformationResult::success(); })); + return header_validator; })); } void HttpConnectionManagerImplMixin::expectUhvTrailerCheck( HeaderValidator::ValidationResult validation_result, - HeaderValidator::TrailersTransformationResult transformation_result) { - EXPECT_CALL(header_validator_factory_, create(codec_->protocol_, _)) - .WillOnce(InvokeWithoutArgs([validation_result, transformation_result]() { + HeaderValidator::TransformationResult transformation_result, bool expect_response) { + EXPECT_CALL(header_validator_factory_, createServerHeaderValidator(codec_->protocol_, _)) + .WillOnce(InvokeWithoutArgs([validation_result, transformation_result, expect_response]() { auto header_validator = std::make_unique>(); EXPECT_CALL(*header_validator, validateRequestHeaders(_)).WillOnce(InvokeWithoutArgs([]() { return HeaderValidator::ValidationResult::success(); })); EXPECT_CALL(*header_validator, transformRequestHeaders(_)).WillOnce(InvokeWithoutArgs([]() { - return HeaderValidator::HeadersTransformationResult::success(); + return HeaderValidator::RequestHeadersTransformationResult::success(); })); EXPECT_CALL(*header_validator, validateRequestTrailers(_)) @@ -354,6 +358,12 @@ void HttpConnectionManagerImplMixin::expectUhvTrailerCheck( .WillOnce( InvokeWithoutArgs([transformation_result]() { return transformation_result; })); } + if (expect_response) { + EXPECT_CALL(*header_validator, transformResponseHeaders(_)) + .WillOnce(InvokeWithoutArgs([]() { + return HeaderValidator::ResponseHeadersTransformationResult::success(); + })); + } return header_validator; })); } diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index 1d07a304298a..f69ea07afbf3 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -160,7 +160,7 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { return proxy_status_config_.get(); } HeaderValidatorPtr makeHeaderValidator(Protocol protocol) override { - return header_validator_factory_.create(protocol, header_validator_stats_); + return header_validator_factory_.createServerHeaderValidator(protocol, header_validator_stats_); } bool appendXForwardedPort() const override { return false; } bool addProxyProtocolConnectionState() const override { @@ -186,10 +186,12 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { callbacks.addAccessLogHandler(handler); }; } - void expectUhvHeaderCheck(HeaderValidator::ValidationResult validation_result, - HeaderValidator::HeadersTransformationResult transformation_result); + void + expectUhvHeaderCheck(HeaderValidatorBase::ValidationResult validation_result, + HeaderValidator::RequestHeadersTransformationResult transformation_result); void expectUhvTrailerCheck(HeaderValidator::ValidationResult validation_result, - HeaderValidator::TrailersTransformationResult transformation_result); + HeaderValidator::TransformationResult transformation_result, + bool expect_response = true); Envoy::Event::SimulatedTimeSystem test_time_; NiceMock route_config_provider_; diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index d617d7573ef1..6a65b3499538 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -220,9 +220,9 @@ class Http1ServerConnectionImplTest : public Http1CodecTestBase { static_cast<::envoy::extensions::http::header_validators::envoy_default::v3:: HeaderValidatorConfig::HeadersWithUnderscoresAction>( headers_with_underscores_action_)); - header_validator_ = - std::make_unique( - header_validator_config_, Protocol::Http11, http1CodecStats()); + header_validator_ = std::make_unique< + Extensions::Http::HeaderValidators::EnvoyDefault::ServerHttp1HeaderValidator>( + header_validator_config_, Protocol::Http11, http1CodecStats()); } protected: diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 98b95632ebed..b610485321a3 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -405,9 +405,9 @@ class Http2CodecImplTestFixture { static_cast<::envoy::extensions::http::header_validators::envoy_default::v3:: HeaderValidatorConfig::HeadersWithUnderscoresAction>( headers_with_underscores_action_)); - header_validator_ = - std::make_unique( - header_validator_config_, Protocol::Http2, server_->http2CodecStats()); + header_validator_ = std::make_unique< + Extensions::Http::HeaderValidators::EnvoyDefault::ServerHttp2HeaderValidator>( + header_validator_config_, Protocol::Http2, server_->http2CodecStats()); request_decoder_.setHeaderValidator(header_validator_.get()); #endif } @@ -3633,6 +3633,10 @@ TEST_P(Http2CodecImplTest, ConnectTest) { client_http2_options_.set_allow_connect(true); server_http2_options_.set_allow_connect(true); initialize(); +#ifdef ENVOY_ENABLE_UHV + // TODO(#26490) : this test needs UHV for both client and server + return; +#endif MockStreamCallbacks callbacks; request_encoder_->getStream().addCallbacks(callbacks); @@ -4386,10 +4390,8 @@ TEST_P(Http2CodecImplTest, CheckHeaderValueValidation) { TEST_P(Http2CodecImplTest, BadResponseHeader) { initialize(); #ifdef ENVOY_ENABLE_UHV - // UHV mode makes no effect on nghttp2 - if (http2_implementation_ != Http2Impl::Oghttp2) { - return; - } + // TODO(#26490): this test needs UHV for both client and server codecs + return; #endif InSequence s; diff --git a/test/extensions/filters/http/compressor/compressor_integration_tests.cc b/test/extensions/filters/http/compressor/compressor_integration_tests.cc index 020a4af326f7..0434b028351d 100644 --- a/test/extensions/filters/http/compressor/compressor_integration_tests.cc +++ b/test/extensions/filters/http/compressor/compressor_integration_tests.cc @@ -196,11 +196,6 @@ void WebsocketWithCompressorIntegrationTest::sendBidirectionalData() { // Technically not a websocket tests, but verifies normal upgrades have parity // with websocket upgrades TEST_P(WebsocketWithCompressorIntegrationTest, NonWebsocketUpgrade) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23286) - add web socket support for H2 UHV - return; -#endif - config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 1f15f24b4692..1e895eb7871d 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -2964,7 +2964,7 @@ class TestHeaderValidatorFactoryConfig : public Http::HeaderValidatorFactoryConf Http::HeaderValidatorFactoryPtr createFromProto(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override { auto header_validator = std::make_unique>(); - EXPECT_CALL(*header_validator, create(Http::Protocol::Http2, _)) + EXPECT_CALL(*header_validator, createServerHeaderValidator(Http::Protocol::Http2, _)) .WillOnce(InvokeWithoutArgs( []() { return std::make_unique>(); })); return header_validator; @@ -2998,7 +2998,7 @@ class DefaultHeaderValidatorFactoryConfigOverride : public Http::HeaderValidator config_ = proto_config; auto header_validator = std::make_unique>(); - EXPECT_CALL(*header_validator, create(Http::Protocol::Http2, _)) + EXPECT_CALL(*header_validator, createServerHeaderValidator(Http::Protocol::Http2, _)) .WillOnce(InvokeWithoutArgs( []() { return std::make_unique>(); })); return header_validator; diff --git a/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc b/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc index 2ca0191208e3..33485c68845c 100644 --- a/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc +++ b/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc @@ -33,7 +33,21 @@ class HeaderValidatorFactoryTest : public testing::Test { TestUtility::loadFromYaml(std::string(config_yaml), typed_config); uhv_factory_ = factory->createFromProto(typed_config.typed_config(), validation_visitor_); - return uhv_factory_->create(protocol, stats_); + return uhv_factory_->createServerHeaderValidator(protocol, stats_); + } + + ::Envoy::Http::ClientHeaderValidatorPtr createClient(absl::string_view config_yaml, + Protocol protocol) { + auto* factory = + Registry::FactoryRegistry::getFactory( + "envoy.http.header_validators.envoy_default"); + ASSERT(factory != nullptr); + + envoy::config::core::v3::TypedExtensionConfig typed_config; + TestUtility::loadFromYaml(std::string(config_yaml), typed_config); + + uhv_factory_ = factory->createFromProto(typed_config.typed_config(), validation_visitor_); + return uhv_factory_->createClientHeaderValidator(protocol, stats_); } NiceMock validation_visitor_; @@ -49,22 +63,34 @@ class HeaderValidatorFactoryTest : public testing::Test { TEST_F(HeaderValidatorFactoryTest, CreateHttp09) { auto uhv = create(empty_config, Protocol::Http10); - EXPECT_NE(dynamic_cast(uhv.get()), nullptr); + EXPECT_NE(dynamic_cast(uhv.get()), nullptr); + + auto client_uhv = createClient(empty_config, Protocol::Http10); + EXPECT_NE(dynamic_cast(client_uhv.get()), nullptr); } TEST_F(HeaderValidatorFactoryTest, CreateHttp1) { auto uhv = create(empty_config, Protocol::Http11); - EXPECT_NE(dynamic_cast(uhv.get()), nullptr); + EXPECT_NE(dynamic_cast(uhv.get()), nullptr); + + auto client_uhv = createClient(empty_config, Protocol::Http11); + EXPECT_NE(dynamic_cast(client_uhv.get()), nullptr); } TEST_F(HeaderValidatorFactoryTest, CreateHttp2) { auto uhv = create(empty_config, Protocol::Http2); - EXPECT_NE(dynamic_cast(uhv.get()), nullptr); + EXPECT_NE(dynamic_cast(uhv.get()), nullptr); + + auto client_uhv = createClient(empty_config, Protocol::Http2); + EXPECT_NE(dynamic_cast(client_uhv.get()), nullptr); } TEST_F(HeaderValidatorFactoryTest, CreateHttp3) { auto uhv = create(empty_config, Protocol::Http3); - EXPECT_NE(dynamic_cast(uhv.get()), nullptr); + EXPECT_NE(dynamic_cast(uhv.get()), nullptr); + + auto client_uhv = createClient(empty_config, Protocol::Http3); + EXPECT_NE(dynamic_cast(client_uhv.get()), nullptr); } } // namespace diff --git a/test/extensions/http/header_validators/envoy_default/header_validator_test.cc b/test/extensions/http/header_validators/envoy_default/header_validator_test.cc index ef9bf83c7a6d..c0da2124c55f 100644 --- a/test/extensions/http/header_validators/envoy_default/header_validator_test.cc +++ b/test/extensions/http/header_validators/envoy_default/header_validator_test.cc @@ -14,58 +14,20 @@ namespace HeaderValidators { namespace EnvoyDefault { using ::Envoy::Http::HeaderString; -using ::Envoy::Http::HeaderValidatorStats; using ::Envoy::Http::Protocol; -using ::Envoy::Http::RequestHeaderMap; -using ::Envoy::Http::RequestTrailerMap; -using ::Envoy::Http::ResponseHeaderMap; -using ::Envoy::Http::ResponseTrailerMap; using ::Envoy::Http::testCharInTable; using ::Envoy::Http::UhvResponseCodeDetail; -class BaseHttpHeaderValidator : public HeaderValidator { -public: - BaseHttpHeaderValidator( - const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& - config, - Protocol protocol, HeaderValidatorStats& stats) - : HeaderValidator(config, protocol, stats) {} - - ValidationResult validateRequestHeaders(const RequestHeaderMap&) override { - return ValidationResult::success(); - } - - HeadersTransformationResult transformRequestHeaders(RequestHeaderMap&) override { - return HeadersTransformationResult::success(); - } - - ValidationResult validateResponseHeaders(const ResponseHeaderMap&) override { - return ValidationResult::success(); - } - - ValidationResult validateRequestTrailers(const RequestTrailerMap&) override { - return ValidationResult::success(); - } - - TrailersTransformationResult transformRequestTrailers(RequestTrailerMap&) override { - return TrailersTransformationResult::success(); - } - - ValidationResult validateResponseTrailers(const ResponseTrailerMap&) override { - return ValidationResult::success(); - } -}; - -using BaseHttpHeaderValidatorPtr = std::unique_ptr; +using HeaderValidatorPtr = std::unique_ptr; class BaseHeaderValidatorTest : public HeaderValidatorTest, public testing::Test { protected: - BaseHttpHeaderValidatorPtr createBase(absl::string_view config_yaml) { + HeaderValidatorPtr createBase(absl::string_view config_yaml) { envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig typed_config; TestUtility::loadFromYaml(std::string(config_yaml), typed_config); - return std::make_unique(typed_config, Protocol::Http11, stats_); + return std::make_unique(typed_config, Protocol::Http11, stats_); } }; diff --git a/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc b/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc index 3d9ab85b9386..9e0f19a27dd7 100644 --- a/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc @@ -23,12 +23,20 @@ using ::Envoy::Http::UhvResponseCodeDetail; class Http1HeaderValidatorTest : public HeaderValidatorTest, public testing::Test { protected: - Http1HeaderValidatorPtr createH1(absl::string_view config_yaml) { + ServerHttp1HeaderValidatorPtr createH1(absl::string_view config_yaml) { envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig typed_config; TestUtility::loadFromYaml(std::string(config_yaml), typed_config); - return std::make_unique(typed_config, Protocol::Http11, stats_); + return std::make_unique(typed_config, Protocol::Http11, stats_); + } + + ClientHttp1HeaderValidatorPtr createH1Client(absl::string_view config_yaml) { + envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig + typed_config; + TestUtility::loadFromYaml(std::string(config_yaml), typed_config); + + return std::make_unique(typed_config, Protocol::Http11, stats_); } TestRequestHeaderMapImpl makeGoodRequestHeaders() { @@ -465,17 +473,41 @@ TEST_F(Http1HeaderValidatorTest, RejectUnderscoreHeadersFromRequestHeadersWhenCo UhvResponseCodeDetail::get().InvalidUnderscore); } -TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapValid) { +TEST_F(Http1HeaderValidatorTest, TransformResponseHeadersServerCodecNoop) { ::Envoy::Http::TestResponseHeaderMapImpl headers{ {":status", "200"}, {"x-foo", "bar"}, {"transfer-encoding", "chunked"}}; auto uhv = createH1(empty_config); + auto result = uhv->transformResponseHeaders(headers); + EXPECT_ACCEPT(result.status); + EXPECT_EQ(result.new_headers, nullptr); +} + +TEST_F(Http1HeaderValidatorTest, TransformRequestHeadersClientCodecNoop) { + auto uhv = createH1Client(empty_config); + auto result = uhv->transformRequestHeaders(makeGoodRequestHeaders()); + EXPECT_ACCEPT(result.status); + EXPECT_EQ(result.new_headers, nullptr); +} + +TEST_F(Http1HeaderValidatorTest, TransformResponseHeadersClientCodecNoop) { + auto uhv = createH1Client(empty_config); + auto headers = makeGoodResponseHeaders(); + EXPECT_ACCEPT(uhv->transformResponseHeaders(headers)); + EXPECT_EQ(headers, TestResponseHeaderMapImpl({{":status", "200"}})); +} + +TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapValid) { + ::Envoy::Http::TestResponseHeaderMapImpl headers{ + {":status", "200"}, {"x-foo", "bar"}, {"transfer-encoding", "chunked"}}; + auto uhv = createH1Client(empty_config); + EXPECT_ACCEPT(uhv->validateResponseHeaders(headers)); } TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapMissingStatus) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{"x-foo", "bar"}}; - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), UhvResponseCodeDetail::get().InvalidStatus); @@ -483,7 +515,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapMissingStatus) { TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapInvalidStatus) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "bar"}, {"x-foo", "bar"}}; - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), UhvResponseCodeDetail::get().InvalidStatus); @@ -491,7 +523,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapInvalidStatus) { TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapExtraPseudoHeader) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "200"}, {":foo", "bar"}}; - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), UhvResponseCodeDetail::get().InvalidPseudoHeader); @@ -499,7 +531,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapExtraPseudoHeader) { TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapEmptyGenericName) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "200"}, {"", "bar"}}; - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), UhvResponseCodeDetail::get().EmptyHeaderName); @@ -508,7 +540,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapEmptyGenericName) { TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapInvaidTransferEncodingStatus100) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "100"}, {"transfer-encoding", "chunked"}}; - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), "uhv.http1.transfer_encoding_not_allowed"); @@ -517,7 +549,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapInvaidTransferEncoding TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapInvaidTransferEncodingStatus204) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "204"}, {"transfer-encoding", "chunked"}}; - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), "uhv.http1.transfer_encoding_not_allowed"); @@ -526,7 +558,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapInvaidTransferEncoding TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapInvaidTransferEncodingChars) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "200"}, {"transfer-encoding", "{chunked}"}}; - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), "uhv.http1.invalid_transfer_encoding"); @@ -551,9 +583,8 @@ TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderMapRejectPath) { auto uhv = createH1(empty_config); EXPECT_TRUE(uhv->validateRequestHeaders(headers).ok()); // Path normalization should fail - auto result = uhv->transformRequestHeaders(headers); - EXPECT_EQ(result.action(), HeaderValidator::RejectOrRedirectAction::Reject); - EXPECT_EQ(result.details(), UhvResponseCodeDetail::get().InvalidUrl); + EXPECT_REJECT_WITH_DETAILS(uhv->transformRequestHeaders(headers), + UhvResponseCodeDetail::get().InvalidUrl); } TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderMapRedirectPath) { @@ -564,7 +595,8 @@ TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderMapRedirectPath) { auto uhv = createH1(redirect_encoded_slash_config); EXPECT_TRUE(uhv->validateRequestHeaders(headers).ok()); auto result = uhv->transformRequestHeaders(headers); - EXPECT_EQ(result.action(), HeaderValidator::RejectOrRedirectAction::Redirect); + EXPECT_EQ(result.action(), + ::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult::Action::Redirect); EXPECT_EQ(result.details(), "uhv.path_noramlization_redirect"); EXPECT_EQ(headers.path(), "/dir1/dir2"); } @@ -587,9 +619,23 @@ TEST_F(Http1HeaderValidatorTest, ValidateRequestTrailerMap) { EXPECT_TRUE(uhv->validateRequestTrailers(request_trailer_map)); } +TEST_F(Http1HeaderValidatorTest, ValidateRequestTrailerMapClientCodec) { + auto uhv = createH1Client(empty_config); + ::Envoy::Http::TestRequestTrailerMapImpl trailer_map{{"trailer1", "value1"}, + {"trailer2", "values"}}; + EXPECT_TRUE(uhv->validateRequestTrailers(trailer_map)); +} + +TEST_F(Http1HeaderValidatorTest, ValidateResponseTrailerMapServerCodec) { + auto uhv = createH1(empty_config); + ::Envoy::Http::TestResponseTrailerMapImpl trailer_map{{"trailer1", "value1"}, + {"trailer2", "values"}}; + EXPECT_TRUE(uhv->validateResponseTrailers(trailer_map)); +} + TEST_F(Http1HeaderValidatorTest, ValidateInvalidRequestTrailerMap) { auto uhv = createH1(empty_config); - // H/2 trailers must not contain pseudo headers + // Trailers must not contain pseudo headers ::Envoy::Http::TestRequestTrailerMapImpl request_trailer_map{{":path", "value1"}, {"trailer2", "values"}}; auto result = uhv->validateRequestTrailers(request_trailer_map); @@ -597,6 +643,26 @@ TEST_F(Http1HeaderValidatorTest, ValidateInvalidRequestTrailerMap) { EXPECT_EQ(result.details(), "uhv.invalid_name_characters"); } +TEST_F(Http1HeaderValidatorTest, ValidateInvalidRequestTrailerMapClientCodec) { + auto uhv = createH1Client(empty_config); + // Trailers must not contain pseudo headers + ::Envoy::Http::TestRequestTrailerMapImpl request_trailer_map{{":path", "value1"}, + {"trailer2", "values"}}; + auto result = uhv->validateRequestTrailers(request_trailer_map); + EXPECT_FALSE(result); + EXPECT_EQ(result.details(), "uhv.invalid_name_characters"); +} + +TEST_F(Http1HeaderValidatorTest, ValidateInvalidResponseTrailerMapServerCodec) { + auto uhv = createH1(empty_config); + // Trailers must not contain pseudo headers + ::Envoy::Http::TestResponseTrailerMapImpl trailer_map{{":path", "value1"}, + {"trailer2", "values"}}; + auto result = uhv->validateResponseTrailers(trailer_map); + EXPECT_FALSE(result); + EXPECT_EQ(result.details(), "uhv.invalid_name_characters"); +} + TEST_F(Http1HeaderValidatorTest, ValidateInvalidValueRequestTrailerMap) { auto uhv = createH1(empty_config); ::Envoy::Http::TestRequestTrailerMapImpl request_trailer_map{{"trailer1", "value1"}, @@ -638,13 +704,13 @@ TEST_F(Http1HeaderValidatorTest, DropUnderscoreHeadersFromRequestTrailers) { } TEST_F(Http1HeaderValidatorTest, ValidateResponseTrailerMap) { - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); ::Envoy::Http::TestResponseTrailerMapImpl response_trailer_map{{"trailer1", "value1"}}; EXPECT_TRUE(uhv->validateResponseTrailers(response_trailer_map).ok()); } TEST_F(Http1HeaderValidatorTest, ValidateInvalidResponseTrailerMap) { - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); // H/2 trailers must not contain pseudo headers ::Envoy::Http::TestResponseTrailerMapImpl response_trailer_map{{":status", "200"}, {"trailer1", "value1"}}; @@ -654,7 +720,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateInvalidResponseTrailerMap) { } TEST_F(Http1HeaderValidatorTest, ValidateInvalidValueResponseTrailerMap) { - auto uhv = createH1(empty_config); + auto uhv = createH1Client(empty_config); // The DEL (0x7F) character is illegal in header values ::Envoy::Http::TestResponseTrailerMapImpl response_trailer_map{{"trailer0", "abcd\x7F\\ef"}, {"trailer1", "value1"}}; @@ -663,6 +729,27 @@ TEST_F(Http1HeaderValidatorTest, ValidateInvalidValueResponseTrailerMap) { EXPECT_EQ(result.details(), "uhv.invalid_value_characters"); } +TEST_F(Http1HeaderValidatorTest, InvalidRequestHeaderBeforeSendingUpstream) { + auto uhv = createH1Client(empty_config); + // The DEL (0x7F) character is illegal in header values + ::Envoy::Http::TestRequestHeaderMapImpl headers{ + {":scheme", "https"}, {":method", "GET"}, {":path", "/dir1%2fdir2"}, + {":authority", "envoy.com"}, {"header0", "abcd\x7F\\ef"}, {"header1", "value1"}}; + auto result = uhv->validateRequestHeaders(headers); + EXPECT_FALSE(result); + EXPECT_EQ(result.details(), "uhv.invalid_value_characters"); +} + +TEST_F(Http1HeaderValidatorTest, InvalidResponseHeaderBeforeSendingDownstream) { + auto uhv = createH1(empty_config); + // The DEL (0x7F) character is illegal in header values + ::Envoy::Http::TestResponseHeaderMapImpl headers{ + {":status", "200"}, {"header0", "abcd\x7F\\ef"}, {"header1", "value1"}}; + auto result = uhv->validateResponseHeaders(headers); + EXPECT_FALSE(result); + EXPECT_EQ(result.details(), "uhv.invalid_value_characters"); +} + TEST_F(Http1HeaderValidatorTest, BackslashInPathIsTranslatedToSlash) { scoped_runtime_.mergeValues( {{"envoy.reloadable_features.uhv_translate_backslash_to_slash", "true"}}); diff --git a/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc b/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc index d2fdc422f352..ca63ff94f2bd 100644 --- a/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc @@ -25,7 +25,23 @@ using ::Envoy::Http::UhvResponseCodeDetail; class Http2HeaderValidatorTest : public HeaderValidatorTest, public testing::Test { protected: - Http2HeaderValidatorPtr createH2(absl::string_view config_yaml) { + ::Envoy::Http::HeaderValidatorPtr createH2ServerUhv(absl::string_view config_yaml) { + envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig + typed_config; + TestUtility::loadFromYaml(std::string(config_yaml), typed_config); + + return std::make_unique(typed_config, Protocol::Http2, stats_); + } + + ::Envoy::Http::ClientHeaderValidatorPtr createH2ClientUhv(absl::string_view config_yaml) { + envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig + typed_config; + TestUtility::loadFromYaml(std::string(config_yaml), typed_config); + + return std::make_unique(typed_config, Protocol::Http2, stats_); + } + + std::unique_ptr createH2BaseUhv(absl::string_view config_yaml) { envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig typed_config; TestUtility::loadFromYaml(std::string(config_yaml), typed_config); @@ -46,9 +62,12 @@ class Http2HeaderValidatorTest : public HeaderValidatorTest, public testing::Tes }; TEST_F(Http2HeaderValidatorTest, GoodHeadersAccepted) { - auto uhv = createH2(empty_config); - EXPECT_ACCEPT(uhv->validateRequestHeaders(makeGoodRequestHeaders())); - EXPECT_ACCEPT(uhv->validateResponseHeaders(makeGoodResponseHeaders())); + auto server_uhv = createH2ServerUhv(empty_config); + auto client_uhv = createH2ClientUhv(empty_config); + EXPECT_ACCEPT(server_uhv->validateRequestHeaders(makeGoodRequestHeaders())); + EXPECT_ACCEPT(server_uhv->validateResponseHeaders(makeGoodResponseHeaders())); + EXPECT_ACCEPT(client_uhv->validateRequestHeaders(makeGoodRequestHeaders())); + EXPECT_ACCEPT(client_uhv->validateResponseHeaders(makeGoodResponseHeaders())); } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapAllowed) { @@ -57,7 +76,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapAllowed) { {":path", "/"}, {":authority", "envoy.com"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); } @@ -65,7 +84,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapAllowed) { TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapMissingPath) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":scheme", "https"}, {":method", "GET"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidUrl); @@ -74,7 +93,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapMissingPath) { TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapMissingMethod) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":scheme", "https"}, {":path", "/"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidMethod); @@ -83,7 +102,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapMissingMethod) { TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapMissingScheme) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":method", "GET"}, {":path", "/"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidScheme); @@ -92,7 +111,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapMissingScheme) { TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapExtraPseudoHeader) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":scheme", "https"}, {":method", "GET"}, {":path", "/"}, {":foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidPseudoHeader); @@ -101,7 +120,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapExtraPseudoHeader) { TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapNoAuthorityIsOk) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":scheme", "https"}, {":method", "GET"}, {":path", "/"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); } @@ -109,15 +128,20 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapNoAuthorityIsOk) { TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapConnect) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":method", "CONNECT"}, {":authority", "envoy.com"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); + EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); + + // Check that UHV does not modify plain vanilla CONNECT method + EXPECT_EQ(headers, ::Envoy::Http::TestRequestHeaderMapImpl( + {{":method", "CONNECT"}, {":authority", "envoy.com"}, {"x-foo", "bar"}})); } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapConnectExtraPseudoHeader) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":method", "CONNECT"}, {":scheme", "https"}, {":authority", "envoy.com"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidScheme); @@ -125,7 +149,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapConnectExtraPseudoHeade TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapConnectMissingAuthority) { ::Envoy::Http::TestRequestHeaderMapImpl headers{{":method", "CONNECT"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidHost); @@ -134,7 +158,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapConnectMissingAuthority TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapConnectWithPath) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":method", "CONNECT"}, {":authority", "envoy.com"}, {":path", "/bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidUrl); @@ -143,7 +167,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapConnectWithPath) { TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapConnectWithScheme) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":method", "CONNECT"}, {":authority", "envoy.com"}, {":scheme", "https"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidScheme); @@ -155,7 +179,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapOptionsAsterisk) { {":path", "*"}, {":authority", "envoy.com"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); } @@ -166,7 +190,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapNotOptionsAsterisk) { {":path", "*"}, {":authority", "envoy.com"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidUrl); @@ -178,7 +202,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapInvalidAuthority) { {":path", "/"}, {":authority", "user:pass@envoy.com"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidHostDeprecatedUserInfo); @@ -190,7 +214,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapEmptyGenericName) { {":path", "/"}, {":authority", "envoy.com"}, {"", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().EmptyHeaderName); @@ -202,7 +226,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapUnderscoreHeadersAllowe {":path", "/"}, {":authority", "envoy.com"}, {"x_foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); @@ -219,7 +243,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapDropUnderscoreHeaders) {":path", "/"}, {":authority", "envoy.com"}, {"x_foo", "bar"}}; - auto uhv = createH2(drop_headers_with_underscores_config); + auto uhv = createH2ServerUhv(drop_headers_with_underscores_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); // Headers with underscores are dropped by the transform method @@ -236,7 +260,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapRejectUnderscoreHeaders {":path", "/"}, {":authority", "envoy.com"}, {"x_foo", "bar"}}; - auto uhv = createH2(reject_headers_with_underscores_config); + auto uhv = createH2ServerUhv(reject_headers_with_underscores_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidUnderscore); @@ -246,8 +270,14 @@ TEST_F(Http2HeaderValidatorTest, RequestExtendedConnect) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":scheme", "https"}, {":method", "CONNECT"}, {":protocol", "websocket"}, {":path", "/foo/bar"}, {":authority", "envoy.com"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); + EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); + // Extended CONNECT is transformed to H/1 upgrade + EXPECT_EQ(headers.method(), "GET"); + EXPECT_EQ(headers.getUpgradeValue(), "websocket"); + EXPECT_EQ(headers.getConnectionValue(), "upgrade"); + EXPECT_EQ(headers.protocol(), ""); } TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoScheme) { @@ -255,9 +285,11 @@ TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoScheme) { {":protocol", "websocket"}, {":path", "/foo/bar"}, {":authority", "envoy.com"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidScheme); + // If validation failed, the extended CONNECT request should not be transformed to H/1 upgrade + EXPECT_EQ(headers.method(), "CONNECT"); } TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoPath) { @@ -265,7 +297,7 @@ TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoPath) { {":method", "CONNECT"}, {":protocol", "websocket"}, {":authority", "envoy.com"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidUrl); } @@ -277,7 +309,7 @@ TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectInvalidPath) { {":protocol", "websocket"}, {":path", "/fo\x7fo/bar"}, {":authority", "envoy.com"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidUrl); } @@ -288,11 +320,16 @@ TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectPathNormalization) { {":protocol", "websocket"}, {":path", "/./dir1/../dir2"}, {":authority", "envoy.com"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); EXPECT_EQ(headers.path(), "/dir2"); + // Expect transformation to H/1 upgrade as well + EXPECT_EQ(headers.method(), "GET"); + EXPECT_EQ(headers.getUpgradeValue(), "websocket"); + EXPECT_EQ(headers.getConnectionValue(), "upgrade"); + EXPECT_EQ(headers.protocol(), ""); } TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoAuthorityIsOk) { @@ -300,20 +337,75 @@ TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoAuthorityIsOk) { {":method", "CONNECT"}, {":path", "/foo/bar"}, {":protocol", "websocket"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); } +TEST_F(Http2HeaderValidatorTest, DownstreamUpgradeResponseTransformedToExtendedConnectResponse) { + ::Envoy::Http::TestResponseHeaderMapImpl headers{ + {":status", "101"}, {"connection", "keep-alive, upgrade"}, {"upgrade", "websocket"}}; + auto uhv = createH2ServerUhv(empty_config); + auto result = uhv->transformResponseHeaders(headers); + EXPECT_ACCEPT(result.status); + // Expect the extended CONNECT response + EXPECT_EQ(result.new_headers->getStatusValue(), "200"); + EXPECT_EQ(result.new_headers->Upgrade(), nullptr); + EXPECT_EQ(result.new_headers->Connection(), nullptr); + // New headers should be valid H/2 response headers + EXPECT_ACCEPT(uhv->validateResponseHeaders(*result.new_headers)); +} + +TEST_F(Http2HeaderValidatorTest, UpstreamUpgradeRequestTransformedToExtendedConnect) { + ::Envoy::Http::TestRequestHeaderMapImpl headers{ + {":scheme", "https"}, {":method", "GET"}, {":path", "/./dir1/../dir2"}, + {":authority", "envoy.com"}, {"upgrade", "websocket"}, {"connection", "upgrade,keep-alive"}}; + auto uhv = createH2ClientUhv(empty_config); + auto result = uhv->transformRequestHeaders(headers); + EXPECT_TRUE(result.status.ok()); + // Expect the extended CONNECT request + EXPECT_EQ(result.new_headers->method(), "CONNECT"); + EXPECT_EQ(result.new_headers->protocol(), "websocket"); + EXPECT_EQ(result.new_headers->Upgrade(), nullptr); + EXPECT_EQ(result.new_headers->Connection(), nullptr); + // No path normalization is expected at the client codec + EXPECT_EQ(result.new_headers->path(), "/./dir1/../dir2"); + EXPECT_EQ(result.new_headers->host(), "envoy.com"); + EXPECT_EQ(result.new_headers->getSchemeValue(), "https"); + + // New headers must be valid H/2 extended CONNECT headers + EXPECT_ACCEPT(uhv->validateRequestHeaders(*result.new_headers)); +} + +TEST_F(Http2HeaderValidatorTest, UpstreamExtendedConnectResponseTransformedToUpgradeResponse) { + // The response transformation is expected when the request was H/1 upgrade + ::Envoy::Http::TestRequestHeaderMapImpl headers{ + {":scheme", "https"}, {":method", "GET"}, {":path", "/./dir1/../dir2"}, + {":authority", "envoy.com"}, {"upgrade", "websocket"}, {"connection", "upgrade,keep-alive"}}; + auto uhv = createH2ClientUhv(empty_config); + auto result = uhv->transformRequestHeaders(headers); + EXPECT_TRUE(result.status.ok()); + // Make sure transformation produced valid H/2 extended CONNECT + EXPECT_ACCEPT(uhv->validateRequestHeaders(*result.new_headers)); + + ::Envoy::Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; + EXPECT_ACCEPT(uhv->validateResponseHeaders(response_headers)); + EXPECT_ACCEPT(uhv->transformResponseHeaders(response_headers)); + // Expect the upgrade response + EXPECT_EQ(response_headers.getStatusValue(), "101"); + EXPECT_EQ(response_headers.getUpgradeValue(), "websocket"); + EXPECT_EQ(response_headers.getConnectionValue(), "upgrade"); +} + TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderMapValid) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "200"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); EXPECT_ACCEPT(uhv->validateResponseHeaders(headers)); } TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderMapMissingStatus) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), UhvResponseCodeDetail::get().InvalidStatus); } @@ -321,28 +413,28 @@ TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderMapMissingStatus) { TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderMapExtraPseudoHeader) { ::Envoy::Http::TestResponseHeaderMapImpl headers{ {":status", "200"}, {":foo", "bar"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), UhvResponseCodeDetail::get().InvalidPseudoHeader); } TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderMapInvalidStatus) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "1024"}, {"x-foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), UhvResponseCodeDetail::get().InvalidStatus); } TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderMapEmptyGenericName) { ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "200"}, {"", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), UhvResponseCodeDetail::get().EmptyHeaderName); } TEST_F(Http2HeaderValidatorTest, ValidateRequestTrailersAllowUnderscoreHeadersByDefault) { TestRequestTrailerMapImpl trailers{{"trailer1", "value1"}, {"x_foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestTrailers(trailers)); EXPECT_ACCEPT(uhv->transformRequestTrailers(trailers)); @@ -352,7 +444,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestTrailersAllowUnderscoreHeadersBy TEST_F(Http2HeaderValidatorTest, ValidateGenericHeaderNameRejectConnectionHeaders) { std::string connection_headers[] = {"transfer-encoding", "connection", "keep-alive", "upgrade", "proxy-connection"}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); for (auto& header : connection_headers) { TestRequestHeaderMapImpl request_headers = makeGoodRequestHeaders(); @@ -363,7 +455,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateGenericHeaderNameRejectConnectionHeader } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderPath) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); TestRequestHeaderMapImpl request_headers = makeGoodRequestHeaders(); request_headers.setPath("/ bad path"); @@ -372,7 +464,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderPath) { } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderTETrailersAllowed) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); TestRequestHeaderMapImpl request_headers = makeGoodRequestHeaders(); request_headers.addCopy("te", "trailers"); @@ -380,7 +472,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderTETrailersAllowed) { } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderInvalidTERejected) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); TestRequestHeaderMapImpl request_headers = makeGoodRequestHeaders(); request_headers.addCopy("te", "chunked"); @@ -388,7 +480,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderInvalidTERejected) { } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderCustomMethod) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); TestRequestHeaderMapImpl request_headers = makeGoodRequestHeaders(); request_headers.setMethod("CUSTOM-METHOD"); @@ -396,7 +488,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderCustomMethod) { } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderContentLength) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); TestRequestHeaderMapImpl request_headers = makeGoodRequestHeaders(); request_headers.setContentLength("100"); EXPECT_ACCEPT(uhv->validateRequestHeaders(request_headers)); @@ -407,7 +499,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderContentLength) { } TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderContentLength) { - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); TestResponseHeaderMapImpl response_headers = makeGoodResponseHeaders(); response_headers.setContentLength("100"); EXPECT_ACCEPT(uhv->validateResponseHeaders(response_headers)); @@ -418,7 +510,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderContentLength) { } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderInvalidScheme) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); TestRequestHeaderMapImpl request_headers = makeGoodRequestHeaders(); request_headers.setScheme("http_ssh"); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(request_headers), @@ -426,7 +518,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderInvalidScheme) { } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderInvalidProtocol) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); TestRequestHeaderMapImpl headers{{":scheme", "https"}, {":method", "CONNECT"}, {":protocol", "something \x7F bad"}, @@ -438,7 +530,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderInvalidProtocol) { } TEST_F(Http2HeaderValidatorTest, InvalidRequestHeaderNameRejected) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); TestRequestHeaderMapImpl request_headers = makeGoodRequestHeaders(); // This header name is valid request_headers.addCopy("x-foo", "bar"); @@ -451,7 +543,7 @@ TEST_F(Http2HeaderValidatorTest, InvalidRequestHeaderNameRejected) { } TEST_F(Http2HeaderValidatorTest, InvalidRequestHeaderValueRejected) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); TestRequestHeaderMapImpl request_headers = makeGoodRequestHeaders(); HeaderString invalid_value{}; setHeaderStringUnvalidated(invalid_value, "hello\nworld"); @@ -461,7 +553,7 @@ TEST_F(Http2HeaderValidatorTest, InvalidRequestHeaderValueRejected) { } TEST_F(Http2HeaderValidatorTest, InvalidResponseHeaderNameRejected) { - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); TestResponseHeaderMapImpl response_headers = makeGoodResponseHeaders(); // This header name is valid response_headers.addCopy("x-foo", "bar"); @@ -474,8 +566,7 @@ TEST_F(Http2HeaderValidatorTest, InvalidResponseHeaderNameRejected) { } TEST_F(Http2HeaderValidatorTest, InvalidResponseHeaderValueRejected) { - - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); TestResponseHeaderMapImpl response_headers = makeGoodResponseHeaders(); HeaderString invalid_value{}; setHeaderStringUnvalidated(invalid_value, "hello\nworld"); @@ -485,7 +576,7 @@ TEST_F(Http2HeaderValidatorTest, InvalidResponseHeaderValueRejected) { } TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderInvalidStatusRejected) { - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); TestResponseHeaderMapImpl response_headers = makeGoodResponseHeaders(); response_headers.setStatus("1024"); @@ -494,7 +585,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateResponseHeaderInvalidStatusRejected) { } TEST_F(Http2HeaderValidatorTest, ValidateUppercaseRequestHeaderRejected) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); HeaderString invalid_name_uppercase; setHeaderStringUnvalidated(invalid_name_uppercase, "X-Foo"); @@ -507,7 +598,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateUppercaseRequestHeaderRejected) { } TEST_F(Http2HeaderValidatorTest, ValidateUppercaseResponseHeaderRejected) { - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); HeaderString invalid_name_uppercase; setHeaderStringUnvalidated(invalid_name_uppercase, "X-Foo"); @@ -520,7 +611,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateUppercaseResponseHeaderRejected) { } TEST_F(Http2HeaderValidatorTest, ValidateRequestGenericHeaderName) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); std::string name{"aaaaa"}; for (int i = 0; i < 0xff; ++i) { char c = static_cast(i); @@ -544,7 +635,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestGenericHeaderName) { } TEST_F(Http2HeaderValidatorTest, ValidateResponseGenericHeaderName) { - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); std::string name{"aaaaa"}; for (int i = 0; i < 0xff; ++i) { char c = static_cast(i); @@ -571,7 +662,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapNormalizePath) { {":method", "GET"}, {":path", "/./dir1/../dir2"}, {":authority", "envoy.com"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_TRUE(uhv->validateRequestHeaders(headers).ok()); EXPECT_TRUE(uhv->transformRequestHeaders(headers).ok()); @@ -581,12 +672,12 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapNormalizePath) { TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapRejectPath) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":scheme", "https"}, {":method", "GET"}, {":path", "/.."}, {":authority", "envoy.com"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); // Path normalization should fail due to /.. path - auto result = uhv->transformRequestHeaders(headers); - EXPECT_EQ(result.action(), HeaderValidator::RejectOrRedirectAction::Reject); - EXPECT_EQ(result.details(), UhvResponseCodeDetail::get().InvalidUrl); + EXPECT_REJECT_WITH_DETAILS(uhv->transformRequestHeaders(headers), + UhvResponseCodeDetail::get().InvalidUrl); + ; } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapRedirectPath) { @@ -594,11 +685,12 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapRedirectPath) { {":method", "GET"}, {":path", "/dir1%2fdir2"}, {":authority", "envoy.com"}}; - auto uhv = createH2(redirect_encoded_slash_config); + auto uhv = createH2ServerUhv(redirect_encoded_slash_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); // Path normalization should result in redirect auto result = uhv->transformRequestHeaders(headers); - EXPECT_EQ(result.action(), HeaderValidator::RejectOrRedirectAction::Redirect); + EXPECT_EQ(result.action(), + ::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult::Action::Redirect); EXPECT_EQ(result.details(), "uhv.path_noramlization_redirect"); EXPECT_EQ(headers.path(), "/dir1/dir2"); } @@ -608,7 +700,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeadersPathNormalizationDisabled {":method", "GET"}, {":path", "/./dir1%2f../dir2"}, {":authority", "envoy.com"}}; - auto uhv = createH2(no_path_normalization); + auto uhv = createH2ServerUhv(no_path_normalization); EXPECT_TRUE(uhv->validateRequestHeaders(headers).ok()); EXPECT_TRUE(uhv->transformRequestHeaders(headers).ok()); @@ -616,14 +708,24 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeadersPathNormalizationDisabled } TEST_F(Http2HeaderValidatorTest, ValidateRequestTrailerMap) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); ::Envoy::Http::TestRequestTrailerMapImpl request_trailer_map{{"trailer1", "value1"}, {"trailer2", "values"}}; EXPECT_TRUE(uhv->validateRequestTrailers(request_trailer_map)); } TEST_F(Http2HeaderValidatorTest, ValidateInvalidRequestTrailerMap) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); + // H/2 trailers must not contain pseudo headers + ::Envoy::Http::TestRequestTrailerMapImpl request_trailer_map{{":path", "value1"}, + {"trailer2", "values"}}; + auto result = uhv->validateRequestTrailers(request_trailer_map); + EXPECT_FALSE(result); + EXPECT_EQ(result.details(), "uhv.invalid_name_characters"); +} + +TEST_F(Http2HeaderValidatorTest, ValidateInvalidRequestTrailerMapClientCodec) { + auto uhv = createH2ClientUhv(empty_config); // H/2 trailers must not contain pseudo headers ::Envoy::Http::TestRequestTrailerMapImpl request_trailer_map{{":path", "value1"}, {"trailer2", "values"}}; @@ -633,7 +735,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateInvalidRequestTrailerMap) { } TEST_F(Http2HeaderValidatorTest, ValidateInvalidValueRequestTrailerMap) { - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); ::Envoy::Http::TestRequestTrailerMapImpl request_trailer_map{{"trailer1", "value1"}, {"trailer2", "values"}}; ::Envoy::Http::HeaderString invalid_value; @@ -647,7 +749,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateInvalidValueRequestTrailerMap) { TEST_F(Http2HeaderValidatorTest, UnderscoreHeadersAllowedInRequestTrailersByDefault) { ::Envoy::Http::TestRequestTrailerMapImpl trailers{{"trailer1", "value1"}, {"x_foo", "bar"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestTrailers(trailers)); EXPECT_ACCEPT(uhv->transformRequestTrailers(trailers)); @@ -657,7 +759,7 @@ TEST_F(Http2HeaderValidatorTest, UnderscoreHeadersAllowedInRequestTrailersByDefa TEST_F(Http2HeaderValidatorTest, ValidateRequestTrailersDropUnderscoreHeaders) { ::Envoy::Http::TestRequestTrailerMapImpl trailers{{"trailer1", "value1"}, {"x_foo", "bar"}}; - auto uhv = createH2(drop_headers_with_underscores_config); + auto uhv = createH2ServerUhv(drop_headers_with_underscores_config); EXPECT_ACCEPT(uhv->validateRequestTrailers(trailers)); EXPECT_ACCEPT(uhv->transformRequestTrailers(trailers)); @@ -666,20 +768,20 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestTrailersDropUnderscoreHeaders) { TEST_F(Http2HeaderValidatorTest, ValidateRequestTrailersRejectUnderscoreHeaders) { ::Envoy::Http::TestRequestTrailerMapImpl trailers{{"trailer1", "value1"}, {"x_foo", "bar"}}; - auto uhv = createH2(reject_headers_with_underscores_config); + auto uhv = createH2ServerUhv(reject_headers_with_underscores_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestTrailers(trailers), UhvResponseCodeDetail::get().InvalidUnderscore); } TEST_F(Http2HeaderValidatorTest, ValidateResponseTrailerMap) { - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); ::Envoy::Http::TestResponseTrailerMapImpl response_trailer_map{{"trailer1", "value1"}}; EXPECT_TRUE(uhv->validateResponseTrailers(response_trailer_map).ok()); } TEST_F(Http2HeaderValidatorTest, ValidateInvalidResponseTrailerMap) { - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); // H/2 trailers must not contain pseudo headers ::Envoy::Http::TestResponseTrailerMapImpl response_trailer_map{{":status", "200"}, {"trailer1", "value1"}}; @@ -689,7 +791,17 @@ TEST_F(Http2HeaderValidatorTest, ValidateInvalidResponseTrailerMap) { } TEST_F(Http2HeaderValidatorTest, ValidateInvalidValueResponseTrailerMap) { - auto uhv = createH2(empty_config); + auto uhv = createH2ClientUhv(empty_config); + // The DEL (0x7F) character is illegal in header values + ::Envoy::Http::TestResponseTrailerMapImpl response_trailer_map{{"trailer0", "abcd\x7F\\ef"}, + {"trailer1", "value1"}}; + auto result = uhv->validateResponseTrailers(response_trailer_map); + EXPECT_FALSE(result); + EXPECT_EQ(result.details(), "uhv.invalid_value_characters"); +} + +TEST_F(Http2HeaderValidatorTest, ValidateInvalidValueResponseTrailerMapServerCodec) { + auto uhv = createH2ServerUhv(empty_config); // The DEL (0x7F) character is illegal in header values ::Envoy::Http::TestResponseTrailerMapImpl response_trailer_map{{"trailer0", "abcd\x7F\\ef"}, {"trailer1", "value1"}}; @@ -705,7 +817,7 @@ TEST_F(Http2HeaderValidatorTest, BackslashInPathIsTranslatedToSlash) { {":path", "/path\\with/back\\/slash%5C"}, {":authority", "envoy.com"}, {":method", "GET"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); @@ -719,7 +831,7 @@ TEST_F(Http2HeaderValidatorTest, BackslashInPathIsRejectedWithOverride) { {":path", "/path\\with/back\\/slash%5c"}, {":authority", "envoy.com"}, {":method", "GET"}}; - auto uhv = createH2(empty_config); + auto uhv = createH2ServerUhv(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), "uhv.invalid_url"); } diff --git a/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc b/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc index d6002d05b554..33d384d0920c 100644 --- a/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc @@ -27,9 +27,9 @@ class HttpCommonValidationTest : public HeaderValidatorTest, TestUtility::loadFromYaml(std::string(config_yaml), typed_config); if (GetParam() == Protocol::Http11) { - return std::make_unique(typed_config, Protocol::Http11, stats_); + return std::make_unique(typed_config, Protocol::Http11, stats_); } - return std::make_unique(typed_config, GetParam(), stats_); + return std::make_unique(typed_config, GetParam(), stats_); } TestScopedRuntime scoped_runtime_; diff --git a/test/extensions/upstreams/http/config_test.cc b/test/extensions/upstreams/http/config_test.cc index 05217ded05d5..6d669bdd4fd5 100644 --- a/test/extensions/upstreams/http/config_test.cc +++ b/test/extensions/upstreams/http/config_test.cc @@ -97,9 +97,10 @@ class TestHeaderValidatorFactoryConfig : public ::Envoy::Http::HeaderValidatorFa createFromProto(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override { auto header_validator = std::make_unique>(); - EXPECT_CALL(*header_validator, create(::Envoy::Http::Protocol::Http2, _)) - .WillOnce(InvokeWithoutArgs( - []() { return std::make_unique>(); })); + EXPECT_CALL(*header_validator, createClientHeaderValidator(::Envoy::Http::Protocol::Http2, _)) + .WillOnce(InvokeWithoutArgs([]() { + return std::make_unique>(); + })); return header_validator; } }; @@ -132,9 +133,10 @@ class DefaultHeaderValidatorFactoryConfigOverride config_ = proto_config; auto header_validator = std::make_unique>(); - EXPECT_CALL(*header_validator, create(::Envoy::Http::Protocol::Http2, _)) - .WillOnce(InvokeWithoutArgs( - []() { return std::make_unique>(); })); + EXPECT_CALL(*header_validator, createClientHeaderValidator(::Envoy::Http::Protocol::Http2, _)) + .WillOnce(InvokeWithoutArgs([]() { + return std::make_unique>(); + })); return header_validator; } @@ -160,8 +162,8 @@ TEST_F(ConfigTest, HeaderValidatorConfig) { #ifdef ENVOY_ENABLE_UHV ProtocolOptionsConfigImpl config(options_, validation_visitor_); NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; - EXPECT_NE(nullptr, - config.header_validator_factory_->create(::Envoy::Http::Protocol::Http2, stats)); + EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( + ::Envoy::Http::Protocol::Http2, stats)); #else // If UHV is disabled, providing config should result in rejection EXPECT_THROW({ ProtocolOptionsConfigImpl config(options_, validation_visitor_); }, @@ -177,8 +179,8 @@ TEST_F(ConfigTest, DefaultHeaderValidatorConfig) { NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; ProtocolOptionsConfigImpl config(options_, validation_visitor_); #ifdef ENVOY_ENABLE_UHV - EXPECT_NE(nullptr, - config.header_validator_factory_->create(::Envoy::Http::Protocol::Http2, stats)); + EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( + ::Envoy::Http::Protocol::Http2, stats)); EXPECT_FALSE(proto_config.http1_protocol_options().allow_chunked_length()); #else // If UHV is disabled, config should be accepted and factory should be nullptr @@ -201,8 +203,8 @@ TEST_F(ConfigTest, TranslateDownstreamLegacyConfigToDefaultHeaderValidatorConfig NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; ProtocolOptionsConfigImpl config(options_, validation_visitor_); #ifdef ENVOY_ENABLE_UHV - EXPECT_NE(nullptr, - config.header_validator_factory_->create(::Envoy::Http::Protocol::Http2, stats)); + EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( + ::Envoy::Http::Protocol::Http2, stats)); EXPECT_TRUE(proto_config.http1_protocol_options().allow_chunked_length()); #else // If UHV is disabled, config should be accepted and factory should be nullptr @@ -225,8 +227,8 @@ TEST_F(ConfigTest, TranslateAutoLegacyConfigToDefaultHeaderValidatorConfig) { NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; ProtocolOptionsConfigImpl config(options_, validation_visitor_); #ifdef ENVOY_ENABLE_UHV - EXPECT_NE(nullptr, - config.header_validator_factory_->create(::Envoy::Http::Protocol::Http2, stats)); + EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( + ::Envoy::Http::Protocol::Http2, stats)); EXPECT_TRUE(proto_config.http1_protocol_options().allow_chunked_length()); #else // If UHV is disabled, config should be accepted and factory should be nullptr @@ -249,8 +251,8 @@ TEST_F(ConfigTest, TranslateExplicitLegacyConfigToDefaultHeaderValidatorConfig) NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; ProtocolOptionsConfigImpl config(options_, validation_visitor_); #ifdef ENVOY_ENABLE_UHV - EXPECT_NE(nullptr, - config.header_validator_factory_->create(::Envoy::Http::Protocol::Http2, stats)); + EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( + ::Envoy::Http::Protocol::Http2, stats)); EXPECT_TRUE(proto_config.http1_protocol_options().allow_chunked_length()); #else // If UHV is disabled, config should be accepted and factory should be nullptr @@ -273,8 +275,8 @@ TEST_F(ConfigTest, TranslateExplicitH2LegacyConfigToDefaultHeaderValidatorConfig NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; ProtocolOptionsConfigImpl config(options_, validation_visitor_); #ifdef ENVOY_ENABLE_UHV - EXPECT_NE(nullptr, - config.header_validator_factory_->create(::Envoy::Http::Protocol::Http2, stats)); + EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( + ::Envoy::Http::Protocol::Http2, stats)); EXPECT_FALSE(proto_config.http1_protocol_options().allow_chunked_length()); #else // If UHV is disabled, config should be accepted and factory should be nullptr diff --git a/test/integration/BUILD b/test/integration/BUILD index a753f1d41ddd..67aad40e1f6c 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -894,6 +894,7 @@ envoy_cc_test_library( "fake_upstream.h", ], deps = [ + ":utility_lib", "//test/mocks/runtime:runtime_mocks", "//test/mocks/protobuf:protobuf_mocks", "//envoy/api:api_interface", @@ -926,6 +927,7 @@ envoy_cc_test_library( "//source/common/stats:isolated_store_lib", "//source/extensions/listener_managers/listener_manager:active_raw_udp_listener_config", "//source/extensions/listener_managers/listener_manager:connection_handler_lib", + "//test/mocks/http:header_validator_mocks", "//test/test_common:network_utility_lib", "//test/test_common:test_time_system_interface", "//test/test_common:utility_lib", @@ -1044,6 +1046,7 @@ envoy_cc_test_library( "//source/server:server_lib", "//test/mocks:common_lib", "//test/mocks/event:event_mocks", + "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/server:transport_socket_factory_context_mocks", "//test/mocks/upstream:cluster_info_mocks", diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 476040508af0..82f38fd2f591 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -24,6 +24,7 @@ #include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" +#include "test/integration/utility.h" #include "test/test_common/network_utility.h" #include "test/test_common/utility.h" @@ -41,13 +42,17 @@ namespace Envoy { FakeStream::FakeStream(FakeHttpConnection& parent, Http::ResponseEncoder& encoder, Event::TestTimeSystem& time_system) - : parent_(parent), encoder_(encoder), time_system_(time_system) { + : parent_(parent), encoder_(encoder), time_system_(time_system), + header_validator_(parent.makeHeaderValidator()) { encoder.getStream().addCallbacks(*this); } void FakeStream::decodeHeaders(Http::RequestHeaderMapSharedPtr&& headers, bool end_stream) { absl::MutexLock lock(&lock_); headers_ = std::move(headers); + if (header_validator_) { + header_validator_->transformRequestHeaders(*headers_); + } setEndStream(end_stream); } @@ -98,6 +103,14 @@ void FakeStream::encodeHeaders(const Http::HeaderMap& headers, bool end_stream) parent_.connection().connectionInfoProvider().localAddress()->asString()); } + if (header_validator_) { + // Ignore validation results + auto result = header_validator_->transformResponseHeaders(*headers_copy); + if (result.new_headers) { + headers_copy = std::move(result.new_headers); + } + } + postToConnectionThread([this, headers_copy, end_stream]() -> void { { absl::MutexLock lock(&lock_); @@ -342,13 +355,28 @@ class TestHttp2ServerConnectionImpl : public Http::Http2::ServerConnectionImpl { } }; +namespace { +// Fake upstream codec will not do path normalization, so the tests can observe +// the path forwarded by Envoy. +constexpr absl::string_view fake_upstream_header_validator_config = R"EOF( + name: envoy.http.header_validators.envoy_default + typed_config: + "@type": type.googleapis.com/envoy.extensions.http.header_validators.envoy_default.v3.HeaderValidatorConfig + uri_path_normalization_options: + skip_path_normalization: true + skip_merging_slashes: true +)EOF"; +} // namespace + FakeHttpConnection::FakeHttpConnection( FakeUpstream& fake_upstream, SharedConnectionWrapper& shared_connection, Http::CodecType type, Event::TestTimeSystem& time_system, uint32_t max_request_headers_kb, uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headers_with_underscores_action) - : FakeConnectionBase(shared_connection, time_system), type_(type) { + : FakeConnectionBase(shared_connection, time_system), type_(type), + header_validator_factory_( + IntegrationUtil::makeHeaderValidationFactory(fake_upstream_header_validator_config)) { ASSERT(max_request_headers_count != 0); if (type == Http::CodecType::HTTP1) { Http::Http1Settings http1_settings; @@ -399,6 +427,26 @@ AssertionResult FakeConnectionBase::readDisable(bool disable, std::chrono::milli [disable](Network::Connection& connection) { connection.readDisable(disable); }, timeout); } +namespace { +Http::Protocol codeTypeToProtocol(Http::CodecType codec_type) { + switch (codec_type) { + case Http::CodecType::HTTP1: + return Http::Protocol::Http11; + case Http::CodecType::HTTP2: + return Http::Protocol::Http2; + case Http::CodecType::HTTP3: + return Http::Protocol::Http3; + } + PANIC_DUE_TO_CORRUPT_ENUM; +} +} // namespace + +Http::HeaderValidatorPtr FakeHttpConnection::makeHeaderValidator() { + return header_validator_factory_ ? header_validator_factory_->createServerHeaderValidator( + codeTypeToProtocol(type_), header_validator_stats_) + : nullptr; +} + Http::RequestDecoder& FakeHttpConnection::newStream(Http::ResponseEncoder& encoder, bool) { absl::MutexLock lock(&lock_); new_streams_.emplace_back(new FakeStream(*this, encoder, time_system_)); diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index d92f1ffd9e2d..adc73410a10b 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -36,6 +36,7 @@ #include "source/common/network/udp_packet_writer_handler_impl.h" #include "source/common/stats/isolated_store_impl.h" +#include "test/mocks/http/header_validator.h" #include "test/mocks/protobuf/mocks.h" #if defined(ENVOY_ENABLE_QUIC) @@ -265,6 +266,7 @@ class FakeStream : public Http::RequestDecoder, std::list access_log_handlers_; bool received_data_{false}; bool grpc_stream_started_{false}; + Http::HeaderValidatorPtr header_validator_; }; using FakeStreamPtr = std::unique_ptr; @@ -493,6 +495,8 @@ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeCo void writeRawData(absl::string_view data); ABSL_MUST_USE_RESULT AssertionResult postWriteRawData(std::string data); + Http::HeaderValidatorPtr makeHeaderValidator(); + private: struct ReadFilter : public Network::ReadFilterBaseImpl { ReadFilter(FakeHttpConnection& parent) : parent_(parent) {} @@ -524,6 +528,8 @@ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeCo Http::ServerConnectionPtr codec_; std::list new_streams_ ABSL_GUARDED_BY(lock_); testing::NiceMock random_; + testing::NiceMock header_validator_stats_; + Http::HeaderValidatorFactoryPtr header_validator_factory_; }; using FakeHttpConnectionPtr = std::unique_ptr; diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 085a95ca7ee4..8436ab90f0ab 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -283,6 +283,16 @@ IntegrationCodecClientPtr HttpIntegrationTest::makeRawHttpConnection( } cluster->http2_options_ = http2_options.value(); cluster->http1_settings_.enable_trailers_ = true; + + static constexpr absl::string_view empty_header_validator_config = R"EOF( + name: envoy.http.header_validators.envoy_default + typed_config: + "@type": type.googleapis.com/envoy.extensions.http.header_validators.envoy_default.v3.HeaderValidatorConfig +)EOF"; + + cluster->header_validator_factory_ = + IntegrationUtil::makeHeaderValidationFactory(empty_header_validator_config); + Upstream::HostDescriptionConstSharedPtr host_description{Upstream::makeTestHostDescription( cluster, fmt::format("tcp://{}:80", Network::Test::getLoopbackAddressUrlString(version_)), timeSystem())}; diff --git a/test/integration/tcp_tunneling_integration_test.cc b/test/integration/tcp_tunneling_integration_test.cc index 8867621bbd69..b824696f3ba5 100644 --- a/test/integration/tcp_tunneling_integration_test.cc +++ b/test/integration/tcp_tunneling_integration_test.cc @@ -126,11 +126,6 @@ class ConnectTerminationIntegrationTest : public HttpProtocolIntegrationTest { // Verify that H/2 extended CONNECT with bytestream protocol is treated like // standard CONNECT request TEST_P(ConnectTerminationIntegrationTest, ExtendedConnectWithBytestreamProtocol) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#24945): This test needs CONNECT/upgrade normalization code which is not yet - // available in UHV. - return; -#endif if (downstream_protocol_ == Http::CodecType::HTTP1) { // Extended CONNECT is applicable to H/2 and H/3 protocols only return; @@ -530,11 +525,6 @@ TEST_P(ProxyingConnectIntegrationTest, ProxyConnect) { } TEST_P(ProxyingConnectIntegrationTest, ProxyExtendedConnect) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#24945): This test needs CONNECT/upgrade normalization code which is not yet - // available in UHV. - return; -#endif add_upgrade_config_ = true; initialize(); diff --git a/test/integration/utility.cc b/test/integration/utility.cc index 56a8105d582c..c03979951245 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -33,6 +33,7 @@ #include "test/integration/ssl_utility.h" #endif #include "test/mocks/common.h" +#include "test/mocks/protobuf/mocks.h" #include "test/mocks/server/transport_socket_factory_context.h" #include "test/mocks/stats/mocks.h" #include "test/mocks/upstream/cluster_info.h" @@ -317,6 +318,24 @@ RawConnectionDriver::RawConnectionDriver(uint32_t port, DoWriteCallback write_re client_->connect(); } +Http::HeaderValidatorFactoryPtr +IntegrationUtil::makeHeaderValidationFactory([[maybe_unused]] absl::string_view config) { +#ifdef ENVOY_ENABLE_UHV + auto* factory = Registry::FactoryRegistry::getFactory( + "envoy.http.header_validators.envoy_default"); + ASSERT(factory != nullptr); + + envoy::config::core::v3::TypedExtensionConfig typed_config; + Thread::SkipAsserts no_main_thread_asserts_in_yaml_parser; + TestUtility::loadFromYaml(std::string(config), typed_config); + + NiceMock validation_visitor; + return factory->createFromProto(typed_config.typed_config(), validation_visitor); +#else + return nullptr; +#endif +} + RawConnectionDriver::~RawConnectionDriver() = default; testing::AssertionResult RawConnectionDriver::waitForConnection() { diff --git a/test/integration/utility.h b/test/integration/utility.h index ea4b5175dbba..f91e2407a6b0 100644 --- a/test/integration/utility.h +++ b/test/integration/utility.h @@ -208,6 +208,8 @@ class IntegrationUtil { createQuicUpstreamTransportSocketFactory(Api::Api& api, Stats::Store& store, Ssl::ContextManager& context_manager, const std::string& san_to_match); + + static Http::HeaderValidatorFactoryPtr makeHeaderValidationFactory(absl::string_view config); }; // A set of connection callbacks which tracks connection state. diff --git a/test/integration/websocket_integration_test.cc b/test/integration/websocket_integration_test.cc index 595c48bab426..66818889e2db 100644 --- a/test/integration/websocket_integration_test.cc +++ b/test/integration/websocket_integration_test.cc @@ -197,11 +197,6 @@ void WebsocketIntegrationTest::sendBidirectionalData() { } TEST_P(WebsocketIntegrationTest, WebSocketConnectionDownstreamDisconnect) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23286) - add web socket support for H2 UHV - return; -#endif - config_helper_.addConfigModifier(setRouteUsingWebsocket()); initialize(); @@ -225,11 +220,6 @@ TEST_P(WebsocketIntegrationTest, PortStrippingForHttp2) { return; } -#ifdef ENVOY_ENABLE_UHV - // TODO(#23286) - add web socket support for H2 UHV - return; -#endif - config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { hcm.set_strip_any_host_port(true); }); @@ -295,11 +285,6 @@ TEST_P(WebsocketIntegrationTest, EarlyData) { } TEST_P(WebsocketIntegrationTest, WebSocketConnectionIdleTimeout) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23286) - add web socket support for H2 UHV - return; -#endif - config_helper_.addConfigModifier(setRouteUsingWebsocket()); config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& @@ -324,11 +309,6 @@ TEST_P(WebsocketIntegrationTest, WebSocketConnectionIdleTimeout) { // Technically not a websocket tests, but verifies normal upgrades have parity // with websocket upgrades TEST_P(WebsocketIntegrationTest, NonWebsocketUpgrade) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23286) - add web socket support for H2 UHV - return; -#endif - config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { @@ -357,11 +337,6 @@ TEST_P(WebsocketIntegrationTest, NonWebsocketUpgrade) { } TEST_P(WebsocketIntegrationTest, RouteSpecificUpgrade) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23286) - add web socket support for H2 UHV - return; -#endif - config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { @@ -398,11 +373,6 @@ TEST_P(WebsocketIntegrationTest, WebsocketCustomFilterChain) { return; } -#ifdef ENVOY_ENABLE_UHV - // TODO(#23286) - add web socket support for H2 UHV - return; -#endif - config_helper_.addConfigModifier(setRouteUsingWebsocket()); // Add a small buffer filter to the standard HTTP filter chain. Websocket diff --git a/test/mocks/http/header_validator.h b/test/mocks/http/header_validator.h index f759b7ed57bd..2162e8b6d0ac 100644 --- a/test/mocks/http/header_validator.h +++ b/test/mocks/http/header_validator.h @@ -11,13 +11,26 @@ class MockHeaderValidator : public HeaderValidator { public: ~MockHeaderValidator() override = default; MOCK_METHOD(ValidationResult, validateRequestHeaders, (const RequestHeaderMap& header_map)); - MOCK_METHOD(HeadersTransformationResult, transformRequestHeaders, + MOCK_METHOD(ValidationResult, validateResponseHeaders, (const ResponseHeaderMap& header_map)); + MOCK_METHOD(ValidationResult, validateRequestTrailers, (const RequestTrailerMap& header_map)); + MOCK_METHOD(ValidationResult, validateResponseTrailers, (const ResponseTrailerMap& header_map)); + MOCK_METHOD(RequestHeadersTransformationResult, transformRequestHeaders, (RequestHeaderMap & header_map)); + MOCK_METHOD(ResponseHeadersTransformationResult, transformResponseHeaders, + (const ResponseHeaderMap& header_map)); + MOCK_METHOD(TransformationResult, transformRequestTrailers, (RequestTrailerMap & trailer_map)); +}; + +class MockClientHeaderValidator : public ClientHeaderValidator { +public: + ~MockClientHeaderValidator() override = default; + MOCK_METHOD(ValidationResult, validateRequestHeaders, (const RequestHeaderMap& header_map)); MOCK_METHOD(ValidationResult, validateResponseHeaders, (const ResponseHeaderMap& header_map)); - MOCK_METHOD(ValidationResult, validateRequestTrailers, (const RequestTrailerMap& trailer_map)); - MOCK_METHOD(TrailersTransformationResult, transformRequestTrailers, - (RequestTrailerMap & trailer_map)); - MOCK_METHOD(ValidationResult, validateResponseTrailers, (const ResponseTrailerMap& trailer_map)); + MOCK_METHOD(ValidationResult, validateRequestTrailers, (const RequestTrailerMap& header_map)); + MOCK_METHOD(ValidationResult, validateResponseTrailers, (const ResponseTrailerMap& header_map)); + MOCK_METHOD(RequestHeadersTransformationResult, transformRequestHeaders, + (const RequestHeaderMap& header_map)); + MOCK_METHOD(TransformationResult, transformResponseHeaders, (ResponseHeaderMap & header_map)); }; class MockHeaderValidatorStats : public HeaderValidatorStats { @@ -29,7 +42,10 @@ class MockHeaderValidatorStats : public HeaderValidatorStats { class MockHeaderValidatorFactory : public HeaderValidatorFactory { public: - MOCK_METHOD(HeaderValidatorPtr, create, (Protocol protocol, HeaderValidatorStats& stats)); + MOCK_METHOD(HeaderValidatorPtr, createServerHeaderValidator, + (Protocol protocol, HeaderValidatorStats& stats)); + MOCK_METHOD(ClientHeaderValidatorPtr, createClientHeaderValidator, + (Protocol protocol, HeaderValidatorStats& stats)); }; } // namespace Http diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc index ccaa69703c92..3262123813b9 100644 --- a/test/mocks/upstream/cluster_info.cc +++ b/test/mocks/upstream/cluster_info.cc @@ -198,10 +198,28 @@ MockClusterInfo::MockClusterInfo() manager.applyFilterFactoryCb({}, factory_cb); return true; })); + ON_CALL(*this, makeHeaderValidator(_)).WillByDefault(Invoke([&](Http::Protocol protocol) { + return header_validator_factory_ ? header_validator_factory_->createClientHeaderValidator( + protocol, codecStats(protocol)) + : nullptr; + })); } MockClusterInfo::~MockClusterInfo() = default; +::Envoy::Http::HeaderValidatorStats& MockClusterInfo::codecStats(Http::Protocol protocol) const { + switch (protocol) { + case ::Envoy::Http::Protocol::Http10: + case ::Envoy::Http::Protocol::Http11: + return http1CodecStats(); + case ::Envoy::Http::Protocol::Http2: + return http2CodecStats(); + case ::Envoy::Http::Protocol::Http3: + return http3CodecStats(); + } + PANIC_DUE_TO_CORRUPT_ENUM; +} + Http::Http1::CodecStats& MockClusterInfo::http1CodecStats() const { return Http::Http1::CodecStats::atomicGet(http1_codec_stats_, *stats_scope_); } diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index 08f3aac22eae..bbcbde483319 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -186,8 +186,9 @@ class MockClusterInfo : public ClusterInfo { const Http::FilterChainFactory::UpgradeMap* upgrade_map, Http::FilterChainManager& manager), (const)); - MOCK_METHOD(Http::HeaderValidatorPtr, makeHeaderValidator, (Http::Protocol), (const)); + MOCK_METHOD(Http::ClientHeaderValidatorPtr, makeHeaderValidator, (Http::Protocol), (const)); + ::Envoy::Http::HeaderValidatorStats& codecStats(Http::Protocol protocol) const; Http::Http1::CodecStats& http1CodecStats() const override; Http::Http2::CodecStats& http2CodecStats() const override; Http::Http3::CodecStats& http3CodecStats() const override; @@ -252,6 +253,7 @@ class MockClusterInfo : public ClusterInfo { mutable Http::Http1::CodecStats::AtomicPtr http1_codec_stats_; mutable Http::Http2::CodecStats::AtomicPtr http2_codec_stats_; mutable Http::Http3::CodecStats::AtomicPtr http3_codec_stats_; + Http::HeaderValidatorFactoryPtr header_validator_factory_; }; class MockIdleTimeEnabledClusterInfo : public MockClusterInfo { From 228ea5a164bf4a119a16c2c5fb4a01f41065a3e1 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Thu, 27 Apr 2023 23:05:51 -0400 Subject: [PATCH 025/740] Update owner file (#27026) ext_proc is now heavily used in our projects. Update the file per offline conversation, @pradeepcrao is not working closely on any projects in this domain. Also, update ext_authz as discussed offline. Signed-off-by: tyxia --- CODEOWNERS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 195ef7302781..e4663e4a2f8a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -23,8 +23,8 @@ extensions/filters/common/original_src @snowp @klarose # cdn_loop extension /*/extensions/filters/http/cdn_loop @justin-mp @penguingao @alyssawilk # external processing filter -/*/extensions/filters/http/ext_proc @gbrail @snowp @pradeepcrao @stevenzzzz -/*/extensions/filters/common/mutation_rules @gbrail @snowp @pradeepcrao @chaoqin-li1123 +/*/extensions/filters/http/ext_proc @gbrail @snowp @stevenzzzz @tyxia +/*/extensions/filters/common/mutation_rules @gbrail @snowp @chaoqin-li1123 @tyxia # jwt_authn http filter extension /*/extensions/filters/http/jwt_authn @qiwzhang @lizan # grpc_http1_reverse_bridge http filter extension @@ -257,9 +257,9 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/tracers/common @wbpcode @Shikugawa @basvanbeek /*/extensions/tracers/common/ot @wbpcode @Shikugawa @basvanbeek # ext_authz -/*/extensions/filters/common/ext_authz @esmet @pradeepcrao @ggreenway -/*/extensions/filters/http/ext_authz @esmet @pradeepcrao @ggreenway -/*/extensions/filters/network/ext_authz @esmet @pradeepcrao @ggreenway +/*/extensions/filters/common/ext_authz @esmet @tyxia @ggreenway +/*/extensions/filters/http/ext_authz @esmet @tyxia @ggreenway +/*/extensions/filters/network/ext_authz @esmet @tyxia @ggreenway # original dst /*/extensions/filters/listener/original_dst @kyessenov @lizan # mongo proxy From dbec6f88d6d814829f1988d92e888cdc98c34fb1 Mon Sep 17 00:00:00 2001 From: LeonGao Date: Thu, 27 Apr 2023 20:31:35 -0700 Subject: [PATCH 026/740] redis filter: Add rate limit for redis client reconnection (#26502) Adding a rate limiting for redis client reconnection rate to avoid reconnection storm on redis server Risk Level: Low Testing: Unit test, Manual test with redis Fixes #26400 Signed-off-by: lgao --- .../network/redis_proxy/v3/redis_proxy.proto | 14 ++- changelogs/current.yaml | 6 ++ .../extensions/clusters/redis/redis_cluster.h | 2 + .../filters/network/common/redis/client.h | 3 + .../network/common/redis/client_impl.cc | 8 ++ .../network/common/redis/client_impl.h | 4 + .../network/redis_proxy/conn_pool_impl.cc | 45 +++++++-- .../network/redis_proxy/conn_pool_impl.h | 5 +- .../extensions/health_checkers/redis/redis.h | 2 + .../network/common/redis/client_impl_test.cc | 6 ++ .../filters/network/common/redis/test_utils.h | 7 +- .../redis_proxy/conn_pool_impl_test.cc | 93 ++++++++++++++++++- 12 files changed, 181 insertions(+), 14 deletions(-) diff --git a/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto b/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto index 47c211a0f5c0..3a64c9cc8665 100644 --- a/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto +++ b/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto @@ -31,7 +31,7 @@ message RedisProxy { "envoy.config.filter.network.redis_proxy.v2.RedisProxy"; // Redis connection pool settings. - // [#next-free-field: 10] + // [#next-free-field: 11] message ConnPoolSettings { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.redis_proxy.v2.RedisProxy.ConnPoolSettings"; @@ -127,6 +127,11 @@ message RedisProxy { // Read policy. The default is to read from the primary. ReadPolicy read_policy = 7 [(validate.rules).enum = {defined_only: true}]; + + // Ops or connection timeout triggers reconnection to redis server which could result in reconnection + // storm to busy redis server. This config is a protection to rate limit reconnection rate. + // If not set, there will be no rate limiting on the reconnection. + ConnectionRateLimit connection_rate_limit = 10; } message PrefixRoutes { @@ -221,6 +226,13 @@ message RedisProxy { repeated string commands = 4; } + // Configuration to limit reconnection rate to redis server to protect redis server + // from client reconnection storm. + message ConnectionRateLimit { + // Reconnection rate per sec. Rate limiting is implemented with TokenBucket. + uint32 connection_rate_limit_per_sec = 1; + } + reserved 2; reserved "cluster"; diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 83dfbe937311..2e558dab801d 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -82,6 +82,12 @@ new_features: added new configuration field :ref:`domain ` to allow for setting rate limit domains on a per-route basis. +- area: redis_proxy + change: | + added new field :ref:`connection_rate_limit + ` + to limit reconnection rate to redis server to avoid reconnection storm. + deprecated: - area: access_log change: | diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index e92673dbf02f..01aabe6c0078 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -221,6 +221,8 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_timeout_; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return true; } + bool connectionRateLimitEnabled() const override { return false; } + uint32_t connectionRateLimitPerSec() const override { return 0; } // For any readPolicy other than Primary, the RedisClientFactory will send a READONLY command // when establishing a new connection. Since we're only using this for making the "cluster // slots" commands, the READONLY command is not relevant in this context. We're setting it to diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index f9835be41238..6a9275882657 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -181,6 +181,9 @@ class Config { * @return the read policy the proxy should use. */ virtual ReadPolicy readPolicy() const PURE; + + virtual bool connectionRateLimitEnabled() const PURE; + virtual uint32_t connectionRateLimitPerSec() const PURE; }; using ConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index c34b7cfda558..53b578d305e5 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -49,6 +49,14 @@ ConfigImpl::ConfigImpl( read_policy_ = ReadPolicy::Any; break; } + + if (config.has_connection_rate_limit()) { + connection_rate_limit_enabled_ = true; + connection_rate_limit_per_sec_ = config.connection_rate_limit().connection_rate_limit_per_sec(); + } else { + connection_rate_limit_enabled_ = false; + connection_rate_limit_per_sec_ = 100; + } } ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 75b67f1509cf..2f41b3786381 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -54,6 +54,8 @@ class ConfigImpl : public Config { } bool enableCommandStats() const override { return enable_command_stats_; } ReadPolicy readPolicy() const override { return read_policy_; } + bool connectionRateLimitEnabled() const override { return connection_rate_limit_enabled_; } + uint32_t connectionRateLimitPerSec() const override { return connection_rate_limit_per_sec_; } private: const std::chrono::milliseconds op_timeout_; @@ -64,6 +66,8 @@ class ConfigImpl : public Config { const uint32_t max_upstream_unknown_connections_; const bool enable_command_stats_; ReadPolicy read_policy_; + bool connection_rate_limit_enabled_; + uint32_t connection_rate_limit_per_sec_; }; class ClientImpl : public Client, public DecoderCallbacks, public Network::ConnectionCallbacks { diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index 3ff0ab6c7f6d..0330bfee1077 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -179,6 +179,7 @@ void InstanceImpl::ThreadLocalPool::onClusterRemoval(const std::string& cluster_ cluster_ = nullptr; host_address_map_.clear(); + cx_rate_limiter_map_.clear(); } void InstanceImpl::ThreadLocalPool::onHostsAdded( @@ -201,6 +202,10 @@ void InstanceImpl::ThreadLocalPool::onHostsAdded( void InstanceImpl::ThreadLocalPool::onHostsRemoved( const std::vector& hosts_removed) { for (const auto& host : hosts_removed) { + auto token_bucket = cx_rate_limiter_map_.find(host); + if (token_bucket != cx_rate_limiter_map_.end()) { + cx_rate_limiter_map_.erase(token_bucket); + } auto it = client_map_.find(host); if (it != client_map_.end()) { if (it->second->redis_client_->active()) { @@ -236,14 +241,24 @@ void InstanceImpl::ThreadLocalPool::drainClients() { InstanceImpl::ThreadLocalActiveClientPtr& InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstSharedPtr host) { + TokenBucketPtr& rate_limiter = cx_rate_limiter_map_[host]; + if (config_->connectionRateLimitEnabled() && !rate_limiter) { + rate_limiter = std::make_unique(config_->connectionRateLimitPerSec(), + dispatcher_.timeSource(), + config_->connectionRateLimitPerSec()); + } ThreadLocalActiveClientPtr& client = client_map_[host]; if (!client) { - client = std::make_unique(*this); - client->host_ = host; - client->redis_client_ = - client_factory_.create(host, dispatcher_, *config_, redis_command_stats_, *(stats_scope_), - auth_username_, auth_password_, false); - client->redis_client_->addConnectionCallbacks(*client); + if (config_->connectionRateLimitEnabled() && rate_limiter->consume(1, false) == 0) { + redis_cluster_stats_.connection_rate_limited_.inc(); + } else { + client = std::make_unique(*this); + client->host_ = host; + client->redis_client_ = + client_factory_.create(host, dispatcher_, *config_, redis_command_stats_, *(stats_scope_), + auth_username_, auth_password_, false); + client->redis_client_->addConnectionCallbacks(*client); + } } return client; } @@ -283,9 +298,16 @@ InstanceImpl::ThreadLocalPool::makeRequest(const std::string& key, RespVariant&& PendingRequest& pending_request = pending_requests_.back(); if (!transaction.active_) { - pending_request.request_handler_ = - this->threadLocalActiveClient(host)->redis_client_->makeRequest( - getRequest(pending_request.incoming_request_), pending_request); + ThreadLocalActiveClientPtr& client = this->threadLocalActiveClient(host); + if (!client) { + ENVOY_LOG(debug, "redis connection is rate limited, erasing empty client"); + pending_request.request_handler_ = nullptr; + onRequestCompleted(); + client_map_.erase(host); + return nullptr; + } + pending_request.request_handler_ = client->redis_client_->makeRequest( + getRequest(pending_request.incoming_request_), pending_request); } else { pending_request.request_handler_ = transaction.client_->makeRequest( getRequest(pending_request.incoming_request_), pending_request); @@ -365,6 +387,11 @@ Common::Redis::Client::PoolRequest* InstanceImpl::ThreadLocalPool::makeRequestTo } ThreadLocalActiveClientPtr& client = threadLocalActiveClient(it->second); + if (!client) { + ENVOY_LOG(debug, "redis connection is rate limited, erasing empty client"); + client_map_.erase(it->second); + return nullptr; + } return client->redis_client_->makeRequest(request, callbacks); } diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index 0e8063893599..a7d43c5fcdb7 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -13,6 +13,7 @@ #include "envoy/upstream/cluster_manager.h" #include "source/common/buffer/buffer_impl.h" +#include "source/common/common/token_bucket_impl.h" #include "source/common/network/address_impl.h" #include "source/common/network/filter_impl.h" #include "source/common/protobuf/utility.h" @@ -41,7 +42,8 @@ namespace ConnPool { #define REDIS_CLUSTER_STATS(COUNTER) \ COUNTER(upstream_cx_drained) \ - COUNTER(max_upstream_unknown_connections_reached) + COUNTER(max_upstream_unknown_connections_reached) \ + COUNTER(connection_rate_limited) struct RedisClusterStats { REDIS_CLUSTER_STATS(GENERATE_COUNTER_STRUCT) @@ -177,6 +179,7 @@ class InstanceImpl : public Instance, public std::enable_shared_from_this client_map_; + absl::node_hash_map cx_rate_limiter_map_; Envoy::Common::CallbackHandlePtr host_set_member_update_cb_handle_; absl::node_hash_map host_address_map_; std::string auth_username_; diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index f1ce5ded62f3..d87dc880b4bb 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -100,6 +100,8 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } + bool connectionRateLimitEnabled() const override { return false; } + uint32_t connectionRateLimitPerSec() const override { return 0; } // Extensions::NetworkFilters::Common::Redis::Client::ClientCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index b95d9bfe14af..619f0ed31f64 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -190,6 +190,8 @@ class ConfigBufferSizeGTSingleRequest : public Config { uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } ReadPolicy readPolicy() const override { return ReadPolicy::Primary; } + bool connectionRateLimitEnabled() const override { return false; } + uint32_t connectionRateLimitPerSec() const override { return 0; } }; TEST_F(RedisClientImplTest, BatchWithTimerFiring) { @@ -349,6 +351,8 @@ class ConfigEnableCommandStats : public Config { ReadPolicy readPolicy() const override { return ReadPolicy::Primary; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return true; } + bool connectionRateLimitEnabled() const override { return false; } + uint32_t connectionRateLimitPerSec() const override { return 0; } }; void initializeRedisSimpleCommand(Common::Redis::RespValue* request, std::string command_name, @@ -735,6 +739,8 @@ class ConfigOutlierDisabled : public Config { ReadPolicy readPolicy() const override { return ReadPolicy::Primary; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } + bool connectionRateLimitEnabled() const override { return false; } + uint32_t connectionRateLimitPerSec() const override { return 0; } }; TEST_F(RedisClientImplTest, OutlierDisabled) { diff --git a/test/extensions/filters/network/common/redis/test_utils.h b/test/extensions/filters/network/common/redis/test_utils.h index 6d18e9dfb971..f86a8f54a637 100644 --- a/test/extensions/filters/network/common/redis/test_utils.h +++ b/test/extensions/filters/network/common/redis/test_utils.h @@ -21,13 +21,18 @@ createConnPoolSettings( uint32_t max_unknown_conns = 100, envoy::extensions::filters::network::redis_proxy::v3::RedisProxy::ConnPoolSettings::ReadPolicy read_policy = envoy::extensions::filters::network::redis_proxy::v3::RedisProxy:: - ConnPoolSettings::MASTER) { + ConnPoolSettings::MASTER, + uint32_t redis_cx_rate_limit_per_sec = 100) { envoy::extensions::filters::network::redis_proxy::v3::RedisProxy::ConnPoolSettings setting{}; setting.mutable_op_timeout()->CopyFrom(Protobuf::util::TimeUtil::MillisecondsToDuration(millis)); setting.set_enable_hashtagging(hashtagging); setting.set_enable_redirection(redirection_support); setting.mutable_max_upstream_unknown_connections()->set_value(max_unknown_conns); setting.set_read_policy(read_policy); + + auto rate_limit = setting.mutable_connection_rate_limit(); + rate_limit->set_connection_rate_limit_per_sec(redis_cx_rate_limit_per_sec); + return setting; } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 27e9bcd0014b..c9ce02d13dc4 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -50,7 +50,8 @@ namespace ConnPool { class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client::ClientFactory { public: void setup(bool cluster_exists = true, bool hashtagging = true, uint32_t max_unknown_conns = 100, - const Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr dns_cache = nullptr) { + const Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr dns_cache = nullptr, + uint32_t redis_cx_rate_limit_per_sec = 100) { EXPECT_CALL(cm_, addThreadLocalClusterUpdateCallbacks_(_)) .WillOnce(DoAll(SaveArgAddress(&update_callbacks_), ReturnNew())); @@ -78,6 +79,16 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client max_upstream_unknown_connections_reached_.value_++; })); + connection_rate_limited_.value_ = 0; + ON_CALL(store_, counter(Eq("connection_rate_limited"))) + .WillByDefault(ReturnRef(connection_rate_limited_)); + ON_CALL(connection_rate_limited_, value()).WillByDefault(Invoke([&]() -> uint64_t { + return connection_rate_limited_.value_; + })); + ON_CALL(connection_rate_limited_, inc()).WillByDefault(Invoke([&]() { + connection_rate_limited_.value_++; + })); + cluster_refresh_manager_ = std::make_shared>(); auto redis_command_stats = @@ -85,7 +96,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client std::shared_ptr conn_pool_impl = std::make_shared( cluster_name_, cm_, *this, tls_, Common::Redis::Client::createConnPoolSettings(20, hashtagging, true, max_unknown_conns, - read_policy_), + read_policy_, redis_cx_rate_limit_per_sec), api_, store_.rootScope(), redis_command_stats, cluster_refresh_manager_, dns_cache); conn_pool_impl->init(); // Set the authentication password for this connection pool. @@ -216,6 +227,11 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client return conn_pool_impl->redis_cluster_stats_.max_upstream_unknown_connections_reached_; } + Stats::Counter& connectionRateLimited() { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + return conn_pool_impl->redis_cluster_stats_.connection_rate_limited_; + } + // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Common::Redis::Client::Config&, @@ -312,6 +328,7 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client ConnPoolSettings::MASTER; NiceMock upstream_cx_drained_; NiceMock max_upstream_unknown_connections_reached_; + NiceMock connection_rate_limited_; std::shared_ptr> cluster_refresh_manager_; Common::Redis::Client::NoOpTransaction transaction_; @@ -408,6 +425,78 @@ TEST_F(RedisConnPoolImplTest, ClientRequestFailed) { tls_.shutdownThread(); }; +TEST_F(RedisConnPoolImplTest, RedisConnectionRateLimited) { + InSequence s; + + setup(true, true, 100, nullptr, 1); + + Common::Redis::RespValue value; + Common::Redis::Client::MockPoolRequest active_request; + MockPoolCallbacks callbacks; + Common::Redis::Client::MockClient* client = new NiceMock(); + EXPECT_CALL(cm_.thread_local_cluster_.lb_, chooseHost(_)) + .WillOnce(Invoke([&](Upstream::LoadBalancerContext* context) -> Upstream::HostConstSharedPtr { + EXPECT_EQ(context->computeHashKey().value(), MurmurHash::murmurHash2("hash_key")); + EXPECT_EQ(context->metadataMatchCriteria(), nullptr); + EXPECT_EQ(context->downstreamConnection(), nullptr); + return cm_.thread_local_cluster_.lb_.host_; + })); + EXPECT_CALL(*this, create_(_)).WillOnce(Return(client)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, address()) + .WillRepeatedly(Return(test_address_)); + EXPECT_CALL(*client, makeRequest_(Eq(value), _)).WillOnce(Return(&active_request)); + Common::Redis::Client::PoolRequest* request = + conn_pool_->makeRequest("hash_key", ConnPool::RespVariant(value), callbacks, transaction_); + EXPECT_NE(nullptr, request); + EXPECT_EQ(connectionRateLimited().value(), 0); + + // close local and reconnect, should be rate limited and get null + client->raiseEvent(Network::ConnectionEvent::LocalClose); + + EXPECT_CALL(cm_.thread_local_cluster_.lb_, chooseHost(_)) + .WillOnce(Invoke([&](Upstream::LoadBalancerContext* context) -> Upstream::HostConstSharedPtr { + EXPECT_EQ(context->computeHashKey().value(), MurmurHash::murmurHash2("hash_key")); + EXPECT_EQ(context->metadataMatchCriteria(), nullptr); + EXPECT_EQ(context->downstreamConnection(), nullptr); + return cm_.thread_local_cluster_.lb_.host_; + })); + EXPECT_CALL(*this, create_(_)).Times(0); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, address()) + .WillRepeatedly(Return(test_address_)); + Common::Redis::Client::PoolRequest* rate_limited_request = + conn_pool_->makeRequest("hash_key", ConnPool::RespVariant(value), callbacks, transaction_); + EXPECT_EQ(nullptr, rate_limited_request); + EXPECT_EQ(connectionRateLimited().value(), 1); + + // wait for a second and rate limiter should recover + tls_.dispatcher_.globalTimeSystem().advanceTimeWait(std::chrono::seconds(1)); + Common::Redis::Client::MockPoolRequest new_active_request; + MockPoolCallbacks new_callbacks; + Common::Redis::Client::MockClient* new_client = new NiceMock(); + EXPECT_CALL(cm_.thread_local_cluster_.lb_, chooseHost(_)) + .WillOnce(Invoke([&](Upstream::LoadBalancerContext* context) -> Upstream::HostConstSharedPtr { + EXPECT_EQ(context->computeHashKey().value(), MurmurHash::murmurHash2("hash_key")); + EXPECT_EQ(context->metadataMatchCriteria(), nullptr); + EXPECT_EQ(context->downstreamConnection(), nullptr); + return cm_.thread_local_cluster_.lb_.host_; + })); + EXPECT_CALL(*this, create_(_)).WillOnce(Return(new_client)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, address()) + .WillRepeatedly(Return(test_address_)); + EXPECT_CALL(*new_client, makeRequest_(Eq(value), _)).WillOnce(Return(&new_active_request)); + Common::Redis::Client::PoolRequest* new_request = conn_pool_->makeRequest( + "hash_key", ConnPool::RespVariant(value), new_callbacks, transaction_); + EXPECT_NE(nullptr, new_request); + EXPECT_EQ(connectionRateLimited().value(), 1); + + EXPECT_CALL(active_request, cancel()); + EXPECT_CALL(callbacks, onFailure_()); + EXPECT_CALL(new_active_request, cancel()); + EXPECT_CALL(new_callbacks, onFailure_()); + EXPECT_CALL(*new_client, close()); + tls_.shutdownThread(); +}; + TEST_F(RedisConnPoolImplTest, BasicWithReadPolicy) { testReadPolicy(envoy::extensions::filters::network::redis_proxy::v3::RedisProxy:: ConnPoolSettings::PREFER_MASTER, From f6a2b567d306fb24dca62d160e488b6353a37371 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Apr 2023 09:33:12 +0100 Subject: [PATCH 027/740] build(deps): bump jaegertracing/all-in-one from `508634b` to `cfdd0e3` in /examples/shared/jaeger (#27042) build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger Bumps jaegertracing/all-in-one from `508634b` to `cfdd0e3`. --- updated-dependencies: - dependency-name: jaegertracing/all-in-one dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/jaeger/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile index 24bd465db25d..f5b859a22d6f 100644 --- a/examples/shared/jaeger/Dockerfile +++ b/examples/shared/jaeger/Dockerfile @@ -1,4 +1,4 @@ -FROM jaegertracing/all-in-one@sha256:508634b55341cfcfd781e2c8ee7d30cabce74f34494df9517a22ca4b0dada673 +FROM jaegertracing/all-in-one@sha256:cfdd0e350ca29aaacc4ae0b2c946c2ef3a33853edd354a15b98e7bb611323099 HEALTHCHECK \ --interval=1s \ --timeout=1s \ From c9294e8d7a69cf0c9eb90326678a96e90be01379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Apr 2023 10:03:43 +0100 Subject: [PATCH 028/740] build(deps): bump github/codeql-action from 2.3.0 to 2.3.2 (#27033) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.3.0 to 2.3.2. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/b2c19fb9a2a485599ccf4ed5d65527d94bc57226...f3feb00acb00f31a6f60280e6ace9ca31d91c76a) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 5c7f93580775..cdf39487c846 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -32,7 +32,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@b2c19fb9a2a485599ccf4ed5d65527d94bc57226 + uses: github/codeql-action/init@f3feb00acb00f31a6f60280e6ace9ca31d91c76a # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -63,4 +63,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@b2c19fb9a2a485599ccf4ed5d65527d94bc57226 + uses: github/codeql-action/analyze@f3feb00acb00f31a6f60280e6ace9ca31d91c76a diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index 20946eb3e5be..bfabcf7f692d 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -45,7 +45,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@b2c19fb9a2a485599ccf4ed5d65527d94bc57226 + uses: github/codeql-action/init@f3feb00acb00f31a6f60280e6ace9ca31d91c76a # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -78,4 +78,4 @@ jobs: - name: Perform CodeQL Analysis if: env.BUILD_TARGETS != '' - uses: github/codeql-action/analyze@b2c19fb9a2a485599ccf4ed5d65527d94bc57226 + uses: github/codeql-action/analyze@f3feb00acb00f31a6f60280e6ace9ca31d91c76a From 809a1f931ca2ffd79117304cc6a15af91dd5266f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Apr 2023 10:04:17 +0100 Subject: [PATCH 029/740] build(deps): bump openpolicyagent/opa from 0.51.0-istio to 0.52.0-istio in /examples/ext_authz (#27041) build(deps): bump openpolicyagent/opa in /examples/ext_authz Bumps openpolicyagent/opa from 0.51.0-istio to 0.52.0-istio. --- updated-dependencies: - dependency-name: openpolicyagent/opa dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/ext_authz/Dockerfile-opa | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ext_authz/Dockerfile-opa b/examples/ext_authz/Dockerfile-opa index d40e0050f358..950698be7305 100644 --- a/examples/ext_authz/Dockerfile-opa +++ b/examples/ext_authz/Dockerfile-opa @@ -1 +1 @@ -FROM openpolicyagent/opa:0.51.0-istio@sha256:fb677a040a7d12f0ff4dea9025204068b7723c81d5d6bb2f5bc25db97e0b4d0c +FROM openpolicyagent/opa:0.52.0-istio@sha256:03fce2c713c1b10c01371aea70aa2630b7a464cce45fe458286a5734ec492a3b From 5d0e530da18fb188f824dfc04fae31f69bd186d3 Mon Sep 17 00:00:00 2001 From: StarryNight Date: Fri, 28 Apr 2023 21:50:39 +0800 Subject: [PATCH 030/740] fix network wasm example doc (#27019) Signed-off-by: wangkai19 --- .../_include/wasm-network-filter.yaml | 42 +++++++++++++++++++ .../listeners/network_filters/wasm_filter.rst | 21 ++++------ 2 files changed, 49 insertions(+), 14 deletions(-) create mode 100644 docs/root/configuration/listeners/network_filters/_include/wasm-network-filter.yaml diff --git a/docs/root/configuration/listeners/network_filters/_include/wasm-network-filter.yaml b/docs/root/configuration/listeners/network_filters/_include/wasm-network-filter.yaml new file mode 100644 index 000000000000..6d4f0397f4a5 --- /dev/null +++ b/docs/root/configuration/listeners/network_filters/_include/wasm-network-filter.yaml @@ -0,0 +1,42 @@ +admin: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + address: 0.0.0.0 + port_value: 10000 + filter_chains: + - filters: + - name: envoy.filters.network.wasm + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm + config: + name: "my_plugin" + vm_config: + code: + local: + filename: "/etc/envoy_filter_network_wasm_example.wasm" + allow_precompiled: true + - name: envoy.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp + cluster: web_service + clusters: + - name: web_service + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service1 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: web_service + port_value: 9000 diff --git a/docs/root/configuration/listeners/network_filters/wasm_filter.rst b/docs/root/configuration/listeners/network_filters/wasm_filter.rst index 232dd56e478d..991c095b63a9 100644 --- a/docs/root/configuration/listeners/network_filters/wasm_filter.rst +++ b/docs/root/configuration/listeners/network_filters/wasm_filter.rst @@ -18,19 +18,12 @@ Example configuration Example filter configuration: -.. code-block:: yaml - - name: envoy.filters.network.wasm - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm - config: - config: - name: "my_plugin" - vm_config: - code: - local: - filename: "/etc/envoy_filter_http_wasm_example.wasm" - allow_precompiled: true - +.. literalinclude:: _include/wasm-network-filter.yaml + :language: yaml + :lines: 13-29 + :emphasize-lines: 3-12 + :linenos: + :lineno-start: 13 + :caption: :download:`wasm-network-filter.yaml <_include/wasm-network-filter.yaml>` The preceding snippet configures a filter from a Wasm binary on local disk. From 6c49c16ac5f7fb8e3eeb3a7cdac943f6b3fc3719 Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Fri, 28 Apr 2023 10:31:57 -0400 Subject: [PATCH 031/740] Revert "fix network wasm example doc (#27019)" (#27050) This reverts commit 5d0e530da18fb188f824dfc04fae31f69bd186d3. Signed-off-by: Adi Suissa-Peleg --- .../_include/wasm-network-filter.yaml | 42 ------------------- .../listeners/network_filters/wasm_filter.rst | 21 ++++++---- 2 files changed, 14 insertions(+), 49 deletions(-) delete mode 100644 docs/root/configuration/listeners/network_filters/_include/wasm-network-filter.yaml diff --git a/docs/root/configuration/listeners/network_filters/_include/wasm-network-filter.yaml b/docs/root/configuration/listeners/network_filters/_include/wasm-network-filter.yaml deleted file mode 100644 index 6d4f0397f4a5..000000000000 --- a/docs/root/configuration/listeners/network_filters/_include/wasm-network-filter.yaml +++ /dev/null @@ -1,42 +0,0 @@ -admin: - address: - socket_address: - address: 127.0.0.1 - port_value: 9901 -static_resources: - listeners: - - name: listener_0 - address: - socket_address: - address: 0.0.0.0 - port_value: 10000 - filter_chains: - - filters: - - name: envoy.filters.network.wasm - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm - config: - name: "my_plugin" - vm_config: - code: - local: - filename: "/etc/envoy_filter_network_wasm_example.wasm" - allow_precompiled: true - - name: envoy.tcp_proxy - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - stat_prefix: tcp - cluster: web_service - clusters: - - name: web_service - type: STRICT_DNS - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: service1 - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: web_service - port_value: 9000 diff --git a/docs/root/configuration/listeners/network_filters/wasm_filter.rst b/docs/root/configuration/listeners/network_filters/wasm_filter.rst index 991c095b63a9..232dd56e478d 100644 --- a/docs/root/configuration/listeners/network_filters/wasm_filter.rst +++ b/docs/root/configuration/listeners/network_filters/wasm_filter.rst @@ -18,12 +18,19 @@ Example configuration Example filter configuration: -.. literalinclude:: _include/wasm-network-filter.yaml - :language: yaml - :lines: 13-29 - :emphasize-lines: 3-12 - :linenos: - :lineno-start: 13 - :caption: :download:`wasm-network-filter.yaml <_include/wasm-network-filter.yaml>` +.. code-block:: yaml + + name: envoy.filters.network.wasm + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm + config: + config: + name: "my_plugin" + vm_config: + code: + local: + filename: "/etc/envoy_filter_http_wasm_example.wasm" + allow_precompiled: true + The preceding snippet configures a filter from a Wasm binary on local disk. From 34068bbfd73bc41e9906d6395e1cf7c70de7c1a9 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Fri, 28 Apr 2023 11:50:58 -0400 Subject: [PATCH 032/740] matcher: Generic matching API (#26861) * matcher: Generic matching API Signed-off-by: tyxia --- .../filters/network/source/match.h | 24 ++---- .../filters/network/test/match_test.cc | 12 +-- .../matching/input_matchers/source/matcher.cc | 6 +- .../matching/input_matchers/source/matcher.h | 2 +- .../input_matchers/test/matcher_test.cc | 2 +- envoy/matcher/matcher.h | 25 +++--- source/common/http/matching/inputs.h | 10 +-- .../common/http/matching/status_code_input.h | 8 +- source/common/matcher/field_matcher.h | 2 +- source/common/matcher/map_matcher.h | 7 +- source/common/matcher/value_input_matcher.h | 13 ++-- source/common/router/router_ratelimit.cc | 13 ++-- source/common/ssl/matching/inputs.h | 12 +-- .../extensions/common/matcher/trie_matcher.h | 4 +- .../filters/http/rate_limit_quota/matcher.cc | 18 ++--- .../environment_variable/config.cc | 2 +- .../environment_variable/input.h | 6 +- .../consistent_hashing/matcher.h | 6 +- .../matching/input_matchers/ip/matcher.cc | 8 +- .../matching/input_matchers/ip/matcher.h | 2 +- .../input_matchers/runtime_fraction/BUILD | 1 + .../input_matchers/runtime_fraction/matcher.h | 7 +- .../network/application_protocol/config.cc | 2 +- .../matching/network/common/inputs.cc | 12 ++- .../matching/network/common/inputs.h | 18 +++-- test/common/http/matching/inputs_test.cc | 26 +++---- .../http/matching/status_code_input_test.cc | 24 +++--- test/common/matcher/exact_map_matcher_test.cc | 2 +- test/common/matcher/field_matcher_test.cc | 6 +- .../common/matcher/prefix_map_matcher_test.cc | 2 +- test/common/matcher/test_utility.h | 27 +++++-- .../matcher/value_input_matcher_test.cc | 6 +- test/common/ssl/matching/inputs_test.cc | 22 +++--- .../common/matcher/trie_matcher_test.cc | 10 +-- .../environment_variable/config_test.cc | 4 +- .../environment_variable/input_test.cc | 6 +- .../consistent_hashing/matcher_test.cc | 4 +- .../input_matchers/ip/matcher_test.cc | 2 +- .../runtime_fraction/matcher_test.cc | 8 +- .../matching/network/common/inputs_test.cc | 76 ++++++++++--------- tools/spelling/spelling_dictionary.txt | 1 + 41 files changed, 239 insertions(+), 209 deletions(-) diff --git a/contrib/generic_proxy/filters/network/source/match.h b/contrib/generic_proxy/filters/network/source/match.h index 6084a9bd2a86..80fb6ea7a2d6 100644 --- a/contrib/generic_proxy/filters/network/source/match.h +++ b/contrib/generic_proxy/filters/network/source/match.h @@ -24,10 +24,8 @@ using PropertyDataInputProto = class ServiceMatchDataInput : public Matcher::DataInput { public: Matcher::DataInputGetResult get(const Request& data) const override { - Matcher::DataInputGetResult result; - result.data_availability_ = Matcher::DataInputGetResult::DataAvailability::AllDataAvailable; - result.data_.emplace(data.host()); - return result; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, + std::string(data.host())}; } }; @@ -50,10 +48,8 @@ class ServiceMatchDataInputFactory : public Matcher::DataInputFactory { class MethodMatchDataInput : public Matcher::DataInput { public: Matcher::DataInputGetResult get(const Request& data) const override { - Matcher::DataInputGetResult result; - result.data_availability_ = Matcher::DataInputGetResult::DataAvailability::AllDataAvailable; - result.data_.emplace(data.method()); - return result; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, + std::string(data.method())}; } }; @@ -78,15 +74,11 @@ class PropertyMatchDataInput : public Matcher::DataInput { PropertyMatchDataInput(const std::string& property_name) : name_(property_name) {} Matcher::DataInputGetResult get(const Request& data) const override { - Matcher::DataInputGetResult result; - result.data_availability_ = Matcher::DataInputGetResult::DataAvailability::AllDataAvailable; - const auto value = data.getByKey(name_); - - if (value.has_value()) { - result.data_.emplace(value.value()); - } - return result; + Matcher::MatchingDataType matching_data = + value.has_value() ? Matcher::MatchingDataType(std::string(value.value())) + : absl::monostate(); + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, matching_data}; } private: diff --git a/contrib/generic_proxy/filters/network/test/match_test.cc b/contrib/generic_proxy/filters/network/test/match_test.cc index 40b924fa74a8..60e691d7bce0 100644 --- a/contrib/generic_proxy/filters/network/test/match_test.cc +++ b/contrib/generic_proxy/filters/network/test/match_test.cc @@ -21,11 +21,11 @@ TEST(ServiceMatchDataInputTest, ServiceMatchDataInputTest) { FakeStreamCodecFactory::FakeRequest request; - EXPECT_EQ("", input->get(request).data_.value()); + EXPECT_EQ("", absl::get(input->get(request).data_)); request.host_ = "fake_host_as_service"; - EXPECT_EQ("fake_host_as_service", input->get(request).data_.value()); + EXPECT_EQ("fake_host_as_service", absl::get(input->get(request).data_)); } TEST(MethodMatchDataInputTest, MethodMatchDataInputTest) { @@ -37,11 +37,11 @@ TEST(MethodMatchDataInputTest, MethodMatchDataInputTest) { FakeStreamCodecFactory::FakeRequest request; - EXPECT_EQ("", input->get(request).data_.value()); + EXPECT_EQ("", absl::get(input->get(request).data_)); request.method_ = "fake_method"; - EXPECT_EQ("fake_method", input->get(request).data_.value()); + EXPECT_EQ("fake_method", absl::get(input->get(request).data_)); } TEST(PropertyMatchDataInputTest, PropertyMatchDataInputTest) { @@ -58,11 +58,11 @@ TEST(PropertyMatchDataInputTest, PropertyMatchDataInputTest) { FakeStreamCodecFactory::FakeRequest request; - EXPECT_EQ(absl::nullopt, input->get(request).data_); + EXPECT_TRUE(absl::holds_alternative(input->get(request).data_)); request.data_["key_0"] = "value_0"; - EXPECT_EQ("value_0", input->get(request).data_.value()); + EXPECT_EQ("value_0", absl::get(input->get(request).data_)); } } // namespace diff --git a/contrib/hyperscan/matching/input_matchers/source/matcher.cc b/contrib/hyperscan/matching/input_matchers/source/matcher.cc index ec476534e8be..39b9ce8e91b6 100644 --- a/contrib/hyperscan/matching/input_matchers/source/matcher.cc +++ b/contrib/hyperscan/matching/input_matchers/source/matcher.cc @@ -125,12 +125,12 @@ std::string Matcher::replaceAll(absl::string_view value, absl::string_view subst return absl::StrJoin(parts, ""); } -bool Matcher::match(absl::optional input) { - if (!input) { +bool Matcher::match(const ::Envoy::Matcher::MatchingDataType& input) { + if (absl::holds_alternative(input)) { return false; } - return static_cast(this)->match(*input); + return static_cast(this)->match(absl::get(input)); } void Matcher::compile(const std::vector& expressions, diff --git a/contrib/hyperscan/matching/input_matchers/source/matcher.h b/contrib/hyperscan/matching/input_matchers/source/matcher.h index b012e4b28306..d49743ae0c5d 100644 --- a/contrib/hyperscan/matching/input_matchers/source/matcher.h +++ b/contrib/hyperscan/matching/input_matchers/source/matcher.h @@ -42,7 +42,7 @@ class Matcher : public Envoy::Regex::CompiledMatcher, public Envoy::Matcher::Inp std::string replaceAll(absl::string_view value, absl::string_view substitution) const override; // Envoy::Matcher::InputMatcher - bool match(absl::optional input) override; + bool match(const ::Envoy::Matcher::MatchingDataType& input) override; private: hs_database_t* database_{}; diff --git a/contrib/hyperscan/matching/input_matchers/test/matcher_test.cc b/contrib/hyperscan/matching/input_matchers/test/matcher_test.cc index e087949257f4..dcd2e9667753 100644 --- a/contrib/hyperscan/matching/input_matchers/test/matcher_test.cc +++ b/contrib/hyperscan/matching/input_matchers/test/matcher_test.cc @@ -111,7 +111,7 @@ TEST_F(MatcherTest, Regex) { TEST_F(MatcherTest, Nullopt) { setup("^/asdf/.+", 0, false); - EXPECT_FALSE(matcher_->match(absl::nullopt)); + EXPECT_FALSE(matcher_->match(absl::monostate())); } // Verify that matching will be performed case-insensitively. diff --git a/envoy/matcher/matcher.h b/envoy/matcher/matcher.h index d360d48d7682..8dc9c7f35275 100644 --- a/envoy/matcher/matcher.h +++ b/envoy/matcher/matcher.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -23,6 +24,8 @@ class ServerFactoryContext; namespace Matcher { +using MatchingDataType = absl::variant; + // This file describes a MatchTree, which traverses a tree of matches until it // either matches (resulting in either an action or a new tree to traverse) or doesn't match. // The matching might stop early if either the data is not available at all yet, or if more data @@ -161,7 +164,7 @@ class InputMatcher { * @param absl::optional the value to match on. Will be absl::nullopt if the * lookup failed. */ - virtual bool match(absl::optional input) PURE; + virtual bool match(const Matcher::MatchingDataType& input) PURE; }; using InputMatcherPtr = std::unique_ptr; @@ -198,16 +201,20 @@ struct DataInputGetResult { }; DataAvailability data_availability_; - // The resulting data. This will be absl::nullopt if we don't have sufficient data available (as - // per data_availability_) or because no value was extracted. For example, consider a DataInput - // which attempts to look a key up in the map: if we don't have access to the map yet, we return - // absl::nullopt with NotAvailable. If we have the entire map, but the key doesn't exist in the - // map, we return absl::nullopt with AllDataAvailable. - absl::optional data_; + // The resulting data. This will be absl::monostate() if we don't have sufficient data available + // (as per data_availability_) or because no value was extracted. For example, consider a + // DataInput which attempts to look a key up in the map: if we don't have access to the map yet, + // we return absl::monostate() with NotAvailable. If we have the entire map, but the key doesn't + // exist in the map, we return absl::monostate() with AllDataAvailable. + MatchingDataType data_; // For pretty printing. friend std::ostream& operator<<(std::ostream& out, const DataInputGetResult& result) { - out << "data input: " << (result.data_ ? result.data_.value() : "n/a"); + out << "data input: " + << (absl::holds_alternative(result.data_) + ? absl::get(result.data_) + : "n/a"); + switch (result.data_availability_) { case DataInputGetResult::DataAvailability::NotAvailable: out << " (not available)"; @@ -266,7 +273,7 @@ template class DataInputFactory : public Config::TypedFactory { class CommonProtocolInput { public: virtual ~CommonProtocolInput() = default; - virtual absl::optional get() PURE; + virtual MatchingDataType get() PURE; }; using CommonProtocolInputPtr = std::unique_ptr; using CommonProtocolInputFactoryCb = std::function; diff --git a/source/common/http/matching/inputs.h b/source/common/http/matching/inputs.h index 2ad1910145f5..95849bc5f22e 100644 --- a/source/common/http/matching/inputs.h +++ b/source/common/http/matching/inputs.h @@ -26,7 +26,7 @@ class HttpHeadersDataInputBase : public Matcher::DataInput { const OptRef maybe_headers = headerMap(data); if (!maybe_headers) { - return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()}; } auto header_string = HeaderUtility::getAllOfHeaderAsString(*maybe_headers, name_, ","); @@ -36,7 +36,7 @@ class HttpHeadersDataInputBase : public Matcher::DataInput { std::string(header_string.result().value())}; } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } private: @@ -153,12 +153,12 @@ class HttpRequestQueryParamsDataInput : public Matcher::DataInputPath(); if (!ret) { - return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()}; } Utility::QueryParams params = @@ -166,7 +166,7 @@ class HttpRequestQueryParamsDataInput : public Matcher::DataInputsecond)}; diff --git a/source/common/http/matching/status_code_input.h b/source/common/http/matching/status_code_input.h index 5742dd3a8985..100824e84ee2 100644 --- a/source/common/http/matching/status_code_input.h +++ b/source/common/http/matching/status_code_input.h @@ -21,7 +21,7 @@ class HttpResponseStatusCodeInput : public Matcher::DataInput const auto maybe_headers = data.responseHeaders(); if (!maybe_headers) { - return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()}; } const auto maybe_status = Http::Utility::getResponseStatusOrNullopt(*maybe_headers); @@ -30,7 +30,7 @@ class HttpResponseStatusCodeInput : public Matcher::DataInput absl::StrCat(*maybe_status)}; } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } }; @@ -42,7 +42,7 @@ class HttpResponseStatusCodeClassInput : public Matcher::DataInput, Logger::Loggablematch(input.data_); + bool current_match = input_matcher_->match(input.data_); if (!current_match && input.data_availability_ == DataInputGetResult::DataAvailability::MoreDataMightBeAvailable) { ENVOY_LOG(trace, "No match yet; delaying result as more data might be available."); diff --git a/source/common/matcher/map_matcher.h b/source/common/matcher/map_matcher.h index 0115ccb1257a..db71454c5645 100644 --- a/source/common/matcher/map_matcher.h +++ b/source/common/matcher/map_matcher.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "envoy/matcher/matcher.h" namespace Envoy { @@ -27,11 +29,12 @@ class MapMatcher : public MatchTree, Logger::Loggable(input.data_)) { return {MatchState::MatchComplete, on_no_match_}; } - const auto result = doMatch(*input.data_); + const auto result = doMatch(absl::get(input.data_)); if (result) { if (result->matcher_) { return result->matcher_->match(data); diff --git a/source/common/matcher/value_input_matcher.h b/source/common/matcher/value_input_matcher.h index 0cfdb3a39ba7..c953f8964d12 100644 --- a/source/common/matcher/value_input_matcher.h +++ b/source/common/matcher/value_input_matcher.h @@ -7,16 +7,17 @@ namespace Envoy { namespace Matcher { -template class StringInputMatcher : public InputMatcher { +template +class StringInputMatcher : public InputMatcher, Logger::Loggable { public: explicit StringInputMatcher(const StringMatcherType& matcher) : matcher_(matcher) {} - bool match(absl::optional input) override { - if (!input) { - return false; + bool match(const MatchingDataType& input) override { + if (absl::holds_alternative(input)) { + return matcher_.match(absl::get(input)); } - - return matcher_.match(*input); + // Return false when input is empty.(i.e., input is absl::monostate). + return false; } private: diff --git a/source/common/router/router_ratelimit.cc b/source/common/router/router_ratelimit.cc index 7d12257df479..24af81716d68 100644 --- a/source/common/router/router_ratelimit.cc +++ b/source/common/router/router_ratelimit.cc @@ -61,13 +61,14 @@ class MatchInputRateLimitDescriptor : public RateLimit::DescriptorProducer { Http::Matching::HttpMatchingDataImpl data(info); data.onRequestHeaders(headers); auto result = data_input_->get(data); - if (result.data_) { - if (!result.data_.value().empty()) { - descriptor_entry = {descriptor_key_, result.data_.value()}; - } - return true; + if (absl::holds_alternative(result.data_)) { + return false; } - return false; + const std::string& str = absl::get(result.data_); + if (!str.empty()) { + descriptor_entry = {descriptor_key_, str}; + } + return true; } private: diff --git a/source/common/ssl/matching/inputs.h b/source/common/ssl/matching/inputs.h index f2de358659ba..2acb27993163 100644 --- a/source/common/ssl/matching/inputs.h +++ b/source/common/ssl/matching/inputs.h @@ -35,14 +35,14 @@ template class UriSanInput : public Matcher::DataInput< Matcher::DataInputGetResult get(const MatchingDataType& data) const override { const auto& ssl = data.ssl(); if (!ssl) { - return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()}; } const auto& uri = ssl->uriSanPeerCertificate(); if (!uri.empty()) { return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::StrJoin(uri, ",")}; } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } }; @@ -63,14 +63,14 @@ template class DnsSanInput : public Matcher::DataInput< Matcher::DataInputGetResult get(const MatchingDataType& data) const override { const auto& ssl = data.ssl(); if (!ssl) { - return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()}; } const auto& dns = ssl->dnsSansPeerCertificate(); if (!dns.empty()) { return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::StrJoin(dns, ",")}; } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } }; @@ -91,14 +91,14 @@ template class SubjectInput : public Matcher::DataInput Matcher::DataInputGetResult get(const MatchingDataType& data) const override { const auto& ssl = data.ssl(); if (!ssl) { - return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()}; } const auto& subject = ssl->subjectPeerCertificate(); if (!subject.empty()) { return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, std::string(subject)}; } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } }; diff --git a/source/extensions/common/matcher/trie_matcher.h b/source/extensions/common/matcher/trie_matcher.h index 8e47d86ff55c..fd5ba0226190 100644 --- a/source/extensions/common/matcher/trie_matcher.h +++ b/source/extensions/common/matcher/trie_matcher.h @@ -69,11 +69,11 @@ template class TrieMatcher : public MatchTree { if (input.data_availability_ != DataInputGetResult::DataAvailability::AllDataAvailable) { return {MatchState::UnableToMatch, absl::nullopt}; } - if (!input.data_) { + if (absl::holds_alternative(input.data_)) { return {MatchState::MatchComplete, on_no_match_}; } const Network::Address::InstanceConstSharedPtr addr = - Network::Utility::parseInternetAddressNoThrow(*input.data_); + Network::Utility::parseInternetAddressNoThrow(absl::get(input.data_)); if (!addr) { return {MatchState::MatchComplete, on_no_match_}; } diff --git a/source/extensions/filters/http/rate_limit_quota/matcher.cc b/source/extensions/filters/http/rate_limit_quota/matcher.cc index a2dcacb6cffc..bed4d857284e 100644 --- a/source/extensions/filters/http/rate_limit_quota/matcher.cc +++ b/source/extensions/filters/http/rate_limit_quota/matcher.cc @@ -39,17 +39,17 @@ RateLimitOnMatchAction::generateBucketId(const Http::Matching::HttpMatchingDataI input_factory_ptr->createDataInput(builder_method.custom_value()); auto result = data_input_cb()->get(data); // If result has data. - if (result.data_) { - if (!result.data_.value().empty()) { - // Build the bucket id from the matched result. - bucket_id.mutable_bucket()->insert({bucket_id_key, result.data_.value()}); - } else { - // TODO(tyxia) Nothing hits this line at this moment. - return absl::InternalError("Empty resulting data from custom value config."); - } - } else { + if (absl::holds_alternative(result.data_)) { return absl::InternalError("Failed to generate the id from custom value config."); } + const std::string& str = absl::get(result.data_); + if (!str.empty()) { + // Build the bucket id from the matched result. + bucket_id.mutable_bucket()->insert({bucket_id_key, str}); + } else { + // TODO(tyxia) Nothing hits this line at this moment. + return absl::InternalError("Empty resulting data from custom value config."); + } break; } case ValueSpecifierCase::VALUE_SPECIFIER_NOT_SET: { diff --git a/source/extensions/matching/common_inputs/environment_variable/config.cc b/source/extensions/matching/common_inputs/environment_variable/config.cc index be8235c2c5a3..62ac602b4441 100644 --- a/source/extensions/matching/common_inputs/environment_variable/config.cc +++ b/source/extensions/matching/common_inputs/environment_variable/config.cc @@ -24,7 +24,7 @@ Config::createCommonProtocolInputFactoryCb(const Protobuf::Message& config, return [s = std::string(value)]() { return std::make_unique(s); }; } - return []() { return std::make_unique(absl::nullopt); }; + return []() { return std::make_unique(absl::monostate()); }; } /** diff --git a/source/extensions/matching/common_inputs/environment_variable/input.h b/source/extensions/matching/common_inputs/environment_variable/input.h index 07b2d67ccffb..f73aadc65dc6 100644 --- a/source/extensions/matching/common_inputs/environment_variable/input.h +++ b/source/extensions/matching/common_inputs/environment_variable/input.h @@ -10,12 +10,12 @@ namespace EnvironmentVariable { class Input : public Matcher::CommonProtocolInput { public: - explicit Input(absl::optional&& value) : storage_(std::move(value)) {} + explicit Input(Matcher::MatchingDataType&& value) : storage_(std::move(value)) {} - absl::optional get() override { return storage_; } + Matcher::MatchingDataType get() override { return storage_; } private: - const absl::optional storage_; + const Matcher::MatchingDataType storage_; }; } // namespace EnvironmentVariable } // namespace CommonInputs diff --git a/source/extensions/matching/input_matchers/consistent_hashing/matcher.h b/source/extensions/matching/input_matchers/consistent_hashing/matcher.h index 4ef0f6eaa20d..f81a177140b4 100644 --- a/source/extensions/matching/input_matchers/consistent_hashing/matcher.h +++ b/source/extensions/matching/input_matchers/consistent_hashing/matcher.h @@ -14,14 +14,14 @@ class Matcher : public Envoy::Matcher::InputMatcher { public: Matcher(uint32_t threshold, uint32_t modulo, uint64_t seed) : threshold_(threshold), modulo_(modulo), seed_(seed) {} - bool match(absl::optional input) override { + bool match(const Envoy::Matcher::MatchingDataType& input) override { // Only match if the value is present. - if (!input) { + if (absl::holds_alternative(input)) { return false; } // Otherwise, match if (hash(input) % modulo) >= threshold. - return HashUtil::xxHash64(*input, seed_) % modulo_ >= threshold_; + return HashUtil::xxHash64(absl::get(input), seed_) % modulo_ >= threshold_; } private: diff --git a/source/extensions/matching/input_matchers/ip/matcher.cc b/source/extensions/matching/input_matchers/ip/matcher.cc index 00db60a8dd62..4ad6744f5052 100644 --- a/source/extensions/matching/input_matchers/ip/matcher.cc +++ b/source/extensions/matching/input_matchers/ip/matcher.cc @@ -25,15 +25,15 @@ Matcher::Matcher(std::vector const& ranges, // store any associated data. trie_({{true, ranges}}), stats_(generateStats(stat_prefix, stat_scope)) {} -bool Matcher::match(absl::optional input) { - if (!input) { +bool Matcher::match(const Envoy::Matcher::MatchingDataType& input) { + if (absl::holds_alternative(input)) { return false; } - const absl::string_view ip_str = *input; + const std::string& ip_str = absl::get(input); if (ip_str.empty()) { return false; } - const auto ip = Network::Utility::parseInternetAddressNoThrow(std::string{ip_str}); + const auto ip = Network::Utility::parseInternetAddressNoThrow(ip_str); if (!ip) { stats_.ip_parsing_failed_.inc(); ENVOY_LOG(debug, "IP matcher: unable to parse address '{}'", ip_str); diff --git a/source/extensions/matching/input_matchers/ip/matcher.h b/source/extensions/matching/input_matchers/ip/matcher.h index ba035ad579ff..583cd03867d6 100644 --- a/source/extensions/matching/input_matchers/ip/matcher.h +++ b/source/extensions/matching/input_matchers/ip/matcher.h @@ -24,7 +24,7 @@ class Matcher : public Envoy::Matcher::InputMatcher, Logger::Loggable const& ranges, absl::string_view stat_prefix, Stats::Scope& stat_scope); - bool match(absl::optional input) override; + bool match(const Envoy::Matcher::MatchingDataType& input) override; absl::optional stats() const { return stats_; } private: diff --git a/source/extensions/matching/input_matchers/runtime_fraction/BUILD b/source/extensions/matching/input_matchers/runtime_fraction/BUILD index 54d7f48e135b..4e26b7256ffe 100644 --- a/source/extensions/matching/input_matchers/runtime_fraction/BUILD +++ b/source/extensions/matching/input_matchers/runtime_fraction/BUILD @@ -16,6 +16,7 @@ envoy_cc_library( "//envoy/matcher:matcher_interface", "//envoy/upstream:retry_interface", "//source/common/common:hash_lib", + "//source/common/matcher:matcher_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/matching/input_matchers/runtime_fraction/matcher.h b/source/extensions/matching/input_matchers/runtime_fraction/matcher.h index e805b1e478ce..a9784be636e5 100644 --- a/source/extensions/matching/input_matchers/runtime_fraction/matcher.h +++ b/source/extensions/matching/input_matchers/runtime_fraction/matcher.h @@ -5,6 +5,7 @@ #include "envoy/runtime/runtime.h" #include "source/common/common/hash.h" +#include "source/common/matcher/matcher.h" namespace Envoy { namespace Extensions { @@ -17,14 +18,14 @@ class Matcher : public Envoy::Matcher::InputMatcher { Matcher(Runtime::Loader& runtime, envoy::config::core::v3::RuntimeFractionalPercent runtime_fraction, uint64_t seed) : runtime_(runtime), runtime_fraction_(runtime_fraction), seed_(seed) {} - bool match(absl::optional input) override { + bool match(const ::Envoy::Matcher::MatchingDataType& input) override { // Only match if the value is present. - if (!input) { + if (absl::holds_alternative(input)) { return false; } // Otherwise, match if feature is enabled for hash(input). - const auto hash_value = HashUtil::xxHash64(*input, seed_); + const auto hash_value = HashUtil::xxHash64(absl::get(input), seed_); return runtime_.snapshot().featureEnabled(runtime_fraction_.runtime_key(), runtime_fraction_.default_value(), hash_value); } diff --git a/source/extensions/matching/network/application_protocol/config.cc b/source/extensions/matching/network/application_protocol/config.cc index c3aec3a727a7..525c4c84b6b0 100644 --- a/source/extensions/matching/network/application_protocol/config.cc +++ b/source/extensions/matching/network/application_protocol/config.cc @@ -15,7 +15,7 @@ Matcher::DataInputGetResult ApplicationProtocolInput::get(const MatchingData& da return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::StrCat("'", absl::StrJoin(protocols, "','"), "'")}; } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } REGISTER_FACTORY(ApplicationProtocolInputFactory, Matcher::DataInputFactory); diff --git a/source/extensions/matching/network/common/inputs.cc b/source/extensions/matching/network/common/inputs.cc index bbc8de261d85..343514a98d5e 100644 --- a/source/extensions/matching/network/common/inputs.cc +++ b/source/extensions/matching/network/common/inputs.cc @@ -15,7 +15,7 @@ Matcher::DataInputGetResult TransportProtocolInput::get(const MatchingData& data return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, std::string(transport_protocol)}; } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } Matcher::DataInputGetResult FilterStateInput::get(const MatchingData& data) const { @@ -23,11 +23,15 @@ Matcher::DataInputGetResult FilterStateInput::get(const MatchingData& data) cons data.filterState().getDataReadOnly(filter_state_key_); if (filter_state_object != nullptr) { - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, - filter_state_object->serializeAsString()}; + auto str = filter_state_object->serializeAsString(); + if (str.has_value()) { + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, str.value()}; + } else { + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; + } } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } class DestinationIPInputFactory : public DestinationIPInputBaseFactory {}; diff --git a/source/extensions/matching/network/common/inputs.h b/source/extensions/matching/network/common/inputs.h index 16da08774417..1ec8732e3d9f 100644 --- a/source/extensions/matching/network/common/inputs.h +++ b/source/extensions/matching/network/common/inputs.h @@ -39,7 +39,7 @@ class DestinationIPInput : public Matcher::DataInput { const auto& address = data.localAddress(); if (address.type() != Network::Address::Type::Ip) { - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, address.ip()->addressAsString()}; @@ -63,13 +63,17 @@ DECLARE_FACTORY(DestinationIPInputFactory); DECLARE_FACTORY(UdpDestinationIPInputFactory); DECLARE_FACTORY(HttpDestinationIPInputFactory); +// With the support of generic matching API, integer is allowed in inputs such as +// `DestinationPortInput` and `SourcePortInput`. As a result, there is no longer a need for the +// redundant conversion of int-to-string. We can change them here once IntInputMatcher (for integer +// type input) is implemented. template class DestinationPortInput : public Matcher::DataInput { public: Matcher::DataInputGetResult get(const MatchingDataType& data) const override { const auto& address = data.localAddress(); if (address.type() != Network::Address::Type::Ip) { - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::StrCat(address.ip()->port())}; @@ -99,7 +103,7 @@ class SourceIPInput : public Matcher::DataInput { Matcher::DataInputGetResult get(const MatchingDataType& data) const override { const auto& address = data.remoteAddress(); if (address.type() != Network::Address::Type::Ip) { - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, address.ip()->addressAsString()}; @@ -128,7 +132,7 @@ class SourcePortInput : public Matcher::DataInput { Matcher::DataInputGetResult get(const MatchingDataType& data) const override { const auto& address = data.remoteAddress(); if (address.type() != Network::Address::Type::Ip) { - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::StrCat(address.ip()->port())}; @@ -157,7 +161,7 @@ class DirectSourceIPInput : public Matcher::DataInput { Matcher::DataInputGetResult get(const MatchingDataType& data) const override { const auto& address = data.connectionInfoProvider().directRemoteAddress(); if (address->type() != Network::Address::Type::Ip) { - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, address->ip()->addressAsString()}; @@ -189,7 +193,7 @@ class SourceTypeInput : public Matcher::DataInput { if (is_local_connection) { return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, "local"}; } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } }; @@ -217,7 +221,7 @@ class ServerNameInput : public Matcher::DataInput { return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, std::string(server_name)}; } - return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}; + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } }; diff --git a/test/common/http/matching/inputs_test.cc b/test/common/http/matching/inputs_test.cc index 9905b3a10110..48d41eca5b35 100644 --- a/test/common/http/matching/inputs_test.cc +++ b/test/common/http/matching/inputs_test.cc @@ -37,7 +37,7 @@ TEST(MatchingData, HttpRequestHeadersDataInput) { TestRequestHeaderMapImpl request_headers({{"header", "bar"}}); data.onRequestHeaders(request_headers); - EXPECT_EQ(input.get(data).data_, "bar"); + EXPECT_EQ(absl::get(input.get(data).data_), "bar"); } { @@ -46,7 +46,7 @@ TEST(MatchingData, HttpRequestHeadersDataInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -58,7 +58,7 @@ TEST(MatchingData, HttpRequestTrailersDataInput) { TestRequestTrailerMapImpl request_trailers({{"header", "bar"}}); data.onRequestTrailers(request_trailers); - EXPECT_EQ(input.get(data).data_, "bar"); + EXPECT_EQ(absl::get(input.get(data).data_), "bar"); } { @@ -67,7 +67,7 @@ TEST(MatchingData, HttpRequestTrailersDataInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -82,7 +82,7 @@ TEST(MatchingData, HttpResponseHeadersDataInput) { TestResponseHeaderMapImpl response_headers({{"header", "bar"}}); data.onResponseHeaders(response_headers); - EXPECT_EQ(input.get(data).data_, "bar"); + EXPECT_EQ(absl::get(input.get(data).data_), "bar"); } { @@ -91,7 +91,7 @@ TEST(MatchingData, HttpResponseHeadersDataInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -106,7 +106,7 @@ TEST(MatchingData, HttpResponseTrailersDataInput) { TestResponseTrailerMapImpl response_trailers({{"header", "bar"}}); data.onResponseTrailers(response_trailers); - EXPECT_EQ(input.get(data).data_, "bar"); + EXPECT_EQ(absl::get(input.get(data).data_), "bar"); } { @@ -115,7 +115,7 @@ TEST(MatchingData, HttpResponseTrailersDataInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -130,7 +130,7 @@ TEST(MatchingData, HttpRequestQueryParamsDataInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::NotAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { @@ -138,7 +138,7 @@ TEST(MatchingData, HttpRequestQueryParamsDataInput) { TestRequestHeaderMapImpl request_headers({{":path", "/test?user%20name=foo%20bar"}}); data.onRequestHeaders(request_headers); - EXPECT_EQ(input.get(data).data_, "foo bar"); + EXPECT_EQ(absl::get(input.get(data).data_), "foo bar"); } { @@ -146,7 +146,7 @@ TEST(MatchingData, HttpRequestQueryParamsDataInput) { TestRequestHeaderMapImpl request_headers({{":path", "/test?username=fooA&username=fooB"}}); data.onRequestHeaders(request_headers); - EXPECT_EQ(input.get(data).data_, "fooA"); + EXPECT_EQ(absl::get(input.get(data).data_), "fooA"); } { @@ -158,7 +158,7 @@ TEST(MatchingData, HttpRequestQueryParamsDataInput) { EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { @@ -170,7 +170,7 @@ TEST(MatchingData, HttpRequestQueryParamsDataInput) { EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::NotAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } diff --git a/test/common/http/matching/status_code_input_test.cc b/test/common/http/matching/status_code_input_test.cc index 1d81e34b7d83..a6f21aa55ff7 100644 --- a/test/common/http/matching/status_code_input_test.cc +++ b/test/common/http/matching/status_code_input_test.cc @@ -37,14 +37,14 @@ TEST(MatchingData, HttpResponseStatusCodeInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::NotAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { TestResponseHeaderMapImpl response_headers({{"header", "bar"}}); response_headers.setStatus(200); data.onResponseHeaders(response_headers); - EXPECT_EQ(input.get(data).data_, "200"); + EXPECT_EQ(absl::get(input.get(data).data_), "200"); } { @@ -53,7 +53,7 @@ TEST(MatchingData, HttpResponseStatusCodeInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -67,14 +67,14 @@ TEST(MatchingData, HttpResponseStatusCodeClassInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::NotAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { TestResponseHeaderMapImpl response_headers({{"header", "bar"}}); response_headers.setStatus(100); data.onResponseHeaders(response_headers); - EXPECT_EQ(input.get(data).data_, "1xx"); + EXPECT_EQ(absl::get(input.get(data).data_), "1xx"); } { @@ -82,28 +82,28 @@ TEST(MatchingData, HttpResponseStatusCodeClassInput) { response_headers.setStatus(200); data.onResponseHeaders(response_headers); - EXPECT_EQ(input.get(data).data_, "2xx"); + EXPECT_EQ(absl::get(input.get(data).data_), "2xx"); } { TestResponseHeaderMapImpl response_headers({{"header", "bar"}}); response_headers.setStatus(300); data.onResponseHeaders(response_headers); - EXPECT_EQ(input.get(data).data_, "3xx"); + EXPECT_EQ(absl::get(input.get(data).data_), "3xx"); } { TestResponseHeaderMapImpl response_headers({{"header", "bar"}}); response_headers.setStatus(400); data.onResponseHeaders(response_headers); - EXPECT_EQ(input.get(data).data_, "4xx"); + EXPECT_EQ(absl::get(input.get(data).data_), "4xx"); } { TestResponseHeaderMapImpl response_headers({{"header", "bar"}}); response_headers.setStatus(500); data.onResponseHeaders(response_headers); - EXPECT_EQ(input.get(data).data_, "5xx"); + EXPECT_EQ(absl::get(input.get(data).data_), "5xx"); } { @@ -112,7 +112,7 @@ TEST(MatchingData, HttpResponseStatusCodeClassInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { TestResponseHeaderMapImpl response_headers({{"not-header", "baz"}}); @@ -121,7 +121,7 @@ TEST(MatchingData, HttpResponseStatusCodeClassInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { TestResponseHeaderMapImpl response_headers({{"not-header", "baz"}}); @@ -130,7 +130,7 @@ TEST(MatchingData, HttpResponseStatusCodeClassInput) { auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } diff --git a/test/common/matcher/exact_map_matcher_test.cc b/test/common/matcher/exact_map_matcher_test.cc index ca6210089e20..be4e60b5eeb5 100644 --- a/test/common/matcher/exact_map_matcher_test.cc +++ b/test/common/matcher/exact_map_matcher_test.cc @@ -25,7 +25,7 @@ TEST(ExactMapMatcherTest, NoMatch) { TEST(ExactMapMatcherTest, NoMatchDueToNoData) { ExactMapMatcher matcher( std::make_unique(DataInputGetResult{ - DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}), + DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}), absl::nullopt); TestData data; diff --git a/test/common/matcher/field_matcher_test.cc b/test/common/matcher/field_matcher_test.cc index 44c40d8c8624..50a8b4cfef27 100644 --- a/test/common/matcher/field_matcher_test.cc +++ b/test/common/matcher/field_matcher_test.cc @@ -19,7 +19,7 @@ class FieldMatcherTest : public testing::Test { matchers.reserve(values.size()); for (const auto& v : values) { matchers.emplace_back(std::make_unique>( - std::make_unique(DataInputGetResult{v.second, absl::nullopt}), + std::make_unique(DataInputGetResult{v.second, absl::monostate()}), std::make_unique(v.first))); } @@ -58,9 +58,7 @@ TEST_F(FieldMatcherTest, SingleFieldMatcher) { createSingleMatcher("foo", [](auto v) { return v == "foo"; })->match(TestData()).result()); EXPECT_FALSE( createSingleMatcher("foo", [](auto v) { return v != "foo"; })->match(TestData()).result()); - EXPECT_TRUE(createSingleMatcher(absl::nullopt, [](auto v) { return v == absl::nullopt; }) - ->match(TestData()) - .result()); + EXPECT_FALSE(createSingleMatcher(absl::nullopt, [](auto v) { return v == "foo"; }) ->match(TestData()) .result()); diff --git a/test/common/matcher/prefix_map_matcher_test.cc b/test/common/matcher/prefix_map_matcher_test.cc index 62db555dfd2f..ffda7ab6a681 100644 --- a/test/common/matcher/prefix_map_matcher_test.cc +++ b/test/common/matcher/prefix_map_matcher_test.cc @@ -25,7 +25,7 @@ TEST(PrefixMapMatcherTest, NoMatch) { TEST(PrefixMapMatcherTest, NoMatchDueToNoData) { PrefixMapMatcher matcher( std::make_unique(DataInputGetResult{ - DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}), + DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}), absl::nullopt); TestData data; diff --git a/test/common/matcher/test_utility.h b/test/common/matcher/test_utility.h index b9e948278005..0301d8ab58d4 100644 --- a/test/common/matcher/test_utility.h +++ b/test/common/matcher/test_utility.h @@ -19,7 +19,7 @@ struct TestData { // A CommonProtocolInput that returns the configured value every time. struct CommonProtocolTestInput : public CommonProtocolInput { explicit CommonProtocolTestInput(const std::string& data) : data_(data) {} - absl::optional get() override { return data_; } + MatchingDataType get() override { return data_; } const std::string data_; }; @@ -101,7 +101,7 @@ class TestDataInputBoolFactory : public DataInputFactory { struct BoolMatcher : public InputMatcher { explicit BoolMatcher(bool value) : value_(value) {} - bool match(absl::optional) override { return value_; } + bool match(const MatchingDataType&) override { return value_; } const bool value_; }; @@ -111,7 +111,12 @@ struct TestMatcher : public InputMatcher { explicit TestMatcher(std::function)> predicate) : predicate_(predicate) {} - bool match(absl::optional input) override { return predicate_(input); } + bool match(const MatchingDataType& input) override { + if (absl::holds_alternative(input)) { + return false; + } + return predicate_(absl::get(input)); + } std::function)> predicate_; }; @@ -143,7 +148,7 @@ class StringActionFactory : public ActionFactory { // An InputMatcher that always returns false. class NeverMatch : public InputMatcher { public: - bool match(absl::optional) override { return false; } + bool match(const MatchingDataType&) override { return false; } }; /** @@ -172,7 +177,13 @@ class NeverMatchFactory : public InputMatcherFactory { class CustomStringMatcher : public InputMatcher { public: explicit CustomStringMatcher(const std::string& str) : str_value_(str) {} - bool match(absl::optional str) override { return str_value_ == str; } + bool match(const MatchingDataType& input) override { + if (absl::holds_alternative(input)) { + return false; + } + + return str_value_ == absl::get(input); + } private: std::string str_value_; @@ -212,9 +223,11 @@ createSingleMatcher(absl::optional input, std::function)> predicate, DataInputGetResult::DataAvailability availability = DataInputGetResult::DataAvailability::AllDataAvailable) { + MatchingDataType data = + input.has_value() ? MatchingDataType(std::string(*input)) : absl::monostate(); + return std::make_unique>( - std::make_unique(DataInputGetResult{ - availability, input ? absl::make_optional(std::string(*input)) : absl::nullopt}), + std::make_unique(DataInputGetResult{availability, std::move(data)}), std::make_unique(predicate)); } diff --git a/test/common/matcher/value_input_matcher_test.cc b/test/common/matcher/value_input_matcher_test.cc index e9d080d31efc..dc3868e98d24 100644 --- a/test/common/matcher/value_input_matcher_test.cc +++ b/test/common/matcher/value_input_matcher_test.cc @@ -11,9 +11,9 @@ TEST(ValueInputMatcher, TestMatch) { StringInputMatcher matcher(matcher_proto); - EXPECT_TRUE(matcher.match("exact")); - EXPECT_FALSE(matcher.match("not")); - EXPECT_FALSE(matcher.match(absl::nullopt)); + EXPECT_TRUE(matcher.match(MatchingDataType("exact"))); + EXPECT_FALSE(matcher.match(MatchingDataType("not"))); + EXPECT_FALSE(matcher.match(MatchingDataType(absl::monostate()))); } } // namespace Matcher diff --git a/test/common/ssl/matching/inputs_test.cc b/test/common/ssl/matching/inputs_test.cc index 56c9510e7299..00f497c23c96 100644 --- a/test/common/ssl/matching/inputs_test.cc +++ b/test/common/ssl/matching/inputs_test.cc @@ -22,7 +22,7 @@ TEST(Authentication, UriSanInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::NotAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } std::shared_ptr ssl = std::make_shared(); @@ -35,7 +35,7 @@ TEST(Authentication, UriSanInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { @@ -45,7 +45,7 @@ TEST(Authentication, UriSanInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "foo"); + EXPECT_EQ(absl::get(result.data_), "foo"); } { @@ -55,7 +55,7 @@ TEST(Authentication, UriSanInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "foo,bar"); + EXPECT_EQ(absl::get(result.data_), "foo,bar"); } } @@ -67,7 +67,7 @@ TEST(Authentication, DnsSanInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::NotAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } std::shared_ptr ssl = std::make_shared(); @@ -79,7 +79,7 @@ TEST(Authentication, DnsSanInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { @@ -89,7 +89,7 @@ TEST(Authentication, DnsSanInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "foo"); + EXPECT_EQ(absl::get(result.data_), "foo"); } { @@ -99,7 +99,7 @@ TEST(Authentication, DnsSanInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "foo,bar"); + EXPECT_EQ(absl::get(result.data_), "foo,bar"); } } @@ -113,7 +113,7 @@ TEST(Authentication, SubjectInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::NotAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } std::shared_ptr ssl = std::make_shared(); @@ -126,7 +126,7 @@ TEST(Authentication, SubjectInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { @@ -134,7 +134,7 @@ TEST(Authentication, SubjectInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "foo"); + EXPECT_EQ(absl::get(result.data_), "foo"); } } diff --git a/test/extensions/common/matcher/trie_matcher_test.cc b/test/extensions/common/matcher/trie_matcher_test.cc index 13518560e63a..56b8b3c3a185 100644 --- a/test/extensions/common/matcher/trie_matcher_test.cc +++ b/test/extensions/common/matcher/trie_matcher_test.cc @@ -182,7 +182,7 @@ TEST_F(TrieMatcherTest, TestMatcherOnNoMatch) { { // Input is nullopt. auto input = TestDataInputStringFactory( - {DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}); + {DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}); validateMatch("bar"); } } @@ -473,26 +473,26 @@ TEST_F(TrieMatcherTest, NoData) { { auto input = TestDataInputStringFactory( - {DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}); + {DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}); auto nested = TestDataInputBoolFactory(""); validateNoMatch(); } { auto input = TestDataInputStringFactory("127.0.0.1"); auto nested = TestDataInputBoolFactory( - {DataInputGetResult::DataAvailability::AllDataAvailable, absl::nullopt}); + {DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}); validateNoMatch(); } { auto input = TestDataInputStringFactory( - {DataInputGetResult::DataAvailability::NotAvailable, absl::nullopt}); + {DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()}); auto nested = TestDataInputBoolFactory(""); validateUnableToMatch(); } { auto input = TestDataInputStringFactory("127.0.0.1"); auto nested = TestDataInputBoolFactory( - {DataInputGetResult::DataAvailability::NotAvailable, absl::nullopt}); + {DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()}); validateUnableToMatch(); } } diff --git a/test/extensions/matching/common_inputs/environment_variable/config_test.cc b/test/extensions/matching/common_inputs/environment_variable/config_test.cc index 94dd3d0bea24..a5bdd10a42ca 100644 --- a/test/extensions/matching/common_inputs/environment_variable/config_test.cc +++ b/test/extensions/matching/common_inputs/environment_variable/config_test.cc @@ -31,7 +31,7 @@ TEST(ConfigTest, TestConfig) { auto input_factory = factory.createCommonProtocolInputFactoryCb( *message, ProtobufMessage::getStrictValidationVisitor()); EXPECT_NE(nullptr, input_factory); - EXPECT_EQ(input_factory()->get(), absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(input_factory()->get())); } TestEnvironment::setEnvVar("foo", "bar", 1); @@ -39,7 +39,7 @@ TEST(ConfigTest, TestConfig) { auto input_factory = factory.createCommonProtocolInputFactoryCb( *message, ProtobufMessage::getStrictValidationVisitor()); EXPECT_NE(nullptr, input_factory); - EXPECT_EQ(input_factory()->get(), absl::make_optional("bar")); + EXPECT_EQ(absl::get(input_factory()->get()), "bar"); } TestEnvironment::unsetEnvVar("foo"); diff --git a/test/extensions/matching/common_inputs/environment_variable/input_test.cc b/test/extensions/matching/common_inputs/environment_variable/input_test.cc index dbc874448e04..5f5b0387b675 100644 --- a/test/extensions/matching/common_inputs/environment_variable/input_test.cc +++ b/test/extensions/matching/common_inputs/environment_variable/input_test.cc @@ -11,13 +11,11 @@ namespace EnvironmentVariable { TEST(InputTest, BasicUsage) { { Input input("foo"); - ASSERT_TRUE(input.get().has_value()); - EXPECT_EQ(input.get().value(), "foo"); + EXPECT_EQ(absl::get(input.get()), "foo"); } Input input("foo"); - ASSERT_TRUE(input.get().has_value()); - EXPECT_EQ(input.get().value(), "foo"); + EXPECT_EQ(absl::get(input.get()), "foo"); } } // namespace EnvironmentVariable } // namespace CommonInputs diff --git a/test/extensions/matching/input_matchers/consistent_hashing/matcher_test.cc b/test/extensions/matching/input_matchers/consistent_hashing/matcher_test.cc index 98d4ea3a9273..e38c08ae8d50 100644 --- a/test/extensions/matching/input_matchers/consistent_hashing/matcher_test.cc +++ b/test/extensions/matching/input_matchers/consistent_hashing/matcher_test.cc @@ -15,8 +15,8 @@ TEST(MatcherTest, BasicUsage) { Matcher matcher1(10, 100, 0); Matcher matcher2(10, 100, 0); - EXPECT_FALSE(matcher1.match(absl::nullopt)); - EXPECT_FALSE(matcher2.match(absl::nullopt)); + EXPECT_FALSE(matcher1.match(absl::monostate())); + EXPECT_FALSE(matcher2.match(absl::monostate())); } { Matcher matcher1(58, 100, 0); diff --git a/test/extensions/matching/input_matchers/ip/matcher_test.cc b/test/extensions/matching/input_matchers/ip/matcher_test.cc index 55cce5fae238..68842784e819 100644 --- a/test/extensions/matching/input_matchers/ip/matcher_test.cc +++ b/test/extensions/matching/input_matchers/ip/matcher_test.cc @@ -69,7 +69,7 @@ TEST_F(MatcherTest, EmptyIP) { ranges.emplace_back(Network::Address::CidrRange::create("192.0.2.0", 24)); initialize(std::move(ranges)); EXPECT_FALSE(m_->match("")); - EXPECT_FALSE(m_->match(absl::optional{})); + EXPECT_FALSE(m_->match(absl::monostate())); } TEST_F(MatcherTest, InvalidIP) { diff --git a/test/extensions/matching/input_matchers/runtime_fraction/matcher_test.cc b/test/extensions/matching/input_matchers/runtime_fraction/matcher_test.cc index 72b74f429a03..05df1388d4cc 100644 --- a/test/extensions/matching/input_matchers/runtime_fraction/matcher_test.cc +++ b/test/extensions/matching/input_matchers/runtime_fraction/matcher_test.cc @@ -45,7 +45,11 @@ class TestMatcher { called_random_value = random_value; return result; }); - EXPECT_EQ(matcher_->match(value), result); + + EXPECT_EQ(matcher_->match(value.has_value() + ? ::Envoy::Matcher::MatchingDataType(std::string(value.value())) + : absl::monostate()), + result); return called_random_value; } @@ -54,7 +58,7 @@ class TestMatcher { runtime_.snapshot_, featureEnabled(key_, testing::Matcher(_), _)) .Times(0); - EXPECT_FALSE(matcher_->match(absl::nullopt)); + EXPECT_FALSE(matcher_->match(absl::monostate())); } private: diff --git a/test/extensions/matching/network/common/inputs_test.cc b/test/extensions/matching/network/common/inputs_test.cc index 2ebbc06baae3..4a1d2e66eae2 100644 --- a/test/extensions/matching/network/common/inputs_test.cc +++ b/test/extensions/matching/network/common/inputs_test.cc @@ -1,3 +1,5 @@ +#include + #include "envoy/http/filter.h" #include "source/common/http/matching/data_impl.h" @@ -27,7 +29,7 @@ TEST(MatchingData, DestinationIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "127.0.0.1"); + EXPECT_EQ(absl::get(result.data_), "127.0.0.1"); } { @@ -36,7 +38,7 @@ TEST(MatchingData, DestinationIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -56,42 +58,42 @@ TEST(MatchingData, HttpDestinationIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "127.0.0.1"); + EXPECT_EQ(absl::get(result.data_), "127.0.0.1"); } { DestinationPortInput input; const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "8080"); + EXPECT_EQ(absl::get(result.data_), "8080"); } { SourceIPInput input; const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "10.0.0.1"); + EXPECT_EQ(absl::get(result.data_), "10.0.0.1"); } { SourcePortInput input; const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "9090"); + EXPECT_EQ(absl::get(result.data_), "9090"); } { DirectSourceIPInput input; const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "127.0.0.2"); + EXPECT_EQ(absl::get(result.data_), "127.0.0.2"); } { ServerNameInput input; const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, host); + EXPECT_EQ(absl::get(result.data_), host); } connection_info_provider->setRemoteAddress( @@ -101,7 +103,7 @@ TEST(MatchingData, HttpDestinationIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "local"); + EXPECT_EQ(absl::get(result.data_), "local"); } } @@ -118,7 +120,7 @@ TEST(MatchingData, DestinationPortInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "8080"); + EXPECT_EQ(absl::get(result.data_), "8080"); } { @@ -127,7 +129,7 @@ TEST(MatchingData, DestinationPortInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -144,7 +146,7 @@ TEST(MatchingData, SourceIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "127.0.0.1"); + EXPECT_EQ(absl::get(result.data_), "127.0.0.1"); } { @@ -153,7 +155,7 @@ TEST(MatchingData, SourceIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -170,7 +172,7 @@ TEST(MatchingData, SourcePortInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "8080"); + EXPECT_EQ(absl::get(result.data_), "8080"); } { @@ -179,7 +181,7 @@ TEST(MatchingData, SourcePortInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -196,7 +198,7 @@ TEST(MatchingData, DirectSourceIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "127.0.0.1"); + EXPECT_EQ(absl::get(result.data_), "127.0.0.1"); } { @@ -205,7 +207,7 @@ TEST(MatchingData, DirectSourceIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -222,7 +224,7 @@ TEST(MatchingData, SourceTypeInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "local"); + EXPECT_EQ(absl::get(result.data_), "local"); } { @@ -231,7 +233,7 @@ TEST(MatchingData, SourceTypeInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -246,7 +248,7 @@ TEST(MatchingData, ServerNameInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { @@ -255,7 +257,7 @@ TEST(MatchingData, ServerNameInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, host); + EXPECT_EQ(absl::get(result.data_), host); } } @@ -271,7 +273,7 @@ TEST(MatchingData, TransportProtocolInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { @@ -280,7 +282,7 @@ TEST(MatchingData, TransportProtocolInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, protocol); + EXPECT_EQ(absl::get(result.data_), protocol); } } @@ -297,7 +299,7 @@ TEST(MatchingData, ApplicationProtocolInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } { @@ -306,7 +308,7 @@ TEST(MatchingData, ApplicationProtocolInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "'h2c'"); + EXPECT_EQ(absl::get(result.data_), "'h2c'"); } { @@ -315,7 +317,7 @@ TEST(MatchingData, ApplicationProtocolInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "'h2','http/1.1'"); + EXPECT_EQ(absl::get(result.data_), "'h2','http/1.1'"); } } @@ -336,7 +338,7 @@ TEST(MatchingData, FilterStateInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } filter_state.setData("unknown_key", std::make_shared("some_value"), @@ -347,7 +349,7 @@ TEST(MatchingData, FilterStateInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } std::string value = "filter_state_value"; @@ -359,7 +361,7 @@ TEST(MatchingData, FilterStateInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, value); + EXPECT_EQ(absl::get(result.data_), value); } } @@ -373,7 +375,7 @@ TEST(UdpMatchingData, UdpDestinationIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "127.0.0.1"); + EXPECT_EQ(absl::get(result.data_), "127.0.0.1"); } { @@ -381,7 +383,7 @@ TEST(UdpMatchingData, UdpDestinationIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -395,7 +397,7 @@ TEST(UdpMatchingData, UdpDestinationPortInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "8080"); + EXPECT_EQ(absl::get(result.data_), "8080"); } { @@ -403,7 +405,7 @@ TEST(UdpMatchingData, UdpDestinationPortInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -417,7 +419,7 @@ TEST(UdpMatchingData, UdpSourceIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "127.0.0.1"); + EXPECT_EQ(absl::get(result.data_), "127.0.0.1"); } { @@ -425,7 +427,7 @@ TEST(UdpMatchingData, UdpSourceIPInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } @@ -439,7 +441,7 @@ TEST(UdpMatchingData, UdpSourcePortInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, "8080"); + EXPECT_EQ(absl::get(result.data_), "8080"); } { @@ -447,7 +449,7 @@ TEST(UdpMatchingData, UdpSourcePortInput) { const auto result = input.get(data); EXPECT_EQ(result.data_availability_, Matcher::DataInputGetResult::DataAvailability::AllDataAvailable); - EXPECT_EQ(result.data_, absl::nullopt); + EXPECT_TRUE(absl::holds_alternative(result.data_)); } } diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 1aa769b424ad..ef68770145af 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -294,6 +294,7 @@ OCSP OD ODCDS middlewildcard +monostate mpd oghttp OID From eedfb0f104161aa3a3e3da13aca777c73ac3ce85 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Fri, 28 Apr 2023 11:56:18 -0400 Subject: [PATCH 033/740] docs: updating life of a request (#26942) Signed-off-by: Alyssa Wilk Signed-off-by: alyssawilk Co-authored-by: Matt Klein --- .../intro/arch_overview/http/http_filters.rst | 7 +++-- docs/root/intro/life_of_a_request.rst | 31 ++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/docs/root/intro/arch_overview/http/http_filters.rst b/docs/root/intro/arch_overview/http/http_filters.rst index 7cd0ed883ef5..5a77045f1c5a 100644 --- a/docs/root/intro/arch_overview/http/http_filters.rst +++ b/docs/root/intro/arch_overview/http/http_filters.rst @@ -9,6 +9,9 @@ HTTP level filter stack within the connection manager. Filters can be written that operate on HTTP level messages without knowledge of the underlying physical protocol (HTTP/1.1, HTTP/2, etc.) or multiplexing capabilities. +HTTP filters can be downstream filters, associated with a given listener and doing stream processing on each +downstream request before routing, or upstream filters, associated with a given cluster and doing stream processing once per upstream request, after the router filter. + There are three types of HTTP level filters: **Decoder** @@ -76,11 +79,11 @@ configure a match tree that can resolve filter configuration to use for a given Filter route mutation --------------------- -During HTTP filter chain processing, when ``decodeHeaders()`` is invoked by a filter, the +During downstream HTTP filter chain processing, when ``decodeHeaders()`` is invoked by a filter, the connection manager performs route resolution and sets a *cached route* pointing to an upstream cluster. -Filters have the capability to directly mutate this *cached route* after route resolution, via the +Downstream filters have the capability to directly mutate this *cached route* after route resolution, via the ``setRoute`` callback and :repo:`DelegatingRoute ` mechanism. diff --git a/docs/root/intro/life_of_a_request.rst b/docs/root/intro/life_of_a_request.rst index 7de3d2b92876..a633cd01404c 100644 --- a/docs/root/intro/life_of_a_request.rst +++ b/docs/root/intro/life_of_a_request.rst @@ -187,7 +187,7 @@ A brief outline of the life cycle of a request and response using the example co 5. The HTTP/2 codec in :ref:`HTTP connection manager ` deframes and demultiplexes the decrypted data stream from the TLS connection to a number of independent streams. Each stream handles a single request and response. -6. For each HTTP stream, an :ref:`HTTP filter ` chain is created and +6. For each HTTP stream, an :ref:`Downstream HTTP filter ` chain is created and runs. The request first passes through CustomFilter which may read and modify the request. The most important HTTP filter is the router filter which sits at the end of the HTTP filter chain. When ``decodeHeaders`` is invoked on the router filter, the route is selected and a cluster is @@ -198,15 +198,21 @@ A brief outline of the life cycle of a request and response using the example co endpoint. The cluster’s circuit breakers are checked to determine if a new stream is allowed. A new connection to the endpoint is created if the endpoint's connection pool is empty or lacks capacity. -8. The upstream endpoint connection's HTTP/2 codec multiplexes and frames the request’s stream with +8. For each stream an :ref:`Upstream HTTP filter ` chain is created and + runs. By default this only includes the CodecFilter, sending data to the appropriate codec, but if + the cluster is configured with an upstream filter chain, that filter chain will be created and run + on each stream, which includes creating and running separate filter chains for retries and shadowed + requests. +9. The upstream endpoint connection's HTTP/2 codec multiplexes and frames the request’s stream with any other streams going to that upstream over a single TCP connection. -9. The upstream endpoint connection's TLS transport socket encrypts these bytes and writes them to a - TCP socket for the upstream connection. -10. The request, consisting of headers, and optional body and trailers, is proxied upstream, and the +10. The upstream endpoint connection's TLS transport socket encrypts these bytes and writes them to a + TCP socket for the upstream connection. +11. The request, consisting of headers, and optional body and trailers, is proxied upstream, and the response is proxied downstream. The response passes through the HTTP filters in the :ref:`opposite order ` from the request, starting at the - router filter and passing through CustomFilter, before being sent downstream. -11. When the response is complete, the stream is destroyed. Post-request processing will update + codec filter, traversing any upstream filters, then going through the router filter and passing + through CustomFilter, before being sent downstream. +12. When the response is complete, the stream is destroyed. Post-request processing will update stats, write to the access log and finalize trace spans. We elaborate on each of these steps in the sections below. @@ -379,7 +385,7 @@ protocol is HTTP/1, HTTP/2 or HTTP/3. 6. HTTP filter chain processing ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -For each HTTP stream, the HCM instantiates an :ref:`HTTP filter ` chain, +For each HTTP stream, the HCM instantiates an :ref:`Downstream HTTP filter ` chain, following the pattern established above for listener and network filter chains. .. image:: /_static/lor-http-filters.svg @@ -454,6 +460,15 @@ The router filter is responsible for all aspects of upstream request lifecycle m stream allocated from the HTTP connection pool. It also is responsible for request timeouts, retries and affinity. +The router filter is also responsible for the creation and running of the `Upstream HTTP filter ` +chain. By default, upstream filters will start running immediately after headers arrive at the router +filter, however C++ filters can pause until the upstream connection is established if they need to +inspect the upstream stream or connection. Upstream filter chains are by default configured via cluster +configuration, so for example a shadowed request can have a separate upstream filter chain for the primary +and shadowed clusters. Also as the upstream filter chain is upstream of the router filter, it is run per each +retry attempt allowing header manipulation per retry and including information about the upstream stream and +connection. Unlike downstream filters, upstream filters can not alter the route. + 7. Load balancing ^^^^^^^^^^^^^^^^^ From e6281a2c5ae5648953882d99898000a226f3a95e Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 28 Apr 2023 16:57:40 +0100 Subject: [PATCH 034/740] docs: Point compiler info to repo (#27049) Signed-off-by: Ryan Northey --- docs/root/start/building.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/start/building.rst b/docs/root/start/building.rst index 4f91867c16c1..0a07818efc94 100644 --- a/docs/root/start/building.rst +++ b/docs/root/start/building.rst @@ -21,7 +21,7 @@ recent Linux including Ubuntu 20.04 LTS. Building Envoy has the following requirements: -* GCC 7+ or Clang/LLVM 7+ (for C++14 support). Clang/LLVM 9+ preferred where Clang is used (see below). +* Recent GCC/Clang versions - please see :repo:`bazel/README.md#supported-compiler-versions` for current requirements. * About 2GB of RAM per core (so 32GB of RAM for 8 cores with hyperthreading). See :ref:`this FAQ entry ` for more information on build performance. * These :repo:`Bazel native ` dependencies. From b1d1e2d53e432117e88585fd8149b13dd92b2bf7 Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Fri, 28 Apr 2023 19:01:26 +0300 Subject: [PATCH 035/740] formatter: use AccessLogType::NotSet instead of default value (#27034) use AccessLogType::NotSet instead of default parameter value Signed-off-by: ohadvano --- envoy/formatter/substitution_formatter.h | 36 +- .../common/formatter/substitution_formatter.h | 286 ++- source/common/local_reply/local_reply.cc | 4 +- source/common/router/header_formatter.cc | 2 +- source/common/router/header_formatter.h | 3 +- source/common/tcp_proxy/tcp_proxy.cc | 2 +- .../open_telemetry/substitution_formatter.h | 16 +- .../extensions/filters/http/oauth2/filter.cc | 12 +- .../filters/network/ratelimit/ratelimit.cc | 3 +- .../network/redis_proxy/router_impl.cc | 10 +- .../req_without_query/req_without_query.cc | 13 +- .../local_response_policy.cc | 4 +- .../matching/actions/format_string/config.cc | 8 +- test/common/formatter/command_extension.h | 28 +- .../substitution_format_string_test.cc | 13 +- .../substitution_formatter_fuzz_test.cc | 2 +- .../substitution_formatter_speed_test.cc | 39 +- .../formatter/substitution_formatter_test.cc | 1624 +++++++++-------- .../substitution_formatter_speed_test.cc | 8 +- .../substitution_formatter_test.cc | 98 +- .../formatter/metadata/metadata_test.cc | 25 +- .../req_without_query_test.cc | 24 +- 22 files changed, 1195 insertions(+), 1065 deletions(-) diff --git a/envoy/formatter/substitution_formatter.h b/envoy/formatter/substitution_formatter.h index 7be401d102ae..6c4c3547cfba 100644 --- a/envoy/formatter/substitution_formatter.h +++ b/envoy/formatter/substitution_formatter.h @@ -29,12 +29,12 @@ class Formatter { * @param local_reply_body supplies the local reply body. * @return std::string string containing the complete formatted substitution line. */ - virtual std::string - format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const PURE; + virtual std::string format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const PURE; }; using FormatterPtr = std::unique_ptr; @@ -58,12 +58,12 @@ class FormatterProvider { * @return absl::optional optional string containing a single value extracted from * the given headers/trailers/stream. */ - virtual absl::optional - format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const PURE; + virtual absl::optional format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const PURE; /** * Extract a value from the provided headers/trailers/stream, preserving the value's type. * @param request_headers supplies the request headers. @@ -74,12 +74,12 @@ class FormatterProvider { * @return ProtobufWkt::Value containing a single value extracted from the given * headers/trailers/stream. */ - virtual ProtobufWkt::Value formatValue( - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const PURE; + virtual ProtobufWkt::Value formatValue(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const PURE; }; using FormatterProviderPtr = std::unique_ptr; diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index 5d8d5bb2c58d..a2a9f95b3c1d 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -145,12 +145,11 @@ class FormatterImpl : public Formatter { const std::vector& command_parsers); // Formatter::format - std::string format( - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const override; + std::string format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const override; private: const std::string& empty_value_string_; @@ -172,12 +171,12 @@ class StructFormatter { StructFormatter(const ProtobufWkt::Struct& format_mapping, bool preserve_types, bool omit_empty_values, const std::vector& commands); - ProtobufWkt::Struct - format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const; + ProtobufWkt::Struct format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const; private: struct StructFormatMapWrapper; @@ -221,13 +220,13 @@ class StructFormatter { }; // Methods for doing the actual formatting. - ProtobufWkt::Value providersCallback( - const std::vector& providers, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const; + ProtobufWkt::Value providersCallback(const std::vector& providers, + const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const; ProtobufWkt::Value structFormatMapCallback(const StructFormatter::StructFormatMapWrapper& format_map, const StructFormatMapVisitor& visitor) const; @@ -254,12 +253,11 @@ class JsonFormatterImpl : public Formatter { : struct_formatter_(format_mapping, preserve_types, omit_empty_values, commands) {} // Formatter::format - std::string format( - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const override; + std::string format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const override; private: const StructFormatter struct_formatter_; @@ -274,14 +272,12 @@ class PlainStringFormatter : public FormatterProvider { PlainStringFormatter(const std::string& str); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; private: ProtobufWkt::Value str_; @@ -295,14 +291,12 @@ class PlainNumberFormatter : public FormatterProvider { PlainNumberFormatter(double num); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; private: ProtobufWkt::Value num_; @@ -316,16 +310,14 @@ class LocalReplyBodyFormatter : public FormatterProvider { LocalReplyBodyFormatter() = default; // Formatter::format - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view local_reply_body, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view local_reply_body, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view local_reply_body, + AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view local_reply_body, + AccessLog::AccessLogType) const override; }; /** @@ -336,16 +328,14 @@ class AccessLogTypeFormatter : public FormatterProvider { AccessLogTypeFormatter() = default; // Formatter::format - absl::optional format( - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value formatValue( - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const override; }; class HeaderFormatter { @@ -375,18 +365,16 @@ class HeadersByteSizeFormatter : public FormatterProvider { HeadersByteSizeFormatter(const HeaderType header_type); - absl::optional - format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, - absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, - absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const override; private: uint64_t extractHeadersByteSize(const Http::RequestHeaderMap& request_headers, @@ -404,14 +392,13 @@ class RequestHeaderFormatter : public FormatterProvider, HeaderFormatter { absl::optional max_length); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; }; /** @@ -423,14 +410,13 @@ class ResponseHeaderFormatter : public FormatterProvider, HeaderFormatter { absl::optional max_length); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; }; /** @@ -442,15 +428,13 @@ class ResponseTrailerFormatter : public FormatterProvider, HeaderFormatter { absl::optional max_length); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, - absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; }; class GrpcStatusFormatter : public FormatterProvider, HeaderFormatter { @@ -465,15 +449,14 @@ class GrpcStatusFormatter : public FormatterProvider, HeaderFormatter { absl::optional max_length, Format format); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo&, - absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; static Format parseFormat(absl::string_view format); @@ -490,14 +473,12 @@ class StreamInfoFormatter : public FormatterProvider { const absl::optional& = absl::nullopt); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; class FieldExtractor { public: @@ -536,17 +517,15 @@ class MetadataFormatter : public FormatterProvider { MetadataFormatter(const std::string& filter_namespace, const std::vector& path, absl::optional max_length, GetMetadataFunction get); - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo& stream_info, - absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view, + AccessLog::AccessLogType) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo& stream_info, - absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view, + AccessLog::AccessLogType) const override; protected: absl::optional @@ -600,14 +579,12 @@ class FilterStateFormatter : public FormatterProvider { bool serialize_as_string, bool is_upstream = false); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; private: const Envoy::StreamInfo::FilterState::Object* @@ -632,14 +609,12 @@ class SystemTimeFormatter : public FormatterProvider { SystemTimeFormatter(const std::string& format, TimeFieldExtractorPtr f); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; private: const Envoy::DateFormatter date_formatter_; @@ -698,14 +673,12 @@ class EnvironmentFormatter : public FormatterProvider { EnvironmentFormatter(const std::string& key, absl::optional max_length); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; private: ProtobufWkt::Value str_; @@ -722,14 +695,13 @@ class StreamInfoRequestHeaderFormatter : public FormatterProvider, HeaderFormatt absl::optional max_length); // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; }; } // namespace Formatter diff --git a/source/common/local_reply/local_reply.cc b/source/common/local_reply/local_reply.cc index 2fabfc2114b0..f09298afc629 100644 --- a/source/common/local_reply/local_reply.cc +++ b/source/common/local_reply/local_reply.cc @@ -37,8 +37,8 @@ class BodyFormatter { const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, std::string& body, absl::string_view& content_type) const { - body = - formatter_->format(request_headers, response_headers, response_trailers, stream_info, body); + body = formatter_->format(request_headers, response_headers, response_trailers, stream_info, + body, AccessLog::AccessLogType::NotSet); content_type = content_type_; } diff --git a/source/common/router/header_formatter.cc b/source/common/router/header_formatter.cc index 5208c855d495..9a59834872e6 100644 --- a/source/common/router/header_formatter.cc +++ b/source/common/router/header_formatter.cc @@ -59,7 +59,7 @@ parseSubstitutionFormatField(absl::string_view field_name, return formatter->format(*Http::StaticEmptyHeaders::get().request_headers, *Http::StaticEmptyHeaders::get().response_headers, *Http::StaticEmptyHeaders::get().response_trailers, stream_info, - absl::string_view()); + absl::string_view(), AccessLog::AccessLogType::NotSet); }; } diff --git a/source/common/router/header_formatter.h b/source/common/router/header_formatter.h index 8079d836967c..1f11fecf51e7 100644 --- a/source/common/router/header_formatter.h +++ b/source/common/router/header_formatter.h @@ -117,7 +117,8 @@ class HttpHeaderFormatterImpl : public HttpHeaderFormatter { const Envoy::StreamInfo::StreamInfo& stream_info) const override { std::string buf; buf = formatter_->format(request_headers, response_headers, - *Http::StaticEmptyHeaders::get().response_trailers, stream_info, ""); + *Http::StaticEmptyHeaders::get().response_trailers, stream_info, "", + AccessLog::AccessLogType::NotSet); return buf; }; diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index 421cf41eb9a1..03519d2a3c8a 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -629,7 +629,7 @@ std::string TunnelingConfigHelperImpl::host(const StreamInfo::StreamInfo& stream return hostname_fmt_->format(*Http::StaticEmptyHeaders::get().request_headers, *Http::StaticEmptyHeaders::get().response_headers, *Http::StaticEmptyHeaders::get().response_trailers, stream_info, - absl::string_view()); + absl::string_view(), AccessLog::AccessLogType::NotSet); } void TunnelingConfigHelperImpl::propagateResponseHeaders( diff --git a/source/extensions/access_loggers/open_telemetry/substitution_formatter.h b/source/extensions/access_loggers/open_telemetry/substitution_formatter.h index 89a050c584b0..3ae2188f8461 100644 --- a/source/extensions/access_loggers/open_telemetry/substitution_formatter.h +++ b/source/extensions/access_loggers/open_telemetry/substitution_formatter.h @@ -33,7 +33,7 @@ class OpenTelemetryFormatter { const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const; + AccessLog::AccessLogType access_log_type) const; private: struct OpenTelemetryFormatMapWrapper; @@ -75,13 +75,13 @@ class OpenTelemetryFormatter { }; // Methods for doing the actual formatting. - ::opentelemetry::proto::common::v1::AnyValue providersCallback( - const std::vector& providers, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) const; + ::opentelemetry::proto::common::v1::AnyValue + providersCallback(const std::vector& providers, + const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, + AccessLog::AccessLogType access_log_type) const; ::opentelemetry::proto::common::v1::AnyValue openTelemetryFormatMapCallback( const OpenTelemetryFormatter::OpenTelemetryFormatMapWrapper& format_map, const OpenTelemetryFormatMapVisitor& visitor) const; diff --git a/source/extensions/filters/http/oauth2/filter.cc b/source/extensions/filters/http/oauth2/filter.cc index e337e233102c..64f3fc07a4d7 100644 --- a/source/extensions/filters/http/oauth2/filter.cc +++ b/source/extensions/filters/http/oauth2/filter.cc @@ -349,9 +349,9 @@ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& he } Formatter::FormatterImpl formatter(config_->redirectUri()); - const auto redirect_uri = formatter.format(headers, *Http::ResponseHeaderMapImpl::create(), - *Http::ResponseTrailerMapImpl::create(), - decoder_callbacks_->streamInfo(), ""); + const auto redirect_uri = formatter.format( + headers, *Http::ResponseHeaderMapImpl::create(), *Http::ResponseTrailerMapImpl::create(), + decoder_callbacks_->streamInfo(), "", AccessLog::AccessLogType::NotSet); oauth_client_->asyncGetAccessToken(auth_code_, config_->clientId(), config_->clientSecret(), redirect_uri, config_->authType()); @@ -405,9 +405,9 @@ void OAuth2Filter::redirectToOAuthServer(Http::RequestHeaderMap& headers) const : Http::Utility::PercentEncoding::encode(state_path, ":/=&?"); Formatter::FormatterImpl formatter(config_->redirectUri()); - const auto redirect_uri = formatter.format(headers, *Http::ResponseHeaderMapImpl::create(), - *Http::ResponseTrailerMapImpl::create(), - decoder_callbacks_->streamInfo(), ""); + const auto redirect_uri = formatter.format( + headers, *Http::ResponseHeaderMapImpl::create(), *Http::ResponseTrailerMapImpl::create(), + decoder_callbacks_->streamInfo(), "", AccessLog::AccessLogType::NotSet); const std::string escaped_redirect_uri = Runtime::runtimeFeatureEnabled("envoy.reloadable_features.oauth_use_url_encoding") ? Http::Utility::PercentEncoding::urlEncodeQueryParameter(redirect_uri) diff --git a/source/extensions/filters/network/ratelimit/ratelimit.cc b/source/extensions/filters/network/ratelimit/ratelimit.cc index bbc92ab1fef7..9930ab455143 100644 --- a/source/extensions/filters/network/ratelimit/ratelimit.cc +++ b/source/extensions/filters/network/ratelimit/ratelimit.cc @@ -50,7 +50,8 @@ Config::applySubstitutionFormatter(StreamInfo::StreamInfo& stream_info) { std::string value = descriptor_entry.value_; value = formatter_it->get()->format(*request_headers_.get(), *response_headers_.get(), - *response_trailers_.get(), stream_info, value); + *response_trailers_.get(), stream_info, value, + AccessLog::AccessLogType::NotSet); formatter_it++; new_descriptor.entries_.push_back({descriptor_entry.key_, value}); } diff --git a/source/extensions/filters/network/redis_proxy/router_impl.cc b/source/extensions/filters/network/redis_proxy/router_impl.cc index eeee6d26c045..87f4770f8927 100644 --- a/source/extensions/filters/network/redis_proxy/router_impl.cc +++ b/source/extensions/filters/network/redis_proxy/router_impl.cc @@ -110,11 +110,11 @@ void PrefixRoutes::formatKey(std::string& key, std::string redis_key_formatter) auto providers = Formatter::SubstitutionFormatParser::parse(redis_key_formatter); std::string formatted_key; for (Formatter::FormatterProviderPtr& provider : providers) { - auto provider_formatted_key = - provider->formatValue(*Http::StaticEmptyHeaders::get().request_headers, - *Http::StaticEmptyHeaders::get().response_headers, - *Http::StaticEmptyHeaders::get().response_trailers, - callbacks_->connection().streamInfo(), absl::string_view()); + auto provider_formatted_key = provider->formatValue( + *Http::StaticEmptyHeaders::get().request_headers, + *Http::StaticEmptyHeaders::get().response_headers, + *Http::StaticEmptyHeaders::get().response_trailers, callbacks_->connection().streamInfo(), + absl::string_view(), AccessLog::AccessLogType::NotSet); if (provider_formatted_key.has_string_value()) { formatted_key = formatted_key + provider_formatted_key.string_value(); diff --git a/source/extensions/formatter/req_without_query/req_without_query.cc b/source/extensions/formatter/req_without_query/req_without_query.cc index 284e546724e1..644eeb859128 100644 --- a/source/extensions/formatter/req_without_query/req_without_query.cc +++ b/source/extensions/formatter/req_without_query/req_without_query.cc @@ -29,8 +29,7 @@ ReqWithoutQuery::ReqWithoutQuery(const std::string& main_header, absl::optional ReqWithoutQuery::format(const Http::RequestHeaderMap& request, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const { + absl::string_view, AccessLog::AccessLogType) const { const Http::HeaderEntry* header = findHeader(request); if (!header) { return absl::nullopt; @@ -42,11 +41,11 @@ ReqWithoutQuery::format(const Http::RequestHeaderMap& request, const Http::Respo return val; } -ProtobufWkt::Value -ReqWithoutQuery::formatValue(const Http::RequestHeaderMap& request, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const { +ProtobufWkt::Value ReqWithoutQuery::formatValue(const Http::RequestHeaderMap& request, + const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo&, absl::string_view, + AccessLog::AccessLogType) const { const Http::HeaderEntry* header = findHeader(request); if (!header) { return ValueUtil::nullValue(); diff --git a/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc b/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc index 8d34828dfca2..d4f9b85f0c46 100644 --- a/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc +++ b/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc @@ -45,8 +45,8 @@ void LocalResponsePolicy::formatBody(const Envoy::Http::RequestHeaderMap& reques if (formatter_) { formatter_->format(request_headers, response_headers, - *Envoy::Http::StaticEmptyHeaders::get().response_trailers, stream_info, - body); + *Envoy::Http::StaticEmptyHeaders::get().response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet); } } diff --git a/source/extensions/matching/actions/format_string/config.cc b/source/extensions/matching/actions/format_string/config.cc index 311bc2bfc6ce..b4d6efb7b52d 100644 --- a/source/extensions/matching/actions/format_string/config.cc +++ b/source/extensions/matching/actions/format_string/config.cc @@ -15,10 +15,10 @@ namespace FormatString { const Network::FilterChain* ActionImpl::get(const Server::Configuration::FilterChainsByName& filter_chains_by_name, const StreamInfo::StreamInfo& info) const { - const std::string name = - formatter_->format(*Http::StaticEmptyHeaders::get().request_headers, - *Http::StaticEmptyHeaders::get().response_headers, - *Http::StaticEmptyHeaders::get().response_trailers, info, ""); + const std::string name = formatter_->format(*Http::StaticEmptyHeaders::get().request_headers, + *Http::StaticEmptyHeaders::get().response_headers, + *Http::StaticEmptyHeaders::get().response_trailers, + info, "", AccessLog::AccessLogType::NotSet); const auto chain_match = filter_chains_by_name.find(name); if (chain_match != filter_chains_by_name.end()) { return chain_match->second.get(); diff --git a/test/common/formatter/command_extension.h b/test/common/formatter/command_extension.h index e0c049b506a3..ad67936078f9 100644 --- a/test/common/formatter/command_extension.h +++ b/test/common/formatter/command_extension.h @@ -13,14 +13,12 @@ namespace Formatter { class TestFormatter : public FormatterProvider { public: // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; }; class TestCommandParser : public CommandParser { @@ -40,14 +38,12 @@ class TestCommandFactory : public CommandParserFactory { class AdditionalFormatter : public FormatterProvider { public: // FormatterProvider - absl::optional - format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; - ProtobufWkt::Value - formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) const override; + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; }; class AdditionalCommandParser : public CommandParser { diff --git a/test/common/formatter/substitution_format_string_test.cc b/test/common/formatter/substitution_format_string_test.cc index e76c6a5a5cc3..756d10d1defd 100644 --- a/test/common/formatter/substitution_format_string_test.cc +++ b/test/common/formatter/substitution_format_string_test.cc @@ -51,7 +51,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigText) { auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("plain text, path=/bar/foo, code=200", formatter->format(request_headers_, response_headers_, response_trailers_, stream_info_, - body_)); + body_, AccessLog::AccessLogType::NotSet)); } TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigJson) { @@ -67,7 +67,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigJson) { auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); const auto out_json = formatter->format(request_headers_, response_headers_, response_trailers_, - stream_info_, body_); + stream_info_, body_, AccessLog::AccessLogType::NotSet); const std::string expected = R"EOF({ "text": "plain text", @@ -111,8 +111,9 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigFormatterExtension) TestUtility::loadFromYaml(yaml, config_); auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); - EXPECT_EQ("plain text TestFormatter", formatter->format(request_headers_, response_headers_, - response_trailers_, stream_info_, body_)); + EXPECT_EQ("plain text TestFormatter", + formatter->format(request_headers_, response_headers_, response_trailers_, stream_info_, + body_, AccessLog::AccessLogType::NotSet)); } TEST_F(SubstitutionFormatStringUtilsTest, @@ -171,7 +172,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigJsonWithExtension) auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); const auto out_json = formatter->format(request_headers_, response_headers_, response_trailers_, - stream_info_, body_); + stream_info_, body_, AccessLog::AccessLogType::NotSet); const std::string expected = R"EOF({ "text": "plain text TestFormatter", @@ -207,7 +208,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigJsonWithMultipleExt auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); const auto out_json = formatter->format(request_headers_, response_headers_, response_trailers_, - stream_info_, body_); + stream_info_, body_, AccessLog::AccessLogType::NotSet); const std::string expected = R"EOF({ "text": "plain text TestFormatter", diff --git a/test/common/formatter/substitution_formatter_fuzz_test.cc b/test/common/formatter/substitution_formatter_fuzz_test.cc index 6a758272a717..23ca22b3b8eb 100644 --- a/test/common/formatter/substitution_formatter_fuzz_test.cc +++ b/test/common/formatter/substitution_formatter_fuzz_test.cc @@ -24,7 +24,7 @@ DEFINE_PROTO_FUZZER(const test::common::substitution::TestCase& input) { Fuzz::fromStreamInfo(input.stream_info(), time_system); for (const auto& it : formatters) { it->format(request_headers, response_headers, response_trailers, *stream_info, - absl::string_view()); + absl::string_view(), AccessLog::AccessLogType::NotSet); } ENVOY_LOG_MISC(trace, "Success"); } catch (const EnvoyException& e) { diff --git a/test/common/formatter/substitution_formatter_speed_test.cc b/test/common/formatter/substitution_formatter_speed_test.cc index d80cefb6377d..79180f4c1217 100644 --- a/test/common/formatter/substitution_formatter_speed_test.cc +++ b/test/common/formatter/substitution_formatter_speed_test.cc @@ -91,9 +91,10 @@ static void BM_AccessLogFormatter(benchmark::State& state) { Http::TestResponseTrailerMapImpl response_trailers; std::string body; for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += - formatter->format(request_headers, response_headers, response_trailers, *stream_info, body) - .length(); + output_bytes += formatter + ->format(request_headers, response_headers, response_trailers, *stream_info, + body, AccessLog::AccessLogType::NotSet) + .length(); } benchmark::DoNotOptimize(output_bytes); } @@ -111,10 +112,10 @@ static void BM_StructAccessLogFormatter(benchmark::State& state) { Http::TestResponseTrailerMapImpl response_trailers; std::string body; for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += - struct_formatter - ->format(request_headers, response_headers, response_trailers, *stream_info, body) - .ByteSize(); + output_bytes += struct_formatter + ->format(request_headers, response_headers, response_trailers, *stream_info, + body, AccessLog::AccessLogType::NotSet) + .ByteSize(); } benchmark::DoNotOptimize(output_bytes); } @@ -133,10 +134,10 @@ static void BM_TypedStructAccessLogFormatter(benchmark::State& state) { Http::TestResponseTrailerMapImpl response_trailers; std::string body; for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += - typed_struct_formatter - ->format(request_headers, response_headers, response_trailers, *stream_info, body) - .ByteSize(); + output_bytes += typed_struct_formatter + ->format(request_headers, response_headers, response_trailers, *stream_info, + body, AccessLog::AccessLogType::NotSet) + .ByteSize(); } benchmark::DoNotOptimize(output_bytes); } @@ -154,10 +155,10 @@ static void BM_JsonAccessLogFormatter(benchmark::State& state) { Http::TestResponseTrailerMapImpl response_trailers; std::string body; for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += - json_formatter - ->format(request_headers, response_headers, response_trailers, *stream_info, body) - .length(); + output_bytes += json_formatter + ->format(request_headers, response_headers, response_trailers, *stream_info, + body, AccessLog::AccessLogType::NotSet) + .length(); } benchmark::DoNotOptimize(output_bytes); } @@ -176,10 +177,10 @@ static void BM_TypedJsonAccessLogFormatter(benchmark::State& state) { Http::TestResponseTrailerMapImpl response_trailers; std::string body; for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += - typed_json_formatter - ->format(request_headers, response_headers, response_trailers, *stream_info, body) - .length(); + output_bytes += typed_json_formatter + ->format(request_headers, response_headers, response_trailers, *stream_info, + body, AccessLog::AccessLogType::NotSet) + .length(); } benchmark::DoNotOptimize(output_bytes); } diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 04750f92d31d..05053d7b49d3 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -182,9 +182,9 @@ TEST(SubstitutionFormatterTest, plainStringFormatter) { std::string body; EXPECT_EQ("plain", formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("plain"))); } @@ -197,9 +197,9 @@ TEST(SubstitutionFormatterTest, plainNumberFormatter) { std::string body; EXPECT_EQ("400", formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(400))); } @@ -216,18 +216,18 @@ TEST(SubstitutionFormatterTest, inFlightDuration) { time_system.setMonotonicTime(MonotonicTime(std::chrono::milliseconds(100))); StreamInfoFormatter duration_format("DURATION"); EXPECT_EQ("100", duration_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { time_system.setMonotonicTime(MonotonicTime(std::chrono::milliseconds(200))); StreamInfoFormatter duration_format("DURATION"); EXPECT_EQ("200", duration_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); time_system.setMonotonicTime(MonotonicTime(std::chrono::milliseconds(300))); EXPECT_THAT(duration_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(300.0))); } } @@ -245,10 +245,12 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { { StreamInfoFormatter request_duration_format("REQUEST_DURATION"); - EXPECT_EQ(absl::nullopt, request_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + request_duration_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(request_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -257,20 +259,23 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_CALL(time_system, monotonicTime) .WillOnce(Return(MonotonicTime(std::chrono::nanoseconds(5000000)))); stream_info.downstream_timing_.onLastDownstreamRxByteReceived(time_system); - EXPECT_EQ("5", request_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("5", + request_duration_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(request_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(5.0))); } { StreamInfoFormatter request_tx_duration_format("REQUEST_TX_DURATION"); - EXPECT_EQ(absl::nullopt, - request_tx_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, request_tx_duration_format.format(request_headers, response_headers, + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); EXPECT_THAT(request_tx_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -280,18 +285,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { .WillOnce(Return(MonotonicTime(std::chrono::nanoseconds(15000000)))); upstream_timing.onLastUpstreamTxByteSent(time_system); EXPECT_EQ("15", request_tx_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); EXPECT_THAT(request_tx_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(15.0))); } { StreamInfoFormatter response_duration_format("RESPONSE_DURATION"); - EXPECT_EQ(absl::nullopt, response_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + response_duration_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(response_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -300,20 +309,24 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_CALL(time_system, monotonicTime) .WillOnce(Return(MonotonicTime(std::chrono::nanoseconds(10000000)))); upstream_timing.onFirstUpstreamRxByteReceived(time_system); - EXPECT_EQ("10", response_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("10", + response_duration_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(response_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(10.0))); } { StreamInfoFormatter ttlb_duration_format("RESPONSE_TX_DURATION"); - EXPECT_EQ(absl::nullopt, ttlb_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + ttlb_duration_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(ttlb_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -324,21 +337,24 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { .WillOnce(Return(MonotonicTime(std::chrono::nanoseconds(25000000)))); stream_info.downstream_timing_.onLastDownstreamTxByteSent(time_system); - EXPECT_EQ("15", ttlb_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("15", + ttlb_duration_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(ttlb_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(15.0))); } { StreamInfoFormatter handshake_duration_format("DOWNSTREAM_HANDSHAKE_DURATION"); - EXPECT_EQ(absl::nullopt, - handshake_duration_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + EXPECT_EQ(absl::nullopt, handshake_duration_format.format(request_headers, response_headers, + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); EXPECT_THAT(handshake_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -350,20 +366,23 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { stream_info.downstream_timing_.onDownstreamHandshakeComplete(time_system); EXPECT_EQ("25", handshake_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); EXPECT_THAT(handshake_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(25.0))); } { StreamInfoFormatter roundtrip_duration_format("ROUNDTRIP_DURATION"); - EXPECT_EQ(absl::nullopt, - roundtrip_duration_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + EXPECT_EQ(absl::nullopt, roundtrip_duration_format.format(request_headers, response_headers, + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); EXPECT_THAT(roundtrip_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -375,19 +394,23 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { stream_info.downstream_timing_.onLastDownstreamAckReceived(time_system); EXPECT_EQ("25", roundtrip_duration_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); EXPECT_THAT(roundtrip_duration_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(25.0))); } { StreamInfoFormatter bytes_received_format("BYTES_RECEIVED"); EXPECT_CALL(stream_info, bytesReceived()).WillRepeatedly(Return(1)); - EXPECT_EQ("1", bytes_received_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("1", + bytes_received_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(bytes_received_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(1.0))); } @@ -395,10 +418,12 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter attempt_count_format("UPSTREAM_REQUEST_ATTEMPT_COUNT"); absl::optional attempt_count{3}; EXPECT_CALL(stream_info, attemptCount()).WillRepeatedly(Return(attempt_count)); - EXPECT_EQ("3", attempt_count_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + EXPECT_EQ("3", + attempt_count_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(attempt_count_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(3.0))); } @@ -406,10 +431,12 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter attempt_count_format("UPSTREAM_REQUEST_ATTEMPT_COUNT"); absl::optional attempt_count; EXPECT_CALL(stream_info, attemptCount()).WillRepeatedly(Return(attempt_count)); - EXPECT_EQ("0", attempt_count_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + EXPECT_EQ("0", + attempt_count_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(attempt_count_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(0.0))); } @@ -421,9 +448,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_CALL(stream_info, getUpstreamBytesMeter()) .WillRepeatedly(ReturnRef(upstream_bytes_meter)); EXPECT_EQ("1", wire_bytes_received_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); EXPECT_THAT(wire_bytes_received_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(1.0))); } @@ -431,10 +460,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter protocol_format("PROTOCOL"); absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - EXPECT_EQ("HTTP/1.1", protocol_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("HTTP/1.1", + protocol_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(protocol_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("HTTP/1.1"))); } { @@ -442,19 +472,21 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter protocol_format("UPSTREAM_PROTOCOL"); EXPECT_CALL(stream_info, upstreamInfo()).WillRepeatedly(Return(nullptr)); - EXPECT_EQ(absl::nullopt, protocol_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + protocol_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(protocol_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; StreamInfoFormatter protocol_format("UPSTREAM_PROTOCOL"); - EXPECT_EQ(absl::nullopt, protocol_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + protocol_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(protocol_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -462,10 +494,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter protocol_format("UPSTREAM_PROTOCOL"); Http::Protocol protocol = Http::Protocol::Http2; stream_info.upstreamInfo()->setUpstreamProtocol(protocol); - EXPECT_EQ("HTTP/2", protocol_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + EXPECT_EQ("HTTP/2", + protocol_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(protocol_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("HTTP/2"))); } @@ -474,9 +507,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { absl::optional response_code{200}; EXPECT_CALL(stream_info, responseCode()).WillRepeatedly(Return(response_code)); EXPECT_EQ("200", response_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(response_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(200.0))); } @@ -484,10 +517,12 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter response_code_format("RESPONSE_CODE"); absl::optional response_code; EXPECT_CALL(stream_info, responseCode()).WillRepeatedly(Return(response_code)); - EXPECT_EQ("0", response_code_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + EXPECT_EQ("0", + response_code_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(response_code_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(0.0))); } @@ -495,10 +530,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter response_format("RESPONSE_CODE_DETAILS"); absl::optional rc_details; EXPECT_CALL(stream_info, responseCodeDetails()).WillRepeatedly(ReturnRef(rc_details)); - EXPECT_EQ(absl::nullopt, response_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + response_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(response_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -506,10 +542,12 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter response_code_format("RESPONSE_CODE_DETAILS"); absl::optional rc_details{"via_upstream"}; EXPECT_CALL(stream_info, responseCodeDetails()).WillRepeatedly(ReturnRef(rc_details)); - EXPECT_EQ("via_upstream", response_code_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("via_upstream", + response_code_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(response_code_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("via_upstream"))); } @@ -517,11 +555,12 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter termination_details_format("CONNECTION_TERMINATION_DETAILS"); absl::optional details; EXPECT_CALL(stream_info, connectionTerminationDetails()).WillRepeatedly(ReturnRef(details)); - EXPECT_EQ(absl::nullopt, - termination_details_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, termination_details_format.format(request_headers, response_headers, + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); EXPECT_THAT(termination_details_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -529,11 +568,12 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter termination_details_format("CONNECTION_TERMINATION_DETAILS"); absl::optional details{"access_denied"}; EXPECT_CALL(stream_info, connectionTerminationDetails()).WillRepeatedly(ReturnRef(details)); - EXPECT_EQ("access_denied", - termination_details_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("access_denied", termination_details_format.format( + request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(termination_details_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("access_denied"))); } @@ -541,9 +581,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter bytes_sent_format("BYTES_SENT"); EXPECT_CALL(stream_info, bytesSent()).WillRepeatedly(Return(1)); EXPECT_EQ("1", bytes_sent_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(bytes_sent_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(1.0))); } @@ -554,10 +594,12 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter wire_bytes_sent_format("UPSTREAM_WIRE_BYTES_SENT"); EXPECT_CALL(stream_info, getUpstreamBytesMeter()) .WillRepeatedly(ReturnRef(upstream_bytes_meter)); - EXPECT_EQ("1", wire_bytes_sent_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("1", + wire_bytes_sent_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(wire_bytes_sent_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(1.0))); } @@ -566,9 +608,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { absl::optional dur = std::chrono::nanoseconds(15000000); EXPECT_CALL(stream_info, currentDuration()).WillRepeatedly(Return(dur)); EXPECT_EQ("15", duration_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(duration_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(15.0))); } @@ -576,10 +618,12 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter response_flags_format("RESPONSE_FLAGS"); ON_CALL(stream_info, hasResponseFlag(StreamInfo::ResponseFlag::LocalReset)) .WillByDefault(Return(true)); - EXPECT_EQ("LR", response_flags_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("LR", + response_flags_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(response_flags_format.formatValue(request_headers, response_headers, - response_trailers, stream_info, body), + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("LR"))); } @@ -590,29 +634,31 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { auto address = Network::Address::InstanceConstSharedPtr{ new Network::Address::Ipv4Instance("127.1.2.3", 18443)}; stream_info.upstreamInfo()->setUpstreamLocalAddress(address); - EXPECT_EQ("127.1.2.3:18443", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("127.1.2.3:18443", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("127.1.2.3:18443"))); // Validate for IPv6 address address = Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv6Instance("::1", 19443)}; stream_info.upstreamInfo()->setUpstreamLocalAddress(address); - EXPECT_EQ("[::1]:19443", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("[::1]:19443", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("[::1]:19443"))); // Validate for Pipe address = Network::Address::InstanceConstSharedPtr{new Network::Address::PipeInstance("/foo")}; stream_info.upstreamInfo()->setUpstreamLocalAddress(address); EXPECT_EQ("/foo", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("/foo"))); } @@ -621,10 +667,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { auto address = Network::Address::InstanceConstSharedPtr{ new Network::Address::Ipv4Instance("127.0.0.3", 18443)}; stream_info.upstreamInfo()->setUpstreamLocalAddress(address); - EXPECT_EQ("127.0.0.3", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("127.0.0.3", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("127.0.0.3"))); } @@ -636,9 +683,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { new Network::Address::Ipv4Instance("127.1.2.3", 18443)}; stream_info.upstreamInfo()->setUpstreamLocalAddress(address); EXPECT_EQ("18443", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(18443))); { @@ -646,7 +693,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("18443"))); } @@ -655,9 +702,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv6Instance("::1", 19443)}; stream_info.upstreamInfo()->setUpstreamLocalAddress(address); EXPECT_EQ("19443", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(19443))); { @@ -665,7 +712,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("19443"))); } @@ -673,43 +720,46 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { address = Network::Address::InstanceConstSharedPtr{new Network::Address::PipeInstance("/foo")}; stream_info.upstreamInfo()->setUpstreamLocalAddress(address); EXPECT_EQ("", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { StreamInfoFormatter upstream_format("UPSTREAM_HOST"); - EXPECT_EQ("10.0.0.1:443", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("10.0.0.1:443", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("10.0.0.1:443"))); } { StreamInfoFormatter upstream_format("UPSTREAM_REMOTE_ADDRESS"); - EXPECT_EQ("10.0.0.1:443", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("10.0.0.1:443", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("10.0.0.1:443"))); } { StreamInfoFormatter upstream_format("UPSTREAM_REMOTE_ADDRESS_WITHOUT_PORT"); - EXPECT_EQ("10.0.0.1", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("10.0.0.1", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("10.0.0.1"))); } { StreamInfoFormatter upstream_format("UPSTREAM_REMOTE_PORT"); EXPECT_EQ("443", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(443))); { @@ -717,7 +767,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("443"))); } } @@ -730,10 +780,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_CALL(stream_info, upstreamClusterInfo()).WillRepeatedly(Return(cluster_info)); EXPECT_CALL(*cluster_info_mock, observabilityName()) .WillRepeatedly(ReturnRef(observable_cluster_name)); - EXPECT_EQ("observability_name", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("observability_name", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("observability_name"))); } @@ -741,20 +792,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter upstream_format("UPSTREAM_CLUSTER"); absl::optional cluster_info = nullptr; EXPECT_CALL(stream_info, upstreamClusterInfo()).WillRepeatedly(Return(cluster_info)); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { StreamInfoFormatter upstream_format("UPSTREAM_HOST"); stream_info.upstreamInfo()->setUpstreamHost(nullptr); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -767,10 +820,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { })); StreamInfoFormatter upstream_format("HOSTNAME"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -784,28 +838,31 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { })); StreamInfoFormatter upstream_format("HOSTNAME"); - EXPECT_EQ("myhostname", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("myhostname", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("myhostname"))); } { StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_ADDRESS"); - EXPECT_EQ("127.0.0.2:0", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("127.0.0.2:0", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("127.0.0.2:0"))); } { StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT"); - EXPECT_EQ("127.0.0.2", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("127.0.0.2", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("127.0.0.2"))); } @@ -817,9 +874,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { new Network::Address::Ipv4Instance("127.1.2.3", 8443)}; stream_info.downstream_connection_info_provider_->setLocalAddress(address); EXPECT_EQ("8443", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(8443))); { @@ -827,7 +884,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("8443"))); } @@ -836,9 +893,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv6Instance("::1", 9443)}; stream_info.downstream_connection_info_provider_->setLocalAddress(address); EXPECT_EQ("9443", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(9443))); { @@ -846,7 +903,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("9443"))); } @@ -854,36 +911,38 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { address = Network::Address::InstanceConstSharedPtr{new Network::Address::PipeInstance("/foo")}; stream_info.downstream_connection_info_provider_->setLocalAddress(address); EXPECT_EQ("", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { StreamInfoFormatter upstream_format("DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT"); - EXPECT_EQ("127.0.0.1", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("127.0.0.1", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("127.0.0.1"))); } { StreamInfoFormatter upstream_format("DOWNSTREAM_REMOTE_ADDRESS"); - EXPECT_EQ("127.0.0.1:0", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("127.0.0.1:0", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("127.0.0.1:0"))); } { StreamInfoFormatter upstream_format("DOWNSTREAM_REMOTE_PORT"); EXPECT_EQ("0", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(0))); { @@ -891,35 +950,37 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("0"))); } } { StreamInfoFormatter upstream_format("DOWNSTREAM_DIRECT_REMOTE_ADDRESS_WITHOUT_PORT"); - EXPECT_EQ("127.0.0.3", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("127.0.0.3", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("127.0.0.3"))); } { StreamInfoFormatter upstream_format("DOWNSTREAM_DIRECT_REMOTE_ADDRESS"); - EXPECT_EQ("127.0.0.3:63443", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("127.0.0.3:63443", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("127.0.0.3:63443"))); } { StreamInfoFormatter upstream_format("DOWNSTREAM_DIRECT_REMOTE_PORT"); EXPECT_EQ("63443", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(63443))); { @@ -927,7 +988,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("63443"))); } } @@ -937,9 +998,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { uint64_t id = 123; stream_info.downstream_connection_info_provider_->setConnectionID(id); EXPECT_EQ("123", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::numberValue(id))); } @@ -952,9 +1013,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_EQ("ffffffff-0012-0110-00ff-0c00400600ff", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("ffffffff-0012-0110-00ff-0c00400600ff"))); } @@ -962,10 +1023,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter upstream_format("REQUESTED_SERVER_NAME"); std::string requested_server_name = "stub_server"; stream_info.downstream_connection_info_provider_->setRequestedServerName(requested_server_name); - EXPECT_EQ("stub_server", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("stub_server", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("stub_server"))); } @@ -973,10 +1035,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter upstream_format("REQUESTED_SERVER_NAME"); std::string requested_server_name; stream_info.downstream_connection_info_provider_->setRequestedServerName(requested_server_name); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -984,20 +1047,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { StreamInfoFormatter listener_format("DOWNSTREAM_TRANSPORT_FAILURE_REASON"); std::string downstream_transport_failure_reason = "TLS error"; stream_info.setDownstreamTransportFailureReason(downstream_transport_failure_reason); - EXPECT_EQ("TLS_error", listener_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("TLS_error", + listener_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(listener_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("TLS_error"))); } { StreamInfoFormatter listener_format("DOWNSTREAM_TRANSPORT_FAILURE_REASON"); std::string downstream_transport_failure_reason; stream_info.setDownstreamTransportFailureReason(downstream_transport_failure_reason); - EXPECT_EQ(absl::nullopt, listener_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + listener_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(listener_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -1006,10 +1071,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { std::string upstream_transport_failure_reason = "SSL error"; stream_info.upstreamInfo()->setUpstreamTransportFailureReason( upstream_transport_failure_reason); - EXPECT_EQ("SSL_error", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("SSL_error", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("SSL_error"))); } { @@ -1017,10 +1083,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { std::string upstream_transport_failure_reason; stream_info.upstreamInfo()->setUpstreamTransportFailureReason( upstream_transport_failure_reason); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } } @@ -1039,14 +1106,15 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { std::string virtual_cluster_name = "authN"; stream_info.setVirtualClusterName(virtual_cluster_name); EXPECT_EQ("authN", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { NiceMock stream_info; StreamInfoFormatter upstream_format("VIRTUAL_CLUSTER_NAME"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); } { @@ -1058,9 +1126,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); EXPECT_EQ("san", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("san"))); } @@ -1071,8 +1139,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { const std::vector sans{"san1", "san2"}; EXPECT_CALL(*connection_info, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ("san1,san2", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("san1,san2", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); } { NiceMock stream_info; @@ -1081,20 +1150,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, uriSanPeerCertificate()) .WillRepeatedly(Return(std::vector())); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_URI_SAN"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1105,9 +1176,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, uriSanLocalCertificate()).WillRepeatedly(Return(sans)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); EXPECT_EQ("san", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("san"))); } { @@ -1117,8 +1188,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { const std::vector sans{"san1", "san2"}; EXPECT_CALL(*connection_info, uriSanLocalCertificate()).WillRepeatedly(Return(sans)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ("san1,san2", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("san1,san2", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); } { NiceMock stream_info; @@ -1127,20 +1199,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, uriSanLocalCertificate()) .WillRepeatedly(Return(std::vector())); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_URI_SAN"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1151,10 +1225,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, subjectLocalCertificate()) .WillRepeatedly(ReturnRef(subject_local)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ("subject", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("subject", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("subject"))); } { @@ -1164,20 +1239,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, subjectLocalCertificate()) .WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_SUBJECT"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1187,10 +1264,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { const std::string subject_peer = "subject"; EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(ReturnRef(subject_peer)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ("subject", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("subject", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("subject"))); } { @@ -1199,20 +1277,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1222,10 +1302,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { const std::string session_id = "deadbeef"; EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(ReturnRef(session_id)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ("deadbeef", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("deadbeef", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("deadbeef"))); } { @@ -1234,20 +1315,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_SESSION_ID"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1259,7 +1342,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); EXPECT_EQ("TLS_DHE_RSA_WITH_AES_256_GCM_SHA384", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { NiceMock stream_info; @@ -1267,20 +1350,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, ciphersuiteString()).WillRepeatedly(Return("")); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_CIPHER"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1290,10 +1375,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { std::string tlsVersion = "TLSv1.2"; EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(ReturnRef(tlsVersion)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ("TLSv1.2", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("TLSv1.2", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("TLSv1.2"))); } { @@ -1302,10 +1388,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1313,10 +1400,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_VERSION"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1327,10 +1415,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, sha256PeerCertificateDigest()) .WillRepeatedly(ReturnRef(expected_sha)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(expected_sha, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(expected_sha, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue(expected_sha))); } { @@ -1341,20 +1430,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, sha256PeerCertificateDigest()) .WillRepeatedly(ReturnRef(expected_sha)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_FINGERPRINT_256"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1365,10 +1456,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, sha1PeerCertificateDigest()) .WillRepeatedly(ReturnRef(expected_sha)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(expected_sha, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(expected_sha, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue(expected_sha))); } { @@ -1379,20 +1471,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, sha1PeerCertificateDigest()) .WillRepeatedly(ReturnRef(expected_sha)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_FINGERPRINT_1"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1403,10 +1497,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, serialNumberPeerCertificate()) .WillRepeatedly(ReturnRef(serial_number)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ("b8b5ecc898f2124a", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("b8b5ecc898f2124a", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("b8b5ecc898f2124a"))); } { @@ -1416,20 +1511,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, serialNumberPeerCertificate()) .WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SERIAL"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1442,7 +1539,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); EXPECT_EQ("CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { NiceMock stream_info; @@ -1450,20 +1547,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, issuerPeerCertificate()).WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_ISSUER"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1476,7 +1575,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); EXPECT_EQ("CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { NiceMock stream_info; @@ -1484,20 +1583,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1508,10 +1609,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) .WillRepeatedly(ReturnRef(expected_cert)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(expected_cert, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(expected_cert, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue(expected_cert))); } { @@ -1522,20 +1624,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) .WillRepeatedly(ReturnRef(expected_cert)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1543,20 +1647,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { StreamInfoFormatter upstream_format("UPSTREAM_TLS_SESSION_ID"); EXPECT_CALL(stream_info, upstreamInfo()).WillRepeatedly(Return(nullptr)); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.upstreamInfo()->setUpstreamSslConnection(nullptr); StreamInfoFormatter upstream_format("UPSTREAM_TLS_SESSION_ID"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1566,10 +1672,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { const std::string session_id = "deadbeef"; EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(ReturnRef(session_id)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ("deadbeef", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("deadbeef", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("deadbeef"))); } { @@ -1578,10 +1685,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1589,20 +1697,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { StreamInfoFormatter upstream_format("UPSTREAM_TLS_CIPHER"); EXPECT_CALL(stream_info, upstreamInfo()).WillRepeatedly(Return(nullptr)); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.upstreamInfo()->setUpstreamSslConnection(nullptr); StreamInfoFormatter upstream_format("UPSTREAM_TLS_CIPHER"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1614,7 +1724,7 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); EXPECT_EQ("TLS_DHE_RSA_WITH_AES_256_GCM_SHA384", upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { NiceMock stream_info; @@ -1622,10 +1732,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, ciphersuiteString()).WillRepeatedly(Return("")); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1633,20 +1744,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { StreamInfoFormatter upstream_format("UPSTREAM_TLS_VERSION"); EXPECT_CALL(stream_info, upstreamInfo()).WillRepeatedly(Return(nullptr)); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.upstreamInfo()->setUpstreamSslConnection(nullptr); StreamInfoFormatter upstream_format("UPSTREAM_TLS_VERSION"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1656,10 +1769,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { std::string tlsVersion = "TLSv1.2"; EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(ReturnRef(tlsVersion)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ("TLSv1.2", upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("TLSv1.2", + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("TLSv1.2"))); } { @@ -1668,10 +1782,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1679,20 +1794,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { StreamInfoFormatter upstream_format("UPSTREAM_PEER_ISSUER"); EXPECT_CALL(stream_info, upstreamInfo()).WillRepeatedly(Return(nullptr)); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.upstreamInfo()->setUpstreamSslConnection(nullptr); StreamInfoFormatter upstream_format("UPSTREAM_PEER_ISSUER"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1701,10 +1818,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, issuerPeerCertificate()).WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1715,30 +1833,33 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"; EXPECT_CALL(*connection_info, issuerPeerCertificate()).WillRepeatedly(ReturnRef(issuer_peer)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(issuer_peer, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(issuer_peer, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue(issuer_peer))); } { NiceMock stream_info; StreamInfoFormatter upstream_format("UPSTREAM_PEER_CERT"); EXPECT_CALL(stream_info, upstreamInfo()).WillRepeatedly(Return(nullptr)); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.upstreamInfo()->setUpstreamSslConnection(nullptr); StreamInfoFormatter upstream_format("UPSTREAM_PEER_CERT"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1748,10 +1869,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) .WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1762,10 +1884,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) .WillRepeatedly(ReturnRef(expected_cert)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(expected_cert, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(expected_cert, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue(expected_cert))); } @@ -1773,20 +1896,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { NiceMock stream_info; StreamInfoFormatter upstream_format("UPSTREAM_PEER_SUBJECT"); EXPECT_CALL(stream_info, upstreamInfo()).WillRepeatedly(Return(nullptr)); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { NiceMock stream_info; stream_info.upstreamInfo()->setUpstreamSslConnection(nullptr); StreamInfoFormatter upstream_format("UPSTREAM_PEER_SUBJECT"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1795,10 +1920,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(ReturnRef(EMPTY_STRING)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { @@ -1809,9 +1935,9 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(ReturnRef(subject)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); EXPECT_EQ(subject, upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue(subject))); } } @@ -1826,46 +1952,46 @@ TEST(SubstitutionFormatterTest, requestHeaderFormatter) { { RequestHeaderFormatter formatter(":Method", "", absl::optional()); EXPECT_EQ("GET", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("GET"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("GET"))); } { RequestHeaderFormatter formatter(":path", ":method", absl::optional()); EXPECT_EQ("/", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("/"))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("/"))); } { RequestHeaderFormatter formatter(":TEST", ":METHOD", absl::optional()); EXPECT_EQ("GET", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("GET"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("GET"))); } { RequestHeaderFormatter formatter("does_not_exist", "", absl::optional()); EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::nullValue())); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); } { RequestHeaderFormatter formatter(":Method", "", absl::optional(2)); EXPECT_EQ("GE", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("GE"))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("GE"))); } } @@ -1878,30 +2004,30 @@ TEST(SubstitutionFormatterTest, headersByteSizeFormatter) { { HeadersByteSizeFormatter formatter(HeadersByteSizeFormatter::HeaderType::RequestHeaders); - EXPECT_EQ( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - "16"); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::numberValue(16))); + EXPECT_EQ(formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet), + "16"); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(16))); } { HeadersByteSizeFormatter formatter(HeadersByteSizeFormatter::HeaderType::ResponseHeaders); - EXPECT_EQ( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - "10"); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::numberValue(10))); + EXPECT_EQ(formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet), + "10"); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(10))); } { HeadersByteSizeFormatter formatter(HeadersByteSizeFormatter::HeaderType::ResponseTrailers); - EXPECT_EQ( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - "23"); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::numberValue(23))); + EXPECT_EQ(formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet), + "23"); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(23))); } } @@ -1915,46 +2041,46 @@ TEST(SubstitutionFormatterTest, responseHeaderFormatter) { { ResponseHeaderFormatter formatter(":method", "", absl::optional()); EXPECT_EQ("PUT", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("PUT"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("PUT"))); } { ResponseHeaderFormatter formatter("test", ":method", absl::optional()); EXPECT_EQ("test", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("test"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("test"))); } { ResponseHeaderFormatter formatter(":path", ":method", absl::optional()); EXPECT_EQ("PUT", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("PUT"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("PUT"))); } { ResponseHeaderFormatter formatter("does_not_exist", "", absl::optional()); EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::nullValue())); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); } { ResponseHeaderFormatter formatter(":method", "", absl::optional(2)); EXPECT_EQ("PU", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("PU"))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("PU"))); } } @@ -1968,46 +2094,46 @@ TEST(SubstitutionFormatterTest, responseTrailerFormatter) { { ResponseTrailerFormatter formatter(":method", "", absl::optional()); EXPECT_EQ("POST", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("POST"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("POST"))); } { ResponseTrailerFormatter formatter("test-2", ":method", absl::optional()); EXPECT_EQ("test-2", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("test-2"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("test-2"))); } { ResponseTrailerFormatter formatter(":path", ":method", absl::optional()); EXPECT_EQ("POST", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("POST"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("POST"))); } { ResponseTrailerFormatter formatter("does_not_exist", "", absl::optional()); EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::nullValue())); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); } { ResponseTrailerFormatter formatter(":method", "", absl::optional(2)); EXPECT_EQ("PO", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("PO"))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("PO"))); } } @@ -2040,46 +2166,47 @@ TEST(SubstitutionFormatterTest, DynamicMetadataFormatter) { { DynamicMetadataFormatter formatter("com.test", {}, absl::optional()); - std::string val = - formatter.format(request_headers, response_headers, response_trailers, stream_info, body) - .value(); + std::string val = formatter + .format(request_headers, response_headers, response_trailers, stream_info, + body, AccessLog::AccessLogType::NotSet) + .value(); EXPECT_TRUE(val.find("\"test_key\":\"test_value\"") != std::string::npos); EXPECT_TRUE(val.find("\"test_obj\":{\"inner_key\":\"inner_value\"}") != std::string::npos); ProtobufWkt::Value expected_val; expected_val.mutable_struct_value()->CopyFrom(metadata.filter_metadata().at("com.test")); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(expected_val)); } { DynamicMetadataFormatter formatter("com.test", {"test_key"}, absl::optional()); EXPECT_EQ("test_value", formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("test_value"))); } { DynamicMetadataFormatter formatter("com.test", {"test_obj"}, absl::optional()); - EXPECT_EQ( - "{\"inner_key\":\"inner_value\"}", - formatter.format(request_headers, response_headers, response_trailers, stream_info, body)); + EXPECT_EQ("{\"inner_key\":\"inner_value\"}", + formatter.format(request_headers, response_headers, response_trailers, stream_info, + body, AccessLog::AccessLogType::NotSet)); ProtobufWkt::Value expected_val; (*expected_val.mutable_struct_value()->mutable_fields())["inner_key"] = ValueUtil::stringValue("inner_value"); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(expected_val)); } { DynamicMetadataFormatter formatter("com.test", {"test_obj", "inner_key"}, absl::optional()); EXPECT_EQ("inner_value", formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("inner_value"))); } @@ -2087,26 +2214,26 @@ TEST(SubstitutionFormatterTest, DynamicMetadataFormatter) { { DynamicMetadataFormatter formatter("com.notfound", {}, absl::optional()); EXPECT_EQ(absl::nullopt, formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { DynamicMetadataFormatter formatter("com.test", {"notfound"}, absl::optional()); EXPECT_EQ(absl::nullopt, formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } { DynamicMetadataFormatter formatter("com.test", {"test_obj", "notfound"}, absl::optional()); EXPECT_EQ(absl::nullopt, formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -2114,11 +2241,11 @@ TEST(SubstitutionFormatterTest, DynamicMetadataFormatter) { { DynamicMetadataFormatter formatter("com.test", {"test_key"}, absl::optional(5)); EXPECT_EQ("test_", formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); // N.B. Does not truncate. EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("test_value"))); } } @@ -2152,25 +2279,26 @@ TEST(SubstitutionFormatterTest, FilterStateFormatter) { { FilterStateFormatter formatter("key", absl::optional(), false); - EXPECT_EQ("\"test_value\"", formatter.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("\"test_value\"", + formatter.format(request_headers, response_headers, response_trailers, stream_info, + body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("test_value"))); } { FilterStateFormatter formatter("key-struct", absl::optional(), false); - EXPECT_EQ( - "{\"inner_key\":\"inner_value\"}", - formatter.format(request_headers, response_headers, response_trailers, stream_info, body)); + EXPECT_EQ("{\"inner_key\":\"inner_value\"}", + formatter.format(request_headers, response_headers, response_trailers, stream_info, + body, AccessLog::AccessLogType::NotSet)); ProtobufWkt::Value expected; (*expected.mutable_struct_value()->mutable_fields())["inner_key"] = ValueUtil::stringValue("inner_value"); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(expected)); } @@ -2179,9 +2307,9 @@ TEST(SubstitutionFormatterTest, FilterStateFormatter) { FilterStateFormatter formatter("key-not-found", absl::optional(), false); EXPECT_EQ(absl::nullopt, formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -2190,9 +2318,9 @@ TEST(SubstitutionFormatterTest, FilterStateFormatter) { FilterStateFormatter formatter("key-no-serialization", absl::optional(), false); EXPECT_EQ(absl::nullopt, formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -2201,9 +2329,9 @@ TEST(SubstitutionFormatterTest, FilterStateFormatter) { FilterStateFormatter formatter("key-serialization-error", absl::optional(), false); EXPECT_EQ(absl::nullopt, formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } @@ -2212,11 +2340,11 @@ TEST(SubstitutionFormatterTest, FilterStateFormatter) { FilterStateFormatter formatter("key", absl::optional(5), false); EXPECT_EQ("\"test", formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); // N.B. Does not truncate. EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("test_value"))); } @@ -2224,8 +2352,9 @@ TEST(SubstitutionFormatterTest, FilterStateFormatter) { { FilterStateFormatter formatter("test_key", absl::optional(), true); - EXPECT_EQ("test_value By PLAIN", formatter.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("test_value By PLAIN", + formatter.format(request_headers, response_headers, response_trailers, stream_info, + body, AccessLog::AccessLogType::NotSet)); } // size limit for serializeAsString @@ -2233,7 +2362,7 @@ TEST(SubstitutionFormatterTest, FilterStateFormatter) { FilterStateFormatter formatter("test_key", absl::optional(10), true); EXPECT_EQ("test_value", formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } // no serialization case for serializeAsString @@ -2241,9 +2370,9 @@ TEST(SubstitutionFormatterTest, FilterStateFormatter) { FilterStateFormatter formatter("key-no-serialization", absl::optional(), true); EXPECT_EQ(absl::nullopt, formatter.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(formatter.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } } @@ -2259,10 +2388,11 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVStartFormatter) { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); DownstreamPeerCertVStartFormatter cert_start_formart("DOWNSTREAM_PEER_CERT_V_START(%Y/%m/%d)"); - EXPECT_EQ(absl::nullopt, cert_start_formart.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_start_formart.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_start_formart.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // No validFromPeerCertificate @@ -2272,10 +2402,11 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVStartFormatter) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(absl::nullopt)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, cert_start_formart.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_start_formart.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_start_formart.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // Default format string @@ -2289,7 +2420,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVStartFormatter) { stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); EXPECT_EQ(AccessLogDateTimeFormatter::fromTime(time), cert_start_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } // Custom format string { @@ -2302,7 +2433,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVStartFormatter) { stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); EXPECT_EQ("Mar 28 23:35:58 2018 UTC", cert_start_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } } @@ -2317,10 +2448,11 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVEndFormatter) { NiceMock stream_info; stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); DownstreamPeerCertVEndFormatter cert_end_format("%Y/%m/%d"); - EXPECT_EQ(absl::nullopt, cert_end_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_end_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_end_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // No expirationPeerCertificate @@ -2331,10 +2463,11 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVEndFormatter) { EXPECT_CALL(*connection_info, expirationPeerCertificate()) .WillRepeatedly(Return(absl::nullopt)); stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, cert_end_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_end_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_end_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // Default format string @@ -2348,7 +2481,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVEndFormatter) { stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); EXPECT_EQ(AccessLogDateTimeFormatter::fromTime(time), cert_end_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } // Custom format string { @@ -2361,7 +2494,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVEndFormatter) { stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); EXPECT_EQ("Mar 28 23:35:58 2018 UTC", cert_end_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } } @@ -2376,10 +2509,11 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVStartFormatter) { NiceMock stream_info; EXPECT_CALL(stream_info, upstreamInfo()).WillRepeatedly(Return(nullptr)); UpstreamPeerCertVStartFormatter cert_start_format("%Y/%m/%d"); - EXPECT_EQ(absl::nullopt, cert_start_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_start_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_start_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // No upstreamSslConnection @@ -2387,10 +2521,11 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVStartFormatter) { NiceMock stream_info; stream_info.upstreamInfo()->setUpstreamSslConnection(nullptr); DownstreamPeerCertVStartFormatter cert_start_format("UPSTREAM_PEER_CERT_V_START(%Y/%m/%d)"); - EXPECT_EQ(absl::nullopt, cert_start_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_start_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_start_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // No validFromPeerCertificate @@ -2400,10 +2535,11 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVStartFormatter) { auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(absl::nullopt)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, cert_start_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_start_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_start_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // Default format string @@ -2417,7 +2553,7 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVStartFormatter) { stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); EXPECT_EQ(AccessLogDateTimeFormatter::fromTime(time), cert_start_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } // Custom format string { @@ -2430,7 +2566,7 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVStartFormatter) { stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); EXPECT_EQ("Mar 28 23:35:58 2018 UTC", cert_start_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } } @@ -2445,10 +2581,11 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVEndFormatter) { NiceMock stream_info; EXPECT_CALL(stream_info, upstreamInfo()).WillRepeatedly(Return(nullptr)); UpstreamPeerCertVEndFormatter cert_end_format("%Y/%m/%d"); - EXPECT_EQ(absl::nullopt, cert_end_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_end_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_end_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // No upstreamSslConnection @@ -2456,10 +2593,11 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVEndFormatter) { NiceMock stream_info; stream_info.upstreamInfo()->setUpstreamSslConnection(nullptr); UpstreamPeerCertVEndFormatter cert_end_format("%Y/%m/%d"); - EXPECT_EQ(absl::nullopt, cert_end_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_end_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_end_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // No expirationPeerCertificate @@ -2470,10 +2608,11 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVEndFormatter) { EXPECT_CALL(*connection_info, expirationPeerCertificate()) .WillRepeatedly(Return(absl::nullopt)); stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); - EXPECT_EQ(absl::nullopt, cert_end_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ(absl::nullopt, + cert_end_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(cert_end_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::nullValue())); } // Default format string @@ -2487,7 +2626,7 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVEndFormatter) { stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); EXPECT_EQ(AccessLogDateTimeFormatter::fromTime(time), cert_end_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } // Custom format string { @@ -2500,7 +2639,7 @@ TEST(SubstitutionFormatterTest, UpstreamPeerCertVEndFormatter) { stream_info.upstreamInfo()->setUpstreamSslConnection(connection_info); EXPECT_EQ("Mar 28 23:35:58 2018 UTC", cert_end_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } } @@ -2516,10 +2655,11 @@ TEST(SubstitutionFormatterTest, StartTimeFormatter) { time_t test_epoch = 1522280158; SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); - EXPECT_EQ("2018/03/28", start_time_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("2018/03/28", + start_time_format.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(start_time_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("2018/03/28"))); } @@ -2529,9 +2669,9 @@ TEST(SubstitutionFormatterTest, StartTimeFormatter) { EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); EXPECT_EQ(AccessLogDateTimeFormatter::fromTime(time), start_time_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); EXPECT_THAT(start_time_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), + stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue(AccessLogDateTimeFormatter::fromTime(time)))); } } @@ -2552,48 +2692,49 @@ TEST(SubstitutionFormatterTest, GrpcStatusFormatterCamelStringTest) { "DataLoss", "Unauthenticated"}; for (size_t i = 0; i < grpc_statuses.size(); ++i) { response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", std::to_string(i)}}; - EXPECT_EQ(grpc_statuses[i], formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue(grpc_statuses[i]))); + EXPECT_EQ(grpc_statuses[i], + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue(grpc_statuses[i]))); } { response_trailer = Http::TestResponseTrailerMapImpl{{"not-a-grpc-status", "13"}}; EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::nullValue())); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); } { response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "-1"}}; EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("-1"))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("-1"))); response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "42738"}}; EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("42738"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("42738"))); response_trailer.clear(); } { response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "-1"}}; EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("-1"))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("-1"))); response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "42738"}}; EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("42738"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("42738"))); response_header.clear(); } } @@ -2626,48 +2767,49 @@ TEST(SubstitutionFormatterTest, GrpcStatusFormatterSnakeStringTest) { "UNAUTHENTICATED"}; for (size_t i = 0; i < grpc_statuses.size(); ++i) { response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", std::to_string(i)}}; - EXPECT_EQ(grpc_statuses[i], formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue(grpc_statuses[i]))); + EXPECT_EQ(grpc_statuses[i], + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue(grpc_statuses[i]))); } { response_trailer = Http::TestResponseTrailerMapImpl{{"not-a-grpc-status", "13"}}; EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::nullValue())); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); } { response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "-1"}}; EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("-1"))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("-1"))); response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "42738"}}; EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("42738"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("42738"))); response_trailer.clear(); } { response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "-1"}}; EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("-1"))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("-1"))); response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "42738"}}; EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::stringValue("42738"))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("42738"))); response_header.clear(); } } @@ -2685,48 +2827,49 @@ TEST(SubstitutionFormatterTest, GrpcStatusFormatterNumberTest) { for (size_t i = 0; i < grpcStatuses; ++i) { response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", std::to_string(i)}}; - EXPECT_EQ(std::to_string(i), formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::numberValue(i))); + EXPECT_EQ(std::to_string(i), + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(i))); } { response_trailer = Http::TestResponseTrailerMapImpl{{"not-a-grpc-status", "13"}}; EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::nullValue())); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); } { response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "-1"}}; EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::numberValue(-1))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(-1))); response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "42738"}}; EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::numberValue(42738))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(42738))); response_trailer.clear(); } { response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "-1"}}; EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::numberValue(-1))); + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(-1))); response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "42738"}}; EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); - EXPECT_THAT( - formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), - ProtoEq(ValueUtil::numberValue(42738))); + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(42738))); response_header.clear(); } } @@ -2763,9 +2906,9 @@ TEST(SubstitutionFormatterTest, StructFormatterPlainStringTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterPlainNumberTest) { @@ -2789,9 +2932,9 @@ TEST(SubstitutionFormatterTest, StructFormatterPlainNumberTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterTypesTest) { @@ -2831,7 +2974,8 @@ TEST(SubstitutionFormatterTest, StructFormatterTypesTest) { ] })EOF"); const ProtobufWkt::Struct out_struct = - formatter.format(request_header, response_header, response_trailer, stream_info, body); + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet); EXPECT_TRUE(TestUtility::protoEqual(out_struct, expected)); } @@ -2955,7 +3099,8 @@ TEST(SubstitutionFormatterTest, StructFormatterNestedObjectsTest) { ], })EOF"); const ProtobufWkt::Struct out_struct = - formatter.format(request_header, response_header, response_trailer, stream_info, body); + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet); EXPECT_TRUE(TestUtility::protoEqual(out_struct, expected)); } @@ -2980,9 +3125,9 @@ TEST(SubstitutionFormatterTest, StructFormatterSingleOperatorTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, EmptyStructFormatterTest) { @@ -3006,9 +3151,9 @@ TEST(SubstitutionFormatterTest, EmptyStructFormatterTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterNonExistentHeaderTest) { @@ -3037,9 +3182,9 @@ TEST(SubstitutionFormatterTest, StructFormatterNonExistentHeaderTest) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterAlternateHeaderTest) { @@ -3074,9 +3219,9 @@ TEST(SubstitutionFormatterTest, StructFormatterAlternateHeaderTest) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterDynamicMetadataTest) { @@ -3105,9 +3250,9 @@ TEST(SubstitutionFormatterTest, StructFormatterDynamicMetadataTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterTypedDynamicMetadataTest) { @@ -3132,7 +3277,8 @@ TEST(SubstitutionFormatterTest, StructFormatterTypedDynamicMetadataTest) { StructFormatter formatter(key_mapping, true, false); ProtobufWkt::Struct output = - formatter.format(request_header, response_header, response_trailer, stream_info, body); + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet); const auto& fields = output.fields(); EXPECT_EQ("test_value", fields.at("test_key").string_value()); @@ -3173,9 +3319,9 @@ TEST(SubstitutionFormatterTest, StructFormatterClusterMetadataTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterTypedClusterMetadataTest) { @@ -3203,7 +3349,8 @@ TEST(SubstitutionFormatterTest, StructFormatterTypedClusterMetadataTest) { StructFormatter formatter(key_mapping, true, false); ProtobufWkt::Struct output = - formatter.format(request_header, response_header, response_trailer, stream_info, body); + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet); const auto& fields = output.fields(); EXPECT_EQ("test_value", fields.at("test_key").string_value()); @@ -3231,16 +3378,16 @@ TEST(SubstitutionFormatterTest, StructFormatterClusterMetadataNoClusterInfoTest) // Empty optional (absl::nullopt) { EXPECT_CALL(Const(stream_info), upstreamClusterInfo()).WillOnce(Return(absl::nullopt)); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } // Empty cluster info (nullptr) { EXPECT_CALL(Const(stream_info), upstreamClusterInfo()).WillOnce(Return(nullptr)); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } } @@ -3278,9 +3425,9 @@ TEST(SubstitutionFormatterTest, StructFormatterUpstreamHostMetadataTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterUpstreamHostMetadataNullPtrs) { @@ -3302,9 +3449,9 @@ TEST(SubstitutionFormatterTest, StructFormatterUpstreamHostMetadataNullPtrs) { // Empty optional (absl::nullopt) { EXPECT_CALL(Const(stream_info), upstreamInfo()).WillOnce(Return(absl::nullopt)); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); testing::Mock::VerifyAndClearExpectations(&stream_info); } // Empty host description info (nullptr) @@ -3312,9 +3459,9 @@ TEST(SubstitutionFormatterTest, StructFormatterUpstreamHostMetadataNullPtrs) { std::shared_ptr mock_upstream_info = std::dynamic_pointer_cast(stream_info.upstreamInfo()); EXPECT_CALL(*mock_upstream_info, upstreamHost()).WillOnce(Return(nullptr)); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } } @@ -3343,9 +3490,9 @@ TEST(SubstitutionFormatterTest, StructFormatterFilterStateTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_headers, response_headers, response_trailers, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterOmitEmptyTest) { @@ -3368,9 +3515,9 @@ TEST(SubstitutionFormatterTest, StructFormatterOmitEmptyTest) { key_mapping); StructFormatter formatter(key_mapping, false, true); - verifyStructOutput( - formatter.format(request_headers, response_headers, response_trailers, stream_info, body), - {}); + verifyStructOutput(formatter.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet), + {}); } TEST(SubstitutionFormatterTest, StructFormatterOmitEmptyNestedTest) { @@ -3394,9 +3541,9 @@ TEST(SubstitutionFormatterTest, StructFormatterOmitEmptyNestedTest) { key_mapping); StructFormatter formatter(key_mapping, false, true); - verifyStructOutput( - formatter.format(request_headers, response_headers, response_trailers, stream_info, body), - {}); + verifyStructOutput(formatter.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet), + {}); } TEST(SubstitutionFormatterTest, StructFormatterTypedFilterStateTest) { @@ -3422,7 +3569,8 @@ TEST(SubstitutionFormatterTest, StructFormatterTypedFilterStateTest) { StructFormatter formatter(key_mapping, true, false); ProtobufWkt::Struct output = - formatter.format(request_headers, response_headers, response_trailers, stream_info, body); + formatter.format(request_headers, response_headers, response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet); const auto& fields = output.fields(); EXPECT_EQ("test_value", fields.at("test_key").string_value()); @@ -3456,9 +3604,9 @@ TEST(SubstitutionFormatterTest, FilterStateSpeciferTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_headers, response_headers, response_trailers, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } // Test new specifier (PLAIN/TYPED) of FilterState and convert the output log string to proto @@ -3483,7 +3631,8 @@ TEST(SubstitutionFormatterTest, TypedFilterStateSpeciferTest) { StructFormatter formatter(key_mapping, true, false); ProtobufWkt::Struct output = - formatter.format(request_headers, response_headers, response_trailers, stream_info, body); + formatter.format(request_headers, response_headers, response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet); const auto& fields = output.fields(); EXPECT_EQ("test_value By PLAIN", fields.at("test_key_plain").string_value()); @@ -3541,9 +3690,9 @@ TEST(SubstitutionFormatterTest, StructFormatterStartTimeTest) { key_mapping); StructFormatter formatter(key_mapping, false, false); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } TEST(SubstitutionFormatterTest, StructFormatterMultiTokenTest) { @@ -3570,9 +3719,9 @@ TEST(SubstitutionFormatterTest, StructFormatterMultiTokenTest) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - verifyStructOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected_json_map); + verifyStructOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected_json_map); } } } @@ -3611,7 +3760,8 @@ TEST(SubstitutionFormatterTest, StructFormatterTypedTest) { StructFormatter formatter(key_mapping, true, false); ProtobufWkt::Struct output = - formatter.format(request_headers, response_headers, response_trailers, stream_info, body); + formatter.format(request_headers, response_headers, response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet); EXPECT_THAT(output.fields().at("request_duration"), ProtoEq(ValueUtil::numberValue(5.0))); EXPECT_THAT(output.fields().at("request_duration_multi"), ProtoEq(ValueUtil::stringValue("5ms"))); @@ -3656,7 +3806,8 @@ TEST(SubstitutionFormatterTest, JsonFormatterTest) { })EOF"; const std::string out_json = - formatter.format(request_header, response_header, response_trailer, stream_info, body); + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet); EXPECT_TRUE(TestUtility::jsonStringEqual(out_json, expected)); } @@ -3676,9 +3827,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - EXPECT_EQ( - "{{HTTP/1.1}} -++test GET PUT\t@POST@\ttest-2[]", - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ("{{HTTP/1.1}} -++test GET PUT\t@POST@\ttest-2[]", + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3687,7 +3838,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { FormatterImpl formatter(format, false); EXPECT_EQ(format, formatter.format(request_header, response_header, response_trailer, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { @@ -3697,8 +3848,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { FormatterImpl formatter(format, false); - EXPECT_EQ("GET|G|PU|GET|POS", formatter.format(request_header, response_header, - response_trailer, stream_info, body)); + EXPECT_EQ("GET|G|PU|GET|POS", + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3711,9 +3863,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { "test_obj)%|%DYNAMIC_METADATA(com.test:test_obj:inner_key)%"; FormatterImpl formatter(format, false); - EXPECT_EQ( - "test_value|{\"inner_key\":\"inner_value\"}|inner_value", - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ("test_value|{\"inner_key\":\"inner_value\"}|inner_value", + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3731,9 +3883,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { "%FILTER_STATE(testing):8%|%FILTER_STATE(nonexisting)%"; FormatterImpl formatter(format, false); - EXPECT_EQ( - "\"test_value\"|-|\"test_va|-", - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ("\"test_value\"|-|\"test_va|-", + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3747,10 +3899,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); FormatterImpl formatter(format, false); - EXPECT_EQ( - fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", - expected_time_in_epoch), - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ(fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", + expected_time_in_epoch), + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3768,10 +3920,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); FormatterImpl formatter(format, false); - EXPECT_EQ( - fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", - expected_time_in_epoch), - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ(fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", + expected_time_in_epoch), + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3789,10 +3941,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); FormatterImpl formatter(format, false); - EXPECT_EQ( - fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", - expected_time_in_epoch), - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ(fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", + expected_time_in_epoch), + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3806,9 +3958,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); FormatterImpl formatter(format, false); - EXPECT_EQ( - "1970/01/01|0|bad_format|1970-01-01T00:00:00.000Z|000000000.0.00.000", - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ("1970/01/01|0|bad_format|1970-01-01T00:00:00.000Z|000000000.0.00.000", + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3819,9 +3971,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(start_time)); FormatterImpl formatter(format, false); - EXPECT_EQ( - "1522796769.123|1522796769.1234|1522796769.12345|1522796769.123456", - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ("1522796769.123|1522796769.1234|1522796769.12345|1522796769.123456", + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3831,10 +3983,10 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(start_time)); FormatterImpl formatter(format, false); - EXPECT_EQ( - "segment1:1522796769.123|segment2:1522796769.1234|seg3:1522796769.123456|1522796769-" - "123-asdf-123456000|.1234560:segm5:2018", - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ("segment1:1522796769.123|segment2:1522796769.1234|seg3:1522796769.123456|1522796769-" + "123-asdf-123456000|.1234560:segm5:2018", + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } { @@ -3845,9 +3997,9 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillOnce(Return(start_time)); FormatterImpl formatter(format, false); - EXPECT_EQ( - "%%|%%123456000|1522796769%%123|1%%1522796769", - formatter.format(request_header, response_header, response_trailer, stream_info, body)); + EXPECT_EQ("%%|%%123456000|1522796769%%123|1%%1522796769", + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); } // The %E formatting option in Absl::FormatTime() behaves differently for non Linux platforms. @@ -3863,7 +4015,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { EXPECT_CALL(stream_info, startTime()).WillOnce(Return(start_time)); FormatterImpl formatter(format); EXPECT_EQ("%E4n", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } #endif } @@ -3884,7 +4036,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(absl::nullopt)); EXPECT_EQ("-|-|-|-|-|-", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { @@ -3896,7 +4048,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(absl::nullopt)); EXPECT_EQ("||||", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { @@ -3908,7 +4060,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { FormatterImpl formatter(format, false); EXPECT_EQ("-|-|-", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { @@ -3920,7 +4072,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { FormatterImpl formatter(format, true); EXPECT_EQ("||", formatter.format(request_header, response_header, response_trailer, stream_info, - body)); + body, AccessLog::AccessLogType::NotSet)); } { @@ -3930,7 +4082,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { FormatterImpl formatter(format, false); EXPECT_EQ("-|-|-|-", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { @@ -3940,7 +4092,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterEmpty) { FormatterImpl formatter(format, true); EXPECT_EQ("|||", formatter.format(request_header, response_header, response_trailer, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } } @@ -4012,7 +4164,7 @@ TEST(SubstitutionFormatterTest, EmptyFormatParse) { EXPECT_EQ(providers.size(), 1); EXPECT_EQ("", providers[0]->format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } TEST(SubstitutionFormatterTest, EscapingFormatParse) { @@ -4026,7 +4178,7 @@ TEST(SubstitutionFormatterTest, EscapingFormatParse) { ASSERT_EQ(providers.size(), 1); EXPECT_EQ("%", providers[0]->format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } TEST(SubstitutionFormatterTest, FormatterExtension) { @@ -4042,8 +4194,9 @@ TEST(SubstitutionFormatterTest, FormatterExtension) { auto providers = SubstitutionFormatParser::parse("foo %COMMAND_EXTENSION(x)%", commands); EXPECT_EQ(providers.size(), 2); - EXPECT_EQ("TestFormatter", providers[1]->format(request_headers, response_headers, - response_trailers, stream_info, body)); + EXPECT_EQ("TestFormatter", + providers[1]->format(request_headers, response_headers, response_trailers, stream_info, + body, AccessLog::AccessLogType::NotSet)); } TEST(SubstitutionFormatterTest, PercentEscapingEdgeCase) { @@ -4065,10 +4218,11 @@ TEST(SubstitutionFormatterTest, PercentEscapingEdgeCase) { auto providers = SubstitutionFormatParser::parse("%HOSTNAME%%PROTOCOL%"); ASSERT_EQ(providers.size(), 2); - EXPECT_EQ("myhostname", providers[0]->format(request_headers, response_headers, response_trailers, - stream_info, body)); + EXPECT_EQ("myhostname", + providers[0]->format(request_headers, response_headers, response_trailers, stream_info, + body, AccessLog::AccessLogType::NotSet)); EXPECT_EQ("HTTP/1.1", providers[1]->format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { @@ -4089,7 +4243,7 @@ TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { ASSERT_EQ(providers.size(), 1); EXPECT_EQ("-", providers[0]->format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { @@ -4107,7 +4261,7 @@ TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { ASSERT_EQ(providers.size(), 1); EXPECT_EQ("test", providers[0]->format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } { @@ -4125,7 +4279,7 @@ TEST(SubstitutionFormatterTest, EnvironmentFormatterTest) { ASSERT_EQ(providers.size(), 1); EXPECT_EQ("te", providers[0]->format(request_headers, response_headers, response_trailers, - stream_info, body)); + stream_info, body, AccessLog::AccessLogType::NotSet)); } } diff --git a/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc b/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc index 622d22b7d883..5cf53412cda3 100644 --- a/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc +++ b/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc @@ -77,10 +77,10 @@ static void BM_OpenTelemetryAccessLogFormatter(benchmark::State& state) { Http::TestResponseTrailerMapImpl response_trailers; std::string body; for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += - otel_formatter - ->format(request_headers, response_headers, response_trailers, *stream_info, body) - .ByteSize(); + output_bytes += otel_formatter + ->format(request_headers, response_headers, response_trailers, *stream_info, + body, AccessLog::AccessLogType::NotSet) + .ByteSize(); } benchmark::DoNotOptimize(output_bytes); } diff --git a/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc b/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc index e9891a5c288f..3adcc6950230 100644 --- a/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc +++ b/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc @@ -133,9 +133,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterPlainStringTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterTypesTest) { @@ -198,8 +198,8 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterTypesTest) { - string_value: "HTTP/1.1" )EOF", expected); - const KeyValueList output = - formatter.format(request_header, response_header, response_trailer, stream_info, body); + const KeyValueList output = formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet); EXPECT_TRUE(TestUtility::protoEqual(output, expected)); } @@ -411,8 +411,8 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterNestedObjectsTest) { - string_value: "HTTP/1.1" )EOF", expected); - const KeyValueList output = - formatter.format(request_header, response_header, response_trailer, stream_info, body); + const KeyValueList output = formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet); EXPECT_TRUE(TestUtility::protoEqual(output, expected)); } @@ -438,9 +438,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterSingleOperatorTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } TEST(SubstitutionFormatterTest, EmptyOpenTelemetryFormatterTest) { @@ -465,9 +465,9 @@ TEST(SubstitutionFormatterTest, EmptyOpenTelemetryFormatterTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterNonExistentHeaderTest) { @@ -504,9 +504,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterNonExistentHeaderTest) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterAlternateHeaderTest) { @@ -546,9 +546,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterAlternateHeaderTest) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterDynamicMetadataTest) { @@ -583,9 +583,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterDynamicMetadataTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataTest) { @@ -629,9 +629,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataNoClusterInfoTest) { @@ -656,16 +656,16 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataNoClusterIn // Empty optional (absl::nullopt) { EXPECT_CALL(Const(stream_info), upstreamClusterInfo()).WillOnce(Return(absl::nullopt)); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } // Empty cluster info (nullptr) { EXPECT_CALL(Const(stream_info), upstreamClusterInfo()).WillOnce(Return(nullptr)); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } } @@ -699,9 +699,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_headers, response_headers, response_trailers, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateTest) { @@ -744,9 +744,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_headers, response_headers, response_trailers, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } // Test new specifier (PLAIN/TYPED) of FilterState. Ensure that after adding additional specifier, @@ -780,9 +780,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateSpeciferTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_headers, response_headers, response_trailers, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } // Test new specifier (PLAIN/TYPED) of FilterState. Ensure that after adding additional specifier, @@ -822,9 +822,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateSpecife key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_headers, response_headers, response_trailers, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } // Error specifier will cause an exception to be thrown. @@ -924,9 +924,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterStartTimeTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterMultiTokenTest) { @@ -954,9 +954,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterMultiTokenTest) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - verifyOpenTelemetryOutput( - formatter.format(request_header, response_header, response_trailer, stream_info, body), - expected); + verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + expected); } } diff --git a/test/extensions/formatter/metadata/metadata_test.cc b/test/extensions/formatter/metadata/metadata_test.cc index 218f70a03612..2f228aeaaeda 100644 --- a/test/extensions/formatter/metadata/metadata_test.cc +++ b/test/extensions/formatter/metadata/metadata_test.cc @@ -70,9 +70,9 @@ TEST_F(MetadataFormatterTest, DynamicMetadata) { EXPECT_CALL(testing::Const(stream_info_), dynamicMetadata()) .WillRepeatedly(testing::ReturnRef(*metadata_)); - EXPECT_EQ("test_value", - getTestMetadataFormatter("DYNAMIC")->format(request_headers_, response_headers_, - response_trailers_, stream_info_, body_)); + EXPECT_EQ("test_value", getTestMetadataFormatter("DYNAMIC")->format( + request_headers_, response_headers_, response_trailers_, stream_info_, + body_, AccessLog::AccessLogType::NotSet)); } // Extensive testing of Cluster Metadata formatter is in @@ -86,9 +86,9 @@ TEST_F(MetadataFormatterTest, ClusterMetadata) { EXPECT_CALL(**cluster, metadata()).WillRepeatedly(testing::ReturnRef(*metadata_)); EXPECT_CALL(stream_info_, upstreamClusterInfo()).WillRepeatedly(testing::ReturnPointee(cluster)); - EXPECT_EQ("test_value", - getTestMetadataFormatter("CLUSTER")->format(request_headers_, response_headers_, - response_trailers_, stream_info_, body_)); + EXPECT_EQ("test_value", getTestMetadataFormatter("CLUSTER")->format( + request_headers_, response_headers_, response_trailers_, stream_info_, + body_, AccessLog::AccessLogType::NotSet)); } // Extensive testing of UpstreamHost Metadata formatter is in @@ -108,7 +108,7 @@ TEST_F(MetadataFormatterTest, UpstreamHostMetadata) { EXPECT_EQ("test_value", getTestMetadataFormatter("UPSTREAM_HOST") ->format(request_headers_, response_headers_, response_trailers_, - stream_info_, body_)); + stream_info_, body_, AccessLog::AccessLogType::NotSet)); } // Test that METADATA(ROUTE accesses stream_info's Route. @@ -117,17 +117,18 @@ TEST_F(MetadataFormatterTest, RouteMetadata) { EXPECT_CALL(*route, metadata()).WillRepeatedly(testing::ReturnRef(*metadata_)); EXPECT_CALL(stream_info_, route()).WillRepeatedly(testing::Return(route)); - EXPECT_EQ("test_value", - getTestMetadataFormatter("ROUTE")->format(request_headers_, response_headers_, - response_trailers_, stream_info_, body_)); + EXPECT_EQ("test_value", getTestMetadataFormatter("ROUTE")->format( + request_headers_, response_headers_, response_trailers_, stream_info_, + body_, AccessLog::AccessLogType::NotSet)); } // Make sure that code handles nullptr returned for stream_info::route(). TEST_F(MetadataFormatterTest, NonExistentRouteMetadata) { EXPECT_CALL(stream_info_, route()).WillRepeatedly(testing::Return(nullptr)); - EXPECT_EQ("-", getTestMetadataFormatter("ROUTE")->format( - request_headers_, response_headers_, response_trailers_, stream_info_, body_)); + EXPECT_EQ("-", getTestMetadataFormatter("ROUTE")->format(request_headers_, response_headers_, + response_trailers_, stream_info_, body_, + AccessLog::AccessLogType::NotSet)); } } // namespace Formatter diff --git a/test/extensions/formatter/req_without_query/req_without_query_test.cc b/test/extensions/formatter/req_without_query/req_without_query_test.cc index e4a1695625be..2155a7e3df35 100644 --- a/test/extensions/formatter/req_without_query/req_without_query_test.cc +++ b/test/extensions/formatter/req_without_query/req_without_query_test.cc @@ -42,8 +42,9 @@ TEST_F(ReqWithoutQueryTest, TestStripQueryString) { auto formatter = Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); - EXPECT_EQ("/request/path", formatter->format(request_headers_, response_headers_, - response_trailers_, stream_info_, body_)); + EXPECT_EQ("/request/path", + formatter->format(request_headers_, response_headers_, response_trailers_, stream_info_, + body_, AccessLog::AccessLogType::NotSet)); } TEST_F(ReqWithoutQueryTest, TestSelectMainHeader) { @@ -60,8 +61,9 @@ TEST_F(ReqWithoutQueryTest, TestSelectMainHeader) { auto formatter = Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); - EXPECT_EQ("/original/path", formatter->format(request_headers_, response_headers_, - response_trailers_, stream_info_, body_)); + EXPECT_EQ("/original/path", + formatter->format(request_headers_, response_headers_, response_trailers_, stream_info_, + body_, AccessLog::AccessLogType::NotSet)); } TEST_F(ReqWithoutQueryTest, TestSelectAlternativeHeader) { @@ -78,8 +80,9 @@ TEST_F(ReqWithoutQueryTest, TestSelectAlternativeHeader) { auto formatter = Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); - EXPECT_EQ("/request/path", formatter->format(request_headers_, response_headers_, - response_trailers_, stream_info_, body_)); + EXPECT_EQ("/request/path", + formatter->format(request_headers_, response_headers_, response_trailers_, stream_info_, + body_, AccessLog::AccessLogType::NotSet)); } TEST_F(ReqWithoutQueryTest, TestTruncateHeader) { @@ -97,7 +100,7 @@ TEST_F(ReqWithoutQueryTest, TestTruncateHeader) { auto formatter = Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("/requ", formatter->format(request_headers_, response_headers_, response_trailers_, - stream_info_, body_)); + stream_info_, body_, AccessLog::AccessLogType::NotSet)); } TEST_F(ReqWithoutQueryTest, TestNonExistingHeader) { @@ -115,7 +118,7 @@ TEST_F(ReqWithoutQueryTest, TestNonExistingHeader) { auto formatter = Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("-", formatter->format(request_headers_, response_headers_, response_trailers_, - stream_info_, body_)); + stream_info_, body_, AccessLog::AccessLogType::NotSet)); } TEST_F(ReqWithoutQueryTest, TestFormatJson) { @@ -142,8 +145,9 @@ TEST_F(ReqWithoutQueryTest, TestFormatJson) { TestUtility::loadFromYaml(yaml, config_); auto formatter = Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); - const std::string actual = formatter->format(request_headers_, response_headers_, - response_trailers_, stream_info_, body_); + const std::string actual = + formatter->format(request_headers_, response_headers_, response_trailers_, stream_info_, + body_, AccessLog::AccessLogType::NotSet); EXPECT_TRUE(TestUtility::jsonStringEqual(actual, expected)); } From 57cb74a3f39305f2c0fddf742f8dab6df12fab42 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 28 Apr 2023 09:51:32 -0700 Subject: [PATCH 036/740] bazel: fix gcc 12+ warning (#27002) * bazel: fix gcc 12+ warning Fixes: https://github.com/envoyproxy/envoy/issues/23783 Signed-off-by: Keith Smiley --- envoy/common/union_string.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/envoy/common/union_string.h b/envoy/common/union_string.h index d0ecbf41e1eb..2eb2061b4049 100644 --- a/envoy/common/union_string.h +++ b/envoy/common/union_string.h @@ -58,7 +58,15 @@ template class UnionStringBase { /** * Default constructor. Sets up for inline storage. */ + +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#endif UnionStringBase() : buffer_(InlinedStringVector()) { +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif ASSERT((getInVec(buffer_).capacity()) >= MaxIntegerLength); ASSERT(valid()); } From 8c17c7a8c91380b2ad5d2e352db94fd1c7571d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20B=C3=A9ky?= Date: Fri, 28 Apr 2023 16:12:34 -0400 Subject: [PATCH 037/740] [balsa] Add config field to enable custom methods. (#26448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [balsa] Add config field to enable custom methods. This is no behavioral change by default: only methods from a hard-coded list (that matches the list hard-coded in http-parser, and is slightly different from the one that will be used by UHV) are accepted. Then the new knob is true, BalsaParser does the exact same validation as UHV will by default: method has to be non-empty and only contain allowed characters. When UHV method validation logic is turned on in the future, all validation can be removed from BalsaParser. When non-UHV mode is deprecated, this new proto field can be removed. Tracking issue: #21245 Signed-off-by: Bence Béky --- api/envoy/config/core/v3/protocol.proto | 15 +++++- envoy/http/codec.h | 5 ++ source/common/http/http1/balsa_parser.cc | 39 ++++++++++----- source/common/http/http1/balsa_parser.h | 3 +- source/common/http/http1/codec_impl.cc | 3 +- source/common/http/http1/settings.cc | 2 + .../transport_sockets/http_11_proxy/connect.h | 3 +- test/common/http/http1/codec_impl_test.cc | 50 ++++++++++++------- test/common/http/utility_test.cc | 14 ++++++ test/integration/protocol_integration_test.cc | 41 +++++---------- 10 files changed, 111 insertions(+), 64 deletions(-) diff --git a/api/envoy/config/core/v3/protocol.proto b/api/envoy/config/core/v3/protocol.proto index 620a2ef59801..95670bb752b4 100644 --- a/api/envoy/config/core/v3/protocol.proto +++ b/api/envoy/config/core/v3/protocol.proto @@ -259,7 +259,7 @@ message HttpProtocolOptions { google.protobuf.UInt32Value max_requests_per_connection = 6; } -// [#next-free-field: 10] +// [#next-free-field: 11] message Http1ProtocolOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.core.Http1ProtocolOptions"; @@ -358,6 +358,19 @@ message Http1ProtocolOptions { // See issue #21245. google.protobuf.BoolValue use_balsa_parser = 9 [(xds.annotations.v3.field_status).work_in_progress = true]; + + // [#not-implemented-hide:] Hiding so that field can be removed. + // If true, and BalsaParser is used (either `use_balsa_parser` above is true, + // or `envoy.reloadable_features.http1_use_balsa_parser` is true and + // `use_balsa_parser` is unset), then every non-empty method with only valid + // characters is accepted. Otherwise, methods not on the hard-coded list are + // rejected. + // Once UHV is enabled, this field should be removed, and BalsaParser should + // allow any method. UHV validates the method, rejecting empty string or + // invalid characters, and provides :ref:`restrict_http_methods + // ` + // to reject custom methods. + bool allow_custom_methods = 10 [(xds.annotations.v3.field_status).work_in_progress = true]; } message KeepaliveSettings { diff --git a/envoy/http/codec.h b/envoy/http/codec.h index f9fb05d857bd..5ad04479d47f 100644 --- a/envoy/http/codec.h +++ b/envoy/http/codec.h @@ -526,6 +526,11 @@ struct Http1Settings { // If true, BalsaParser is used for HTTP/1 parsing; if false, http-parser is // used. See issue #21245. bool use_balsa_parser_{false}; + + // If true, any non-empty method composed of valid characters is accepted. + // If false, only methods from a hard-coded list of known methods are accepted. + // Only implemented in BalsaParser. http-parser only accepts known methods. + bool allow_custom_methods_{false}; }; /** diff --git a/source/common/http/http1/balsa_parser.cc b/source/common/http/http1/balsa_parser.cc index 707f3c2391e3..a274a76e6723 100644 --- a/source/common/http/http1/balsa_parser.cc +++ b/source/common/http/http1/balsa_parser.cc @@ -23,9 +23,6 @@ constexpr absl::string_view kColonSlashSlash = "://"; // Response must start with "HTTP". constexpr char kResponseFirstByte = 'H'; -#ifndef ENVOY_ENABLE_UHV -// When UHV is enabled, BalsaParser allows any method and defers to UHV for -// validation. When UHV is disabled, BalsaParser behavior matches http-parser. bool isFirstCharacterOfValidMethod(char c) { static constexpr char kValidFirstCharacters[] = {'A', 'B', 'C', 'D', 'G', 'H', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U'}; @@ -35,7 +32,27 @@ bool isFirstCharacterOfValidMethod(char c) { return std::binary_search(begin, end, c); } -bool isMethodValid(absl::string_view method) { +// TODO(#21245): Skip method validation altogether when UHV method validation is +// enabled. +bool isMethodValid(absl::string_view method, bool allow_custom_methods) { + if (allow_custom_methods) { + // Allowed characters in method according to RFC 9110, + // https://www.rfc-editor.org/rfc/rfc9110.html#section-5.1. + static constexpr char kValidCharacters[] = { + '!', '#', '$', '%', '&', '\'', '*', '+', '-', '.', '0', '1', '2', '3', '4', '5', + '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', + 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '^', '_', + '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '|', '~'}; + const auto* begin = &kValidCharacters[0]; + const auto* end = &kValidCharacters[ABSL_ARRAYSIZE(kValidCharacters) - 1] + 1; + + return !method.empty() && + std::all_of(method.begin(), method.end(), [begin, end](absl::string_view::value_type c) { + return std::binary_search(begin, end, c); + }); + } + static constexpr absl::string_view kValidMethods[] = { "ACL", "BIND", "CHECKOUT", "CONNECT", "COPY", "DELETE", "GET", "HEAD", "LINK", "LOCK", "MERGE", "MKACTIVITY", "MKCALENDAR", "MKCOL", @@ -47,7 +64,6 @@ bool isMethodValid(absl::string_view method) { const auto* end = &kValidMethods[ABSL_ARRAYSIZE(kValidMethods) - 1] + 1; return std::binary_search(begin, end, method); } -#endif // This function is crafted to match the URL validation behavior of the http-parser library. bool isUrlValid(absl::string_view url, bool is_connect) { @@ -125,8 +141,8 @@ bool isVersionValid(absl::string_view version_input) { } // anonymous namespace BalsaParser::BalsaParser(MessageType type, ParserCallbacks* connection, size_t max_header_length, - bool enable_trailers) - : message_type_(type), connection_(connection) { + bool enable_trailers, bool allow_custom_methods) + : message_type_(type), connection_(connection), allow_custom_methods_(allow_custom_methods) { ASSERT(connection_ != nullptr); quiche::HttpValidationPolicy http_validation_policy; @@ -158,13 +174,12 @@ size_t BalsaParser::execute(const char* slice, int len) { ASSERT(status_ != ParserStatus::Error); if (len > 0 && !first_byte_processed_) { -#ifndef ENVOY_ENABLE_UHV - if (message_type_ == MessageType::Request && !isFirstCharacterOfValidMethod(*slice)) { + if (message_type_ == MessageType::Request && !allow_custom_methods_ && + !isFirstCharacterOfValidMethod(*slice)) { status_ = ParserStatus::Error; error_message_ = "HPE_INVALID_METHOD"; return 0; } -#endif if (message_type_ == MessageType::Response && *slice != kResponseFirstByte) { status_ = ParserStatus::Error; error_message_ = "HPE_INVALID_CONSTANT"; @@ -266,13 +281,11 @@ void BalsaParser::OnRequestFirstLineInput(absl::string_view /*line_input*/, if (status_ == ParserStatus::Error) { return; } -#ifndef ENVOY_ENABLE_UHV - if (!isMethodValid(method_input)) { + if (!isMethodValid(method_input, allow_custom_methods_)) { status_ = ParserStatus::Error; error_message_ = "HPE_INVALID_METHOD"; return; } -#endif const bool is_connect = method_input == Headers::get().MethodValues.Connect; if (!isUrlValid(request_uri, is_connect)) { status_ = ParserStatus::Error; diff --git a/source/common/http/http1/balsa_parser.h b/source/common/http/http1/balsa_parser.h index 38607f2100c1..088051612523 100644 --- a/source/common/http/http1/balsa_parser.h +++ b/source/common/http/http1/balsa_parser.h @@ -17,7 +17,7 @@ namespace Http1 { class BalsaParser : public Parser, public quiche::BalsaVisitorInterface { public: BalsaParser(MessageType type, ParserCallbacks* connection, size_t max_header_length, - bool enable_trailers); + bool enable_trailers, bool allow_custom_methods); ~BalsaParser() override = default; // Http1::Parser implementation @@ -72,6 +72,7 @@ class BalsaParser : public Parser, public quiche::BalsaVisitorInterface { const MessageType message_type_ = MessageType::Request; ParserCallbacks* connection_ = nullptr; + const bool allow_custom_methods_ = false; bool first_byte_processed_ = false; bool headers_done_ = false; ParserStatus status_ = ParserStatus::Ok; diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index 405b75983f2b..4b36728f02c6 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -526,7 +526,8 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stat deferred_end_stream_headers_(false), dispatching_(false), max_headers_kb_(max_headers_kb), max_headers_count_(max_headers_count) { if (codec_settings_.use_balsa_parser_) { - parser_ = std::make_unique(type, this, max_headers_kb_ * 1024, enableTrailers()); + parser_ = std::make_unique(type, this, max_headers_kb_ * 1024, enableTrailers(), + codec_settings_.allow_custom_methods_); } else { parser_ = std::make_unique(type, this); } diff --git a/source/common/http/http1/settings.cc b/source/common/http/http1/settings.cc index 7c68a49c7c52..e74af4f49135 100644 --- a/source/common/http/http1/settings.cc +++ b/source/common/http/http1/settings.cc @@ -39,6 +39,8 @@ Http1Settings parseHttp1Settings(const envoy::config::core::v3::Http1ProtocolOpt Runtime::runtimeFeatureEnabled("envoy.reloadable_features.http1_use_balsa_parser"); } + ret.allow_custom_methods_ = config.allow_custom_methods(); + return ret; } diff --git a/source/extensions/transport_sockets/http_11_proxy/connect.h b/source/extensions/transport_sockets/http_11_proxy/connect.h index e570cc7ac743..2e06915a11f1 100644 --- a/source/extensions/transport_sockets/http_11_proxy/connect.h +++ b/source/extensions/transport_sockets/http_11_proxy/connect.h @@ -62,7 +62,8 @@ class UpstreamHttp11ConnectSocketFactory : public PassthroughFactory { class SelfContainedParser : public Http::Http1::ParserCallbacks { public: SelfContainedParser() - : parser_(Http::Http1::MessageType::Response, this, 2000, /* enable_trailers = */ false) {} + : parser_(Http::Http1::MessageType::Response, this, 2000, /* enable_trailers = */ false, + /* allow_custom_methods = */ false) {} Http::Http1::CallbackResult onMessageBegin() override { return Http::Http1::CallbackResult::Success; } diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 6a65b3499538..227b498f1c9a 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -1226,36 +1226,48 @@ TEST_P(Http1ServerConnectionImplTest, BadRequestNoStream) { EXPECT_TRUE(isCodecProtocolError(status)); } -TEST_P(Http1ServerConnectionImplTest, CustomMethod) { - bool reject_custom_methods = true; -#ifdef ENVOY_ENABLE_UHV - if (parser_impl_ == Http1ParserImpl::BalsaParser) { - // When both BalsaParser and UHV are enabled, custom methods are allowed by - // default. - reject_custom_methods = false; - } -#endif - +TEST_P(Http1ServerConnectionImplTest, RejectCustomMethod) { initialize(); MockRequestDecoder decoder; EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); - if (reject_custom_methods) { - EXPECT_CALL(decoder, sendLocalReply(_, _, _, _, _)); - } + EXPECT_CALL(decoder, sendLocalReply(_, _, _, _, _)); Buffer::OwnedImpl buffer("BAD / HTTP/1.1\r\n"); auto status = codec_->dispatch(buffer); - if (reject_custom_methods) { - // http-parser rejects unknown methods. - // This behavior was observed during CVE-2019-18801 and helped to limit the - // scope of affected Envoy configurations. - EXPECT_TRUE(isCodecProtocolError(status)); - EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_METHOD"); + EXPECT_TRUE(isCodecProtocolError(status)); + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_METHOD"); +} + +TEST_P(Http1ServerConnectionImplTest, RejectInvalidCharacterInMethod) { + codec_settings_.allow_custom_methods_ = true; + initialize(); + + MockRequestDecoder decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + EXPECT_CALL(decoder, sendLocalReply(_, _, _, _, _)); + + Buffer::OwnedImpl buffer("B{}D / HTTP/1.1\r\n"); + auto status = codec_->dispatch(buffer); + + EXPECT_TRUE(isCodecProtocolError(status)); + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_METHOD"); +} + +TEST_P(Http1ServerConnectionImplTest, AllowCustomMethod) { + if (parser_impl_ == Http1ParserImpl::HttpParser) { return; } + codec_settings_.allow_custom_methods_ = true; + initialize(); + + MockRequestDecoder decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + + Buffer::OwnedImpl buffer("BAD / HTTP/1.1\r\n"); + auto status = codec_->dispatch(buffer); ASSERT_TRUE(status.ok()); TestRequestHeaderMapImpl expected_headers{ diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index 4cadf8ec167d..aeecabe7f94e 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -674,6 +674,20 @@ TEST(HttpUtility, UseBalsaParser) { .use_balsa_parser_); } +TEST(HttpUtility, AllowCustomMethods) { + envoy::config::core::v3::Http1ProtocolOptions http1_options; + ProtobufWkt::BoolValue hcm_value; + NiceMock validation_visitor; + + EXPECT_FALSE(http1_options.allow_custom_methods()); + EXPECT_FALSE(Http1::parseHttp1Settings(http1_options, validation_visitor, hcm_value, false) + .allow_custom_methods_); + + http1_options.set_allow_custom_methods(true); + EXPECT_TRUE(Http1::parseHttp1Settings(http1_options, validation_visitor, hcm_value, false) + .allow_custom_methods_); +} + TEST(HttpUtility, getLastAddressFromXFF) { { const std::string first_address = "192.0.2.10"; diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index d8227d3178ab..f742e4cad2d5 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -2390,33 +2390,19 @@ TEST_P(DownstreamProtocolIntegrationTest, ManyTrailerHeaders) { // :method request headers, since the case of other large headers is // covered in the various testLargeRequest-based integration tests here. // -// Tests are run with two HTTP/1 parsers: http-parser and BalsaParser. -// http-parser rejects large method strings, because it only accepts known -// methods from a hardcoded list. BalsaParser mimics http-parser behavior when -// UHV is disabled, but defers method validation to UHV when it is enabled. +// Both HTTP/1 parsers (http-parser and BalsaParser) reject large method strings +// by default, because they only accepts known methods from a hardcoded list. +// HTTP/2 and HTTP/3 codecs accept large methods. The table below describes the +// expected behaviors (in addition we should never see an ASSERT or ASAN failure +// trigger). // -// In addition to BalsaParser with UHV enabled, H2 and H3 codecs also accept -// large methods. The table below describes the expected behaviors (in addition -// we should never see an ASSERT or ASAN failure trigger). -// -// Downstream proto Upstream proto Behavior expected -// -------------------------------------------------------- -// accepts accepts Success -// accepts rejects Envoy will forward; backend will reject -// rejects accepts Envoy will reject -// rejects rejects Envoy will reject +// Downstream proto Upstream proto Behavior expected +// ---------------------------------------------------------- +// HTTP/2 or HTTP/3 HTTP/2 or HTTP/3 Success +// HTTP/2 or HTTP/3 HTTP/1 Envoy will forward; backend will reject +// HTTP/1 HTTP/2 or HTTP/3 Envoy will reject +// HTTP/1 HTTP/1 Envoy will reject TEST_P(ProtocolIntegrationTest, LargeRequestMethod) { - bool http1_codec_rejects_large_method = true; -#ifdef ENVOY_ENABLE_UHV - if (GetParam().http1_implementation == Http1ParserImpl::BalsaParser) { - http1_codec_rejects_large_method = false; - } -#endif - const bool downstream_proto_rejects_large_method = - downstreamProtocol() == Http::CodecType::HTTP1 && http1_codec_rejects_large_method; - const bool upstream_proto_rejects_large_method = - upstreamProtocol() == Http::CodecType::HTTP1 && http1_codec_rejects_large_method; - // There will be no upstream connections for HTTP/1 downstream, we need to // test the full mesh regardless. testing_upstream_intentionally_ = true; @@ -2428,15 +2414,14 @@ TEST_P(ProtocolIntegrationTest, LargeRequestMethod) { initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); - - if (downstream_proto_rejects_large_method) { + if (downstreamProtocol() == Http::CodecType::HTTP1) { auto encoder_decoder = codec_client_->startRequest(request_headers); request_encoder_ = &encoder_decoder.first; auto response = std::move(encoder_decoder.second); ASSERT_TRUE(codec_client_->waitForDisconnect()); EXPECT_TRUE(response->complete()); EXPECT_EQ("400", response->headers().getStatusValue()); - } else if (upstream_proto_rejects_large_method) { + } else if (upstreamProtocol() == Http::CodecType::HTTP1) { auto response = codec_client_->makeHeaderOnlyRequest(request_headers); ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); ASSERT_TRUE(response->waitForEndStream()); From fe274c0277a8633b277f02b720930137280f7529 Mon Sep 17 00:00:00 2001 From: Raven Black Date: Sun, 30 Apr 2023 13:15:58 -0700 Subject: [PATCH 038/740] [rbac] Make deprecation of Principal.source_ip documented (#26762) Make deprecation of source_ip documented Signed-off-by: Raven Black --- api/envoy/config/rbac/v3/rbac.proto | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/envoy/config/rbac/v3/rbac.proto b/api/envoy/config/rbac/v3/rbac.proto index e2f9b5c8db83..ce32ea7d6b59 100644 --- a/api/envoy/config/rbac/v3/rbac.proto +++ b/api/envoy/config/rbac/v3/rbac.proto @@ -320,6 +320,11 @@ message Principal { // A CIDR block that describes the downstream IP. // This address will honor proxy protocol, but will not honor XFF. + // + // This field is deprecated; either use :ref:`direct_remote_ip + // ` for the same + // behavior, or use + // :ref:`remote_ip `. core.v3.CidrRange source_ip = 5 [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; From 15baa25e2290314917d566565dea6fef9008df59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 07:52:38 +0000 Subject: [PATCH 039/740] build(deps): bump envoy-base-utils from 0.4.2 to 0.4.7 in /tools/base (#27068) Bumps [envoy-base-utils](https://github.com/envoyproxy/pytooling) from 0.4.2 to 0.4.7. - [Release notes](https://github.com/envoyproxy/pytooling/releases) - [Commits](https://github.com/envoyproxy/pytooling/commits) --- updated-dependencies: - dependency-name: envoy-base-utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 50 +++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 5432f8417c3e..07d9e70166a5 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -419,8 +419,9 @@ docutils==0.19 \ # via # envoy-docs-sphinx-runner # sphinx -envoy-base-utils==0.4.2 \ - --hash=sha256:7c3162f69584abbf80ce01e95417415dcde86cb3aad06f31f0bf114d8347bea7 +envoy-base-utils==0.4.7 \ + --hash=sha256:1726809d5edbed6093c0fd779fe189d6ddc0491f2bfddf84b929deb0795292f0 \ + --hash=sha256:af918b0d9db86c1e907d93df7996b036e5bfc212fe7e491daa793dba5795c7e4 # via # -r requirements.in # envoy-code-check @@ -1283,6 +1284,51 @@ yarl==1.9.2 \ # via # -r requirements.in # aiohttp +zstandard==0.21.0 \ + --hash=sha256:0aad6090ac164a9d237d096c8af241b8dcd015524ac6dbec1330092dba151657 \ + --hash=sha256:0bdbe350691dec3078b187b8304e6a9c4d9db3eb2d50ab5b1d748533e746d099 \ + --hash=sha256:0e1e94a9d9e35dc04bf90055e914077c80b1e0c15454cc5419e82529d3e70728 \ + --hash=sha256:1243b01fb7926a5a0417120c57d4c28b25a0200284af0525fddba812d575f605 \ + --hash=sha256:144a4fe4be2e747bf9c646deab212666e39048faa4372abb6a250dab0f347a29 \ + --hash=sha256:14e10ed461e4807471075d4b7a2af51f5234c8f1e2a0c1d37d5ca49aaaad49e8 \ + --hash=sha256:1545fb9cb93e043351d0cb2ee73fa0ab32e61298968667bb924aac166278c3fc \ + --hash=sha256:1e6e131a4df2eb6f64961cea6f979cdff22d6e0d5516feb0d09492c8fd36f3bc \ + --hash=sha256:25fbfef672ad798afab12e8fd204d122fca3bc8e2dcb0a2ba73bf0a0ac0f5f07 \ + --hash=sha256:2769730c13638e08b7a983b32cb67775650024632cd0476bf1ba0e6360f5ac7d \ + --hash=sha256:48b6233b5c4cacb7afb0ee6b4f91820afbb6c0e3ae0fa10abbc20000acdf4f11 \ + --hash=sha256:4af612c96599b17e4930fe58bffd6514e6c25509d120f4eae6031b7595912f85 \ + --hash=sha256:52b2b5e3e7670bd25835e0e0730a236f2b0df87672d99d3bf4bf87248aa659fb \ + --hash=sha256:57ac078ad7333c9db7a74804684099c4c77f98971c151cee18d17a12649bc25c \ + --hash=sha256:62957069a7c2626ae80023998757e27bd28d933b165c487ab6f83ad3337f773d \ + --hash=sha256:649a67643257e3b2cff1c0a73130609679a5673bf389564bc6d4b164d822a7ce \ + --hash=sha256:67829fdb82e7393ca68e543894cd0581a79243cc4ec74a836c305c70a5943f07 \ + --hash=sha256:7d3bc4de588b987f3934ca79140e226785d7b5e47e31756761e48644a45a6766 \ + --hash=sha256:7f2afab2c727b6a3d466faee6974a7dad0d9991241c498e7317e5ccf53dbc766 \ + --hash=sha256:8070c1cdb4587a8aa038638acda3bd97c43c59e1e31705f2766d5576b329e97c \ + --hash=sha256:8257752b97134477fb4e413529edaa04fc0457361d304c1319573de00ba796b1 \ + --hash=sha256:9980489f066a391c5572bc7dc471e903fb134e0b0001ea9b1d3eff85af0a6f1b \ + --hash=sha256:9cff89a036c639a6a9299bf19e16bfb9ac7def9a7634c52c257166db09d950e7 \ + --hash=sha256:a8d200617d5c876221304b0e3fe43307adde291b4a897e7b0617a61611dfff6a \ + --hash=sha256:a9fec02ce2b38e8b2e86079ff0b912445495e8ab0b137f9c0505f88ad0d61296 \ + --hash=sha256:b1367da0dde8ae5040ef0413fb57b5baeac39d8931c70536d5f013b11d3fc3a5 \ + --hash=sha256:b69cccd06a4a0a1d9fb3ec9a97600055cf03030ed7048d4bcb88c574f7895773 \ + --hash=sha256:b72060402524ab91e075881f6b6b3f37ab715663313030d0ce983da44960a86f \ + --hash=sha256:c053b7c4cbf71cc26808ed67ae955836232f7638444d709bfc302d3e499364fa \ + --hash=sha256:cff891e37b167bc477f35562cda1248acc115dbafbea4f3af54ec70821090965 \ + --hash=sha256:d12fa383e315b62630bd407477d750ec96a0f438447d0e6e496ab67b8b451d39 \ + --hash=sha256:d2d61675b2a73edcef5e327e38eb62bdfc89009960f0e3991eae5cc3d54718de \ + --hash=sha256:db62cbe7a965e68ad2217a056107cc43d41764c66c895be05cf9c8b19578ce9c \ + --hash=sha256:ddb086ea3b915e50f6604be93f4f64f168d3fc3cef3585bb9a375d5834392d4f \ + --hash=sha256:df28aa5c241f59a7ab524f8ad8bb75d9a23f7ed9d501b0fed6d40ec3064784e8 \ + --hash=sha256:e1e0c62a67ff425927898cf43da2cf6b852289ebcc2054514ea9bf121bec10a5 \ + --hash=sha256:e6048a287f8d2d6e8bc67f6b42a766c61923641dd4022b7fd3f7439e17ba5a4d \ + --hash=sha256:e7d560ce14fd209db6adacce8908244503a009c6c39eee0c10f138996cd66d3e \ + --hash=sha256:ea68b1ba4f9678ac3d3e370d96442a6332d431e5050223626bdce748692226ea \ + --hash=sha256:f08e3a10d01a247877e4cb61a82a319ea746c356a3786558bed2481e6c405546 \ + --hash=sha256:f1b9703fe2e6b6811886c44052647df7c37478af1b4a1a9078585806f42e5b15 \ + --hash=sha256:fe6c821eb6870f81d73bf10e5deed80edcac1e63fbc40610e61f340723fd5f7c \ + --hash=sha256:ff0852da2abe86326b20abae912d0367878dd0854b8931897d44cfeb18985472 + # via envoy-base-utils # The following packages are considered to be unsafe in a requirements file: setuptools==67.7.2 \ From 9591e065dad88c617fec9cafe1349a54a03b6c30 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 08:58:24 +0100 Subject: [PATCH 040/740] build(deps): bump slack-sdk from 3.21.2 to 3.21.3 in /.github/actions/pr_notifier (#27067) build(deps): bump slack-sdk in /.github/actions/pr_notifier Bumps [slack-sdk](https://github.com/slackapi/python-slack-sdk) from 3.21.2 to 3.21.3. - [Release notes](https://github.com/slackapi/python-slack-sdk/releases) - [Changelog](https://github.com/slackapi/python-slack-sdk/blob/main/docs-v2/changelog.html) - [Commits](https://github.com/slackapi/python-slack-sdk/compare/v3.21.2...v3.21.3) --- updated-dependencies: - dependency-name: slack-sdk dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/pr_notifier/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/pr_notifier/requirements.txt b/.github/actions/pr_notifier/requirements.txt index a003308802d5..2d3efaf76d9b 100644 --- a/.github/actions/pr_notifier/requirements.txt +++ b/.github/actions/pr_notifier/requirements.txt @@ -138,9 +138,9 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via pynacl -slack-sdk==3.21.2 \ - --hash=sha256:cb74aa764ae32cd61971baa745d8bdf50ffea4ad06e4a0a2ab1e879ae6918496 \ - --hash=sha256:fd04ca4aae3071fbd6b284b11ef39b6c72d687b1a2118f40dbcba9921da07d62 +slack-sdk==3.21.3 \ + --hash=sha256:20829bdc1a423ec93dac903470975ebf3bc76fd3fd91a4dadc0eeffc940ecb0c \ + --hash=sha256:de3c07b92479940b61cd68c566f49fbc9974c8f38f661d26244078f3903bb9cc # via -r requirements.in urllib3==1.26.6 \ --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \ From 3bd4519e0ac5fddef4e9508311a76329984bd4af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 07:58:51 +0000 Subject: [PATCH 041/740] build(deps): bump envoy-gpg-identity from 0.1.0 to 0.1.1 in /tools/base (#27069) Bumps [envoy-gpg-identity](https://github.com/envoyproxy/pytooling) from 0.1.0 to 0.1.1. - [Release notes](https://github.com/envoyproxy/pytooling/releases) - [Commits](https://github.com/envoyproxy/pytooling/compare/0.1.0...0.1.1) --- updated-dependencies: - dependency-name: envoy-gpg-identity dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 07d9e70166a5..f277fd91ce97 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -474,9 +474,9 @@ envoy-github-release==0.0.15 \ --hash=sha256:955671aa0c664ade3d8dbcd60bc9c72284139b81aeaa3fe72157c4e8383fd35b \ --hash=sha256:d571422a79e535ec36130dae5a3c56b429040fb20ca60cd3b8cf8588c52b546c # via envoy-distribution-release -envoy-gpg-identity==0.1.0 \ - --hash=sha256:1df6989e1a6a3a9f5809ac056829b840f57326d41ae1181db415a722216bff48 \ - --hash=sha256:9e11625e25ad6a031f1c85c57067673f50c0427b0e71ef0a7e926ce77ebb75a9 +envoy-gpg-identity==0.1.1 \ + --hash=sha256:03f615278b2ca0de652be9d9e3a45faffae74f85f483347c1e0d690edd4019f3 \ + --hash=sha256:c41505491f906bd5ab22504b0ae2f9e76430ae492c9f59278a306225ed19c785 # via # -r requirements.in # envoy-gpg-sign From d117fd17e62fd7fe9975a7797718e85e4c046f46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 08:03:56 +0000 Subject: [PATCH 042/740] build(deps): bump otel/opentelemetry-collector from `e31d081` to `da930e5` in /examples/opentelemetry (#27074) build(deps): bump otel/opentelemetry-collector Bumps otel/opentelemetry-collector from `e31d081` to `da930e5`. --- updated-dependencies: - dependency-name: otel/opentelemetry-collector dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/opentelemetry/Dockerfile-opentelemetry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry index 16af29d51ddd..1512b770cd94 100644 --- a/examples/opentelemetry/Dockerfile-opentelemetry +++ b/examples/opentelemetry/Dockerfile-opentelemetry @@ -1,7 +1,7 @@ FROM alpine:3.17@sha256:124c7d2707904eea7431fffe91522a01e5a861a624ee31d03372cc1d138a3126 as otelc_curl RUN apk --update add curl -FROM otel/opentelemetry-collector:latest@sha256:e31d081310a83e461f335eda8390436e7ee414615664415666f7ac71c1b3b3db +FROM otel/opentelemetry-collector:latest@sha256:da930e519a80e34de471794435aa9c24ce8ef84b231fe8f4b9b6f0148797cd60 COPY --from=otelc_curl / / From 01aed8aa302d707a5598dbfd063dcfe3b31655f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 08:04:15 +0000 Subject: [PATCH 043/740] build(deps): bump openzipkin/zipkin from `cd6878d` to `2d797c2` in /examples/zipkin (#27076) build(deps): bump openzipkin/zipkin in /examples/zipkin Bumps openzipkin/zipkin from `cd6878d` to `2d797c2`. --- updated-dependencies: - dependency-name: openzipkin/zipkin dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/zipkin/Dockerfile-zipkin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/zipkin/Dockerfile-zipkin b/examples/zipkin/Dockerfile-zipkin index 0340c60b1b79..c09f6bb90525 100644 --- a/examples/zipkin/Dockerfile-zipkin +++ b/examples/zipkin/Dockerfile-zipkin @@ -1 +1 @@ -FROM openzipkin/zipkin:latest@sha256:cd6878def8c23d4efbfb7e7c5a0a8e963fa530473c2c8e84608dff42d143ad4e +FROM openzipkin/zipkin:latest@sha256:2d797c2da4aa98aee166eef82852cfb3cf09c1c3c3e48905be3564ce7e6acfc4 From 4dd9cbc306aa042eaeef1a14d908cd18adc801ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 08:11:41 +0000 Subject: [PATCH 044/740] build(deps): bump jaegertracing/all-in-one from `cfdd0e3` to `ac83e8d` in /examples/shared/jaeger (#27075) build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger Bumps jaegertracing/all-in-one from `cfdd0e3` to `ac83e8d`. --- updated-dependencies: - dependency-name: jaegertracing/all-in-one dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/jaeger/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile index f5b859a22d6f..d05f454acee7 100644 --- a/examples/shared/jaeger/Dockerfile +++ b/examples/shared/jaeger/Dockerfile @@ -1,4 +1,4 @@ -FROM jaegertracing/all-in-one@sha256:cfdd0e350ca29aaacc4ae0b2c946c2ef3a33853edd354a15b98e7bb611323099 +FROM jaegertracing/all-in-one@sha256:ac83e8d52ce8365390beda54de65235d392077da6e1e35f8b848f95db7e9d6b0 HEALTHCHECK \ --interval=1s \ --timeout=1s \ From bb8e262b2bb9768189f968608bc2fe797210c215 Mon Sep 17 00:00:00 2001 From: danzh Date: Mon, 1 May 2023 08:57:22 -0400 Subject: [PATCH 045/740] http3: deprecate envoy.reloadable_features.quic_defer_send_in_response_to_packet (#26951) Commit Message: deprecate above runtime guard Risk Level: low Testing: existing tests passed Docs Changes: N/A Release Notes: runtime guard deprecation Platform Specific Features: N/A Fixes #26906 Deprecated: envoy.reloadable_features.quic_defer_send_in_response_to_packet Signed-off-by: Dan Zhang Co-authored-by: Dan Zhang --- changelogs/current.yaml | 3 + .../quic/envoy_quic_server_connection.cc | 16 ++-- .../quic/envoy_quic_server_connection.h | 5 - .../common/quic/envoy_quic_server_session.cc | 8 +- .../quic_filter_manager_connection_impl.cc | 7 +- source/common/runtime/runtime_features.cc | 1 - test/common/quic/active_quic_listener_test.cc | 4 +- .../common/quic/envoy_quic_dispatcher_test.cc | 94 +++++++++---------- 8 files changed, 58 insertions(+), 80 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 2e558dab801d..d643ce209de0 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -41,6 +41,9 @@ removed_config_or_runtime: - area: http change: | removed runtime key ``envoy.reloadable_features.allow_upstream_filters`` and legacy code paths. +- area: quic + change: | + removed runtime key ``envoy.reloadable_features.quic_defer_send_in_response_to_packet`` and legacy code paths. - area: upstream change: | removed runtime key ``envoy.reloadable_features.fix_hash_key`` and legacy code paths. diff --git a/source/common/quic/envoy_quic_server_connection.cc b/source/common/quic/envoy_quic_server_connection.cc index dec29a480205..66d72c08b3f9 100644 --- a/source/common/quic/envoy_quic_server_connection.cc +++ b/source/common/quic/envoy_quic_server_connection.cc @@ -17,17 +17,13 @@ EnvoyQuicServerConnection::EnvoyQuicServerConnection( : quic::QuicConnection(server_connection_id, initial_self_address, initial_peer_address, &helper, &alarm_factory, writer, owns_writer, quic::Perspective::IS_SERVER, supported_versions, generator), - QuicNetworkConnection(std::move(connection_socket)), - defer_send_(Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.quic_defer_send_in_response_to_packet")) { + QuicNetworkConnection(std::move(connection_socket)) { #ifndef WIN32 - if (defer_send_) { - // Defer sending while processing UDP packets till the end of the current event loop to optimize - // UDP GSO sendmsg efficiency. But this optimization causes some test failures under Windows, - // and Windows doesn't support GSO, do not apply this optimization on Windows. - // TODO(#22976) Figure out if this is needed on Windows. - set_defer_send_in_response_to_packets(GetQuicFlag(quic_defer_send_in_response)); - } + // Defer sending while processing UDP packets till the end of the current event loop to optimize + // UDP GSO sendmsg efficiency. But this optimization causes some test failures under Windows, + // and Windows doesn't support GSO, do not apply this optimization on Windows. + // TODO(#22976) Figure out if this is needed on Windows. + set_defer_send_in_response_to_packets(GetQuicFlag(quic_defer_send_in_response)); #endif } diff --git a/source/common/quic/envoy_quic_server_connection.h b/source/common/quic/envoy_quic_server_connection.h index 487fe81ad168..5bb79ac0872e 100644 --- a/source/common/quic/envoy_quic_server_connection.h +++ b/source/common/quic/envoy_quic_server_connection.h @@ -27,12 +27,7 @@ class EnvoyQuicServerConnection : public quic::QuicConnection, public QuicNetwor bool OnPacketHeader(const quic::QuicPacketHeader& header) override; void OnCanWrite() override; - bool deferSend() const { return defer_send_; } - bool actuallyDeferSend() const { return defer_send_in_response_to_packets(); } - -private: - const bool defer_send_; }; // An implementation that issues connection IDs with stable first 4 types. diff --git a/source/common/quic/envoy_quic_server_session.cc b/source/common/quic/envoy_quic_server_session.cc index 5abe726f3c02..b0b016b7a898 100644 --- a/source/common/quic/envoy_quic_server_session.cc +++ b/source/common/quic/envoy_quic_server_session.cc @@ -199,11 +199,9 @@ quic::QuicSSLConfig EnvoyQuicServerSession::GetSSLConfig() const { void EnvoyQuicServerSession::ProcessUdpPacket(const quic::QuicSocketAddress& self_address, const quic::QuicSocketAddress& peer_address, const quic::QuicReceivedPacket& packet) { - if (quic_connection_->deferSend()) { - // If L4 filters causes the connection to be closed early during initialization, now - // is the time to actually close the connection. - maybeHandleCloseDuringInitialize(); - } + // If L4 filters causes the connection to be closed early during initialization, now + // is the time to actually close the connection. + maybeHandleCloseDuringInitialize(); quic::QuicServerSessionBase::ProcessUdpPacket(self_address, peer_address, packet); if (connection()->sent_server_preferred_address().IsInitialized() && self_address == connection()->sent_server_preferred_address()) { diff --git a/source/common/quic/quic_filter_manager_connection_impl.cc b/source/common/quic/quic_filter_manager_connection_impl.cc index 591d8f1f25ec..cf8c9032d0c0 100644 --- a/source/common/quic/quic_filter_manager_connection_impl.cc +++ b/source/common/quic/quic_filter_manager_connection_impl.cc @@ -147,12 +147,7 @@ void QuicFilterManagerConnectionImpl::updateBytesBuffered(uint64_t old_buffered_ void QuicFilterManagerConnectionImpl::maybeUpdateDelayCloseTimer(bool has_sent_any_data) { if (!inDelayedClose()) { - if (close_type_during_initialize_.has_value()) { - ASSERT(!Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.quic_defer_send_in_response_to_packet")); - close(close_type_during_initialize_.value()); - close_type_during_initialize_ = absl::nullopt; - } + ASSERT(!close_type_during_initialize_.has_value()); return; } if (has_sent_any_data && delayed_close_timer_ != nullptr) { diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index e7cc369a4b05..b89591a516c9 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -55,7 +55,6 @@ RUNTIME_GUARD(envoy_reloadable_features_oauth_use_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_original_dst_rely_on_idle_timeout); RUNTIME_GUARD(envoy_reloadable_features_prohibit_route_refresh_after_response_headers_sent); RUNTIME_GUARD(envoy_reloadable_features_quic_defer_logging_to_ack_listener); -RUNTIME_GUARD(envoy_reloadable_features_quic_defer_send_in_response_to_packet); RUNTIME_GUARD(envoy_reloadable_features_reject_require_client_certificate_with_quic); RUNTIME_GUARD(envoy_reloadable_features_sanitize_original_path); RUNTIME_GUARD(envoy_reloadable_features_service_sanitize_non_utf8_strings); diff --git a/test/common/quic/active_quic_listener_test.cc b/test/common/quic/active_quic_listener_test.cc index 92292b1369c5..5e4ff37453f7 100644 --- a/test/common/quic/active_quic_listener_test.cc +++ b/test/common/quic/active_quic_listener_test.cc @@ -381,9 +381,7 @@ TEST_P(ActiveQuicListenerTest, ReceiveCHLO) { ->max_time_before_crypto_handshake() .ToMilliseconds()); #ifndef WIN32 - EXPECT_EQ( - Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.quic_defer_send_in_response_to_packet"), + EXPECT_TRUE( static_cast(session->connection())->actuallyDeferSend()); #endif readFromClientSockets(); diff --git a/test/common/quic/envoy_quic_dispatcher_test.cc b/test/common/quic/envoy_quic_dispatcher_test.cc index e2c8d7d905e2..ce0bb7d806de 100644 --- a/test/common/quic/envoy_quic_dispatcher_test.cc +++ b/test/common/quic/envoy_quic_dispatcher_test.cc @@ -249,60 +249,54 @@ TEST_P(EnvoyQuicDispatcherTest, CreateNewConnectionUponCHLO) { } TEST_P(EnvoyQuicDispatcherTest, CloseConnectionDuringFilterInstallation) { - for (const std::string& value : {"true", "false"}) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.quic_defer_send_in_response_to_packet", value}}); - Network::MockFilterChainManager filter_chain_manager; - std::shared_ptr read_filter(new Network::MockReadFilter()); - Network::MockConnectionCallbacks network_connection_callbacks; - testing::StrictMock read_total; - testing::StrictMock read_current; - testing::StrictMock write_total; - testing::StrictMock write_current; + Network::MockFilterChainManager filter_chain_manager; + std::shared_ptr read_filter(new Network::MockReadFilter()); + Network::MockConnectionCallbacks network_connection_callbacks; + testing::StrictMock read_total; + testing::StrictMock read_current; + testing::StrictMock write_total; + testing::StrictMock write_current; - std::vector filter_factory( - {[&](Network::FilterManager& filter_manager) { - filter_manager.addReadFilter(read_filter); - read_filter->callbacks_->connection().addConnectionCallbacks( - network_connection_callbacks); - read_filter->callbacks_->connection().setConnectionStats( - {read_total, read_current, write_total, write_current, nullptr, nullptr}); - // This will not close connection right away, but during processing the first packet. - read_filter->callbacks_->connection().close(Network::ConnectionCloseType::NoFlush); - }}); + std::vector filter_factory( + {[&](Network::FilterManager& filter_manager) { + filter_manager.addReadFilter(read_filter); + read_filter->callbacks_->connection().addConnectionCallbacks(network_connection_callbacks); + read_filter->callbacks_->connection().setConnectionStats( + {read_total, read_current, write_total, write_current, nullptr, nullptr}); + // This will not close connection right away, but during processing the first packet. + read_filter->callbacks_->connection().close(Network::ConnectionCloseType::NoFlush); + }}); - EXPECT_CALL(listener_config_, filterChainManager()).WillOnce(ReturnRef(filter_chain_manager)); - EXPECT_CALL(filter_chain_manager, findFilterChain(_, _)) - .WillOnce(Return(&proof_source_->filterChain())); - EXPECT_CALL(proof_source_->filterChain(), networkFilterFactories()) - .WillOnce(ReturnRef(filter_factory)); - EXPECT_CALL(listener_config_, filterChainFactory()); - EXPECT_CALL(listener_config_.filter_chain_factory_, createNetworkFilterChain(_, _)) - .WillOnce(Invoke([](Network::Connection& connection, - const std::vector& filter_factories) { - EXPECT_EQ(1u, filter_factories.size()); - Server::Configuration::FilterChainUtility::buildFilterChain(connection, filter_factories); - return true; - })); - EXPECT_CALL(*read_filter, onNewConnection()) - // Stop iteration to avoid calling getRead/WriteBuffer(). - .WillOnce(Return(Network::FilterStatus::StopIteration)); + EXPECT_CALL(listener_config_, filterChainManager()).WillOnce(ReturnRef(filter_chain_manager)); + EXPECT_CALL(filter_chain_manager, findFilterChain(_, _)) + .WillOnce(Return(&proof_source_->filterChain())); + EXPECT_CALL(proof_source_->filterChain(), networkFilterFactories()) + .WillOnce(ReturnRef(filter_factory)); + EXPECT_CALL(listener_config_, filterChainFactory()); + EXPECT_CALL(listener_config_.filter_chain_factory_, createNetworkFilterChain(_, _)) + .WillOnce(Invoke([](Network::Connection& connection, + const std::vector& filter_factories) { + EXPECT_EQ(1u, filter_factories.size()); + Server::Configuration::FilterChainUtility::buildFilterChain(connection, filter_factories); + return true; + })); + EXPECT_CALL(*read_filter, onNewConnection()) + // Stop iteration to avoid calling getRead/WriteBuffer(). + .WillOnce(Return(Network::FilterStatus::StopIteration)); - EXPECT_CALL(network_connection_callbacks, onEvent(Network::ConnectionEvent::LocalClose)); - // Set QuicDispatcher::new_sessions_allowed_per_event_loop_ to - // |kNumSessionsToCreatePerLoopForTests| so that received CHLOs can be - // processed immediately. - envoy_quic_dispatcher_.ProcessBufferedChlos(kNumSessionsToCreatePerLoopForTests); - quic::QuicSocketAddress peer_addr(version_ == Network::Address::IpVersion::v4 - ? quic::QuicIpAddress::Loopback4() - : quic::QuicIpAddress::Loopback6(), - 54321); + EXPECT_CALL(network_connection_callbacks, onEvent(Network::ConnectionEvent::LocalClose)); + // Set QuicDispatcher::new_sessions_allowed_per_event_loop_ to + // |kNumSessionsToCreatePerLoopForTests| so that received CHLOs can be + // processed immediately. + envoy_quic_dispatcher_.ProcessBufferedChlos(kNumSessionsToCreatePerLoopForTests); + quic::QuicSocketAddress peer_addr(version_ == Network::Address::IpVersion::v4 + ? quic::QuicIpAddress::Loopback4() + : quic::QuicIpAddress::Loopback6(), + 54321); - processValidChloPacket(peer_addr); - connection_id_ = - quic::test::TestConnectionId(quic::test::TestConnectionIdToUInt64(connection_id_) + 1); - } + processValidChloPacket(peer_addr); + connection_id_ = + quic::test::TestConnectionId(quic::test::TestConnectionIdToUInt64(connection_id_) + 1); } TEST_P(EnvoyQuicDispatcherTest, CreateNewConnectionUponBufferedCHLO) { From 1cf0187dd75f265ede6ea9151532bb8e8f4a46fc Mon Sep 17 00:00:00 2001 From: rulex123 <29862113+rulex123@users.noreply.github.com> Date: Mon, 1 May 2023 17:33:57 +0200 Subject: [PATCH 046/740] stats: prometheus performance improvements (#24998) Add support for streaming/chunking of prometheus stats - basically adapts the algorithm introduced in #19693 for prometheus stats (while keeping common parts of the algorithm in a shared class to avoid code duplication). With this patch we should be able to improve the issue described here #16139 Signed-off-by: rulex123 --- source/server/admin/BUILD | 27 +- source/server/admin/admin.cc | 10 +- source/server/admin/grouped_stats_request.cc | 117 +++++++ source/server/admin/grouped_stats_request.h | 61 ++++ source/server/admin/prometheus_stats.cc | 285 ------------------ source/server/admin/prometheus_stats.h | 51 ---- source/server/admin/stats_handler.cc | 108 +++---- source/server/admin/stats_handler.h | 56 +--- source/server/admin/stats_render.cc | 165 ++++++++++ source/server/admin/stats_render.h | 122 +++++++- source/server/admin/stats_request.cc | 190 ++++-------- source/server/admin/stats_request.h | 124 +++++--- .../server/admin/ungrouped_stats_request.cc | 129 ++++++++ source/server/admin/ungrouped_stats_request.h | 49 +++ test/server/admin/BUILD | 38 ++- test/server/admin/admin_test.cc | 7 +- .../admin/grouped_stats_request_test.cc | 90 ++++++ test/server/admin/prometheus_stats_test.cc | 279 +++++++++-------- test/server/admin/stats_handler_speed_test.cc | 7 +- test/server/admin/stats_handler_test.cc | 7 +- test/server/admin/stats_render_test.cc | 2 - test/server/admin/stats_request_test.cc | 167 ---------- test/server/admin/stats_request_test_base.cc | 65 ++++ test/server/admin/stats_request_test_base.h | 82 +++++ .../admin/ungrouped_stats_request_test.cc | 88 ++++++ tools/code_format/config.yaml | 7 +- 26 files changed, 1360 insertions(+), 973 deletions(-) create mode 100644 source/server/admin/grouped_stats_request.cc create mode 100644 source/server/admin/grouped_stats_request.h delete mode 100644 source/server/admin/prometheus_stats.cc delete mode 100644 source/server/admin/prometheus_stats.h create mode 100644 source/server/admin/ungrouped_stats_request.cc create mode 100644 source/server/admin/ungrouped_stats_request.h create mode 100644 test/server/admin/grouped_stats_request_test.cc delete mode 100644 test/server/admin/stats_request_test.cc create mode 100644 test/server/admin/stats_request_test_base.cc create mode 100644 test/server/admin/stats_request_test_base.h create mode 100644 test/server/admin/ungrouped_stats_request_test.cc diff --git a/source/server/admin/BUILD b/source/server/admin/BUILD index 7fa1d22fac92..7d4123c7d5aa 100644 --- a/source/server/admin/BUILD +++ b/source/server/admin/BUILD @@ -139,11 +139,18 @@ envoy_cc_library( envoy_cc_library( name = "stats_request_lib", - srcs = ["stats_request.cc"], - hdrs = ["stats_request.h"], + srcs = [ + "grouped_stats_request.cc", + "stats_request.cc", + "ungrouped_stats_request.cc", + ], + hdrs = [ + "grouped_stats_request.h", + "stats_request.h", + "ungrouped_stats_request.h", + ], deps = [ ":handler_ctx_lib", - ":prometheus_stats_lib", ":stats_params_lib", ":stats_render_lib", ":utils_lib", @@ -183,7 +190,6 @@ envoy_cc_library( hdrs = ["stats_handler.h"], deps = [ ":handler_ctx_lib", - ":prometheus_stats_lib", ":stats_render_lib", ":stats_request_lib", ":utils_lib", @@ -197,19 +203,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "prometheus_stats_lib", - srcs = ["prometheus_stats.cc"], - hdrs = ["prometheus_stats.h"], - deps = [ - ":stats_params_lib", - ":utils_lib", - "//envoy/stats:custom_stat_namespaces_interface", - "//source/common/buffer:buffer_lib", - "//source/common/stats:histogram_lib", - ], -) - envoy_cc_library( name = "listeners_handler_lib", srcs = ["listeners_handler.cc"], diff --git a/source/server/admin/admin.cc b/source/server/admin/admin.cc index aab5216ecdfa..3e5bea114e49 100644 --- a/source/server/admin/admin.cc +++ b/source/server/admin/admin.cc @@ -187,15 +187,7 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server, makeHandler("/ready", "print server state, return 200 if LIVE, otherwise return 503", MAKE_ADMIN_HANDLER(server_info_handler_.handlerReady), false, false), stats_handler_.statsHandler(false /* not active mode */), - makeHandler("/stats/prometheus", "print server stats in prometheus format", - MAKE_ADMIN_HANDLER(stats_handler_.handlerPrometheusStats), false, false, - {{ParamDescriptor::Type::Boolean, "usedonly", - "Only include stats that have been written by system since restart"}, - {ParamDescriptor::Type::Boolean, "text_readouts", - "Render text_readouts as new gaugues with value 0 (increases Prometheus " - "data size)"}, - {ParamDescriptor::Type::String, "filter", - "Regular expression (Google re2) for filtering stats"}}), + stats_handler_.prometheusStatsHandler(), makeHandler("/stats/recentlookups", "Show recent stat-name lookups", MAKE_ADMIN_HANDLER(stats_handler_.handlerStatsRecentLookups), false, false), makeHandler("/stats/recentlookups/clear", "clear list of stat-name lookups and counter", diff --git a/source/server/admin/grouped_stats_request.cc b/source/server/admin/grouped_stats_request.cc new file mode 100644 index 000000000000..d4b86a81f6fb --- /dev/null +++ b/source/server/admin/grouped_stats_request.cc @@ -0,0 +1,117 @@ +#include "source/server/admin/grouped_stats_request.h" + +#include +#include + +#include "source/server/admin/stats_render.h" + +namespace Envoy { +namespace Server { + +GroupedStatsRequest::GroupedStatsRequest(Stats::Store& stats, const StatsParams& params, + Stats::CustomStatNamespaces& custom_namespaces, + UrlHandlerFn url_handler_fn) + : StatsRequest(stats, params, url_handler_fn), custom_namespaces_(custom_namespaces), + global_symbol_table_(stats.constSymbolTable()) { + + // The "type" query param is ignored for prometheus stats, so always start from + // counters; also, skip the TextReadouts phase unless that stat type is explicitly + // requested via query param. + if (params_.prometheus_text_readouts_) { + phases_ = {Phase::Counters, Phase::Gauges, Phase::TextReadouts, Phase::Histograms}; + } else { + phases_ = {Phase::Counters, Phase::Gauges, Phase::Histograms}; + } + phase_index_ = 0; +} + +template Stats::IterateFn GroupedStatsRequest::saveMatchingStat() { + return [this](const Stats::RefcountPtr& stat) -> bool { + // Check if unused. + if (params_.used_only_ && !stat->used()) { + return true; + } + + // Check if filtered. + if (params_.re2_filter_ != nullptr && + !re2::RE2::PartialMatch(stat->name(), *params_.re2_filter_)) { + return true; + } + + // Capture stat. + std::string tag_extracted_name = global_symbol_table_.toString(stat->tagExtractedStatName()); + stat_map_.insert({tag_extracted_name, std::vector>({})}); + absl::get>>(stat_map_[tag_extracted_name]) + .emplace_back(stat); + return true; + }; +} + +Stats::IterateFn GroupedStatsRequest::saveMatchingStatForTextReadout() { + return saveMatchingStat(); +} + +Stats::IterateFn GroupedStatsRequest::saveMatchingStatForGauge() { + return saveMatchingStat(); +} + +Stats::IterateFn GroupedStatsRequest::saveMatchingStatForCounter() { + return saveMatchingStat(); +} + +Stats::IterateFn GroupedStatsRequest::saveMatchingStatForHistogram() { + return saveMatchingStat(); +} + +template +void GroupedStatsRequest::renderStat(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + auto prefixed_tag_extracted_name = prefixedTagExtractedName(name); + if (prefixed_tag_extracted_name.has_value()) { + ++phase_stat_count_; + + // Sort group. + std::vector group = absl::get>(variant); + global_symbol_table_.sortByStatNames( + group.begin(), group.end(), + [](const SharedStatType& stat_ptr) -> Stats::StatName { return stat_ptr->statName(); }); + + // Render group. + render_->generate(response, prefixed_tag_extracted_name.value(), group); + } +} + +void GroupedStatsRequest::processTextReadout(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + renderStat(name, response, variant); +} + +void GroupedStatsRequest::processCounter(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + + renderStat(name, response, variant); +} + +void GroupedStatsRequest::processGauge(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + renderStat(name, response, variant); +} + +void GroupedStatsRequest::processHistogram(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + renderStat(name, response, variant); +} + +template +absl::optional +GroupedStatsRequest::prefixedTagExtractedName(const std::string& tag_extracted_name) { + return Envoy::Server::PrometheusStatsRender::metricName(tag_extracted_name, custom_namespaces_); +} + +void GroupedStatsRequest::setRenderPtr(Http::ResponseHeaderMap&) { + ASSERT(params_.format_ == StatsFormat::Prometheus); + render_ = std::make_unique(); +} + +} // namespace Server +} // namespace Envoy diff --git a/source/server/admin/grouped_stats_request.h b/source/server/admin/grouped_stats_request.h new file mode 100644 index 000000000000..8fce7a9cf901 --- /dev/null +++ b/source/server/admin/grouped_stats_request.h @@ -0,0 +1,61 @@ +#pragma once + +#include "source/server/admin/stats_request.h" + +namespace Envoy { +namespace Server { + +// In the context of this class, a stat is represented by a group of +// gauges, counters etc. with the same tag-extracted name. This class +// is currently used for Prometheus stats only (and has some Prometheus-specific +// logic in it), but if needed could be generalized for other +// formats. +// TODO(rulex123): cleanup any Prometheus-specific logic if we decide to have a grouped view +// for HTML or JSON stats. +class GroupedStatsRequest + : public StatsRequest, + std::vector, std::vector, + std::vector> { + +public: + GroupedStatsRequest(Stats::Store& stats, const StatsParams& params, + Stats::CustomStatNamespaces& custom_namespaces, + UrlHandlerFn url_handler_fn = nullptr); + +protected: + Stats::IterateFn saveMatchingStatForTextReadout() override; + Stats::IterateFn saveMatchingStatForGauge() override; + Stats::IterateFn saveMatchingStatForCounter() override; + Stats::IterateFn saveMatchingStatForHistogram() override; + template Stats::IterateFn saveMatchingStat(); + + void processTextReadout(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) override; + void processGauge(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) override; + void processCounter(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) override; + void processHistogram(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) override; + + // GroupedStatsRequest + template + absl::optional prefixedTagExtractedName(const std::string& tag_extracted_name); + + template + void renderStat(const std::string& name, Buffer::Instance& response, const StatOrScopes& variant); + + void setRenderPtr(Http::ResponseHeaderMap& response_headers) override; + StatsRenderBase& render() override { + ASSERT(render_ != nullptr); + return *render_; + } + +private: + Stats::CustomStatNamespaces& custom_namespaces_; + const Stats::SymbolTable& global_symbol_table_; + std::unique_ptr render_; +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/admin/prometheus_stats.cc b/source/server/admin/prometheus_stats.cc deleted file mode 100644 index d65891f1c29a..000000000000 --- a/source/server/admin/prometheus_stats.cc +++ /dev/null @@ -1,285 +0,0 @@ -#include "source/server/admin/prometheus_stats.h" - -#include "source/common/common/empty_string.h" -#include "source/common/common/macros.h" -#include "source/common/common/regex.h" -#include "source/common/stats/histogram_impl.h" - -#include "absl/strings/str_cat.h" -#include "absl/strings/str_replace.h" - -namespace Envoy { -namespace Server { - -namespace { - -const Regex::CompiledGoogleReMatcher& promRegex() { - CONSTRUCT_ON_FIRST_USE(Regex::CompiledGoogleReMatcher, "[^a-zA-Z0-9_]", false); -} - -/** - * Take a string and sanitize it according to Prometheus conventions. - */ -std::string sanitizeName(const absl::string_view name) { - // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by - // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/. - // The initial [a-zA-Z_] constraint is always satisfied by the namespace prefix. - return promRegex().replaceAll(name, "_"); -} - -/** - * Take tag values and sanitize it for text serialization, according to - * Prometheus conventions. - */ -std::string sanitizeValue(const absl::string_view value) { - // Removes problematic characters from Prometheus tag values to prevent - // text serialization issues. This matches the prometheus text formatting code: - // https://github.com/prometheus/common/blob/88f1636b699ae4fb949d292ffb904c205bf542c9/expfmt/text_create.go#L419-L420. - // The goal is to replace '\' with "\\", newline with "\n", and '"' with "\"". - return absl::StrReplaceAll(value, { - {R"(\)", R"(\\)"}, - {"\n", R"(\n)"}, - {R"(")", R"(\")"}, - }); -} - -/* - * Determine whether a metric has never been emitted and choose to - * not show it if we only wanted used metrics. - */ -template -static bool shouldShowMetric(const StatType& metric, const StatsParams& params) { - // This duplicates logic in StatsRequest::populateStatsFromScopes, but differs - // in one subtle way: in Prometheus we only use metric.name() for filtering, - // not rendering, so we only construct the name if there's a filter. - if (params.used_only_ && !metric.used()) { - return false; - } - if (params.re2_filter_ != nullptr && - !re2::RE2::PartialMatch(metric.name(), *params.re2_filter_)) { - return false; - } - return true; -} - -/* - * Comparator for Stats::Metric that does not require a string representation - * to make the comparison, for memory efficiency. - */ -struct MetricLessThan { - bool operator()(const Stats::Metric* a, const Stats::Metric* b) const { - ASSERT(&a->constSymbolTable() == &b->constSymbolTable()); - return a->constSymbolTable().lessThan(a->statName(), b->statName()); - } -}; - -/** - * Processes a stat type (counter, gauge, histogram) by generating all output lines, sorting - * them by tag-extracted metric name, and then outputting them in the correct sorted order into - * response. - * - * @param response The buffer to put the output into. - * @param used_only Whether to only output stats that are used. - * @param regex A filter on which stats to output. - * @param metrics The metrics to output stats for. This must contain all stats of the given type - * to be included in the same output. - * @param generate_output A function which returns the output text for this metric. - * @param type The name of the prometheus metric type for used in TYPE annotations. - */ -template -uint64_t outputStatType( - Buffer::Instance& response, const StatsParams& params, - const std::vector>& metrics, - const std::function& generate_output, - absl::string_view type, const Stats::CustomStatNamespaces& custom_namespaces) { - - /* - * From - * https:*github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting: - * - * All lines for a given metric must be provided as one single group, with the optional HELP and - * TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated - * expositions is preferred but not required, i.e. do not sort if the computational cost is - * prohibitive. - */ - - // This is an unsorted collection of dumb-pointers (no need to increment then decrement every - // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will - // be sorted before producing the final output to satisfy the "preferred" ordering from the - // prometheus spec: metrics will be sorted by their tags' textual representation, which will be - // consistent across calls. - using StatTypeUnsortedCollection = std::vector; - - // Return early to avoid crashing when getting the symbol table from the first metric. - if (metrics.empty()) { - return 0; - } - - // There should only be one symbol table for all of the stats in the admin - // interface. If this assumption changes, the name comparisons in this function - // will have to change to compare to convert all StatNames to strings before - // comparison. - const Stats::SymbolTable& global_symbol_table = metrics.front()->constSymbolTable(); - - // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements - // of the exposition format. - std::map groups( - global_symbol_table); - - for (const auto& metric : metrics) { - ASSERT(&global_symbol_table == &metric->constSymbolTable()); - if (!shouldShowMetric(*metric, params)) { - continue; - } - groups[metric->tagExtractedStatName()].push_back(metric.get()); - } - - auto result = groups.size(); - for (auto& group : groups) { - const absl::optional prefixed_tag_extracted_name = - PrometheusStatsFormatter::metricName(global_symbol_table.toString(group.first), - custom_namespaces); - if (!prefixed_tag_extracted_name.has_value()) { - --result; - continue; - } - response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name.value(), type)); - - // Sort before producing the final output to satisfy the "preferred" ordering from the - // prometheus spec: metrics will be sorted by their tags' textual representation, which will - // be consistent across calls. - std::sort(group.second.begin(), group.second.end(), MetricLessThan()); - - for (const auto& metric : group.second) { - response.add(generate_output(*metric, prefixed_tag_extracted_name.value())); - } - } - return result; -} - -/* - * Return the prometheus output for a numeric Stat (Counter or Gauge). - */ -template -std::string generateNumericOutput(const StatType& metric, - const std::string& prefixed_tag_extracted_name) { - const std::string tags = PrometheusStatsFormatter::formattedTags(metric.tags()); - return fmt::format("{0}{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, metric.value()); -} - -/* - * Returns the prometheus output for a TextReadout in gauge format. - * It is a workaround of a limitation of prometheus which stores only numeric metrics. - * The output is a gauge named the same as a given text-readout. The value of returned gauge is - * always equal to 0. Returned gauge contains all tags of a given text-readout and one additional - * tag {"text_value":"textReadout.value"}. - */ -std::string generateTextReadoutOutput(const Stats::TextReadout& text_readout, - const std::string& prefixed_tag_extracted_name) { - auto tags = text_readout.tags(); - tags.push_back(Stats::Tag{"text_value", text_readout.value()}); - const std::string formattedTags = PrometheusStatsFormatter::formattedTags(tags); - return fmt::format("{0}{{{1}}} 0\n", prefixed_tag_extracted_name, formattedTags); -} - -/* - * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded - * newlines) that contains all the individual bucket counts and sum/count for a single histogram - * (metric_name plus all tags). - */ -std::string generateHistogramOutput(const Stats::ParentHistogram& histogram, - const std::string& prefixed_tag_extracted_name) { - const std::string tags = PrometheusStatsFormatter::formattedTags(histogram.tags()); - const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ","); - - const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics(); - Stats::ConstSupportedBuckets& supported_buckets = stats.supportedBuckets(); - const std::vector& computed_buckets = stats.computedBuckets(); - std::string output; - for (size_t i = 0; i < supported_buckets.size(); ++i) { - double bucket = supported_buckets[i]; - uint64_t value = computed_buckets[i]; - // We want to print the bucket in a fixed point (non-scientific) format. The fmt library - // doesn't have a specific modifier to format as a fixed-point value only so we use the - // 'g' operator which prints the number in general fixed point format or scientific format - // with precision 50 to round the number up to 32 significant digits in fixed point format - // which should cover pretty much all cases - output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", prefixed_tag_extracted_name, - hist_tags, bucket, value)); - } - - output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", prefixed_tag_extracted_name, - hist_tags, stats.sampleCount())); - output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", prefixed_tag_extracted_name, tags, - stats.sampleSum())); - output.append(fmt::format("{0}_count{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, - stats.sampleCount())); - - return output; -}; - -} // namespace - -std::string PrometheusStatsFormatter::formattedTags(const std::vector& tags) { - std::vector buf; - buf.reserve(tags.size()); - for (const Stats::Tag& tag : tags) { - buf.push_back(fmt::format("{}=\"{}\"", sanitizeName(tag.name_), sanitizeValue(tag.value_))); - } - return absl::StrJoin(buf, ","); -} - -absl::optional -PrometheusStatsFormatter::metricName(const std::string& extracted_name, - const Stats::CustomStatNamespaces& custom_namespaces) { - const absl::optional custom_namespace_stripped = - custom_namespaces.stripRegisteredPrefix(extracted_name); - if (custom_namespace_stripped.has_value()) { - // This case the name has a custom namespace, and it is a custom metric. - const std::string sanitized_name = sanitizeName(custom_namespace_stripped.value()); - // We expose these metrics without modifying (e.g. without "envoy_"), - // so we have to check the "user-defined" stat name complies with the Prometheus naming - // convention. Specifically the name must start with the "[a-zA-Z_]" pattern. - // All the characters in sanitized_name are already in "[a-zA-Z0-9_]" pattern - // thanks to sanitizeName above, so the only thing we have to do is check - // if it does not start with digits. - if (sanitized_name.empty() || absl::ascii_isdigit(sanitized_name.front())) { - return absl::nullopt; - } - return sanitized_name; - } - - // If it does not have a custom namespace, add namespacing prefix to avoid conflicts, as per best - // practice: https://prometheus.io/docs/practices/naming/#metric-names Also, naming conventions on - // https://prometheus.io/docs/concepts/data_model/ - return absl::StrCat("envoy_", sanitizeName(extracted_name)); -} - -uint64_t PrometheusStatsFormatter::statsAsPrometheus( - const std::vector& counters, - const std::vector& gauges, - const std::vector& histograms, - const std::vector& text_readouts, Buffer::Instance& response, - const StatsParams& params, const Stats::CustomStatNamespaces& custom_namespaces) { - - uint64_t metric_name_count = 0; - metric_name_count += outputStatType(response, params, counters, - generateNumericOutput, - "counter", custom_namespaces); - - metric_name_count += outputStatType( - response, params, gauges, generateNumericOutput, "gauge", custom_namespaces); - - // TextReadout stats are returned in gauge format, so "gauge" type is set intentionally. - metric_name_count += outputStatType( - response, params, text_readouts, generateTextReadoutOutput, "gauge", custom_namespaces); - - metric_name_count += outputStatType( - response, params, histograms, generateHistogramOutput, "histogram", custom_namespaces); - - return metric_name_count; -} - -} // namespace Server -} // namespace Envoy diff --git a/source/server/admin/prometheus_stats.h b/source/server/admin/prometheus_stats.h deleted file mode 100644 index 9677e86685c1..000000000000 --- a/source/server/admin/prometheus_stats.h +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/buffer/buffer.h" -#include "envoy/stats/custom_stat_namespaces.h" -#include "envoy/stats/histogram.h" -#include "envoy/stats/stats.h" - -#include "source/server/admin/stats_params.h" - -namespace Envoy { -namespace Server { -/** - * Formatter for metric/labels exported to Prometheus. - * - * See: https://prometheus.io/docs/concepts/data_model - */ -class PrometheusStatsFormatter { -public: - /** - * Extracts counters and gauges and relevant tags, appending them to - * the response buffer after sanitizing the metric / label names. - * @return uint64_t total number of metric types inserted in response. - */ - static uint64_t statsAsPrometheus(const std::vector& counters, - const std::vector& gauges, - const std::vector& histograms, - const std::vector& text_readouts, - Buffer::Instance& response, const StatsParams& params, - const Stats::CustomStatNamespaces& custom_namespaces); - /** - * Format the given tags, returning a string as a comma-separated list - * of ="" pairs. - */ - static std::string formattedTags(const std::vector& tags); - - /** - * Format the given metric name, and prefixed with "envoy_" if it does not have a custom - * stat namespace. If it has a custom stat namespace AND the name without the custom namespace - * has a valid prometheus namespace, the trimmed name is returned. - * Otherwise, return nullopt. - */ - static absl::optional - metricName(const std::string& extracted_name, - const Stats::CustomStatNamespaces& custom_namespace_factory); -}; - -} // namespace Server -} // namespace Envoy diff --git a/source/server/admin/stats_handler.cc b/source/server/admin/stats_handler.cc index be645ebddd4f..9cdc923bad17 100644 --- a/source/server/admin/stats_handler.cc +++ b/source/server/admin/stats_handler.cc @@ -10,8 +10,9 @@ #include "source/common/common/empty_string.h" #include "source/common/http/headers.h" #include "source/common/http/utility.h" -#include "source/server/admin/prometheus_stats.h" -#include "source/server/admin/stats_request.h" +#include "source/server/admin/grouped_stats_request.h" +#include "source/server/admin/stats_params.h" +#include "source/server/admin/ungrouped_stats_request.h" #include "absl/strings/numbers.h" @@ -71,31 +72,22 @@ Http::Code StatsHandler::handlerStatsRecentLookupsEnable(Http::ResponseHeaderMap return Http::Code::OK; } -Admin::RequestPtr StatsHandler::makeRequest(AdminStream& admin_stream) { - StatsParams params; +Admin::RequestPtr StatsHandler::makeRequest(AdminStream& admin_stream, StatsParams& params) { Buffer::OwnedImpl response; Http::Code code = params.parse(admin_stream.getRequestHeaders().getPathValue(), response); if (code != Http::Code::OK) { return Admin::makeStaticTextRequest(response, code); } - if (params.format_ == StatsFormat::Prometheus) { - // TODO(#16139): modify streaming algorithm to cover Prometheus. - // - // This may be easiest to accomplish by populating the set - // with tagExtractedName(), and allowing for vectors of - // stats as multiples will have the same tag-extracted names. - // Ideally we'd find a way to do this without slowing down - // the non-Prometheus implementations. - Buffer::OwnedImpl response; - prometheusFlushAndRender(params, response); - return Admin::makeStaticTextRequest(response, code); - } - if (server_.statsConfig().flushOnAdmin()) { server_.flushStats(); } + if (params.format_ == StatsFormat::Prometheus) { + return makePrometheusRequest( + server_.stats(), params, server_.api().customStatNamespaces(), + [this]() -> Admin::UrlHandler { return prometheusStatsHandler(); }); + } #ifdef ENVOY_ADMIN_HTML const bool active_mode = params.format_ == StatsFormat::ActiveHtml; return makeRequest(server_.stats(), params, [this, active_mode]() -> Admin::UrlHandler { @@ -108,48 +100,15 @@ Admin::RequestPtr StatsHandler::makeRequest(AdminStream& admin_stream) { } Admin::RequestPtr StatsHandler::makeRequest(Stats::Store& stats, const StatsParams& params, - StatsRequest::UrlHandlerFn url_handler_fn) { - return std::make_unique(stats, params, url_handler_fn); + UngroupedStatsRequest::UrlHandlerFn url_handler_fn) { + return std::make_unique(stats, params, url_handler_fn); } -Http::Code StatsHandler::handlerPrometheusStats(Http::ResponseHeaderMap&, - Buffer::Instance& response, - AdminStream& admin_stream) { - return prometheusStats(admin_stream.getRequestHeaders().getPathValue(), response); -} - -Http::Code StatsHandler::prometheusStats(absl::string_view path_and_query, - Buffer::Instance& response) { - StatsParams params; - Http::Code code = params.parse(path_and_query, response); - if (code != Http::Code::OK) { - return code; - } - - if (server_.statsConfig().flushOnAdmin()) { - server_.flushStats(); - } - - prometheusFlushAndRender(params, response); - return Http::Code::OK; -} - -void StatsHandler::prometheusFlushAndRender(const StatsParams& params, Buffer::Instance& response) { - if (server_.statsConfig().flushOnAdmin()) { - server_.flushStats(); - } - prometheusRender(server_.stats(), server_.api().customStatNamespaces(), params, response); -} - -void StatsHandler::prometheusRender(Stats::Store& stats, - const Stats::CustomStatNamespaces& custom_namespaces, - const StatsParams& params, Buffer::Instance& response) { - const std::vector& text_readouts_vec = - params.prometheus_text_readouts_ ? stats.textReadouts() - : std::vector(); - PrometheusStatsFormatter::statsAsPrometheus(stats.counters(), stats.gauges(), stats.histograms(), - text_readouts_vec, response, params, - custom_namespaces); +Admin::RequestPtr +StatsHandler::makePrometheusRequest(Stats::Store& stats, const StatsParams& params, + Stats::CustomStatNamespaces& custom_namespaces, + GroupedStatsRequest::UrlHandlerFn url_handler_fn) { + return std::make_unique(stats, params, custom_namespaces, url_handler_fn); } Http::Code StatsHandler::handlerContention(Http::ResponseHeaderMap& response_headers, @@ -195,13 +154,34 @@ Admin::UrlHandler StatsHandler::statsHandler(bool active_mode) { } params.insert(params.end(), common_params.begin(), common_params.end()); - return { - "/stats", - "print server stats", - [this](AdminStream& admin_stream) -> Admin::RequestPtr { return makeRequest(admin_stream); }, - false, - false, - params}; + return {"/stats", + "print server stats", + [this](AdminStream& admin_stream) -> Admin::RequestPtr { + StatsParams params; + return makeRequest(admin_stream, params); + }, + false, + false, + params}; +} + +Admin::UrlHandler StatsHandler::prometheusStatsHandler() { + return {"/stats/prometheus", + "print server stats in prometheus format", + [this](AdminStream& admin_stream) -> Admin::RequestPtr { + StatsParams params; + params.format_ = StatsFormat::Prometheus; + return makeRequest(admin_stream, params); + }, + false, + false, + {{Admin::ParamDescriptor::Type::Boolean, "usedonly", + "Only include stats that have been written by system since restart"}, + {Admin::ParamDescriptor::Type::Boolean, "text_readouts", + "Render text_readouts as new gaugues with value 0 (increases Prometheus " + "data size)"}, + {Admin::ParamDescriptor::Type::String, "filter", + "Regular expression (Google re2) for filtering stats"}}}; } } // namespace Server diff --git a/source/server/admin/stats_handler.h b/source/server/admin/stats_handler.h index fb29712600b8..9570bd96ce94 100644 --- a/source/server/admin/stats_handler.h +++ b/source/server/admin/stats_handler.h @@ -9,8 +9,9 @@ #include "envoy/server/admin.h" #include "envoy/server/instance.h" +#include "source/server/admin/grouped_stats_request.h" #include "source/server/admin/handler_ctx.h" -#include "source/server/admin/stats_request.h" +#include "source/server/admin/ungrouped_stats_request.h" #include "source/server/admin/utils.h" #include "absl/strings/string_view.h" @@ -33,42 +34,6 @@ class StatsHandler : public HandlerContextBase { Buffer::Instance& response, AdminStream&); Http::Code handlerStatsRecentLookupsEnable(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&); - Http::Code handlerPrometheusStats(Http::ResponseHeaderMap& response_headers, - Buffer::Instance& response, AdminStream&); - - /** - * Parses and executes a prometheus stats request. - * - * @param path_and_query the URL path and query - * @param response buffer into which to write response - * @return http response code - */ - Http::Code prometheusStats(absl::string_view path_and_query, Buffer::Instance& response); - - /** - * Checks the server_ to see if a flush is needed, and then renders the - * prometheus stats request. - * - * @params params the already-parsed parameters. - * @param response buffer into which to write response - */ - void prometheusFlushAndRender(const StatsParams& params, Buffer::Instance& response); - - /** - * Renders the stats as prometheus. This is broken out as a separately - * callable API to facilitate the benchmark - * (test/server/admin/stats_handler_speed_test.cc) which does not have a - * server object. - * - * @params stats the stats store to read - * @param custom_namespaces namespace mappings used for prometheus - * @params params the already-parsed parameters. - * @param response buffer into which to write response - */ - static void prometheusRender(Stats::Store& stats, - const Stats::CustomStatNamespaces& custom_namespaces, - const StatsParams& params, Buffer::Instance& response); - Http::Code handlerContention(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&); @@ -86,14 +51,17 @@ class StatsHandler : public HandlerContextBase { */ Admin::UrlHandler statsHandler(bool active_mode); - static Admin::RequestPtr makeRequest(Stats::Store& stats, const StatsParams& params, - StatsRequest::UrlHandlerFn url_handler_fn = nullptr); - Admin::RequestPtr makeRequest(AdminStream&); + Admin::UrlHandler prometheusStatsHandler(); + + static Admin::RequestPtr + makeRequest(Stats::Store& stats, const StatsParams& params, + UngroupedStatsRequest::UrlHandlerFn url_handler_fn = nullptr); -private: - static Http::Code prometheusStats(absl::string_view path_and_query, Buffer::Instance& response, - Stats::Store& stats, - Stats::CustomStatNamespaces& custom_namespaces); + static Admin::RequestPtr + makePrometheusRequest(Stats::Store& stats, const StatsParams& params, + Stats::CustomStatNamespaces& custom_namespaces, + GroupedStatsRequest::UrlHandlerFn url_handler_fn = nullptr); + Admin::RequestPtr makeRequest(AdminStream& admin_stream, StatsParams& params); }; } // namespace Server diff --git a/source/server/admin/stats_render.cc b/source/server/admin/stats_render.cc index e75dab1b0a28..bc3fb3eb1fe0 100644 --- a/source/server/admin/stats_render.cc +++ b/source/server/admin/stats_render.cc @@ -1,14 +1,24 @@ #include "source/server/admin/stats_render.h" +#include + +#include "source/common/common/empty_string.h" +#include "source/common/common/regex.h" #include "source/common/json/json_sanitizer.h" #include "source/common/stats/histogram_impl.h" +#include "absl/strings/str_replace.h" + namespace { constexpr absl::string_view JsonNameTag = "{\"name\":\""; constexpr absl::string_view JsonValueTag = "\",\"value\":"; constexpr absl::string_view JsonValueTagQuote = "\",\"value\":\""; constexpr absl::string_view JsonCloseBrace = "}"; constexpr absl::string_view JsonQuoteCloseBrace = "\"}"; + +const Envoy::Regex::CompiledGoogleReMatcher& prometheusRegex() { + CONSTRUCT_ON_FIRST_USE(Envoy::Regex::CompiledGoogleReMatcher, "[^a-zA-Z0-9_]", false); +} } // namespace namespace Envoy { @@ -262,5 +272,160 @@ void StatsJsonRender::collectBuckets(const std::string& name, *histogram_array_->add_values() = ValueUtil::structValue(histogram_obj); } +// Writes output for a Prometheus stat of type Gauge. +void PrometheusStatsRender::generate(Buffer::Instance& response, + const std::string& prefixed_tag_extracted_name, + const std::vector& gauge) { + outputStatType(response, gauge, prefixed_tag_extracted_name, + generateNumericOutput, "gauge"); +} + +// Writes output for a Prometheus stat of type Counter. +void PrometheusStatsRender::generate(Buffer::Instance& response, + const std::string& prefixed_tag_extracted_name, + const std::vector& counter) { + outputStatType(response, counter, prefixed_tag_extracted_name, + generateNumericOutput, + "counter"); +} + +// Writes output for a Prometheus stat of type Text Readout. +void PrometheusStatsRender::generate(Buffer::Instance& response, + const std::string& prefixed_tag_extracted_name, + const std::vector& text_readout) { + // text readout stats are returned in gauge format, so "gauge" type is set intentionally. + outputStatType(response, text_readout, prefixed_tag_extracted_name, + generateTextReadoutOutput, "gauge"); +} + +// Writes output for a Prometheus stat of type Histogram. +void PrometheusStatsRender::generate(Buffer::Instance& response, + const std::string& prefixed_tag_extracted_name, + const std::vector& histogram) { + outputStatType(response, histogram, prefixed_tag_extracted_name, + generateHistogramOutput, "histogram"); +} + +void PrometheusStatsRender::finalize(Buffer::Instance&) {} + +std::string PrometheusStatsRender::formattedTags(const std::vector& tags) { + std::vector buf; + buf.reserve(tags.size()); + for (const Stats::Tag& tag : tags) { + buf.push_back(absl::StrCat(sanitizeName(tag.name_), "=\"", sanitizeValue(tag.value_), "\"")); + } + return absl::StrJoin(buf, ","); +} + +absl::optional +PrometheusStatsRender::metricName(const std::string& extracted_name, + const Stats::CustomStatNamespaces& custom_namespaces) { + const absl::optional custom_namespace_stripped = + custom_namespaces.stripRegisteredPrefix(extracted_name); + if (custom_namespace_stripped.has_value()) { + // This case the name has a custom namespace, and it is a custom metric. + const std::string sanitized_name = sanitizeName(custom_namespace_stripped.value()); + // We expose these metrics without modifying (e.g. without "envoy_"), + // so we have to check the "user-defined" stat name complies with the Prometheus naming + // convention. Specifically the name must start with the "[a-zA-Z_]" pattern. + // All the characters in sanitized_name are already in "[a-zA-Z0-9_]" pattern + // thanks to sanitizeName above, so the only thing we have to do is check + // if it does not start with digits. + if (sanitized_name.empty() || absl::ascii_isdigit(sanitized_name.front())) { + return absl::nullopt; + } + return sanitized_name; + } + + // If it does not have a custom namespace, add namespacing prefix to avoid conflicts, as per best + // practice: https://prometheus.io/docs/practices/naming/#metric-names Also, naming conventions on + // https://prometheus.io/docs/concepts/data_model/ + return absl::StrCat("envoy_", sanitizeName(extracted_name)); +} + +std::string PrometheusStatsRender::sanitizeName(const absl::string_view name) { + // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by + // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/. + // The initial [a-zA-Z_] constraint is always satisfied by the namespace prefix. + return prometheusRegex().replaceAll(name, "_"); +} + +std::string PrometheusStatsRender::sanitizeValue(const absl::string_view value) { + // Removes problematic characters from Prometheus tag values to prevent + // text serialization issues. This matches the prometheus text formatting code: + // https://github.com/prometheus/common/blob/88f1636b699ae4fb949d292ffb904c205bf542c9/expfmt/text_create.go#L419-L420. + // The goal is to replace '\' with "\\", newline with "\n", and '"' with "\"". + return absl::StrReplaceAll(value, { + {R"(\)", R"(\\)"}, + {"\n", R"(\n)"}, + {R"(")", R"(\")"}, + }); +} + +template +void PrometheusStatsRender::outputStatType( + Buffer::Instance& response, const std::vector& metrics, + const std::string& prefixed_tag_extracted_name, + const std::function& generate_output, + absl::string_view type) { + response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name, type)); + for (const auto& metric : metrics) { + response.add(generate_output(metric, prefixed_tag_extracted_name)); + } +} + +template +std::string +PrometheusStatsRender::generateNumericOutput(const StatType& metric, + const std::string& prefixed_tag_extracted_name) { + const std::string tags = formattedTags(metric->tags()); + return absl::StrCat(prefixed_tag_extracted_name, "{", tags, "} ", metric->value(), "\n"); +} + +std::string +PrometheusStatsRender::generateTextReadoutOutput(const Stats::TextReadoutSharedPtr& metric, + const std::string& prefixed_tag_extracted_name) { + auto tags = metric->tags(); + tags.push_back(Stats::Tag{"text_value", metric->value()}); + const std::string formTags = formattedTags(tags); + return absl::StrCat(prefixed_tag_extracted_name, "{", formTags, "} 0\n"); +} + +std::string +PrometheusStatsRender::generateHistogramOutput(const Stats::HistogramSharedPtr& metric, + const std::string& prefixed_tag_extracted_name) { + auto parent_histogram = dynamic_cast(metric.get()); + if (parent_histogram != nullptr) { + const std::string tags = formattedTags(parent_histogram->tags()); + const std::string hist_tags = parent_histogram->tags().empty() ? EMPTY_STRING : (tags + ","); + + const Stats::HistogramStatistics& stats = parent_histogram->cumulativeStatistics(); + Stats::ConstSupportedBuckets& supported_buckets = stats.supportedBuckets(); + const std::vector& computed_buckets = stats.computedBuckets(); + std::string output; + for (size_t i = 0; i < supported_buckets.size(); ++i) { + double bucket = supported_buckets[i]; + uint64_t value = computed_buckets[i]; + // We want to print the bucket in a fixed point (non-scientific) format. The fmt library + // doesn't have a specific modifier to format as a fixed-point value only so we use the + // 'g' operator which prints the number in general fixed point format or scientific format + // with precision 50 to round the number up to 32 significant digits in fixed point format + // which should cover pretty much all cases + output.append(absl::StrCat(prefixed_tag_extracted_name, "_bucket{", hist_tags, "le=\"", + fmt::format("{0:.32g}", bucket), "\"} ", value, "\n")); + } + + output.append(absl::StrCat(prefixed_tag_extracted_name, "_bucket{", hist_tags, "le=\"+Inf\"} ", + stats.sampleCount(), "\n")); + output.append(absl::StrCat(prefixed_tag_extracted_name, "_sum{", tags, "} ", + fmt::format("{0:.32g}", stats.sampleSum()), "\n")); + output.append(absl::StrCat(prefixed_tag_extracted_name, "_count{", tags, "} ", + stats.sampleCount(), "\n")); + return output; + } + return EMPTY_STRING; +} + } // namespace Server } // namespace Envoy diff --git a/source/server/admin/stats_render.h b/source/server/admin/stats_render.h index 477c3eb6695e..efc55c804c91 100644 --- a/source/server/admin/stats_render.h +++ b/source/server/admin/stats_render.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "envoy/server/admin.h" #include "envoy/stats/stats.h" @@ -10,14 +12,29 @@ namespace Envoy { namespace Server { -// Abstract class for rendering stats. Every method is called "generate" -// differing only by the data type, to facilitate templatized call-sites. -// -// There are currently Json and Text implementations of this interface, and in -// #19546 an HTML version will be added to provide a hierarchical view. -class StatsRender { +// Abstract base class for rendering stats, which captures logic +// that is shared across all formats (e.g. finalizing rendering of stats). +// The APIs for generating stats output vary between formats (e.g. +// JSON vs. Prometheus) and are defined in derived classes: having a base +// render class avoids code duplication while affording the flexibility to +// capture any differences in how we generate stats output. +class StatsRenderBase { +public: + virtual ~StatsRenderBase() = default; + + // Completes rendering any buffered data. + virtual void finalize(Buffer::Instance& response) PURE; + + // Indicates that no stats for a particular type have been found. + virtual void noStats(Buffer::Instance&, absl::string_view) {} +}; + +// Abstract class for rendering ungrouped stats. +// Every method is called "generate" differing only by the data type, to +// facilitate templatized call-sites. +class StatsRender : public StatsRenderBase { public: - virtual ~StatsRender() = default; + ~StatsRender() override = default; // Writes a fragment for a numeric value, for counters and gauges. virtual void generate(Buffer::Instance& response, const std::string& name, uint64_t value) PURE; @@ -29,12 +46,6 @@ class StatsRender { // Writes a histogram value. virtual void generate(Buffer::Instance& response, const std::string& name, const Stats::ParentHistogram& histogram) PURE; - - // Completes rendering any buffered data. - virtual void finalize(Buffer::Instance& response) PURE; - - // Indicates that no stats for a particular type have been found. - virtual void noStats(Buffer::Instance&, absl::string_view) {} }; // Implements the Render interface for simple textual representation of stats. @@ -95,5 +106,90 @@ class StatsJsonRender : public StatsRender { std::string value_buffer_; // Used for Json::sanitize for text-readout values. }; +// Implements the Render base interface for textual representation of Prometheus stats +// (see: https://prometheus.io/docs/concepts/data_model). +// The APIs for rendering Prometheus stats take as input a string (the stat's name) +// and a vector of counters, gauges etc. (each entry in the vector holds a set of labels +// and the stat's value for that set of labels). The Prometheus stat is rendered +// as per the text-based format described at +// https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format. +class PrometheusStatsRender : public StatsRenderBase { +public: + void generate(Buffer::Instance& response, const std::string& prefixed_tag_extracted_name, + const std::vector& histogram); + + void generate(Buffer::Instance& response, const std::string& prefixed_tag_extracted_name, + const std::vector& gauge); + + void generate(Buffer::Instance& response, const std::string& prefixed_tag_extracted_name, + const std::vector& counter); + + void generate(Buffer::Instance& response, const std::string& prefixed_tag_extracted_name, + const std::vector& text_readout); + + void finalize(Buffer::Instance&) override; + + /** + * Format the given tags, returning a string as a comma-separated list + * of ="" pairs. + */ + static std::string formattedTags(const std::vector& tags); + + /** + * Format the given metric name, and prefixed with "envoy_" if it does not have a custom + * stat namespace. If it has a custom stat namespace AND the name without the custom namespace + * has a valid prometheus namespace, the trimmed name is returned. + * Otherwise, return nullopt. + */ + static absl::optional + metricName(const std::string& extracted_name, + const Stats::CustomStatNamespaces& custom_namespace_factory); + +private: + /** + * Take a string and sanitize it according to Prometheus conventions. + */ + static std::string sanitizeName(const absl::string_view name); + + /** + * Take tag values and sanitize it for text serialization, according to + * Prometheus conventions. + */ + static std::string sanitizeValue(const absl::string_view value); + + template + void outputStatType( + Envoy::Buffer::Instance& response, const std::vector& metrics, + const std::string& prefixed_tag_extracted_name, + const std::function& generate_output, + absl::string_view type); + + /* + * Return the prometheus output for a numeric Stat (Counter or Gauge). + */ + template + static std::string generateNumericOutput(const StatType& metric, + const std::string& prefixed_tag_extracted_name); + + /* + * Returns the prometheus output for a TextReadout in gauge format. + * It is a workaround of a limitation of prometheus which stores only numeric metrics. + * The output is a gauge named the same as a given text-readout. The value of returned gauge is + * always equal to 0. Returned gauge contains all tags of a given text-readout and one additional + * tag {"text_value":"textReadout.value"}. + */ + static std::string generateTextReadoutOutput(const Stats::TextReadoutSharedPtr& metric, + const std::string& prefixed_tag_extracted_name); + + /* + * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded + * newlines) that contains all the individual bucket counts and sum/count for a single histogram + * (metric_name plus all tags). + */ + static std::string generateHistogramOutput(const Stats::HistogramSharedPtr& histogram, + const std::string& prefixed_tag_extracted_name); +}; + } // namespace Server } // namespace Envoy diff --git a/source/server/admin/stats_request.cc b/source/server/admin/stats_request.cc index 954eb1127dbf..410c10909ae8 100644 --- a/source/server/admin/stats_request.cc +++ b/source/server/admin/stats_request.cc @@ -1,55 +1,22 @@ #include "source/server/admin/stats_request.h" -#ifdef ENVOY_ADMIN_HTML -#include "source/server/admin/stats_html_render.h" -#endif - namespace Envoy { namespace Server { -StatsRequest::StatsRequest(Stats::Store& stats, const StatsParams& params, - UrlHandlerFn url_handler_fn) - : params_(params), stats_(stats), url_handler_fn_(url_handler_fn) { - switch (params_.type_) { - case StatsType::TextReadouts: - case StatsType::All: - phase_ = Phase::TextReadouts; - break; - case StatsType::Counters: - case StatsType::Gauges: - phase_ = Phase::CountersAndGauges; - break; - case StatsType::Histograms: - phase_ = Phase::Histograms; - break; - } -} +template +StatsRequest::StatsRequest( + Stats::Store& stats, const StatsParams& params, UrlHandlerFn url_handler_fn) + : params_(params), url_handler_fn_(url_handler_fn), stats_(stats) {} -Http::Code StatsRequest::start(Http::ResponseHeaderMap& response_headers) { - switch (params_.format_) { - case StatsFormat::Json: - render_ = std::make_unique(response_headers, response_, params_); - break; - case StatsFormat::Text: - render_ = std::make_unique(params_); - break; +template +Http::Code StatsRequest::start( + Http::ResponseHeaderMap& response_headers) { + setRenderPtr(response_headers); #ifdef ENVOY_ADMIN_HTML - case StatsFormat::ActiveHtml: - case StatsFormat::Html: { - auto html_render = std::make_unique(response_headers, response_, params_); - html_render->setupStatsPage(url_handler_fn_(), params_, response_); - render_ = std::move(html_render); - if (params_.format_ == StatsFormat::ActiveHtml) { - return Http::Code::OK; - } - break; + if (params_.format_ == StatsFormat::ActiveHtml) { + return Http::Code::OK; } #endif - case StatsFormat::Prometheus: - // TODO(#16139): once Prometheus shares this algorithm here, this becomes a legitimate choice. - IS_ENVOY_BUG("reached Prometheus case in switch unexpectedly"); - return Http::Code::BadRequest; - } // Populate the top-level scopes and the stats underneath any scopes with an empty name. // We will have to de-dup, but we can do that after sorting. @@ -64,7 +31,9 @@ Http::Code StatsRequest::start(Http::ResponseHeaderMap& response_headers) { return Http::Code::OK; } -bool StatsRequest::nextChunk(Buffer::Instance& response) { +template +bool StatsRequest::nextChunk( + Buffer::Instance& response) { if (response_.length() > 0) { ASSERT(response.length() == 0); response.move(response_); @@ -73,38 +42,35 @@ bool StatsRequest::nextChunk(Buffer::Instance& response) { // nextChunk's contract is to add up to chunk_size_ additional bytes. The // caller is not required to drain the bytes after each call to nextChunk. + StatsRenderBase& stats_render = render(); const uint64_t starting_response_length = response.length(); while (response.length() - starting_response_length < chunk_size_) { while (stat_map_.empty()) { if (phase_stat_count_ == 0) { - render_->noStats(response, phase_string_); + stats_render.noStats(response, phase_labels_[phases_.at(phase_index_)]); } else { phase_stat_count_ = 0; } if (params_.type_ != StatsType::All) { - render_->finalize(response); + stats_render.finalize(response); return false; } - switch (phase_) { - case Phase::TextReadouts: - phase_ = Phase::CountersAndGauges; - phase_string_ = "Counters and Gauges"; - startPhase(); - break; - case Phase::CountersAndGauges: - phase_ = Phase::Histograms; - phase_string_ = "Histograms"; - startPhase(); - break; - case Phase::Histograms: - render_->finalize(response); + + // Check if we are at the last phase: in that case, we are done; + // if not, increment phase index and start next phase. + if (phase_index_ == phases_.size() - 1) { + stats_render.finalize(response); return false; + } else { + phase_index_++; + startPhase(); } } auto iter = stat_map_.begin(); StatOrScopes variant = std::move(iter->second); StatOrScopesIndex index = static_cast(variant.index()); + switch (index) { case StatOrScopesIndex::Scopes: // Erase the current element before adding new ones, as absl::btree_map @@ -115,100 +81,78 @@ bool StatsRequest::nextChunk(Buffer::Instance& response) { populateStatsForCurrentPhase(absl::get(variant)); break; case StatOrScopesIndex::TextReadout: - renderStat(iter->first, response, variant); + processTextReadout(iter->first, response, variant); stat_map_.erase(iter); - ++phase_stat_count_; break; case StatOrScopesIndex::Counter: - renderStat(iter->first, response, variant); + processCounter(iter->first, response, variant); stat_map_.erase(iter); - ++phase_stat_count_; break; case StatOrScopesIndex::Gauge: - renderStat(iter->first, response, variant); + processGauge(iter->first, response, variant); stat_map_.erase(iter); - ++phase_stat_count_; break; - case StatOrScopesIndex::Histogram: { - auto histogram = absl::get(variant); - auto parent_histogram = dynamic_cast(histogram.get()); - if (parent_histogram != nullptr) { - render_->generate(response, iter->first, *parent_histogram); - ++phase_stat_count_; - } + case StatOrScopesIndex::Histogram: + processHistogram(iter->first, response, variant); stat_map_.erase(iter); - } + break; } } return true; } -void StatsRequest::startPhase() { +template +void StatsRequest::startPhase() { ASSERT(stat_map_.empty()); // Insert all the scopes in the alphabetically ordered map. As we iterate // through the map we'll erase the scopes and replace them with the stats held // in the scopes. for (const Stats::ConstScopeSharedPtr& scope : scopes_) { + // The operator[] of btree_map runs a try_emplace behind the scenes, + // inserting the variant into the map when the lookup key does not exist. StatOrScopes& variant = stat_map_[stats_.symbolTable().toString(scope->prefix())]; - if (variant.index() == absl::variant_npos) { - variant = ScopeVec(); - } + ASSERT(static_cast(variant.index()) == StatOrScopesIndex::Scopes); absl::get(variant).emplace_back(scope); } } -void StatsRequest::populateStatsForCurrentPhase(const ScopeVec& scope_vec) { - switch (phase_) { - case Phase::TextReadouts: - populateStatsFromScopes(scope_vec); - break; - case Phase::CountersAndGauges: - if (params_.type_ != StatsType::Gauges) { - populateStatsFromScopes(scope_vec); - } - if (params_.type_ != StatsType::Counters) { - populateStatsFromScopes(scope_vec); +template +void StatsRequest::populateStatsForCurrentPhase(const ScopeVec& scope_vec) { + Phase current_phase = phases_.at(phase_index_); + for (const Stats::ConstScopeSharedPtr& scope : scope_vec) { + switch (current_phase) { + case Phase::TextReadouts: + scope->iterate(saveMatchingStatForTextReadout()); + break; + case Phase::CountersAndGauges: + if (params_.type_ != StatsType::Gauges) { + scope->iterate(saveMatchingStatForCounter()); + } + if (params_.type_ != StatsType::Counters) { + scope->iterate(saveMatchingStatForGauge()); + } + break; + case Phase::Counters: + scope->iterate(saveMatchingStatForCounter()); + break; + case Phase::Gauges: + scope->iterate(saveMatchingStatForGauge()); + break; + case Phase::Histograms: + scope->iterate(saveMatchingStatForHistogram()); + break; } - break; - case Phase::Histograms: - populateStatsFromScopes(scope_vec); - break; } } -template void StatsRequest::populateStatsFromScopes(const ScopeVec& scope_vec) { - Stats::IterateFn check_stat = [this](const Stats::RefcountPtr& stat) -> bool { - if (params_.used_only_ && !stat->used()) { - return true; - } - - // Capture the name if we did not early-exit due to used_only -- we'll use - // the name for both filtering and for capturing the stat in the map. - // stat->name() takes a symbol table lock and builds a string, so we only - // want to call it once. - // - // This duplicates logic in shouldShowMetric in prometheus_stats.cc, but - // differs in that Prometheus only uses stat->name() for filtering, not - // rendering, so it only grab the name if there's a filter. - std::string name = stat->name(); - if (params_.re2_filter_ != nullptr && !re2::RE2::PartialMatch(name, *params_.re2_filter_)) { - return true; - } - stat_map_[name] = stat; - return true; - }; - for (const Stats::ConstScopeSharedPtr& scope : scope_vec) { - scope->iterate(check_stat); - } -} +template class StatsRequest; -template -void StatsRequest::renderStat(const std::string& name, Buffer::Instance& response, - StatOrScopes& variant) { - auto stat = absl::get(variant); - render_->generate(response, name, stat->value()); -} +template class StatsRequest< + std::vector, std::vector, + std::vector, std::vector>; } // namespace Server } // namespace Envoy diff --git a/source/server/admin/stats_request.h b/source/server/admin/stats_request.h index 41d43632a83f..711b246c0218 100644 --- a/source/server/admin/stats_request.h +++ b/source/server/admin/stats_request.h @@ -1,5 +1,9 @@ #pragma once +#include +#include +#include + #include "envoy/server/admin.h" #include "source/server/admin/stats_params.h" @@ -12,35 +16,16 @@ namespace Envoy { namespace Server { -// Captures context for a streaming request, implementing the AdminHandler interface. +// Captures context for a streaming request, implementing the AdminHandler interface. The class +// templating allows derived classes to specify how each stat type (Text Readout, Counter, etc.) +// is represented, catering for different exposition formats (e.g. grouped, ungrouped). +template class StatsRequest : public Admin::Request { - using ScopeVec = std::vector; - using StatOrScopes = absl::variant; - - // Ordered to match the StatsOrScopes variant. - enum class StatOrScopesIndex { Scopes, TextReadout, Counter, Gauge, Histogram }; - - // In order to keep the output consistent with the fully buffered behavior - // prior to the chunked implementation that buffered each type, we iterate - // over all scopes for each type. This enables the complex chunking - // implementation to pass the tests that capture the buffered behavior. There - // is not a significant cost to this, but in a future PR we may choose to - // co-mingle the types. Note that histograms are groups together in the data - // JSON data model, so we won't be able to fully co-mingle. - enum class Phase { - TextReadouts, - CountersAndGauges, - Histograms, - }; public: - using UrlHandlerFn = std::function; - static constexpr uint64_t DefaultChunkSize = 2 * 1000 * 1000; - StatsRequest(Stats::Store& stats, const StatsParams& params, - UrlHandlerFn url_handler_fn = nullptr); + using UrlHandlerFn = std::function; // Admin::Request Http::Code start(Http::ResponseHeaderMap& response_headers) override; @@ -67,15 +52,23 @@ class StatsRequest : public Admin::Request { // introduce flow-control so that we don't buffer the all the serialized stats // while waiting for a slow client. // - // Note that we do 3 passes through all the scopes_, so that we can emit + // For text/JSON/HTML stats, we do 3 passes through all the scopes_, and we emit // text-readouts first, then the intermingled counters and gauges, and finally - // the histograms. + // the histograms. For prometheus stats, we do 3 or 4 passes (this depends on whether Text + // Readouts are explicitly requested) through all the scopes_, and we emit counters first, + // then gauges, possibly text readouts and finally histograms. bool nextChunk(Buffer::Instance& response) override; - // To duplicate prior behavior for this class, we do three passes over all the stats: + // To duplicate prior behavior for this class, we do several passes over all the stats. For + // text/JSON/HTML stats, we do 3 passes: // 1. text readouts across all scopes // 2. counters and gauges, co-mingled, across all scopes // 3. histograms across all scopes. + // For prometheus stats, we do 3 or 4 passes: + // 1. counters across all scopes + // 2. gauges across all scopes + // 3. text readouts (only if explicitly requested via query param) + // 4. histograms across all scopes. // It would be little more efficient to co-mingle all the stats, but three // passes over the scopes is OK. In the future we may decide to organize the // result data differently, but in the process of changing from buffering @@ -83,35 +76,78 @@ class StatsRequest : public Admin::Request { // to reason about if the tests don't change their expectations. void startPhase(); + // Sets the chunk size. + void setChunkSize(uint64_t chunk_size) { chunk_size_ = chunk_size; } + +protected: + // Ordered to match the StatsOrScopes variant. + enum class StatOrScopesIndex { Scopes, TextReadout, Counter, Gauge, Histogram }; + + // In order to keep the output consistent with the fully buffered behavior + // prior to the chunked implementation that buffered each type, we iterate + // over all scopes for each type. This enables the complex chunking + // implementation to pass the tests that capture the buffered behavior. There + // is not a significant cost to this, but in a future PR we may choose to + // co-mingle the types. Note that histograms are grouped together in the data + // JSON data model, so we won't be able to fully co-mingle. + enum class Phase { + TextReadouts, + CountersAndGauges, + Counters, + Gauges, + Histograms, + }; + + using ScopeVec = std::vector; + + using StatOrScopes = + absl::variant; + + StatsRequest(Stats::Store& stats, const StatsParams& params, + UrlHandlerFn url_handler_fn = nullptr); + + ~StatsRequest() override = default; + // Iterates over scope_vec and populates the metric types associated with the // current phase. void populateStatsForCurrentPhase(const ScopeVec& scope_vec); - // Populates all the metrics of the templatized type from scope_vec. Here we - // exploit that Scope::iterate is a generic templatized function to avoid code - // duplication. - template void populateStatsFromScopes(const ScopeVec& scope_vec); + virtual Stats::IterateFn saveMatchingStatForTextReadout() PURE; + virtual Stats::IterateFn saveMatchingStatForGauge() PURE; + virtual Stats::IterateFn saveMatchingStatForCounter() PURE; + virtual Stats::IterateFn saveMatchingStatForHistogram() PURE; - // Renders the templatized type, exploiting the fact that Render::generate is - // generic to avoid code duplication. - template - void renderStat(const std::string& name, Buffer::Instance& response, StatOrScopes& variant); + virtual void processTextReadout(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) PURE; + virtual void processGauge(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) PURE; + virtual void processCounter(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) PURE; + virtual void processHistogram(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) PURE; - // Sets the chunk size. - void setChunkSize(uint64_t chunk_size) { chunk_size_ = chunk_size; } + virtual void setRenderPtr(Http::ResponseHeaderMap& response_headers) PURE; + virtual StatsRenderBase& render() PURE; -private: StatsParams params_; - std::unique_ptr render_; - Stats::Store& stats_; - ScopeVec scopes_; absl::btree_map stat_map_; - Phase phase_{Phase::TextReadouts}; - uint64_t phase_stat_count_{0}; - absl::string_view phase_string_{"text readouts"}; Buffer::OwnedImpl response_; UrlHandlerFn url_handler_fn_; + + // Phase-related state. + uint64_t phase_stat_count_{0}; + uint32_t phase_index_{0}; + std::vector phases_; + +private: + Stats::Store& stats_; + ScopeVec scopes_; uint64_t chunk_size_{DefaultChunkSize}; + std::map phase_labels_{{Phase::TextReadouts, "Text Readouts"}, + {Phase::CountersAndGauges, "Counters and Gauges"}, + {Phase::Histograms, "Histograms"}, + {Phase::Counters, "Counters"}, + {Phase::Gauges, "Gauges"}}; }; } // namespace Server diff --git a/source/server/admin/ungrouped_stats_request.cc b/source/server/admin/ungrouped_stats_request.cc new file mode 100644 index 000000000000..0b3d02c37cce --- /dev/null +++ b/source/server/admin/ungrouped_stats_request.cc @@ -0,0 +1,129 @@ +#include "source/server/admin/ungrouped_stats_request.h" + +#ifdef ENVOY_ADMIN_HTML +#include "source/server/admin/stats_html_render.h" +#endif + +namespace Envoy { +namespace Server { + +UngroupedStatsRequest::UngroupedStatsRequest(Stats::Store& stats, const StatsParams& params, + UrlHandlerFn url_handler_fn) + : StatsRequest(stats, params, url_handler_fn) { + phases_ = {Phase::TextReadouts, Phase::CountersAndGauges, Phase::Histograms}; + switch (params_.type_) { + case StatsType::TextReadouts: + case StatsType::All: + phase_index_ = 0; + break; + case StatsType::Counters: + case StatsType::Gauges: + phase_index_ = 1; + break; + case StatsType::Histograms: + phase_index_ = 2; + break; + } +} + +template Stats::IterateFn UngroupedStatsRequest::saveMatchingStat() { + return [this](const Stats::RefcountPtr& stat) -> bool { + // Check if used. + if (params_.used_only_ && !stat->used()) { + return true; + } + + // Capture the name if we did not early-exit due to used_only -- we'll use + // the name for both filtering and for capturing the stat in the map. + // stat->name() takes a symbol table lock and builds a string, so we only + // want to call it once. + // + // This duplicates logic in saveMatchingStat in grouped_stats_request.cc, but + // differs in that Prometheus only uses stat->name() for filtering, not + // rendering, so it only grabs the name if there's a filter. + std::string name = stat->name(); + + // Check if filtered. + if (params_.re2_filter_ != nullptr && !re2::RE2::PartialMatch(name, *params_.re2_filter_)) { + return true; + } + + stat_map_[name] = stat; + return true; + }; +} + +Stats::IterateFn UngroupedStatsRequest::saveMatchingStatForTextReadout() { + return saveMatchingStat(); +} + +Stats::IterateFn UngroupedStatsRequest::saveMatchingStatForGauge() { + return saveMatchingStat(); +} + +Stats::IterateFn UngroupedStatsRequest::saveMatchingStatForCounter() { + return saveMatchingStat(); +} + +Stats::IterateFn UngroupedStatsRequest::saveMatchingStatForHistogram() { + return saveMatchingStat(); +} + +template +void UngroupedStatsRequest::renderStat(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + auto stat = absl::get(variant); + render_.get()->generate(response, name, stat->value()); + phase_stat_count_++; +} + +void UngroupedStatsRequest::processTextReadout(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + renderStat(name, response, variant); +} + +void UngroupedStatsRequest::processCounter(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + renderStat(name, response, variant); +} + +void UngroupedStatsRequest::processGauge(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + renderStat(name, response, variant); +} + +void UngroupedStatsRequest::processHistogram(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) { + auto histogram = absl::get(variant); + auto parent_histogram = dynamic_cast(histogram.get()); + if (parent_histogram != nullptr) { + dynamic_cast(render_.get())->generate(response, name, *parent_histogram); + ++phase_stat_count_; + } +} + +void UngroupedStatsRequest::setRenderPtr(Http::ResponseHeaderMap& response_headers) { + switch (params_.format_) { + case StatsFormat::Json: + render_ = std::make_unique(response_headers, response_, params_); + break; + case StatsFormat::Text: + render_ = std::make_unique(params_); + break; +#ifdef ENVOY_ADMIN_HTML + case StatsFormat::ActiveHtml: + case StatsFormat::Html: { + auto html_render = std::make_unique(response_headers, response_, params_); + html_render->setupStatsPage(url_handler_fn_(), params_, response_); + render_ = std::move(html_render); + break; + } +#endif + case StatsFormat::Prometheus: + IS_ENVOY_BUG("reached Prometheus case in switch unexpectedly"); + break; + } +} + +} // namespace Server +} // namespace Envoy diff --git a/source/server/admin/ungrouped_stats_request.h b/source/server/admin/ungrouped_stats_request.h new file mode 100644 index 000000000000..47f402aee9a7 --- /dev/null +++ b/source/server/admin/ungrouped_stats_request.h @@ -0,0 +1,49 @@ +#pragma once + +#include "source/server/admin/stats_request.h" + +namespace Envoy { +namespace Server { + +// In the context of this class, a stat is represented by an individual +// gauge, counter etc.; this is in contrast to the GroupedStatsRequest class, +// where a stat is a group of gauges, counters etc. with a common tag-extracted name. +// This class is currently used for all non-Prometheus formats (e.g. text, JSON). +class UngroupedStatsRequest + : public StatsRequest { + +public: + UngroupedStatsRequest(Stats::Store& stats, const StatsParams& params, + UrlHandlerFn url_handler_fn = nullptr); + +protected: + Stats::IterateFn saveMatchingStatForTextReadout() override; + Stats::IterateFn saveMatchingStatForGauge() override; + Stats::IterateFn saveMatchingStatForCounter() override; + Stats::IterateFn saveMatchingStatForHistogram() override; + template Stats::IterateFn saveMatchingStat(); + + void processTextReadout(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) override; + void processGauge(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) override; + void processCounter(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) override; + void processHistogram(const std::string& name, Buffer::Instance& response, + const StatOrScopes& variant) override; + + template + void renderStat(const std::string& name, Buffer::Instance& response, const StatOrScopes& variant); + + void setRenderPtr(Http::ResponseHeaderMap& response_headers) override; + StatsRenderBase& render() override { + ASSERT(render_ != nullptr); + return *render_; + } + +private: + std::unique_ptr render_; +}; +} // namespace Server +} // namespace Envoy diff --git a/test/server/admin/BUILD b/test/server/admin/BUILD index 8c1a28f7d2ee..36e37168dec7 100644 --- a/test/server/admin/BUILD +++ b/test/server/admin/BUILD @@ -127,8 +127,25 @@ envoy_cc_test_library( ) envoy_cc_test( - name = "stats_request_test", - srcs = envoy_select_admin_functionality(["stats_request_test.cc"]), + name = "ungrouped_stats_request_test", + srcs = envoy_select_admin_functionality(["ungrouped_stats_request_test.cc"]), + deps = [ + ":stats_request_test_base", + ], +) + +envoy_cc_test( + name = "grouped_stats_request_test", + srcs = envoy_select_admin_functionality(["grouped_stats_request_test.cc"]), + deps = [ + ":stats_request_test_base", + ], +) + +envoy_cc_test_library( + name = "stats_request_test_base", + srcs = envoy_select_admin_functionality(["stats_request_test_base.cc"]), + hdrs = ["stats_request_test_base.h"], deps = [ "//source/common/stats:thread_local_store_lib", "//source/server/admin:stats_request_lib", @@ -139,6 +156,14 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "prometheus_stats_test", + srcs = envoy_select_admin_functionality(["prometheus_stats_test.cc"]), + deps = [ + ":stats_request_test_base", + ], +) + envoy_cc_benchmark_binary( name = "stats_handler_speed_test", srcs = envoy_select_admin_functionality(["stats_handler_speed_test.cc"]), @@ -164,15 +189,6 @@ envoy_cc_test( deps = [":admin_instance_lib"], ) -envoy_cc_test( - name = "prometheus_stats_test", - srcs = envoy_select_admin_functionality(["prometheus_stats_test.cc"]), - deps = [ - "//source/server/admin:prometheus_stats_lib", - "//test/test_common:utility_lib", - ], -) - envoy_cc_test( name = "logs_handler_test", srcs = envoy_select_admin_functionality(["logs_handler_test.cc"]), diff --git a/test/server/admin/admin_test.cc b/test/server/admin/admin_test.cc index ec9d0f39d1d9..b030005493cd 100644 --- a/test/server/admin/admin_test.cc +++ b/test/server/admin/admin_test.cc @@ -15,7 +15,7 @@ #include "source/common/protobuf/utility.h" #include "source/common/upstream/upstream_impl.h" #include "source/extensions/access_loggers/common/file_access_log_impl.h" -#include "source/server/admin/stats_request.h" +#include "source/server/admin/ungrouped_stats_request.h" #include "test/server/admin/admin_instance.h" #include "test/test_common/logging.h" @@ -254,7 +254,8 @@ TEST_P(AdminInstanceTest, StatsWithMultipleChunks) { uint32_t expected_size = 0; // Declare enough counters so that we are sure to exceed the chunk size. - const uint32_t n = (StatsRequest::DefaultChunkSize + prefix.size() / 2) / prefix.size() + 1; + const uint32_t n = + (UngroupedStatsRequest::DefaultChunkSize + prefix.size() / 2) / prefix.size() + 1; for (uint32_t i = 0; i <= n; ++i) { const std::string name = absl::StrCat(prefix, i); store.counterFromString(name); @@ -262,7 +263,7 @@ TEST_P(AdminInstanceTest, StatsWithMultipleChunks) { } EXPECT_EQ(Http::Code::OK, getCallback("/stats", header_map, response)); EXPECT_LT(expected_size, response.length()); - EXPECT_LT(StatsRequest::DefaultChunkSize, response.length()); + EXPECT_LT(UngroupedStatsRequest::DefaultChunkSize, response.length()); EXPECT_THAT(response.toString(), StartsWith(absl::StrCat(prefix, "0: 0\n", prefix))); } diff --git a/test/server/admin/grouped_stats_request_test.cc b/test/server/admin/grouped_stats_request_test.cc new file mode 100644 index 000000000000..c731c8aa18c8 --- /dev/null +++ b/test/server/admin/grouped_stats_request_test.cc @@ -0,0 +1,90 @@ +#include "source/common/stats/custom_stat_namespaces_impl.h" + +#include "test/mocks/stats/mocks.h" +#include "test/server/admin/stats_request_test_base.h" + +namespace Envoy { +namespace Server { + +class GroupedStatsRequestTest : public StatsRequestTestBase { +protected: + std::unique_ptr makeRequest(bool used_only, bool text_readouts = false) { + StatsParams params; + params.used_only_ = used_only; + params.prometheus_text_readouts_ = text_readouts; + params.format_ = StatsFormat::Prometheus; + return std::make_unique(store_, params, custom_namespaces_); + } + + Stats::CustomStatNamespacesImpl custom_namespaces_; +}; + +TEST_F(GroupedStatsRequestTest, Empty) { EXPECT_EQ(0, iterateChunks(*makeRequest(false))); } + +TEST_F(GroupedStatsRequestTest, OneCounter) { + Stats::StatNameTagVector c1Tags{{makeStatName("cluster"), makeStatName("c1")}}; + Stats::Counter& c1 = store_.rootScope()->counterFromStatNameWithTags( + makeStatName("cluster.upstream.cx.total"), c1Tags); + c1.add(10); + + EXPECT_EQ(1, iterateChunks(*makeRequest(false))); +} + +TEST_F(GroupedStatsRequestTest, OneGauge) { + Stats::StatNameTagVector c1Tags{{makeStatName("cluster"), makeStatName("c1")}}; + store_.rootScope()->gaugeFromStatNameWithTags(makeStatName("foo"), c1Tags, + Stats::Gauge::ImportMode::Accumulate); + EXPECT_EQ(1, iterateChunks(*makeRequest(false))); +} + +TEST_F(GroupedStatsRequestTest, OneHistogram) { + Stats::StatNameTagVector c1Tags{{makeStatName("cluster"), makeStatName("c1")}}; + store_.rootScope()->histogramFromStatNameWithTags(makeStatName("foo"), c1Tags, + Stats::Histogram::Unit::Milliseconds); + EXPECT_EQ(1, iterateChunks(*makeRequest(false))); +} + +TEST_F(GroupedStatsRequestTest, OneTextReadout) { + Stats::StatNameTagVector c1Tags{{makeStatName("cluster"), makeStatName("c1")}}; + store_.rootScope()->textReadoutFromStatNameWithTags(makeStatName("foo"), c1Tags); + // text readouts are not included in the returned prometheus stats, unless specifically asked for + // via query param + EXPECT_EQ(0, iterateChunks(*makeRequest(false))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, true))); +} + +TEST_F(GroupedStatsRequestTest, OneScope) { + Stats::ScopeSharedPtr scope = store_.createScope("foo"); + EXPECT_EQ(0, iterateChunks(*makeRequest(false))); +} + +// example output: +// # TYPE envoy_foo6 counter +// envoy_foo6{cluster="c1"} 0 +TEST_F(GroupedStatsRequestTest, ManyStatsSmallChunkSize) { + for (uint32_t i = 0; i < 10; ++i) { + Stats::StatNameTagVector tags{{makeStatName("cluster"), makeStatName("c1")}}; + store_.rootScope()->counterFromStatNameWithTags(makeStatName(absl::StrCat("foo", i)), tags); + } + std::unique_ptr request = makeRequest(false); + request->setChunkSize(50); + EXPECT_EQ(10, iterateChunks(*request)); +} + +TEST_F(GroupedStatsRequestTest, ManyStatsSmallChunkSizeNoDrain) { + for (uint32_t i = 0; i < 10; ++i) { + Stats::StatNameTagVector tags{{makeStatName("cluster"), makeStatName("c1")}}; + store_.rootScope()->counterFromStatNameWithTags(makeStatName(absl::StrCat("foo", i)), tags); + } + std::unique_ptr request = makeRequest(false); + request->setChunkSize(50); + EXPECT_EQ(10, iterateChunks(*request, false)); +} + +TEST_F(GroupedStatsRequestTest, OneStatUsedOnly) { + Stats::ScopeSharedPtr scope = store_.createScope("foo"); + EXPECT_EQ(0, iterateChunks(*makeRequest(true))); +} + +} // namespace Server +} // namespace Envoy diff --git a/test/server/admin/prometheus_stats_test.cc b/test/server/admin/prometheus_stats_test.cc index c27af6121130..5b57be204ec0 100644 --- a/test/server/admin/prometheus_stats_test.cc +++ b/test/server/admin/prometheus_stats_test.cc @@ -1,14 +1,10 @@ -#include -#include -#include +#include "envoy/stats/histogram.h" #include "source/common/stats/custom_stat_namespaces_impl.h" -#include "source/server/admin/prometheus_stats.h" #include "test/mocks/stats/mocks.h" -#include "test/test_common/utility.h" +#include "test/server/admin/stats_request_test_base.h" -using testing::NiceMock; using testing::ReturnRef; namespace Envoy { @@ -38,6 +34,9 @@ class HistogramWrapper { histogram_t* histogram_; }; +// TODO(rulex123): these tests should be moved to grouped_stats_request_test.cc, +// but are left here to ease code review while we move Prometheus stats to the +// streaming API. class PrometheusStatsFormatterTest : public testing::Test { protected: PrometheusStatsFormatterTest() : alloc_(*symbol_table_), pool_(*symbol_table_) {} @@ -47,16 +46,16 @@ class PrometheusStatsFormatterTest : public testing::Test { void addCounter(const std::string& name, Stats::StatNameTagVector cluster_tags) { Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_); Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_); - counters_.push_back(alloc_.makeCounter(name_storage.statName(), - tag_extracted_name_storage.statName(), cluster_tags)); + test_scope_ptr_->counters_.push_back(alloc_.makeCounter( + name_storage.statName(), tag_extracted_name_storage.statName(), cluster_tags)); } void addGauge(const std::string& name, Stats::StatNameTagVector cluster_tags) { Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_); Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_); - gauges_.push_back(alloc_.makeGauge(name_storage.statName(), - tag_extracted_name_storage.statName(), cluster_tags, - Stats::Gauge::ImportMode::Accumulate)); + test_scope_ptr_->gauges_.push_back( + alloc_.makeGauge(name_storage.statName(), tag_extracted_name_storage.statName(), + cluster_tags, Stats::Gauge::ImportMode::Accumulate)); } void addTextReadout(const std::string& name, const std::string& value, @@ -66,15 +65,17 @@ class PrometheusStatsFormatterTest : public testing::Test { Stats::TextReadoutSharedPtr textReadout = alloc_.makeTextReadout( name_storage.statName(), tag_extracted_name_storage.statName(), cluster_tags); textReadout->set(value); - textReadouts_.push_back(textReadout); + test_scope_ptr_->text_readouts_.push_back(textReadout); } - using MockHistogramSharedPtr = Stats::RefcountPtr>; - void addHistogram(MockHistogramSharedPtr histogram) { histograms_.push_back(histogram); } + using MockHistogramSharedPtr = Stats::RefcountPtr>; + void addHistogram(MockHistogramSharedPtr histogram) { + test_scope_ptr_->histograms_.push_back(histogram); + } MockHistogramSharedPtr makeHistogram(const std::string& name, Stats::StatNameTagVector cluster_tags) { - auto histogram = MockHistogramSharedPtr(new NiceMock()); + auto histogram = MockHistogramSharedPtr(new testing::NiceMock()); histogram->name_ = baseName(name, cluster_tags); histogram->setTagExtractedName(name); histogram->setTags(cluster_tags); @@ -98,67 +99,93 @@ class PrometheusStatsFormatterTest : public testing::Test { void clearStorage() { pool_.clear(); - counters_.clear(); - gauges_.clear(); - histograms_.clear(); - textReadouts_.clear(); + test_scope_ptr_->counters_.clear(); + test_scope_ptr_->histograms_.clear(); + test_scope_ptr_->gauges_.clear(); + test_scope_ptr_->text_readouts_.clear(); EXPECT_EQ(0, symbol_table_->numSymbols()); } + std::unique_ptr makeRequest(bool used_only, bool text_readouts = false) { + StatsParams params; + params.used_only_ = used_only; + params.prometheus_text_readouts_ = text_readouts; + params.format_ = StatsFormat::Prometheus; + return std::make_unique(mock_store_, params, custom_namespaces_); + } + + std::unique_ptr makeRequest(StatsParams& params) { + return std::make_unique(mock_store_, params, custom_namespaces_); + } + + std::string response(GroupedStatsRequest& request) { + Http::TestResponseHeaderMapImpl response_headers; + Http::Code code = request.start(response_headers); + EXPECT_EQ(Http::Code::OK, code); + Buffer::OwnedImpl data; + while (request.nextChunk(data)) { + } + return data.toString(); + } + Stats::TestUtil::TestSymbolTable symbol_table_; Stats::AllocatorImpl alloc_; Stats::StatNamePool pool_; - std::vector counters_; - std::vector gauges_; - std::vector histograms_; - std::vector textReadouts_; + Stats::CustomStatNamespacesImpl custom_namespaces_; + testing::NiceMock mock_store_; + std::shared_ptr test_scope_ptr_ = std::make_shared("", mock_store_); }; -TEST_F(PrometheusStatsFormatterTest, MetricName) { +// TODO(rulex123): these tests should be moved to stats_render_test.cc, +// but are left here to ease code review while we move Prometheus stats to the +// streaming API. +class PrometheusStatsRenderTest : public testing::Test {}; + +TEST_F(PrometheusStatsRenderTest, MetricName) { Stats::CustomStatNamespacesImpl custom_namespaces; std::string raw = "vulture.eats-liver"; std::string expected = "envoy_vulture_eats_liver"; - auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); EXPECT_TRUE(actual.has_value()); EXPECT_EQ(expected, actual.value()); } -TEST_F(PrometheusStatsFormatterTest, SanitizeMetricName) { +TEST_F(PrometheusStatsRenderTest, SanitizeMetricName) { Stats::CustomStatNamespacesImpl custom_namespaces; std::string raw = "An.artist.plays-violin@019street"; std::string expected = "envoy_An_artist_plays_violin_019street"; - auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); EXPECT_EQ(expected, actual.value()); } -TEST_F(PrometheusStatsFormatterTest, SanitizeMetricNameDigitFirst) { +TEST_F(PrometheusStatsRenderTest, SanitizeMetricNameDigitFirst) { Stats::CustomStatNamespacesImpl custom_namespaces; std::string raw = "3.artists.play-violin@019street"; std::string expected = "envoy_3_artists_play_violin_019street"; - auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); EXPECT_TRUE(actual.has_value()); EXPECT_EQ(expected, actual.value()); } -TEST_F(PrometheusStatsFormatterTest, CustomNamespace) { +TEST_F(PrometheusStatsRenderTest, CustomNamespace) { Stats::CustomStatNamespacesImpl custom_namespaces; custom_namespaces.registerStatNamespace("promstattest"); std::string raw = "promstattest.vulture.eats-liver"; std::string expected = "vulture_eats_liver"; - auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); EXPECT_TRUE(actual.has_value()); EXPECT_EQ(expected, actual.value()); } -TEST_F(PrometheusStatsFormatterTest, CustomNamespaceWithInvalidPromnamespace) { +TEST_F(PrometheusStatsRenderTest, CustomNamespaceWithInvalidPromnamespace) { Stats::CustomStatNamespacesImpl custom_namespaces; custom_namespaces.registerStatNamespace("promstattest"); std::string raw = "promstattest.1234abcd.eats-liver"; - auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); EXPECT_FALSE(actual.has_value()); } -TEST_F(PrometheusStatsFormatterTest, FormattedTags) { +TEST_F(PrometheusStatsRenderTest, FormattedTags) { std::vector tags; Stats::Tag tag1 = {"a.tag-name", "a.tag-value"}; Stats::Tag tag2 = {"another_tag_name", "another_tag-value"}; @@ -169,16 +196,13 @@ TEST_F(PrometheusStatsFormatterTest, FormattedTags) { tags.push_back(tag3); std::string expected = "a_tag_name=\"a.tag-value\",another_tag_name=\"another_tag-value\"," "replace_problematic=\"val\\\"ue with\\\\ some\\n issues\""; - auto actual = PrometheusStatsFormatter::formattedTags(tags); + auto actual = PrometheusStatsRender::formattedTags(tags); EXPECT_EQ(expected, actual); } -TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) { - Stats::CustomStatNamespacesImpl custom_namespaces; - +TEST_F(PrometheusStatsFormatterTest, MetricNameCollision) { // Create two counters and two gauges with each pair having the same name, - // but having different tag names and values. - //`statsAsPrometheus()` should return two implying it found two unique stat names + // but having different tag names and values. 2 groups should be rendered. addCounter("cluster.test_cluster_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); @@ -189,18 +213,23 @@ TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) { addGauge("cluster.test_cluster_2.upstream_cx_total", {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(2UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + + const std::string expected_output = + R"EOF(# TYPE envoy_cluster_test_cluster_1_upstream_cx_total counter +envoy_cluster_test_cluster_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 +envoy_cluster_test_cluster_1_upstream_cx_total{another_tag_name="another_tag-value"} 0 +# TYPE envoy_cluster_test_cluster_2_upstream_cx_total gauge +envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 +envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0 +)EOF"; + + EXPECT_EQ(expected_output, response(*makeRequest(false))); } TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) { - Stats::CustomStatNamespacesImpl custom_namespaces; - // Create two counters and two gauges, all with unique names. - // statsAsPrometheus() should return four implying it found - // four unique stat names. + // 4 groups should be rendered. addCounter("cluster.test_cluster_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); @@ -211,14 +240,23 @@ TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) { addGauge("cluster.test_cluster_4.upstream_cx_total", {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(4UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + + const std::string expected_output = + R"EOF(# TYPE envoy_cluster_test_cluster_1_upstream_cx_total counter +envoy_cluster_test_cluster_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 +# TYPE envoy_cluster_test_cluster_2_upstream_cx_total counter +envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name="another_tag-value"} 0 +# TYPE envoy_cluster_test_cluster_3_upstream_cx_total gauge +envoy_cluster_test_cluster_3_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 +# TYPE envoy_cluster_test_cluster_4_upstream_cx_total gauge +envoy_cluster_test_cluster_4_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0 +)EOF"; + + EXPECT_EQ(expected_output, response(*makeRequest(false))); } TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { - Stats::CustomStatNamespacesImpl custom_namespaces; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(std::vector(0)); Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); @@ -227,11 +265,7 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { ON_CALL(*histogram, cumulativeStatistics()).WillByDefault(ReturnRef(h1_cumulative_statistics)); addHistogram(histogram); - - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(1UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram envoy_histogram1_bucket{le="0.5"} 0 @@ -258,11 +292,10 @@ envoy_histogram1_sum{} 0 envoy_histogram1_count{} 0 )EOF"; - EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(expected_output, response(*makeRequest(false))); } TEST_F(PrometheusStatsFormatterTest, HistogramWithNonDefaultBuckets) { - Stats::CustomStatNamespacesImpl custom_namespaces; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(std::vector(0)); Stats::ConstSupportedBuckets buckets{10, 20}; @@ -274,10 +307,7 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNonDefaultBuckets) { addHistogram(histogram); - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(1UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram envoy_histogram1_bucket{le="10"} 0 @@ -287,13 +317,12 @@ envoy_histogram1_sum{} 0 envoy_histogram1_count{} 0 )EOF"; - EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(expected_output, response(*makeRequest(false))); } // Test that scaled percents are emitted in the expected 0.0-1.0 range, and that the buckets // apply to the final output range, not the internal scaled range. TEST_F(PrometheusStatsFormatterTest, HistogramWithScaledPercent) { - Stats::CustomStatNamespacesImpl custom_namespaces; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(std::vector(0)); Stats::ConstSupportedBuckets buckets{0.5, 1.0}; @@ -313,10 +342,7 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithScaledPercent) { addHistogram(histogram); - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(1UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram envoy_histogram1_bucket{le="0.5"} 1 @@ -326,11 +352,10 @@ envoy_histogram1_sum{} 2.2599999999999997868371792719699 envoy_histogram1_count{} 3 )EOF"; - EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(expected_output, response(*makeRequest(false))); } TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) { - Stats::CustomStatNamespacesImpl custom_namespaces; HistogramWrapper h1_cumulative; // Force large counts to prove that the +Inf bucket doesn't overflow to scientific notation. @@ -347,10 +372,7 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) { addHistogram(histogram); - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(1UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram envoy_histogram1_bucket{le="0.5"} 0 @@ -377,12 +399,11 @@ envoy_histogram1_sum{} 105105105000 envoy_histogram1_count{} 101100000 )EOF"; - EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(expected_output, response(*makeRequest(false))); } TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { - Stats::CustomStatNamespacesImpl custom_namespaces; - custom_namespaces.registerStatNamespace("promtest"); + custom_namespaces_.registerStatNamespace("promtest"); addCounter("cluster.test_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); @@ -409,10 +430,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { addHistogram(histogram1); EXPECT_CALL(*histogram1, cumulativeStatistics()).WillOnce(ReturnRef(h1_cumulative_statistics)); - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(7UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 @@ -451,12 +469,10 @@ envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 )EOF"; - EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(expected_output, response(*makeRequest(false))); } TEST_F(PrometheusStatsFormatterTest, OutputWithTextReadoutsInGaugeFormat) { - Stats::CustomStatNamespacesImpl custom_namespaces; - addCounter("cluster.upstream_cx_total_count", {{makeStat("cluster"), makeStat("c1")}}); addGauge("cluster.upstream_cx_total", {{makeStat("cluster"), makeStat("c1")}}); // Text readouts that should be returned in gauge format. @@ -466,10 +482,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithTextReadoutsInGaugeFormat) { {makeStat("tag2"), makeStat("\n")}, {makeStat("tag3"), makeStat(R"(")")}}); - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(4UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); const std::string expected_output = R"EOF(# TYPE envoy_cluster_upstream_cx_total_count counter envoy_cluster_upstream_cx_total_count{cluster="c1"} 0 @@ -481,7 +494,7 @@ envoy_control_plane_identifier{cluster="c1",text_value="CP-1"} 0 envoy_invalid_tag_values{tag1="\\",tag2="\n",tag3="\"",text_value="test"} 0 )EOF"; - EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(expected_output, response(*makeRequest(false, true))); } // Test that output groups all metrics of the same name (with different tags) together, @@ -489,7 +502,6 @@ envoy_invalid_tag_values{tag1="\\",tag2="\n",tag3="\"",text_value="test"} 0 // should be sorted by their tags; the format specifies that it is preferred that metrics // are always grouped in the same order, and sorting is an easy way to ensure this. TEST_F(PrometheusStatsFormatterTest, OutputSortedByMetricName) { - Stats::CustomStatNamespacesImpl custom_namespaces; const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(h1_values); @@ -514,10 +526,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputSortedByMetricName) { } } - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(6UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); const std::string expected_output = R"EOF(# TYPE envoy_cluster_upstream_cx_connect_fail counter envoy_cluster_upstream_cx_connect_fail{cluster="aaa"} 0 @@ -671,11 +680,10 @@ envoy_cluster_upstream_rq_time_sum{cluster="ccc"} 5532 envoy_cluster_upstream_rq_time_count{cluster="ccc"} 7 )EOF"; - EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(expected_output, response(*makeRequest(false))); } TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { - Stats::CustomStatNamespacesImpl custom_namespaces; addCounter("cluster.test_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); addCounter("cluster.test_2.upstream_cx_total", @@ -697,12 +705,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { addHistogram(histogram1); EXPECT_CALL(*histogram1, cumulativeStatistics()).WillOnce(ReturnRef(h1_cumulative_statistics)); - Buffer::OwnedImpl response; - StatsParams params; - params.used_only_ = true; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); - EXPECT_EQ(1UL, size); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0 @@ -729,11 +732,10 @@ envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 )EOF"; - EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(expected_output, response(*makeRequest(true))); } TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { - Stats::CustomStatNamespacesImpl custom_namespaces; const std::vector h1_values = {}; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(h1_values); @@ -745,31 +747,49 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; histogram1->used_ = false; addHistogram(histogram1); - StatsParams params; + + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); { - params.used_only_ = true; EXPECT_CALL(*histogram1, cumulativeStatistics()).Times(0); - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); - EXPECT_EQ(0UL, size); + std::string result = response(*makeRequest(true)); + EXPECT_EQ(EMPTY_STRING, result); } { - params.used_only_ = false; EXPECT_CALL(*histogram1, cumulativeStatistics()).WillOnce(ReturnRef(h1_cumulative_statistics)); - Buffer::OwnedImpl response; - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); - EXPECT_EQ(1UL, size); + const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 0 +envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 0 +envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 0 +)EOF"; + + EXPECT_EQ(expected_output, response(*makeRequest(false))); } } TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) { - Stats::CustomStatNamespacesImpl custom_namespaces; addCounter("cluster.test_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); addCounter("cluster.test_2.upstream_cx_total", @@ -790,30 +810,27 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) { histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; addHistogram(histogram1); + ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 )EOF"; { - Buffer::OwnedImpl response; + Buffer::OwnedImpl res; StatsParams params; - ASSERT_EQ(Http::Code::OK, - params.parse("/stats?filter=cluster.test_1.upstream_cx_total", response)); - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); - EXPECT_EQ(1UL, size); - EXPECT_EQ(expected_output, response.toString()); + ASSERT_EQ(Http::Code::OK, params.parse("/stats?filter=cluster.test_1.upstream_cx_total", res)); + params.format_ = StatsFormat::Prometheus; + EXPECT_EQ(expected_output, response(*makeRequest(params))); } { - Buffer::OwnedImpl response; + Buffer::OwnedImpl res; StatsParams params; ASSERT_EQ(Http::Code::OK, - params.parse("/stats?filter=cluster.test_1.upstream_cx_total&safe", response)); - const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); - EXPECT_EQ(1UL, size); - EXPECT_EQ(expected_output, response.toString()); + params.parse("/stats?filter=cluster.test_1.upstream_cx_total&safe", res)); + params.format_ = StatsFormat::Prometheus; + EXPECT_EQ(expected_output, response(*makeRequest(params))); } } diff --git a/test/server/admin/stats_handler_speed_test.cc b/test/server/admin/stats_handler_speed_test.cc index 4c045936d094..8dc9ed7b0230 100644 --- a/test/server/admin/stats_handler_speed_test.cc +++ b/test/server/admin/stats_handler_speed_test.cc @@ -36,11 +36,12 @@ class StatsHandlerTest { */ uint64_t handlerStats(const StatsParams& params) { Buffer::OwnedImpl data; + Admin::RequestPtr request; if (params.format_ == Envoy::Server::StatsFormat::Prometheus) { - Envoy::Server::StatsHandler::prometheusRender(store_, custom_namespaces_, params, data); - return data.length(); + request = StatsHandler::makePrometheusRequest(store_, params, custom_namespaces_); + } else { + request = StatsHandler::makeRequest(store_, params); } - Admin::RequestPtr request = StatsHandler::makeRequest(store_, params); auto response_headers = Http::ResponseHeaderMapImpl::create(); request->start(*response_headers); uint64_t count = 0; diff --git a/test/server/admin/stats_handler_test.cc b/test/server/admin/stats_handler_test.cc index 529f0f0993e3..aa3e3c245903 100644 --- a/test/server/admin/stats_handler_test.cc +++ b/test/server/admin/stats_handler_test.cc @@ -5,7 +5,7 @@ #include "source/common/stats/custom_stat_namespaces_impl.h" #include "source/common/stats/thread_local_store.h" #include "source/server/admin/stats_handler.h" -#include "source/server/admin/stats_request.h" +#include "source/server/admin/ungrouped_stats_request.h" #include "test/mocks/server/admin_stream.h" #include "test/mocks/server/instance.h" @@ -87,7 +87,8 @@ class StatsHandlerTest { EXPECT_CALL(api_, customStatNamespaces()).WillRepeatedly(ReturnRef(custom_namespaces_)); StatsHandler handler(instance); request_headers_.setPath(url); - Admin::RequestPtr request = handler.makeRequest(admin_stream_); + StatsParams params; + Admin::RequestPtr request = handler.makeRequest(admin_stream_, params); Http::TestResponseHeaderMapImpl response_headers; Http::Code code = request->start(response_headers); Buffer::OwnedImpl data; @@ -1171,7 +1172,7 @@ class ThreadedTest : public testing::Test { } void statsEndpoint() { - StatsRequest request(*store_, StatsParams()); + UngroupedStatsRequest request(*store_, StatsParams()); Http::TestResponseHeaderMapImpl response_headers; request.start(response_headers); Buffer::OwnedImpl data; diff --git a/test/server/admin/stats_render_test.cc b/test/server/admin/stats_render_test.cc index b2f9ce223153..ff1bd37caff3 100644 --- a/test/server/admin/stats_render_test.cc +++ b/test/server/admin/stats_render_test.cc @@ -1,5 +1,3 @@ -#include - #include "test/server/admin/stats_render_test_base.h" namespace Envoy { diff --git a/test/server/admin/stats_request_test.cc b/test/server/admin/stats_request_test.cc deleted file mode 100644 index 23bd46a4e2c1..000000000000 --- a/test/server/admin/stats_request_test.cc +++ /dev/null @@ -1,167 +0,0 @@ -#include - -#include "source/common/buffer/buffer_impl.h" -#include "source/common/stats/thread_local_store.h" -#include "source/server/admin/stats_request.h" - -#include "test/mocks/event/mocks.h" -#include "test/mocks/stats/mocks.h" -#include "test/mocks/thread_local/mocks.h" -#include "test/test_common/utility.h" - -#include "gtest/gtest.h" - -using testing::NiceMock; -using testing::StartsWith; - -namespace Envoy { -namespace Server { - -class StatsRequestTest : public testing::Test { -protected: - StatsRequestTest() : pool_(symbol_table_), alloc_(symbol_table_), store_(alloc_) { - store_.addSink(sink_); - store_.initializeThreading(main_thread_dispatcher_, tls_); - } - - ~StatsRequestTest() override { - tls_.shutdownGlobalThreading(); - store_.shutdownThreading(); - tls_.shutdownThread(); - } - - std::unique_ptr makeRequest(bool used_only, StatsFormat format, StatsType type) { - StatsParams params; - params.used_only_ = used_only; - params.type_ = type; - params.format_ = format; - return std::make_unique(store_, params); - } - - // Executes a request, counting the chunks that were generated. - uint32_t iterateChunks(StatsRequest& request, bool drain = true, - Http::Code expect_code = Http::Code::OK) { - Http::TestResponseHeaderMapImpl response_headers; - Http::Code code = request.start(response_headers); - EXPECT_EQ(expect_code, code); - if (code != Http::Code::OK) { - return 0; - } - Buffer::OwnedImpl data; - uint32_t num_chunks = 0; - bool more = true; - do { - more = request.nextChunk(data); - uint64_t size = data.length(); - if (size > 0) { - ++num_chunks; - if (drain) { - data.drain(size); - } - } - } while (more); - return num_chunks; - } - - // Executes a request, returning the rendered buffer as a string. - std::string response(StatsRequest& request) { - Http::TestResponseHeaderMapImpl response_headers; - Http::Code code = request.start(response_headers); - EXPECT_EQ(Http::Code::OK, code); - Buffer::OwnedImpl data; - while (request.nextChunk(data)) { - } - return data.toString(); - } - - Stats::StatName makeStatName(absl::string_view name) { return pool_.add(name); } - - Stats::SymbolTableImpl symbol_table_; - Stats::StatNamePool pool_; - Stats::AllocatorImpl alloc_; - NiceMock sink_; - NiceMock main_thread_dispatcher_; - NiceMock tls_; - Stats::ThreadLocalStoreImpl store_; - Buffer::OwnedImpl response_; -}; - -TEST_F(StatsRequestTest, Empty) { - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); -} - -TEST_F(StatsRequestTest, OneCounter) { - store_.rootScope()->counterFromStatName(makeStatName("foo")); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); -} - -TEST_F(StatsRequestTest, OneGauge) { - store_.rootScope()->gaugeFromStatName(makeStatName("foo"), Stats::Gauge::ImportMode::Accumulate); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); -} - -TEST_F(StatsRequestTest, OneHistogram) { - store_.rootScope()->histogramFromStatName(makeStatName("foo"), - Stats::Histogram::Unit::Milliseconds); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Histograms))); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); -} - -TEST_F(StatsRequestTest, OneTextReadout) { - store_.rootScope()->textReadoutFromStatName(makeStatName("foo")); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::TextReadouts))); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); -} - -TEST_F(StatsRequestTest, OneScope) { - Stats::ScopeSharedPtr scope = store_.createScope("foo"); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); -} - -TEST_F(StatsRequestTest, ManyStatsSmallChunkSize) { - for (uint32_t i = 0; i < 100; ++i) { - store_.rootScope()->counterFromStatName(makeStatName(absl::StrCat("foo", i))); - } - std::unique_ptr request = makeRequest(false, StatsFormat::Text, StatsType::All); - request->setChunkSize(100); - EXPECT_EQ(9, iterateChunks(*request)); -} - -TEST_F(StatsRequestTest, ManyStatsSmallChunkSizeNoDrain) { - for (uint32_t i = 0; i < 100; ++i) { - store_.rootScope()->counterFromStatName(makeStatName(absl::StrCat("foo", i))); - } - std::unique_ptr request = makeRequest(false, StatsFormat::Text, StatsType::All); - request->setChunkSize(100); - EXPECT_EQ(9, iterateChunks(*request, false)); -} - -TEST_F(StatsRequestTest, OneStatUsedOnly) { - store_.rootScope()->counterFromStatName(makeStatName("foo")); - EXPECT_EQ(0, iterateChunks(*makeRequest(true, StatsFormat::Text, StatsType::All))); -} - -TEST_F(StatsRequestTest, OneStatJson) { - store_.rootScope()->counterFromStatName(makeStatName("foo")); - EXPECT_THAT(response(*makeRequest(false, StatsFormat::Json, StatsType::All)), StartsWith("{")); -} - -TEST_F(StatsRequestTest, OneStatPrometheus) { - // Currently the rendering infrastructure does not support Prometheus -- that - // gets rendered using a different code-path. This will be fixed at some - // point, to make Prometheus consume less resource, and when that occurs this - // test can exercise that. - store_.rootScope()->counterFromStatName(makeStatName("foo")); - EXPECT_ENVOY_BUG(iterateChunks(*makeRequest(false, StatsFormat::Prometheus, StatsType::All), true, - Http::Code::BadRequest), - "reached Prometheus case in switch unexpectedly"); -} - -} // namespace Server -} // namespace Envoy diff --git a/test/server/admin/stats_request_test_base.cc b/test/server/admin/stats_request_test_base.cc new file mode 100644 index 000000000000..52c655027ab2 --- /dev/null +++ b/test/server/admin/stats_request_test_base.cc @@ -0,0 +1,65 @@ +#include "test/server/admin/stats_request_test_base.h" + +#include + +namespace Envoy { +namespace Server { + +template +StatsRequestTestBase::StatsRequestTestBase() + : pool_(symbol_table_), alloc_(symbol_table_), store_(alloc_) { + store_.addSink(sink_); + store_.initializeThreading(main_thread_dispatcher_, tls_); +} + +template StatsRequestTestBase::~StatsRequestTestBase() { + tls_.shutdownGlobalThreading(); + store_.shutdownThreading(); + tls_.shutdownThread(); +} + +// Executes a request, counting the chunks that were generated. +template +uint32_t StatsRequestTestBase::iterateChunks(T& request, bool drain, Http::Code expect_code) { + Http::TestResponseHeaderMapImpl response_headers; + Http::Code code = request.start(response_headers); + EXPECT_EQ(expect_code, code); + if (code != Http::Code::OK) { + return 0; + } + Buffer::OwnedImpl data; + uint32_t num_chunks = 0; + bool more = true; + do { + uint64_t prev_size = data.length(); + more = request.nextChunk(data); + uint64_t size = data.length(); + if ((drain && size > 0) || (!drain && size != prev_size)) { + ++num_chunks; + if (drain) { + data.drain(size); + } + } + } while (more); + return num_chunks; +} + +// Executes a request, returning the rendered buffer as a string. +template std::string StatsRequestTestBase::response(T& request) { + Http::TestResponseHeaderMapImpl response_headers; + Http::Code code = request.start(response_headers); + EXPECT_EQ(Http::Code::OK, code); + Buffer::OwnedImpl data; + while (request.nextChunk(data)) { + } + return data.toString(); +} + +TestScope::TestScope(const std::string& prefix, Stats::MockStore& store) + : Stats::IsolatedScopeImpl(prefix, store) {} + +template class StatsRequestTestBase; +template class StatsRequestTestBase; + +} // namespace Server +} // namespace Envoy diff --git a/test/server/admin/stats_request_test_base.h b/test/server/admin/stats_request_test_base.h new file mode 100644 index 000000000000..2a6bbcf7faa4 --- /dev/null +++ b/test/server/admin/stats_request_test_base.h @@ -0,0 +1,82 @@ +#pragma once + +#include +#include + +#include "source/common/buffer/buffer_impl.h" +#include "source/common/stats/thread_local_store.h" +#include "source/server/admin/grouped_stats_request.h" +#include "source/server/admin/ungrouped_stats_request.h" + +#include "test/mocks/event/mocks.h" +#include "test/mocks/stats/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Server { + +template class StatsRequestTestBase : public testing::Test { +protected: + StatsRequestTestBase(); + ~StatsRequestTestBase() override; + + uint32_t iterateChunks(T& request, bool drain = true, Http::Code expect_code = Http::Code::OK); + + Stats::StatName makeStatName(absl::string_view name) { return pool_.add(name); } + + std::string response(T& request); + + Stats::SymbolTableImpl symbol_table_; + Stats::StatNamePool pool_; + Stats::AllocatorImpl alloc_; + testing::NiceMock sink_; + testing::NiceMock main_thread_dispatcher_; + testing::NiceMock tls_; + Stats::ThreadLocalStoreImpl store_; + Buffer::OwnedImpl response_; +}; + +// TODO(jmarantz): investigate whether we can reuse one of the pre-existing +// scope classes instead of introducing a new subclass of IsolatedScopeImpl +// (used only by a set of tests in prometheus_stats_test.cc). +class TestScope : public Stats::IsolatedScopeImpl { +public: + TestScope(const std::string& prefix, Stats::MockStore& store); + + bool iterate(const Stats::IterateFn& fn) const override { + std::for_each(histograms_.begin(), histograms_.end(), fn); + return true; + } + + bool iterate(const Stats::IterateFn& fn) const override { + std::for_each(counters_.begin(), counters_.end(), fn); + return true; + } + + bool iterate(const Stats::IterateFn& fn) const override { + std::for_each(gauges_.begin(), gauges_.end(), fn); + return true; + } + + bool iterate(const Stats::IterateFn& fn) const override { + std::for_each(text_readouts_.begin(), text_readouts_.end(), fn); + return true; + } + +public: + std::vector histograms_; + std::vector counters_; + std::vector gauges_; + std::vector text_readouts_; +}; + +class MockStore : public Stats::MockStore { +public: + MOCK_METHOD(Stats::ScopeSharedPtr, rootScope, ()); +}; + +} // namespace Server +} // namespace Envoy diff --git a/test/server/admin/ungrouped_stats_request_test.cc b/test/server/admin/ungrouped_stats_request_test.cc new file mode 100644 index 000000000000..65f7d9e0551b --- /dev/null +++ b/test/server/admin/ungrouped_stats_request_test.cc @@ -0,0 +1,88 @@ +#include "test/server/admin/stats_request_test_base.h" + +namespace Envoy { +namespace Server { + +class UngroupedStatsRequestTest : public StatsRequestTestBase { +protected: + std::unique_ptr makeRequest(bool used_only, StatsFormat format, + StatsType type) { + StatsParams params; + params.used_only_ = used_only; + params.type_ = type; + params.format_ = format; + return std::make_unique(store_, params); + } +}; + +TEST_F(UngroupedStatsRequestTest, Empty) { + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); +} + +TEST_F(UngroupedStatsRequestTest, OneCounter) { + store_.rootScope()->counterFromStatName(makeStatName("foo")); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); +} + +TEST_F(UngroupedStatsRequestTest, OneGauge) { + store_.rootScope()->gaugeFromStatName(makeStatName("foo"), Stats::Gauge::ImportMode::Accumulate); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); +} + +TEST_F(UngroupedStatsRequestTest, OneHistogram) { + store_.rootScope()->histogramFromStatName(makeStatName("foo"), + Stats::Histogram::Unit::Milliseconds); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Histograms))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); +} + +TEST_F(UngroupedStatsRequestTest, OneTextReadout) { + store_.rootScope()->textReadoutFromStatName(makeStatName("foo")); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::TextReadouts))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); +} + +TEST_F(UngroupedStatsRequestTest, OneScope) { + Stats::ScopeSharedPtr scope = store_.createScope("foo"); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); +} + +TEST_F(UngroupedStatsRequestTest, ManyStatsSmallChunkSize) { + for (uint32_t i = 0; i < 100; ++i) { + store_.rootScope()->counterFromStatName(makeStatName(absl::StrCat("foo", i))); + } + std::unique_ptr request = + makeRequest(false, StatsFormat::Text, StatsType::All); + request->setChunkSize(100); + EXPECT_EQ(9, iterateChunks(*request)); +} + +TEST_F(UngroupedStatsRequestTest, ManyStatsSmallChunkSizeNoDrain) { + for (uint32_t i = 0; i < 100; ++i) { + store_.rootScope()->counterFromStatName(makeStatName(absl::StrCat("foo", i))); + } + std::unique_ptr request = + makeRequest(false, StatsFormat::Text, StatsType::All); + request->setChunkSize(100); + EXPECT_EQ(9, iterateChunks(*request, false)); +} + +TEST_F(UngroupedStatsRequestTest, OneStatUsedOnly) { + store_.rootScope()->counterFromStatName(makeStatName("foo")); + EXPECT_EQ(0, iterateChunks(*makeRequest(true, StatsFormat::Text, StatsType::All))); +} + +TEST_F(UngroupedStatsRequestTest, OneStatJson) { + store_.rootScope()->counterFromStatName(makeStatName("foo")); + EXPECT_THAT(response(*makeRequest(false, StatsFormat::Json, StatsType::All)), + testing::StartsWith("{")); +} + +} // namespace Server +} // namespace Envoy diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index 39a31a4a42fe..f95619d87c4a 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -193,10 +193,11 @@ paths: - source/common/stats/tag_extractor_impl.h - source/common/stats/tag_extractor_impl.cc - source/common/version/version.cc - - source/server/admin/prometheus_stats.h - - source/server/admin/prometheus_stats.cc + - source/server/admin/grouped_stats_request.cc + - source/server/admin/ungrouped_stats_request.cc + - source/server/admin/prometheus_stats_render.h + - source/server/admin/prometheus_stats_render.cc - source/server/admin/stats_params.h - - source/server/admin/stats_request.cc - source/server/admin/utils.h - source/server/admin/utils.cc - tools/clang_tools/api_booster/main.cc From 6301fefe405d5c9bade12658034f17f5ad4cdd3c Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 1 May 2023 16:46:03 +0100 Subject: [PATCH 047/740] deps: Bump pytooling publishing deps (#27059) Signed-off-by: Ryan Northey --- distribution/packages.bzl | 7 ++----- tools/base/requirements.in | 6 +++--- tools/base/requirements.txt | 7 +++---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/distribution/packages.bzl b/distribution/packages.bzl index 1f0750c1a3d5..d788330cbad1 100644 --- a/distribution/packages.bzl +++ b/distribution/packages.bzl @@ -63,14 +63,11 @@ def envoy_pkg_distros( SIGNING_ARGS+=("--maintainer-email" "$${PACKAGES_MAINTAINER_EMAIL}"); \ fi \ && $(location //tools/distribution:sign) \ - --extract \ - --tar $@ \ + --out $@ \ "$${SIGNING_ARGS[@]}" \ $(location :distro_packages) """, outs = ["%s.tar.gz" % name], srcs = [":distro_packages"], - tools = [ - "//tools/distribution:sign", - ], + tools = ["//tools/distribution:sign"], ) diff --git a/tools/base/requirements.in b/tools/base/requirements.in index dbde1c5a313a..66ca2c322174 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -5,15 +5,15 @@ cffi>=1.15.0 colorama coloredlogs dependatool>=0.2.2 -envoy.base.utils>=0.4.2 +envoy.base.utils>=0.4.7 envoy.code.check>=0.4.1 envoy.dependency.check>=0.1.7 envoy.distribution.release>=0.0.9 envoy.distribution.repo>=0.0.8 envoy.distribution.verify>=0.0.11 envoy.docs.sphinx_runner>=0.2.5 -envoy.gpg.identity>=0.1.0 -envoy.gpg.sign>=0.1.1 +envoy.gpg.identity>=0.1.1 +envoy.gpg.sign>=0.2.0 flake8>=6 frozendict>=2.3.7 gitpython diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index f277fd91ce97..6e449fc57644 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -480,9 +480,9 @@ envoy-gpg-identity==0.1.1 \ # via # -r requirements.in # envoy-gpg-sign -envoy-gpg-sign==0.1.1 \ - --hash=sha256:5149360c2394a450ee9293d0d2a3a5d62ba8a06c5b92820c201241f61406c790 \ - --hash=sha256:533154f983b3ccbbbf085aba9d79aa7f5452563f7854a6cdc4080ce60d1f3d04 +envoy-gpg-sign==0.2.0 \ + --hash=sha256:53ef217a05555d725d467ceb70fbf7bc623eeb973a41996e8bbe1f295d8c9aab \ + --hash=sha256:8bca326766a2b82864ec6274c51d99c9924f2ec773316c2f13034925ddf50772 # via -r requirements.in flake8==6.0.0 \ --hash=sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7 \ @@ -1129,7 +1129,6 @@ verboselogs==1.7 \ # envoy-distribution-repo # envoy-github-abstract # envoy-github-release - # envoy-gpg-sign wrapt==1.14.1 \ --hash=sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3 \ --hash=sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b \ From d699a6329de2cd0a2c8aba88907d49cc6dcdd121 Mon Sep 17 00:00:00 2001 From: code Date: Tue, 2 May 2023 00:09:03 +0800 Subject: [PATCH 048/740] generic proxy: virtual host support for the generic proxy routing (#26932) * update proto api of generic proxy Signed-off-by: wbpcode * update comment Signed-off-by: wbpcode * initial common matcher Signed-off-by: wbpcode * complete test Signed-off-by: wbpcode * minor update of new custom matcher Signed-off-by: wbpcode * add const to name() methods Signed-off-by: wbpcode * address all comments Signed-off-by: wbpcode * fix error after merge main Signed-off-by: wbpcode --------- Signed-off-by: wbpcode --- .../generic_proxy/action/v3/action.proto | 4 + .../network/generic_proxy/matcher/v3/BUILD | 1 + .../generic_proxy/matcher/v3/matcher.proto | 53 +- .../network/generic_proxy/v3/route.proto | 35 +- .../filters/network/source/interface/route.h | 8 + .../filters/network/source/match.cc | 4 + .../filters/network/source/match.h | 52 ++ .../filters/network/source/route.cc | 140 ++++- .../filters/network/source/route.h | 45 +- .../filters/network/test/integration_test.cc | 56 +- .../filters/network/test/match_test.cc | 32 + .../filters/network/test/mocks/route.h | 1 + .../filters/network/test/route_test.cc | 550 +++++++++++++++--- 13 files changed, 868 insertions(+), 113 deletions(-) diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/action.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/action.proto index 1cc8f52e6eb2..119b8e7f85ef 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/action.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/action.proto @@ -22,7 +22,11 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // [#protodoc-title: Generic Proxy Route Action Configuration] // Configuration for the route match action. +// [#next-free-field: 6] message RouteAction { + // The name of the route action. This should be unique across all route actions. + string name = 5; + oneof cluster_specifier { option (validate.required) = true; diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/BUILD b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/BUILD index ec1e778e06e5..081219249cbd 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/type/matcher/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", "@com_github_cncf_udpa//xds/annotations/v3:pkg", ], diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto index b03a6088d37d..021d1f85346a 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package envoy.extensions.filters.network.generic_proxy.matcher.v3; +import "envoy/type/matcher/v3/string.proto"; + import "xds/annotations/v3/status.proto"; import "udpa/annotations/status.proto"; @@ -18,17 +20,64 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // Used to match request service of the downstream request. Only applicable if a service provided // by the application protocol. +// This is deprecated and should be replaced by HostMatchInput. This is kept for backward compatibility. message ServiceMatchInput { } -// Used to match request method of the downstream request. Only applicable if a method provided +// Used to match request host of the generic downstream request. Only applicable if a host provided +// by the application protocol. +// This is same with the ServiceMatchInput and this should be preferred over ServiceMatchInput. +message HostMatchInput { +} + +// Used to match request path of the generic downstream request. Only applicable if a path provided +// by the application protocol. +message PathMatchInput { +} + +// Used to match request method of the generic downstream request. Only applicable if a method provided // by the application protocol. message MethodMatchInput { } -// Used to match an arbitrary property of the downstream request. +// Used to match an arbitrary property of the generic downstream request. // These properties are populated by the codecs of application protocols. message PropertyMatchInput { // The property name to match on. string property_name = 1 [(validate.rules).string = {min_len: 1}]; } + +// [#not-implemented-hide:] +// Used to match an whole generic downstream request. +message RequestMatchInput { +} + +// [#not-implemented-hide:] +// Used to match an arbitrary key-value pair for headers, trailers or properties. +message KeyValueMatchEntry { + // The key name to match on. + string name = 1 [(validate.rules).string = {min_len: 1}]; + + // The key value pattern. + type.matcher.v3.StringMatcher string_match = 2 [(validate.rules).message = {required: true}]; +} + +// [#not-implemented-hide:] +// Custom matcher to match on the generic downstream request. This is used to match +// multiple fields of the downstream request and avoid complex combinations of +// HostMatchInput, PathMatchInput, MethodMatchInput and PropertyMatchInput. +message RequestMatcher { + // Optional host pattern to match on. If not specified, any host will match. + type.matcher.v3.StringMatcher host = 1; + + // Optional path pattern to match on. If not specified, any path will match. + type.matcher.v3.StringMatcher path = 2; + + // Optional method pattern to match on. If not specified, any method will match. + type.matcher.v3.StringMatcher method = 3; + + // Optional arbitrary properties to match on. If not specified, any properties + // will match. The key is the property name and the value is the property value + // to match on. + repeated KeyValueMatchEntry properties = 4; +} diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/route.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/route.proto index 629850470fb4..01442fa85d6d 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/route.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/route.proto @@ -17,6 +17,32 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // [#protodoc-title: Generic Proxy Route Configuration] +message VirtualHost { + // The name of the virtual host. + string name = 1 [(validate.rules).string = {min_len: 1}]; + + // A list of hosts that will be matched to this virtual host. Wildcard hosts are supported in + // the suffix or prefix form. + // + // Host search order: + // 1. Exact names: ``www.foo.com``. + // 2. Suffix wildcards: ``*.foo.com`` or ``*-bar.foo.com``. + // 3. Prefix wildcards: ``foo.*`` or ``foo-*``. + // 4. Special wildcard ``*`` matching any host and will be the default virtual host. + // + // .. note:: + // + // The wildcard will not match the empty string. + // e.g. ``*-bar.foo.com`` will match ``baz-bar.foo.com`` but not ``-bar.foo.com``. + // The longest wildcards match first. + // Only a single virtual host in the entire route configuration can match on ``*``. A domain + // must be unique across all virtual hosts or the config will fail to load. + repeated string hosts = 2 [(validate.rules).repeated = {min_items: 1}]; + + // The match tree to use when resolving route actions for incoming requests. + xds.type.matcher.v3.Matcher routes = 3 [(validate.rules).message = {required: true}]; +} + // The generic proxy makes use of the `xds matching API` for routing configurations. // // In the below example, we combine a top level tree matcher with a linear matcher to match @@ -68,5 +94,12 @@ message RouteConfiguration { string name = 1 [(validate.rules).string = {min_len: 1}]; // The match tree to use when resolving route actions for incoming requests. - xds.type.matcher.v3.Matcher routes = 2 [(validate.rules).message = {required: true}]; + // If no any virtual host is configured in the ``virtual_hosts`` field or no special wildcard + // virtual host is configured, the ``routes`` field will be used as the default route table. + // If both the wildcard virtual host and ``routes`` are configured, the configuration will fail + // to load. + xds.type.matcher.v3.Matcher routes = 2; + + // An array of virtual hosts that make up the route table. + repeated VirtualHost virtual_hosts = 3; } diff --git a/contrib/generic_proxy/filters/network/source/interface/route.h b/contrib/generic_proxy/filters/network/source/interface/route.h index 688d9612f5ed..a417a7a39068 100644 --- a/contrib/generic_proxy/filters/network/source/interface/route.h +++ b/contrib/generic_proxy/filters/network/source/interface/route.h @@ -28,6 +28,14 @@ class RouteEntry { public: virtual ~RouteEntry() = default; + /** + * @return absl::string_view the name of the route. + */ + virtual absl::string_view name() const PURE; + + /** + * @return const std::string& the name of the target cluster. + */ virtual const std::string& clusterName() const PURE; /** diff --git a/contrib/generic_proxy/filters/network/source/match.cc b/contrib/generic_proxy/filters/network/source/match.cc index 3858c026895c..37ddc4bd15fa 100644 --- a/contrib/generic_proxy/filters/network/source/match.cc +++ b/contrib/generic_proxy/filters/network/source/match.cc @@ -7,6 +7,10 @@ namespace GenericProxy { REGISTER_FACTORY(ServiceMatchDataInputFactory, Matcher::DataInputFactory); +REGISTER_FACTORY(HostMatchDataInputFactory, Matcher::DataInputFactory); + +REGISTER_FACTORY(PathMatchDataInputFactory, Matcher::DataInputFactory); + REGISTER_FACTORY(MethodMatchDataInputFactory, Matcher::DataInputFactory); REGISTER_FACTORY(PropertyMatchDataInputFactory, Matcher::DataInputFactory); diff --git a/contrib/generic_proxy/filters/network/source/match.h b/contrib/generic_proxy/filters/network/source/match.h index 80fb6ea7a2d6..d339f4c54ef5 100644 --- a/contrib/generic_proxy/filters/network/source/match.h +++ b/contrib/generic_proxy/filters/network/source/match.h @@ -16,6 +16,10 @@ namespace GenericProxy { using ServiceDataInputProto = envoy::extensions::filters::network::generic_proxy::matcher::v3::ServiceMatchInput; +using HostDataInputProto = + envoy::extensions::filters::network::generic_proxy::matcher::v3::HostMatchInput; +using PathDataInputProto = + envoy::extensions::filters::network::generic_proxy::matcher::v3::PathMatchInput; using MethodDataInputProto = envoy::extensions::filters::network::generic_proxy::matcher::v3::MethodMatchInput; using PropertyDataInputProto = @@ -45,6 +49,54 @@ class ServiceMatchDataInputFactory : public Matcher::DataInputFactory { std::string name() const override { return "envoy.matching.generic_proxy.input.service"; } }; +class HostMatchDataInput : public Matcher::DataInput { +public: + Matcher::DataInputGetResult get(const Request& data) const override { + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, + std::string(data.host())}; + } +}; + +class HostMatchDataInputFactory : public Matcher::DataInputFactory { +public: + HostMatchDataInputFactory() = default; + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + Matcher::DataInputFactoryCb + createDataInputFactoryCb(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override { + return []() { return std::make_unique(); }; + } + + std::string name() const override { return "envoy.matching.generic_proxy.input.host"; } +}; + +class PathMatchDataInput : public Matcher::DataInput { +public: + Matcher::DataInputGetResult get(const Request& data) const override { + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, + std::string(data.path())}; + } +}; + +class PathMatchDataInputFactory : public Matcher::DataInputFactory { +public: + PathMatchDataInputFactory() = default; + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + Matcher::DataInputFactoryCb + createDataInputFactoryCb(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override { + return []() { return std::make_unique(); }; + } + + std::string name() const override { return "envoy.matching.generic_proxy.input.path"; } +}; + class MethodMatchDataInput : public Matcher::DataInput { public: Matcher::DataInputGetResult get(const Request& data) const override { diff --git a/contrib/generic_proxy/filters/network/source/route.cc b/contrib/generic_proxy/filters/network/source/route.cc index d2eb2df5aae6..bcbedb2d3136 100644 --- a/contrib/generic_proxy/filters/network/source/route.cc +++ b/contrib/generic_proxy/filters/network/source/route.cc @@ -19,7 +19,8 @@ namespace GenericProxy { RouteEntryImpl::RouteEntryImpl(const ProtoRouteAction& route_action, Envoy::Server::Configuration::ServerFactoryContext& context) - : cluster_name_(route_action.cluster()), metadata_(route_action.metadata()) { + : name_(route_action.name()), cluster_name_(route_action.cluster()), + metadata_(route_action.metadata()) { for (const auto& proto_filter_config : route_action.per_filter_config()) { auto& factory = Config::Utility::getAndCheckFactoryByName( @@ -49,18 +50,16 @@ Matcher::ActionFactoryCb RouteMatchActionFactory::createActionFactoryCb( } REGISTER_FACTORY(RouteMatchActionFactory, Matcher::ActionFactory); -RouteMatcherImpl::RouteMatcherImpl(const ProtoRouteConfiguration& route_config, - Envoy::Server::Configuration::ServerFactoryContext& context, - bool) - : name_(route_config.name()) { - +VirtualHostImpl::VirtualHostImpl(const ProtoVirtualHost& virtual_host_config, + Envoy::Server::Configuration::ServerFactoryContext& context, bool) + : name_(virtual_host_config.name()) { RouteActionValidationVisitor validation_visitor; RouteActionContext action_context{context}; Matcher::MatchTreeFactory factory(action_context, context, validation_visitor); - matcher_ = factory.create(route_config.routes())(); + matcher_ = factory.create(virtual_host_config.routes())(); if (!validation_visitor.errors().empty()) { throw EnvoyException(fmt::format("requirement violation while creating route match tree: {}", @@ -68,7 +67,7 @@ RouteMatcherImpl::RouteMatcherImpl(const ProtoRouteConfiguration& route_config, } } -RouteEntryConstSharedPtr RouteMatcherImpl::routeEntry(const Request& request) const { +RouteEntryConstSharedPtr VirtualHostImpl::routeEntry(const Request& request) const { auto match = Matcher::evaluateMatch(*matcher_, request); if (match.result_) { @@ -88,6 +87,131 @@ RouteEntryConstSharedPtr RouteMatcherImpl::routeEntry(const Request& request) co return nullptr; } +RouteMatcherImpl::RouteMatcherImpl(const ProtoRouteConfiguration& route_config, + Envoy::Server::Configuration::ServerFactoryContext& context, + bool validate_clusters) + : name_(route_config.name()) { + constexpr absl::string_view wildcard_flag{"*"}; + + // TODO(wbpcode): maybe share the same code with common/router/config_impl.cc by using template. + for (const auto& virtual_host_config : route_config.virtual_hosts()) { + VirtualHostSharedPtr virtual_host = + std::make_shared(virtual_host_config, context, validate_clusters); + for (const std::string& host_name : virtual_host_config.hosts()) { + if (host_name.empty()) { + throw EnvoyException( + fmt::format("Invalid empty host name in route {}", route_config.name())); + } + + absl::string_view host_name_view{host_name}; + bool duplicate_found = false; + if (wildcard_flag == host_name_view) { + // Catch all virtual host. + if (default_virtual_host_ != nullptr) { + throw EnvoyException(fmt::format("Only a single wildcard domain is permitted in route {}", + route_config.name())); + } + default_virtual_host_ = virtual_host; + } else if (absl::StartsWith(host_name_view, wildcard_flag)) { + duplicate_found = !wildcard_virtual_host_suffixes_[host_name_view.size() - 1] + .emplace(host_name_view.substr(1), virtual_host) + .second; + } else if (absl::EndsWith(host_name_view, wildcard_flag)) { + duplicate_found = + !wildcard_virtual_host_prefixes_[host_name_view.size() - 1] + .emplace(host_name_view.substr(0, host_name_view.size() - 1), virtual_host) + .second; + } else { + duplicate_found = !virtual_hosts_.emplace(host_name_view, virtual_host).second; + } + if (duplicate_found) { + throw EnvoyException(fmt::format("Only unique values for host are permitted. Duplicate " + "entry of domain {} in route {}", + host_name_view, route_config.name())); + } + } + } + + // 'routes' is supported for backwards compatibility. It will be used as default virtual host + // if no catch-all virtual host is specified. + if (route_config.has_routes()) { + if (default_virtual_host_ == nullptr) { + ProtoVirtualHost proto_virtual_host; + proto_virtual_host.mutable_routes()->MergeFrom(route_config.routes()); + default_virtual_host_ = + std::make_shared(proto_virtual_host, context, validate_clusters); + } else { + throw EnvoyException(fmt::format("'routes' cannot be specified at the same time as a " + "catch-all ('*') virtual host in route {}", + route_config.name())); + } + } +} + +const VirtualHostImpl* RouteMatcherImpl::findWildcardVirtualHost( + absl::string_view host, const RouteMatcherImpl::WildcardVirtualHosts& wildcard_virtual_hosts, + RouteMatcherImpl::SubstringFunction substring_function) const { + // We do a longest wildcard match against the host that's passed in + // (e.g. "foo-bar.baz.com" should match "*-bar.baz.com" before matching "*.baz.com" for suffix + // wildcards). This is done by scanning the length => wildcards map looking for every wildcard + // whose size is < length. + for (const auto& iter : wildcard_virtual_hosts) { + const uint32_t wildcard_length = iter.first; + const auto& wildcard_map = iter.second; + // >= because *.foo.com shouldn't match .foo.com. + if (wildcard_length >= host.size()) { + continue; + } + const auto& match = wildcard_map.find(substring_function(host, wildcard_length)); + if (match != wildcard_map.end()) { + return match->second.get(); + } + } + return nullptr; +} + +const VirtualHostImpl* RouteMatcherImpl::findVirtualHost(const Request& request) const { + absl::string_view host = request.host(); + + // Fast path the case where we only have a default virtual host or host property is provided. + if (host.empty() || (virtual_hosts_.empty() && wildcard_virtual_host_suffixes_.empty() && + wildcard_virtual_host_prefixes_.empty())) { + return default_virtual_host_.get(); + } + + const auto iter = virtual_hosts_.find(host); + if (iter != virtual_hosts_.end()) { + return iter->second.get(); + } + + if (!wildcard_virtual_host_suffixes_.empty()) { + const VirtualHostImpl* vhost = findWildcardVirtualHost( + host, wildcard_virtual_host_suffixes_, + [](absl::string_view h, int l) -> absl::string_view { return h.substr(h.size() - l); }); + if (vhost != nullptr) { + return vhost; + } + } + + if (!wildcard_virtual_host_prefixes_.empty()) { + const VirtualHostImpl* vhost = findWildcardVirtualHost( + host, wildcard_virtual_host_prefixes_, + [](absl::string_view h, int l) -> absl::string_view { return h.substr(0, l); }); + if (vhost != nullptr) { + return vhost; + } + } + return default_virtual_host_.get(); +} + +RouteEntryConstSharedPtr RouteMatcherImpl::routeEntry(const Request& request) const { + const auto* virtual_host = findVirtualHost(request); + if (virtual_host != nullptr) { + return virtual_host->routeEntry(request); + } + return nullptr; +} + } // namespace GenericProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/contrib/generic_proxy/filters/network/source/route.h b/contrib/generic_proxy/filters/network/source/route.h index a4ca169bcd93..bea6961e4e22 100644 --- a/contrib/generic_proxy/filters/network/source/route.h +++ b/contrib/generic_proxy/filters/network/source/route.h @@ -28,6 +28,7 @@ using ProtoRouteAction = envoy::extensions::filters::network::generic_proxy::action::v3::RouteAction; using ProtoRouteConfiguration = envoy::extensions::filters::network::generic_proxy::v3::RouteConfiguration; +using ProtoVirtualHost = envoy::extensions::filters::network::generic_proxy::v3::VirtualHost; class RouteEntryImpl : public RouteEntry { public: @@ -35,6 +36,7 @@ class RouteEntryImpl : public RouteEntry { Envoy::Server::Configuration::ServerFactoryContext& context); // RouteEntry + absl::string_view name() const override { return name_; } const std::string& clusterName() const override { return cluster_name_; } const RouteSpecificFilterConfig* perFilterConfig(absl::string_view name) const override { auto iter = per_filter_configs_.find(name); @@ -45,6 +47,7 @@ class RouteEntryImpl : public RouteEntry { private: static const uint64_t DEFAULT_ROUTE_TIMEOUT_MS = 15000; + std::string name_; std::string cluster_name_; const envoy::config::core::v3::Metadata metadata_; @@ -94,6 +97,21 @@ class NullRouteMatcherImpl : public RouteMatcher { RouteEntryConstSharedPtr routeEntry(const Request&) const override { return nullptr; } }; +class VirtualHostImpl : Logger::Loggable { +public: + VirtualHostImpl(const ProtoVirtualHost& virtual_host_config, + Envoy::Server::Configuration::ServerFactoryContext& context, + bool validate_clusters_default = false); + + RouteEntryConstSharedPtr routeEntry(const Request& request) const; + absl::string_view name() const { return name_; } + +private: + std::string name_; + Matcher::MatchTreeSharedPtr matcher_; +}; +using VirtualHostSharedPtr = std::shared_ptr; + class RouteMatcherImpl : public RouteMatcher, Logger::Loggable { public: RouteMatcherImpl(const ProtoRouteConfiguration& route_config, @@ -102,11 +120,34 @@ class RouteMatcherImpl : public RouteMatcher, Logger::Loggable, std::greater<>>; + using SubstringFunction = std::function; + const VirtualHostImpl* findWildcardVirtualHost(absl::string_view host, + const WildcardVirtualHosts& wildcard_virtual_hosts, + SubstringFunction substring_function) const; + + const VirtualHostImpl* findVirtualHost(const Request& request) const; + std::string name_; - Matcher::MatchTreeSharedPtr matcher_; + + absl::flat_hash_map virtual_hosts_; + // std::greater as a minor optimization to iterate from more to less specific + // + // A note on using an unordered_map versus a vector of (string, VirtualHostSharedPtr) pairs: + // + // Based on local benchmarks, each vector entry costs around 20ns for recall and (string) + // comparison with a fixed cost of about 25ns. For unordered_map, the empty map costs about 65ns + // and climbs to about 110ns once there are any entries. + // + // The break-even is 4 entries. + WildcardVirtualHosts wildcard_virtual_host_suffixes_; + WildcardVirtualHosts wildcard_virtual_host_prefixes_; + + VirtualHostSharedPtr default_virtual_host_; }; } // namespace GenericProxy diff --git a/contrib/generic_proxy/filters/network/test/integration_test.cc b/contrib/generic_proxy/filters/network/test/integration_test.cc index 14b1421765cd..53c948f247da 100644 --- a/contrib/generic_proxy/filters/network/test/integration_test.cc +++ b/contrib/generic_proxy/filters/network/test/integration_test.cc @@ -134,33 +134,37 @@ class IntegrationTest : public testing::TestWithParam(input->get(request).data_)); } +TEST(HostMatchDataInputTest, HostMatchDataInputTest) { + NiceMock factory_context; + HostMatchDataInputFactory factory; + auto proto_config = factory.createEmptyConfigProto(); + auto input = + factory.createDataInputFactoryCb(*proto_config, factory_context.messageValidationVisitor())(); + + FakeStreamCodecFactory::FakeRequest request; + + EXPECT_EQ("", absl::get(input->get(request).data_)); + + request.host_ = "fake_host_as_service"; + + EXPECT_EQ("fake_host_as_service", absl::get(input->get(request).data_)); +} + +TEST(PathMatchDataInputTest, PathMatchDataInputTest) { + NiceMock factory_context; + PathMatchDataInputFactory factory; + auto proto_config = factory.createEmptyConfigProto(); + auto input = + factory.createDataInputFactoryCb(*proto_config, factory_context.messageValidationVisitor())(); + + FakeStreamCodecFactory::FakeRequest request; + + EXPECT_EQ("", absl::get(input->get(request).data_)); + + request.path_ = "fake_path"; + + EXPECT_EQ("fake_path", absl::get(input->get(request).data_)); +} + TEST(MethodMatchDataInputTest, MethodMatchDataInputTest) { NiceMock factory_context; MethodMatchDataInputFactory factory; diff --git a/contrib/generic_proxy/filters/network/test/mocks/route.h b/contrib/generic_proxy/filters/network/test/mocks/route.h index d2cc0e1da900..428f4285a672 100644 --- a/contrib/generic_proxy/filters/network/test/mocks/route.h +++ b/contrib/generic_proxy/filters/network/test/mocks/route.h @@ -17,6 +17,7 @@ class MockRouteEntry : public RouteEntry { MOCK_METHOD(const std::string&, clusterName, (), (const)); MOCK_METHOD(const RouteSpecificFilterConfig*, perFilterConfig, (absl::string_view), (const)); MOCK_METHOD(const envoy::config::core::v3::Metadata&, metadata, (), (const)); + MOCK_METHOD(absl::string_view, name, (), (const)); std::string cluster_name_{"fake_cluster_name"}; diff --git a/contrib/generic_proxy/filters/network/test/route_test.cc b/contrib/generic_proxy/filters/network/test/route_test.cc index 416b2abcdc29..7e948fef3191 100644 --- a/contrib/generic_proxy/filters/network/test/route_test.cc +++ b/contrib/generic_proxy/filters/network/test/route_test.cc @@ -218,54 +218,185 @@ class RouteMatcherImplTest : public testing::Test { static const std::string RouteConfigurationYaml = R"EOF( name: test_matcher_tree -routes: - matcher_list: - matchers: - - predicate: - and_matcher: - predicate: - - single_predicate: - input: - name: envoy.matching.generic_proxy.input.service - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.ServiceMatchInput - value_match: - exact: "service_0" - - single_predicate: - input: - name: envoy.matching.generic_proxy.input.method - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.MethodMatchInput - value_match: - exact: "method_0" - - or_matcher: - predicate: - - single_predicate: - input: - name: envoy.matching.generic_proxy.input.property - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.PropertyMatchInput - property_name: "key_0" - value_match: - exact: "value_0" - - single_predicate: - input: - name: envoy.matching.generic_proxy.input.property - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.PropertyMatchInput - property_name: "key_1" - value_match: - exact: "value_1" - on_match: - action: - name: envoy.matching.action.generic_proxy.route - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction - cluster: "cluster_0" - metadata: - filter_metadata: - mock_filter: - key_0: value_0 +virtual_hosts: +- name: service + hosts: + - service_0 + routes: + matcher_list: + matchers: + - predicate: + and_matcher: + predicate: + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.host + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.HostMatchInput + value_match: + exact: "service_0" + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.method + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.MethodMatchInput + value_match: + exact: "method_0" + - or_matcher: + predicate: + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.property + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.PropertyMatchInput + property_name: "key_0" + value_match: + exact: "value_0" + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.property + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.PropertyMatchInput + property_name: "key_1" + value_match: + exact: "value_1" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_0" + metadata: + filter_metadata: + mock_filter: + match_service: match_service +- name: prefix + hosts: + - "prefix*" + routes: + matcher_list: + matchers: + - predicate: + and_matcher: + predicate: + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.host + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.HostMatchInput + value_match: + exact: "prefix_service_0" + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.method + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.MethodMatchInput + value_match: + exact: "method_0" + - or_matcher: + predicate: + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.property + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.PropertyMatchInput + property_name: "key_0" + value_match: + exact: "value_0" + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.property + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.PropertyMatchInput + property_name: "key_1" + value_match: + exact: "value_1" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_1" + metadata: + filter_metadata: + mock_filter: + match_prefix: match_prefix +- name: suffix + hosts: + - "*suffix" + routes: + matcher_list: + matchers: + - predicate: + and_matcher: + predicate: + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.host + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.HostMatchInput + value_match: + exact: "service_0_suffix" + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.method + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.MethodMatchInput + value_match: + exact: "method_0" + - or_matcher: + predicate: + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.property + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.PropertyMatchInput + property_name: "key_0" + value_match: + exact: "value_0" + - single_predicate: + input: + name: envoy.matching.generic_proxy.input.property + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.PropertyMatchInput + property_name: "key_1" + value_match: + exact: "value_1" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_2" + metadata: + filter_metadata: + mock_filter: + match_suffix: match_suffix +- name: catch_all + hosts: + - "*" + routes: + matcher_list: + matchers: + - predicate: + single_predicate: + input: + name: envoy.matching.generic_proxy.input.property + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.PropertyMatchInput + property_name: "catch_all" + value_match: + exact: "catch_all" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_3" + metadata: + filter_metadata: + mock_filter: + catch_all: catch_all )EOF"; /** @@ -282,23 +413,89 @@ TEST_F(RouteMatcherImplTest, SimpleNameMethod) { TEST_F(RouteMatcherImplTest, RouteMatch) { initialize(RouteConfigurationYaml); - FakeStreamCodecFactory::FakeRequest fake_request_0; - fake_request_0.host_ = "service_0"; - fake_request_0.method_ = "method_0"; - fake_request_0.data_.insert({"key_0", "value_0"}); + // Exact host searching. + { + FakeStreamCodecFactory::FakeRequest fake_request_0; + fake_request_0.host_ = "service_0"; + fake_request_0.method_ = "method_0"; + fake_request_0.data_.insert({"key_0", "value_0"}); + + FakeStreamCodecFactory::FakeRequest fake_request_1; + fake_request_1.host_ = "service_0"; + fake_request_1.method_ = "method_0"; + fake_request_1.data_.insert({"key_1", "value_1"}); + + auto route_entry_0 = route_matcher_->routeEntry(fake_request_0); + auto route_entry_1 = route_matcher_->routeEntry(fake_request_1); + + EXPECT_EQ(route_entry_0.get(), route_entry_1.get()); + EXPECT_NE(route_entry_0.get(), nullptr); + + EXPECT_EQ(route_entry_0->clusterName(), "cluster_0"); + } + + // Prefix host searching. + { + FakeStreamCodecFactory::FakeRequest fake_request_0; + fake_request_0.host_ = "prefix_service_0"; + fake_request_0.method_ = "method_0"; + fake_request_0.data_.insert({"key_0", "value_0"}); + + FakeStreamCodecFactory::FakeRequest fake_request_1; + fake_request_1.host_ = "prefix_service_0"; + fake_request_1.method_ = "method_0"; + fake_request_1.data_.insert({"key_1", "value_1"}); + + auto route_entry_0 = route_matcher_->routeEntry(fake_request_0); + auto route_entry_1 = route_matcher_->routeEntry(fake_request_1); + + EXPECT_EQ(route_entry_0.get(), route_entry_1.get()); + EXPECT_NE(route_entry_0.get(), nullptr); + + EXPECT_EQ(route_entry_0->clusterName(), "cluster_1"); + } + + // Suffix host searching. + { + FakeStreamCodecFactory::FakeRequest fake_request_0; + fake_request_0.host_ = "service_0_suffix"; + fake_request_0.method_ = "method_0"; + fake_request_0.data_.insert({"key_0", "value_0"}); + + FakeStreamCodecFactory::FakeRequest fake_request_1; + fake_request_1.host_ = "service_0_suffix"; + fake_request_1.method_ = "method_0"; + fake_request_1.data_.insert({"key_1", "value_1"}); - FakeStreamCodecFactory::FakeRequest fake_request_1; - fake_request_1.host_ = "service_0"; - fake_request_1.method_ = "method_0"; - fake_request_1.data_.insert({"key_1", "value_1"}); + auto route_entry_0 = route_matcher_->routeEntry(fake_request_0); + auto route_entry_1 = route_matcher_->routeEntry(fake_request_1); - auto route_entry_0 = route_matcher_->routeEntry(fake_request_0); - auto route_entry_1 = route_matcher_->routeEntry(fake_request_1); + EXPECT_EQ(route_entry_0.get(), route_entry_1.get()); + EXPECT_NE(route_entry_0.get(), nullptr); - EXPECT_EQ(route_entry_0.get(), route_entry_1.get()); - EXPECT_NE(route_entry_0.get(), nullptr); + EXPECT_EQ(route_entry_0->clusterName(), "cluster_2"); + } + + // Catch all host. + { + FakeStreamCodecFactory::FakeRequest fake_request_0; + fake_request_0.host_ = "any_service"; + fake_request_0.method_ = "method_0"; + fake_request_0.data_.insert({"catch_all", "catch_all"}); + + FakeStreamCodecFactory::FakeRequest fake_request_1; + fake_request_1.host_ = "any_service"; + fake_request_1.method_ = "method_0"; + fake_request_1.data_.insert({"catch_all", "catch_all"}); + + auto route_entry_0 = route_matcher_->routeEntry(fake_request_0); + auto route_entry_1 = route_matcher_->routeEntry(fake_request_1); - EXPECT_EQ(route_entry_0->clusterName(), "cluster_0"); + EXPECT_EQ(route_entry_0.get(), route_entry_1.get()); + EXPECT_NE(route_entry_0.get(), nullptr); + + EXPECT_EQ(route_entry_0->clusterName(), "cluster_3"); + } } /** @@ -310,7 +507,7 @@ TEST_F(RouteMatcherImplTest, RouteNotMatch) { // Test the service not match. { FakeStreamCodecFactory::FakeRequest fake_request; - fake_request.host_ = "service_x"; + fake_request.host_ = "prefix_service_1"; fake_request.method_ = "method_0"; fake_request.data_.insert({"key_0", "value_0"}); @@ -338,24 +535,193 @@ TEST_F(RouteMatcherImplTest, RouteNotMatch) { static const std::string RouteConfigurationYamlWithUnknownInput = R"EOF( name: test_matcher_tree +virtual_hosts: +- hosts: + - "*" + routes: + matcher_list: + matchers: + - predicate: + single_predicate: + input: + name: envoy.matching.generic_proxy.input.unknown_input + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.UnknownInput + value_match: + exact: "service_0" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_0" + metadata: + filter_metadata: + mock_filter: + key_0: value_0 +)EOF"; + +TEST_F(RouteMatcherImplTest, RouteConfigurationWithUnknownInput) { + EXPECT_THROW(initialize(RouteConfigurationYamlWithUnknownInput), EnvoyException); + EXPECT_EQ(nullptr, route_matcher_.get()); +} + +static const std::string RouteConfigurationYamlWithoutDefaultHost = R"EOF( +name: test_matcher_tree +virtual_hosts: +- hosts: + - "service_0" + routes: + matcher_list: + matchers: + - predicate: + single_predicate: + input: + name: envoy.matching.generic_proxy.input.host + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.HostMatchInput + value_match: + exact: "service_0" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_0" + metadata: + filter_metadata: + mock_filter: + key_0: value_0 +)EOF"; + +TEST_F(RouteMatcherImplTest, NoHostMatch) { + initialize(RouteConfigurationYamlWithoutDefaultHost); + + // Test the host not match. + { + FakeStreamCodecFactory::FakeRequest fake_request; + fake_request.host_ = "any_service"; + fake_request.method_ = "method_0"; + fake_request.data_.insert({"key_0", "value_0"}); + + EXPECT_EQ(nullptr, route_matcher_->routeEntry(fake_request)); + } +} + +static const std::string RouteConfigurationYamlWithRepeatedHost = R"EOF( +name: test_matcher_tree +virtual_hosts: +- hosts: + - "service_0" + - "service_0" + routes: + matcher_list: + matchers: + - predicate: + single_predicate: + input: + name: envoy.matching.generic_proxy.input.host + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.HostMatchInput + value_match: + exact: "service_0" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_0" + metadata: + filter_metadata: + mock_filter: + key_0: value_0 +)EOF"; + +TEST_F(RouteMatcherImplTest, RouteConfigurationYamlWithRepeatedHost) { + EXPECT_THROW_WITH_MESSAGE(initialize(RouteConfigurationYamlWithRepeatedHost), EnvoyException, + "Only unique values for host are permitted. Duplicate " + "entry of domain service_0 in route test_matcher_tree"); + EXPECT_EQ(nullptr, route_matcher_.get()); +} + +static const std::string RouteConfigurationYamlWithMultipleWildcard = R"EOF( +name: test_matcher_tree +virtual_hosts: +- hosts: + - "*" + - "*" + routes: + matcher_list: + matchers: + - predicate: + single_predicate: + input: + name: envoy.matching.generic_proxy.input.host + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.HostMatchInput + value_match: + exact: "service_0" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_0" + metadata: + filter_metadata: + mock_filter: + key_0: value_0 +)EOF"; + +TEST_F(RouteMatcherImplTest, RouteConfigurationYamlWithMultipleWildcard) { + EXPECT_THROW_WITH_MESSAGE( + initialize(RouteConfigurationYamlWithMultipleWildcard), EnvoyException, + "Only a single wildcard domain is permitted in route test_matcher_tree"); + EXPECT_EQ(nullptr, route_matcher_.get()); +} + +static const std::string RouteConfigurationYamlWithMultipleWildcard2 = R"EOF( +name: test_matcher_tree +virtual_hosts: +- hosts: + - "*" + routes: + matcher_list: + matchers: + - predicate: + single_predicate: + input: + name: envoy.matching.generic_proxy.input.host + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.HostMatchInput + value_match: + exact: "service_0" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_0" + metadata: + filter_metadata: + mock_filter: + key_0: value_0 routes: matcher_list: matchers: - predicate: - and_matcher: - predicate: - - single_predicate: - input: - name: envoy.matching.generic_proxy.input.unknown_input - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.v3.UnknownInput - value_match: - exact: "service_0" + single_predicate: + input: + name: envoy.matching.generic_proxy.input.host + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.HostMatchInput + value_match: + exact: "service_0" on_match: action: name: envoy.matching.action.generic_proxy.route typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.v3.RouteAction + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction cluster: "cluster_0" metadata: filter_metadata: @@ -363,8 +729,44 @@ name: test_matcher_tree key_0: value_0 )EOF"; -TEST_F(RouteMatcherImplTest, RouteConfigurationWithUnknownInput) { - EXPECT_THROW(initialize(RouteConfigurationYamlWithUnknownInput), EnvoyException); +TEST_F(RouteMatcherImplTest, RouteConfigurationYamlWithMultipleWildcard2) { + EXPECT_THROW_WITH_MESSAGE(initialize(RouteConfigurationYamlWithMultipleWildcard2), EnvoyException, + "'routes' cannot be specified at the same time as a " + "catch-all ('*') virtual host in route test_matcher_tree"); + EXPECT_EQ(nullptr, route_matcher_.get()); +} + +static const std::string RouteConfigurationYamlWithEmptyHost = R"EOF( +name: test_matcher_tree +virtual_hosts: +- hosts: + - "" + routes: + matcher_list: + matchers: + - predicate: + single_predicate: + input: + name: envoy.matching.generic_proxy.input.host + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.matcher.v3.HostMatchInput + value_match: + exact: "service_0" + on_match: + action: + name: envoy.matching.action.generic_proxy.route + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.action.v3.RouteAction + cluster: "cluster_0" + metadata: + filter_metadata: + mock_filter: + key_0: value_0 +)EOF"; + +TEST_F(RouteMatcherImplTest, RouteConfigurationYamlWithEmptyHost) { + EXPECT_THROW_WITH_MESSAGE(initialize(RouteConfigurationYamlWithEmptyHost), EnvoyException, + "Invalid empty host name in route test_matcher_tree"); EXPECT_EQ(nullptr, route_matcher_.get()); } From 65918a8c59e25b01d21116faa3070a64390f574e Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Mon, 1 May 2023 20:16:56 +0300 Subject: [PATCH 049/740] tools: setup build in local_fix_format (#27060) Signed-off-by: ohadvano <49730675+ohadvano@users.noreply.github.com> Co-authored-by: phlax --- tools/local_fix_format.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/local_fix_format.sh b/tools/local_fix_format.sh index a16e4957b69c..c1cc3d6b6ea2 100755 --- a/tools/local_fix_format.sh +++ b/tools/local_fix_format.sh @@ -22,6 +22,19 @@ # If DISPLAY is set, then tkdiff pops up for some BUILD changes. unset DISPLAY +# The following optional argument is added to be able to run this script using Docker, +# due to a problem to locate clang using WSL on Windows. https://learn.microsoft.com/en-us/windows/wsl/about +# Call with -docker as the first arument. +if [[ $# -gt 0 && "$1" == "-docker" ]]; then + shift + exec ./ci/run_envoy_docker.sh "$0" -run-build-setup "$@" +fi + +if [[ $# -gt 0 && "$1" == "-run-build-setup" ]]; then + shift + . ci/build_setup.sh +fi + if [[ $# -gt 0 && "$1" == "-verbose" ]]; then verbose=1 shift From 8f02c8008abc04e18270a1bc91f8adfb5b73122c Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 1 May 2023 13:34:36 -0400 Subject: [PATCH 050/740] Remove unused member from GrpcStream (#27055) Remove unused `random_` member from GrpcMux Signed-off-by: Yan Avlasov --- source/common/config/grpc_mux_impl.cc | 5 ++--- source/common/config/grpc_mux_impl.h | 5 ++--- source/common/config/grpc_stream.h | 8 +++----- source/common/config/new_grpc_mux_impl.cc | 5 ++--- source/common/config/new_grpc_mux_impl.h | 4 ++-- .../common/config/subscription_factory_impl.cc | 10 +++++----- source/common/config/xds_mux/grpc_mux_impl.cc | 16 +++++++--------- source/common/config/xds_mux/grpc_mux_impl.h | 17 +++++++---------- source/common/upstream/cluster_manager_impl.cc | 8 ++++---- .../config/delta_subscription_impl_test.cc | 4 ++-- .../config/delta_subscription_test_harness.h | 4 ++-- test/common/config/grpc_mux_impl_test.cc | 9 ++++----- test/common/config/grpc_stream_test.cc | 7 +++---- .../config/grpc_subscription_test_harness.h | 4 ++-- test/common/config/new_grpc_mux_impl_test.cc | 6 +++--- test/common/config/xds_grpc_mux_impl_test.cc | 11 +++++------ test/extensions/clusters/eds/eds_speed_test.cc | 5 ++--- 17 files changed, 57 insertions(+), 71 deletions(-) diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index dc0af5e560fc..1c26cdc439c9 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -58,15 +58,14 @@ std::string convertToWildcard(const std::string& resource_name) { GrpcMuxImpl::GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Random::RandomGenerator& random, Stats::Scope& scope, + const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node, CustomConfigValidatorsPtr&& config_validators, JitteredExponentialBackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, XdsResourcesDelegateOptRef xds_resources_delegate, const std::string& target_xds_authority) - : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + : grpc_stream_(this, std::move(async_client), service_method, dispatcher, scope, std::move(backoff_strategy), rate_limit_settings), local_info_(local_info), skip_subsequent_node_(skip_subsequent_node), config_validators_(std::move(config_validators)), xds_config_tracker_(xds_config_tracker), diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 7ed7c43f35a9..7bc66280e290 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -40,9 +40,8 @@ class GrpcMuxImpl : public GrpcMux, public: GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - Random::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node, - CustomConfigValidatorsPtr&& config_validators, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + bool skip_subsequent_node, CustomConfigValidatorsPtr&& config_validators, JitteredExponentialBackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, XdsResourcesDelegateOptRef xds_resources_delegate, diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index fa4d7a3becd3..b46d1d9b718b 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -25,13 +25,12 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, public Logger::Loggable { public: GrpcStream(GrpcStreamCallbacks* callbacks, Grpc::RawAsyncClientPtr async_client, - const Protobuf::MethodDescriptor& service_method, Random::RandomGenerator& random, - Event::Dispatcher& dispatcher, Stats::Scope& scope, - JitteredExponentialBackOffStrategyPtr backoff_strategy, + const Protobuf::MethodDescriptor& service_method, Event::Dispatcher& dispatcher, + Stats::Scope& scope, JitteredExponentialBackOffStrategyPtr backoff_strategy, const RateLimitSettings& rate_limit_settings) : callbacks_(callbacks), async_client_(std::move(async_client)), service_method_(service_method), - control_plane_stats_(Utility::generateControlPlaneStats(scope)), random_(random), + control_plane_stats_(Utility::generateControlPlaneStats(scope)), time_source_(dispatcher.timeSource()), backoff_strategy_(std::move(backoff_strategy)), rate_limiting_enabled_(rate_limit_settings.enabled_) { retry_timer_ = dispatcher.createTimer([this]() -> void { establishNewStream(); }); @@ -228,7 +227,6 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, // Reestablishes the gRPC channel when necessary, with some backoff politeness. Event::TimerPtr retry_timer_; - Random::RandomGenerator& random_; TimeSource& time_source_; JitteredExponentialBackOffStrategyPtr backoff_strategy_; diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index b83ed713e14f..7f0133846a2a 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -37,13 +37,12 @@ using AllMuxes = ThreadSafeSingleton; NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - Random::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, JitteredExponentialBackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker) - : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + : grpc_stream_(this, std::move(async_client), service_method, dispatcher, scope, std::move(backoff_strategy), rate_limit_settings), local_info_(local_info), config_validators_(std::move(config_validators)), dynamic_update_callback_handle_(local_info.contextProvider().addDynamicContextUpdateCallback( diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h index 765b8b04975c..e6a9c13aa358 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/new_grpc_mux_impl.h @@ -32,8 +32,8 @@ class NewGrpcMuxImpl Logger::Loggable { public: NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, Random::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, JitteredExponentialBackOffStrategyPtr backoff_strategy, diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 02e1bbb7a6aa..81fa75e3609d 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -83,7 +83,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), api_config_source, scope, true) ->createUncachedRawAsyncClient(), - dispatcher_, sotwGrpcMethod(type_url), api_.randomGenerator(), scope, + dispatcher_, sotwGrpcMethod(type_url), scope, Utility::parseRateLimitSettings(api_config_source), local_info_, api_config_source.set_node_on_first_message_only(), std::move(custom_config_validators), std::move(backoff_strategy), xds_config_tracker_, xds_resources_delegate_, @@ -94,7 +94,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), api_config_source, scope, true) ->createUncachedRawAsyncClient(), - dispatcher_, sotwGrpcMethod(type_url), api_.randomGenerator(), scope, + dispatcher_, sotwGrpcMethod(type_url), scope, Utility::parseRateLimitSettings(api_config_source), api_config_source.set_node_on_first_message_only(), std::move(custom_config_validators), std::move(backoff_strategy), xds_config_tracker_, xds_resources_delegate_, @@ -120,7 +120,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), api_config_source, scope, true) ->createUncachedRawAsyncClient(), - dispatcher_, deltaGrpcMethod(type_url), api_.randomGenerator(), scope, + dispatcher_, deltaGrpcMethod(type_url), scope, Utility::parseRateLimitSettings(api_config_source), local_info_, api_config_source.set_node_on_first_message_only(), std::move(custom_config_validators), std::move(backoff_strategy), xds_config_tracker_); @@ -129,7 +129,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), api_config_source, scope, true) ->createUncachedRawAsyncClient(), - dispatcher_, deltaGrpcMethod(type_url), api_.randomGenerator(), scope, + dispatcher_, deltaGrpcMethod(type_url), scope, Utility::parseRateLimitSettings(api_config_source), local_info_, std::move(custom_config_validators), std::move(backoff_strategy), xds_config_tracker_); } @@ -220,7 +220,7 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), api_config_source, scope, true) ->createUncachedRawAsyncClient(), - dispatcher_, deltaGrpcMethod(type_url), api_.randomGenerator(), scope, + dispatcher_, deltaGrpcMethod(type_url), scope, Utility::parseRateLimitSettings(api_config_source), local_info_, std::move(custom_config_validators), std::move(backoff_strategy), xds_config_tracker_), diff --git a/source/common/config/xds_mux/grpc_mux_impl.cc b/source/common/config/xds_mux/grpc_mux_impl.cc index 9d3eece2c3da..af981cd4dbec 100644 --- a/source/common/config/xds_mux/grpc_mux_impl.cc +++ b/source/common/config/xds_mux/grpc_mux_impl.cc @@ -40,12 +40,12 @@ GrpcMuxImpl::GrpcMuxImpl( std::unique_ptr subscription_state_factory, bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - Random::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, CustomConfigValidatorsPtr&& config_validators, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + CustomConfigValidatorsPtr&& config_validators, JitteredExponentialBackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, XdsResourcesDelegateOptRef xds_resources_delegate, const std::string& target_xds_authority) - : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + : grpc_stream_(this, std::move(async_client), service_method, dispatcher, scope, std::move(backoff_strategy), rate_limit_settings), subscription_state_factory_(std::move(subscription_state_factory)), skip_subsequent_node_(skip_subsequent_node), local_info_(local_info), @@ -363,15 +363,14 @@ template class GrpcMuxImpl(dispatcher), skip_subsequent_node, - local_info, std::move(async_client), dispatcher, service_method, random, scope, + local_info, std::move(async_client), dispatcher, service_method, scope, rate_limit_settings, std::move(config_validators), std::move(backoff_strategy), xds_config_tracker) {} @@ -387,8 +386,7 @@ void GrpcMuxDelta::requestOnDemandUpdate(const std::string& type_url, } GrpcMuxSotw::GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Random::RandomGenerator& random, Stats::Scope& scope, + const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node, CustomConfigValidatorsPtr&& config_validators, @@ -397,7 +395,7 @@ GrpcMuxSotw::GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatch XdsResourcesDelegateOptRef xds_resources_delegate, const std::string& target_xds_authority) : GrpcMuxImpl(std::make_unique(dispatcher), skip_subsequent_node, - local_info, std::move(async_client), dispatcher, service_method, random, scope, + local_info, std::move(async_client), dispatcher, service_method, scope, rate_limit_settings, std::move(config_validators), std::move(backoff_strategy), xds_config_tracker, xds_resources_delegate, target_xds_authority) {} diff --git a/source/common/config/xds_mux/grpc_mux_impl.h b/source/common/config/xds_mux/grpc_mux_impl.h index 9b67bb7c4971..d91639847372 100644 --- a/source/common/config/xds_mux/grpc_mux_impl.h +++ b/source/common/config/xds_mux/grpc_mux_impl.h @@ -62,8 +62,7 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, GrpcMuxImpl(std::unique_ptr subscription_state_factory, bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - Random::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, CustomConfigValidatorsPtr&& config_validators, JitteredExponentialBackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, @@ -223,10 +222,9 @@ class GrpcMuxDelta : public GrpcMuxImpl { public: GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, Random::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node, - CustomConfigValidatorsPtr&& config_validators, + const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, + bool skip_subsequent_node, CustomConfigValidatorsPtr&& config_validators, JitteredExponentialBackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker); @@ -240,10 +238,9 @@ class GrpcMuxSotw : public GrpcMuxImpl { public: GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, Random::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node, - CustomConfigValidatorsPtr&& config_validators, + const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, + bool skip_subsequent_node, CustomConfigValidatorsPtr&& config_validators, JitteredExponentialBackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, XdsResourcesDelegateOptRef xds_resources_delegate = absl::nullopt, diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index a5019aadc602..6e65198fa18c 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -404,7 +404,7 @@ ClusterManagerImpl::ClusterManagerImpl( *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v3.AggregatedDiscoveryService." "DeltaAggregatedResources"), - random_, *stats_.rootScope(), + *stats_.rootScope(), Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info, dyn_resources.ads_config().set_node_on_first_message_only(), std::move(custom_config_validators), std::move(backoff_strategy), @@ -417,7 +417,7 @@ ClusterManagerImpl::ClusterManagerImpl( main_thread_dispatcher, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v3.AggregatedDiscoveryService.DeltaAggregatedResources"), - random_, *stats_.rootScope(), + *stats_.rootScope(), Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info, std::move(custom_config_validators), std::move(backoff_strategy), makeOptRefFromPtr(xds_config_tracker_.get())); @@ -437,7 +437,7 @@ ClusterManagerImpl::ClusterManagerImpl( *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v3.AggregatedDiscoveryService." "StreamAggregatedResources"), - random_, *stats_.rootScope(), + *stats_.rootScope(), Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info, bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only(), std::move(custom_config_validators), std::move(backoff_strategy), @@ -452,7 +452,7 @@ ClusterManagerImpl::ClusterManagerImpl( main_thread_dispatcher, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), + *stats_.rootScope(), Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only(), std::move(custom_config_validators), std::move(backoff_strategy), diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 513d08fc22d9..48bd147168e4 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -157,13 +157,13 @@ TEST_P(DeltaSubscriptionNoGrpcStreamTest, NoGrpcStream) { if (GetParam() == LegacyOrUnified::Unified) { xds_context = std::make_shared( std::unique_ptr(async_client), dispatcher, *method_descriptor, - random, *stats_store.rootScope(), rate_limit_settings, local_info, false, + *stats_store.rootScope(), rate_limit_settings, local_info, false, std::make_unique>(), std::move(backoff_strategy), /*xds_config_tracker=*/XdsConfigTrackerOptRef()); } else { xds_context = std::make_shared( std::unique_ptr(async_client), dispatcher, *method_descriptor, - random, *stats_store.rootScope(), rate_limit_settings, local_info, + *stats_store.rootScope(), rate_limit_settings, local_info, std::make_unique>(), std::move(backoff_strategy), /*xds_config_tracker=*/XdsConfigTrackerOptRef()); } diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 50522f58da4a..bf1dfa0df637 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -53,13 +53,13 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { if (should_use_unified_) { xds_context_ = std::make_shared( std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, - random_, *stats_store_.rootScope(), rate_limit_settings_, local_info_, false, + *stats_store_.rootScope(), rate_limit_settings_, local_info_, false, std::make_unique>(), std::move(backoff_strategy), /*xds_config_tracker=*/XdsConfigTrackerOptRef()); } else { xds_context_ = std::make_shared( std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, - random_, *stats_store_.rootScope(), rate_limit_settings_, local_info_, + *stats_store_.rootScope(), rate_limit_settings_, local_info_, std::make_unique>(), std::move(backoff_strategy), /*xds_config_tracker=*/XdsConfigTrackerOptRef()); } diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 4ac23876ac6f..e8b41a7cc08c 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -64,7 +64,7 @@ class GrpcMuxImplTestBase : public testing::Test { local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), rate_limit_settings_, true, std::move(config_validators_), + *stats_.rootScope(), rate_limit_settings_, true, std::move(config_validators_), std::make_unique( SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), @@ -77,8 +77,7 @@ class GrpcMuxImplTestBase : public testing::Test { local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), custom_rate_limit_settings, true, - std::move(config_validators_), + *stats_.rootScope(), custom_rate_limit_settings, true, std::move(config_validators_), std::make_unique( SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), @@ -956,7 +955,7 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), rate_limit_settings_, true, + *stats_.rootScope(), rate_limit_settings_, true, std::make_unique>(), std::make_unique( SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, @@ -976,7 +975,7 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), rate_limit_settings_, true, + *stats_.rootScope(), rate_limit_settings_, true, std::make_unique>(), std::make_unique( SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, diff --git a/test/common/config/grpc_stream_test.cc b/test/common/config/grpc_stream_test.cc index f7b8f8ee59c1..259d2619cecd 100644 --- a/test/common/config/grpc_stream_test.cc +++ b/test/common/config/grpc_stream_test.cc @@ -36,8 +36,8 @@ class GrpcStreamTest : public testing::Test { &callbacks_, std::move(async_client_owner_), *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.endpoint.v3.EndpointDiscoveryService.StreamEndpoints"), - random_, dispatcher_, *stats_.rootScope(), std::move(backoff_strategy_), - rate_limit_settings_)) {} + dispatcher_, *stats_.rootScope(), std::move(backoff_strategy_), rate_limit_settings_)) { + } void setUpCustomBackoffRetryTimer(uint32_t retry_initial_delay_ms, absl::optional retry_max_delay_ms, @@ -54,8 +54,7 @@ class GrpcStreamTest : public testing::Test { &callbacks_, std::move(async_client_owner_), *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.endpoint.v3.EndpointDiscoveryService.StreamEndpoints"), - random_, dispatcher_, *stats_.rootScope(), std::move(backoff_strategy_), - rate_limit_settings_); + dispatcher_, *stats_.rootScope(), std::move(backoff_strategy_), rate_limit_settings_); } NiceMock dispatcher_; diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index f4fedb207c97..0b0da0302bc8 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -62,13 +62,13 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { if (should_use_unified_) { mux_ = std::make_shared( std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, - random_, *stats_store_.rootScope(), rate_limit_settings_, local_info_, true, + *stats_store_.rootScope(), rate_limit_settings_, local_info_, true, std::move(config_validators_), std::move(backoff_strategy), /*xds_config_tracker=*/XdsConfigTrackerOptRef()); } else { mux_ = std::make_shared( local_info_, std::unique_ptr(async_client_), dispatcher_, - *method_descriptor_, random_, *stats_store_.rootScope(), rate_limit_settings_, true, + *method_descriptor_, *stats_store_.rootScope(), rate_limit_settings_, true, std::move(config_validators_), std::move(backoff_strategy), /*xds_config_tracker=*/XdsConfigTrackerOptRef(), /*xds_resources_delegate=*/ diff --git a/test/common/config/new_grpc_mux_impl_test.cc b/test/common/config/new_grpc_mux_impl_test.cc index 472f494cf2f8..0abf1f935841 100644 --- a/test/common/config/new_grpc_mux_impl_test.cc +++ b/test/common/config/new_grpc_mux_impl_test.cc @@ -68,7 +68,7 @@ class NewGrpcMuxImplTestBase : public testing::TestWithParam { std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), rate_limit_settings_, local_info_, false, + *stats_.rootScope(), rate_limit_settings_, local_info_, false, std::move(config_validators_), std::move(backoff_strategy), /*xds_config_tracker=*/XdsConfigTrackerOptRef()); return; @@ -77,8 +77,8 @@ class NewGrpcMuxImplTestBase : public testing::TestWithParam { std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), rate_limit_settings_, local_info_, - std::move(config_validators_), std::move(backoff_strategy), + *stats_.rootScope(), rate_limit_settings_, local_info_, std::move(config_validators_), + std::move(backoff_strategy), /*xds_config_tracker=*/XdsConfigTrackerOptRef()); } diff --git a/test/common/config/xds_grpc_mux_impl_test.cc b/test/common/config/xds_grpc_mux_impl_test.cc index 343295f9609e..3946bad36e76 100644 --- a/test/common/config/xds_grpc_mux_impl_test.cc +++ b/test/common/config/xds_grpc_mux_impl_test.cc @@ -63,8 +63,7 @@ class GrpcMuxImplTestBase : public testing::Test { std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), rate_limit_settings_, local_info_, true, - std::move(config_validators_), + *stats_.rootScope(), rate_limit_settings_, local_info_, true, std::move(config_validators_), std::make_unique( SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), @@ -76,7 +75,7 @@ class GrpcMuxImplTestBase : public testing::Test { std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), custom_rate_limit_settings, local_info_, true, + *stats_.rootScope(), custom_rate_limit_settings, local_info_, true, std::move(config_validators_), std::make_unique( SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, @@ -903,7 +902,7 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), rate_limit_settings_, local_info_, true, + *stats_.rootScope(), rate_limit_settings_, local_info_, true, std::make_unique>(), std::make_unique( SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, @@ -921,7 +920,7 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), rate_limit_settings_, local_info_, true, + *stats_.rootScope(), rate_limit_settings_, local_info_, true, std::make_unique>(), nullptr, /*xds_config_tracker=*/XdsConfigTrackerOptRef()), EnvoyException, @@ -1034,7 +1033,7 @@ TEST_F(GrpcMuxImplTest, AllMuxesStateTest) { std::unique_ptr(), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, *stats_.rootScope(), rate_limit_settings_, local_info_, true, + *stats_.rootScope(), rate_limit_settings_, local_info_, true, std::make_unique>(), std::make_unique( SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_), diff --git a/test/extensions/clusters/eds/eds_speed_test.cc b/test/extensions/clusters/eds/eds_speed_test.cc index 5c7541931709..fc5af2ad8916 100644 --- a/test/extensions/clusters/eds/eds_speed_test.cc +++ b/test/extensions/clusters/eds/eds_speed_test.cc @@ -57,8 +57,7 @@ class EdsSpeedTest { std::unique_ptr(async_client_), server_context_.dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.endpoint.v3.EndpointDiscoveryService.StreamEndpoints"), - random_, scope_, {}, local_info_, true, std::move(config_validators_), - std::move(backoff_strategy), + scope_, {}, local_info_, true, std::move(config_validators_), std::move(backoff_strategy), /*xds_config_tracker=*/Config::XdsConfigTrackerOptRef())); } else { grpc_mux_.reset(new Config::GrpcMuxImpl( @@ -66,7 +65,7 @@ class EdsSpeedTest { server_context_.dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.endpoint.v3.EndpointDiscoveryService.StreamEndpoints"), - random_, scope_, {}, true, std::move(config_validators_), std::move(backoff_strategy), + scope_, {}, true, std::move(config_validators_), std::move(backoff_strategy), /*xds_config_tracker=*/Config::XdsConfigTrackerOptRef(), /*xds_resources_delegate=*/Config::XdsResourcesDelegateOptRef(), /*target_xds_authority=*/"")); From 03b77acaf75a1124038580c26d03d5a5c2b5d5e7 Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Mon, 1 May 2023 21:22:48 +0300 Subject: [PATCH 051/740] access_log: pass access log type parameter to evaluate function (#27063) pass access log type parameter to evaluate function Signed-off-by: ohadvano --- envoy/access_log/access_log.h | 3 +- source/common/access_log/access_log_impl.cc | 40 +++++++++++-------- source/common/access_log/access_log_impl.h | 22 +++++----- source/common/local_reply/local_reply.cc | 3 +- .../access_loggers/common/access_log_base.cc | 4 +- .../access_loggers/filters/cel/cel.cc | 9 +++-- .../access_loggers/filters/cel/cel.h | 3 +- .../wasm/wasm_access_log_impl.h | 7 ++-- .../common/access_log/access_log_impl_test.cc | 33 +++++++++------ .../common/access_log_base_test.cc | 2 +- .../grpc/http_grpc_access_log_impl_test.cc | 2 +- .../open_telemetry/access_log_impl_test.cc | 2 +- test/mocks/access_log/mocks.h | 2 +- 13 files changed, 75 insertions(+), 57 deletions(-) diff --git a/envoy/access_log/access_log.h b/envoy/access_log/access_log.h index a024091a524d..5a6676d04d78 100644 --- a/envoy/access_log/access_log.h +++ b/envoy/access_log/access_log.h @@ -71,7 +71,8 @@ class Filter { virtual bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const PURE; + const Http::ResponseTrailerMap& response_trailers, + AccessLogType access_log_type) const PURE; }; using FilterPtr = std::unique_ptr; diff --git a/source/common/access_log/access_log_impl.cc b/source/common/access_log/access_log_impl.cc index c780ed4d8400..e3fbe056d352 100644 --- a/source/common/access_log/access_log_impl.cc +++ b/source/common/access_log/access_log_impl.cc @@ -96,14 +96,14 @@ FilterPtr FilterFactory::fromProto(const envoy::config::accesslog::v3::AccessLog bool TraceableRequestFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&) const { + const Http::ResponseTrailerMap&, AccessLogType) const { const Tracing::Decision decision = Tracing::TracerUtility::shouldTraceRequest(info); return decision.traced && decision.reason == Tracing::Reason::ServiceForced; } bool StatusCodeFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&) const { + const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + AccessLogType) const { if (!info.responseCode()) { return compareAgainstValue(0ULL); } @@ -112,8 +112,8 @@ bool StatusCodeFilter::evaluate(const StreamInfo::StreamInfo& info, const Http:: } bool DurationFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&) const { + const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + AccessLogType) const { absl::optional duration = info.currentDuration(); if (!duration.has_value()) { return false; @@ -131,7 +131,7 @@ RuntimeFilter::RuntimeFilter(const envoy::config::accesslog::v3::RuntimeFilter& bool RuntimeFilter::evaluate(const StreamInfo::StreamInfo& stream_info, const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&) const { + const Http::ResponseTrailerMap&, AccessLogType) const { // This code is verbose to avoid preallocating a random number that is not needed. uint64_t random_value; if (use_independent_randomness_) { @@ -179,10 +179,12 @@ AndFilter::AndFilter(const envoy::config::accesslog::v3::AndFilter& config, bool OrFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const { + const Http::ResponseTrailerMap& response_trailers, + AccessLogType access_log_type) const { bool result = false; for (auto& filter : filters_) { - result |= filter->evaluate(info, request_headers, response_headers, response_trailers); + result |= filter->evaluate(info, request_headers, response_headers, response_trailers, + access_log_type); if (result) { break; @@ -195,10 +197,12 @@ bool OrFilter::evaluate(const StreamInfo::StreamInfo& info, bool AndFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const { + const Http::ResponseTrailerMap& response_trailers, + AccessLogType access_log_type) const { bool result = true; for (auto& filter : filters_) { - result &= filter->evaluate(info, request_headers, response_headers, response_trailers); + result &= filter->evaluate(info, request_headers, response_headers, response_trailers, + access_log_type); if (!result) { break; @@ -210,7 +214,7 @@ bool AndFilter::evaluate(const StreamInfo::StreamInfo& info, bool NotHealthCheckFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&) const { + const Http::ResponseTrailerMap&, AccessLogType) const { return !info.healthCheck(); } @@ -219,7 +223,8 @@ HeaderFilter::HeaderFilter(const envoy::config::accesslog::v3::HeaderFilter& con bool HeaderFilter::evaluate(const StreamInfo::StreamInfo&, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&) const { + const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + AccessLogType) const { return Http::HeaderUtility::matchHeaders(request_headers, *header_data_); } @@ -235,8 +240,8 @@ ResponseFlagFilter::ResponseFlagFilter( } bool ResponseFlagFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&) const { + const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + AccessLogType) const { if (configured_flags_ != 0) { return info.intersectResponseFlags(configured_flags_); } @@ -253,7 +258,8 @@ GrpcStatusFilter::GrpcStatusFilter(const envoy::config::accesslog::v3::GrpcStatu bool GrpcStatusFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const { + const Http::ResponseTrailerMap& response_trailers, + AccessLogType) const { Grpc::Status::GrpcStatus status = Grpc::Status::WellKnownGrpcStatus::Unknown; const auto& optional_status = @@ -294,8 +300,8 @@ MetadataFilter::MetadataFilter(const envoy::config::accesslog::v3::MetadataFilte } bool MetadataFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&) const { + const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + AccessLogType) const { const auto& value = Envoy::Config::Metadata::metadataValue(&info.dynamicMetadata(), filter_, path_); // If the key corresponds to a set value in dynamic metadata, return true if the value matches the diff --git a/source/common/access_log/access_log_impl.h b/source/common/access_log/access_log_impl.h index c4856306d5e2..eaf15ca75e00 100644 --- a/source/common/access_log/access_log_impl.h +++ b/source/common/access_log/access_log_impl.h @@ -62,7 +62,7 @@ class StatusCodeFilter : public ComparisonFilter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; }; /** @@ -77,7 +77,7 @@ class DurationFilter : public ComparisonFilter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; }; /** @@ -106,7 +106,7 @@ class AndFilter : public OperatorFilter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; }; /** @@ -120,7 +120,7 @@ class OrFilter : public OperatorFilter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; }; /** @@ -133,7 +133,7 @@ class NotHealthCheckFilter : public Filter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; }; /** @@ -144,7 +144,7 @@ class TraceableRequestFilter : public Filter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; }; /** @@ -158,7 +158,7 @@ class RuntimeFilter : public Filter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; private: Runtime::Loader& runtime_; @@ -178,7 +178,7 @@ class HeaderFilter : public Filter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; private: const Http::HeaderUtility::HeaderDataPtr header_data_; @@ -194,7 +194,7 @@ class ResponseFlagFilter : public Filter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; private: uint64_t configured_flags_{}; @@ -215,7 +215,7 @@ class GrpcStatusFilter : public Filter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; private: GrpcStatusHashSet statuses_; @@ -238,7 +238,7 @@ class MetadataFilter : public Filter { bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; private: Matchers::ValueMatcherConstSharedPtr present_matcher_; diff --git a/source/common/local_reply/local_reply.cc b/source/common/local_reply/local_reply.cc index f09298afc629..edc51b177be3 100644 --- a/source/common/local_reply/local_reply.cc +++ b/source/common/local_reply/local_reply.cc @@ -80,7 +80,8 @@ class ResponseMapper { BodyFormatter*& final_formatter) const { // If not matched, just bail out. if (filter_ == nullptr || - !filter_->evaluate(stream_info, request_headers, response_headers, response_trailers)) { + !filter_->evaluate(stream_info, request_headers, response_headers, response_trailers, + AccessLog::AccessLogType::NotSet)) { return false; } diff --git a/source/extensions/access_loggers/common/access_log_base.cc b/source/extensions/access_loggers/common/access_log_base.cc index 73efca022219..f90781a86125 100644 --- a/source/extensions/access_loggers/common/access_log_base.cc +++ b/source/extensions/access_loggers/common/access_log_base.cc @@ -22,8 +22,8 @@ void ImplBase::log(const Http::RequestHeaderMap* request_headers, if (!response_trailers) { response_trailers = Http::StaticEmptyHeaders::get().response_trailers.get(); } - if (filter_ && - !filter_->evaluate(stream_info, *request_headers, *response_headers, *response_trailers)) { + if (filter_ && !filter_->evaluate(stream_info, *request_headers, *response_headers, + *response_trailers, access_log_type)) { return; } return emitLog(*request_headers, *response_headers, *response_trailers, stream_info, diff --git a/source/extensions/access_loggers/filters/cel/cel.cc b/source/extensions/access_loggers/filters/cel/cel.cc index a9c87da2e1a0..f0c694396363 100644 --- a/source/extensions/access_loggers/filters/cel/cel.cc +++ b/source/extensions/access_loggers/filters/cel/cel.cc @@ -14,10 +14,11 @@ CELAccessLogExtensionFilter::CELAccessLogExtensionFilter( compiled_expr_ = Expr::createExpression(builder, parsed_expr_); } -bool CELAccessLogExtensionFilter::evaluate( - const StreamInfo::StreamInfo& stream_info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const { +bool CELAccessLogExtensionFilter::evaluate(const StreamInfo::StreamInfo& stream_info, + const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + AccessLog::AccessLogType) const { Protobuf::Arena arena; auto eval_status = Expr::evaluate(*compiled_expr_, arena, stream_info, &request_headers, &response_headers, &response_trailers); diff --git a/source/extensions/access_loggers/filters/cel/cel.h b/source/extensions/access_loggers/filters/cel/cel.h index 911f04106b60..dc851eadb748 100644 --- a/source/extensions/access_loggers/filters/cel/cel.h +++ b/source/extensions/access_loggers/filters/cel/cel.h @@ -22,7 +22,8 @@ class CELAccessLogExtensionFilter : public AccessLog::Filter { bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers) const override; + const Http::ResponseTrailerMap& response_trailers, + AccessLog::AccessLogType access_log_type) const override; private: const google::api::expr::v1alpha1::Expr parsed_expr_; diff --git a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h index 824df76b0644..120686773ac7 100644 --- a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h +++ b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h @@ -23,10 +23,11 @@ class WasmAccessLog : public AccessLog::Instance { void log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) override { + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) override { if (filter_ && request_headers && response_headers && response_trailers) { - if (!filter_->evaluate(stream_info, *request_headers, *response_headers, - *response_trailers)) { + if (!filter_->evaluate(stream_info, *request_headers, *response_headers, *response_trailers, + access_log_type)) { return; } } diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index f4e30d271540..fd97e9dd2956 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -691,24 +691,28 @@ TEST(AccessLogFilterTest, DurationWithRuntimeKey) { stream_info.end_time_ = stream_info.startTimeMonotonic() + std::chrono::microseconds(100000); EXPECT_CALL(runtime.snapshot_, getInteger("key", 1000000)).WillOnce(Return(1)); - EXPECT_TRUE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers)); + EXPECT_TRUE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers, + AccessLog::AccessLogType::NotSet)); EXPECT_CALL(runtime.snapshot_, getInteger("key", 1000000)).WillOnce(Return(1000)); - EXPECT_FALSE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers)); + EXPECT_FALSE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers, + AccessLog::AccessLogType::NotSet)); stream_info.end_time_ = stream_info.startTimeMonotonic() + std::chrono::microseconds(100000001000); EXPECT_CALL(runtime.snapshot_, getInteger("key", 1000000)).WillOnce(Return(100000000)); - EXPECT_TRUE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers)); + EXPECT_TRUE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers, + AccessLog::AccessLogType::NotSet)); stream_info.end_time_ = stream_info.startTimeMonotonic() + std::chrono::microseconds(10000); EXPECT_CALL(runtime.snapshot_, getInteger("key", 1000000)).WillOnce(Return(100000000)); - EXPECT_FALSE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers)); + EXPECT_FALSE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers, + AccessLog::AccessLogType::NotSet)); StreamInfo::MockStreamInfo mock_stream_info; EXPECT_CALL(mock_stream_info, currentDuration()).WillOnce(testing::Return(std::nullopt)); - EXPECT_FALSE( - filter.evaluate(mock_stream_info, request_headers, response_headers, response_trailers)); + EXPECT_FALSE(filter.evaluate(mock_stream_info, request_headers, response_headers, + response_trailers, AccessLog::AccessLogType::NotSet)); } TEST(AccessLogFilterTest, MidStreamDuration) { @@ -734,11 +738,11 @@ TEST(AccessLogFilterTest, MidStreamDuration) { .WillOnce(testing::Return(std::chrono::microseconds(1000))) // 1ms .WillOnce(testing::Return(std::chrono::microseconds(1000000))); // 1000ms - EXPECT_FALSE( - filter.evaluate(mock_stream_info, request_headers, response_headers, response_trailers)); + EXPECT_FALSE(filter.evaluate(mock_stream_info, request_headers, response_headers, + response_trailers, AccessLog::AccessLogType::NotSet)); - EXPECT_TRUE( - filter.evaluate(mock_stream_info, request_headers, response_headers, response_trailers)); + EXPECT_TRUE(filter.evaluate(mock_stream_info, request_headers, response_headers, + response_trailers, AccessLog::AccessLogType::NotSet)); } TEST(AccessLogFilterTest, StatusCodeWithRuntimeKey) { @@ -765,10 +769,12 @@ TEST(AccessLogFilterTest, StatusCodeWithRuntimeKey) { info.response_code_ = 400; EXPECT_CALL(runtime.snapshot_, getInteger("key", 300)).WillOnce(Return(350)); - EXPECT_TRUE(filter.evaluate(info, request_headers, response_headers, response_trailers)); + EXPECT_TRUE(filter.evaluate(info, request_headers, response_headers, response_trailers, + AccessLog::AccessLogType::NotSet)); EXPECT_CALL(runtime.snapshot_, getInteger("key", 300)).WillOnce(Return(500)); - EXPECT_FALSE(filter.evaluate(info, request_headers, response_headers, response_trailers)); + EXPECT_FALSE(filter.evaluate(info, request_headers, response_headers, response_trailers, + AccessLog::AccessLogType::NotSet)); } TEST_F(AccessLogImplTest, StatusCodeLessThan) { @@ -1545,7 +1551,8 @@ class SampleExtensionFilter : public Filter { // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo&, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&) const override { + const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + AccessLogType) const override { if (current_++ == 0) { return true; } diff --git a/test/extensions/access_loggers/common/access_log_base_test.cc b/test/extensions/access_loggers/common/access_log_base_test.cc index fbcd559b4754..349738d398ba 100644 --- a/test/extensions/access_loggers/common/access_log_base_test.cc +++ b/test/extensions/access_loggers/common/access_log_base_test.cc @@ -44,7 +44,7 @@ TEST(AccessLogBaseTest, FilterReject) { StreamInfo::MockStreamInfo stream_info; std::unique_ptr filter = std::make_unique(); - EXPECT_CALL(*filter, evaluate(_, _, _, _)).WillOnce(Return(false)); + EXPECT_CALL(*filter, evaluate(_, _, _, _, _)).WillOnce(Return(false)); TestImpl logger(std::move(filter)); EXPECT_EQ(logger.count(), 0); logger.log(nullptr, nullptr, nullptr, stream_info); diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index 1b9c45708f6c..c760a705b4be 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -83,7 +83,7 @@ TEST(HttpGrpcAccessLog, TlsLifetimeCheck) { class HttpGrpcAccessLogTest : public testing::Test { public: void init() { - ON_CALL(*filter_, evaluate(_, _, _, _)).WillByDefault(Return(true)); + ON_CALL(*filter_, evaluate(_, _, _, _, _)).WillByDefault(Return(true)); config_.mutable_common_config()->set_log_name("hello_log"); config_.mutable_common_config()->add_filter_state_objects_to_log("string_accessor"); config_.mutable_common_config()->add_filter_state_objects_to_log("uint32_accessor"); diff --git a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc index ed53487c37b8..0bb888df76df 100644 --- a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc @@ -64,7 +64,7 @@ class MockGrpcAccessLoggerCache : public GrpcAccessLoggerCache { class AccessLogTest : public testing::Test { public: AccessLogPtr makeAccessLog(const AnyValue& body_config, const KeyValueList& attributes_config) { - ON_CALL(*filter_, evaluate(_, _, _, _)).WillByDefault(Return(true)); + ON_CALL(*filter_, evaluate(_, _, _, _, _)).WillByDefault(Return(true)); *config_.mutable_body() = body_config; *config_.mutable_attributes() = attributes_config; config_.mutable_common_config()->set_log_name("test_log"); diff --git a/test/mocks/access_log/mocks.h b/test/mocks/access_log/mocks.h index db6809530c59..c4d5ea6837d1 100644 --- a/test/mocks/access_log/mocks.h +++ b/test/mocks/access_log/mocks.h @@ -30,7 +30,7 @@ class MockFilter : public Filter { MOCK_METHOD(bool, evaluate, (const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers), + const Http::ResponseTrailerMap& response_trailers, AccessLogType access_log_type), (const)); }; From 456580900aa69aa697fc6f1c8fca2e2f45681796 Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Mon, 1 May 2023 21:24:03 +0300 Subject: [PATCH 052/740] access_log: use AccessLogType::NotSet instead of default value (#27058) * use AccessLogType::NotSet instead of default parameter value * pass access log type parameter in wasm extension Signed-off-by: ohadvano --- .../filters/http/source/golang_filter.h | 3 +- envoy/access_log/access_log.h | 3 +- source/common/quic/quic_stats_gatherer.cc | 3 +- .../access_loggers/common/access_log_base.h | 13 +- .../common/file_access_log_impl.h | 11 +- .../grpc/http_grpc_access_log_impl.h | 11 +- .../grpc/tcp_grpc_access_log_impl.h | 11 +- .../wasm/wasm_access_log_impl.h | 2 +- source/extensions/common/wasm/context.h | 2 +- source/extensions/common/wasm/wasm.cc | 5 +- source/extensions/common/wasm/wasm.h | 2 +- .../filters/http/composite/filter.h | 2 +- .../extensions/filters/http/tap/tap_filter.h | 2 +- .../network/thrift_proxy/conn_manager.cc | 3 +- .../filters/udp/udp_proxy/udp_proxy_filter.cc | 6 +- .../active_stream_listener_base.cc | 2 +- .../common/access_log/access_log_impl_test.cc | 246 ++++++++++++------ .../common/access_log_base_test.cc | 6 +- .../access_loggers/file/config_test.cc | 3 +- .../grpc/http_grpc_access_log_impl_test.cc | 38 +-- .../open_telemetry/access_log_impl_test.cc | 6 +- .../access_loggers/stream/stream_test_base.h | 3 +- .../access_loggers/wasm/config_test.cc | 21 +- .../http/common/fuzz/http_filter_fuzzer.h | 3 +- .../filters/http/composite/filter_test.cc | 3 +- .../filters/http/tap/tap_filter_test.cc | 6 +- .../filters/http/wasm/wasm_filter_test.cc | 24 +- 27 files changed, 275 insertions(+), 165 deletions(-) diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h index 3edc149d64d6..610b229f1337 100644 --- a/contrib/golang/filters/http/source/golang_filter.h +++ b/contrib/golang/filters/http/source/golang_filter.h @@ -154,8 +154,7 @@ class Filter : public Http::StreamFilter, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, const StreamInfo::StreamInfo& stream_info, - Envoy::AccessLog::AccessLogType access_log_type = - Envoy::AccessLog::AccessLogType::NotSet) override; + Envoy::AccessLog::AccessLogType access_log_type) override; void onStreamComplete() override {} diff --git a/envoy/access_log/access_log.h b/envoy/access_log/access_log.h index 5a6676d04d78..ff0caadad5ca 100644 --- a/envoy/access_log/access_log.h +++ b/envoy/access_log/access_log.h @@ -97,8 +97,7 @@ class Instance { virtual void log(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLogType access_log_type = AccessLogType::NotSet) PURE; + const StreamInfo::StreamInfo& stream_info, AccessLogType access_log_type) PURE; }; using InstanceSharedPtr = std::shared_ptr; diff --git a/source/common/quic/quic_stats_gatherer.cc b/source/common/quic/quic_stats_gatherer.cc index be4b83ba70b3..32552993751f 100644 --- a/source/common/quic/quic_stats_gatherer.cc +++ b/source/common/quic/quic_stats_gatherer.cc @@ -25,7 +25,8 @@ void QuicStatsGatherer::maybeDoDeferredLog(bool record_ack_timing) { const Http::ResponseHeaderMap* response_headers = response_header_map_.get(); const Http::ResponseTrailerMap* response_trailers = response_trailer_map_.get(); for (const AccessLog::InstanceSharedPtr& log_handler : access_log_handlers_) { - log_handler->log(request_headers, response_headers, response_trailers, *stream_info_); + log_handler->log(request_headers, response_headers, response_trailers, *stream_info_, + AccessLog::AccessLogType::NotSet); } } diff --git a/source/extensions/access_loggers/common/access_log_base.h b/source/extensions/access_loggers/common/access_log_base.h index 648d73226eb7..c15b6003c310 100644 --- a/source/extensions/access_loggers/common/access_log_base.h +++ b/source/extensions/access_loggers/common/access_log_base.h @@ -30,7 +30,7 @@ class ImplBase : public AccessLog::Instance { const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; + AccessLog::AccessLogType access_log_type) override; private: /** @@ -41,12 +41,11 @@ class ImplBase : public AccessLog::Instance { * @param stream_info supplies additional information about the request not * contained in the request headers. */ - virtual void - emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) PURE; + virtual void emitLog(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) PURE; AccessLog::FilterPtr filter_; }; diff --git a/source/extensions/access_loggers/common/file_access_log_impl.h b/source/extensions/access_loggers/common/file_access_log_impl.h index 45a8be54ab2d..4b866879fbbb 100644 --- a/source/extensions/access_loggers/common/file_access_log_impl.h +++ b/source/extensions/access_loggers/common/file_access_log_impl.h @@ -19,12 +19,11 @@ class FileAccessLog : public Common::ImplBase { private: // Common::ImplBase - void - emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; + void emitLog(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) override; AccessLog::AccessLogFileSharedPtr log_file_; Formatter::FormatterPtr formatter_; diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h index 47411915f38a..f11d22cbd61c 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h @@ -44,12 +44,11 @@ class HttpGrpcAccessLog : public Common::ImplBase { }; // Common::ImplBase - void - emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; + void emitLog(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) override; const HttpGrpcAccessLogConfigConstSharedPtr config_; const ThreadLocal::SlotPtr tls_slot_; diff --git a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h index acd552e78fab..71b5dfd81f9b 100644 --- a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h @@ -43,12 +43,11 @@ class TcpGrpcAccessLog : public Common::ImplBase { }; // Common::ImplBase - void - emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; + void emitLog(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) override; const TcpGrpcAccessLogConfigConstSharedPtr config_; const ThreadLocal::SlotPtr tls_slot_; diff --git a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h index 120686773ac7..21dd30273eeb 100644 --- a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h +++ b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h @@ -41,7 +41,7 @@ class WasmAccessLog : public AccessLog::Instance { } if (handle->wasmHandle()) { handle->wasmHandle()->wasm()->log(plugin_, request_headers, response_headers, - response_trailers, stream_info); + response_trailers, stream_info, access_log_type); } } diff --git a/source/extensions/common/wasm/context.h b/source/extensions/common/wasm/context.h index e093df09c2a8..51778de53881 100644 --- a/source/extensions/common/wasm/context.h +++ b/source/extensions/common/wasm/context.h @@ -152,7 +152,7 @@ class Context : public proxy_wasm::ContextBase, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; + AccessLog::AccessLogType access_log_type) override; uint32_t getLogLevel() override; diff --git a/source/extensions/common/wasm/wasm.cc b/source/extensions/common/wasm/wasm.cc index 9926fb3cbb8d..17d40d003a9c 100644 --- a/source/extensions/common/wasm/wasm.cc +++ b/source/extensions/common/wasm/wasm.cc @@ -229,9 +229,10 @@ ContextBase* Wasm::createVmContext() { return new Context(this); } void Wasm::log(const PluginSharedPtr& plugin, const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) { + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { auto context = getRootContext(plugin, true); - context->log(request_headers, response_headers, response_trailers, stream_info); + context->log(request_headers, response_headers, response_trailers, stream_info, access_log_type); } void Wasm::onStatsUpdate(const PluginSharedPtr& plugin, Envoy::Stats::MetricSnapshot& snapshot) { diff --git a/source/extensions/common/wasm/wasm.h b/source/extensions/common/wasm/wasm.h index 04d5338407c1..dc2b5d704d16 100644 --- a/source/extensions/common/wasm/wasm.h +++ b/source/extensions/common/wasm/wasm.h @@ -69,7 +69,7 @@ class Wasm : public WasmBase, Logger::Loggable { void log(const PluginSharedPtr& plugin, const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info); + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType access_log_type); void onStatsUpdate(const PluginSharedPtr& plugin, Envoy::Stats::MetricSnapshot& snapshot); diff --git a/source/extensions/filters/http/composite/filter.h b/source/extensions/filters/http/composite/filter.h index ea6bfc35c7f7..e76de9750b84 100644 --- a/source/extensions/filters/http/composite/filter.h +++ b/source/extensions/filters/http/composite/filter.h @@ -71,7 +71,7 @@ class Filter : public Http::StreamFilter, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override { + AccessLog::AccessLogType access_log_type) override { for (const auto& log : access_loggers_) { log->log(request_headers, response_headers, response_trailers, stream_info, access_log_type); } diff --git a/source/extensions/filters/http/tap/tap_filter.h b/source/extensions/filters/http/tap/tap_filter.h index e18334fcdfee..6621327645ca 100644 --- a/source/extensions/filters/http/tap/tap_filter.h +++ b/source/extensions/filters/http/tap/tap_filter.h @@ -111,7 +111,7 @@ class Filter : public Http::StreamFilter, public AccessLog::Instance { const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers, const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type = AccessLog::AccessLogType::NotSet) override; + AccessLog::AccessLogType access_log_type) override; private: FilterConfigSharedPtr config_; diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.cc b/source/extensions/filters/network/thrift_proxy/conn_manager.cc index 375648c87b6b..301066f93860 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.cc +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.cc @@ -55,7 +55,8 @@ void ConnectionManager::emitLogEntry(const Http::RequestHeaderMap* request_heade const Http::ResponseHeaderMap* response_headers, const StreamInfo::StreamInfo& stream_info) { for (const auto& access_log : config_.accessLogs()) { - access_log->log(request_headers, response_headers, nullptr, stream_info); + access_log->log(request_headers, response_headers, nullptr, stream_info, + AccessLog::AccessLogType::NotSet); } } diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc index 32e52135853e..903e46fc1ec3 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc @@ -30,7 +30,8 @@ UdpProxyFilter::~UdpProxyFilter() { if (!config_->proxyAccessLogs().empty()) { fillProxyStreamInfo(); for (const auto& access_log : config_->proxyAccessLogs()) { - access_log->log(nullptr, nullptr, nullptr, udp_proxy_stats_.value()); + access_log->log(nullptr, nullptr, nullptr, udp_proxy_stats_.value(), + AccessLog::AccessLogType::NotSet); } } } @@ -297,7 +298,8 @@ UdpProxyFilter::ActiveSession::~ActiveSession() { if (!cluster_.filter_.config_->sessionAccessLogs().empty()) { fillSessionStreamInfo(); for (const auto& access_log : cluster_.filter_.config_->sessionAccessLogs()) { - access_log->log(nullptr, nullptr, nullptr, udp_session_stats_.value()); + access_log->log(nullptr, nullptr, nullptr, udp_session_stats_.value(), + AccessLog::AccessLogType::NotSet); } } } diff --git a/source/extensions/listener_managers/listener_manager/active_stream_listener_base.cc b/source/extensions/listener_managers/listener_manager/active_stream_listener_base.cc index a281668b2e75..bded60cc1f39 100644 --- a/source/extensions/listener_managers/listener_manager/active_stream_listener_base.cc +++ b/source/extensions/listener_managers/listener_manager/active_stream_listener_base.cc @@ -20,7 +20,7 @@ void ActiveStreamListenerBase::emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info) { stream_info.onRequestComplete(); for (const auto& access_log : config.accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, stream_info); + access_log->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } } diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index fd97e9dd2956..96f5b1519351 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -92,7 +92,8 @@ name: accesslog request_headers_.addCopy(Http::Headers::get().Host, "host"); request_headers_.addCopy(Http::Headers::get().ForwardedFor, "x.x.x.x"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 UF 1 2 3 - \"x.x.x.x\" " "\"user-agent-set\" \"id\" \"host\" \"-\"\n", output_); @@ -115,7 +116,8 @@ name: accesslog Upstream::makeTestHostDescription(cluster, "tcp://10.0.0.5:1234", simTime())); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::DownstreamConnectionTermination); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 DC 1 2 3 - \"-\" \"-\" \"-\" \"-\" " "\"10.0.0.5:1234\"\n", output_); @@ -142,7 +144,8 @@ name: accesslog request_headers_.addCopy(Http::Headers::get().Host, "host"); request_headers_.addCopy(Http::Headers::get().ForwardedFor, "x.x.x.x"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 UF route-test-name 1 2 0 0 3 - " "\"x.x.x.x\" " @@ -179,7 +182,8 @@ name: accesslog // response trailers: // response_trailer_key: response_trailer_val - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ(output_, "52 38 40"); } @@ -197,7 +201,8 @@ name: accesslog EXPECT_CALL(*file_, write(_)); response_headers_.addCopy(Http::Headers::get().EnvoyUpstreamServiceTime, "999"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 - 1 2 3 999 \"-\" \"-\" \"-\" \"-\" " "\"-\"\n", output_); @@ -214,7 +219,8 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ( "[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 - 1 2 3 - \"-\" \"-\" \"-\" \"-\" \"-\"\n", output_); @@ -235,7 +241,8 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 - 1 2 3 - \"-\" \"-\" \"-\" \"-\" " "\"10.0.0.5:1234\"\n", output_); @@ -267,10 +274,12 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.response_code_ = 200; - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, WithFilterHit) { @@ -305,15 +314,18 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(3); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.response_code_ = 500; - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.response_code_ = 200; stream_info_.end_time_ = stream_info_.startTimeMonotonic() + std::chrono::microseconds(1001000000000000); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, RuntimeFilter) { @@ -335,25 +347,29 @@ name: accesslog EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 0, 42, 100)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_CALL(context_.api_.random_, random()).WillOnce(Return(43)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 0, 43, 100)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.stream_id_provider_ = std::make_shared("000000ff-0000-0000-0000-000000000000"); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 0, 55, 100)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 0, 55, 100)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, RuntimeFilterV2) { @@ -378,25 +394,29 @@ name: accesslog EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 42, 10000)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_CALL(context_.api_.random_, random()).WillOnce(Return(43)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 43, 10000)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.stream_id_provider_ = std::make_shared("000000ff-0000-0000-0000-000000000000"); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 255, 10000)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 255, 10000)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, RuntimeFilterV2IndependentRandomness) { @@ -422,13 +442,15 @@ name: accesslog EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 42, 1000000)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_CALL(context_.api_.random_, random()).WillOnce(Return(43)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 43, 1000000)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, PathRewrite) { @@ -444,7 +466,8 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET /bar HTTP/1.1\" 0 - 1 2 3 - \"-\" \"-\" \"-\" \"-\" " "\"-\"\n", output_); @@ -466,7 +489,8 @@ name: accesslog stream_info_.health_check_request_ = true; EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_); + log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, HealthCheckFalse) { @@ -484,7 +508,8 @@ name: accesslog Http::TestRequestHeaderMapImpl header_map{}; EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, RequestTracing) { @@ -504,19 +529,22 @@ name: accesslog { stream_info_.setTraceReason(Tracing::Reason::ServiceForced); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } { stream_info_.setTraceReason(Tracing::Reason::NotTraceable); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } { stream_info_.setTraceReason(Tracing::Reason::Sampling); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } } @@ -577,14 +605,16 @@ name: accesslog EXPECT_CALL(*file_, write(_)); Http::TestRequestHeaderMapImpl header_map{{"user-agent", "NOT/Envoy/HC"}}; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_); + log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } { EXPECT_CALL(*file_, write(_)).Times(0); Http::TestRequestHeaderMapImpl header_map{}; stream_info_.health_check_request_ = true; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_); + log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } } @@ -613,13 +643,15 @@ name: accesslog EXPECT_CALL(*file_, write(_)); Http::TestRequestHeaderMapImpl header_map{{"user-agent", "NOT/Envoy/HC"}}; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_); + log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } { EXPECT_CALL(*file_, write(_)); Http::TestRequestHeaderMapImpl header_map{{"user-agent", "Envoy/HC"}}; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_); + log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } } @@ -656,7 +688,8 @@ name: accesslog EXPECT_CALL(*file_, write(_)); Http::TestRequestHeaderMapImpl header_map{}; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_); + log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } { @@ -664,7 +697,8 @@ name: accesslog Http::TestRequestHeaderMapImpl header_map{}; stream_info_.health_check_request_ = true; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_); + log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } } @@ -797,12 +831,14 @@ name: accesslog stream_info_.response_code_ = 499; EXPECT_CALL(runtime_.snapshot_, getInteger("hello", 499)).WillOnce(Return(499)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.response_code_ = 500; EXPECT_CALL(runtime_.snapshot_, getInteger("hello", 499)).WillOnce(Return(499)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, HeaderPresence) { @@ -820,11 +856,13 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.addCopy("test-header", "present"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, HeaderExactMatch) { @@ -845,16 +883,19 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.addCopy("test-header", "exact-match-value"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "not-exact-match-value"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, HeaderRegexMatch) { @@ -875,21 +916,25 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.addCopy("test-header", "123"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "1234"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "123.456"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, HeaderRangeMatch) { @@ -910,31 +955,37 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.addCopy("test-header", "-1"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "0"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "somestring"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "10.9"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "-1somestring"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, ResponseFlagFilterAnyFlag) { @@ -950,11 +1001,13 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, ResponseFlagFilterSpecificFlag) { @@ -972,15 +1025,18 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::UpstreamOverflow); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, ResponseFlagFilterSeveralFlags) { @@ -999,15 +1055,18 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::UpstreamOverflow); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, ResponseFlagFilterAllFlagsInPGV) { @@ -1056,7 +1115,8 @@ name: accesslog TestStreamInfo stream_info(time_source_); stream_info.setResponseFlag(response_flag); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, + AccessLog::AccessLogType::NotSet); } } @@ -1160,7 +1220,8 @@ name: accesslog { EXPECT_CALL(*file_, write(_)); response_trailers_.addCopy(Http::Headers::get().GrpcStatus, "0"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ("OK 0 OK OK OK 0\n", output_); response_trailers_.remove(Http::Headers::get().GrpcStatus); } @@ -1168,7 +1229,8 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); response_headers_.addCopy(Http::Headers::get().GrpcStatus, "1"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ("Canceled 1 Canceled Canceled CANCELLED 1\n", output_); response_headers_.remove(Http::Headers::get().GrpcStatus); } @@ -1176,7 +1238,8 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); response_headers_.addCopy(Http::Headers::get().GrpcStatus, "-1"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ("-1 -1 -1 -1 -1 -1\n", output_); response_headers_.remove(Http::Headers::get().GrpcStatus); } @@ -1208,7 +1271,8 @@ name: accesslog EXPECT_CALL(*file_, write(_)); response_trailers_.addCopy(Http::Headers::get().GrpcStatus, std::to_string(i)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); response_trailers_.remove(Http::Headers::get().GrpcStatus); } } @@ -1247,7 +1311,8 @@ name: accesslog response_trailers_.addCopy(Http::Headers::get().GrpcStatus, "1"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, GrpcStatusFilterHttpCodes) { @@ -1278,7 +1343,8 @@ name: accesslog parseAccessLogFromV3Yaml(fmt::format(yaml_template, response_string)), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } } @@ -1298,7 +1364,8 @@ name: accesslog AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, GrpcStatusFilterExclude) { @@ -1321,7 +1388,8 @@ name: accesslog EXPECT_CALL(*file_, write(_)).Times(i == 0 ? 0 : 1); response_trailers_.addCopy(Http::Headers::get().GrpcStatus, std::to_string(i)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); response_trailers_.remove(Http::Headers::get().GrpcStatus); } } @@ -1345,7 +1413,8 @@ name: accesslog response_trailers_.addCopy(Http::Headers::get().GrpcStatus, "0"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, GrpcStatusFilterHeader) { @@ -1366,7 +1435,8 @@ name: accesslog EXPECT_CALL(*file_, write(_)); response_headers_.addCopy(Http::Headers::get().GrpcStatus, "0"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, MetadataFilter) { @@ -1404,7 +1474,8 @@ name: accesslog EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, + AccessLog::AccessLogType::NotSet); fields_c["c"].set_bool_value(false); EXPECT_CALL(*file_, write(_)).Times(0); @@ -1433,7 +1504,8 @@ name: accesslog // If no matcher is set, then expect no logs. EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, MetadataFilterNoKey) { @@ -1484,13 +1556,15 @@ name: accesslog AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(default_false_yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - default_false_log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info); + default_false_log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, + AccessLog::AccessLogType::NotSet); const InstanceSharedPtr default_true_log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(default_true_yaml), context_); EXPECT_CALL(*file_, write(_)); - default_true_log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info); + default_true_log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, + AccessLog::AccessLogType::NotSet); } class TestHeaderFilterFactory : public ExtensionFilterFactory { @@ -1535,11 +1609,13 @@ name: accesslog InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.addCopy("test-header", "foo/bar"); EXPECT_CALL(*file_, write(_)); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } /** @@ -1613,13 +1689,16 @@ name: accesslog InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); // For rate=5 expect 1st request to be recorded, 2nd-5th skipped, and 6th recorded. EXPECT_CALL(*file_, write(_)); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); for (int i = 0; i <= 3; ++i) { EXPECT_CALL(*file_, write(_)).Times(0); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } EXPECT_CALL(*file_, write(_)); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, UnregisteredExtensionFilter) { @@ -1678,11 +1757,13 @@ name: accesslog request_headers_.addCopy("log", "true"); stream_info_.response_code_ = 404; EXPECT_CALL(*file_, write(_)); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); request_headers_.remove("log"); EXPECT_CALL(*file_, write(_)).Times(0); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, CelExtensionFilterExpressionError) { @@ -1702,7 +1783,8 @@ name: accesslog InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_F(AccessLogImplTest, CelExtensionFilterExpressionUnparsable) { diff --git a/test/extensions/access_loggers/common/access_log_base_test.cc b/test/extensions/access_loggers/common/access_log_base_test.cc index 349738d398ba..72d88b87c3eb 100644 --- a/test/extensions/access_loggers/common/access_log_base_test.cc +++ b/test/extensions/access_loggers/common/access_log_base_test.cc @@ -25,7 +25,7 @@ class TestImpl : public ImplBase { private: void emitLog(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - AccessLog::AccessLogType = AccessLog::AccessLogType::NotSet) override { + AccessLog::AccessLogType) override { count_++; } @@ -36,7 +36,7 @@ TEST(AccessLogBaseTest, NoFilter) { StreamInfo::MockStreamInfo stream_info; TestImpl logger(nullptr); EXPECT_EQ(logger.count(), 0); - logger.log(nullptr, nullptr, nullptr, stream_info); + logger.log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); EXPECT_EQ(logger.count(), 1); } @@ -47,7 +47,7 @@ TEST(AccessLogBaseTest, FilterReject) { EXPECT_CALL(*filter, evaluate(_, _, _, _, _)).WillOnce(Return(false)); TestImpl logger(std::move(filter)); EXPECT_EQ(logger.count(), 0); - logger.log(nullptr, nullptr, nullptr, stream_info); + logger.log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); EXPECT_EQ(logger.count(), 0); } diff --git a/test/extensions/access_loggers/file/config_test.cc b/test/extensions/access_loggers/file/config_test.cc index 1644c756c3f8..84b55e72013e 100644 --- a/test/extensions/access_loggers/file/config_test.cc +++ b/test/extensions/access_loggers/file/config_test.cc @@ -73,7 +73,8 @@ class FileAccessLogTest : public testing::Test { EXPECT_EQ(got, expected); } })); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } Http::TestRequestHeaderMapImpl request_headers_{{":method", "GET"}, {":path", "/bar/foo"}}; diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index c760a705b4be..ad8ac6fc267f 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -155,7 +155,8 @@ class HttpGrpcAccessLogTest : public testing::Test { response: {{}} )EOF", request_method, request_method.length() + 7)); - access_log_->log(&request_headers, nullptr, nullptr, stream_info); + access_log_->log(&request_headers, nullptr, nullptr, stream_info, + AccessLog::AccessLogType::NotSet); } Stats::IsolatedStoreImpl scope_; @@ -247,7 +248,7 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } { @@ -285,7 +286,7 @@ response: {} request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } { @@ -404,7 +405,8 @@ protocol_version: HTTP10 response_body_bytes: 20 response_code_details: "via_upstream" )EOF"); - access_log_->log(&request_headers, &response_headers, nullptr, stream_info); + access_log_->log(&request_headers, &response_headers, nullptr, stream_info, + AccessLog::AccessLogType::NotSet); } { @@ -445,7 +447,8 @@ protocol_version: HTTP10 request_headers_bytes: 16 response: {} )EOF"); - access_log_->log(&request_headers, nullptr, nullptr, stream_info); + access_log_->log(&request_headers, nullptr, nullptr, stream_info, + AccessLog::AccessLogType::NotSet); } { @@ -517,7 +520,8 @@ response: {} request_headers_bytes: 16 response: {} )EOF"); - access_log_->log(&request_headers, nullptr, nullptr, stream_info); + access_log_->log(&request_headers, nullptr, nullptr, stream_info, + AccessLog::AccessLogType::NotSet); } // TLSv1.2 @@ -573,7 +577,7 @@ response: {} request_method: "METHOD_UNSPECIFIED" response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } // TLSv1.1 @@ -629,7 +633,7 @@ response: {} request_method: "METHOD_UNSPECIFIED" response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } // TLSv1 @@ -685,7 +689,7 @@ response: {} request_method: "METHOD_UNSPECIFIED" response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } // Unknown TLS version (TLSv1.4) @@ -741,7 +745,7 @@ response: {} request_method: "METHOD_UNSPECIFIED" response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } // Intermediate log entry. @@ -809,7 +813,7 @@ response: {} request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } } @@ -905,7 +909,8 @@ TEST_F(HttpGrpcAccessLogTest, MarshallingAdditionalHeaders) { "x-logged-trailer": "value,response_trailer_value" "x-empty-trailer": "" )EOF"); - access_log_->log(&request_headers, &response_headers, &response_trailers, stream_info); + access_log_->log(&request_headers, &response_headers, &response_trailers, stream_info, + AccessLog::AccessLogType::NotSet); } } @@ -988,7 +993,8 @@ TEST_F(HttpGrpcAccessLogTest, SanitizeUTF8) { "x-trailer": "{0},{0}" )EOF", "prefix!!suffix")); - access_log_->log(&request_headers, &response_headers, &response_trailers, stream_info); + access_log_->log(&request_headers, &response_headers, &response_trailers, stream_info, + AccessLog::AccessLogType::NotSet); } } @@ -1050,7 +1056,7 @@ tag: ltag request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } TEST_F(HttpGrpcAccessLogTest, CustomTagTestMetadata) { @@ -1109,7 +1115,7 @@ tag: mtag request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } TEST_F(HttpGrpcAccessLogTest, CustomTagTestMetadataDefaultValue) { @@ -1165,7 +1171,7 @@ tag: mtag request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info); + access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); } } // namespace diff --git a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc index 0bb888df76df..75ad47a32c04 100644 --- a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc @@ -145,7 +145,8 @@ TEST_F(AccessLogTest, Marshalling) { value: string_value: "10" )EOF"); - access_log->log(&request_headers, &response_headers, nullptr, stream_info); + access_log->log(&request_headers, &response_headers, nullptr, stream_info, + Envoy::AccessLog::AccessLogType::NotSet); } // Test log with empty config. @@ -159,7 +160,8 @@ TEST_F(AccessLogTest, EmptyConfig) { expectLog(R"EOF( time_unix_nano: 3600000000000 )EOF"); - access_log->log(&request_headers, &response_headers, nullptr, stream_info); + access_log->log(&request_headers, &response_headers, nullptr, stream_info, + Envoy::AccessLog::AccessLogType::NotSet); } } // namespace diff --git a/test/extensions/access_loggers/stream/stream_test_base.h b/test/extensions/access_loggers/stream/stream_test_base.h index 021c26766f3d..080579e91b3d 100644 --- a/test/extensions/access_loggers/stream/stream_test_base.h +++ b/test/extensions/access_loggers/stream/stream_test_base.h @@ -52,7 +52,8 @@ class StreamAccessLogTest : public testing::Test { EXPECT_EQ(got, expected); } })); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::NotSet); } Http::TestRequestHeaderMapImpl request_headers_{{":method", "GET"}, {":path", "/bar/foo"}}; diff --git a/test/extensions/access_loggers/wasm/config_test.cc b/test/extensions/access_loggers/wasm/config_test.cc index bf20daf1f121..1ba26b57e53d 100644 --- a/test/extensions/access_loggers/wasm/config_test.cc +++ b/test/extensions/access_loggers/wasm/config_test.cc @@ -121,12 +121,14 @@ TEST_P(WasmAccessLogConfigTest, CreateWasmFromWASM) { Http::TestRequestHeaderMapImpl request_header; Http::TestResponseHeaderMapImpl response_header; Http::TestResponseTrailerMapImpl response_trailer; - instance->log(&request_header, &response_header, &response_trailer, log_stream_info_); + instance->log(&request_header, &response_header, &response_trailer, log_stream_info_, + AccessLog::AccessLogType::NotSet); filter = std::make_unique>(); AccessLog::InstanceSharedPtr filter_instance = factory->createAccessLogInstance(config, std::move(filter), context_); - filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info_); + filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_P(WasmAccessLogConfigTest, YamlLoadFromFileWasmInvalidConfig) { @@ -180,7 +182,8 @@ TEST_P(WasmAccessLogConfigTest, YamlLoadFromFileWasmInvalidConfig) { AccessLog::InstanceSharedPtr filter_instance = factory->createAccessLogInstance(proto_config, nullptr, context_); filter_instance = factory->createAccessLogInstance(proto_config, nullptr, context_); - filter_instance->log(nullptr, nullptr, nullptr, log_stream_info_); + filter_instance->log(nullptr, nullptr, nullptr, log_stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_P(WasmAccessLogConfigTest, YamlLoadFromRemoteWasmCreateFilter) { @@ -225,7 +228,8 @@ TEST_P(WasmAccessLogConfigTest, YamlLoadFromRemoteWasmCreateFilter) { })); AccessLog::InstanceSharedPtr filter_instance = factory.createAccessLogInstance(proto_config, nullptr, context_); - filter_instance->log(nullptr, nullptr, nullptr, log_stream_info_); + filter_instance->log(nullptr, nullptr, nullptr, log_stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); auto response = Http::ResponseMessagePtr{new Http::ResponseMessageImpl( @@ -233,7 +237,8 @@ TEST_P(WasmAccessLogConfigTest, YamlLoadFromRemoteWasmCreateFilter) { response->body().add(code); async_callbacks->onSuccess(request, std::move(response)); EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); - filter_instance->log(nullptr, nullptr, nullptr, log_stream_info_); + filter_instance->log(nullptr, nullptr, nullptr, log_stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_P(WasmAccessLogConfigTest, FailedToGetThreadLocalPlugin) { @@ -274,11 +279,13 @@ TEST_P(WasmAccessLogConfigTest, FailedToGetThreadLocalPlugin) { Http::TestResponseHeaderMapImpl response_header; Http::TestResponseTrailerMapImpl response_trailer; - filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info_); + filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info_, + AccessLog::AccessLogType::NotSet); // Even if the thread local plugin handle returns nullptr, `log` should not raise error or // exception. threadlocal.data_[0] = std::make_shared(nullptr); - filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info_); + filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info_, + AccessLog::AccessLogType::NotSet); } } // namespace Wasm diff --git a/test/extensions/filters/http/common/fuzz/http_filter_fuzzer.h b/test/extensions/filters/http/common/fuzz/http_filter_fuzzer.h index b5150504a1e2..7e415e679992 100644 --- a/test/extensions/filters/http/common/fuzz/http_filter_fuzzer.h +++ b/test/extensions/filters/http/common/fuzz/http_filter_fuzzer.h @@ -38,7 +38,8 @@ class HttpFilterFuzzer { // This executes the access logger with the fuzzed headers/trailers. void accessLog(AccessLog::Instance* access_logger, const StreamInfo::StreamInfo& stream_info) { ENVOY_LOG_MISC(debug, "Access logging"); - access_logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info); + access_logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, + AccessLog::AccessLogType::NotSet); } // Fuzzed headers and trailers are needed for access logging, reset the data and destroy filters. diff --git a/test/extensions/filters/http/composite/filter_test.cc b/test/extensions/filters/http/composite/filter_test.cc index 7ce2e7ffd641..c7595e762198 100644 --- a/test/extensions/filters/http/composite/filter_test.cc +++ b/test/extensions/filters/http/composite/filter_test.cc @@ -226,7 +226,8 @@ TEST_F(FilterTest, StreamFilterDelegationMultipleAccessLoggers) { EXPECT_CALL(*access_log_1, log(_, _, _, _, _)); EXPECT_CALL(*access_log_2, log(_, _, _, _, _)); - filter_.log(nullptr, nullptr, nullptr, StreamInfo::MockStreamInfo()); + filter_.log(nullptr, nullptr, nullptr, StreamInfo::MockStreamInfo(), + AccessLog::AccessLogType::NotSet); } } // namespace diff --git a/test/extensions/filters/http/tap/tap_filter_test.cc b/test/extensions/filters/http/tap/tap_filter_test.cc index 38546a7b2886..6184deff191a 100644 --- a/test/extensions/filters/http/tap/tap_filter_test.cc +++ b/test/extensions/filters/http/tap/tap_filter_test.cc @@ -89,7 +89,8 @@ TEST_F(TapFilterTest, NoConfig) { Http::MetadataMap metadata; EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata)); - filter_->log(&request_headers, &response_headers, &response_trailers, stream_info_); + filter_->log(&request_headers, &response_headers, &response_trailers, stream_info_, + AccessLog::AccessLogType::NotSet); } // Verify filter functionality when there is a tap config. @@ -119,7 +120,8 @@ TEST_F(TapFilterTest, Config) { EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers)); EXPECT_CALL(*http_per_request_tapper_, onDestroyLog()).WillOnce(Return(true)); - filter_->log(&request_headers, &response_headers, &response_trailers, stream_info_); + filter_->log(&request_headers, &response_headers, &response_trailers, stream_info_, + AccessLog::AccessLogType::NotSet); EXPECT_EQ(1UL, filter_config_->stats_.rq_tapped_.value()); // Workaround InSequence/shared_ptr mock leak. diff --git a/test/extensions/filters/http/wasm/wasm_filter_test.cc b/test/extensions/filters/http/wasm/wasm_filter_test.cc index 297ec0e2a308..5ca67af6eb6f 100644 --- a/test/extensions/filters/http/wasm/wasm_filter_test.cc +++ b/test/extensions/filters/http/wasm/wasm_filter_test.cc @@ -678,7 +678,8 @@ TEST_P(WasmHttpFilterTest, AccessLog) { StreamInfo::MockStreamInfo log_stream_info; EXPECT_CALL(log_stream_info, requestComplete()) .WillRepeatedly(testing::Return(std::chrono::milliseconds(30))); - filter().log(&request_headers, &response_headers, &response_trailers, log_stream_info); + filter().log(&request_headers, &response_headers, &response_trailers, log_stream_info, + AccessLog::AccessLogType::NotSet); filter().onDestroy(); } @@ -697,7 +698,8 @@ TEST_P(WasmHttpFilterTest, AccessLogClientDisconnected) { StreamInfo::MockStreamInfo log_stream_info; EXPECT_CALL(log_stream_info, requestComplete()) .WillRepeatedly(testing::Return(std::chrono::milliseconds(30))); - filter().log(&request_headers, nullptr, nullptr, log_stream_info); + filter().log(&request_headers, nullptr, nullptr, log_stream_info, + AccessLog::AccessLogType::NotSet); filter().onDestroy(); } @@ -714,7 +716,8 @@ TEST_P(WasmHttpFilterTest, AccessLogDisabledForIncompleteStream) { StreamInfo::MockStreamInfo log_stream_info; EXPECT_CALL(log_stream_info, requestComplete()).WillRepeatedly(testing::Return(absl::nullopt)); - filter().log(&request_headers, nullptr, nullptr, log_stream_info); + filter().log(&request_headers, nullptr, nullptr, log_stream_info, + AccessLog::AccessLogType::NotSet); filter().onDestroy(); } @@ -730,7 +733,8 @@ TEST_P(WasmHttpFilterTest, AccessLogCreate) { Http::TestRequestHeaderMapImpl request_headers{{":path", "/"}}; Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; Http::TestResponseTrailerMapImpl response_trailers{}; - filter().log(&request_headers, &response_headers, &response_trailers, log_stream_info); + filter().log(&request_headers, &response_headers, &response_trailers, log_stream_info, + AccessLog::AccessLogType::NotSet); filter().onDestroy(); } @@ -1759,7 +1763,8 @@ TEST_P(WasmHttpFilterTest, Metadata) { Buffer::OwnedImpl data("hello"); EXPECT_EQ(Http::FilterDataStatus::Continue, filter().decodeData(data, true)); - filter().log(&request_headers, nullptr, nullptr, request_stream_info_); + filter().log(&request_headers, nullptr, nullptr, request_stream_info_, + AccessLog::AccessLogType::NotSet); const auto* result = request_stream_info_.filterState()->getDataReadOnly( @@ -1826,7 +1831,8 @@ TEST_P(WasmHttpFilterTest, Property) { request_stream_info_.upstreamInfo()->setUpstreamHost(host_description); EXPECT_CALL(request_stream_info_, requestComplete()) .WillRepeatedly(Return(std::chrono::milliseconds(30))); - filter().log(&request_headers, nullptr, nullptr, request_stream_info_); + filter().log(&request_headers, nullptr, nullptr, request_stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_P(WasmHttpFilterTest, ClusterMetadata) { @@ -1857,14 +1863,16 @@ TEST_P(WasmHttpFilterTest, ClusterMetadata) { request_stream_info_.upstreamInfo()->setUpstreamHost(host_description); EXPECT_CALL(request_stream_info_, requestComplete) .WillRepeatedly(Return(std::chrono::milliseconds(30))); - filter().log(&request_headers, nullptr, nullptr, request_stream_info_); + filter().log(&request_headers, nullptr, nullptr, request_stream_info_, + AccessLog::AccessLogType::NotSet); // If upstream host is empty, fallback to upstream cluster info for cluster metadata. request_stream_info_.upstreamInfo()->setUpstreamHost(nullptr); EXPECT_CALL(request_stream_info_, upstreamClusterInfo()).WillRepeatedly(Return(cluster)); EXPECT_CALL(filter(), log_(spdlog::level::warn, Eq(absl::string_view("cluster metadata: cluster")))); - filter().log(&request_headers, nullptr, nullptr, request_stream_info_); + filter().log(&request_headers, nullptr, nullptr, request_stream_info_, + AccessLog::AccessLogType::NotSet); } TEST_P(WasmHttpFilterTest, SharedData) { From 5dffc0240e56d248e53868962eea965a47c7c284 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 1 May 2023 14:35:48 -0400 Subject: [PATCH 053/740] Plumb ServerFactoryContext into header validator factory (#27008) Signed-off-by: Yan Avlasov --- envoy/http/BUILD | 12 +++++- envoy/http/header_validator.h | 14 ------- envoy/http/header_validator_factory.h | 23 +++++++++++ envoy/server/transport_socket_config.h | 5 +++ .../network/http_connection_manager/BUILD | 2 +- .../network/http_connection_manager/config.cc | 7 ++-- .../header_validators/envoy_default/BUILD | 2 +- .../header_validators/envoy_default/config.cc | 7 ++-- .../header_validators/envoy_default/config.h | 4 +- source/extensions/upstreams/http/BUILD | 2 +- source/extensions/upstreams/http/config.cc | 13 +++--- source/extensions/upstreams/http/config.h | 4 +- source/server/server.h | 1 + source/server/transport_socket_config_impl.h | 1 + .../network/http_connection_manager/BUILD | 1 + .../http_connection_manager/config_test.cc | 12 +++--- .../header_validators/envoy_default/BUILD | 4 +- .../envoy_default/config_test.cc | 6 +-- .../header_validator_factory_test.cc | 8 ++-- test/extensions/upstreams/http/BUILD | 3 +- test/extensions/upstreams/http/config_test.cc | 41 ++++++++++--------- test/integration/BUILD | 4 +- test/integration/fake_upstream.h | 1 + test/integration/utility.cc | 10 +++-- test/mocks/server/instance.h | 33 +++++++++++++++ .../server/transport_socket_factory_context.h | 1 + 26 files changed, 146 insertions(+), 75 deletions(-) create mode 100644 envoy/http/header_validator_factory.h diff --git a/envoy/http/BUILD b/envoy/http/BUILD index 91d43202a7e8..dbb2e8f80ab3 100644 --- a/envoy/http/BUILD +++ b/envoy/http/BUILD @@ -239,10 +239,18 @@ envoy_cc_library( name = "header_validator_interface", hdrs = ["header_validator.h"], deps = [ - "//envoy/config:typed_config_interface", "//envoy/http:header_map_interface", "//envoy/http:protocol_interface", - "//envoy/protobuf:message_validator_interface", + ], +) + +envoy_cc_library( + name = "header_validator_factory_interface", + hdrs = ["header_validator_factory.h"], + deps = [ + ":header_validator_interface", + "//envoy/config:typed_config_interface", + "//envoy/server:factory_context_interface", ], ) diff --git a/envoy/http/header_validator.h b/envoy/http/header_validator.h index c89b7e2a3c59..b79380c513a9 100644 --- a/envoy/http/header_validator.h +++ b/envoy/http/header_validator.h @@ -3,10 +3,8 @@ #include #include -#include "envoy/config/typed_config.h" #include "envoy/http/header_map.h" #include "envoy/http/protocol.h" -#include "envoy/protobuf/message_validator.h" namespace Envoy { namespace Http { @@ -197,17 +195,5 @@ class HeaderValidatorFactory { using HeaderValidatorFactoryPtr = std::unique_ptr; -/** - * Extension configuration for header validators. - */ -class HeaderValidatorFactoryConfig : public Config::TypedFactory { -public: - virtual HeaderValidatorFactoryPtr - createFromProto(const Protobuf::Message& config, - ProtobufMessage::ValidationVisitor& validation_visitor) PURE; - - std::string category() const override { return "envoy.http.header_validators"; } -}; - } // namespace Http } // namespace Envoy diff --git a/envoy/http/header_validator_factory.h b/envoy/http/header_validator_factory.h new file mode 100644 index 000000000000..10449e1d1d38 --- /dev/null +++ b/envoy/http/header_validator_factory.h @@ -0,0 +1,23 @@ +#pragma once + +#include "envoy/config/typed_config.h" +#include "envoy/http/header_validator.h" +#include "envoy/server/factory_context.h" + +namespace Envoy { +namespace Http { + +/** + * Extension configuration for header validators. + */ +class HeaderValidatorFactoryConfig : public Config::TypedFactory { +public: + virtual HeaderValidatorFactoryPtr + createFromProto(const Protobuf::Message& config, + Server::Configuration::ServerFactoryContext& server_context) PURE; + + std::string category() const override { return "envoy.http.header_validators"; } +}; + +} // namespace Http +} // namespace Envoy diff --git a/envoy/server/transport_socket_config.h b/envoy/server/transport_socket_config.h index 64401ca61a49..756c27f64365 100644 --- a/envoy/server/transport_socket_config.h +++ b/envoy/server/transport_socket_config.h @@ -29,6 +29,11 @@ class TransportSocketFactoryContext { public: virtual ~TransportSocketFactoryContext() = default; + /** + * @return ServerFactoryContext which lifetime is no shorter than the server. + */ + virtual ServerFactoryContext& getServerFactoryContext() PURE; + /** * @return OptRef the global HTTP admin endpoint for the server. */ diff --git a/source/extensions/filters/network/http_connection_manager/BUILD b/source/extensions/filters/network/http_connection_manager/BUILD index 0d589272e0cb..730d87b77cc1 100644 --- a/source/extensions/filters/network/http_connection_manager/BUILD +++ b/source/extensions/filters/network/http_connection_manager/BUILD @@ -24,7 +24,7 @@ envoy_cc_extension( "//envoy/http:codec_interface", "//envoy/http:early_header_mutation_interface", "//envoy/http:filter_interface", - "//envoy/http:header_validator_interface", + "//envoy/http:header_validator_factory_interface", "//envoy/http:original_ip_detection_interface", "//envoy/http:request_id_extension_interface", "//envoy/registry", diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 7b1e38b5eae5..da548397fe13 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -12,6 +12,7 @@ #include "envoy/extensions/http/original_ip_detection/xff/v3/xff.pb.h" #include "envoy/extensions/request_id/uuid/v3/uuid.pb.h" #include "envoy/filesystem/filesystem.h" +#include "envoy/http/header_validator_factory.h" #include "envoy/registry/registry.h" #include "envoy/server/admin.h" #include "envoy/tracing/tracer.h" @@ -128,7 +129,7 @@ envoy::extensions::filters::network::http_connection_manager::v3::HttpConnection Http::HeaderValidatorFactoryPtr createHeaderValidatorFactory( [[maybe_unused]] const envoy::extensions::filters::network::http_connection_manager::v3:: HttpConnectionManager& config, - [[maybe_unused]] ProtobufMessage::ValidationVisitor& validation_visitor) { + [[maybe_unused]] Server::Configuration::ServerFactoryContext& server_context) { Http::HeaderValidatorFactoryPtr header_validator_factory; #ifdef ENVOY_ENABLE_UHV @@ -170,7 +171,7 @@ Http::HeaderValidatorFactoryPtr createHeaderValidatorFactory( } header_validator_factory = - factory->createFromProto(header_validator_config.typed_config(), validation_visitor); + factory->createFromProto(header_validator_config.typed_config(), server_context); if (!header_validator_factory) { throw EnvoyException(fmt::format("Header validator extension could not be created: '{}'", header_validator_config.name())); @@ -379,7 +380,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( config.proxy_status_config()) : nullptr), header_validator_factory_( - createHeaderValidatorFactory(config, context.messageValidationVisitor())), + createHeaderValidatorFactory(config, context.getServerFactoryContext())), append_x_forwarded_port_(config.append_x_forwarded_port()), add_proxy_protocol_connection_state_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, add_proxy_protocol_connection_state, true)) { diff --git a/source/extensions/http/header_validators/envoy_default/BUILD b/source/extensions/http/header_validators/envoy_default/BUILD index 18252f0f142a..3c2d11327bc9 100644 --- a/source/extensions/http/header_validators/envoy_default/BUILD +++ b/source/extensions/http/header_validators/envoy_default/BUILD @@ -143,7 +143,7 @@ envoy_cc_extension( hdrs = ["config.h"], deps = [ ":header_validator_factory", - "//envoy/http:header_validator_interface", + "//envoy/http:header_validator_factory_interface", "//envoy/registry", "//source/common/config:utility_lib", "@envoy_api//envoy/extensions/http/header_validators/envoy_default/v3:pkg_cc_proto", diff --git a/source/extensions/http/header_validators/envoy_default/config.cc b/source/extensions/http/header_validators/envoy_default/config.cc index ab10a5bdfd1e..3ea9da26527e 100644 --- a/source/extensions/http/header_validators/envoy_default/config.cc +++ b/source/extensions/http/header_validators/envoy_default/config.cc @@ -13,13 +13,14 @@ namespace HeaderValidators { namespace EnvoyDefault { ::Envoy::Http::HeaderValidatorFactoryPtr HeaderValidatorFactoryConfig::createFromProto( - const Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor) { + const Protobuf::Message& message, Server::Configuration::ServerFactoryContext& server_context) { auto mptr = ::Envoy::Config::Utility::translateAnyToFactoryConfig( - dynamic_cast(message), validation_visitor, *this); + dynamic_cast(message), server_context.messageValidationVisitor(), + *this); const auto& proto_config = MessageUtil::downcastAndValidate( - *mptr, validation_visitor); + *mptr, server_context.messageValidationVisitor()); return std::make_unique(proto_config); } diff --git a/source/extensions/http/header_validators/envoy_default/config.h b/source/extensions/http/header_validators/envoy_default/config.h index f3abd307285f..72632dcae373 100644 --- a/source/extensions/http/header_validators/envoy_default/config.h +++ b/source/extensions/http/header_validators/envoy_default/config.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/http/header_validator.h" +#include "envoy/http/header_validator_factory.h" #include "envoy/registry/registry.h" #include "source/common/protobuf/protobuf.h" @@ -19,7 +19,7 @@ class HeaderValidatorFactoryConfig : public ::Envoy::Http::HeaderValidatorFactor public: ::Envoy::Http::HeaderValidatorFactoryPtr createFromProto(const Protobuf::Message& config, - ProtobufMessage::ValidationVisitor& validation_visitor) override; + Server::Configuration::ServerFactoryContext& server_context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; diff --git a/source/extensions/upstreams/http/BUILD b/source/extensions/upstreams/http/BUILD index f2ad70306beb..02b41dcb1bb4 100644 --- a/source/extensions/upstreams/http/BUILD +++ b/source/extensions/upstreams/http/BUILD @@ -15,7 +15,7 @@ envoy_cc_extension( # This is core Envoy config. visibility = ["//visibility:public"], deps = [ - "//envoy/http:header_validator_interface", + "//envoy/http:header_validator_factory_interface", "//envoy/registry", "//source/common/common:minimal_logger_lib", "//source/common/config:utility_lib", diff --git a/source/extensions/upstreams/http/config.cc b/source/extensions/upstreams/http/config.cc index 8852f1230430..39ee82441ef4 100644 --- a/source/extensions/upstreams/http/config.cc +++ b/source/extensions/upstreams/http/config.cc @@ -8,6 +8,7 @@ #include "envoy/config/cluster/v3/cluster.pb.h" #include "envoy/config/core/v3/base.pb.h" #include "envoy/extensions/http/header_validators/envoy_default/v3/header_validator.pb.h" +#include "envoy/http/header_validator_factory.h" #include "envoy/upstream/upstream.h" #include "source/common/config/utility.h" @@ -96,7 +97,7 @@ getAlternateProtocolsCacheOptions( Envoy::Http::HeaderValidatorFactoryPtr createHeaderValidatorFactory( [[maybe_unused]] const envoy::extensions::upstreams::http::v3::HttpProtocolOptions& options, - [[maybe_unused]] ProtobufMessage::ValidationVisitor& validation_visitor) { + [[maybe_unused]] Server::Configuration::ServerFactoryContext& server_context) { Envoy::Http::HeaderValidatorFactoryPtr header_validator_factory; #ifdef ENVOY_ENABLE_UHV @@ -124,7 +125,7 @@ Envoy::Http::HeaderValidatorFactoryPtr createHeaderValidatorFactory( } header_validator_factory = - factory->createFromProto(header_validator_config.typed_config(), validation_visitor); + factory->createFromProto(header_validator_config.typed_config(), server_context); if (!header_validator_factory) { throw EnvoyException(fmt::format("Header validator extension could not be created: '{}'", header_validator_config.name())); @@ -165,9 +166,9 @@ uint64_t ProtocolOptionsConfigImpl::parseFeatures(const envoy::config::cluster:: ProtocolOptionsConfigImpl::ProtocolOptionsConfigImpl( const envoy::extensions::upstreams::http::v3::HttpProtocolOptions& options, - ProtobufMessage::ValidationVisitor& validation_visitor) - : http1_settings_( - Envoy::Http::Http1::parseHttp1Settings(getHttpOptions(options), validation_visitor)), + Server::Configuration::ServerFactoryContext& server_context) + : http1_settings_(Envoy::Http::Http1::parseHttp1Settings( + getHttpOptions(options), server_context.messageValidationVisitor())), http2_options_(Http2::Utility::initializeAndValidateOptions(getHttp2Options(options))), http3_options_(getHttp3Options(options)), common_http_protocol_options_(options.common_http_protocol_options()), @@ -178,7 +179,7 @@ ProtocolOptionsConfigImpl::ProtocolOptionsConfigImpl( : absl::nullopt), http_filters_(options.http_filters()), alternate_protocol_cache_options_(getAlternateProtocolsCacheOptions(options)), - header_validator_factory_(createHeaderValidatorFactory(options, validation_visitor)), + header_validator_factory_(createHeaderValidatorFactory(options, server_context)), use_downstream_protocol_(options.has_use_downstream_protocol_config()), use_http2_(useHttp2(options)), use_http3_(useHttp3(options)), use_alpn_(options.has_auto_config()) {} diff --git a/source/extensions/upstreams/http/config.h b/source/extensions/upstreams/http/config.h index e53f3206b16a..93879e9deb6b 100644 --- a/source/extensions/upstreams/http/config.h +++ b/source/extensions/upstreams/http/config.h @@ -28,7 +28,7 @@ class ProtocolOptionsConfigImpl : public Upstream::ProtocolOptionsConfig { public: ProtocolOptionsConfigImpl( const envoy::extensions::upstreams::http::v3::HttpProtocolOptions& options, - ProtobufMessage::ValidationVisitor& validation_visitor); + Server::Configuration::ServerFactoryContext& server_context); // Constructor for legacy (deprecated) config. ProtocolOptionsConfigImpl( const envoy::config::core::v3::Http1ProtocolOptions& http1_settings, @@ -71,7 +71,7 @@ class ProtocolOptionsConfigFactory : public Server::Configuration::ProtocolOptio const envoy::extensions::upstreams::http::v3::HttpProtocolOptions&>( config, context.messageValidationVisitor()); return std::make_shared(typed_config, - context.messageValidationVisitor()); + context.getServerFactoryContext()); } std::string category() const override { return "envoy.upstream_options"; } std::string name() const override { diff --git a/source/server/server.h b/source/server/server.h index 714205b7afd9..d9c9f175a6dd 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -200,6 +200,7 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return server_.bootstrap(); } // Configuration::TransportSocketFactoryContext + ServerFactoryContext& getServerFactoryContext() override { return *this; }; Ssl::ContextManager& sslContextManager() override { return server_.sslContextManager(); } Secret::SecretManager& secretManager() override { return server_.secretManager(); } Stats::Store& stats() override { return server_.stats(); } diff --git a/source/server/transport_socket_config_impl.h b/source/server/transport_socket_config_impl.h index f4c192c7e3e8..edbb65282e19 100644 --- a/source/server/transport_socket_config_impl.h +++ b/source/server/transport_socket_config_impl.h @@ -27,6 +27,7 @@ class TransportSocketFactoryContextImpl : public TransportSocketFactoryContext { void setInitManager(Init::Manager& init_manager) { init_manager_ = &init_manager; } // TransportSocketFactoryContext + ServerFactoryContext& getServerFactoryContext() override { return server_context_; }; OptRef admin() override { return server_context_.admin(); } Ssl::ContextManager& sslContextManager() override { return context_manager_; } Stats::Scope& scope() override { return stats_scope_; } diff --git a/test/extensions/filters/network/http_connection_manager/BUILD b/test/extensions/filters/network/http_connection_manager/BUILD index 5c71996bb22f..dcdd0ab23ef9 100644 --- a/test/extensions/filters/network/http_connection_manager/BUILD +++ b/test/extensions/filters/network/http_connection_manager/BUILD @@ -47,6 +47,7 @@ envoy_extension_cc_test( deps = [ ":config_cc_proto", ":config_test_base", + "//envoy/http:header_validator_factory_interface", "//source/common/buffer:buffer_lib", "//source/extensions/access_loggers/file:config", "//source/extensions/filters/network/http_connection_manager:config", diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 1e895eb7871d..8e547b6a69e7 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -6,6 +6,7 @@ #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.validate.h" #include "envoy/extensions/http/header_validators/envoy_default/v3/header_validator.pb.h" #include "envoy/extensions/http/header_validators/envoy_default/v3/header_validator.pb.validate.h" +#include "envoy/http/header_validator_factory.h" #include "envoy/server/request_id_extension_config.h" #include "envoy/type/v3/percent.pb.h" @@ -2961,8 +2962,8 @@ class TestHeaderValidatorFactoryConfig : public Http::HeaderValidatorFactoryConf return std::make_unique(); } - Http::HeaderValidatorFactoryPtr createFromProto(const Protobuf::Message&, - ProtobufMessage::ValidationVisitor&) override { + Http::HeaderValidatorFactoryPtr + createFromProto(const Protobuf::Message&, Server::Configuration::ServerFactoryContext&) override { auto header_validator = std::make_unique>(); EXPECT_CALL(*header_validator, createServerHeaderValidator(Http::Protocol::Http2, _)) .WillOnce(InvokeWithoutArgs( @@ -2988,13 +2989,14 @@ class DefaultHeaderValidatorFactoryConfigOverride : public Http::HeaderValidator Http::HeaderValidatorFactoryPtr createFromProto(const Protobuf::Message& message, - ProtobufMessage::ValidationVisitor& validation_visitor) override { + Server::Configuration::ServerFactoryContext& server_context) override { auto mptr = ::Envoy::Config::Utility::translateAnyToFactoryConfig( - dynamic_cast(message), validation_visitor, *this); + dynamic_cast(message), server_context.messageValidationVisitor(), + *this); const auto& proto_config = MessageUtil::downcastAndValidate( - *mptr, validation_visitor); + *mptr, server_context.messageValidationVisitor()); config_ = proto_config; auto header_validator = std::make_unique>(); diff --git a/test/extensions/http/header_validators/envoy_default/BUILD b/test/extensions/http/header_validators/envoy_default/BUILD index 8d2feff1ed72..2eda8c51e913 100644 --- a/test/extensions/http/header_validators/envoy_default/BUILD +++ b/test/extensions/http/header_validators/envoy_default/BUILD @@ -99,7 +99,7 @@ envoy_extension_cc_test( "//source/common/network:utility_lib", "//source/extensions/http/header_validators/envoy_default:config", "//test/mocks/http:header_validator_mocks", - "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/server:instance_mocks", "//test/test_common:utility_lib", "@envoy_api//envoy/extensions/http/header_validators/envoy_default/v3:pkg_cc_proto", ], @@ -125,7 +125,7 @@ envoy_extension_cc_test( deps = [ "//envoy/registry", "//source/extensions/http/header_validators/envoy_default:config", - "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/server:instance_mocks", "//test/test_common:utility_lib", "@envoy_api//envoy/extensions/http/header_validators/envoy_default/v3:pkg_cc_proto", ], diff --git a/test/extensions/http/header_validators/envoy_default/config_test.cc b/test/extensions/http/header_validators/envoy_default/config_test.cc index e7cada790386..f2f5254ac0c4 100644 --- a/test/extensions/http/header_validators/envoy_default/config_test.cc +++ b/test/extensions/http/header_validators/envoy_default/config_test.cc @@ -3,7 +3,7 @@ #include "source/extensions/http/header_validators/envoy_default/config.h" -#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/server/instance.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -29,8 +29,8 @@ TEST(EnvoyDefaultUhvFactoryTest, Basic) { )EOF"; TestUtility::loadFromYaml(yaml, typed_config); - ::testing::NiceMock validation_visitor; - EXPECT_NE(factory->createFromProto(typed_config.typed_config(), validation_visitor), nullptr); + ::testing::NiceMock server_context; + EXPECT_NE(factory->createFromProto(typed_config.typed_config(), server_context), nullptr); } } // namespace EnvoyDefault diff --git a/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc b/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc index 33485c68845c..e7b3cc7796fc 100644 --- a/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc +++ b/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc @@ -6,7 +6,7 @@ #include "source/extensions/http/header_validators/envoy_default/http2_header_validator.h" #include "test/mocks/http/header_validator.h" -#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/server/instance.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -32,7 +32,7 @@ class HeaderValidatorFactoryTest : public testing::Test { envoy::config::core::v3::TypedExtensionConfig typed_config; TestUtility::loadFromYaml(std::string(config_yaml), typed_config); - uhv_factory_ = factory->createFromProto(typed_config.typed_config(), validation_visitor_); + uhv_factory_ = factory->createFromProto(typed_config.typed_config(), server_context_); return uhv_factory_->createServerHeaderValidator(protocol, stats_); } @@ -46,11 +46,11 @@ class HeaderValidatorFactoryTest : public testing::Test { envoy::config::core::v3::TypedExtensionConfig typed_config; TestUtility::loadFromYaml(std::string(config_yaml), typed_config); - uhv_factory_ = factory->createFromProto(typed_config.typed_config(), validation_visitor_); + uhv_factory_ = factory->createFromProto(typed_config.typed_config(), server_context_); return uhv_factory_->createClientHeaderValidator(protocol, stats_); } - NiceMock validation_visitor_; + NiceMock server_context_; ::Envoy::Http::HeaderValidatorFactoryPtr uhv_factory_; NiceMock stats_; diff --git a/test/extensions/upstreams/http/BUILD b/test/extensions/upstreams/http/BUILD index 37864fc38bb7..de84aa12fd30 100644 --- a/test/extensions/upstreams/http/BUILD +++ b/test/extensions/upstreams/http/BUILD @@ -19,12 +19,13 @@ envoy_cc_test( srcs = ["config_test.cc"], deps = [ ":config_cc_proto", + "//envoy/http:header_validator_factory_interface", "//source/common/config:utility_lib", "//source/common/upstream:upstream_includes", "//source/common/upstream:upstream_lib", "//source/extensions/upstreams/http:config", "//test/mocks/http:header_validator_mocks", - "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/server:instance_mocks", "//test/test_common:registry_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/extensions/http/header_validators/envoy_default/v3:pkg_cc_proto", diff --git a/test/extensions/upstreams/http/config_test.cc b/test/extensions/upstreams/http/config_test.cc index 6d669bdd4fd5..60f5ae3b8a32 100644 --- a/test/extensions/upstreams/http/config_test.cc +++ b/test/extensions/upstreams/http/config_test.cc @@ -1,5 +1,6 @@ #include "envoy/extensions/http/header_validators/envoy_default/v3/header_validator.pb.h" #include "envoy/extensions/http/header_validators/envoy_default/v3/header_validator.pb.validate.h" +#include "envoy/http/header_validator_factory.h" #include "source/common/config/utility.h" #include "source/extensions/upstreams/http/config.h" @@ -7,7 +8,7 @@ #include "test/extensions/upstreams/http/config.pb.h" #include "test/extensions/upstreams/http/config.pb.validate.h" #include "test/mocks/http/header_validator.h" -#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/server/instance.h" #include "test/test_common/registry.h" #include "test/test_common/utility.h" @@ -26,11 +27,11 @@ using ::testing::StrictMock; class ConfigTest : public ::testing::Test { public: envoy::extensions::upstreams::http::v3::HttpProtocolOptions options_; - NiceMock validation_visitor_; + NiceMock server_context_; }; TEST_F(ConfigTest, Basic) { - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); EXPECT_FALSE(config.use_downstream_protocol_); EXPECT_FALSE(config.use_http2_); } @@ -38,14 +39,14 @@ TEST_F(ConfigTest, Basic) { TEST_F(ConfigTest, Downstream) { options_.mutable_use_downstream_protocol_config(); { - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); EXPECT_TRUE(config.use_downstream_protocol_); EXPECT_FALSE(config.use_http2_); } options_.mutable_use_downstream_protocol_config()->mutable_http2_protocol_options(); { - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); EXPECT_TRUE(config.use_downstream_protocol_); EXPECT_TRUE(config.use_http2_); } @@ -58,7 +59,7 @@ TEST(FactoryTest, EmptyProto) { TEST_F(ConfigTest, Auto) { options_.mutable_auto_config(); - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); EXPECT_FALSE(config.use_downstream_protocol_); EXPECT_TRUE(config.use_http2_); EXPECT_FALSE(config.use_http3_); @@ -69,7 +70,7 @@ TEST_F(ConfigTest, AutoHttp3) { options_.mutable_auto_config(); options_.mutable_auto_config()->mutable_http3_protocol_options(); options_.mutable_auto_config()->mutable_alternate_protocols_cache_options(); - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); EXPECT_TRUE(config.use_http2_); EXPECT_TRUE(config.use_http3_); EXPECT_TRUE(config.use_alpn_); @@ -79,7 +80,7 @@ TEST_F(ConfigTest, AutoHttp3NoCache) { options_.mutable_auto_config(); options_.mutable_auto_config()->mutable_http3_protocol_options(); EXPECT_THROW_WITH_MESSAGE( - ProtocolOptionsConfigImpl config(options_, validation_visitor_), EnvoyException, + ProtocolOptionsConfigImpl config(options_, server_context_), EnvoyException, "alternate protocols cache must be configured when HTTP/3 is enabled with auto_config"); } @@ -94,7 +95,7 @@ class TestHeaderValidatorFactoryConfig : public ::Envoy::Http::HeaderValidatorFa } ::Envoy::Http::HeaderValidatorFactoryPtr - createFromProto(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override { + createFromProto(const Protobuf::Message&, Server::Configuration::ServerFactoryContext&) override { auto header_validator = std::make_unique>(); EXPECT_CALL(*header_validator, createClientHeaderValidator(::Envoy::Http::Protocol::Http2, _)) @@ -123,13 +124,14 @@ class DefaultHeaderValidatorFactoryConfigOverride ::Envoy::Http::HeaderValidatorFactoryPtr createFromProto(const Protobuf::Message& message, - ProtobufMessage::ValidationVisitor& validation_visitor) override { + Server::Configuration::ServerFactoryContext& server_context) override { auto mptr = ::Envoy::Config::Utility::translateAnyToFactoryConfig( - dynamic_cast(message), validation_visitor, *this); + dynamic_cast(message), server_context.messageValidationVisitor(), + *this); const auto& proto_config = MessageUtil::downcastAndValidate( - *mptr, validation_visitor); + *mptr, server_context.messageValidationVisitor()); config_ = proto_config; auto header_validator = std::make_unique>(); @@ -160,14 +162,13 @@ TEST_F(ConfigTest, HeaderValidatorConfig) { Registry::InjectFactory<::Envoy::Http::HeaderValidatorFactoryConfig> registration(factory); TestUtility::loadFromYamlAndValidate(yaml_string, options_); #ifdef ENVOY_ENABLE_UHV - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( ::Envoy::Http::Protocol::Http2, stats)); #else // If UHV is disabled, providing config should result in rejection - EXPECT_THROW({ ProtocolOptionsConfigImpl config(options_, validation_visitor_); }, - EnvoyException); + EXPECT_THROW({ ProtocolOptionsConfigImpl config(options_, server_context_); }, EnvoyException); #endif } @@ -177,7 +178,7 @@ TEST_F(ConfigTest, DefaultHeaderValidatorConfig) { DefaultHeaderValidatorFactoryConfigOverride factory(proto_config); Registry::InjectFactory<::Envoy::Http::HeaderValidatorFactoryConfig> registration(factory); NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); #ifdef ENVOY_ENABLE_UHV EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( ::Envoy::Http::Protocol::Http2, stats)); @@ -201,7 +202,7 @@ TEST_F(ConfigTest, TranslateDownstreamLegacyConfigToDefaultHeaderValidatorConfig DefaultHeaderValidatorFactoryConfigOverride factory(proto_config); Registry::InjectFactory<::Envoy::Http::HeaderValidatorFactoryConfig> registration(factory); NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); #ifdef ENVOY_ENABLE_UHV EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( ::Envoy::Http::Protocol::Http2, stats)); @@ -225,7 +226,7 @@ TEST_F(ConfigTest, TranslateAutoLegacyConfigToDefaultHeaderValidatorConfig) { DefaultHeaderValidatorFactoryConfigOverride factory(proto_config); Registry::InjectFactory<::Envoy::Http::HeaderValidatorFactoryConfig> registration(factory); NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); #ifdef ENVOY_ENABLE_UHV EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( ::Envoy::Http::Protocol::Http2, stats)); @@ -249,7 +250,7 @@ TEST_F(ConfigTest, TranslateExplicitLegacyConfigToDefaultHeaderValidatorConfig) DefaultHeaderValidatorFactoryConfigOverride factory(proto_config); Registry::InjectFactory<::Envoy::Http::HeaderValidatorFactoryConfig> registration(factory); NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); #ifdef ENVOY_ENABLE_UHV EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( ::Envoy::Http::Protocol::Http2, stats)); @@ -273,7 +274,7 @@ TEST_F(ConfigTest, TranslateExplicitH2LegacyConfigToDefaultHeaderValidatorConfig DefaultHeaderValidatorFactoryConfigOverride factory(proto_config); Registry::InjectFactory<::Envoy::Http::HeaderValidatorFactoryConfig> registration(factory); NiceMock<::Envoy::Http::MockHeaderValidatorStats> stats; - ProtocolOptionsConfigImpl config(options_, validation_visitor_); + ProtocolOptionsConfigImpl config(options_, server_context_); #ifdef ENVOY_ENABLE_UHV EXPECT_NE(nullptr, config.header_validator_factory_->createClientHeaderValidator( ::Envoy::Http::Protocol::Http2, stats)); diff --git a/test/integration/BUILD b/test/integration/BUILD index 67aad40e1f6c..39938958fd7f 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1017,7 +1017,7 @@ envoy_cc_test_library( ":tcp_dump", "//envoy/api:api_interface", "//envoy/http:codec_interface", - "//envoy/http:header_map_interface", + "//envoy/http:header_validator_factory_interface", "//envoy/network:filter_interface", "//envoy/server:options_interface", "//envoy/server:process_context_interface", @@ -1046,8 +1046,8 @@ envoy_cc_test_library( "//source/server:server_lib", "//test/mocks:common_lib", "//test/mocks/event:event_mocks", - "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:instance_mocks", "//test/mocks/server:transport_socket_factory_context_mocks", "//test/mocks/upstream:cluster_info_mocks", "//test/test_common:environment_lib", diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index adc73410a10b..c5b915f64bb9 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -38,6 +38,7 @@ #include "test/mocks/http/header_validator.h" #include "test/mocks/protobuf/mocks.h" +#include "test/mocks/server/instance.h" #if defined(ENVOY_ENABLE_QUIC) #include "source/common/quic/active_quic_listener.h" diff --git a/test/integration/utility.cc b/test/integration/utility.cc index c03979951245..db03dded6453 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -8,6 +8,7 @@ #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/event/dispatcher.h" #include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.h" +#include "envoy/http/header_validator_factory.h" #include "envoy/network/connection.h" #include "source/common/api/api_impl.h" @@ -33,7 +34,7 @@ #include "test/integration/ssl_utility.h" #endif #include "test/mocks/common.h" -#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/server/instance.h" #include "test/mocks/server/transport_socket_factory_context.h" #include "test/mocks/stats/mocks.h" #include "test/mocks/upstream/cluster_info.h" @@ -327,10 +328,13 @@ IntegrationUtil::makeHeaderValidationFactory([[maybe_unused]] absl::string_view envoy::config::core::v3::TypedExtensionConfig typed_config; Thread::SkipAsserts no_main_thread_asserts_in_yaml_parser; + testing::NiceMock server_context; + ON_CALL(server_context, messageValidationVisitor()) + .WillByDefault(testing::ReturnRef(ProtobufMessage::getNullValidationVisitor())); + TestUtility::loadFromYaml(std::string(config), typed_config); - NiceMock validation_visitor; - return factory->createFromProto(typed_config.typed_config(), validation_visitor); + return factory->createFromProto(typed_config.typed_config(), server_context); #else return nullptr; #endif diff --git a/test/mocks/server/instance.h b/test/mocks/server/instance.h index ab792d5cf52c..e8d077a1624e 100644 --- a/test/mocks/server/instance.h +++ b/test/mocks/server/instance.h @@ -198,6 +198,39 @@ class MockServerFactoryContext : public virtual ServerFactoryContext { testing::NiceMock options_; }; +// Stateless mock ServerFactoryContext for cases where it needs to be used concurrently in different +// threads. Global state in the MockServerFactoryContext causes thread safety issues in this case. +class StatelessMockServerFactoryContext : public virtual ServerFactoryContext { +public: + StatelessMockServerFactoryContext() = default; + ~StatelessMockServerFactoryContext() override = default; + + MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); + MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); + MOCK_METHOD(const Server::Options&, options, ()); + MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); + MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); + MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); + MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(Stats::Scope&, serverScope, ()); + MOCK_METHOD(Singleton::Manager&, singletonManager, ()); + MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); + MOCK_METHOD(OptRef, admin, ()); + MOCK_METHOD(TimeSource&, timeSource, ()); + MOCK_METHOD(Event::TestTimeSystem&, timeSystem, ()); + MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); + MOCK_METHOD(Api::Api&, api, ()); + MOCK_METHOD(Grpc::Context&, grpcContext, ()); + MOCK_METHOD(Router::Context&, routerContext, ()); + MOCK_METHOD(envoy::config::bootstrap::v3::Bootstrap&, bootstrap, ()); + MOCK_METHOD(Server::DrainManager&, drainManager, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); + MOCK_METHOD(StatsConfig&, statsConfig, (), ()); + MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); +}; + } // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/transport_socket_factory_context.h b/test/mocks/server/transport_socket_factory_context.h index 283f84892965..ee23330262d7 100644 --- a/test/mocks/server/transport_socket_factory_context.h +++ b/test/mocks/server/transport_socket_factory_context.h @@ -24,6 +24,7 @@ class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { Secret::SecretManager& secretManager() override { return *(secret_manager_); } + MOCK_METHOD(ServerFactoryContext&, getServerFactoryContext, ()); MOCK_METHOD(OptRef, admin, ()); MOCK_METHOD(Ssl::ContextManager&, sslContextManager, ()); MOCK_METHOD(Stats::Scope&, scope, ()); From 4f752679d4003eabaf11e47832ca5df2647329b1 Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Mon, 1 May 2023 14:56:27 -0400 Subject: [PATCH 054/740] Overload Manager: LoadShedPoint for HCM decode headers (#26769) * Reject new request load shed point. Signed-off-by: Kevin Baichoo --- changelogs/current.yaml | 4 ++ .../overload_manager/overload_manager.rst | 5 +++ source/common/http/conn_manager_impl.cc | 11 ++++- source/common/http/conn_manager_impl.h | 1 + test/common/http/conn_manager_impl_test_2.cc | 38 ++++++++++++++++ test/integration/overload_integration_test.cc | 45 ++++++++++++++++++- 6 files changed, 101 insertions(+), 3 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index d643ce209de0..c4463dde2882 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -67,6 +67,10 @@ new_features: added Runtime feature ``envoy.reloadable_features.max_request_headers_size_kb`` to override the default value of :ref:`max request headers size `. +- area: load shed point + change: | + added load shed point ``envoy.load_shed_points.http_connection_manager_decode_headers`` that rejects new http streams + by sending a local reply. - area: matchers change: | Added :ref:`RuntimeFraction ` input diff --git a/docs/root/configuration/operations/overload_manager/overload_manager.rst b/docs/root/configuration/operations/overload_manager/overload_manager.rst index 78b93ccc19e0..b2629e7381db 100644 --- a/docs/root/configuration/operations/overload_manager/overload_manager.rst +++ b/docs/root/configuration/operations/overload_manager/overload_manager.rst @@ -151,6 +151,11 @@ The following core load shed points are supported: - Envoy will reject (close) new TCP connections. This occurs before the :ref:`Listener Filter Chain ` is created. + * - envoy.load_shed_points.http_connection_manager_decode_headers + - Envoy will reject new HTTP streams by sending a local reply. This occurs + right after the http codec has finished parsing headers but before the + :ref:`HTTP Filter Chain is instantiated `. + .. _config_overload_manager_reducing_timeouts: Reducing timeouts diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 006ac3f93471..3d6fdae25f76 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -100,6 +100,8 @@ ConnectionManagerImpl::ConnectionManagerImpl(ConnectionManagerConfig& config, local_info_(local_info), cluster_manager_(cluster_manager), listener_stats_(config_.listenerStats()), overload_state_(overload_manager.getThreadLocalOverloadState()), + accept_new_http_stream_(overload_manager.getLoadShedPoint( + "envoy.load_shed_points.http_connection_manager_decode_headers")), overload_stop_accepting_requests_ref_( overload_state_.getState(Server::OverloadActionNames::get().StopAcceptingRequests)), overload_disable_keepalive_ref_( @@ -1086,8 +1088,13 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt } // Drop new requests when overloaded as soon as we have decoded the headers. - if (connection_manager_.random_generator_.bernoulli( - connection_manager_.overload_stop_accepting_requests_ref_.value())) { + const bool drop_request_due_to_overload = + (connection_manager_.accept_new_http_stream_ != nullptr && + connection_manager_.accept_new_http_stream_->shouldShedLoad()) || + connection_manager_.random_generator_.bernoulli( + connection_manager_.overload_stop_accepting_requests_ref_.value()); + + if (drop_request_due_to_overload) { // In this one special case, do not create the filter chain. If there is a risk of memory // overload it is more important to avoid unnecessary allocation than to create the filters. filter_manager_.skipFilterChainCreation(); diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 8500efbf9ae0..2a7b19cd112c 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -575,6 +575,7 @@ class ConnectionManagerImpl : Logger::Loggable, Network::ReadFilterCallbacks* read_callbacks_{}; ConnectionManagerListenerStats& listener_stats_; Server::ThreadLocalOverloadState& overload_state_; + Server::LoadShedPoint* accept_new_http_stream_{nullptr}; // References into the overload manager thread local state map. Using these lets us avoid a // map lookup in the hot path of processing each request. const Server::OverloadActionState& overload_stop_accepting_requests_ref_; diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index 0b99d09ae01d..18ed7b9799e3 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -2477,6 +2477,44 @@ TEST_F(HttpConnectionManagerImplTest, DisableHttp2KeepAliveWhenOverloaded) { EXPECT_EQ(1, stats_.named_.downstream_cx_overload_disable_keepalive_.value()); } +TEST_F(HttpConnectionManagerImplTest, DecodeHeaderLoadShedPointCanRejectNewStreams) { + Server::MockLoadShedPoint accept_new_stream_point; + EXPECT_CALL(overload_manager_, getLoadShedPoint(testing::_)) + .WillOnce(Return(&accept_new_stream_point)); + + setup(false, ""); + setupFilterChain(1, 0); + + EXPECT_CALL(accept_new_stream_point, shouldShedLoad()).WillOnce(Return(true)); + + // 503 direct response when overloaded. + EXPECT_CALL(response_encoder_, encodeHeaders(_, false)) + .WillOnce(Invoke([](const ResponseHeaderMap& headers, bool) -> void { + EXPECT_EQ("503", headers.getStatusValue()); + })); + std::string response_body; + EXPECT_CALL(response_encoder_, encodeData(_, true)).WillOnce(AddBufferToString(&response_body)); + + startRequest(); + + EXPECT_EQ("envoy overloaded", response_body); + EXPECT_EQ(1U, stats_.named_.downstream_rq_overload_close_.value()); + + // Let the load shed point allow a new stream. + EXPECT_CALL(accept_new_stream_point, shouldShedLoad()).WillOnce(Return(false)); + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, true)); + EXPECT_CALL(*decoder_filters_[0], decodeComplete()); + startRequest(true); + + // Clean up. + EXPECT_CALL(response_encoder_, encodeHeaders(_, true)); + expectOnDestroy(); + + decoder_filters_[0]->callbacks_->streamInfo().setResponseCodeDetails(""); + decoder_filters_[0]->callbacks_->encodeHeaders( + ResponseHeaderMapPtr{new TestResponseHeaderMapImpl{{":status", "200"}}}, true, "details"); +} + TEST_F(HttpConnectionManagerImplTest, TestStopAllIterationAndBufferOnDecodingPathFirstFilter) { setup(false, "envoy-custom-server", false); setUpEncoderAndDecoder(true, true); diff --git a/test/integration/overload_integration_test.cc b/test/integration/overload_integration_test.cc index 2f803f9f3568..1e4b56e8951d 100644 --- a/test/integration/overload_integration_test.cc +++ b/test/integration/overload_integration_test.cc @@ -403,11 +403,16 @@ class LoadShedPointIntegrationTest : public BaseOverloadIntegrationTest, INSTANTIATE_TEST_SUITE_P(Protocols, LoadShedPointIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( - {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, + {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2, + Http::CodecClient::Type::HTTP3}, {FakeHttpConnection::Type::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); TEST_P(LoadShedPointIntegrationTest, ListenerAcceptShedsLoad) { + // QUIC uses UDP, not TCP. + if (downstreamProtocol() == Http::CodecClient::Type::HTTP3) { + return; + } autonomous_upstream_ = true; initializeOverloadManager( TestUtility::parseYaml(R"EOF( @@ -442,4 +447,42 @@ TEST_P(LoadShedPointIntegrationTest, ListenerAcceptShedsLoad) { ASSERT_TRUE(response->waitForEndStream()); } +TEST_P(LoadShedPointIntegrationTest, AcceptNewHttpStreamShedsLoad) { + autonomous_upstream_ = true; + initializeOverloadManager( + TestUtility::parseYaml(R"EOF( + name: "envoy.load_shed_points.http_connection_manager_decode_headers" + triggers: + - name: "envoy.resource_monitors.testonly.fake_resource_monitor" + threshold: + value: 0.90 + )EOF")); + + // Put envoy in overloaded state and check that it sends a local reply for the + // new stream. + updateResource(0.95); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http_connection_manager_decode_headers.scale_percent", 100); + + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + auto response_that_will_be_local_reply = + codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 1); + ASSERT_TRUE(response_that_will_be_local_reply->waitForEndStream()); + EXPECT_EQ(response_that_will_be_local_reply->headers().getStatusValue(), "503"); + + // Disable overload, Envoy should proxy the request. + updateResource(0.80); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http_connection_manager_decode_headers.scale_percent", 0); + + auto response_that_will_be_proxied = + codec_client_->makeHeaderOnlyRequest(default_request_headers_); + ASSERT_TRUE(response_that_will_be_proxied->waitForEndStream()); + EXPECT_EQ(response_that_will_be_proxied->headers().getStatusValue(), "200"); + // Should not be incremented as we didn't reject the request. + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 1); +} + } // namespace Envoy From 35943743480395d570a88775534a92beb7fd35ea Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 1 May 2023 15:14:31 -0400 Subject: [PATCH 055/740] lb: moving maglev to extensions (#27037) Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: inline Signed-off-by: Alyssa Wilk --- changelogs/current.yaml | 4 ++-- source/common/upstream/BUILD | 14 ------------- .../common/upstream/cluster_manager_impl.cc | 10 ++++----- .../load_balancing_policies/maglev/BUILD | 20 +++++++++++++++++- .../load_balancing_policies/maglev/config.cc | 12 ++++++----- .../maglev}/maglev_lb.cc | 2 +- .../maglev}/maglev_lb.h | 0 .../load_balancing_policies/subset/BUILD | 2 +- .../subset/subset_lb.cc | 2 +- test/common/upstream/BUILD | 21 ++----------------- .../upstream/load_balancer_benchmark.cc | 2 +- .../load_balancing_policies/maglev/BUILD | 19 +++++++++++++++++ .../maglev}/maglev_lb_test.cc | 2 +- test/integration/BUILD | 1 + test/mocks/upstream/cluster_info.cc | 1 + test/mocks/upstream/cluster_info.h | 1 + 16 files changed, 62 insertions(+), 51 deletions(-) rename source/{common/upstream => extensions/load_balancing_policies/maglev}/maglev_lb.cc (99%) rename source/{common/upstream => extensions/load_balancing_policies/maglev}/maglev_lb.h (100%) rename test/{common/upstream => extensions/load_balancing_policies/maglev}/maglev_lb_test.cc (99%) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index c4463dde2882..3cf56cbeb5ed 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -4,8 +4,8 @@ behavior_changes: # *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* - area: build change: | - Moved the subset LB code into extensions. If you use the subset LB and override extensions_build_config.bzl you will - need to include it explicitly. + Moved the subset and maglev LB code into extensions. If you use these load balancers and override + extensions_build_config.bzl you will need to include them explicitly. minor_behavior_changes: # *Changes that may cause incompatibilities for some users, but should not for most* diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 49b06978e780..16eec190993a 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -85,7 +85,6 @@ envoy_cc_library( ":load_balancer_lib", ":load_stats_reporter_lib", ":od_cds_api_lib", - ":maglev_lb_lib", ":ring_hash_lb_lib", ":host_utility_lib", "//envoy/api:api_interface", @@ -384,19 +383,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "maglev_lb_lib", - srcs = ["maglev_lb.cc"], - hdrs = ["maglev_lb.h"], - deps = [ - ":thread_aware_lb_lib", - ":upstream_lib", - "//source/common/common:bit_array_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/load_balancing_policies/maglev/v3:pkg_cc_proto", - ], -) - envoy_cc_library( name = "ring_hash_lb_lib", srcs = ["ring_hash_lb.cc"], diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 6e65198fa18c..150cbbae7299 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -44,7 +44,6 @@ #include "source/common/upstream/cds_api_impl.h" #include "source/common/upstream/cluster_factory_impl.h" #include "source/common/upstream/load_balancer_impl.h" -#include "source/common/upstream/maglev_lb.h" #include "source/common/upstream/priority_conn_pool_map_impl.h" #include "source/common/upstream/ring_hash_lb.h" @@ -915,10 +914,11 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust } } else if (cluster_reference.info()->lbType() == LoadBalancerType::Maglev) { if (!cluster_reference.info()->lbSubsetInfo().isEnabled()) { - cluster_entry_it->second->thread_aware_lb_ = std::make_unique( - cluster_reference.prioritySet(), cluster_reference.info()->lbStats(), - cluster_reference.info()->statsScope(), runtime_, random_, - cluster_reference.info()->lbMaglevConfig(), cluster_reference.info()->lbConfig()); + auto& factory = Config::Utility::getAndCheckFactoryByName( + "envoy.load_balancing_policies.maglev"); + cluster_entry_it->second->thread_aware_lb_ = + factory.create(*cluster_reference.info(), cluster_reference.prioritySet(), runtime_, + random_, time_source_); } } else if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided) { cluster_entry_it->second->thread_aware_lb_ = std::move(new_cluster_pair.second); diff --git a/source/extensions/load_balancing_policies/maglev/BUILD b/source/extensions/load_balancing_policies/maglev/BUILD index a3f88251c56c..02852a17f644 100644 --- a/source/extensions/load_balancing_policies/maglev/BUILD +++ b/source/extensions/load_balancing_policies/maglev/BUILD @@ -1,6 +1,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_extension", + "envoy_cc_library", "envoy_extension_package", ) @@ -8,15 +9,32 @@ licenses(["notice"]) # Apache 2 envoy_extension_package() +envoy_cc_library( + name = "maglev_lb_lib", + srcs = ["maglev_lb.cc"], + hdrs = ["maglev_lb.h"], + deps = [ + "//source/common/common:bit_array_lib", + "//source/common/upstream:thread_aware_lb_lib", + "//source/common/upstream:upstream_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/maglev/v3:pkg_cc_proto", + ], +) + envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + extra_visibility = [ + # previously considered core code. + "//test:__subpackages__", + ], deps = [ + ":maglev_lb_lib", "//source/common/common:minimal_logger_lib", "//source/common/upstream:load_balancer_factory_base_lib", "//source/common/upstream:load_balancer_lib", - "//source/common/upstream:maglev_lb_lib", "@envoy_api//envoy/extensions/load_balancing_policies/maglev/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/load_balancing_policies/maglev/config.cc b/source/extensions/load_balancing_policies/maglev/config.cc index 494b5bd7b61d..0b1a4e89105f 100644 --- a/source/extensions/load_balancing_policies/maglev/config.cc +++ b/source/extensions/load_balancing_policies/maglev/config.cc @@ -2,7 +2,7 @@ #include "envoy/extensions/load_balancing_policies/maglev/v3/maglev.pb.h" -#include "source/common/upstream/maglev_lb.h" +#include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" namespace Envoy { namespace Extensions { @@ -18,10 +18,12 @@ Upstream::ThreadAwareLoadBalancerPtr Factory::create(const Upstream::ClusterInfo dynamic_cast( cluster_info.loadBalancingPolicy().get()); - // The load balancing policy configuration will be loaded and validated in the main thread when we - // load the cluster configuration. So we can assume the configuration is valid here. - ASSERT(typed_config != nullptr, - "Invalid load balancing policy configuration for maglev load balancer"); + // Assume legacy config. + if (!typed_config) { + return std::make_unique( + priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random, + cluster_info.lbMaglevConfig(), cluster_info.lbConfig()); + } return std::make_unique( priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random, diff --git a/source/common/upstream/maglev_lb.cc b/source/extensions/load_balancing_policies/maglev/maglev_lb.cc similarity index 99% rename from source/common/upstream/maglev_lb.cc rename to source/extensions/load_balancing_policies/maglev/maglev_lb.cc index 9b847b9781a1..03332ec6c376 100644 --- a/source/common/upstream/maglev_lb.cc +++ b/source/extensions/load_balancing_policies/maglev/maglev_lb.cc @@ -1,4 +1,4 @@ -#include "source/common/upstream/maglev_lb.h" +#include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" #include "envoy/config/cluster/v3/cluster.pb.h" diff --git a/source/common/upstream/maglev_lb.h b/source/extensions/load_balancing_policies/maglev/maglev_lb.h similarity index 100% rename from source/common/upstream/maglev_lb.h rename to source/extensions/load_balancing_policies/maglev/maglev_lb.h diff --git a/source/extensions/load_balancing_policies/subset/BUILD b/source/extensions/load_balancing_policies/subset/BUILD index 951b7f4e09b9..14d5eceaf0f2 100644 --- a/source/extensions/load_balancing_policies/subset/BUILD +++ b/source/extensions/load_balancing_policies/subset/BUILD @@ -22,9 +22,9 @@ envoy_cc_library( "//source/common/protobuf", "//source/common/protobuf:utility_lib", "//source/common/upstream:load_balancer_lib", - "//source/common/upstream:maglev_lb_lib", "//source/common/upstream:ring_hash_lb_lib", "//source/common/upstream:upstream_lib", + "//source/extensions/load_balancing_policies/maglev:maglev_lb_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.cc b/source/extensions/load_balancing_policies/subset/subset_lb.cc index 20dedf08e161..16ac15fcf551 100644 --- a/source/extensions/load_balancing_policies/subset/subset_lb.cc +++ b/source/extensions/load_balancing_policies/subset/subset_lb.cc @@ -12,8 +12,8 @@ #include "source/common/config/well_known_names.h" #include "source/common/protobuf/utility.h" #include "source/common/upstream/load_balancer_impl.h" -#include "source/common/upstream/maglev_lb.h" #include "source/common/upstream/ring_hash_lb.h" +#include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" #include "absl/container/node_hash_set.h" diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 8fdc80087cd8..9b6e8408af00 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -79,6 +79,7 @@ envoy_cc_test( "//source/extensions/clusters/strict_dns:strict_dns_cluster_lib", "//source/extensions/health_checkers/http:health_checker_lib", "//source/extensions/health_checkers/tcp:health_checker_lib", + "//source/extensions/load_balancing_policies/maglev:config", "//source/extensions/network/dns_resolver/cares:config", "//source/extensions/transport_sockets/tls:config", "//test/config:v2_link_hacks", @@ -445,24 +446,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "maglev_lb_test", - srcs = ["maglev_lb_test.cc"], - deps = [ - ":utility_lib", - "//source/common/upstream:maglev_lb_lib", - "//test/mocks:common_lib", - "//test/mocks/upstream:cluster_info_mocks", - "//test/mocks/upstream:host_mocks", - "//test/mocks/upstream:host_set_mocks", - "//test/mocks/upstream:load_balancer_context_mock", - "//test/mocks/upstream:priority_set_mocks", - "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "bounded_load_hlb_test", srcs = ["bounded_load_hlb_test.cc"], @@ -484,9 +467,9 @@ envoy_cc_benchmark_binary( deps = [ "//source/common/common:random_generator_lib", "//source/common/memory:stats_lib", - "//source/common/upstream:maglev_lb_lib", "//source/common/upstream:ring_hash_lb_lib", "//source/common/upstream:upstream_lib", + "//source/extensions/load_balancing_policies/maglev:config", "//source/extensions/load_balancing_policies/subset:config", "//test/common/upstream:utility_lib", "//test/mocks/upstream:cluster_info_mocks", diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc index 95cc869af6fc..c882d88e7a10 100644 --- a/test/common/upstream/load_balancer_benchmark.cc +++ b/test/common/upstream/load_balancer_benchmark.cc @@ -6,9 +6,9 @@ #include "source/common/common/random_generator.h" #include "source/common/memory/stats.h" -#include "source/common/upstream/maglev_lb.h" #include "source/common/upstream/ring_hash_lb.h" #include "source/common/upstream/upstream_impl.h" +#include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" #include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include "test/benchmark/main.h" diff --git a/test/extensions/load_balancing_policies/maglev/BUILD b/test/extensions/load_balancing_policies/maglev/BUILD index 19201eba617b..87d543682990 100644 --- a/test/extensions/load_balancing_policies/maglev/BUILD +++ b/test/extensions/load_balancing_policies/maglev/BUILD @@ -11,6 +11,25 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_extension_cc_test( + name = "maglev_lb_test", + srcs = ["maglev_lb_test.cc"], + extension_names = ["envoy.load_balancing_policies.maglev"], + deps = [ + "//source/extensions/load_balancing_policies/maglev:maglev_lb_lib", + "//test/common/upstream:utility_lib", + "//test/mocks:common_lib", + "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:host_mocks", + "//test/mocks/upstream:host_set_mocks", + "//test/mocks/upstream:load_balancer_context_mock", + "//test/mocks/upstream:priority_set_mocks", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + ], +) + envoy_extension_cc_test( name = "config_test", srcs = ["config_test.cc"], diff --git a/test/common/upstream/maglev_lb_test.cc b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc similarity index 99% rename from test/common/upstream/maglev_lb_test.cc rename to test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc index 11401249e6b8..850a2fa00493 100644 --- a/test/common/upstream/maglev_lb_test.cc +++ b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc @@ -2,7 +2,7 @@ #include "envoy/config/cluster/v3/cluster.pb.h" -#include "source/common/upstream/maglev_lb.h" +#include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" #include "test/common/upstream/utility.h" #include "test/mocks/common.h" diff --git a/test/integration/BUILD b/test/integration/BUILD index 39938958fd7f..3f02e24f4b08 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -952,6 +952,7 @@ envoy_cc_test_library( "base_integration_test.h", ], deps = [ + "//source/extensions/load_balancing_policies/maglev:config", "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib", "//source/server:proto_descriptors_lib", "//source/extensions/request_id/uuid:config", diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc index 3262123813b9..6fd33414a641 100644 --- a/test/mocks/upstream/cluster_info.cc +++ b/test/mocks/upstream/cluster_info.cc @@ -198,6 +198,7 @@ MockClusterInfo::MockClusterInfo() manager.applyFilterFactoryCb({}, factory_cb); return true; })); + ON_CALL(*this, loadBalancingPolicy).WillByDefault(ReturnRef(load_balancing_policy_)); ON_CALL(*this, makeHeaderValidator(_)).WillByDefault(Invoke([&](Http::Protocol protocol) { return header_validator_factory_ ? header_validator_factory_->createClientHeaderValidator( protocol, codecStats(protocol)) diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index bbcbde483319..3c92b980557d 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -253,6 +253,7 @@ class MockClusterInfo : public ClusterInfo { mutable Http::Http1::CodecStats::AtomicPtr http1_codec_stats_; mutable Http::Http2::CodecStats::AtomicPtr http2_codec_stats_; mutable Http::Http3::CodecStats::AtomicPtr http3_codec_stats_; + ProtobufTypes::MessagePtr load_balancing_policy_; Http::HeaderValidatorFactoryPtr header_validator_factory_; }; From 9224e44ea37dc1531fc0c2b3a0bd9a6335f5c69c Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 1 May 2023 20:20:36 +0100 Subject: [PATCH 056/740] ci: Consolidate artifact targets (#27079) Signed-off-by: Ryan Northey --- .azure-pipelines/bazel.yml | 2 +- .azure-pipelines/stage/publish.yml | 22 ++++++++-------------- .azure-pipelines/stage/verify.yml | 6 +++--- ci/build_setup.sh | 7 ++++++- ci/do_ci.sh | 8 ++++---- 5 files changed, 22 insertions(+), 23 deletions(-) diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index 1a819712efd1..3eb11e8499bb 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -220,6 +220,6 @@ steps: - task: PublishBuildArtifacts@1 inputs: pathtoPublish: "$(Build.StagingDirectory)/envoy" - artifactName: ${{ parameters.ciTarget }}${{ parameters.artifactSuffix }} + artifactName: ${{ parameters.ciTarget }} timeoutInMinutes: 10 condition: eq('${{ parameters.publishEnvoy }}', 'true') diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index e4f6102d285d..c74654654e55 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -95,13 +95,7 @@ jobs: inputs: buildType: current artifactName: "bazel.release" - itemPattern: "bazel.release/bin/*" - targetPath: $(Build.StagingDirectory) - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "bazel.release.arm64" - itemPattern: "bazel.release.arm64/bin/*" + itemPattern: "bazel.release/**/bin/*" targetPath: $(Build.StagingDirectory) - bash: | set -e @@ -109,12 +103,12 @@ jobs: mkdir -p linux/amd64 linux/arm64 # x64 - cp -a $(Build.StagingDirectory)/bazel.release/bin/release.tar.zst linux/amd64/release.tar.zst - cp -a $(Build.StagingDirectory)/bazel.release/bin/schema_validator_tool linux/amd64/schema_validator_tool + cp -a $(Build.StagingDirectory)/bazel.release/x64/bin/release.tar.zst linux/amd64/release.tar.zst + cp -a $(Build.StagingDirectory)/bazel.release/x64/bin/schema_validator_tool linux/amd64/schema_validator_tool # arm64 - cp -a $(Build.StagingDirectory)/bazel.release.arm64/bin/release.tar.zst linux/arm64/release.tar.zst - cp -a $(Build.StagingDirectory)/bazel.release.arm64/bin/schema_validator_tool linux/arm64/schema_validator_tool + cp -a $(Build.StagingDirectory)/bazel.release/arm64/bin/release.tar.zst linux/arm64/release.tar.zst + cp -a $(Build.StagingDirectory)/bazel.release/arm64/bin/schema_validator_tool linux/arm64/schema_validator_tool # Debug what files appear to have been downloaded find linux -type f -name "*" | xargs ls -l @@ -156,7 +150,7 @@ jobs: inputs: buildType: current artifactName: "bazel.release" - itemPattern: "bazel.release/bin/*" + itemPattern: "bazel.release/x64/bin/*" targetPath: $(Build.StagingDirectory) - template: ../bazel.yml parameters: @@ -173,8 +167,8 @@ jobs: - task: DownloadBuildArtifacts@0 inputs: buildType: current - artifactName: "bazel.release.arm64" - itemPattern: "bazel.release.arm64/bin/*" + artifactName: "bazel.release" + itemPattern: "bazel.release/arm64/bin/*" targetPath: $(Build.StagingDirectory) - template: ../bazel.yml parameters: diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml index 8f51023fe5a9..d5f514766b4a 100644 --- a/.azure-pipelines/stage/verify.yml +++ b/.azure-pipelines/stage/verify.yml @@ -38,7 +38,7 @@ jobs: inputs: buildType: current artifactName: "bazel.distribution" - itemPattern: "bazel.distribution/packages.x64.tar.gz" + itemPattern: "bazel.distribution/x64/packages.x64.tar.gz" downloadType: single targetPath: $(Build.StagingDirectory) - script: ci/run_envoy_docker.sh 'ci/do_ci.sh verify_distro' @@ -60,8 +60,8 @@ jobs: - task: DownloadBuildArtifacts@0 inputs: buildType: current - artifactName: "bazel.distribution.arm64" - itemPattern: "bazel.distribution.arm64/packages.arm64.tar.gz" + artifactName: "bazel.distribution" + itemPattern: "bazel.distribution/arm64/packages.arm64.tar.gz" downloadType: single targetPath: $(Build.StagingDirectory) - script: ci/run_envoy_docker.sh 'ci/do_ci.sh verify_distro' diff --git a/ci/build_setup.sh b/ci/build_setup.sh index 84e186f985fd..a2398af0e6be 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -161,9 +161,14 @@ fi [[ "${BAZEL_EXPUNGE}" == "1" ]] && bazel clean "${BAZEL_BUILD_OPTIONS[@]}" --expunge +if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + ENVOY_BUILD_DIR="${BUILD_DIR}/envoy/x64" +else + ENVOY_BUILD_DIR="${BUILD_DIR}/envoy/arm64" +fi # Also setup some space for building Envoy standalone. -export ENVOY_BUILD_DIR="${BUILD_DIR}"/envoy +export ENVOY_BUILD_DIR mkdir -p "${ENVOY_BUILD_DIR}" # This is where we copy build deliverables to. diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 10d072243830..24c83c5d95e6 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -253,9 +253,9 @@ case $CI_TARGET in # Extract the Envoy binary from the tarball mkdir -p distribution/custom if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - ENVOY_RELEASE_TARBALL="/build/bazel.release/bin/release.tar.zst" + ENVOY_RELEASE_TARBALL="/build/bazel.release/x64/bin/release.tar.zst" else - ENVOY_RELEASE_TARBALL="/build/bazel.release.arm64/bin/release.tar.zst" + ENVOY_RELEASE_TARBALL="/build/bazel.release/arm64/bin/release.tar.zst" fi bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/zstd -- --stdout -d "$ENVOY_RELEASE_TARBALL" | tar xfO - envoy > distribution/custom/envoy @@ -564,9 +564,9 @@ case $CI_TARGET in ;; verify_distro) if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - PACKAGE_BUILD=/build/bazel.distribution/packages.x64.tar.gz + PACKAGE_BUILD=/build/bazel.distribution/x64/packages.x64.tar.gz else - PACKAGE_BUILD=/build/bazel.distribution.arm64/packages.arm64.tar.gz + PACKAGE_BUILD=/build/bazel.distribution/arm64/packages.arm64.tar.gz fi bazel run "${BAZEL_BUILD_OPTIONS[@]}" //distribution:verify_packages "$PACKAGE_BUILD" ;; From dcaa10a56db0e3ac1c727e3cc244ea40d9943608 Mon Sep 17 00:00:00 2001 From: Kuo-Chung Hsu Date: Mon, 1 May 2023 12:24:08 -0700 Subject: [PATCH 057/740] thrift_proxy: add access log support for local reply (#27057) * thrift_proxy: add access log support for local reply Signed-off-by: kuochunghsu --- .../network/thrift_proxy/conn_manager.cc | 68 +++++++++++++------ .../network/thrift_proxy/conn_manager.h | 8 ++- .../network/thrift_proxy/conn_manager_test.cc | 23 +++---- 3 files changed, 67 insertions(+), 32 deletions(-) diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.cc b/source/extensions/filters/network/thrift_proxy/conn_manager.cc index 301066f93860..c71d621bd512 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.cc +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.cc @@ -107,10 +107,11 @@ void ConnectionManager::dispatch() { resetAllRpcs(true); } -void ConnectionManager::sendLocalReply(MessageMetadata& metadata, const DirectResponse& response, - bool end_stream) { +absl::optional +ConnectionManager::sendLocalReply(MessageMetadata& metadata, const DirectResponse& response, + bool end_stream) { if (read_callbacks_->connection().state() == Network::Connection::State::Closed) { - return; + return absl::nullopt; } DirectResponse::ResponseType result = DirectResponse::ResponseType::Exception; @@ -140,6 +141,7 @@ void ConnectionManager::sendLocalReply(MessageMetadata& metadata, const DirectRe stats_.response_exception_.inc(); break; } + return result; } void ConnectionManager::continueDecoding() { @@ -357,20 +359,7 @@ FilterStatus ConnectionManager::ResponseDecoder::messageBegin(MessageMetadataSha } } - ProtobufWkt::Struct stats_obj; - auto& fields_map = *stats_obj.mutable_fields(); - auto& response_fields_map = *fields_map["response"].mutable_struct_value()->mutable_fields(); - - response_fields_map["transport_type"] = - ValueUtil::stringValue(TransportNames::get().fromType(decoder_->transportType())); - response_fields_map["protocol_type"] = - ValueUtil::stringValue(ProtocolNames::get().fromType(decoder_->protocolType())); - response_fields_map["message_type"] = ValueUtil::stringValue( - metadata->hasMessageType() ? MessageTypeNames::get().fromType(metadata->messageType()) : "-"); - response_fields_map["reply_type"] = ValueUtil::stringValue( - metadata->hasReplyType() ? ReplyTypeNames::get().fromType(metadata->replyType()) : "-"); - - parent_.streamInfo().setDynamicMetadata("thrift.proxy", stats_obj); + parent_.recordResponseAccessLog(metadata); return parent_.applyEncoderFilters(DecoderEvent::MessageBegin, metadata, protocol_converter_); } @@ -817,6 +806,43 @@ bool ConnectionManager::ActiveRpc::passthroughSupported() const { return true; } +void ConnectionManager::ActiveRpc::recordResponseAccessLog( + DirectResponse::ResponseType direct_response_type, const MessageMetadataSharedPtr& metadata) { + if (direct_response_type == DirectResponse::ResponseType::Exception) { + recordResponseAccessLog(MessageTypeNames::get().fromType(MessageType::Exception), "-"); + return; + } + const std::string& message_type_name = + metadata->hasMessageType() ? MessageTypeNames::get().fromType(metadata->messageType()) : "-"; + const std::string& reply_type_name = ReplyTypeNames::get().fromType( + direct_response_type == DirectResponse::ResponseType::SuccessReply ? ReplyType::Success + : ReplyType::Error); + recordResponseAccessLog(message_type_name, reply_type_name); +} + +void ConnectionManager::ActiveRpc::recordResponseAccessLog( + const MessageMetadataSharedPtr& metadata) { + recordResponseAccessLog( + metadata->hasMessageType() ? MessageTypeNames::get().fromType(metadata->messageType()) : "-", + metadata->hasReplyType() ? ReplyTypeNames::get().fromType(metadata->replyType()) : "-"); +} + +void ConnectionManager::ActiveRpc::recordResponseAccessLog(const std::string& message_type, + const std::string& reply_type) { + ProtobufWkt::Struct stats_obj; + auto& fields_map = *stats_obj.mutable_fields(); + auto& response_fields_map = *fields_map["response"].mutable_struct_value()->mutable_fields(); + + response_fields_map["transport_type"] = + ValueUtil::stringValue(TransportNames::get().fromType(downstreamTransportType())); + response_fields_map["protocol_type"] = + ValueUtil::stringValue(ProtocolNames::get().fromType(downstreamProtocolType())); + response_fields_map["message_type"] = ValueUtil::stringValue(message_type); + response_fields_map["reply_type"] = ValueUtil::stringValue(reply_type); + + streamInfo().setDynamicMetadata("thrift.proxy", stats_obj); +} + FilterStatus ConnectionManager::ActiveRpc::passthroughData(Buffer::Instance& data) { passthrough_ = true; return applyDecoderFilters(DecoderEvent::PassthroughData, &data); @@ -876,7 +902,7 @@ FilterStatus ConnectionManager::ActiveRpc::messageBegin(MessageMetadataSharedPtr request_fields_map["transport_type"] = ValueUtil::stringValue(TransportNames::get().fromType(downstreamTransportType())); request_fields_map["protocol_type"] = - ValueUtil::stringValue(ProtocolNames::get().fromType(parent_.decoder_->protocolType())); + ValueUtil::stringValue(ProtocolNames::get().fromType(downstreamProtocolType())); request_fields_map["message_type"] = ValueUtil::stringValue( metadata->hasMessageType() ? MessageTypeNames::get().fromType(metadata->messageType()) : "-"); @@ -1026,7 +1052,11 @@ void ConnectionManager::ActiveRpc::sendLocalReply(const DirectResponse& response cm.stats_.downstream_response_drain_close_.inc(); } - parent_.sendLocalReply(*localReplyMetadata_, response, end_stream); + auto result = parent_.sendLocalReply(*localReplyMetadata_, response, end_stream); + // Only report while the local reply is successfully written. + if (result.has_value()) { + recordResponseAccessLog(*result, localReplyMetadata_); + } if (end_stream) { return; diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.h b/source/extensions/filters/network/thrift_proxy/conn_manager.h index 9c9613acfa41..5829b1b0a00b 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.h +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.h @@ -324,6 +324,11 @@ class ConnectionManager : public Network::ReadFilter, bool passthroughSupported() const; + void recordResponseAccessLog(const MessageMetadataSharedPtr& metadata); + void recordResponseAccessLog(DirectResponse::ResponseType direct_response_type, + const MessageMetadataSharedPtr& metadata); + void recordResponseAccessLog(const std::string& message_type, const std::string& reply_type); + // Apply filters to the decoder_event. // @param filter the last filter which is already applied to the decoder_event. // nullptr indicates none is applied and the decoder_event is applied from the @@ -372,7 +377,8 @@ class ConnectionManager : public Network::ReadFilter, void continueDecoding(); void dispatch(); - void sendLocalReply(MessageMetadata& metadata, const DirectResponse& response, bool end_stream); + absl::optional + sendLocalReply(MessageMetadata& metadata, const DirectResponse& response, bool end_stream); void doDeferredRpcDestroy(ActiveRpc& rpc); void resetAllRpcs(bool local_reset); void emitLogEntry(const Http::RequestHeaderMap* request_headers, diff --git a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc index af6a4bceea0b..76c7e887dfef 100644 --- a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc @@ -852,9 +852,8 @@ TEST_F(ThriftConnectionManagerTest, OnDataHandlesProtocolError) { filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0U, stats_.request_active_.value()); - - EXPECT_EQ(access_log_data_, - "name cluster passthrough_enabled=false framed binary call - - - - 0 0 0 -\n"); + EXPECT_EQ(access_log_data_, "name cluster passthrough_enabled=false framed binary call framed " + "binary exception - 0 0 0 -\n"); } TEST_F(ThriftConnectionManagerTest, OnDataHandlesProtocolErrorDuringMessageBegin) { @@ -1360,7 +1359,7 @@ TEST_F(ThriftConnectionManagerTest, RequestAndResponseProtocolError) { EXPECT_EQ(1U, store_.counter("test.response_decoding_error").value()); EXPECT_EQ(access_log_data_, "name cluster passthrough_enabled=false framed binary call framed " - "binary reply success 0 0 0 -\n"); + "binary exception - 0 0 0 -\n"); } TEST_F(ThriftConnectionManagerTest, RequestAndTransportApplicationException) { @@ -1404,8 +1403,8 @@ TEST_F(ThriftConnectionManagerTest, RequestAndTransportApplicationException) { EXPECT_EQ(0U, store_.counter("test.response_error").value()); EXPECT_EQ(1U, store_.counter("test.response_decoding_error").value()); - EXPECT_EQ(access_log_data_, - "name cluster passthrough_enabled=false header binary call - - - - 0 0 0 -\n"); + EXPECT_EQ(access_log_data_, "name cluster passthrough_enabled=false header binary call header " + "binary exception - 0 0 0 -\n"); } TEST_F(ThriftConnectionManagerTest, BadFunctionCallExceptionHandling) { @@ -1468,8 +1467,8 @@ TEST_F(ThriftConnectionManagerTest, RequestAndGarbageResponse) { EXPECT_EQ(0U, store_.counter("test.response_success").value()); EXPECT_EQ(0U, store_.counter("test.response_error").value()); - EXPECT_EQ(access_log_data_, - "name cluster passthrough_enabled=false framed binary call - - - - 0 0 0 -\n"); + EXPECT_EQ(access_log_data_, "name cluster passthrough_enabled=false framed binary call framed " + "binary exception - 0 0 0 -\n"); } TEST_F(ThriftConnectionManagerTest, PipelinedRequestAndResponse) { @@ -1956,8 +1955,8 @@ TEST_F(ThriftConnectionManagerTest, OnDataWithFilterSendsLocalReply) { EXPECT_EQ(0U, stats_.request_active_.value()); EXPECT_EQ(1U, store_.counter("test.response_success").value()); - EXPECT_EQ(access_log_data_, - "name cluster passthrough_enabled=false framed binary call - - - - 0 0 0 -\n"); + EXPECT_EQ(access_log_data_, "name cluster passthrough_enabled=false framed binary call framed " + "binary call success 0 0 0 -\n"); } // Tests multiple filters where one invokes sendLocalReply with an error reply. @@ -2010,8 +2009,8 @@ TEST_F(ThriftConnectionManagerTest, OnDataWithFilterSendsLocalErrorReply) { EXPECT_EQ(0U, stats_.request_active_.value()); EXPECT_EQ(1U, store_.counter("test.response_error").value()); - EXPECT_EQ(access_log_data_, - "name cluster passthrough_enabled=false framed binary call - - - - 0 0 0 -\n"); + EXPECT_EQ(access_log_data_, "name cluster passthrough_enabled=false framed binary call framed " + "binary call error 0 0 0 -\n"); } // sendLocalReply does nothing, when the remote closed the connection. From 21b6356463e256dc64d37edb09e5df1c99273e05 Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 1 May 2023 21:09:07 +0100 Subject: [PATCH 058/740] ci/mobile: Hide CI progress in .bazelrc (#27045) Signed-off-by: Ryan Northey --- .azure-pipelines/bazel.yml | 2 +- .bazelrc | 4 ++++ .github/workflows/mobile-compile_time_options.yml | 1 + ci/build_setup.sh | 2 -- mobile/.bazelrc | 1 + 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index 3eb11e8499bb..6a5d8e6961f4 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -172,7 +172,7 @@ steps: BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance ${{ if eq(parameters.rbe, false) }}: - BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelBuildExtraOptions }}" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}" BAZEL_REMOTE_CACHE: $(LocalBuildCache) ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)" diff --git a/.bazelrc b/.bazelrc index 637a9a65bea2..ef72b14b6ee0 100644 --- a/.bazelrc +++ b/.bazelrc @@ -333,6 +333,10 @@ build:docker-tsan --config=rbe-toolchain-tsan # CI configurations build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com +build:remote-ci --config=ci +# Note this config is used by mobile CI also. +build:ci --noshow_progress +build:ci --noshow_loading_progress # Build Event Service build:google-bes --bes_backend=grpcs://buildeventservice.googleapis.com diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index f085ab5dce04..309cb69b4cea 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -115,6 +115,7 @@ jobs: --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ //:android_dist \ && ./bazelw test \ + --config=ci \ --fat_apk_cpu=x86_64 \ --define=admin_html=enabled \ --define=envoy_mobile_request_compression=disabled \ diff --git a/ci/build_setup.sh b/ci/build_setup.sh index a2398af0e6be..9ad3a8dd0577 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -131,8 +131,6 @@ BAZEL_BUILD_OPTIONS=( "--verbose_failures" "--experimental_generate_json_trace_profile" "--test_output=errors" - "--noshow_progress" - "--noshow_loading_progress" "--action_env=CLANG_FORMAT" "${BAZEL_BUILD_EXTRA_OPTIONS[@]}" "${BAZEL_EXTRA_TEST_OPTIONS[@]}") diff --git a/mobile/.bazelrc b/mobile/.bazelrc index 52fe52ffbda4..fcc169dd3b0e 100644 --- a/mobile/.bazelrc +++ b/mobile/.bazelrc @@ -138,6 +138,7 @@ build:coverage --instrumentation_filter="//library/common[/:]" # to add platform-specific options. Avoid using this config directly and # instead use a platform-specific config ############################################################################# +build:remote-ci-common --config=ci build:remote-ci-common --config=remote build:remote-ci-common --google_default_credentials=false build:remote-ci-common --remote_cache=grpcs://envoy.cluster.engflow.com From 7e1c8f926795c6516c898f7e2ea68ac4b3d3cee7 Mon Sep 17 00:00:00 2001 From: Michael Kaufmann Date: Mon, 1 May 2023 22:12:58 +0200 Subject: [PATCH 059/740] docs: fix the license URL of the dependency "dd-trace-cpp" (#27054) Signed-off-by: Michael Kaufmann --- bazel/repository_locations.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 6191b6905e94..aef44d2ca1cd 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -533,7 +533,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2023-03-31", cpe = "N/A", license = "Apache-2.0", - license_url = "https://github.com/DataDog/dd-trace-cpp/blob/v{version}/LICENSE", + license_url = "https://github.com/DataDog/dd-trace-cpp/blob/v{version}/LICENSE.md", ), com_github_google_benchmark = dict( project_name = "Benchmark", From 8232c0d4a58fede9d82816e2b283d8b4065ce239 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 1 May 2023 16:53:36 -0400 Subject: [PATCH 060/740] Fix compliance issues for iOS builds (#27027) Risk Level: Low Testing: Unit Tests Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A Signed-off-by: Yan Avlasov --- source/common/http/conn_manager_impl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 3d6fdae25f76..d9b968ae239c 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -949,7 +949,7 @@ bool ConnectionManagerImpl::ActiveStream::validateHeaders() { failure = !transformation_result.ok(); redirect = transformation_result.action() == Http::HeaderValidator::RequestHeadersTransformationResult::Action::Redirect; - failure_details = transformation_result.details(); + failure_details = std::string(transformation_result.details()); } if (failure) { std::function modify_headers; @@ -998,7 +998,7 @@ bool ConnectionManagerImpl::ActiveStream::validateTrailers() { if (transformation_result.ok()) { return true; } - failure_details = transformation_result.details(); + failure_details = std::string(transformation_result.details()); } Code response_code = Code::BadRequest; From 0f2cd3f8fb2fe26cb1a9b37da450a02b34ec2c00 Mon Sep 17 00:00:00 2001 From: RenjieTang Date: Mon, 1 May 2023 13:58:35 -0700 Subject: [PATCH 061/740] [mobile]remove Java and GMScore impl from Cronvoy (#27039) Commit Message: remove Java and GMScore impl from Cronvoy Additional Description: This change removes all the unnecessary Cronet implementations and only keeps the native Envoy Mobile one. Note that the other impls are never used. This change should make Cronvoy a lot more simpler and should marginally reduce the binary size. Risk Level: Low Testing: Existing test Docs Changes: n/a Release Notes: n/a Platform Specific Features: Android change only Signed-off-by: Renjie Tang --- mobile/envoy_build_config/BUILD | 2 +- .../java/org/chromium/net/CronetProvider.java | 56 -- .../library/java/org/chromium/net/impl/BUILD | 6 - .../chromium/net/impl/InputStreamChannel.java | 69 -- .../chromium/net/impl/JavaCronetEngine.java | 186 ---- .../net/impl/JavaCronetEngineBuilderImpl.java | 25 - .../chromium/net/impl/JavaCronetProvider.java | 53 - .../net/impl/JavaUploadDataSinkBase.java | 245 ----- .../org/chromium/net/impl/JavaUrlRequest.java | 916 ------------------ mobile/library/swift/BUILD | 2 +- .../chromium/net/BidirectionalStreamTest.java | 61 +- .../java/org/chromium/net/BrotliTest.java | 4 - .../chromium/net/CronetEngineBuilderTest.java | 2 - .../org/chromium/net/CronetStressTest.java | 2 - .../net/CronetUrlRequestContextTest.java | 33 - .../chromium/net/CronetUrlRequestTest.java | 30 +- .../org/chromium/net/DiskStorageTest.java | 4 - .../java/org/chromium/net/GetStatusTest.java | 2 - .../chromium/net/RequestFinishedInfoTest.java | 13 - .../chromium/net/UploadDataProvidersTest.java | 2 - .../chromium/net/testing/CronetTestRule.java | 99 +- .../net/testing/CronetTestRuleTest.java | 31 - .../net/testing/MockUrlRequestJobFactory.java | 16 +- 23 files changed, 22 insertions(+), 1837 deletions(-) delete mode 100644 mobile/library/java/org/chromium/net/impl/InputStreamChannel.java delete mode 100644 mobile/library/java/org/chromium/net/impl/JavaCronetEngine.java delete mode 100644 mobile/library/java/org/chromium/net/impl/JavaCronetEngineBuilderImpl.java delete mode 100644 mobile/library/java/org/chromium/net/impl/JavaCronetProvider.java delete mode 100644 mobile/library/java/org/chromium/net/impl/JavaUploadDataSinkBase.java delete mode 100644 mobile/library/java/org/chromium/net/impl/JavaUrlRequest.java diff --git a/mobile/envoy_build_config/BUILD b/mobile/envoy_build_config/BUILD index f9f6f7a0a556..3b3c27bddcf1 100644 --- a/mobile/envoy_build_config/BUILD +++ b/mobile/envoy_build_config/BUILD @@ -33,8 +33,8 @@ envoy_cc_library( "@envoy//source/extensions/request_id/uuid:config", "@envoy//source/extensions/transport_sockets/http_11_proxy:upstream_config", "@envoy//source/extensions/transport_sockets/raw_buffer:config", - "@envoy//source/extensions/transport_sockets/tls/cert_validator:cert_validator_lib", "@envoy//source/extensions/transport_sockets/tls:config", + "@envoy//source/extensions/transport_sockets/tls/cert_validator:cert_validator_lib", "@envoy//source/extensions/upstreams/http/generic:config", "@envoy_mobile//library/common/extensions/cert_validator/platform_bridge:config", "@envoy_mobile//library/common/extensions/filters/http/local_error:config", diff --git a/mobile/library/java/org/chromium/net/CronetProvider.java b/mobile/library/java/org/chromium/net/CronetProvider.java index 0ff90c0597d8..62c2249a0507 100644 --- a/mobile/library/java/org/chromium/net/CronetProvider.java +++ b/mobile/library/java/org/chromium/net/CronetProvider.java @@ -106,25 +106,10 @@ public String toString() { + "enabled=" + isEnabled() + "]"; } - /** Name of the Java {@link CronetProvider} class. */ - private static final String JAVA_CRONET_PROVIDER_CLASS = - "org.chromium.net.impl.JavaCronetProvider"; - /** Name of the native {@link CronetProvider} class. */ private static final String NATIVE_CRONET_PROVIDER_CLASS = "org.chromium.net.impl.NativeCronetProvider"; - /** {@link CronetProvider} class that is packaged with Google Play Services. */ - private static final String PLAY_SERVICES_CRONET_PROVIDER_CLASS = - "com.google.android.gms.net.PlayServicesCronetProvider"; - - /** - * {@link CronetProvider} a deprecated class that may be packaged with some old versions of Google - * Play Services. - */ - private static final String GMS_CORE_CRONET_PROVIDER_CLASS = - "com.google.android.gms.net.GmsCoreCronetProvider"; - /** * Returns an unmodifiable list of all available {@link CronetProvider}s. The providers are * returned in no particular order. Some of the returned providers may be in a disabled state and @@ -135,12 +120,7 @@ public String toString() { public static List getAllProviders(Context context) { // Use LinkedHashSet to preserve the order and eliminate duplicate providers. Set providers = new LinkedHashSet<>(); - addCronetProviderFromResourceFile(context, providers); - addCronetProviderImplByClassName(context, PLAY_SERVICES_CRONET_PROVIDER_CLASS, providers, - false); - addCronetProviderImplByClassName(context, GMS_CORE_CRONET_PROVIDER_CLASS, providers, false); addCronetProviderImplByClassName(context, NATIVE_CRONET_PROVIDER_CLASS, providers, false); - addCronetProviderImplByClassName(context, JAVA_CRONET_PROVIDER_CLASS, providers, false); return Collections.unmodifiableList(new ArrayList<>(providers)); } @@ -191,40 +171,4 @@ private static void logReflectiveOperationException(String className, boolean lo } } } - - /** - * Attempts to add a provider specified in the app resource file to a set. - * - * @param providers the set of providers to add the new provider to. - * @return {@code true} if the provider was added to the set; {@code false} if the app resources - * do not include the string with {@link #RES_KEY_CRONET_IMPL_CLASS} key. - * @throws RuntimeException if the provider cannot be found or instantiated. - */ - private static boolean addCronetProviderFromResourceFile(Context context, - Set providers) { - int resId = context.getResources().getIdentifier(RES_KEY_CRONET_IMPL_CLASS, "string", - context.getPackageName()); - // Resource not found - if (resId == 0) { - // The resource wasn't included in the app; therefore, there is nothing to add. - return false; - } - String className = context.getResources().getString(resId); - - // If the resource specifies a well known provider, don't load it because - // there will be an attempt to load it anyways. - if (className == null || className.equals(PLAY_SERVICES_CRONET_PROVIDER_CLASS) || - className.equals(GMS_CORE_CRONET_PROVIDER_CLASS) || - className.equals(JAVA_CRONET_PROVIDER_CLASS) || - className.equals(NATIVE_CRONET_PROVIDER_CLASS)) { - return false; - } - - if (!addCronetProviderImplByClassName(context, className, providers, true)) { - Log.e(TAG, "Unable to instantiate Cronet implementation class " + className + - " that is listed as in the app string resource file under " + - RES_KEY_CRONET_IMPL_CLASS + " key"); - } - return true; - } } diff --git a/mobile/library/java/org/chromium/net/impl/BUILD b/mobile/library/java/org/chromium/net/impl/BUILD index 07f0023d2871..d6d8ec07ad23 100644 --- a/mobile/library/java/org/chromium/net/impl/BUILD +++ b/mobile/library/java/org/chromium/net/impl/BUILD @@ -28,12 +28,6 @@ android_library( "Executors.java", "HttpReason.java", "ImplVersion.java", - "InputStreamChannel.java", - "JavaCronetEngine.java", - "JavaCronetEngineBuilderImpl.java", - "JavaCronetProvider.java", - "JavaUploadDataSinkBase.java", - "JavaUrlRequest.java", "NativeCronetEngineBuilderImpl.java", "NativeCronetEngineBuilderWithLibraryLoaderImpl.java", "NativeCronetProvider.java", diff --git a/mobile/library/java/org/chromium/net/impl/InputStreamChannel.java b/mobile/library/java/org/chromium/net/impl/InputStreamChannel.java deleted file mode 100644 index 760915d7c519..000000000000 --- a/mobile/library/java/org/chromium/net/impl/InputStreamChannel.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.chromium.net.impl; - -import androidx.annotation.NonNull; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Adapts an {@link InputStream} into a {@link ReadableByteChannel}, exactly like - * {@link java.nio.channels.Channels#newChannel(InputStream)} does, but more efficiently, since it - * does not allocate a temporary buffer if it doesn't have to, and it freely takes advantage of - * {@link FileInputStream}'s trivial conversion to {@link java.nio.channels.FileChannel}. - */ -final class InputStreamChannel implements ReadableByteChannel { - private static final int MAX_TMP_BUFFER_SIZE = 16384; - private static final int MIN_TMP_BUFFER_SIZE = 4096; - private final InputStream mInputStream; - private final AtomicBoolean mIsOpen = new AtomicBoolean(true); - - private InputStreamChannel(@NonNull InputStream inputStream) { mInputStream = inputStream; } - - static ReadableByteChannel wrap(@NonNull InputStream inputStream) { - if (inputStream instanceof FileInputStream) { - return ((FileInputStream)inputStream).getChannel(); - } - return new InputStreamChannel(inputStream); - } - - @Override - public int read(ByteBuffer dst) throws IOException { - final int read; - if (dst.hasArray()) { - read = mInputStream.read(dst.array(), dst.arrayOffset() + dst.position(), dst.remaining()); - if (read > 0) { - dst.position(dst.position() + read); - } - } else { - // Since we're allocating a buffer for every read, we want to choose a good size - on - // Android, the only case where a ByteBuffer won't have a backing byte[] is if it was - // created wrapping a void * in native code, or if it represents a memory-mapped file. - // Especially in the latter case, we want to avoid allocating a buffer that could be - // very large. - final int possibleToRead = - Math.min(Math.max(mInputStream.available(), MIN_TMP_BUFFER_SIZE), dst.remaining()); - final int reasonableToRead = Math.min(MAX_TMP_BUFFER_SIZE, possibleToRead); - byte[] tmpBuf = new byte[reasonableToRead]; - read = mInputStream.read(tmpBuf); - if (read > 0) { - dst.put(tmpBuf, 0, read); - } - } - return read; - } - - @Override - public boolean isOpen() { - return mIsOpen.get(); - } - - @Override - public void close() throws IOException { - if (mIsOpen.compareAndSet(true, false)) { - mInputStream.close(); - } - } -} diff --git a/mobile/library/java/org/chromium/net/impl/JavaCronetEngine.java b/mobile/library/java/org/chromium/net/impl/JavaCronetEngine.java deleted file mode 100644 index f6e6d0e8c8b7..000000000000 --- a/mobile/library/java/org/chromium/net/impl/JavaCronetEngine.java +++ /dev/null @@ -1,186 +0,0 @@ -package org.chromium.net.impl; - -import static android.os.Process.THREAD_PRIORITY_BACKGROUND; -import static android.os.Process.THREAD_PRIORITY_MORE_FAVORABLE; - -import java.io.IOException; -import java.net.Proxy; -import java.net.URL; -import java.net.URLConnection; -import java.net.URLStreamHandler; -import java.net.URLStreamHandlerFactory; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import org.chromium.net.BidirectionalStream; -import org.chromium.net.ExperimentalBidirectionalStream; -import org.chromium.net.NetworkQualityRttListener; -import org.chromium.net.NetworkQualityThroughputListener; -import org.chromium.net.RequestFinishedInfo; -import org.chromium.net.UrlRequest; - -/** - * {@link java.net.HttpURLConnection} backed CronetEngine. - * - *

Does not support netlogs, transferred data measurement, bidistream, cache, or priority. - */ -public final class JavaCronetEngine extends CronetEngineBase { - private final String mUserAgent; - private final ExecutorService mExecutorService; - - public JavaCronetEngine(CronetEngineBuilderImpl builder) { - // On android, all background threads (and all threads that are part - // of background processes) are put in a cgroup that is allowed to - // consume up to 5% of CPU - these worker threads spend the vast - // majority of their time waiting on I/O, so making them contend with - // background applications for a slice of CPU doesn't make much sense. - // We want to hurry up and get idle. - final int threadPriority = - builder.threadPriority(THREAD_PRIORITY_BACKGROUND + THREAD_PRIORITY_MORE_FAVORABLE); - mUserAgent = builder.getUserAgent(); - mExecutorService = new ThreadPoolExecutor( - 10, 20, 50, TimeUnit.SECONDS, new LinkedBlockingQueue(), new ThreadFactory() { - @Override - public Thread newThread(final Runnable r) { - return Executors.defaultThreadFactory().newThread(new Runnable() { - @Override - public void run() { - Thread.currentThread().setName("JavaCronetEngine"); - android.os.Process.setThreadPriority(threadPriority); - r.run(); - } - }); - } - }); - } - - @Override - public UrlRequestBase - createRequest(String url, UrlRequest.Callback callback, Executor executor, int priority, - Collection connectionAnnotations, boolean disableCache, - boolean disableConnectionMigration, boolean allowDirectExecutor, - boolean trafficStatsTagSet, int trafficStatsTag, boolean trafficStatsUidSet, - int trafficStatsUid, RequestFinishedInfo.Listener requestFinishedListener, - int idempotency) { - return new JavaUrlRequest(callback, mExecutorService, executor, url, mUserAgent, - allowDirectExecutor, trafficStatsTagSet, trafficStatsTag, - trafficStatsUidSet, trafficStatsUid); - } - - @Override - protected ExperimentalBidirectionalStream - createBidirectionalStream(String url, BidirectionalStream.Callback callback, Executor executor, - String httpMethod, List> requestHeaders, - @StreamPriority int priority, - boolean delayRequestHeadersUntilFirstFlush, - Collection connectionAnnotations, boolean trafficStatsTagSet, - int trafficStatsTag, boolean trafficStatsUidSet, int trafficStatsUid) { - throw new UnsupportedOperationException( - "Can't create a bidi stream - httpurlconnection doesn't have those APIs"); - } - - @Override - public ExperimentalBidirectionalStream.Builder - newBidirectionalStreamBuilder(String url, BidirectionalStream.Callback callback, - Executor executor) { - throw new UnsupportedOperationException( - "The bidirectional stream API is not supported by the Java implementation " - + "of Cronet Engine"); - } - - @Override - public String getVersionString() { - return "CronetHttpURLConnection/" + ImplVersion.getCronetVersionWithLastChange(); - } - - @Override - public void shutdown() { - mExecutorService.shutdown(); - } - - @Override - public void startNetLogToFile(String fileName, boolean logAll) {} - - @Override - public void startNetLogToDisk(String dirPath, boolean logAll, int maxSize) {} - - @Override - public void stopNetLog() {} - - @Override - public byte[] getGlobalMetricsDeltas() { - return new byte[0]; - } - - @Override - public int getEffectiveConnectionType() { - return EFFECTIVE_CONNECTION_TYPE_UNKNOWN; - } - - @Override - public int getHttpRttMs() { - return CONNECTION_METRIC_UNKNOWN; - } - - @Override - public int getTransportRttMs() { - return CONNECTION_METRIC_UNKNOWN; - } - - @Override - public int getDownstreamThroughputKbps() { - return CONNECTION_METRIC_UNKNOWN; - } - - @Override - public void configureNetworkQualityEstimatorForTesting(boolean useLocalHostRequests, - boolean useSmallerResponses, - boolean disableOfflineCheck) {} - - @Override - public void addRttListener(NetworkQualityRttListener listener) {} - - @Override - public void removeRttListener(NetworkQualityRttListener listener) {} - - @Override - public void addThroughputListener(NetworkQualityThroughputListener listener) {} - - @Override - public void removeThroughputListener(NetworkQualityThroughputListener listener) {} - - @Override - public void addRequestFinishedListener(RequestFinishedInfo.Listener listener) {} - - @Override - public void removeRequestFinishedListener(RequestFinishedInfo.Listener listener) {} - - @Override - public URLConnection openConnection(URL url) throws IOException { - return url.openConnection(); - } - - @Override - public URLConnection openConnection(URL url, Proxy proxy) throws IOException { - return url.openConnection(proxy); - } - - @Override - public URLStreamHandlerFactory createURLStreamHandlerFactory() { - // Returning null causes this factory to pass though, which ends up using the platform's - // implementation. - return new URLStreamHandlerFactory() { - @Override - public URLStreamHandler createURLStreamHandler(String protocol) { - return null; - } - }; - } -} diff --git a/mobile/library/java/org/chromium/net/impl/JavaCronetEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/JavaCronetEngineBuilderImpl.java deleted file mode 100644 index 631af5b09e4b..000000000000 --- a/mobile/library/java/org/chromium/net/impl/JavaCronetEngineBuilderImpl.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.chromium.net.impl; - -import android.content.Context; -import org.chromium.net.ExperimentalCronetEngine; -import org.chromium.net.ICronetEngineBuilder; - -/** - * Implementation of {@link ICronetEngineBuilder} that builds Java-based Cronet engine. - */ -final class JavaCronetEngineBuilderImpl extends CronetEngineBuilderImpl { - /** - * Builder for Platform Cronet Engine. - * - * @param context Android {@link Context} for engine to use. - */ - public JavaCronetEngineBuilderImpl(Context context) { super(context); } - - @Override - public ExperimentalCronetEngine build() { - if (getUserAgent() == null) { - setUserAgent(getDefaultUserAgent()); - } - return new JavaCronetEngine(this); - } -} diff --git a/mobile/library/java/org/chromium/net/impl/JavaCronetProvider.java b/mobile/library/java/org/chromium/net/impl/JavaCronetProvider.java deleted file mode 100644 index 28bb3686348d..000000000000 --- a/mobile/library/java/org/chromium/net/impl/JavaCronetProvider.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.chromium.net.impl; - -import android.content.Context; -import java.util.Arrays; -import org.chromium.net.CronetEngine; -import org.chromium.net.CronetProvider; -import org.chromium.net.ExperimentalCronetEngine; -import org.chromium.net.ICronetEngineBuilder; - -/** - * Implementation of {@link CronetProvider} that creates {@link CronetEngine.Builder} - * for building the Java-based implementation of {@link CronetEngine}. - */ -public class JavaCronetProvider extends CronetProvider { - /** - * Constructor. - * - * @param context Android context to use. - */ - public JavaCronetProvider(Context context) { super(context); } - - @Override - public CronetEngine.Builder createBuilder() { - ICronetEngineBuilder impl = new JavaCronetEngineBuilderImpl(mContext); - return new ExperimentalCronetEngine.Builder(impl); - } - - @Override - public String getName() { - return CronetProvider.PROVIDER_NAME_FALLBACK; - } - - @Override - public String getVersion() { - return ImplVersion.getCronetVersion(); - } - - @Override - public boolean isEnabled() { - return true; - } - - @Override - public int hashCode() { - return Arrays.hashCode(new Object[] {JavaCronetProvider.class, mContext}); - } - - @Override - public boolean equals(Object other) { - return other == this || (other instanceof JavaCronetProvider && - this.mContext.equals(((JavaCronetProvider)other).mContext)); - } -} diff --git a/mobile/library/java/org/chromium/net/impl/JavaUploadDataSinkBase.java b/mobile/library/java/org/chromium/net/impl/JavaUploadDataSinkBase.java deleted file mode 100644 index 4bfc8a7478ce..000000000000 --- a/mobile/library/java/org/chromium/net/impl/JavaUploadDataSinkBase.java +++ /dev/null @@ -1,245 +0,0 @@ -package org.chromium.net.impl; - -import androidx.annotation.IntDef; -import java.io.IOException; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.nio.ByteBuffer; -import java.util.Locale; -import java.util.concurrent.Executor; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.AtomicInteger; -import org.chromium.net.UploadDataProvider; -import org.chromium.net.UploadDataSink; -import org.chromium.net.impl.Executors.CheckedRunnable; - -/** - * Base class for Java UrlRequest implementations of UploadDataSink. Handles asynchronicity and - * manages the executors for this upload. - */ -public abstract class JavaUploadDataSinkBase extends UploadDataSink { - @IntDef({SinkState.AWAITING_READ_RESULT, SinkState.AWAITING_REWIND_RESULT, SinkState.UPLOADING, - SinkState.NOT_STARTED}) - @Retention(RetentionPolicy.SOURCE) - @interface SinkState { - int AWAITING_READ_RESULT = 0; - int AWAITING_REWIND_RESULT = 1; - int UPLOADING = 2; - int NOT_STARTED = 3; - } - - public static final int DEFAULT_UPLOAD_BUFFER_SIZE = 8192; - - private final AtomicInteger /*SinkState*/ mSinkState = new AtomicInteger(SinkState.NOT_STARTED); - private final Executor mUserUploadExecutor; - private final Executor mExecutor; - private final UploadDataProvider mUploadProvider; - private ByteBuffer mBuffer; - /** This holds the total bytes to send (the content-length). -1 if unknown. */ - private long mTotalBytes; - /** This holds the bytes written so far */ - private long mWrittenBytes; - - public JavaUploadDataSinkBase(final Executor userExecutor, Executor executor, - UploadDataProvider provider) { - mUserUploadExecutor = new Executor() { - @Override - public void execute(Runnable runnable) { - try { - userExecutor.execute(runnable); - } catch (RejectedExecutionException e) { - processUploadError(e); - } - } - }; - mExecutor = executor; - mUploadProvider = provider; - } - - @Override - public void onReadSucceeded(final boolean finalChunk) { - if (!mSinkState.compareAndSet(/* expected= */ SinkState.AWAITING_READ_RESULT, - /* updated= */ SinkState.UPLOADING)) { - throw new IllegalStateException( - "onReadSucceeded() called when not awaiting a read result; in state: " + - mSinkState.get()); - } - mExecutor.execute(getErrorSettingRunnable(new CheckedRunnable() { - @Override - public void run() throws Exception { - mBuffer.flip(); - if (mTotalBytes != -1 && mTotalBytes - mWrittenBytes < mBuffer.remaining()) { - processUploadError(new IllegalArgumentException(String.format( - Locale.getDefault(), "Read upload data length %d exceeds expected length %d", - mWrittenBytes + mBuffer.remaining(), mTotalBytes))); - return; - } - - mWrittenBytes += processSuccessfulRead(mBuffer); - - if (mWrittenBytes < mTotalBytes || (mTotalBytes == -1 && !finalChunk)) { - mBuffer.clear(); - mSinkState.set(SinkState.AWAITING_READ_RESULT); - executeOnUploadExecutor(new CheckedRunnable() { - @Override - public void run() throws Exception { - mUploadProvider.read(JavaUploadDataSinkBase.this, mBuffer); - } - }); - } else if (mTotalBytes == -1) { - finish(); - } else if (mTotalBytes == mWrittenBytes) { - finish(); - } else { - processUploadError(new IllegalArgumentException(String.format( - Locale.getDefault(), "Read upload data length %d exceeds expected length %d", - mWrittenBytes, mTotalBytes))); - } - } - })); - } - - @Override - public void onRewindSucceeded() { - if (!mSinkState.compareAndSet(/* expected= */ SinkState.AWAITING_REWIND_RESULT, - /* updated= */ SinkState.UPLOADING)) { - throw new IllegalStateException( - "onRewindSucceeded() called when not awaiting a rewind; in state: " + mSinkState.get()); - } - startRead(); - } - - @Override - public void onReadError(Exception exception) { - processUploadError(exception); - } - - @Override - public void onRewindError(Exception exception) { - processUploadError(exception); - } - - private void startRead() { - mExecutor.execute(getErrorSettingRunnable(new CheckedRunnable() { - @Override - public void run() throws Exception { - initializeRead(); - mSinkState.set(SinkState.AWAITING_READ_RESULT); - executeOnUploadExecutor(new CheckedRunnable() { - @Override - public void run() throws Exception { - mUploadProvider.read(JavaUploadDataSinkBase.this, mBuffer); - } - }); - } - })); - } - - /** - * Helper method to execute a checked runnable on the upload executor and process any errors - * that occur as upload errors. - * - * @param runnable the runnable to attempt to run and check for errors - */ - private void executeOnUploadExecutor(CheckedRunnable runnable) { - try { - mUserUploadExecutor.execute(getUploadErrorSettingRunnable(runnable)); - } catch (RejectedExecutionException e) { - processUploadError(e); - } - } - - /** - * Starts the upload. This method can be called multiple times. If it is not the first time it - * is called the {@link UploadDataProvider} must rewind. - * - * @param firstTime true if this is the first time this {@link UploadDataSink} has started an - * upload - */ - public void start(final boolean firstTime) { - executeOnUploadExecutor(new CheckedRunnable() { - @Override - public void run() throws Exception { - mTotalBytes = mUploadProvider.getLength(); - if (mTotalBytes == 0) { - finish(); - } else { - // If we know how much data we have to upload, and it's small, we can save - // memory by allocating a reasonably sized buffer to read into. - if (mTotalBytes > 0 && mTotalBytes < DEFAULT_UPLOAD_BUFFER_SIZE) { - // Allocate one byte more than necessary, to detect callers uploading - // more bytes than they specified in length. - mBuffer = ByteBuffer.allocateDirect((int)mTotalBytes + 1); - } else { - mBuffer = ByteBuffer.allocateDirect(DEFAULT_UPLOAD_BUFFER_SIZE); - } - - initializeStart(mTotalBytes); - - if (firstTime) { - startRead(); - } else { - mSinkState.set(SinkState.AWAITING_REWIND_RESULT); - mUploadProvider.rewind(JavaUploadDataSinkBase.this); - } - } - } - }); - } - - /** - * Gets a runnable that checks for errors and processes them by setting an error state when - * executing a {@link CheckedRunnable}. - * - * @param runnable The runnable to run. - * @return a runnable that checks for errors - */ - protected abstract Runnable getErrorSettingRunnable(CheckedRunnable runnable); - - /** - * Gets a runnable that checks for errors and processes them by setting an upload error state - * when executing a {@link CheckedRunnable}. - * - * @param runnable The runnable to run. - * @return a runnable that checks for errors - */ - protected abstract Runnable getUploadErrorSettingRunnable(CheckedRunnable runnable); - - /** - * Processes an error encountered while uploading data. - * - * @param error the {@link Throwable} to process - */ - protected abstract void processUploadError(final Throwable error); - - /** - * Called when a successful read has occurred and there is new data in the {@code mBuffer} to - * process. - * - * @return the number of bytes processed in this read - * @throws IOException - */ - protected abstract int processSuccessfulRead(ByteBuffer buffer) throws IOException; - - /** - * Finishes this upload. Called when the upload is complete. - * - * @throws IOException - */ - protected abstract void finish() throws IOException; - - /** - * Initializes the {@link UploadDataSink} before each call to {@code read} in the - * {@link UploadDataProvider}. - * - * @throws IOException - */ - protected abstract void initializeRead() throws IOException; - - /** - * Initializes the {@link UploadDataSink} at the start of the upload. - * - * @param totalBytes the total number of bytes to be retrieved in this upload - */ - protected abstract void initializeStart(long totalBytes); -} diff --git a/mobile/library/java/org/chromium/net/impl/JavaUrlRequest.java b/mobile/library/java/org/chromium/net/impl/JavaUrlRequest.java deleted file mode 100644 index 26cee0bebf9a..000000000000 --- a/mobile/library/java/org/chromium/net/impl/JavaUrlRequest.java +++ /dev/null @@ -1,916 +0,0 @@ -package org.chromium.net.impl; - -import android.annotation.TargetApi; -import android.net.TrafficStats; -import android.os.Build; -import android.util.Log; -import androidx.annotation.GuardedBy; -import androidx.annotation.IntDef; -import androidx.annotation.Nullable; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; -import java.util.AbstractMap.SimpleEntry; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.Executor; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import org.chromium.net.CronetException; -import org.chromium.net.InlineExecutionProhibitedException; -import org.chromium.net.UploadDataProvider; -import org.chromium.net.UrlResponseInfo; -import org.chromium.net.impl.Executors.CheckedRunnable; -import org.chromium.net.impl.Executors.DirectPreventingExecutor; -import io.envoyproxy.envoymobile.utilities.ThreadStatsUid; - -/** - * Pure java UrlRequest, backed by {@link HttpURLConnection}. - */ -@TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH) // TrafficStats only available on ICS -final class JavaUrlRequest extends UrlRequestBase { - - /** - * State interface for keeping track of the internal state of a {@link UrlRequestBase}. - * 
-   *               /- AWAITING_FOLLOW_REDIRECT <- REDIRECT_RECEIVED <-\     /- READING <--\
-   *               |                                                  |     |             |
-   *               V                                                  /     V             /
-   * NOT_STARTED -> STARTED -----------------------------------------------> AWAITING_READ -------
-   * --> COMPLETE
-   *
- */ - @IntDef({State.NOT_STARTED, State.STARTED, State.REDIRECT_RECEIVED, - State.AWAITING_FOLLOW_REDIRECT, State.AWAITING_READ, State.READING, State.ERROR, - State.COMPLETE, State.CANCELLED}) - @Retention(RetentionPolicy.SOURCE) - @interface State { - - int NOT_STARTED = 0; - int STARTED = 1; - int REDIRECT_RECEIVED = 2; - int AWAITING_FOLLOW_REDIRECT = 3; - int AWAITING_READ = 4; - int READING = 5; - int ERROR = 6; - int COMPLETE = 7; - int CANCELLED = 8; - } - - private static final String X_ANDROID = "X-Android"; - private static final String X_ANDROID_SELECTED_TRANSPORT = "X-Android-Selected-Transport"; - private static final String TAG = JavaUrlRequest.class.getSimpleName(); - private static final int DEFAULT_CHUNK_LENGTH = JavaUploadDataSinkBase.DEFAULT_UPLOAD_BUFFER_SIZE; - private static final String USER_AGENT = "User-Agent"; - private final AsyncUrlRequestCallback mCallbackAsync; - private final Executor mExecutor; - private final String mUserAgent; - private final Map mRequestHeaders = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - private final List mUrlChain = new ArrayList<>(); - /** - * This is the source of thread safety in this class - no other synchronization is performed. By - * compare-and-swapping from one state to another, we guarantee that operations aren't running - * concurrently. Only the winner of a compare-and-swapping proceeds. - * - *

A caller can lose a compare-and-swapping for three reasons - user error (two calls to - * read() without waiting for the read to succeed), runtime error (network code or user code - * throws an exception), or cancellation. - */ - private final AtomicInteger /* State */ mState = new AtomicInteger(State.NOT_STARTED); - private final AtomicBoolean mUploadProviderClosed = new AtomicBoolean(false); - - private final boolean mAllowDirectExecutor; - - /* These don't change with redirects */ - private String mInitialMethod; - private VersionSafeCallbacks.UploadDataProviderWrapper mUploadDataProvider; - private Executor mUploadExecutor; - - /** - * Holds a subset of StatusValues - {@link State#STARTED} can represent {@link - * Status#SENDING_REQUEST} or {@link Status#WAITING_FOR_RESPONSE}. While the distinction isn't - * needed to implement the logic in this class, it is needed to implement {@link - * #getStatus(StatusListener)}. - * - *

Concurrency notes - this value is not atomically updated with mState, so there is some - * risk that we'd get an inconsistent snapshot of both - however, it also happens that this value - * is only used with the STARTED state, so it's inconsequential. - */ - @StatusValues private volatile int mAdditionalStatusDetails = Status.INVALID; - - /* These change with redirects. */ - private String mCurrentUrl; - @Nullable private ReadableByteChannel mResponseChannel; // Only accessed on mExecutor. - private UrlResponseInfoImpl mUrlResponseInfo; - private String mPendingRedirectUrl; - private HttpURLConnection mCurrentUrlConnection; // Only accessed on mExecutor. - private OutputStreamDataSink mOutputStreamDataSink; // Only accessed on mExecutor. - - /** - * @param executor The executor used for reading and writing from sockets - * @param userExecutor The executor used to dispatch to {@code callback} - */ - JavaUrlRequest(Callback callback, final Executor executor, Executor userExecutor, String url, - String userAgent, boolean allowDirectExecutor, boolean trafficStatsTagSet, - int trafficStatsTag, final boolean trafficStatsUidSet, final int trafficStatsUid) { - if (url == null) { - throw new NullPointerException("URL is required"); - } - if (callback == null) { - throw new NullPointerException("Listener is required"); - } - if (executor == null) { - throw new NullPointerException("Executor is required"); - } - if (userExecutor == null) { - throw new NullPointerException("userExecutor is required"); - } - - this.mAllowDirectExecutor = allowDirectExecutor; - this.mCallbackAsync = new AsyncUrlRequestCallback(callback, userExecutor); - final int trafficStatsTagToUse = - trafficStatsTagSet ? trafficStatsTag : TrafficStats.getThreadStatsTag(); - this.mExecutor = new SerializingExecutor(new Executor() { - @Override - public void execute(final Runnable command) { - executor.execute(new Runnable() { - @Override - public void run() { - int oldTag = TrafficStats.getThreadStatsTag(); - TrafficStats.setThreadStatsTag(trafficStatsTagToUse); - if (trafficStatsUidSet) { - ThreadStatsUid.set(trafficStatsUid); - } - try { - command.run(); - } finally { - if (trafficStatsUidSet) { - ThreadStatsUid.clear(); - } - TrafficStats.setThreadStatsTag(oldTag); - } - } - }); - } - }); - this.mCurrentUrl = url; - this.mUserAgent = userAgent; - } - - @Override - public void setHttpMethod(String method) { - checkNotStarted(); - if (method == null) { - throw new NullPointerException("Method is required."); - } - if ("OPTIONS".equalsIgnoreCase(method) || "GET".equalsIgnoreCase(method) || - "HEAD".equalsIgnoreCase(method) || "POST".equalsIgnoreCase(method) || - "PUT".equalsIgnoreCase(method) || "DELETE".equalsIgnoreCase(method) || - "TRACE".equalsIgnoreCase(method) || "PATCH".equalsIgnoreCase(method)) { - mInitialMethod = method; - } else { - throw new IllegalArgumentException("Invalid http method " + method); - } - } - - private void checkNotStarted() { - @State int state = mState.get(); - if (state != State.NOT_STARTED) { - throw new IllegalStateException("Request is already started. State is: " + state); - } - } - - @Override - public void addHeader(String header, String value) { - checkNotStarted(); - if (!isValidHeaderName(header) || value.contains("\r\n")) { - throw new IllegalArgumentException("Invalid header " + header + "=" + value); - } - if (mRequestHeaders.containsKey(header)) { - mRequestHeaders.remove(header); - } - mRequestHeaders.put(header, value); - } - - private boolean isValidHeaderName(String header) { - for (int i = 0; i < header.length(); i++) { - char c = header.charAt(i); - switch (c) { - case '(': - case ')': - case '<': - case '>': - case '@': - case ',': - case ';': - case ':': - case '\\': - case '\'': - case '/': - case '[': - case ']': - case '?': - case '=': - case '{': - case '}': - return false; - default: { - if (Character.isISOControl(c) || Character.isWhitespace(c)) { - return false; - } - } - } - } - return true; - } - - @Override - public void setUploadDataProvider(UploadDataProvider uploadDataProvider, Executor executor) { - if (uploadDataProvider == null) { - throw new NullPointerException("Invalid UploadDataProvider."); - } - if (!mRequestHeaders.containsKey("Content-Type")) { - throw new IllegalArgumentException("Requests with upload data must have a Content-Type."); - } - checkNotStarted(); - if (mInitialMethod == null) { - mInitialMethod = "POST"; - } - this.mUploadDataProvider = - new VersionSafeCallbacks.UploadDataProviderWrapper(uploadDataProvider); - if (mAllowDirectExecutor) { - this.mUploadExecutor = executor; - } else { - this.mUploadExecutor = new DirectPreventingExecutor(executor); - } - } - - private final class OutputStreamDataSink extends JavaUploadDataSinkBase { - - private final HttpURLConnection mUrlConnection; - private final AtomicBoolean mOutputChannelClosed = new AtomicBoolean(false); - private WritableByteChannel mOutputChannel; - private OutputStream mUrlConnectionOutputStream; - - OutputStreamDataSink(final Executor userExecutor, Executor executor, - HttpURLConnection urlConnection, - VersionSafeCallbacks.UploadDataProviderWrapper provider) { - super(userExecutor, executor, provider); - mUrlConnection = urlConnection; - } - - @Override - protected void initializeRead() throws IOException { - if (mOutputChannel == null) { - mAdditionalStatusDetails = Status.CONNECTING; - mUrlConnection.setDoOutput(true); - mUrlConnection.connect(); - mAdditionalStatusDetails = Status.SENDING_REQUEST; - mUrlConnectionOutputStream = mUrlConnection.getOutputStream(); - mOutputChannel = Channels.newChannel(mUrlConnectionOutputStream); - } - } - - void closeOutputChannel() throws IOException { - if (mOutputChannel != null && mOutputChannelClosed.compareAndSet( - /* expected= */ false, /* updated= */ true)) { - mOutputChannel.close(); - } - } - - @Override - protected void finish() throws IOException { - closeOutputChannel(); - fireGetHeaders(); - } - - @Override - protected void initializeStart(long totalBytes) { - if (totalBytes > 0 && totalBytes <= Integer.MAX_VALUE) { - mUrlConnection.setFixedLengthStreamingMode((int)totalBytes); - } else if (totalBytes > Integer.MAX_VALUE && - Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { - mUrlConnection.setFixedLengthStreamingMode(totalBytes); - } else { - // Even if we know the length, but we're running pre-kitkat and it's larger - // than an int can hold, we have to use chunked - otherwise we'll end up - // buffering the whole response in memory. - mUrlConnection.setChunkedStreamingMode(DEFAULT_CHUNK_LENGTH); - } - } - - @Override - protected int processSuccessfulRead(ByteBuffer buffer) throws IOException { - int totalBytesProcessed = 0; - while (buffer.hasRemaining()) { - totalBytesProcessed += mOutputChannel.write(buffer); - } - // Forces a chunk to be sent, rather than buffering to the DEFAULT_CHUNK_LENGTH. - // This allows clients to trickle-upload bytes as they become available without - // introducing latency due to buffering. - mUrlConnectionOutputStream.flush(); - return totalBytesProcessed; - } - - @Override - protected Runnable getErrorSettingRunnable(CheckedRunnable runnable) { - return errorSetting(runnable); - } - - @Override - protected Runnable getUploadErrorSettingRunnable(CheckedRunnable runnable) { - return uploadErrorSetting(runnable); - } - - @Override - protected void processUploadError(Throwable exception) { - enterUploadErrorState(exception); - } - } - - @Override - public void start() { - mAdditionalStatusDetails = Status.CONNECTING; - transitionStates(State.NOT_STARTED, State.STARTED, new Runnable() { - @Override - public void run() { - mUrlChain.add(mCurrentUrl); - fireOpenConnection(); - } - }); - } - - private void enterErrorState(final CronetException error) { - if (setTerminalState(State.ERROR)) { - fireDisconnect(); - fireCloseUploadDataProvider(); - mCallbackAsync.onFailed(mUrlResponseInfo, error); - } - } - - private boolean setTerminalState(@State int error) { - while (true) { - @State int oldState = mState.get(); - switch (oldState) { - case State.NOT_STARTED: - throw new IllegalStateException("Can't enter error state before start"); - case State.ERROR: // fallthrough - case State.COMPLETE: // fallthrough - case State.CANCELLED: - return false; // Already in a terminal state - default: { - if (mState.compareAndSet(/* expected= */ oldState, /* updated= */ error)) { - return true; - } - } - } - } - } - - /** - * Ends the request with an error, caused by an exception thrown from user code. - */ - private void enterUserErrorState(final Throwable error) { - enterErrorState( - new CallbackExceptionImpl("Exception received from UrlRequest.Callback", error)); - } - - /** - * Ends the request with an error, caused by an exception thrown from user code. - */ - private void enterUploadErrorState(final Throwable error) { - enterErrorState(new CallbackExceptionImpl("Exception received from UploadDataProvider", error)); - } - - private void enterCronetErrorState(final Throwable error) { - // TODO(clm) mapping from Java exception (UnknownHostException, for example) to net error - // code goes here. - enterErrorState(new CronetExceptionImpl("System error", error)); - } - - /** - * Atomically swaps from the expected state to a new state. If the swap fails, and it's not due to - * an earlier error or cancellation, throws an exception. - * - * @param afterTransition Callback to run after transition completes successfully. - */ - private void transitionStates(@State int expected, @State int newState, - Runnable afterTransition) { - if (!mState.compareAndSet(expected, newState)) { - @State int state = mState.get(); - if (!(state == State.CANCELLED || state == State.ERROR)) { - throw new IllegalStateException("Invalid state transition - expected " + expected + - " but was " + state); - } - } else { - afterTransition.run(); - } - } - - @Override - public void followRedirect() { - transitionStates(State.AWAITING_FOLLOW_REDIRECT, State.STARTED, new Runnable() { - @Override - public void run() { - mCurrentUrl = mPendingRedirectUrl; - mPendingRedirectUrl = null; - fireOpenConnection(); - } - }); - } - - private void fireGetHeaders() { - mAdditionalStatusDetails = Status.WAITING_FOR_RESPONSE; - mExecutor.execute(errorSetting(new CheckedRunnable() { - @Override - public void run() throws Exception { - if (mCurrentUrlConnection == null) { - return; // We've been cancelled - } - final List> headerList = new ArrayList<>(); - String selectedTransport = "http/1.1"; - String headerValue; - for (int i = 0; (headerValue = mCurrentUrlConnection.getHeaderField(i)) != null; i++) { - String headerKey = mCurrentUrlConnection.getHeaderFieldKey(i); - if (headerKey != null) { - if (X_ANDROID_SELECTED_TRANSPORT.equalsIgnoreCase(headerKey)) { - selectedTransport = headerValue; - } - if (!headerKey.startsWith(X_ANDROID)) { - headerList.add(new SimpleEntry<>(headerKey, headerValue)); - } - } - } - - int responseCode = mCurrentUrlConnection.getResponseCode(); - // Important to copy the list here, because although we never concurrently modify - // the list ourselves, user code might iterate over it while we're redirecting, and - // that would throw ConcurrentModificationException. - mUrlResponseInfo = new UrlResponseInfoImpl( - new ArrayList<>(mUrlChain), responseCode, mCurrentUrlConnection.getResponseMessage(), - Collections.unmodifiableList(headerList), false, selectedTransport, "", 0); - // TODO(clm) actual redirect handling? post -> get and whatnot? - if (responseCode >= 300 && responseCode < 400) { - List locationFields = mUrlResponseInfo.getAllHeaders().get("location"); - if (locationFields != null) { - fireRedirectReceived(locationFields.get(0)); - return; - } - } - fireCloseUploadDataProvider(); - if (responseCode >= 400) { - InputStream inputStream = mCurrentUrlConnection.getErrorStream(); - mResponseChannel = inputStream == null ? null : InputStreamChannel.wrap(inputStream); - mCallbackAsync.onResponseStarted(mUrlResponseInfo); - } else { - mResponseChannel = InputStreamChannel.wrap(mCurrentUrlConnection.getInputStream()); - mCallbackAsync.onResponseStarted(mUrlResponseInfo); - } - } - })); - } - - private void fireCloseUploadDataProvider() { - if (mUploadDataProvider != null && mUploadProviderClosed.compareAndSet( - /* expected= */ false, /* updated= */ true)) { - try { - mUploadExecutor.execute(uploadErrorSetting(new CheckedRunnable() { - @Override - public void run() throws Exception { - mUploadDataProvider.close(); - } - })); - } catch (RejectedExecutionException e) { - Log.e(TAG, "Exception when closing uploadDataProvider", e); - } - } - } - - private void fireRedirectReceived(final String locationField) { - transitionStates(State.STARTED, State.REDIRECT_RECEIVED, new Runnable() { - @Override - public void run() { - mPendingRedirectUrl = URI.create(mCurrentUrl).resolve(locationField).toString(); - mUrlChain.add(mPendingRedirectUrl); - transitionStates(State.REDIRECT_RECEIVED, State.AWAITING_FOLLOW_REDIRECT, new Runnable() { - @Override - public void run() { - mCallbackAsync.onRedirectReceived(mUrlResponseInfo, mPendingRedirectUrl); - } - }); - } - }); - } - - private void fireOpenConnection() { - mExecutor.execute(errorSetting(new CheckedRunnable() { - @Override - public void run() throws Exception { - // If we're cancelled, then our old connection will be disconnected for us and - // we shouldn't open a new one. - if (mState.get() == State.CANCELLED) { - return; - } - - final URL url = new URL(mCurrentUrl); - if (mCurrentUrlConnection != null) { - mCurrentUrlConnection.disconnect(); - mCurrentUrlConnection = null; - } - mCurrentUrlConnection = (HttpURLConnection)url.openConnection(); - mCurrentUrlConnection.setInstanceFollowRedirects(false); - if (!mRequestHeaders.containsKey(USER_AGENT)) { - mRequestHeaders.put(USER_AGENT, mUserAgent); - } - for (Map.Entry entry : mRequestHeaders.entrySet()) { - mCurrentUrlConnection.setRequestProperty(entry.getKey(), entry.getValue()); - } - if (mInitialMethod == null) { - mInitialMethod = "GET"; - } - mCurrentUrlConnection.setRequestMethod(mInitialMethod); - if (mUploadDataProvider != null) { - mOutputStreamDataSink = new OutputStreamDataSink( - mUploadExecutor, mExecutor, mCurrentUrlConnection, mUploadDataProvider); - mOutputStreamDataSink.start(mUrlChain.size() == 1); - } else { - mAdditionalStatusDetails = Status.CONNECTING; - mCurrentUrlConnection.connect(); - fireGetHeaders(); - } - } - })); - } - - private Runnable errorSetting(final CheckedRunnable delegate) { - return new Runnable() { - @Override - public void run() { - try { - delegate.run(); - } catch (Throwable t) { - enterCronetErrorState(t); - } - } - }; - } - - private Runnable userErrorSetting(final CheckedRunnable delegate) { - return new Runnable() { - @Override - public void run() { - try { - delegate.run(); - } catch (Throwable t) { - enterUserErrorState(t); - } - } - }; - } - - private Runnable uploadErrorSetting(final CheckedRunnable delegate) { - return new Runnable() { - @Override - public void run() { - try { - delegate.run(); - } catch (Throwable t) { - enterUploadErrorState(t); - } - } - }; - } - - @Override - public void read(final ByteBuffer buffer) { - Preconditions.checkDirect(buffer); - Preconditions.checkHasRemaining(buffer); - transitionStates(State.AWAITING_READ, State.READING, new Runnable() { - @Override - public void run() { - mExecutor.execute(errorSetting(new CheckedRunnable() { - @Override - public void run() throws Exception { - int read = mResponseChannel == null ? -1 : mResponseChannel.read(buffer); - processReadResult(read, buffer); - } - })); - } - }); - } - - private void processReadResult(int read, final ByteBuffer buffer) throws IOException { - if (read != -1) { - mCallbackAsync.onReadCompleted(mUrlResponseInfo, buffer); - } else { - if (mResponseChannel != null) { - mResponseChannel.close(); - } - if (mState.compareAndSet( - /* expected= */ State.READING, /* updated= */ State.COMPLETE)) { - fireDisconnect(); - mCallbackAsync.onSucceeded(mUrlResponseInfo); - } - } - } - - private void fireDisconnect() { - mExecutor.execute(new Runnable() { - @Override - public void run() { - if (mOutputStreamDataSink != null) { - try { - mOutputStreamDataSink.closeOutputChannel(); - } catch (IOException e) { - Log.e(TAG, "Exception when closing OutputChannel", e); - } - } - if (mCurrentUrlConnection != null) { - mCurrentUrlConnection.disconnect(); - mCurrentUrlConnection = null; - } - } - }); - } - - @Override - public void cancel() { - @State int oldState = mState.getAndSet(State.CANCELLED); - switch (oldState) { - // We've just scheduled some user code to run. When they perform their next operation, - // they'll observe it and fail. However, if user code is cancelling in response to one - // of these callbacks, we'll never actually cancel! - // TODO(clm) figure out if it's possible to avoid concurrency in user callbacks. - case State.REDIRECT_RECEIVED: - case State.AWAITING_FOLLOW_REDIRECT: - case State.AWAITING_READ: - - // User code is waiting on us - cancel away! - case State.STARTED: - case State.READING: - fireDisconnect(); - fireCloseUploadDataProvider(); - mCallbackAsync.onCanceled(mUrlResponseInfo); - break; - // The rest are all termination cases - we're too late to cancel. - case State.ERROR: - case State.COMPLETE: - case State.CANCELLED: - break; - default: - break; - } - } - - @Override - public boolean isDone() { - @State int state = mState.get(); - return state == State.COMPLETE || state == State.ERROR || state == State.CANCELLED; - } - - @Override - public void getStatus(StatusListener listener) { - @State int state = mState.get(); - int extraStatus = this.mAdditionalStatusDetails; - - @StatusValues final int status; - switch (state) { - case State.ERROR: - case State.COMPLETE: - case State.CANCELLED: - case State.NOT_STARTED: - status = Status.INVALID; - break; - case State.STARTED: - status = extraStatus; - break; - case State.REDIRECT_RECEIVED: - case State.AWAITING_FOLLOW_REDIRECT: - case State.AWAITING_READ: - status = Status.IDLE; - break; - case State.READING: - status = Status.READING_RESPONSE; - break; - default: - throw new IllegalStateException("Switch is exhaustive: " + state); - } - - mCallbackAsync.sendStatus(new VersionSafeCallbacks.UrlRequestStatusListener(listener), status); - } - - /** - * This wrapper ensures that callbacks are always called on the correct executor - */ - private final class AsyncUrlRequestCallback { - - final VersionSafeCallbacks.UrlRequestCallback mCallback; - final Executor mUserExecutor; - final Executor mFallbackExecutor; - - AsyncUrlRequestCallback(Callback callback, final Executor userExecutor) { - this.mCallback = new VersionSafeCallbacks.UrlRequestCallback(callback); - if (mAllowDirectExecutor) { - this.mUserExecutor = userExecutor; - this.mFallbackExecutor = null; - } else { - mUserExecutor = new DirectPreventingExecutor(userExecutor); - mFallbackExecutor = userExecutor; - } - } - - void sendStatus(final VersionSafeCallbacks.UrlRequestStatusListener listener, - final int status) { - mUserExecutor.execute(new Runnable() { - @Override - public void run() { - listener.onStatus(status); - } - }); - } - - void execute(CheckedRunnable runnable) { - try { - mUserExecutor.execute(userErrorSetting(runnable)); - } catch (RejectedExecutionException e) { - enterErrorState(new CronetExceptionImpl("Exception posting task to executor", e)); - } - } - - void onRedirectReceived(final UrlResponseInfo info, final String newLocationUrl) { - execute(new CheckedRunnable() { - @Override - public void run() throws Exception { - mCallback.onRedirectReceived(JavaUrlRequest.this, info, newLocationUrl); - } - }); - } - - void onResponseStarted(UrlResponseInfo info) { - execute(new CheckedRunnable() { - @Override - public void run() throws Exception { - if (mState.compareAndSet(/* expected= */ State.STARTED, - /* updated= */ State.AWAITING_READ)) { - mCallback.onResponseStarted(JavaUrlRequest.this, mUrlResponseInfo); - } - } - }); - } - - void onReadCompleted(final UrlResponseInfo info, final ByteBuffer byteBuffer) { - execute(new CheckedRunnable() { - @Override - public void run() throws Exception { - if (mState.compareAndSet(/* expected= */ State.READING, - /* updated= */ State.AWAITING_READ)) { - mCallback.onReadCompleted(JavaUrlRequest.this, info, byteBuffer); - } - } - }); - } - - void onCanceled(final UrlResponseInfo info) { - closeResponseChannel(); - mUserExecutor.execute(new Runnable() { - @Override - public void run() { - try { - mCallback.onCanceled(JavaUrlRequest.this, info); - } catch (Exception exception) { - Log.e(TAG, "Exception in onCanceled method", exception); - } - } - }); - } - - void onSucceeded(final UrlResponseInfo info) { - mUserExecutor.execute(new Runnable() { - @Override - public void run() { - try { - mCallback.onSucceeded(JavaUrlRequest.this, info); - } catch (Exception exception) { - Log.e(TAG, "Exception in onSucceeded method", exception); - } - } - }); - } - - void onFailed(final UrlResponseInfo urlResponseInfo, final CronetException e) { - closeResponseChannel(); - Runnable runnable = new Runnable() { - @Override - public void run() { - try { - mCallback.onFailed(JavaUrlRequest.this, urlResponseInfo, e); - } catch (Exception exception) { - Log.e(TAG, "Exception in onFailed method", exception); - } - } - }; - try { - mUserExecutor.execute(runnable); - } catch (InlineExecutionProhibitedException wasDirect) { - if (mFallbackExecutor != null) { - mFallbackExecutor.execute(runnable); - } - } - } - } - - private void closeResponseChannel() { - mExecutor.execute(new Runnable() { - @Override - public void run() { - if (mResponseChannel != null) { - try { - mResponseChannel.close(); - } catch (IOException e) { - // Silently ignored, willingly. - } - mResponseChannel = null; - } - } - }); - } - - // Executor that runs one task at a time on an underlying Executor. - // NOTE: Do not use to wrap user supplied Executor as lock is held while underlying execute() - // is called. - private static final class SerializingExecutor implements Executor { - - private final Executor mUnderlyingExecutor; - private final Runnable mRunTasks = new Runnable() { - @Override - public void run() { - Runnable task; - synchronized (mTaskQueue) { - if (mRunning) { - return; - } - task = mTaskQueue.pollFirst(); - mRunning = task != null; - } - while (task != null) { - boolean threw = true; - try { - task.run(); - threw = false; - } finally { - synchronized (mTaskQueue) { - if (threw) { - // If task.run() threw, this method will abort without looping - // again, so repost to keep running tasks. - mRunning = false; - try { - mUnderlyingExecutor.execute(mRunTasks); - } catch (RejectedExecutionException e) { - // Give up if a task run at shutdown throws. - } - } else { - task = mTaskQueue.pollFirst(); - mRunning = task != null; - } - } - } - } - } - }; - // Queue of tasks to run. Tasks are added to the end and taken from the front. - // Synchronized on itself. - @GuardedBy("mTaskQueue") private final Deque mTaskQueue = new ArrayDeque<>(); - // Indicates if mRunTasks is actively running tasks. Synchronized on mTaskQueue. - @GuardedBy("mTaskQueue") private boolean mRunning; - - SerializingExecutor(Executor underlyingExecutor) { mUnderlyingExecutor = underlyingExecutor; } - - @Override - public void execute(Runnable command) { - synchronized (mTaskQueue) { - mTaskQueue.addLast(command); - try { - mUnderlyingExecutor.execute(mRunTasks); - } catch (RejectedExecutionException e) { - // If shutting down, do not add new tasks to the queue. - mTaskQueue.removeLast(); - } - } - } - } -} diff --git a/mobile/library/swift/BUILD b/mobile/library/swift/BUILD index 9e5f6f0ee1b2..9ddeb5d61ca0 100644 --- a/mobile/library/swift/BUILD +++ b/mobile/library/swift/BUILD @@ -103,7 +103,7 @@ swift_library( "@envoy_mobile_extra_swift_sources//:extra_private_dep", ] + select({ "//bazel:envoy_mobile_swift_cxx_interop": [ - "//library/swift/EnvoyCxxSwiftInterop:EnvoyCxxSwiftInterop", + "//library/swift/EnvoyCxxSwiftInterop", ], "//conditions:default": [], }), diff --git a/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java b/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java index 2ff6bc2ced34..607061acbef5 100644 --- a/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java +++ b/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java @@ -36,7 +36,6 @@ import org.junit.Test; import org.junit.runner.RunWith; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.CronetTestRule.RequiresMinApi; import org.chromium.net.testing.MetricsTestUtil.TestRequestFinishedListener; import org.chromium.net.testing.TestBidirectionalStreamCallback.FailureType; @@ -147,11 +146,7 @@ private void runSimpleGetWithExpectedReceivedByteCount(int expectedReceivedBytes @SmallTest @Feature({"Cronet"}) public void testBuilderCheck() throws Exception { - if (mTestRule.testingJavaImpl()) { - runBuilderCheckJavaImpl(); - } else { - runBuilderCheckNativeImpl(); - } + runBuilderCheckNativeImpl(); } private void runBuilderCheckNativeImpl() throws Exception { @@ -198,24 +193,9 @@ private void runBuilderCheckNativeImpl() throws Exception { } } - private void runBuilderCheckJavaImpl() { - try { - TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); - CronetTestRule.createJavaEngineBuilder(CronetTestRule.getContext()) - .build() - .newBidirectionalStreamBuilder(Http2TestServer.getServerUrl(), callback, - callback.getExecutor()); - fail("JavaCronetEngine doesn't support BidirectionalStream." - + " Expected UnsupportedOperationException"); - } catch (UnsupportedOperationException e) { - // Expected. - } - } - @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testFailPlainHttp() throws Exception { String url = "http://example.com"; TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -233,7 +213,6 @@ public void testFailPlainHttp() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("fix expected ReceivedByteCount - quite unpredictable") public void testSimpleGet() throws Exception { // Since this is the first request on the connection, the expected received bytes count @@ -244,7 +223,6 @@ public void testSimpleGet() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("To be investigated - head does not work") public void testSimpleHead() throws Exception { String url = Http2TestServer.getEchoMethodUrl(); @@ -268,7 +246,6 @@ public void testSimpleHead() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testSimplePost() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -307,7 +284,6 @@ public void testSimplePost() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testSimpleGetWithCombinedHeader() throws Exception { String url = Http2TestServer.getCombinedHeadersUrl(); TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -334,7 +310,6 @@ public void testSimpleGetWithCombinedHeader() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testSimplePostWithFlush() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -367,7 +342,6 @@ public void testSimplePostWithFlush() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Tests that a delayed flush() only sends buffers that have been written // before it is called, and it doesn't flush buffers in mPendingQueue. public void testFlushData() throws Exception { @@ -450,7 +424,6 @@ public void onWriteCompleted(BidirectionalStream stream, UrlResponseInfo info, @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Regression test for crbug.com/692168. public void testCancelWhileWriteDataPending() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); @@ -506,7 +479,6 @@ private ByteBuffer getDummyData() { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testSimpleGetWithFlush() throws Exception { for (int i = 0; i < 2; i++) { String url = Http2TestServer.getEchoStreamUrl(); @@ -559,7 +531,6 @@ public void onStreamReady(BidirectionalStream stream) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testSimplePostWithFlushAfterOneWrite() throws Exception { for (int i = 0; i < 2; i++) { String url = Http2TestServer.getEchoStreamUrl(); @@ -587,7 +558,6 @@ public void testSimplePostWithFlushAfterOneWrite() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testSimplePostWithFlushTwice() throws Exception { for (int i = 0; i < 2; i++) { String url = Http2TestServer.getEchoStreamUrl(); @@ -620,7 +590,6 @@ public void testSimplePostWithFlushTwice() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Tests that it is legal to call read() in onStreamReady(). public void testReadDuringOnStreamReady() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); @@ -658,7 +627,6 @@ public void onResponseHeadersReceived(BidirectionalStream stream, UrlResponseInf @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Tests that it is legal to call flush() when previous nativeWritevData has // yet to complete. public void testSimplePostWithFlushBeforePreviousWriteCompleted() throws Exception { @@ -698,7 +666,6 @@ public void onStreamReady(BidirectionalStream stream) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testSimplePut() throws Exception { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); callback.addWriteData("Put This Data!".getBytes()); @@ -716,7 +683,6 @@ public void testSimplePut() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testBadMethod() throws Exception { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); BidirectionalStream.Builder builder = mCronetEngine.newBidirectionalStreamBuilder( @@ -733,7 +699,6 @@ public void testBadMethod() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testBadHeaderName() throws Exception { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); BidirectionalStream.Builder builder = mCronetEngine.newBidirectionalStreamBuilder( @@ -752,7 +717,6 @@ public void testBadHeaderName() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testBadHeaderValue() throws Exception { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); BidirectionalStream.Builder builder = mCronetEngine.newBidirectionalStreamBuilder( @@ -769,7 +733,6 @@ public void testBadHeaderValue() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testAddHeader() throws Exception { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); String headerName = "header-name"; @@ -787,7 +750,6 @@ public void testAddHeader() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Cronet does not support multi-headers - EM does") public void testMultiRequestHeaders() throws Exception { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -816,7 +778,6 @@ public void testMultiRequestHeaders() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("okhttp returns :status as a header") public void testEchoTrailers() throws Exception { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -837,7 +798,6 @@ public void testEchoTrailers() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testCustomUserAgent() throws Exception { String userAgentName = "User-Agent"; String userAgentValue = "User-Agent-Value"; @@ -855,7 +815,6 @@ public void testCustomUserAgent() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testCustomCronetEngineUserAgent() throws Exception { String userAgentName = "User-Agent"; String userAgentValue = "User-Agent-Value"; @@ -877,7 +836,6 @@ public void testCustomCronetEngineUserAgent() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testDefaultUserAgent() throws Exception { String userAgentName = "User-Agent"; TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -894,7 +852,6 @@ public void testDefaultUserAgent() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testEchoStream() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -923,7 +880,6 @@ public void testEchoStream() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testEchoStreamEmptyWrite() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -941,7 +897,6 @@ public void testEchoStreamEmptyWrite() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testDoubleWrite() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback() { @@ -970,7 +925,6 @@ public void onStreamReady(BidirectionalStream stream) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testDoubleRead() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback() { @@ -1003,7 +957,6 @@ public void onResponseHeadersReceived(BidirectionalStream stream, UrlResponseInf @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Disabled due to timeout. See crbug.com/591112") public void testReadAndWrite() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); @@ -1036,7 +989,6 @@ public void onResponseHeadersReceived(BidirectionalStream stream, UrlResponseInf @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testEchoStreamWriteFirst() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -1080,7 +1032,6 @@ public void testEchoStreamWriteFirst() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testEchoStreamStepByStep() throws Exception { String url = Http2TestServer.getEchoStreamUrl(); TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -1125,7 +1076,6 @@ public void testEchoStreamStepByStep() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testSimpleGetBufferUpdates() throws Exception { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); callback.setAutoAdvance(false); @@ -1220,7 +1170,6 @@ public void testSimpleGetBufferUpdates() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testBadBuffers() throws Exception { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); callback.setAutoAdvance(false); @@ -1329,7 +1278,6 @@ private void throwOrCancel(FailureType failureType, ResponseStep failureStep, @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Flaky: crashes EM") public void testFailures() throws Exception { // TODO(https://github.com/envoyproxy/envoy-mobile/issues/2192): start/end time are not set. @@ -1353,7 +1301,6 @@ public void testFailures() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testThrowOnSucceeded() { TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); callback.setFailure(FailureType.THROW_SYNC, ResponseStep.ON_SUCCEEDED); @@ -1373,7 +1320,6 @@ public void testThrowOnSucceeded() { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testExecutorShutdownBeforeStreamIsDone() { // Test that stream is destroyed even if executor is shut down and rejects posting tasks. TestBidirectionalStreamCallback callback = new TestBidirectionalStreamCallback(); @@ -1440,7 +1386,6 @@ public void onCanceled(BidirectionalStream stream, UrlResponseInfo info) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testCronetEngineShutdown() throws Exception { // Test that CronetEngine cannot be shut down if there are any active streams. TestBidirectionalStreamCallback callback = new ShutdownTestBidirectionalStreamCallback(); @@ -1488,7 +1433,6 @@ public void testCronetEngineShutdown() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testCronetEngineShutdownAfterStreamFailure() throws Exception { // Test that CronetEngine can be shut down after stream reports a failure. TestBidirectionalStreamCallback callback = new ShutdownTestBidirectionalStreamCallback(); @@ -1506,7 +1450,6 @@ public void testCronetEngineShutdownAfterStreamFailure() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testCronetEngineShutdownAfterStreamCancel() throws Exception { // Test that CronetEngine can be shut down after stream is canceled. TestBidirectionalStreamCallback callback = new ShutdownTestBidirectionalStreamCallback(); @@ -1539,7 +1482,6 @@ public void testCronetEngineShutdownAfterStreamCancel() throws Exception { @SmallTest @Feature({"Cronet"}) @Test - @OnlyRunNativeCronet public void testErrorCodes() throws Exception { // Non-BidirectionalStream specific error codes. checkSpecificErrorCode(NetError.ERR_NAME_NOT_RESOLVED, @@ -1589,7 +1531,6 @@ private static void checkSpecificErrorCode(NetError netError, int errorCode, @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @RequiresMinApi(10) // Tagging support added in API level 10: crrev.com/c/chromium/src/+/937583 @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1521") public void testTagging() throws Exception { diff --git a/mobile/test/java/org/chromium/net/BrotliTest.java b/mobile/test/java/org/chromium/net/BrotliTest.java index 046a856bc0f5..651682466e82 100644 --- a/mobile/test/java/org/chromium/net/BrotliTest.java +++ b/mobile/test/java/org/chromium/net/BrotliTest.java @@ -9,7 +9,6 @@ import androidx.test.filters.SmallTest; import org.chromium.net.testing.CronetTestRule; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.CronetTestUtil; import org.chromium.net.testing.Feature; import org.chromium.net.testing.Http2TestServer; @@ -49,7 +48,6 @@ public void tearDown() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testBrotliAdvertised() throws Exception { ExperimentalCronetEngine.Builder builder = new ExperimentalCronetEngine.Builder(getContext()); builder.enableBrotli(true); @@ -65,7 +63,6 @@ public void testBrotliAdvertised() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testBrotliNotAdvertised() throws Exception { ExperimentalCronetEngine.Builder builder = new ExperimentalCronetEngine.Builder(getContext()); CronetTestUtil.setMockCertVerifierForTesting(builder); @@ -79,7 +76,6 @@ public void testBrotliNotAdvertised() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testBrotliDecoded() throws Exception { ExperimentalCronetEngine.Builder builder = new ExperimentalCronetEngine.Builder(getContext()); builder.enableBrotli(true); diff --git a/mobile/test/java/org/chromium/net/CronetEngineBuilderTest.java b/mobile/test/java/org/chromium/net/CronetEngineBuilderTest.java index 35982d7f9a12..441bcafeb4a1 100644 --- a/mobile/test/java/org/chromium/net/CronetEngineBuilderTest.java +++ b/mobile/test/java/org/chromium/net/CronetEngineBuilderTest.java @@ -14,7 +14,6 @@ import java.util.Arrays; import java.util.List; import org.chromium.net.testing.CronetTestRule; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.Feature; import org.junit.Rule; import org.junit.Test; @@ -33,7 +32,6 @@ public class CronetEngineBuilderTest { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testVersionComparison() { assertVersionIsHigher("22.44", "22.43.12"); assertVersionIsLower("22.43.12", "022.124"); diff --git a/mobile/test/java/org/chromium/net/CronetStressTest.java b/mobile/test/java/org/chromium/net/CronetStressTest.java index 20bd213aa5fa..866b6423a9dd 100644 --- a/mobile/test/java/org/chromium/net/CronetStressTest.java +++ b/mobile/test/java/org/chromium/net/CronetStressTest.java @@ -8,7 +8,6 @@ import androidx.test.filters.LargeTest; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.Feature; import org.chromium.net.testing.NativeTestServer; import org.chromium.net.testing.TestUrlRequestCallback; @@ -40,7 +39,6 @@ public void tearDown() throws Exception { @Test @LargeTest - @OnlyRunNativeCronet @Feature({"Cronet"}) public void testLargeNumberOfUploads() throws Exception { final int kNumRequest = 1000; diff --git a/mobile/test/java/org/chromium/net/CronetUrlRequestContextTest.java b/mobile/test/java/org/chromium/net/CronetUrlRequestContextTest.java index f5e14d4e28eb..e99eba9844a0 100644 --- a/mobile/test/java/org/chromium/net/CronetUrlRequestContextTest.java +++ b/mobile/test/java/org/chromium/net/CronetUrlRequestContextTest.java @@ -32,7 +32,6 @@ import org.chromium.net.impl.NativeCronetEngineBuilderImpl; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.CronetTestRule.RequiresMinApi; import org.chromium.net.testing.Feature; import org.chromium.net.testing.FileUtils; @@ -140,9 +139,6 @@ public void testConfigUserAgent() throws Exception { String userAgentValue = "User-Agent-Value"; ExperimentalCronetEngine.Builder cronetEngineBuilder = new ExperimentalCronetEngine.Builder(getContext()); - if (mTestRule.testingJavaImpl()) { - cronetEngineBuilder = CronetTestRule.createJavaEngineBuilder(getContext()); - } cronetEngineBuilder.setUserAgent(userAgentValue); final CronetEngine cronetEngine = cronetEngineBuilder.build(); NativeTestServer.shutdownNativeTestServer(); // startNativeTestServer returns false if it's @@ -160,7 +156,6 @@ public void testConfigUserAgent() throws Exception { @SmallTest @Feature({"Cronet"}) // TODO: Remove the annotation after fixing http://crbug.com/637979 & http://crbug.com/637972 - @OnlyRunNativeCronet @Ignore("Properly implement shutdown sequence") public void testShutdown() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -211,7 +206,6 @@ public void testShutdown() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Test is running on main thread") public void testShutdownDuringInit() throws Exception { final ConditionVariable block = new ConditionVariable(false); @@ -249,7 +243,6 @@ public void run() { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Properly implement shutdown sequence") public void testInitAndShutdownOnMainThread() throws Exception { final ConditionVariable block = new ConditionVariable(false); @@ -276,7 +269,6 @@ public void run() { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // JavaCronetEngine doesn't support throwing on repeat shutdown() @Ignore("Properly implement shutdown sequence") public void testMultipleShutdown() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -293,7 +285,6 @@ public void testMultipleShutdown() throws Exception { @SmallTest @Feature({"Cronet"}) // TODO: Remove the annotation after fixing http://crbug.com/637972 - @OnlyRunNativeCronet @Ignore("Properly implement shutdown sequence") public void testShutdownAfterError() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -311,7 +302,6 @@ public void testShutdownAfterError() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // JavaCronetEngine doesn't support throwing on shutdown() @Ignore("Shutdown not properly implemented") public void testShutdownAfterCancel() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -338,7 +328,6 @@ public void testShutdownAfterCancel() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // No netlogs for pure java impl @Ignore("Netlog not implemented") public void testNetLog() throws Exception { Context context = getContext(); @@ -367,7 +356,6 @@ public void testNetLog() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // No netlogs for pure java impl @Ignore("Netlog not implemented") public void testBoundedFileNetLog() throws Exception { Context context = getContext(); @@ -399,7 +387,6 @@ public void testBoundedFileNetLog() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // No netlogs for pure java impl @Ignore("Netlog not implemented") // Tests that if stopNetLog is not explicitly called, CronetEngine.shutdown() // will take care of it. crbug.com/623701. @@ -428,7 +415,6 @@ public void testNoStopNetLog() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // No netlogs for pure java impl @Ignore("Netlog not implemented") // Tests that if stopNetLog is not explicitly called, CronetEngine.shutdown() // will take care of it. crbug.com/623701. @@ -460,7 +446,6 @@ public void testNoStopBoundedFileNetLog() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") // Tests that NetLog contains events emitted by all live CronetEngines. public void testNetLogContainEventsFromAllLiveEngines() throws Exception { @@ -502,7 +487,6 @@ public void testNetLogContainEventsFromAllLiveEngines() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") // Tests that NetLog contains events emitted by all live CronetEngines. public void testBoundedFileNetLogContainEventsFromAllLiveEngines() throws Exception { @@ -572,7 +556,6 @@ private CronetEngine createCronetEngineWithCache(int cacheType) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Caching not implemented yet: https://github.com/envoyproxy/envoy-mobile/issues/1578") // Tests that if CronetEngine is shut down on the network thread, an appropriate exception // is thrown. @@ -631,7 +614,6 @@ public void execute(Runnable command) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Caching not implemented yet: https://github.com/envoyproxy/envoy-mobile/issues/1578") // Tests that if CronetEngine is shut down when reading from disk cache, // there isn't a crash. See crbug.com/486120. @@ -678,7 +660,6 @@ public void onFailed(UrlRequest request, UrlResponseInfo info, CronetException e @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") public void testNetLogAfterShutdown() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -705,7 +686,6 @@ public void testNetLogAfterShutdown() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") public void testBoundedFileNetLogAfterShutdown() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -735,7 +715,6 @@ public void testBoundedFileNetLogAfterShutdown() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") public void testNetLogStartMultipleTimes() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -763,7 +742,6 @@ public void testNetLogStartMultipleTimes() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") public void testBoundedFileNetLogStartMultipleTimes() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -795,7 +773,6 @@ public void testBoundedFileNetLogStartMultipleTimes() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") public void testNetLogStopMultipleTimes() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -824,7 +801,6 @@ public void testNetLogStopMultipleTimes() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") public void testBoundedFileNetLogStopMultipleTimes() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -857,7 +833,6 @@ public void testBoundedFileNetLogStopMultipleTimes() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") public void testNetLogWithBytes() throws Exception { Context context = getContext(); @@ -883,7 +858,6 @@ public void testNetLogWithBytes() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Netlog not implemented") public void testBoundedFileNetLogWithBytes() throws Exception { Context context = getContext(); @@ -962,7 +936,6 @@ private void checkRequestCaching(CronetEngine engine, String url, boolean expect @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Caching not implemented yet: https://github.com/envoyproxy/envoy-mobile/issues/1578") public void testEnableHttpCacheDisabled() throws Exception { CronetEngine cronetEngine = @@ -1006,7 +979,6 @@ public void testEnableHttpCacheDisk() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Caching not implemented yet: https://github.com/envoyproxy/envoy-mobile/issues/1578") public void testNoConcurrentDiskUsage() throws Exception { CronetEngine cronetEngine = createCronetEngineWithCache(CronetEngine.Builder.HTTP_CACHE_DISK); @@ -1027,7 +999,6 @@ public void testNoConcurrentDiskUsage() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Caching not implemented yet: https://github.com/envoyproxy/envoy-mobile/issues/1578") public void testEnableHttpCacheDiskNoHttp() throws Exception { CronetEngine cronetEngine = @@ -1194,7 +1165,6 @@ public void testInitTwoEnginesInSequence() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Java engine doesn't produce metrics @Ignore("UMA not implemented yet: https://github.com/envoyproxy/envoy-mobile/issues/1615") public void testGetGlobalMetricsDeltas() throws Exception { final CronetTestFramework testFramework = mTestRule.startCronetTestFramework(); @@ -1292,7 +1262,6 @@ public void loadLibrary(String libName) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Library Loader not needed") public void testSetLibraryLoaderIsEnforcedByDefaultEmbeddedProvider() throws Exception { CronetEngine.Builder builder = new CronetEngine.Builder(getContext()); @@ -1309,7 +1278,6 @@ public void testSetLibraryLoaderIsEnforcedByDefaultEmbeddedProvider() throws Exc @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("Library Loader not needed") public void testSetLibraryLoaderIsIgnoredInNativeCronetEngineBuilderImpl() throws Exception { CronetEngine.Builder builder = @@ -1324,7 +1292,6 @@ public void testSetLibraryLoaderIsIgnoredInNativeCronetEngineBuilderImpl() throw @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testNativeCronetEngineBuilderImplSetsCorrectVersionString() throws Exception { CronetEngine.Builder builder = new CronetEngine.Builder(new NativeCronetEngineBuilderImpl(getContext())); diff --git a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java index abada86a531b..53e3a9602ce6 100644 --- a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java +++ b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java @@ -42,7 +42,6 @@ import org.chromium.net.impl.UrlResponseInfoImpl; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.CronetTestRule.RequiresMinApi; import org.chromium.net.testing.FailurePhase; import org.chromium.net.testing.Feature; @@ -85,8 +84,8 @@ public class CronetUrlRequestTest { @Before public void setUp() { - mMockUrlRequestJobFactory = new MockUrlRequestJobFactory( - mTestRule.buildCronetTestFramework().mBuilder, mTestRule.testingJavaImpl()); + mMockUrlRequestJobFactory = + new MockUrlRequestJobFactory(mTestRule.buildCronetTestFramework().mBuilder); assertTrue(NativeTestServer.startNativeTestServer(getContext())); } @@ -199,7 +198,6 @@ private static UrlResponseInfo createUrlResponseInfo(String[] urls, String messa @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1522") public void testLoadFlagsWithConnectionMigration() throws Exception {} @@ -400,7 +398,6 @@ public void testNotFound() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // No canonical exception to assert on @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1550") public void testContentLengthMismatchFailsOnce() throws Exception { String url = NativeTestServer.getFileURL("/content_length_mismatch.html"); @@ -568,7 +565,6 @@ public void testCustomReferer_verbatim() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1551") public void testCustomReferer_changeToCanonical() throws Exception { TestUrlRequestCallback callback = new TestUrlRequestCallback(); @@ -585,7 +581,6 @@ public void testCustomReferer_changeToCanonical() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1551") public void testCustomReferer_discardInvalid() throws Exception { TestUrlRequestCallback callback = new TestUrlRequestCallback(); @@ -724,7 +719,6 @@ public void testMockNotFound() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Java impl doesn't support MockUrlRequestJobFactory @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1549") public void testMockStartAsyncError() throws Exception { final int arbitraryNetError = -3; @@ -743,7 +737,6 @@ public void testMockStartAsyncError() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Java impl doesn't support MockUrlRequestJobFactory @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1549") public void testMockReadDataSyncError() throws Exception { final int arbitraryNetError = -4; @@ -766,7 +759,6 @@ public void testMockReadDataSyncError() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1549") public void testMockClientCertificateRequested() throws Exception { TestUrlRequestCallback callback = @@ -786,7 +778,6 @@ public void testMockClientCertificateRequested() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Java impl doesn't support MockUrlRequestJobFactory @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1549") public void testMockSSLCertificateError() throws Exception { TestUrlRequestCallback callback = @@ -808,7 +799,6 @@ public void testMockSSLCertificateError() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Java impl doesn't support MockUrlRequestJobFactory @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1549") public void testSSLCertificateError() throws Exception {} @@ -1775,13 +1765,8 @@ public void testUploadFailsWithoutInitializingStream() throws Exception { dataProvider.assertClosed(); assertNull(callback.mResponseInfo); - if (mTestRule.testingJavaImpl()) { - Throwable cause = callback.mError.getCause(); - assertTrue("Exception was: " + cause, cause instanceof ConnectException); - } else { - assertContains("Exception in CronetUrlRequest: net::ERR_CONNECTION_REFUSED", - callback.mError.getMessage()); - } + assertContains("Exception in CronetUrlRequest: net::ERR_CONNECTION_REFUSED", + callback.mError.getMessage()); } private void throwOrCancel(FailureType failureType, ResponseStep failureStep, @@ -1930,7 +1915,6 @@ public void onResponseStarted(UrlRequest request, UrlResponseInfo info) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // No destroyed callback for tests @Ignore("Not yet implemented") public void testExecutorShutdown() { TestUrlRequestCallback callback = new TestUrlRequestCallback(); @@ -2030,7 +2014,6 @@ public void onSucceeded(UrlRequest request, UrlResponseInfo info) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // No adapter to destroy in pure java @Ignore("Not yet implemented") public void testDestroyUploadDataStreamAdapterOnSucceededCallback() throws Exception { TestUrlRequestCallback callback = new QuitOnSuccessCallback(); @@ -2064,7 +2047,6 @@ public void testDestroyUploadDataStreamAdapterOnSucceededCallback() throws Excep @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Java impl doesn't support MockUrlRequestJobFactory public void testErrorCodes() throws Exception { checkSpecificErrorCode(EnvoyMobileError.DNS_RESOLUTION_FAILED, NetError.ERR_NAME_NOT_RESOLVED, NetworkException.ERROR_HOSTNAME_NOT_RESOLVED, "NAME_NOT_RESOLVED", @@ -2089,7 +2071,6 @@ public void testErrorCodes() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Java impl doesn't support MockUrlRequestJobFactory public void testInternetDisconnectedError() throws Exception { AndroidNetworkMonitor androidNetworkMonitor = AndroidNetworkMonitor.getInstance(); Intent intent = new Intent(ConnectivityManager.CONNECTIVITY_ACTION); @@ -2138,7 +2119,6 @@ public void testCookiesArentSavedOrSent() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testQuicErrorCode() throws Exception { long envoyMobileError = 0x2000; TestUrlRequestCallback callback = startAndWaitForComplete( @@ -2161,7 +2141,6 @@ public void testQuicErrorCode() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testLegacyOnFailedCallback() throws Exception { final long envoyMobileError = 0x2000; final int netError = NetError.ERR_OTHER.getErrorCode(); @@ -2258,7 +2237,6 @@ private void assertResponseStepCanceled(TestUrlRequestCallback callback) { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1550") public void testCleartextTrafficBlocked() throws Exception { // This feature only works starting from N. diff --git a/mobile/test/java/org/chromium/net/DiskStorageTest.java b/mobile/test/java/org/chromium/net/DiskStorageTest.java index 110b63ba505a..2f88abdf51ab 100644 --- a/mobile/test/java/org/chromium/net/DiskStorageTest.java +++ b/mobile/test/java/org/chromium/net/DiskStorageTest.java @@ -14,7 +14,6 @@ import java.io.FileReader; import java.util.Arrays; import org.chromium.net.testing.CronetTestRule; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.Feature; import org.chromium.net.testing.FileUtils; import org.chromium.net.testing.NativeTestServer; @@ -53,7 +52,6 @@ public void tearDown() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Crashing on Android Cronet Builder, see crbug.com/601409. public void testReadOnlyStorageDirectory() throws Exception { mReadOnlyStoragePath = PathUtils.getDataDirectory() + "/read_only"; @@ -98,7 +96,6 @@ public void testReadOnlyStorageDirectory() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Crashing on Android Cronet Builder, see crbug.com/601409. public void testPurgeOldVersion() throws Exception { String testStorage = getTestStorage(); @@ -161,7 +158,6 @@ public void testPurgeOldVersion() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Tests that if cache version is current, Cronet does not purge the directory. public void testCacheVersionCurrent() throws Exception { // Initialize a CronetEngine and shut it down. diff --git a/mobile/test/java/org/chromium/net/GetStatusTest.java b/mobile/test/java/org/chromium/net/GetStatusTest.java index cdef4bc8e4ac..e1fe04a389d8 100644 --- a/mobile/test/java/org/chromium/net/GetStatusTest.java +++ b/mobile/test/java/org/chromium/net/GetStatusTest.java @@ -15,7 +15,6 @@ import org.chromium.net.UrlRequest.StatusListener; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.Feature; import org.chromium.net.testing.NativeTestServer; import org.chromium.net.testing.TestUploadDataProvider; @@ -149,7 +148,6 @@ public void testInvalidLoadState() throws Exception { @SmallTest @Feature({"Cronet"}) // Regression test for crbug.com/606872. - @OnlyRunNativeCronet @Ignore("https://github.com/envoyproxy/envoy-mobile/issues/1519") public void testGetStatusForUpload() throws Exception { TestUrlRequestCallback callback = new TestUrlRequestCallback(); diff --git a/mobile/test/java/org/chromium/net/RequestFinishedInfoTest.java b/mobile/test/java/org/chromium/net/RequestFinishedInfoTest.java index 9c8d7125d39c..8f04f18de196 100644 --- a/mobile/test/java/org/chromium/net/RequestFinishedInfoTest.java +++ b/mobile/test/java/org/chromium/net/RequestFinishedInfoTest.java @@ -21,7 +21,6 @@ import org.chromium.net.impl.CronetMetrics; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.CronetTestRule.RequiresMinApi; import org.chromium.net.testing.Feature; import org.chromium.net.testing.MetricsTestUtil; @@ -108,7 +107,6 @@ public void joinAll() throws InterruptedException { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) @SuppressWarnings("deprecation") public void testRequestFinishedListener() throws Exception { @@ -137,7 +135,6 @@ public void testRequestFinishedListener() throws Exception { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) @SuppressWarnings("deprecation") public void testRequestFinishedListenerDirectExecutor() throws Exception { @@ -170,7 +167,6 @@ public void testRequestFinishedListenerDirectExecutor() throws Exception { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) @SuppressWarnings("deprecation") public void testRequestFinishedListenerDifferentThreads() throws Exception { @@ -212,7 +208,6 @@ public void testRequestFinishedListenerDifferentThreads() throws Exception { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) @Ignore("Needs debugging") @SuppressWarnings("deprecation") @@ -260,7 +255,6 @@ public void testRequestFinishedListenerFailedRequest() throws Exception { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) @SuppressWarnings("deprecation") public void testRequestFinishedListenerRemoved() throws Exception { @@ -283,7 +277,6 @@ public void testRequestFinishedListenerRemoved() throws Exception { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) public void testRequestFinishedListenerCanceledRequest() throws Exception { TestRequestFinishedListener requestFinishedListener = new TestRequestFinishedListener(); @@ -326,7 +319,6 @@ public void execute(Runnable task) { // collection is enabled and the URLRequest hasn't been created. See http://crbug.com/675629. @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) public void testExceptionInRequestStart() throws Exception { // The listener in this test shouldn't get any tasks. @@ -390,7 +382,6 @@ public void testMetricsGetters() throws Exception { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) @SuppressWarnings("deprecation") public void testOrderSuccessfulRequest() throws Exception { @@ -420,7 +411,6 @@ public void testOrderSuccessfulRequest() throws Exception { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) @RequiresMinApi(11) public void testUpdateAnnotationOnSucceeded() throws Exception { @@ -458,7 +448,6 @@ public void onSucceeded(UrlRequest request, UrlResponseInfo info) { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) // Tests a failed request where the error originates from Java. public void testOrderFailedRequestJava() throws Exception { @@ -491,7 +480,6 @@ public void onResponseStarted(UrlRequest request, UrlResponseInfo info) { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) @Ignore("Needs debugging") // Tests a failed request where the error originates from native code. @@ -521,7 +509,6 @@ public void testOrderFailedRequestNative() throws Exception { @Test @SmallTest - @OnlyRunNativeCronet @Feature({"Cronet"}) public void testOrderCanceledRequest() throws Exception { final TestUrlRequestCallback callback = new TestUrlRequestCallback() { diff --git a/mobile/test/java/org/chromium/net/UploadDataProvidersTest.java b/mobile/test/java/org/chromium/net/UploadDataProvidersTest.java index eff72fb68bfd..9536d052e576 100644 --- a/mobile/test/java/org/chromium/net/UploadDataProvidersTest.java +++ b/mobile/test/java/org/chromium/net/UploadDataProvidersTest.java @@ -16,7 +16,6 @@ import java.nio.ByteBuffer; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.Feature; import org.chromium.net.testing.NativeTestServer; import org.chromium.net.testing.TestUrlRequestCallback; @@ -136,7 +135,6 @@ public void testBufferProvider() throws Exception { @Test @SmallTest @Feature({"Cronet"}) - @OnlyRunNativeCronet // Tests that ByteBuffer's limit cannot be changed by the caller. @Ignore("Blocked by envoy-mobile flow control impl. ByteBuffer impl for cronvoy is different") public void testUploadChangeBufferLimit() throws Exception { diff --git a/mobile/test/java/org/chromium/net/testing/CronetTestRule.java b/mobile/test/java/org/chromium/net/testing/CronetTestRule.java index 774b95467d6f..ba3abf38e012 100644 --- a/mobile/test/java/org/chromium/net/testing/CronetTestRule.java +++ b/mobile/test/java/org/chromium/net/testing/CronetTestRule.java @@ -25,8 +25,6 @@ import org.chromium.net.ExperimentalCronetEngine; import org.chromium.net.UrlResponseInfo; import org.chromium.net.impl.CronetEngineBuilderImpl; -import org.chromium.net.impl.JavaCronetEngine; -import org.chromium.net.impl.JavaCronetProvider; import org.chromium.net.impl.NativeCronetProvider; import org.chromium.net.impl.UserAgent; import org.junit.Assert; @@ -45,7 +43,6 @@ public final class CronetTestRule implements TestRule { // {@code true} when test is being run against system HttpURLConnection implementation. private boolean mTestingSystemHttpURLConnection; - private boolean mTestingJavaImpl; private StrictMode.VmPolicy mOldVmPolicy; private CronetEngine mUrlConnectionCronetEngine; private static Context mContext; @@ -72,30 +69,20 @@ public static class CronetTestFramework { public ExperimentalCronetEngine.Builder mBuilder; private Context mContext; - private boolean mIsTestingJavaImpl; - private CronetTestFramework(Context context, boolean isTestingJavaImpl) { + private CronetTestFramework(Context context) { mContext = context; - mIsTestingJavaImpl = isTestingJavaImpl; - mBuilder = mIsTestingJavaImpl ? createJavaEngineBuilder() : createNativeEngineBuilder(); - } - - private static CronetTestFramework createUsingJavaImpl(Context context) { - return new CronetTestFramework(context, true /* isTestingJavaImpl */); + mBuilder = createNativeEngineBuilder(); } private static CronetTestFramework createUsingNativeImpl(Context context) { - return new CronetTestFramework(context, false /* isTestingJavaImpl */); + return new CronetTestFramework(context); } public ExperimentalCronetEngine startEngine() { assert mCronetEngine == null; mCronetEngine = mBuilder.build(); - if (mIsTestingJavaImpl) { - // Make sure that the instantiated engine is JavaCronetEngine. - assert mCronetEngine.getClass() == JavaCronetEngine.class; - } // Start collecting metrics. mCronetEngine.getGlobalMetricsDeltas(); @@ -110,12 +97,6 @@ public void shutdownEngine() { mCronetEngine = null; } - private ExperimentalCronetEngine.Builder createJavaEngineBuilder() { - return CronetTestRule.createJavaEngineBuilder(mContext) - .setUserAgent(UserAgent.from(getContext())) - .enableQuic(true); - } - private ExperimentalCronetEngine.Builder createNativeEngineBuilder() { return CronetTestRule.createNativeEngineBuilder(mContext).enableQuic(true); } @@ -157,26 +138,10 @@ public void evaluate() throws Throwable { */ public boolean testingSystemHttpURLConnection() { return mTestingSystemHttpURLConnection; } - /** - * Returns {@code true} when test is being run against the java implementation of CronetEngine. - */ - public boolean testingJavaImpl() { return mTestingJavaImpl; } - private void runBase(Statement base, Description desc) throws Throwable { setTestingSystemHttpURLConnection(false); - setTestingJavaImpl(false); String packageName = desc.getTestClass().getPackage().getName(); - boolean onlyRunTestForNative = desc.getAnnotation(OnlyRunNativeCronet.class) != null; - boolean onlyRunTestForJava = desc.getAnnotation(OnlyRunJavaCronet.class) != null; - if (onlyRunTestForNative && onlyRunTestForJava) { - throw new IllegalArgumentException(desc.getMethodName() + - " skipped because it specified both " - + "OnlyRunNativeCronet and OnlyRunJavaCronet annotations"); - } - boolean doRunTestForNative = onlyRunTestForNative || !onlyRunTestForJava; - boolean doRunTestForJava = onlyRunTestForJava || !onlyRunTestForNative; - // Find the API version required by the test. int requiredApiVersion = getMaximumAvailableApiLevel(); int requiredAndroidApiVersion = Build.VERSION_CODES.KITKAT; @@ -226,18 +191,10 @@ private void runBase(Statement base, Description desc) throws Throwable { } } else if (packageName.startsWith("org.chromium.net")) { try { - if (doRunTestForNative) { - Log.i(TAG, "Running test against Native implementation."); - base.evaluate(); - } - if (doRunTestForJava) { - Log.i(TAG, "Running test against Java implementation."); - setTestingJavaImpl(true); - base.evaluate(); - } + Log.i(TAG, "Running test against Native implementation."); + base.evaluate(); } catch (Throwable e) { - Log.e(TAG, String.format("CronetTestBase#runTest failed for %s implementation.", - testingJavaImpl() ? "Java" : "Native")); + Log.e(TAG, "CronetTestBase#runTest failed for Native implementation."); throw e; } } else { @@ -290,9 +247,7 @@ private void tearDown() { } private CronetTestFramework createCronetTestFramework() { - mCronetTestFramework = testingJavaImpl() - ? CronetTestFramework.createUsingJavaImpl(getContext()) - : CronetTestFramework.createUsingNativeImpl(getContext()); + mCronetTestFramework = CronetTestFramework.createUsingNativeImpl(getContext()); return mCronetTestFramework; } @@ -310,16 +265,6 @@ public CronetTestFramework startCronetTestFramework() { */ public CronetTestFramework buildCronetTestFramework() { return createCronetTestFramework(); } - /** - * Creates and returns {@link ExperimentalCronetEngine.Builder} that creates - * Java (platform) based {@link CronetEngine.Builder}. - * - * @return the {@code CronetEngine.Builder} that builds Java-based {@code Cronet engine}. - */ - public static ExperimentalCronetEngine.Builder createJavaEngineBuilder(Context context) { - return (ExperimentalCronetEngine.Builder) new JavaCronetProvider(context).createBuilder(); - } - /** * Creates and returns {@link ExperimentalCronetEngine.Builder} that creates * Chromium (native) based {@link CronetEngine.Builder}. @@ -337,13 +282,9 @@ public void assertResponseEquals(UrlResponseInfo expected, UrlResponseInfo actua assertEquals(expected.getHttpStatusText(), actual.getHttpStatusText()); assertEquals(expected.getUrlChain(), actual.getUrlChain()); assertEquals(expected.getUrl(), actual.getUrl()); - // Transferred bytes and proxy server are not supported in pure java - if (!testingJavaImpl()) { - assertEquals(expected.getReceivedByteCount(), actual.getReceivedByteCount()); - assertEquals(expected.getProxyServer(), actual.getProxyServer()); - // This is a place where behavior intentionally differs between native and java - assertEquals(expected.getNegotiatedProtocol(), actual.getNegotiatedProtocol()); - } + assertEquals(expected.getReceivedByteCount(), actual.getReceivedByteCount()); + assertEquals(expected.getProxyServer(), actual.getProxyServer()); + assertEquals(expected.getNegotiatedProtocol(), actual.getNegotiatedProtocol()); } public static void assertContains(String expectedSubstring, String actualString) { @@ -404,24 +345,6 @@ private void resetURLStreamHandlerFactory() { @Retention(RetentionPolicy.RUNTIME) public @interface OnlyRunCronetHttpURLConnection {} - /** - * Annotation for test methods in org.chromium.net package that disables rerunning the test - * against the Java-only implementation. When this annotation is present the test is only run - * against the native implementation. - */ - @Target(ElementType.METHOD) - @Retention(RetentionPolicy.RUNTIME) - public @interface OnlyRunNativeCronet {} - - /** - * Annotation for test methods in org.chromium.net package that disables rerunning the test - * against the Native/Chromium implementation. When this annotation is present the test is only - * run against the Java implementation. - */ - @Target(ElementType.METHOD) - @Retention(RetentionPolicy.RUNTIME) - public @interface OnlyRunJavaCronet {} - /** * Annotation allowing classes or individual tests to be skipped based on the version of the * Cronet API present. Takes the minimum API version upon which the test should be run. @@ -500,6 +423,4 @@ private static boolean recursiveDelete(File path) { private void setTestingSystemHttpURLConnection(boolean value) { mTestingSystemHttpURLConnection = value; } - - private void setTestingJavaImpl(boolean value) { mTestingJavaImpl = value; } } diff --git a/mobile/test/java/org/chromium/net/testing/CronetTestRuleTest.java b/mobile/test/java/org/chromium/net/testing/CronetTestRuleTest.java index c2339fcc8008..82cbb265fa35 100644 --- a/mobile/test/java/org/chromium/net/testing/CronetTestRuleTest.java +++ b/mobile/test/java/org/chromium/net/testing/CronetTestRuleTest.java @@ -8,10 +8,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.filters.SmallTest; import org.chromium.net.impl.CronetUrlRequestContext; -import org.chromium.net.impl.JavaCronetEngine; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; -import org.chromium.net.testing.CronetTestRule.OnlyRunJavaCronet; -import org.chromium.net.testing.CronetTestRule.OnlyRunNativeCronet; import org.chromium.net.testing.CronetTestRule.RequiresMinApi; import org.junit.After; import org.junit.Before; @@ -67,37 +64,9 @@ public void testRequiresMinApiMustRun() { @Test @SmallTest @Feature({"Cronet"}) - public void testRunBothImplsMustRun() { - if (mTestRule.testingJavaImpl()) { - assertFalse(mTestWasRun); - mTestWasRun = true; - assertEquals(mTestFramework.mCronetEngine.getClass(), JavaCronetEngine.class); - } else { - assertFalse(mTestWasRun); - mTestWasRun = true; - assertEquals(mTestFramework.mCronetEngine.getClass(), CronetUrlRequestContext.class); - } - } - - @Test - @SmallTest - @Feature({"Cronet"}) - @OnlyRunNativeCronet public void testRunOnlyNativeMustRun() { - assertFalse(mTestRule.testingJavaImpl()); assertFalse(mTestWasRun); mTestWasRun = true; assertEquals(mTestFramework.mCronetEngine.getClass(), CronetUrlRequestContext.class); } - - @Test - @SmallTest - @Feature({"Cronet"}) - @OnlyRunJavaCronet - public void testRunOnlyJavaMustRun() { - assertTrue(mTestRule.testingJavaImpl()); - assertFalse(mTestWasRun); - mTestWasRun = true; - assertEquals(mTestFramework.mCronetEngine.getClass(), JavaCronetEngine.class); - } } diff --git a/mobile/test/java/org/chromium/net/testing/MockUrlRequestJobFactory.java b/mobile/test/java/org/chromium/net/testing/MockUrlRequestJobFactory.java index eaf9a3cda39b..a65cf41cb984 100644 --- a/mobile/test/java/org/chromium/net/testing/MockUrlRequestJobFactory.java +++ b/mobile/test/java/org/chromium/net/testing/MockUrlRequestJobFactory.java @@ -22,18 +22,12 @@ public final class MockUrlRequestJobFactory { /** * Sets up request filters. - * This optionally adds a request filter to the cronetEngine before building. - * JavaCronetEngine does not support these filters. + * This adds a request filter to the cronetEngine before building. */ - public MockUrlRequestJobFactory(ExperimentalCronetEngine.Builder builder, - boolean testingJavaImpl) { - if (testingJavaImpl) { - mCronetEngine = builder.build(); - } else { - // Add a filter to immediately return a response - mCronetEngine = - CronetTestUtil.getCronetEngineBuilderImpl(builder).addUrlInterceptorsForTesting().build(); - } + public MockUrlRequestJobFactory(ExperimentalCronetEngine.Builder builder) { + // Add a filter to immediately return a response + mCronetEngine = + CronetTestUtil.getCronetEngineBuilderImpl(builder).addUrlInterceptorsForTesting().build(); } /** From 27be56a1c373c3a86c1eb71d331fb994c8b339cc Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 1 May 2023 22:01:38 +0100 Subject: [PATCH 062/740] mobile/ci: Increase test_timeout for ios tests (#27044) Signed-off-by: Ryan Northey --- mobile/.bazelrc | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/.bazelrc b/mobile/.bazelrc index fcc169dd3b0e..5168401fb5eb 100644 --- a/mobile/.bazelrc +++ b/mobile/.bazelrc @@ -56,6 +56,7 @@ build:dbg-common --copt="-fdebug-compilation-dir" --copt="/proc/self/cwd" # More information on stamping can be found here: # https://github.com/envoyproxy/envoy/tree/master/bazel#enabling-optional-features build:ios --define=manual_stamp=manual_stamp +build:ios --test_timeout=390,750,1500,5700 # Default flags for builds targeting Android build:android --define=logger=android From ecf3720d44a83277a56d14aa72208b84ca343871 Mon Sep 17 00:00:00 2001 From: Raven Black Date: Mon, 1 May 2023 14:03:58 -0700 Subject: [PATCH 063/740] Add createPath to Filesystem abstraction. (#27052) Add createPath to Filesystem abstraction Additional Description: createPath support is a prerequisite of file_system_http_cache's create_cache_path support. This is a revert of a revert. Now that mobile CI will run as part of the change, this should be safer to land when it passes than the two previous attempts. The revert is the first commit in this PR, so the changes that are new/fixes can be easily reviewed separately. Compared to the previous commit that broke mobile CI, this change performs posix file system operations with C style operating system calls, thereby avoiding the prior problem of C++17 filesystem functions failing to link. Signed-off-by: Raven Black --- envoy/filesystem/filesystem.h | 8 ++++ .../base_client_integration_test.cc | 2 +- .../filesystem/posix/filesystem_impl.cc | 32 +++++++++++++ .../common/filesystem/posix/filesystem_impl.h | 1 + .../filesystem/win32/filesystem_impl.cc | 10 +++++ .../common/filesystem/win32/filesystem_impl.h | 1 + .../common/filesystem/filesystem_impl_test.cc | 45 +++++++++++++++++++ test/mocks/filesystem/mocks.h | 1 + test/per_file_coverage.sh | 2 +- test/test_common/file_system_for_test.cc | 5 +++ test/test_common/file_system_for_test.h | 2 + 11 files changed, 107 insertions(+), 2 deletions(-) diff --git a/envoy/filesystem/filesystem.h b/envoy/filesystem/filesystem.h index 366d9b71b349..fca513f38a83 100644 --- a/envoy/filesystem/filesystem.h +++ b/envoy/filesystem/filesystem.h @@ -174,6 +174,14 @@ class Instance { */ virtual Api::IoCallResult stat(absl::string_view path) PURE; + /** + * Attempts to create the given path, recursively if necessary. + * @return bool true if one or more directories was created and the path exists, + * false if the path already existed, an error status if the path does + * not exist after the call. + */ + virtual Api::IoCallBoolResult createPath(absl::string_view path) PURE; + /** * @return bool whether a directory exists on disk and can be opened for read. */ diff --git a/mobile/test/common/integration/base_client_integration_test.cc b/mobile/test/common/integration/base_client_integration_test.cc index 9d1f99b9076e..a382597060ce 100644 --- a/mobile/test/common/integration/base_client_integration_test.cc +++ b/mobile/test/common/integration/base_client_integration_test.cc @@ -206,7 +206,7 @@ void BaseClientIntegrationTest::createEnvoy() { << "Bootstrap config should not have `admin` configured in Envoy Mobile"; builder_.setOverrideConfig(MessageUtil::getYamlStringFromMessage(config_helper_.bootstrap())); } else { - ENVOY_LOG_MISC(warn, "Using builder config and ignoring config modifiers."); + ENVOY_LOG_MISC(warn, "Using builder config and ignoring config modifiers"); } absl::Notification engine_running; diff --git a/source/common/filesystem/posix/filesystem_impl.cc b/source/common/filesystem/posix/filesystem_impl.cc index f6ec22a8d9c8..24c0bf1c05f3 100644 --- a/source/common/filesystem/posix/filesystem_impl.cc +++ b/source/common/filesystem/posix/filesystem_impl.cc @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -184,6 +185,37 @@ Api::IoCallResult InstanceImplPosix::stat(absl::string_view path) { return resultSuccess(infoFromStat(path, s)); } +Api::IoCallBoolResult InstanceImplPosix::createPath(absl::string_view path) { + // Ideally we could just use std::filesystem::create_directories for this, + // identical to ../win32/filesystem_impl.cc, but the OS version used in mobile + // CI doesn't support it, so we have to do recursive path creation manually. + while (!path.empty() && path.back() == '/') { + path.remove_suffix(1); + } + if (directoryExists(std::string{path})) { + return resultSuccess(false); + } + absl::string_view subpath = path; + do { + size_t slashpos = subpath.find_last_of('/'); + if (slashpos == absl::string_view::npos) { + return resultFailure(false, ENOENT); + } + subpath = subpath.substr(0, slashpos); + } while (!directoryExists(std::string{subpath})); + // We're now at the most-nested directory that already exists. + // Time to create paths recursively. + while (subpath != path) { + size_t slashpos = path.find_first_of('/', subpath.size() + 2); + subpath = (slashpos == absl::string_view::npos) ? path : path.substr(0, slashpos); + std::string dir_to_create{subpath}; + if (mkdir(dir_to_create.c_str(), 0777)) { + return resultFailure(false, errno); + } + } + return resultSuccess(true); +} + FileImplPosix::FlagsAndMode FileImplPosix::translateFlag(FlagSet in) { int out = 0; mode_t mode = 0; diff --git a/source/common/filesystem/posix/filesystem_impl.h b/source/common/filesystem/posix/filesystem_impl.h index 0b7616653039..e952a129f93e 100644 --- a/source/common/filesystem/posix/filesystem_impl.h +++ b/source/common/filesystem/posix/filesystem_impl.h @@ -62,6 +62,7 @@ class InstanceImplPosix : public Instance { PathSplitResult splitPathFromFilename(absl::string_view path) override; bool illegalPath(const std::string& path) override; Api::IoCallResult stat(absl::string_view path) override; + Api::IoCallBoolResult createPath(absl::string_view path) override; private: Api::SysCallStringResult canonicalPath(const std::string& path); diff --git a/source/common/filesystem/win32/filesystem_impl.cc b/source/common/filesystem/win32/filesystem_impl.cc index d7ea1a26b9a9..d400cea68b13 100644 --- a/source/common/filesystem/win32/filesystem_impl.cc +++ b/source/common/filesystem/win32/filesystem_impl.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -186,6 +187,15 @@ Api::IoCallResult InstanceImplWin32::stat(absl::string_view path) { return resultSuccess(fileInfoFromAttributeData(full_path, data)); } +Api::IoCallBoolResult InstanceImplWin32::createPath(absl::string_view path) { + std::error_code ec; + while (!path.empty() && path.back() == '/') { + path.remove_suffix(1); + } + bool result = std::filesystem::create_directories(std::string{path}, ec); + return ec ? resultFailure(false, ec.value()) : resultSuccess(result); +} + FileImplWin32::FlagsAndMode FileImplWin32::translateFlag(FlagSet in) { DWORD access = 0; DWORD creation = OPEN_EXISTING; diff --git a/source/common/filesystem/win32/filesystem_impl.h b/source/common/filesystem/win32/filesystem_impl.h index 8f6f2050935b..6a4ccf5521f3 100644 --- a/source/common/filesystem/win32/filesystem_impl.h +++ b/source/common/filesystem/win32/filesystem_impl.h @@ -96,6 +96,7 @@ class InstanceImplWin32 : public Instance { PathSplitResult splitPathFromFilename(absl::string_view path) override; bool illegalPath(const std::string& path) override; Api::IoCallResult stat(absl::string_view path) override; + Api::IoCallBoolResult createPath(absl::string_view path) override; }; using FileImpl = FileImplWin32; diff --git a/test/common/filesystem/filesystem_impl_test.cc b/test/common/filesystem/filesystem_impl_test.cc index 1645a9bc48c6..f909d987cbb7 100644 --- a/test/common/filesystem/filesystem_impl_test.cc +++ b/test/common/filesystem/filesystem_impl_test.cc @@ -361,6 +361,51 @@ TEST_F(FileSystemImplTest, StatOnDirectoryReturnsDirectoryType) { EXPECT_EQ(info_result.return_value_.file_type_, FileType::Directory); } +TEST_F(FileSystemImplTest, CreatePathCreatesDirectoryAndReturnsSuccessForExistingDirectory) { + const std::string new_dir_path = TestEnvironment::temporaryPath("envoy_test_dir"); + Api::IoCallBoolResult result = file_system_.createPath(new_dir_path); + Cleanup cleanup{[new_dir_path]() { TestEnvironment::removePath(new_dir_path); }}; + EXPECT_TRUE(result.return_value_); + EXPECT_THAT(result.err_, ::testing::IsNull()) << result.err_->getErrorDetails(); + // A bit awkwardly using file_system_.stat to test that the directory exists, because + // otherwise we might have to do windows/linux conditional code for this. + const Api::IoCallResult info_result = file_system_.stat(new_dir_path); + EXPECT_THAT(info_result.err_, ::testing::IsNull()) << info_result.err_->getErrorDetails(); + EXPECT_EQ(info_result.return_value_.name_, "envoy_test_dir"); + EXPECT_EQ(info_result.return_value_.file_type_, FileType::Directory); + // Ensure it returns true if we 'create' an existing path too. + result = file_system_.createPath(new_dir_path); + EXPECT_FALSE(result.return_value_); + EXPECT_THAT(result.err_, ::testing::IsNull()) << result.err_->getErrorDetails(); +} + +TEST_F(FileSystemImplTest, CreatePathCreatesDirectoryGivenTrailingSlash) { + const std::string new_dir_path = TestEnvironment::temporaryPath("envoy_test_dir/"); + const Api::IoCallBoolResult result = file_system_.createPath(new_dir_path); + Cleanup cleanup{[new_dir_path]() { + TestEnvironment::removePath(new_dir_path.substr(0, new_dir_path.size() - 1)); + }}; + EXPECT_TRUE(result.return_value_); + EXPECT_THAT(result.err_, ::testing::IsNull()) << result.err_->getErrorDetails(); + // A bit awkwardly using file_system_.stat to test that the directory exists, because + // otherwise we might have to do windows/linux conditional code for this. + const Api::IoCallResult info_result = + file_system_.stat(absl::string_view{new_dir_path}.substr(0, new_dir_path.size() - 1)); + EXPECT_THAT(info_result.err_, ::testing::IsNull()) << info_result.err_->getErrorDetails(); + EXPECT_EQ(info_result.return_value_.name_, "envoy_test_dir"); + EXPECT_EQ(info_result.return_value_.file_type_, FileType::Directory); +} + +TEST_F(FileSystemImplTest, CreatePathReturnsErrorOnNoPermissionToWriteDir) { +#ifdef WIN32 + GTEST_SKIP() << "It's hard to not have permission to create a directory on Windows"; +#endif + const std::string new_dir_path = "/should_fail_to_create_directory_in_root/x/y"; + const Api::IoCallBoolResult result = file_system_.createPath(new_dir_path); + EXPECT_FALSE(result.return_value_); + EXPECT_THAT(result.err_, testing::Not(testing::IsNull())); +} + TEST_F(FileSystemImplTest, StatOnFileOpenOrClosedMeasuresTheExpectedValues) { const std::string file_path = TestEnvironment::writeStringToFileForTest("test_envoy", "0123456789"); diff --git a/test/mocks/filesystem/mocks.h b/test/mocks/filesystem/mocks.h index 5ecb53b97e45..2ae9642ed474 100644 --- a/test/mocks/filesystem/mocks.h +++ b/test/mocks/filesystem/mocks.h @@ -67,6 +67,7 @@ class MockInstance : public Instance { MOCK_METHOD(PathSplitResult, splitPathFromFilename, (absl::string_view)); MOCK_METHOD(bool, illegalPath, (const std::string&)); MOCK_METHOD(Api::IoCallResult, stat, (absl::string_view)); + MOCK_METHOD(Api::IoCallBoolResult, createPath, (absl::string_view)); }; class MockWatcher : public Watcher { diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 62ecae6eab39..661903315d4c 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -10,7 +10,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/config:96.2" "source/common/crypto:88.1" "source/common/event:95.1" # Emulated edge events guards don't report LCOV -"source/common/filesystem/posix:96.5" # FileReadToEndNotReadable keeps failing +"source/common/filesystem/posix:96.2" # FileReadToEndNotReadable fails in some env; createPath can't test all failure branches. "source/common/http:96.3" "source/common/http/http2:95.0" "source/common/json:93.4" diff --git a/test/test_common/file_system_for_test.cc b/test/test_common/file_system_for_test.cc index 4563547f0343..1c5b4e219bcb 100644 --- a/test/test_common/file_system_for_test.cc +++ b/test/test_common/file_system_for_test.cc @@ -121,6 +121,11 @@ Api::IoCallResult MemfileInstanceImpl::stat(absl::string_view path) { return file_system_->stat(path); } +Api::IoCallBoolResult MemfileInstanceImpl::createPath(absl::string_view) { + // Creating an imaginary path is always successful. + return resultSuccess(true); +} + MemfileInstanceImpl::MemfileInstanceImpl() : file_system_{new InstanceImpl()}, use_memfiles_(false) {} diff --git a/test/test_common/file_system_for_test.h b/test/test_common/file_system_for_test.h index 5711d334cf25..1cb712144b8a 100644 --- a/test/test_common/file_system_for_test.h +++ b/test/test_common/file_system_for_test.h @@ -42,6 +42,8 @@ class MemfileInstanceImpl : public Instance { Api::IoCallResult stat(absl::string_view path) override; + Api::IoCallBoolResult createPath(absl::string_view path) override; + private: friend class ScopedUseMemfiles; From eed541de6159337db7a6a2f1b93a13f466efa979 Mon Sep 17 00:00:00 2001 From: Greg Greenway Date: Mon, 1 May 2023 17:35:18 -0700 Subject: [PATCH 064/740] ssl: upgrade FIPS boringssl version (#27087) * ssl: upgrade FIPS boringssl version Signed-off-by: Greg Greenway --- bazel/external/boringssl_fips.genrule_cmd | 48 +++++++++---- bazel/external/boringssl_fips.patch | 18 ----- bazel/repositories.bzl | 1 - bazel/repository_locations.bzl | 8 +-- .../transport_sockets/tls/utility.cc | 69 +------------------ .../transport_sockets/tls/utility_test.cc | 2 +- 6 files changed, 40 insertions(+), 106 deletions(-) delete mode 100644 bazel/external/boringssl_fips.patch diff --git a/bazel/external/boringssl_fips.genrule_cmd b/bazel/external/boringssl_fips.genrule_cmd index b4036e9bb9f0..37fc301a9f9f 100755 --- a/bazel/external/boringssl_fips.genrule_cmd +++ b/bazel/external/boringssl_fips.genrule_cmd @@ -16,20 +16,21 @@ fi ROOT=./external/boringssl_fips pushd "$ROOT" -# Build tools requirements: -# - Clang compiler version 7.0.1 (https://releases.llvm.org/download.html) -# - Go programming language version 1.12.7 (https://golang.org/dl/) -# - Ninja build system version 1.9.0 (https://github.com/ninja-build/ninja/releases) +# Build tools requirements (from section 12.1 of https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp4407.pdf): +# - Clang compiler version 12.0.0 (https://releases.llvm.org/download.html) +# - Go programming language version 1.16.5 (https://golang.org/dl/) +# - Ninja build system version 1.10.2 (https://github.com/ninja-build/ninja/releases) +# - Cmake version 3.20.1 (https://cmake.org/download/) # Override $PATH for build tools, to avoid picking up anything else. export PATH="$(dirname `which cmake`):/usr/bin:/bin" -# Clang 7.0.1 -VERSION=7.0.1 -SHA256=02ad925add5b2b934d64c3dd5cbd1b2002258059f7d962993ba7f16524c3089c -PLATFORM="x86_64-linux-gnu-ubuntu-16.04" +# Clang +VERSION=12.0.0 +SHA256=a9ff205eb0b73ca7c86afc6432eed1c2d49133bd0d49e47b15be59bbf0dd292e +PLATFORM="x86_64-linux-gnu-ubuntu-20.04" -curl -sLO https://releases.llvm.org/"$VERSION"/clang+llvm-"$VERSION"-"$PLATFORM".tar.xz \ +curl -sLO https://github.com/llvm/llvm-project/releases/download/llvmorg-"$VERSION"/clang+llvm-"$VERSION"-"$PLATFORM".tar.xz \ && echo "$SHA256" clang+llvm-"$VERSION"-"$PLATFORM".tar.xz | sha256sum --check tar xf clang+llvm-"$VERSION"-"$PLATFORM".tar.xz @@ -42,9 +43,9 @@ if [[ `clang --version | head -1 | awk '{print $3}'` != "$VERSION" ]]; then exit 1 fi -# Go 1.12.7 -VERSION=1.12.7 -SHA256=66d83bfb5a9ede000e33c6579a91a29e6b101829ad41fffb5c5bb6c900e109d9 +# Go +VERSION=1.16.5 +SHA256=b12c23023b68de22f74c0524f10b753e7b08b1504cb7e417eccebdd3fae49061 PLATFORM="linux-amd64" curl -sLO https://dl.google.com/go/go"$VERSION"."$PLATFORM".tar.gz \ @@ -60,9 +61,9 @@ if [[ `go version | awk '{print $3}'` != "go$VERSION" ]]; then exit 1 fi -# Ninja 1.9.0 -VERSION=1.9.0 -SHA256=1b1235f2b0b4df55ac6d80bbe681ea3639c9d2c505c7ff2159a3daf63d196305 +# Ninja +VERSION=1.10.2 +SHA256=763464859c7ef2ea3a0a10f4df40d2025d3bb9438fcb1228404640410c0ec22d PLATFORM="linux" curl -sLO https://github.com/ninja-build/ninja/releases/download/v"$VERSION"/ninja-"$PLATFORM".zip \ @@ -76,6 +77,22 @@ if [[ `ninja --version` != "$VERSION" ]]; then exit 1 fi +# CMake +VERSION=3.20.1 +SHA256=b8c141bd7a6d335600ab0a8a35e75af79f95b837f736456b5532f4d717f20a09 +PLATFORM="linux-x86_64" + +curl -sLO https://github.com/Kitware/CMake/releases/download/v"$VERSION"/cmake-"$VERSION"-"$PLATFORM".tar.gz \ + && echo "$SHA256" cmake-"$VERSION"-"$PLATFORM".tar.gz | sha256sum --check +tar xf cmake-"$VERSION"-"$PLATFORM".tar.gz + +export PATH="$PWD/cmake-$VERSION-$PLATFORM/bin:$PATH" + +if [[ `cmake --version | head -n1` != "cmake version $VERSION" ]]; then + echo "ERROR: CMake version doesn't match." + exit 1 +fi + # Clean after previous build. rm -rf boringssl/build @@ -84,6 +101,7 @@ cd boringssl mkdir build && cd build && cmake -GNinja -DCMAKE_TOOLCHAIN_FILE=${HOME}/toolchain -DFIPS=1 -DCMAKE_BUILD_TYPE=Release .. ninja ninja run_tests +./crypto/crypto_test # Verify correctness of the FIPS build. if [[ `tool/bssl isfips` != "1" ]]; then diff --git a/bazel/external/boringssl_fips.patch b/bazel/external/boringssl_fips.patch deleted file mode 100644 index 37247dc2f5c5..000000000000 --- a/bazel/external/boringssl_fips.patch +++ /dev/null @@ -1,18 +0,0 @@ -# Fix FIPS build (from BoringSSL commit 4ca15d5dcbe6e8051a4654df7c971ea8307abfe0). -# -# The modulewrapper is not a part of the FIPS module, so it can be patched without -# concern about breaking the FIPS validation. ---- boringssl/util/fipstools/acvp/modulewrapper/modulewrapper.cc -+++ boringssl/util/fipstools/acvp/modulewrapper/modulewrapper.cc -@@ -12,9 +12,11 @@ - * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ - -+#include - #include - - #include -+#include - #include - #include - #include diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index daa63ea13f27..1d8cbb9f33b7 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -353,7 +353,6 @@ def _boringssl_fips(): external_http_archive( name = "boringssl_fips", build_file = "@envoy//bazel/external:boringssl_fips.BUILD", - patches = ["@envoy//bazel/external:boringssl_fips.patch"], ) def _com_github_circonus_labs_libcircllhist(): diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index aef44d2ca1cd..94c00e973cbb 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -125,11 +125,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_url = "https://boringssl.googlesource.com/boringssl/+/master/crypto/fipsmodule/FIPS.md", # When this is updated to a revision newer than 2022-08-12, # CertValidatorUtil::setIgnoreCertificateExpiration can be simplified. - version = "fips-20190808", - sha256 = "3b5fdf23274d4179c2077b5e8fa625d9debd7a390aac1d165b7e47234f648bb8", - urls = ["https://commondatastorage.googleapis.com/chromium-boringssl-fips/boringssl-ae223d6138807a13006342edfeef32e813246b39.tar.xz"], + version = "fips-20210429", + sha256 = "a4d069ccef6f3c7bc0c68de82b91414f05cb817494cd1ab483dcf3368883c7c2", + urls = ["https://commondatastorage.googleapis.com/chromium-boringssl-fips/boringssl-853ca1ea1168dff08011e5d42d94609cc0ca2e27.tar.xz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2019-08-08", + release_date = "2021-04-29", cpe = "cpe:2.3:a:google:boringssl:*", ), aspect_bazel_lib = dict( diff --git a/source/extensions/transport_sockets/tls/utility.cc b/source/extensions/transport_sockets/tls/utility.cc index 788c07f6ca2e..2219998916de 100644 --- a/source/extensions/transport_sockets/tls/utility.cc +++ b/source/extensions/transport_sockets/tls/utility.cc @@ -16,29 +16,6 @@ namespace Extensions { namespace TransportSockets { namespace Tls { -#if BORINGSSL_API_VERSION < 10 -static constexpr absl::string_view SSL_ERROR_NONE_MESSAGE = "NONE"; -static constexpr absl::string_view SSL_ERROR_SSL_MESSAGE = "SSL"; -static constexpr absl::string_view SSL_ERROR_WANT_READ_MESSAGE = "WANT_READ"; -static constexpr absl::string_view SSL_ERROR_WANT_WRITE_MESSAGE = "WANT_WRITE"; -static constexpr absl::string_view SSL_ERROR_WANT_X509_LOOPUP_MESSAGE = "WANT_X509_LOOKUP"; -static constexpr absl::string_view SSL_ERROR_SYSCALL_MESSAGE = "SYSCALL"; -static constexpr absl::string_view SSL_ERROR_ZERO_RETURN_MESSAGE = "ZERO_RETURN"; -static constexpr absl::string_view SSL_ERROR_WANT_CONNECT_MESSAGE = "WANT_CONNECT"; -static constexpr absl::string_view SSL_ERROR_WANT_ACCEPT_MESSAGE = "WANT_ACCEPT"; -static constexpr absl::string_view SSL_ERROR_WANT_CHANNEL_ID_LOOKUP_MESSAGE = - "WANT_CHANNEL_ID_LOOKUP"; -static constexpr absl::string_view SSL_ERROR_PENDING_SESSION_MESSAGE = "PENDING_SESSION"; -static constexpr absl::string_view SSL_ERROR_PENDING_CERTIFICATE_MESSAGE = "PENDING_CERTIFICATE"; -static constexpr absl::string_view SSL_ERROR_WANT_PRIVATE_KEY_OPERATION_MESSAGE = - "WANT_PRIVATE_KEY_OPERATION"; -static constexpr absl::string_view SSL_ERROR_PENDING_TICKET_MESSAGE = "PENDING_TICKET"; -static constexpr absl::string_view SSL_ERROR_EARLY_DATA_REJECTED_MESSAGE = "EARLY_DATA_REJECTED"; -static constexpr absl::string_view SSL_ERROR_WANT_CERTIFICATE_VERIFY_MESSAGE = - "WANT_CERTIFICATE_VERIFY"; -static constexpr absl::string_view SSL_ERROR_HANDOFF_MESSAGE = "HANDOFF"; -static constexpr absl::string_view SSL_ERROR_HANDBACK_MESSAGE = "HANDBACK"; -#endif static constexpr absl::string_view SSL_ERROR_UNKNOWN_ERROR_MESSAGE = "UNKNOWN_ERROR"; Envoy::Ssl::CertificateDetailsPtr Utility::certificateDetails(X509* cert, const std::string& path, @@ -332,54 +309,12 @@ absl::optional Utility::getLastCryptoError() { } absl::string_view Utility::getErrorDescription(int err) { -#if BORINGSSL_API_VERSION < 10 - // TODO(davidben): Remove this and the corresponding SSL_ERROR_*_MESSAGE constants when the FIPS - // build is updated to a later version. - switch (err) { - case SSL_ERROR_NONE: - return SSL_ERROR_NONE_MESSAGE; - case SSL_ERROR_SSL: - return SSL_ERROR_SSL_MESSAGE; - case SSL_ERROR_WANT_READ: - return SSL_ERROR_WANT_READ_MESSAGE; - case SSL_ERROR_WANT_WRITE: - return SSL_ERROR_WANT_WRITE_MESSAGE; - case SSL_ERROR_WANT_X509_LOOKUP: - return SSL_ERROR_WANT_X509_LOOPUP_MESSAGE; - case SSL_ERROR_SYSCALL: - return SSL_ERROR_SYSCALL_MESSAGE; - case SSL_ERROR_ZERO_RETURN: - return SSL_ERROR_ZERO_RETURN_MESSAGE; - case SSL_ERROR_WANT_CONNECT: - return SSL_ERROR_WANT_CONNECT_MESSAGE; - case SSL_ERROR_WANT_ACCEPT: - return SSL_ERROR_WANT_ACCEPT_MESSAGE; - case SSL_ERROR_WANT_CHANNEL_ID_LOOKUP: - return SSL_ERROR_WANT_CHANNEL_ID_LOOKUP_MESSAGE; - case SSL_ERROR_PENDING_SESSION: - return SSL_ERROR_PENDING_SESSION_MESSAGE; - case SSL_ERROR_PENDING_CERTIFICATE: - return SSL_ERROR_PENDING_CERTIFICATE_MESSAGE; - case SSL_ERROR_WANT_PRIVATE_KEY_OPERATION: - return SSL_ERROR_WANT_PRIVATE_KEY_OPERATION_MESSAGE; - case SSL_ERROR_PENDING_TICKET: - return SSL_ERROR_PENDING_TICKET_MESSAGE; - case SSL_ERROR_EARLY_DATA_REJECTED: - return SSL_ERROR_EARLY_DATA_REJECTED_MESSAGE; - case SSL_ERROR_WANT_CERTIFICATE_VERIFY: - return SSL_ERROR_WANT_CERTIFICATE_VERIFY_MESSAGE; - case SSL_ERROR_HANDOFF: - return SSL_ERROR_HANDOFF_MESSAGE; - case SSL_ERROR_HANDBACK: - return SSL_ERROR_HANDBACK_MESSAGE; - } -#else const char* description = SSL_error_description(err); if (description) { return description; } -#endif - ENVOY_BUG(false, "Unknown BoringSSL error had occurred"); + + IS_ENVOY_BUG("BoringSSL error had occurred: SSL_error_description() returned nullptr"); return SSL_ERROR_UNKNOWN_ERROR_MESSAGE; } diff --git a/test/extensions/transport_sockets/tls/utility_test.cc b/test/extensions/transport_sockets/tls/utility_test.cc index 406240a7fcde..e2707bb40935 100644 --- a/test/extensions/transport_sockets/tls/utility_test.cc +++ b/test/extensions/transport_sockets/tls/utility_test.cc @@ -194,7 +194,7 @@ TEST(UtilityTest, SslErrorDescriptionTest) { } EXPECT_ENVOY_BUG(EXPECT_EQ(Utility::getErrorDescription(-1), "UNKNOWN_ERROR"), - "Unknown BoringSSL error had occurred"); + "BoringSSL error had occurred: SSL_error_description() returned nullptr"); } TEST(UtilityTest, TestGetX509ErrorInfo) { From 4917e4bcca996b0da1a98d3c3acaa5a8c8702dfa Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Mon, 1 May 2023 23:12:09 -0400 Subject: [PATCH 065/740] ci: run Kotlin tests with signal_trace disabled (#27090) * ci: run Kotlin tests with signal_trace disabled There are several Java tests that fail without the `--define=signal_trace=disabled` flag. The flag is already on for Java tests; this commit turns them on for Kotlin tests on CI as well. Also, this solves one of the problems with flaky tests in StatsFlushIntegrationTest.kt, as described in https://github.com/envoyproxy/envoy/issues/24657, so we re-enable the test that now passes. The other StatsFlushIntegrationTest test still fails and will be debugged separately. Signed-off-by: Ali Beyad --- .github/workflows/mobile-android_tests.yml | 2 ++ mobile/test/kotlin/integration/StatFlushIntegrationTest.kt | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml index 86998d2cfc3e..c198cef9b363 100644 --- a/.github/workflows/mobile-android_tests.yml +++ b/.github/workflows/mobile-android_tests.yml @@ -45,6 +45,7 @@ jobs: --test_output=all \ --build_tests_only \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ + --define=signal_trace=disabled \ //test/kotlin/io/... javatestsmac: if: github.repository == 'envoyproxy/envoy' @@ -113,4 +114,5 @@ jobs: --build_tests_only \ --config test-android \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-linux-clang") \ + --define=signal_trace=disabled \ //test/kotlin/... diff --git a/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt b/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt index 05f9b58346b4..733cd816272c 100644 --- a/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt +++ b/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt @@ -12,9 +12,6 @@ import org.junit.After import org.junit.Ignore import org.junit.Test -// TODO(abeyad): Fix the test and re-enable. -// See https://github.com/envoyproxy/envoy/issues/24657. -@Ignore class StatFlushIntegrationTest { private var engine: Engine? = null @@ -43,6 +40,9 @@ class StatFlushIntegrationTest { assertThat(countDownLatch.await(30, TimeUnit.SECONDS)).isTrue() } + // TODO(abeyad): Fix the test and re-enable. + // See https://github.com/envoyproxy/envoy/issues/24657. + @Ignore @Test fun `flush flushes to stats sink`() { val countDownLatch = CountDownLatch(1) From 7897f75a1f0cc744cbd83191fa37fb485ceabab8 Mon Sep 17 00:00:00 2001 From: Easwar Swaminathan Date: Mon, 1 May 2023 20:16:55 -0700 Subject: [PATCH 066/740] xds: add config for pick_first LB policy extension (#26952) Signed-off-by: Easwar Swaminathan --- api/BUILD | 1 + .../pick_first/v3/BUILD | 9 ++++++++ .../pick_first/v3/pick_first.proto | 22 +++++++++++++++++++ api/versioning/BUILD | 1 + 4 files changed, 33 insertions(+) create mode 100644 api/envoy/extensions/load_balancing_policies/pick_first/v3/BUILD create mode 100644 api/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.proto diff --git a/api/BUILD b/api/BUILD index 26ac05ccad40..a40ea1204b00 100644 --- a/api/BUILD +++ b/api/BUILD @@ -254,6 +254,7 @@ proto_library( "//envoy/extensions/load_balancing_policies/common/v3:pkg", "//envoy/extensions/load_balancing_policies/least_request/v3:pkg", "//envoy/extensions/load_balancing_policies/maglev/v3:pkg", + "//envoy/extensions/load_balancing_policies/pick_first/v3:pkg", "//envoy/extensions/load_balancing_policies/random/v3:pkg", "//envoy/extensions/load_balancing_policies/ring_hash/v3:pkg", "//envoy/extensions/load_balancing_policies/round_robin/v3:pkg", diff --git a/api/envoy/extensions/load_balancing_policies/pick_first/v3/BUILD b/api/envoy/extensions/load_balancing_policies/pick_first/v3/BUILD new file mode 100644 index 000000000000..ee92fb652582 --- /dev/null +++ b/api/envoy/extensions/load_balancing_policies/pick_first/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.proto b/api/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.proto new file mode 100644 index 000000000000..2a7db058e217 --- /dev/null +++ b/api/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package envoy.extensions.load_balancing_policies.pick_first.v3; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.load_balancing_policies.pick_first.v3"; +option java_outer_classname = "PickFirstProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3;pick_firstv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Pick First Load Balancing Policy] +// [#not-implemented-hide:] + +// This configuration allows the built-in PICK_FIRST LB policy to be configured +// via the LB policy extension point. +message PickFirst { + // If set to true, instructs the LB policy to shuffle the list of addresses + // received from the name resolver before attempting to connect to them. + bool shuffle_address_list = 1; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 5d0548a4bbc3..82f27feda2b5 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -193,6 +193,7 @@ proto_library( "//envoy/extensions/load_balancing_policies/common/v3:pkg", "//envoy/extensions/load_balancing_policies/least_request/v3:pkg", "//envoy/extensions/load_balancing_policies/maglev/v3:pkg", + "//envoy/extensions/load_balancing_policies/pick_first/v3:pkg", "//envoy/extensions/load_balancing_policies/random/v3:pkg", "//envoy/extensions/load_balancing_policies/ring_hash/v3:pkg", "//envoy/extensions/load_balancing_policies/round_robin/v3:pkg", From a0a0273f6ef89977e859b9709226b5091a568b3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 May 2023 08:21:15 +0100 Subject: [PATCH 067/740] build(deps): bump orjson from 3.8.10 to 3.8.11 in /tools/base (#27040) Bumps [orjson](https://github.com/ijl/orjson) from 3.8.10 to 3.8.11. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.8.10...3.8.11) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 109 +++++++++++++++++------------------- 1 file changed, 52 insertions(+), 57 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 6e449fc57644..61983d5b761b 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -784,63 +784,58 @@ multidict==6.0.4 \ # -r requirements.in # aiohttp # yarl -orjson==3.8.10 \ - --hash=sha256:0826ad2dc1cea1547edff14ce580374f0061d853cbac088c71162dbfe2e52205 \ - --hash=sha256:0b470d31244a6f647e5402aac7d2abaf7bb4f52379acf67722a09d35a45c9417 \ - --hash=sha256:11ae68f995a50724032af297c92f20bcde31005e0bf3653b12bff9356394615b \ - --hash=sha256:1486600bc1dd1db26c588dd482689edba3d72d301accbe4301db4b2b28bd7aa4 \ - --hash=sha256:1810e5446fe68d61732e9743592da0ec807e63972eef076d09e02878c2f5958e \ - --hash=sha256:2073b62822738d6740bd2492f6035af5c2fd34aa198322b803dc0e70559a17b7 \ - --hash=sha256:26aee557cf8c93b2a971b5a4a8e3cca19780573531493ce6573aa1002f5c4378 \ - --hash=sha256:27bb26e171e9cfdbec39c7ca4739b6bef8bd06c293d56d92d5e3a3fc017df17d \ - --hash=sha256:2b8cdaacecb92997916603ab232bb096d0fa9e56b418ca956b9754187d65ca06 \ - --hash=sha256:344ea91c556a2ce6423dc13401b83ab0392aa697a97fa4142c2c63a6fd0bbfef \ - --hash=sha256:345e41abd1d9e3ecfb554e1e75ff818cf42e268bd06ad25a96c34e00f73a327e \ - --hash=sha256:34b6901c110c06ab9e8d7d0496db4bc9a0c162ca8d77f67539d22cb39e0a1ef4 \ - --hash=sha256:35d879b46b8029e1e01e9f6067928b470a4efa1ca749b6d053232b873c2dcf66 \ - --hash=sha256:3cfe32b1227fe029a5ad989fbec0b453a34e5e6d9a977723f7c3046d062d3537 \ - --hash=sha256:4355c9aedfefe60904e8bd7901315ebbc8bb828f665e4c9bc94b1432e67cb6f7 \ - --hash=sha256:45a5afc9cda6b8aac066dd50d8194432fbc33e71f7164f95402999b725232d78 \ - --hash=sha256:48824649019a25d3e52f6454435cf19fe1eb3d05ee697e65d257f58ae3aa94d9 \ - --hash=sha256:4bf2556ba99292c4dc550560384dd22e88b5cdbe6d98fb4e202e902b5775cf9f \ - --hash=sha256:4dfe0651e26492d5d929bbf4322de9afbd1c51ac2e3947a7f78492b20359711d \ - --hash=sha256:595e1e7d04aaaa3d41113e4eb9f765ab642173c4001182684ae9ddc621bb11c8 \ - --hash=sha256:5a0b1f4e4fa75e26f814161196e365fc0e1a16e3c07428154505b680a17df02f \ - --hash=sha256:61e2e51cefe7ef90c4fbbc9fd38ecc091575a3ea7751d56fad95cbebeae2a054 \ - --hash=sha256:64ffd92328473a2f9af059410bd10c703206a4bbc7b70abb1bedcd8761e39eb8 \ - --hash=sha256:6a286ad379972e4f46579e772f0477e6b505f1823aabcd64ef097dbb4549e1a4 \ - --hash=sha256:6bbd7b3a3e2030b03c68c4d4b19a2ef5b89081cbb43c05fe2010767ef5e408db \ - --hash=sha256:6fa3a26dcf0f5f2912a8ce8e87273e68b2a9526854d19fd09ea671b154418e88 \ - --hash=sha256:7d27b6182f75896dd8c10ea0f78b9265a3454be72d00632b97f84d7031900dd4 \ - --hash=sha256:81aa3f321d201bff0bd0f4014ea44e51d58a9a02d8f2b0eeab2cee22611be8e1 \ - --hash=sha256:8c1825997232a324911d11c75d91e1e0338c7b723c149cf53a5fc24496c048a4 \ - --hash=sha256:979f231e3bad1c835627eef1a30db12a8af58bfb475a6758868ea7e81897211f \ - --hash=sha256:9b23fb0264bbdd7218aa685cb6fc71f0dcecf34182f0a8596a3a0dff010c06f9 \ - --hash=sha256:a3fdee68c4bb3c5d6f89ed4560f1384b5d6260e48fbf868bae1a245a3c693d4d \ - --hash=sha256:a7bce6e61cea6426309259b04c6ee2295b3f823ea51a033749459fe2dd0423b2 \ - --hash=sha256:abce8d319aae800fd2d774db1106f926dee0e8a5ca85998fd76391fcb58ef94f \ - --hash=sha256:ad632dc330a7b39da42530c8d146f76f727d476c01b719dc6743c2b5701aaf6b \ - --hash=sha256:af7601a78b99f0515af2f8ab12c955c0072ffcc1e437fb2556f4465783a4d813 \ - --hash=sha256:b1f648ec89c6a426098868460c0ef8c86b457ce1378d7569ff4acb6c0c454048 \ - --hash=sha256:b2c4faf20b6bb5a2d7ac0c16f58eb1a3800abcef188c011296d1dc2bb2224d48 \ - --hash=sha256:b6e79d8864794635974b18821b49a7f27859d17b93413d4603efadf2e92da7a5 \ - --hash=sha256:b7b0ba074375e25c1594e770e2215941e2017c3cd121889150737fa1123e8bfe \ - --hash=sha256:b88afd662190f19c3bb5036a903589f88b1d2c2608fbb97281ce000db6b08897 \ - --hash=sha256:bc30de5c7b3a402eb59cc0656b8ee53ca36322fc52ab67739c92635174f88336 \ - --hash=sha256:bce970f293825e008dbf739268dfa41dfe583aa2a1b5ef4efe53a0e92e9671ea \ - --hash=sha256:c08b426fae7b9577b528f99af0f7e0ff3ce46858dd9a7d1bf86d30f18df89a4c \ - --hash=sha256:c2ef690335b24f9272dbf6639353c1ffc3f196623a92b851063e28e9515cf7dd \ - --hash=sha256:cb62ec16a1c26ad9487727b529103cb6a94a1d4969d5b32dd0eab5c3f4f5a6f2 \ - --hash=sha256:ce49999bcbbc14791c61844bc8a69af44f5205d219be540e074660038adae6bf \ - --hash=sha256:d2874cee6856d7c386b596e50bc517d1973d73dc40b2bd6abec057b5e7c76b2f \ - --hash=sha256:d953e6c2087dcd990e794f8405011369ee11cf13e9aaae3172ee762ee63947f2 \ - --hash=sha256:dcf6adb4471b69875034afab51a14b64f1026bc968175a2bb02c5f6b358bd413 \ - --hash=sha256:ddabc5e44702d13137949adee3c60b7091e73a664f6e07c7b428eebb2dea7bbf \ - --hash=sha256:e5d7f82506212e047b184c06e4bcd48c1483e101969013623cebcf51cf12cad9 \ - --hash=sha256:e999abca892accada083f7079612307d94dd14cc105a699588a324f843216509 \ - --hash=sha256:f3e9ac9483c2b4cd794e760316966b7bd1e6afb52b0218f068a4e80c9b2db4f6 \ - --hash=sha256:f7e85d4682f3ed7321d36846cad0503e944ea9579ef435d4c162e1b73ead8ac9 \ - --hash=sha256:faee89e885796a9cc493c930013fa5cfcec9bfaee431ddf00f0fbfb57166a8b3 +orjson==3.8.11 \ + --hash=sha256:04b60dfc1251742e79bb075d7a7c4e37078b932a02e6f005c45761bd90c69189 \ + --hash=sha256:08729e339ff3146e6de56c1166f014c3d2ec3e79ffb76d6c55d52cc892e5e477 \ + --hash=sha256:0a53b3c02a38aadc5302661c2ca18645093971488992df77ce14fef16f598b2e \ + --hash=sha256:0bc3d1b93a73b46a698c054697eb2d27bdedbc5ea0d11ec5f1a6bfbec36346b5 \ + --hash=sha256:0f9415b86ef154bf247fa78a6918aac50089c296e26fb6cf15bc9d7e6402a1f8 \ + --hash=sha256:1103e597c16f82c241e1b02beadc9c91cecd93e60433ca73cb6464dcc235f37c \ + --hash=sha256:12f647d4da0aab1997e25bed4fa2b76782b5b9d2d1bf3066b5f0a57d34d833c4 \ + --hash=sha256:173b8f8c750590f432757292cfb197582e5c14347b913b4017561d47af0e759b \ + --hash=sha256:176d742f53434541e50a5e659694073aa51dcbd8f29a1708a4fa1a320193c615 \ + --hash=sha256:1e97fdbb779a3b8f5d9fc7dfddef5325f81ee45897eb7cb4638d5d9734d42514 \ + --hash=sha256:1fedcc428416e23a6c9de62a000c22ae33bbe0108302ad5d5935e29ea739bf37 \ + --hash=sha256:235926b38ed9b76ab2bca99ff26ece79c1c46bc10079b06e660b087aecffbe69 \ + --hash=sha256:2fc050f8e7f2e4061c8c9968ad0be745b11b03913b77ffa8ceca65914696886c \ + --hash=sha256:3485c458670c0edb79ca149fe201f199dd9ccfe7ca3acbdef617e3c683e7b97f \ + --hash=sha256:358e515b8b19a275b259f5ee1e0efa2859b1d976b5ed5d016ac59f9e6c8788a3 \ + --hash=sha256:37f38c8194ce086e6a9816b4b8dde5e7f383feeed92feec0385d99baf64f9b6e \ + --hash=sha256:382f15861a4bf447ab9d07106010e61b217ef6d4245c6cf64af0c12c4c5e2346 \ + --hash=sha256:3afccf7f8684dca7f017837a315de0a1ab5c095de22a4eed206d079f9325ed72 \ + --hash=sha256:3b65424ceee82b94e3613233b67ef110dc58f9d83b0076ec47a506289552a861 \ + --hash=sha256:3c55065bc2075a5ea6ffb30462d84fd3aa5bbb7ae600855c325ee5753feec715 \ + --hash=sha256:4118dcd2b5a27a22af5ad92414073f25d93bca1868f1f580056003c84841062f \ + --hash=sha256:553fdaf9f4b5060a0dcc517ae0c511c289c184a83d6719d03c5602ed0eef0390 \ + --hash=sha256:58c068f93d701f9466f667bf3b5cb4e4946aee940df2b07ca5101f1cf1b60ce4 \ + --hash=sha256:5c3b5405edc3a5f9e34516ee1a729f6c46aecf6de960ae07a7b3e95ebdd0e1d9 \ + --hash=sha256:5ff10789cbc08a9fd94507c907ba55b9315e99f20345ff8ef34fac432dacd948 \ + --hash=sha256:62eb8bdcf6f4cdbe12743e88ad98696277a75f91a35e8fb93a7ea2b9f4a7000c \ + --hash=sha256:66f0c9e4e8f6641497a7dc50591af3704b11468e9fc90cfb5874f28b0a61edb5 \ + --hash=sha256:6d7b050135669d2335e40120215ad4120e29958c139f8bab68ce06a1cb1a1b2c \ + --hash=sha256:714c3e2be6ed7e4ff6e887926d6e171bfd94fdee76d7d3bfa74ee19237a2d49d \ + --hash=sha256:71a656f1c62e84c69060093e20cedff6a92e472d53ff5b8b9026b1b298542a68 \ + --hash=sha256:7c7b4fae3b8fc69c8e76f1c0694f3decfe8a57f87e7ac7779ebb59cd71135438 \ + --hash=sha256:7e4ded77ac7432a155d1d27a83bcadf722750aea3b9e6c4d47f2a92054ab71cb \ + --hash=sha256:882c77126c42dd93bb35288632d69b1e393863a2b752de3e5fe0112833609496 \ + --hash=sha256:9486963d2e65482c565dacb366adb36d22aa22acf7274b61490244c3d87fa631 \ + --hash=sha256:982ab319b7a5ece4199caf2a2b3a28e62a8e289cb6418548ef98bced7e2a6cfe \ + --hash=sha256:98befa717efaab7ddb847ebe47d473f6bd6f0cb53e98e6c3d487c7c58ba2e174 \ + --hash=sha256:9fa900bdd84b4576c8dd6f3e2a00b35797f29283af328c6e3d70addfa4c2d599 \ + --hash=sha256:b369019e597b59c4b97e9f925a3b725321fa1481c129d76c74c6ea3823f5d1e8 \ + --hash=sha256:b68a07794834b7bd53ae2a8b4fe4bf010734cae3f0917d434c83b97acf8e5bce \ + --hash=sha256:bf48ed8d4b6ab9f23b7ee642462369d7133412d72824bad89f9bf4311c06c6a1 \ + --hash=sha256:c2d3e6b65458ed71b6797f321d6e8bfeeadee9d3d31cac47806a608ea745edd7 \ + --hash=sha256:c67ac094a4dde914297543af19f22532d7124f3a35245580d8b756c4ff2f5884 \ + --hash=sha256:cdf201e77d3fac9d8d6f68d872ef45dccfe46f30b268bb88b6c5af5065b433aa \ + --hash=sha256:d47f97b99beb9bcac6e288a76b559543a61e0187443d8089204b757726b1d000 \ + --hash=sha256:d70b6db9d4e1e6057829cd7fe119c217cebaf989f88d14b2445fa69fc568d03e \ + --hash=sha256:d7d5aecccfaf2052cd07ed5bec8efba9ddfea055682fcd346047b1a3e9da3034 \ + --hash=sha256:e14903bfeb591a9117b7d40d81e3ebca9700b4e77bd829d6f22ea57941bb0ebf \ + --hash=sha256:ef52f1d5a2f89ef9049781c90ea35d5edf74374ed6ed515c286a706d1b290267 \ + --hash=sha256:f2ef933da50b31c112b252be03d1ef59e0d0552c1a08e48295bd529ce42aaab8 \ + --hash=sha256:f4e4a1001933166fd1c257b920b241b35322bef99ed7329338bf266ac053abe7 \ + --hash=sha256:f7aeefac55848aeb29f20b91fa55f9e488f446201bb1bb31dc17480d113d8955 # via # -r requirements.in # envoy-base-utils From 307fb1f5e6716070bbea889305899df41c7257fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 May 2023 08:21:45 +0100 Subject: [PATCH 068/740] build(deps): bump frozendict from 2.3.7 to 2.3.8 in /tools/base (#27071) Bumps [frozendict](https://github.com/Marco-Sulla/python-frozendict) from 2.3.7 to 2.3.8. - [Release notes](https://github.com/Marco-Sulla/python-frozendict/releases) - [Commits](https://github.com/Marco-Sulla/python-frozendict/compare/v2.3.7...v2.3.8) --- updated-dependencies: - dependency-name: frozendict dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 75 +++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 61983d5b761b..fa6c98056653 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -491,43 +491,44 @@ flake8==6.0.0 \ # -r requirements.in # envoy-code-check # pep8-naming -frozendict==2.3.7 \ - --hash=sha256:01c01ba4b1b77aa4385f27b2514ac284af42b331eeb4b853a05cf39145853e5f \ - --hash=sha256:0f63205aebde38645dce0732f90b59dace205104f2284fcee42bedbafb2068f3 \ - --hash=sha256:13b9cba0c86ce433ef866984a5f9dfeb4297b9b41449845066a64bb2cff28a92 \ - --hash=sha256:23a13ae84c09405f9a4e535e0799cadc9f2537ae624f7a7dcf381389caa5f311 \ - --hash=sha256:31ec7cfffaf669f5556228eff7b0ee469d2540bcc7de5e67c6e0b9fc9ae450ee \ - --hash=sha256:326bb8d6e057e12a9d1f146b7180fdf38c66243a50e4a4a516e22830db75cc72 \ - --hash=sha256:35e9185c936c7841566b455b5f7a0e7bbd5f28c74812c36c466faf3a5482bf8c \ - --hash=sha256:3c259eb24f167ef7f2b4562c2ebc915f457985ea5c18fd5a0682b681d0e61180 \ - --hash=sha256:499417e55e383c4d17635d6ed4170ac1632a0b2d39ee3a7a9f17c44345a69550 \ - --hash=sha256:4b576d84ff9ba7180cc2f507a2c2fda904536fae3d25fed674ba5611a1de5304 \ - --hash=sha256:4cd49b8bd04795fce6d58316510078f1ffdcd848d83e376e7f39fb98b704009b \ - --hash=sha256:54247f8dfdd9fdf0c8c2071d66ad69089766706e68ba2b07d546204a091e5756 \ - --hash=sha256:552633b1bf31d4c0091d09937c4e9571bc2a1cb1a28caa7487fa7385598fc2c9 \ - --hash=sha256:557fce99d9414c921ccd70a6b854977690400b8583656fc6fcb65de3f7ed0a5a \ - --hash=sha256:564373fb730f16d8ded8b5958a574d6fdcff3114a17a3e9835fd683ea5014860 \ - --hash=sha256:57bdac4370ee91fc03b7c7e66119e390506e347dfc31633073724551e3e7a30a \ - --hash=sha256:5c72e84c653b5ee978f6d3f552de94eb48969a4d4679cbdbe7f15f4ca9374e27 \ - --hash=sha256:6556f9d261f07ada93fd51020a412e146d5df3200e14456010d8d11bb534e691 \ - --hash=sha256:725834044799e401d6d5f5ffe43b2beb1768aa23ed4f5fcca91e98a88068bb02 \ - --hash=sha256:743b2947811d8348639b5d8bfcc658833e5fab03e3a97aa7ab4287d08faf320d \ - --hash=sha256:7771949613615c96a972715a66e02ac14a921a9f433f49ae97b7887582399d11 \ - --hash=sha256:799540d7517943e696bec0959c4fd0ecbd023fe17ea13b110905bc10947f93cb \ - --hash=sha256:7d1ba64144a7941bcfe905d4f6d6fd52f4918f07f0def754d3007fe242e6cf5c \ - --hash=sha256:84da41912b1ddfaaefc75b202c525be0ca16f19a42f5f8746bdadffcc8424730 \ - --hash=sha256:8ed8da6dae2dd98f08a2b7e93c16709157370ac65dd7dfef556eaad416dae8b0 \ - --hash=sha256:9409dc3359216d28aa068eb90cae11d4ab3a841c603b2e0991c5dca244794288 \ - --hash=sha256:a7ec47ada61a6c1f6b82c16432a87a59693031dc14167282d502a1d99ec967b0 \ - --hash=sha256:aa16ed2e4c3ab268e922999c62b748caf3d7f189c8e03ed10a26ad60d57f572a \ - --hash=sha256:aa4adfd36cd5c8a14f429fbbd18438ca5ca74d6658e3cd583cfc27bac7bd5bbf \ - --hash=sha256:b2c7b6e2dc2e7a71145803bbd846a967e8009ce30c8ab752b3d466773a5f7cdc \ - --hash=sha256:b63fcf53dddb1b9cbc7e7d19cde3a2a6be8f9c2bb0f16da0a143116992e1514a \ - --hash=sha256:dd381d26ea752b01f206ae159a2c5784d5d60bbdcc258b330149a443feae2e6d \ - --hash=sha256:e2484540116288fcb4eb8d20463c282db04e0332fd4d863b8242ef1d2b5f0115 \ - --hash=sha256:e7864b2f64d76e19367eecc331fbb7174460e4b5fd51e8cf7df5c974777e67bf \ - --hash=sha256:ef54086cb00d8cc7811640040c90ec90fbaced17d4e730bb6279ba1bf46da667 \ - --hash=sha256:f9825a2462421cf0e28bb82d86555e0f11f74417cd7f1abc1f6739c23096bcad +frozendict==2.3.8 \ + --hash=sha256:0bc4767e2f83db5b701c787e22380296977368b0c57e485ca71b2eedfa11c4a3 \ + --hash=sha256:145afd033ebfade28416093335261b8ec1af5cccc593482309e7add062ec8668 \ + --hash=sha256:23c4bb46e6b8246e1e7e49b5593c2bc09221db0d8f31f7c092be8dfb42b9e620 \ + --hash=sha256:2b2fd8ce36277919b36e3c834d2389f3cd7ac068ae730c312671dd4439a5dd65 \ + --hash=sha256:2b3435e5f1ca5ae68a5e95e64b09d6d5c645cadd6b87569a0b3019dd248c8d00 \ + --hash=sha256:313ed8d9ba6bac35d7635cd9580ee5721a0fb016f4d2d20f0efa05dbecbdb1be \ + --hash=sha256:3957d52f1906b0c85f641a1911d214255873f6408ab4e5ad657cc27a247fb145 \ + --hash=sha256:4742e76c4111bd09198d3ab66cef94be8506212311338f9182d6ef5f5cb60493 \ + --hash=sha256:47fc26468407fdeb428cfc89495b7921419e670355c21b383765482fdf6c5c14 \ + --hash=sha256:4c258aab9c8488338634f2ec670ef049dbf0ab0e7a2fa9bc2c7b5009cb614801 \ + --hash=sha256:5526559eca8f1780a4ee5146896f59afc31435313560208dd394a3a5e537d3ff \ + --hash=sha256:5e82befa7c385a668d569cebbebbdf49cee6fea4083f08e869a1b08cfb640a9f \ + --hash=sha256:638cf363d3cbca31a341503cf2219eac52a5f5140449676fae3d9644cd3c5487 \ + --hash=sha256:6ea638228692db2bf94bce40ea4b25f4077588497b516bd16576575560094bd9 \ + --hash=sha256:72cfe08ab8ae524e54848fa90b22d02c1b1ecfb3064438696bcaa4b953f18772 \ + --hash=sha256:750632cc890d8ee9484fe6d31b261159144b6efacc08e1317fe46accd1410373 \ + --hash=sha256:7a75bf87e76c4386caecdbdd02a99e53ad43a6b5c38fb3d5a634a9fc9ce41462 \ + --hash=sha256:7ee5fe2658a8ac9a57f748acaf563f6a47f80b8308cbf0a04fac0ba057d41f75 \ + --hash=sha256:80abe81d36e889ceec665e06ec764a7638000fa3e7be09786ac4d3ddc64b76db \ + --hash=sha256:8ccc94ac781710db44e142e1a11ff9b31d02c032c01c6868d51fcbef73086225 \ + --hash=sha256:8cf35ddd25513428ec152614def9696afb93ae5ec0eb54fa6aa6206eda77ac4c \ + --hash=sha256:9a506d807858fa961aaa5b48dab6154fdc6bd045bbe9310788bbff141bb42d13 \ + --hash=sha256:9ea5520e85447ff8d4681e181941e482662817ccba921b7cb3f87922056d892a \ + --hash=sha256:ba41a7ed019bd03b62d63ed3f8dea35b8243d1936f7c9ed4b5298ca45a01928e \ + --hash=sha256:c31abc8acea309b132dde441856829f6003a3d242da8b54bce4c0f2a3c8c63f0 \ + --hash=sha256:d086440328a465dea9bef2dbad7548d75d1a0a0d21f43a08c03e1ec79ac5240e \ + --hash=sha256:d188d062084fba0e4bf32719ff7380b26c050b932ff164043ce82ab90587c52b \ + --hash=sha256:d3c6ce943946c2a61501c8cf116fff0892d11dd579877eb36e2aea2c27fddfef \ + --hash=sha256:da98427de26b5a2865727947480cbb53860089c4d195baa29c539da811cea617 \ + --hash=sha256:e27c5c1d29d0eda7979253ec88abc239da1313b38f39f4b16984db3b3e482300 \ + --hash=sha256:e4c785de7f1a13f15963945f400656b18f057c2fc76c089dacf127a2bb188c03 \ + --hash=sha256:e72dbc1bcc2203cef38d205f692396f5505921a5680f66aa9a7e8bb71fd38f28 \ + --hash=sha256:ed5a6c5c7a0f57269577c2a338a6002949aea21a23b7b7d06da7e7dced8b605b \ + --hash=sha256:f0f573dc4861dd7ec9e055c8cceaf45355e894e749f621f199aab7b311ac4bdb \ + --hash=sha256:f2a4e818ac457f6354401dcb631527af25e5a20fcfc81e6b5054b45fc245caca \ + --hash=sha256:f83fed36497af9562ead5e9fb8443224ba2781786bd3b92b1087cb7d0ff20135 \ + --hash=sha256:ffc684773de7c88724788fa9787d0016fd75830412d58acbd9ed1a04762c675b # via # -r requirements.in # aio-run-runner From 34126dad1d57a81cad460326a850e2ebe8e7bca4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 May 2023 08:11:30 +0000 Subject: [PATCH 069/740] build(deps): bump distroless/base-nossl-debian11 from `9523ef8` to `f10e1fb` in /ci (#27097) build(deps): bump distroless/base-nossl-debian11 in /ci Bumps distroless/base-nossl-debian11 from `9523ef8` to `f10e1fb`. --- updated-dependencies: - dependency-name: distroless/base-nossl-debian11 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ci/Dockerfile-envoy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Dockerfile-envoy b/ci/Dockerfile-envoy index 189d6525e5cd..973278f82fa2 100644 --- a/ci/Dockerfile-envoy +++ b/ci/Dockerfile-envoy @@ -56,7 +56,7 @@ COPY --chown=0:0 --chmod=755 \ # STAGE: envoy-distroless # gcr.io/distroless/base-nossl-debian11:nonroot -FROM gcr.io/distroless/base-nossl-debian11:nonroot@sha256:9523ef8cf054e23a81e722d231c6f604ab43a03c5b174b5c8386c78c0b6473d0 AS envoy-distroless +FROM gcr.io/distroless/base-nossl-debian11:nonroot@sha256:f10e1fbf558c630a4b74a987e6c754d45bf59f9ddcefce090f6b111925996767 AS envoy-distroless EXPOSE 10000 ENTRYPOINT ["/usr/local/bin/envoy"] CMD ["-c", "/etc/envoy/envoy.yaml"] From 338c4fc9a0f65106e3a2410be94ed7f065795471 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 2 May 2023 09:33:03 +0100 Subject: [PATCH 070/740] repo: Sync version histories (#27102) Signed-off-by: Ryan Northey --- changelogs/1.23.9.yaml | 6 ++++++ changelogs/1.24.7.yaml | 6 ++++++ changelogs/1.25.6.yaml | 6 ++++++ changelogs/1.26.1.yaml | 6 ++++++ docs/inventories/v1.23/objects.inv | Bin 138055 -> 138089 bytes docs/inventories/v1.24/objects.inv | Bin 141541 -> 141606 bytes docs/inventories/v1.25/objects.inv | Bin 149503 -> 149576 bytes docs/inventories/v1.26/objects.inv | Bin 0 -> 153489 bytes docs/versions.yaml | 7 ++++--- 9 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 changelogs/1.23.9.yaml create mode 100644 changelogs/1.24.7.yaml create mode 100644 changelogs/1.25.6.yaml create mode 100644 changelogs/1.26.1.yaml create mode 100644 docs/inventories/v1.26/objects.inv diff --git a/changelogs/1.23.9.yaml b/changelogs/1.23.9.yaml new file mode 100644 index 000000000000..571d92c3ef35 --- /dev/null +++ b/changelogs/1.23.9.yaml @@ -0,0 +1,6 @@ +date: April 26, 2023 + +bug_fixes: +- area: tls + change: | + Fix build FIPS compliance when using both FIPS mode and Wasm extensions (``--define boringssl=fips`` and ``--define wasm=v8``). diff --git a/changelogs/1.24.7.yaml b/changelogs/1.24.7.yaml new file mode 100644 index 000000000000..571d92c3ef35 --- /dev/null +++ b/changelogs/1.24.7.yaml @@ -0,0 +1,6 @@ +date: April 26, 2023 + +bug_fixes: +- area: tls + change: | + Fix build FIPS compliance when using both FIPS mode and Wasm extensions (``--define boringssl=fips`` and ``--define wasm=v8``). diff --git a/changelogs/1.25.6.yaml b/changelogs/1.25.6.yaml new file mode 100644 index 000000000000..453a1ca3830e --- /dev/null +++ b/changelogs/1.25.6.yaml @@ -0,0 +1,6 @@ +date: April 27, 2023 + +bug_fixes: +- area: tls + change: | + Fix build FIPS compliance when using both FIPS mode and Wasm extensions (``--define boringssl=fips`` and ``--define wasm=v8``). diff --git a/changelogs/1.26.1.yaml b/changelogs/1.26.1.yaml new file mode 100644 index 000000000000..4b3f0f961997 --- /dev/null +++ b/changelogs/1.26.1.yaml @@ -0,0 +1,6 @@ +date: April 25, 2023 + +bug_fixes: +- area: tls + change: | + Fix build FIPS compliance when using both FIPS mode and Wasm extensions (``--define boringssl=fips`` and ``--define wasm=v8``). diff --git a/docs/inventories/v1.23/objects.inv b/docs/inventories/v1.23/objects.inv index 68538b31026fcd5c72afaded699811ef16ba9dc5..b9648fb76c8f6bd8731b0cc7bec1eb1c9a886092 100644 GIT binary patch delta 20825 zcmV(*K;FN{wg~CA2#`JqIW1){G&3_|fkw4P0h+~s$#LS$kae-OrYv)bn)ZtIQUP{@ zO-yw++5nmq{j`5$|K9$RQ<-%IP?xN3lJa^G)hJZv7*!|~>Xw;Oqnx=9im`W%ywa&Z zi^RQZ`-Xorns#1nv}+`tvTqG*1=^LCId#S#6lL3{;<{p~78px@OWmQZ)2b@6bsnQC-0OmWAFPL$SW$3#SjpmwVyTuE9bCy}g+;}Z zkFq|>JA@Tlr^0#Bh&>$q@i3K#s_-ibKOU~~P#u01-CW6KeFZ-rpz@Y*Msv|ro9i0X zT9*$~k(NTJPV4gFD%Da7V!<*b*2hm}_mW?}|&e-dxKQ+I+@w{5Ch@HYZ`WSswPwZIW!bHDBC^_3M2PbA3fJomcnZt4ZCu{uGH zl}`Ot(Wu0VlxlH*P^-?Q>nGX3D?(5ELVxs z68u_RCe(_pMI4Q=w5HSv3d+?paR6dSCsu$9xE3pDggP+Zf7qggC{kN-#P7?tksDDm zEmjR~MaQ7%7;|pP9|BO73*SMOsvK0v?Iy*EUXa&1EnFQ}#r#N*hId=h!r(+1ET-Y= ze}+t}wvZyTMk|M_J=O#Jp<1+P8KkKpIl0a_f*JBFZ7}u+ZMgd3B%W0Ef626WTUgKj zMPv=vC(NE$xmi{lF}7LO^>S4<TOP?CSmr{p>T6}eP_qX!KhqWBZ$`uJuV zDH(czVb$Vjx2y1g2mCLSE8?fhXgLOHe{2b$ja)($&#OfaV)&RHB*L`+#s8Q7Y~E2m zDlndc-i58}O@%Mnb@ioOz+u#_IKjiO;DTBO19itn9n^=VoLnoi|5O@t)B)#ix8fjA zpI}1S-on+-{9>rkmOw_sJlO+LLAf8NQ& zk1~x^zg=^GbW^w8Q4QVxyx0`YJ%a>%k0DW{VbT^(u+0glby4pz$Q+OKLY8!lkltO;t*0HQKeDaXul$RYIRNaQWH-( z4r>&D*#?as;wfc^8il%47LREye^Vbv)lW#J?A<8)DtiZU_ic{)Za=MjZT?T^$#!XP_bSpP@6{p|j;UCq(5~H4DU$0$RrG~UCA3dC z`On&F^SM1*HpTX);zMUhYjHfFl0*t*i&d`CcRbLC1GfPLu_hsRj!JW-e<2qda40t> zNiN_|GU*aiE_pU;ZKb%Nu_|RB?=x2MeU4`w#G*cD^x*=zp41){t4^9_bWy9ek@PW0 z38M>o$D_5#HK{fSX<<*%^$gJbY+v&K6AKDPFKjX~*86CyIKfLb+tE_#HPKVwo6@N} za(7|>YC_AO$;5as2$mqNe=^Xta)RCr)HF{YB1x=hZ8M-aiD9lte1MuB=@%p2A?V)K ztZgUsG#W>`KK|)-@lGZ_ZAEMU4Z?TZ>z_H!W&Hc)@!L&7(Vsk2;i ziSf(5N_c?xp^wn(dIsQoKNZpAC_QpvGE|7fD#3n7omD&ZwZLUjf36h%U(|P-gDV)9 zczGuHpPBb*@7lzbUY^-UkW{+=k{_aj~;)(RhgHeI*E@tFEen~jYtM#Li{EM%VGM@@7QtkGyTSXp*!2#@#0o3_FTH9yu~gjA z*NDdz(uu`kz^Cy1RBTfVINyp%DBVcYQBtKug^m&$_b>Ha_!4N{hM193?DwEWqnVdY zS6$<}kn}CW^GiR4d(?|~FT@ev1FA#35J7H~!o4yIe$xu1fA7RSs|!meUe1H6=%8Lh z)z6R&sZLNxbPo?#50G$GZOX-?w5P|YtenJnpBjY()co+x&l;xwHJYH45Tzv6nEAk_ zxGXniQ_b6Q-jw@Sh7dovKKf)R^nVf(mngAc4VW`d>ip8z?ztU|Cuf2L*RGa@<)tKAN5316mPoU{FYTC1Ug+*Vn z>cBppcv$VRyJ$BlQcVfTm#Ype_9^GjysBW(UPXJ8AeNhdrQg@fy^Fjp8s2)q&F){|v0G$_rN7V;d)I(;_}5i%gV+6Ep(4hdi1i^I z6n;WM-Zb{C7QMRs*O=Zo=uk?b7rJRv8+2x0pk>EhQ^~*&JW<+(h_vfs&Ob>}_v|BG z%2B^Ee_#=*Osb>S5PIuJwYg9rz4kjP#)|f@#a0@v{ElgFh1_0sv~24u_yRplC2?&G z)je=ExmgnG>8G~-5d;$|O06Ob_o8w_YE6`lb>+6MS6HxUeJP#7hpJR0tj+KzvrINY zE4jXuVsWTKYO^NNF0!P;%4zj0Eq}q~aceWBfA;W?Jfm`Wz2P75HVvmf9;U|dk352s zR#&QMaN^{^TI%Lv&tU!lb;OmZHT8$7Ay`_6Z`(Yveq^CO%@3^|*?bcH%MYqWwHl>` zNi};`ifl0f5AowdKtoA!`cA5bqhi<9vhfn8>R+Q)@&U!vN;-PU<>gxzo1xR{to#DL ze4G(_hT+CG~9-rMM$%W8vY2MZ71-{z|nV{&!qd0uRbt7=ts z_f5$wvzp6&G{T#;&&h0c96JQ-VEIaCfB$H*6J7-+Z#h3!zK`3-ES?k3!Rw95_nXQ` zD$sVdTUJl-oO(D89v65&;YolKu}%gwd3^c~GzX_kD_25MZr6`rfZMg7_HXM1vhY7R z2dLux*dcsv7zYb5vA7hsBdq!Cc~Rp|13H2)L_iX802oq=q>f5xJML?fz6Nalf8^2% zg}D^Zg}6N0JEBJ`*o8=W8SL9+-5wS5dD*tBdMVBi!beaiwQK%=v2;IYso+@W76vHc z@sAYBwU9U)=s|T+6n-UMV$yq_`4^G-uma-9NvHBYw5dGZ`lFXRcT-8Iild)DeY()8 zZ-j;e>!(_9Ce?N2*|&Q9TCoYFf7Hc0Eq7oI@NwZ^q=7h@q}0i$R}a-Z_>gPMJEN!W zAPg*}*dJx?KpA)44I1BdyfUtKT^ZVY(8c0HSRy6F)LA%E`M5Nuk01w}UaAb9&k@P>)kYfg3B^MH%dce1-aokBsZyn#=`-UT?|^cu5>7oA#; z`tuW(VF)(h^6O_B_eIst>suLxdQq;rLZ?t(0G%1#4{BYMg6Q1hi8zs6Bws_;*D59Y zb_5GZcv6p5!ohP37BTH1e}0wcM|*U8KDDp&z=+Rk-^X;nDc03oAL;>D=Ur1&o34GD z;)U>?1r-ANOwlVNaCniA%x{Qx`L$~xAJR#zkR$UO!gM}B{j}9)xhhS{SFfQlq0~CM zL=7h85o+nE{KBih{mWHWu6P|P5wYr;kSm9Wt`7s|qN1F)E+sNw)$_vVVf3*I zpfN+`crP*~76(o>KWSM6V5lH)T-sj>K3;{rWZZSUsu%Z{5z+x4e|d&qWCnUsNP~+< zD9UJfF)T`aU|~y$W42$TFhgVqIbnsER-k3Gor4weu6@eF*r*OtKk?g_c$-#;o3dDx z&DVHahR6`wrmjgyOtFKTH5`*h0$tRhCR+MVJE*Dk7seOf)!O{T}9B|1T;0}rxI#+bL!B?`U ze`qfBNZC>zn`+-*qnGf~0W~)9n0w&Hr4Sk4sEj}3orDsJ1l$>T?9%k0d?gVDz`Y1c zDO3$T^79zNjCiOAK(GJ4O46(SaF<)t0UkBP&nkriq13-N5AsagZ8?vh(|MqBTVSCQ zUaD-AvcQrM^!K_~eYAo&kVb6Y%RIX4-m3_Ym#)(RF#!*k;L`ys91#i8vzw8!QI91) zZZTbF6C))i9ZU2Tp43<$sh20z0VOf=i}!F5?N)qcAv~xDEzIEI_)4?yqY-jeI6A`O z@gtNk9}^%I*a|-vTVRgVfKGYxmwnU$BLT&iq0|8$f4sumF$)VfX=P;iWzjxmDe^IF zuvp3f^1l{sYB|Lbh>${{8PBIgnv}Av#kBV2{uRzyGOsKe+@)HSa;bPFZpT9Vyk(uPWyO z<>Qm5W?8nYqP_7LlRZsg^Bxa!$&Y>ku}dL#e@HcKAZ=CsSwgPndOmyc{OoI1hm#Mz zT0SUo$o28qz|3dkSW6lL`^S9dfkH$>dQSxc3h7CKXU_us(eW$H0Li${pVH ze=>@%-&c5w9$*kLL!-QAIQb~5_6E*t7vyA+;uGD6!t@)ArTc8Z(rv4S#HZEc#V0oQsg#hq(u0k zy%c%lrQ6htv=1oHpHvbXUyh?@&>p{Rc#heQWU`P^XGRHNvngbPeopxW(^lY z9cN7~%uU^PPgW8H3m4<(x=n3GL9a4@k5LST7cPl+Yc5ilIy@$Q;G)!KOzJI2?H`ZA z76goJU`K(szk1yjOFwy;8}}73H{;pk?4=lPFKCb<{42cksaPg3wOyb-32WO2s=oxb z2h^0(h_$DuDTu`KmqDy;rO;nNT!>=zH2f8qzX0z=j+7FpSLkVqpwawgQ17vqK-d8q z1`*QyRIVGBVAuf`DiC-4^lpmwW?QeS`F)TVWMd*9iJPTXA^psgE-;D0=IFXwLGY%o z=*mZzpx6Nv2Uv8qeh1e}m%G>jB7cW`KP`m{;~gYL-cBxdcJjbHltPQg*-3=QL-!5C z9wFKfw0xEdQN?cg3lZGe$D<%3FO!)`jD`RT_)@0-Cer;3yZUpvx!Qr>`ye~a$3#XF zGh49&`vhq}V%nII_UsJKAg}7Ci+ZTmZmGcoBuG%aP;1xIkT!y`NxX!+elc&-O?eGQ zkRIPR{67Z)BXc{;US7P*biy)r^+5s>Mr=q6d2+SZ_1V(P8_MocQz~u@q zkCR+%S_b}fSGQ$!P9`V1(l<1GH}3d=1H7Gm5iWhXF8YU9@#a#wW)NWf$F7=#;`*8e z96vj169iepgdFnc2qag3=%cJ)+{mBSyt&_YH50~gkIGH}<^oV!--TF$#X3Wib!J)e zKi;PZ@N!{OZ{vj+UKR5rcZWG3#xo)>5Fl~k8A-h)9>?Ur`H-r7tRXsLiBPeIMeeaqcDWSBSggE zm*BT;Dn6R;s>i2hA%$20z)@K=t9u4p^DMBZ{4nG|C`Z5?&j3ll0LQl4ijU7xXfZwT zs%Ql|_Onq*;mmXAn47Y!j-IAiTWvlEAF<^|Ncg5A5kYTPLU({dchgkY-C=cR>%iSP zTPKC95TMsvw`z}n=4I0{y+j(LjAo`=*pw6Tw|UB{m8;!r;+&65T ztrmn@^+*)>jS@p-wJx(b?%dHFlD?fs`(mC&57YQ>IVNC;b6tdAQbP-X zLPRJJlKSKfs^_7pgE2?Na@mwi-$7k_MNmo{Yl-N0yMG2JH&6jS&J{;8H}8673Qwsd zmrBI+CBzx#{w6CV(7Ftdlo?S6t7kN;ooP2_FmL!NDP^F+|Whb0c#5 zjtqJErW_b@FdiNo+KaRIxX_sO^3Q6!XB~s+poUQ!kt^iQ(|GP%x5#u@v&mc&YV}V} zZqHf;(Lt@EHX>8Vo2T*Iw^otquvU|~Mr(C&qU~s1ZL0Ndt@@jmzQ!VFYev`cP#5lh zYYx)Fr5PNA%cZpG>fCEFqHuiA~DZ zL$8D}h6o%XcwOvPX+r>%z>O04K_(=CVr(itvuW@7G-}QE_~4Jl)rL+#l}?(A=0cP> zC|J(7*p|}HfNLpSrBD}Snp`3mpfPq}J_5fc?5e32%d}7ONSZeD>{A7Qdlb}>O9phx zO$kQa<$=`uS`I6M)Rj_{f4BT5Zkx~RMPmDQDUcx%VAaxB+5>nKWSOY8G^9#sm(Trwt8-h(6X%9|%gU2&1;__8Yv|g2IPWmp)+OLgb-J`^T!iI&$Qce-qDvq^(nQ!xE0Mtrcn4QWy%;?aUrnRpPL1Ju&KG* z!o6?Tf;QmPy!H2M5gc@mStopL$E^4K;*ME|zq!!P%LdPg^zcVApCOIj&O+=WGr z=0afKt=a>cc~2Vq1Un&#I+(6d+PF5Y#(1o(ZC5yVOSNGdh>Np;y(9{IZFr5MXf2Lp z2=t8!X40@t^B>5|scSDSsRYJ!JT{=ycH_dYMKHo>pacW)w7Ivo92Z_LGzS9f-PKi{ zwo{}T8&TYUaxs@lltC=IBc$k-^P4)AC&=L53;#cb8DbGAp&!b1eO2kBQX_lfuVu09 zDVT8TfQ;3StZwd#F`?k8+hO{F?7btVOF-BZQ#AiaJd*vN!sp`pbHT2f`g3}jY*}1O zv#wLvqg*aIP${p=_GUlAAgKrcCMlgUQi${2bBRrVk$#|6HbtGj(5CpW>&dC22CBJuFG{E-kSr>s;es;P^Widtm7Qc?P| z=0nkPTQ>VuUOmiL;FJZtDoWHKJ!3|XzGG7Q51^@qloJ_pe_?wU7j9&H-Gy=&d_;cxltes# z4B?QRk_^5-nov{z!S1T=hBpKD$F7XK`7q2zqGDV;NThM)V!0H{1&uN3p-*%)xvIjh ziv}{vqN)me(il2MM@y=zSW`&hiQ=oGEf?Xg)=$u(A&D#=Oh)-o6Bu|;ry2VXga$VM zA(jJ2>Ryw7zt?gZ{f`A=FWO9dr&w%%J;yygR(wE_5G;h(W4R0#sDd>nmFvs(^#P^f z(ohEe-=53?-ow}LX8i~(T)=}XgIGo+`Xe33Jso1_&`O9F$M2LqSBi3E7b{(>!_m~A zD_UVjM=yk|*aU)KboQ+dMuX^MvLRRON5yJY--$2ag5k$J`Vx=y9Xpf=d*4xiGK+6< zg?zZ@s8W11Wk}pMwZ#1AZB_5uy-Pb#ARC^YbZ9A+x;;zS7T5e0XzI3QE$=Ynj^4&) z9I3!5{V?&+a)QDf73PxYxTHF!l=WqdmB+-PGBULNRULxL`U*Nox#I=cL5ewkQx#Wn z#|zp$dr#G|OO^2b#BYa6;0`l?V23l#HMLN|#5YYjjXvs{s#I4G?RvyjR|CSMZKN4(iDoPeW1@ zKAu_rRIQa<2Hx!&KB3b!!(uGTw&Mc^EJn5cngy_!@X<>mb?`y@S0RsoTnO~VM?rcb z|H^eQ`hDrIoL2jMvfn1Wxqa4ci}N5j$g~IB4Sbg5E5X4rL6L|_Um6Y#`)K^Xm1D>x4uFASd*Lx4pK=a(PMcdpDTWpVkPu^gJM)$ zVLO~g)7i=SV{;uXR#6Lg$N6h1eaxlxOo{&vQS}Gx%7KHQv>mH|3uj3sm9Q;&jFPxS z1wU9%D}lBhuPYZUh?M6=6JK(2QbtMP-wTe39Yk5MAW~k!9>AihHrMrD#4UOBgkUZY zD}>E!ToepS){91_LX^RsxNkMcb=;H3y6-6rFvbvNz1U=GtR&$*jP2lVa*vYDZ;EO| z;%iGwOB$ms#?}sh_a?U}a{)c(>JFxMRDRh;H&)!D#~>!qR zc)RDtrjYmAc!40s2V^}S$)YjiFfm*bNBFrMROCWME;uEX1{0~LFNVeG3G4FoQVClr zfpTGDV6)zTT`;|CmHLeOg1rr}J)e%=@d`g06S&WikAw{zl|bgdKXz#76~!czAC3_* zCGrFt48|BdT9hQ`c}7t9n?xZtRu>}f4my}C24l{!^DKh+9WCmm@V>P;Jf5Ek<0qo{ zIn__!Y@li#9iB)Gt@^~OA9Ch`Qh*Xt-6p8Q6J^$oaUml^xl;$A3)U6^Z z5orvhh=in6uCRi{9<3o!9t!15v(<8_$C)h_o|DYu*p5f-wz|G9_YUz0Ht0fy(MjmR z#^D}wNyJUM7q{_9g=u_}sE?jtkGUq{U@QVM9zhq!I~G#VN0~RsL8#4@+5oV$g{V6k zw$4s}f|oWDbr)c;s+Koh+m%}q0iBzIZ&dr?Ar9M|-`UY3Zkp^A0+#YHgK#)H7YQvv zL}_@9&D$XWzo+kcY`mAz>-^Gx*XJv^s!{6jW5`$NRI|e)cS53=u_E3Z_)* zL>&YwEmrX%?zl?@DH*yV)(!`;N)eAj40jEGCllJi^D)DocyCQY-I0r|Y4q`o;bh`H zl0GhnK13D_2^%cSNJTH&ok~nAh(3Vk3?Az%OjFzuokWjIVPu8<#~r~l^q4o7^Sgx} z6XpYrrsH44bh=7Xj;io?8dZ%=Axdzzo`3Mn|T9 z4x00=*AKCS>^@J~j13d15g)7Wb<*5Hjfsx^L3vcCl|!$yMqNaW0!ULQ<0C<35Txei zB9&bi;NmIxO&!IQ4ot_#e3xy#H&-spu3&NcAhy83!BhpPmQJ<7?x~8J_-iTYRa;1j ztt9+udt%wv;fA4=NiW-yLX<1~NhFGYe;m$Y5-B1wiJyKmSiJ?LHj%L6I#iQLR=-=h zq7h}!`@cl4TtfA?`&TW%hPd(j>5UYaJyaXTXvf|rYKL+^+U!=Vhl-MGhjGtQaRmJK z`J+H{wGrUEV%*9}N4tgwHv)#^+&H^ITOtRrU)SdlY-(`zBdzXpH=dLNU zH;^Ig>TK7RS$9)rT+hpkcmFWjb$Oin?dY2tp=?1%2BAFoEtrzygoHECz+N>gvQ2rH zX;}kSZRSl0<{HJyg=XCraKGPw6kS%eNF2tJyEdz@Gh54-*Oytl+rqHOz`%FASe9^- zR4qmfkoq|+FkXFFOVhven&SUHWMwx$%GPDE;ka0ADVxCo(bj<8-S7utYQzEB&!Ir? zv!W4?Mm#Gh5r=72^_Gi&^y=Y?_np_3EbY~rpI zv>9(;s0zpOE}&~LJhY>KuzCUQoL*qal*vpkxK(*uu2^?hhp}`vLJ2d>oPkbnDfKY` zJP*@`!3UWKeUt!(JxktKWYD>0=6DV7o^Fa=+v)zPuese+VQ8MgTjI+P@3LzSq5L4U zE(G`>)w@^;RW2`<%HV|HU9zJua%S=ai2E-6fAZdm6?gpIK^xJ3DAYK=7ZtK*m^G7G z1%X9*wG&}ypf{~r@dalm-=BaTo*a;>NGhYcO_Me~Ql%PGTW%ImOfJzvfAs3t4F%}6S< zv4Q|Ul{wRyJtWtE_rma*ZR(C)*SpOkN^30ABeK57w#-KkHSdn#_7M8Z^_JcC44j>C zy=;!M*4q-^I9^)*9s95Rt#C3{F7gW)hw>`m-BPiDq0|o&ig#A9mz~3yqY$k6LE$?b z-3=HyvIsWsaHzuH-6GSny!6mEGU>f$&HLW0SYWQM7Lcodx>(4Q*W#SM4pYwLA#Mvi zn7Kcg*!*v6Tw4Fe^+l4td6&;YXad_zy5~sTCc^gVJ3y z?vkPc1D;|4ndy9)$|vxZk@*ltr3-CuU;ZcN=C3z@`t<+v4l0%f^Yj3gf81c|G*zhi-RUD01wP!b60H^^<4}9 zv+HWhyFW4Q8*=dF6UeX4z>qhK-$hQ|vKp>Q1dxZT5(sX~C$3Vk;IO4f zBxT?tXo(!T%#mx=iK7};j=~ZLR!jN9 z%Rm_1B-7Vc7_KZ$V`%-wD#Vn6>FZXkH)iXB_C{@pqVvijoEBy{t4mRl3yIfiHi+t6 zt7d_EaIKpmh&Gs!diGmOW8}4&9>+a@R%ehj&#l{KE^;d)++3rjFXJ%ZN(`C#X!T}H zS7ymUZD?u7vYKS|W_8bYTRwj&<6f)ud!nuKZC#R;Wf+-*_FQUzEBZ5GF$p&^apT#t z-0%@mIXLgHQsvYVe)61Mhs#IhYU^Ta-xq9TN{$jgiHi&_p6U3TJf$1^AsgL)mZM(i znLo;z>L*A?MNFQzjGx}bzk-|CVwZxb_>;JJW~xID8?BlUPn4(caMudnlV`99Qw+Da zaaw&WZwbV|@p{BdJsnb_thenG(jhN$EpXA4IMVARwbw9YX|ifvEX!8w$8Sk>MOLr! zzmoqC)Q#p71D!bZ)Z+}Lv?uj{zS``*9RE_x`EzVw`|6{0R|BGT6V#KD|C7`&+*JHLVpIhJri)=e9Ms+-n}1k$g5 z{dgGwXq)Q@0MFqC0O?nkAK&@`7WEDWn2y82Lm-h;oNx01+Ie_Oh70k3na;k&-RSkF z_m^3@K|2cm=6&ZPBn=b9YtM!!cSfwFWkP>$ActQ0mJg)854ta!bEMZIWiYP)Rt@w} z8-eq6yANmhkhk~z3$Z2=<}e<@MwN@LC(PQArHP~Fp<`mEz4;;+^5L>>sy`(Ut>Qa> z0~ALWg)Q)CZZ_mJ$n0Z(hS_qX^DQ22G}nGI4S&>>m%+;DB>`FFtMC?ni#Nf6iaxfl znrbuy8=_hneW>VdTk2YVym*y~-!9ZYtH`?rIlMR^{S*J=(ERVBT%z-^uDV4RXYJ?k%-#A?wg6y(XXi~2Me?`3-SdkKbV!F62qxUH6JmgfVZ&5s*ct-jr03hn4G$gy8L|qyENhR_fsQ7kRb)I_V8eW ztXYR$757k&YEAxIJ`DWM4d@G&?SMAntOF#hnAb4&e5bE}-pk3joLGC`Qdegl83DMk@UsXWw3ac=w$jo`Koxjxm$E20ZUPpZlQkw$$@G81KLOU;oSB zJ=~R7u6>gcrpo3;)p~v}uf+a{t4@XGE3GH$=k)=LT)GGPZ(&;N?|uH{m=Rol*NX2SUka7YOaR^2|TKS>3#iw^6(EDqJhw{TgZW|=6E>pz?_&4 z>p8Ibh=*q|IfLW@HF5B%6FWy#YHlGHHQ~{v6WdOP+DAN3CDsDH)bNZ`Vf&4s@(W11 zNQ&&1uL+CkCEm;Qy*^z&)psq9;$O24$S)SVxm-RB1Sn8H`pc070kK^9QeH(sI#}e> z(MCvrFj_FY2Qdi`l{3OK{VuFCGNEH|<A*E;jtT1MBHzM7OxL z_hRm6=6-|{@$DW7Sg@ft*T|W5Hxi6leG?#Zc{G!OeD!i$NQdLBJW0TUl_`Fp-4;$b zmb3~~nJoa7 zj^CerM|{-%IO!yG6BJ#%&aW;4ow*Tz>xb7RA(?o(h{R|70i^m0R?Lg53f6v)i#ePp z<#RbaZ4!eF$;z<}re!jXK%jS(hTppq@W0wg1azGMNxV@Aa@#ouV}2%pnV<4)F-rcQekDiYc^a<&i1Bxm zLSi5)smN40pl@OXqyvToT;&S_;XX=nLZRVouC};1zA840yK2$hkVjcj1%lC!Dm^^V z8fb$f^NkFSyQ;35k=6dbI@IugPv7DL2YHLaYq_egeMvEp_KeS;_uf#lHlM_ODY)12q-uPLxNe?n`Lr;p-@P!ios7} zkp|bPsDo>}@#sVnkKV3+0)gH>zuL^FLE@%(DxTS97yIZ=B(=e8&aM$B)Lub(Xjq+Z zdug9CVkw8YCs>DN; z`*E^01gY5>6Bxqg1IoaEIo~}w_H_hV_OtHAlP4calP42d@|+1Idvme_Ds-$TNOzol zZ_ayH?nq4n zAH&EfHrSsP%S{anQDiE96{fAAEvr8HCc4i-W)e?ndjn?FyB0hdEb6K2cl3 z3dno~k3HJ+`X%MQg-Xs|iR3VT$aPbapj>AgI5UWV3x?ExwO5~@d z3O*7Y?J&#qFa;eAcSqS{@Q9gh=8rlp3@hAOG3CyRj60B%+X;Q&SYhO$GY#}K6+em> zlb2PXwAlxw>B`D4yiFZIM#-z5Q)=6fjCD!aC8lYSU8{DYWchpZ?HLmkr!=6T7Pag< zFxyE{Cryce?$8Eke^xy;)@n3-&+e3cQQuo$64AP;mK9E7+PzuYI^X7=I9~xv=YM&lF*G`G9${#$(X$1;oS-L zsR+0_JV1?HMTW#oC#La|QCIgSB;_R&r47B!r1e&REqDwf?VHCn$W9cz2?>obz)NGZ zyBa!}-9voTTY}RzaVv?9@jAROA)`TX`tOl*Vgezz?Jb#&+YDD!&qr-tdpWSK4QS=W zS5?ArROXkju@qjET{(|;!;&)iB%8xs`WrKnRI?}P$WtYU@u{=kk$^m*rNrfsyrrR0 zPr!M91qbNzeB9s&+@jpB>U%g+&AR(7czPHljNzjqYP6|RGfedY4&8Y~z^v^3r`grY zJpAw3weeo?_pI0~vW4Z@!^dRkN+C>S=ug4HBir&PZgcS=t9FXM&pfX;LCDL)T5VP? z|6JpEb5B1)dHrncZfPt&a>ms6zZ5+89X{iKy{fKDR$sG1`R^Jl7-dl>V6RIs4=vEy zPpL;QGCT&msalBTHP$!CAhi7^60@I0qSSSFQ@9=II&SyfekzYlJR5Vqcmsz8%WOU>%34-(i~Mi5|wV?7v;QOjBGU1;4og*G}o$5kCgL& zc%D$sGj+p30&IZ!ZRyeHAQE5%Y^jkJv0=ftwrpHc7W@A8bXUJ# zu9cJ^WR&G1S;kDXPN=S-7Dm3-w)eTZp$>v9cD#QSUe^Za4P6$FgVTlI{LMaJ2pw** zSXUd)L7uj=MDdROnhftLJ3?E5@VtM2UCc{c6$Iz5u3_V*|Dp0^k&93!CHuu#6xlMX z%BM~G`hvr9YbfTb;)6LE)y-CM-LmUdvGlQ0x20!paEK_pA7z;Q%zZw!Ki=}?8J%Ly zF7CSHgYDQ|NC z*=HS%5$Ou$0b;D*omsDmTIZeh&1XbQYzm^wc-v=}O|-HO!UhbpeKyxA#?wLUn9+66 zlzODiSm68c@UPs_fS)}bO>un|=kbLm6>keJCtODn^7j_Ll;+eMqc^c$0! zV`^@s3Ek`vkwKr@9)_C>9eEIcM$VTGMN(?QVB%F-?AF^u@q`}))I^13SNCjPw4Wai zI=|wdc+i+a0UW*X56N;kTsV{n=*@-R6gtYE@WYbPTKA!X+sqIDq_pQ`u0HXxRU6z$ zVdbKIKu%I?4qAg$(1z$hV4))jBevE=UoN7`9E518%TXN}y#D-v&~@a0Ip6RxVN-o! z=r@~Lt565P3za#l%%#d839wpvh<*|D5PC%o_dtE3hNbn0eKX(vVU3Jc}GoQ$njc1IvDEc z!Hnk7F72b4v<=eJ5S?Lvv-!}kwquUnQ_u59rrZoeCW~3LHTQNQ^k&~S42Dn^DsrJB zG&N`AsksYnJUO=^`Q6;;AkErG5u9A8!`ZcsY0JAO-8KYML8cGI($T83-{lV*G1fiV z6MY*%$&Kg!k+;P$>9_R`SJGUo0L(SRb2ML1^;8h}8R4ev4 zL~FC$3;3!yjrC<8xd}XrnB$4fzwW|C;Y(~J(z!4gpzLl2z>#{4* z?O6axJL)9@Xg=!RunquEEmuE zo}lfe8Am(`?d$nj)Pb~Llh509%#pOvBE-hpMY`PUZP7b_t_}t-)x5LR*nvZOdTf`H zbj}dSuc75 znWYI~un)ySEQKIHlQ&8Z-`#l%AtB);fgi)U19KF<#*;;8{ZB|1Ssc1H5TpT|>b7Bp zCZBTkgT`)u{MwfoAJ#z=K2?1DPZd5cIXM~!N4imzrAEU+uSv8UrSs+OW?f#D?*PdS zu2UsGBgJ6}Y4rzg@dRWEhgcy%xiEanJgmgJQmIQQ;+?4nufAm9pi~@DQ*de3{ zGpIogBdw9|NZ3&#Mmej^Y|4n= z*HLDvUIv1{LIeKLH`fu7!UN@qM==aH;R+=YRH;Huh?o-Al88gfSZ+AiNtm_zFj!=< z)*rsX6d5Fouo%HqpasA7UZ4x1UIk5%LUO4>g6t4fKn+0F#k_96B!WnxJ_szY7W3?X z*USGJK-|fek%pn9Mj*1GNf4Y?GVfxSd#){^tefN6x2>6(Wu1V2YoBSPUu-vp7JSBd z@;t-guyTGOa3g4rE$ss!#l|)srZN_-KV%k^myTmqelH?I&xishJHTmm-d_=MU`sW6 zsU*XxE=FT2*Y&{X=9q=$3>M~|%uI5B1Ei>nd=9$qCB?+WKxYZm|LQ1D`vj#>{~OE^ zivZ)n)IFK%F#M~Zr-hWVuIXlN1+QJSkIvkRVaw4>Md!wxi^O&Hq+C}6uKF|zjqHJK zcrVI(w&9j->P7hkJP`)|1la6PCF!-NP!Y@cPud#^$lQkPb8yhrmSl%~gHXwTZ*aWd z3^(Vl(;{Ph*hOGww;{k{u;1XIVeMQDHzRtO%E12Ma%)A}K3{xyaqThs>E-GzleI&5 znBA4Zj!6%%w__53>+P91(6k=cc8U`bM;H8wv=0THYlYgpRv4LTw_h+DSD8Z0)}Dx^ zd$B@ge7c^kcipZKO9J1*uETeKU<_Jru4_2rOAyM}t3%V}AO~(R=uDS}2lmJx`e6-DqPA-P6OYj(8EEeEYUGTpA z@Giphq3aeRluDJ-`4UuSVP*&E^?(2O|7m-F&o{f{EmQ{s zQ|lppx@NDN{GqdUyIpLB#=thXgMi0wyjO?D{$q|UWA))ocTsT1NXz9kt4Yt7SINW-<-XD5fFW3+Dih}}_ zW&9x`<9%BN52;uk2A*4gie8$ZJ%6=s2D~#l6dP5*9}k3oaWVm9uvpn!pmYP`8N<3Q z9|Sm8%K8NG03lEI5b+KU6YzzOS1a&&V+QB2BiUByLctBathxh%L4zvoHJp1q6W;&F zXGUA785X|vacPVU)0BkVa&2ZVskzQD@sT^ZWz6_Qs(q5#GIK2?xh$8WTq0sfp^)5i zDI~cix9>i`@8|d5`*AL>=llIW=lu0P=h$Xe2<>m*5%jF5QuZh9UaIQteLV{`Iw8Iv zo#4O6QSr##KCY_%lX93b?Da{`iZb8a1Zx-ndY0&?Hs~x+arM-0v%TQx;+4FGml3tM zw9hn>)_$Yv<)*;j#{)bLO^jHB3`%td1Yf$3i`SmqzMvXKXbvMEh`pLIb{Fs7hQ5@K91MS_%oJw)`RK_q!9`t1Iui-fJ6IutLsN+p46fIW z8-(e8>yJvx#uXckcec6u;VHPdFYDaD@4o5Z{MGRpxlV>PJp9pJapRHo2?^M^J7e^T ztH(+m>Ev$g>D~CyjT{jYm7)kx%$Pm9qLvnVaai&a{nv(I5XmxPUD;a#@lxOK2&6>h+9-osExfxV zL`ba*-gRFjx&uxYCSXhD9YnnEU*BG6EX%LbKb(kdAD5=_$dBS>-@z+la_k(06>OqU zD+J&E@Ur(xKKAj48}=BBFV9)dDviHAvciTd;%pOHcAjK754F8#xh@$<#-q%(ykYy#BxXZg8edA5UiuR1Mk`@+fV;;pRI zCM`);Ph%*TUi(X}7~?;LSzJo%c``A+@O2Z^yK)KV`xLKSoLlsiH=_To5o+WRj~vXt>~bCEgJu{jDmEi7=7?J|Zc9pX6Haa>gT zdR$Jngz&nUeYJCqq(*wBU3@Oy(PN2^C8$ur+X8$G;|eI+5gHW8>scb1gj2y z=TZ*xb)>P9jbh6(5nFFKm!U{i+o^kL!Tt(}#zE_y^t3SYB-?KMv{D?-Ot%i-d$Hl} z^8EwT6vv%g3Y`J7ZhB&VH=F)ETS$(xj{Wfegf-;jwqi$iX)?y9gJl;&hWDnA+`ab$ zA~n-yqAT;X$Vxit69cE7?jSu{^0nabBzEW4*Q{Zzi-CyW&AN*@cB=+lB%#Iq;u~Cv zyo5*_$1Axb(>lgFndOXj@K>{M#hjpY@l06#&VR!U@=5H=(nv_T%4b}JxyYoW=2@(* ziToUYN12c@HtYID>Yp7W%x$(lao999bBJw;iiAjV-C*6!#pAO^#ga@_DYAbH*00<$ zwYNkXBjRQ5XJ-9_mKID>U_)K{!&Y>J6}T*?wSRQd!udMS2#KPX8Y!?Gyp}TjHA2kG zx}E7P{a!m=M#_uqNtdO=^z#+jm%*uGD(7&6GU{~9cIIod-LC^;t0p_Q5<3GP=DY;+ z!*s~8%ZT@cbA0n-?CwgOXltV~p=us)dQ0RnBHj$=t(BA_j59hW0J*&geyu7sJ*3O|`DR&lVnAte_nljQS;L}(%cdi@JAaFgFK11N0m}vEvTh#&CQ|xJRr=KL z&23ytT`OqJYi+C~x|e9x@BEBLlmB<##wJkwsKmad-(TTjb&_7>-DTqyl;N~fV{AL4 zhmQ18AyQ7*eYOL>v&N8%U))Vo6`G-?g_82wcn^qDoVA&54Ia02i3$ocPuArUd)L&o z3W;n7tzPx?uy;rb-s)VjiGH0tXJJ`xrswwNA7=}bS4ilFFt0rC$Lnd>=X1<0DYFK# zOdIi>)XbFFElxCCHrPUE=(m}2&(F3Bl`eZ92h_?J)1b_5Q%Rjwk|w>gD3$YD$}II! zK$=S6e->~`ZIX3iq~*NMA{K|Q#p1VrWf4fQ?C6X+3XW6n35~tO7hCpsYQM(>SovZus0S7!E(vBJONX6ct z7ilU`vE#dWy+2lfs(v92cLgkwl;^AIuq^IMp^?&tO?kXTnFdgkZhP}d&nwn{+x}c$ zr7TM3M8c35;tK%~ZLSkbWV{fBY$6h3 zO-|8>3?UF?T{|Jh8gq-tkTQjGgE}T!^(E9<-k+OM4Z^TR1VGOU9Yth35QId@0yyt% z4VeFi8DOdd9-Clm`zs16!Y2@G+>009tTj$!dr41%m85~58`7w~;Fh1LeGOi-_D zI7_xesRTF_F<=FVFgeBd(*^{Ikfj*q*xyoO`Y0KjEj?DKo9m-RG#FvycQ_XNm-`nknDX3#{aV91fC)@$>{+ zU^CKjnm_*{KQ6(NV(l;y^5St5x`7DfK;Fj``Hwc}?luMRM;%INuyLBY`u!zAZu%dV zl^K9VM2jqyK!zeF`~Xq+iE=HdS76u=6hObw8H?_Aq|#zwMh5xAcyxCZl@=}AtgqyJ z5#7zEf@0tRgX1Fvbayk~{sUlubaqE~TPlJ_bTWby({_mk>mV*q*|I(Z@Ey^@p%PLg z|3OfqtZOQX3_l@A1l%a6KR|=&6JXq`4)dqut0YZs(pm-F8qIKf6z-GJUKjXX1P6Xs;K4Vq zsWOE(prk%jYodzN%hcB}9@7enM#l3XJ#o}A-29LjuF~c5m5}xbVqKBxhVm5k!g`xb zN+RnA7~vpPv9UbnDZxx78=Wm(9yAQG&?Ub1p%{L3t-7>7u+)_+v;U|RG0iVWa?D&@{aq4ZGF}Y_-3`o zmlwX*bT%LRoAEDAGqQ!w>m?CM4mVyo9TMcY9ibf6>?#`zeY1MZB3Y>}d%Mx!jFR5v zN`93P`QDYZ6Caj)u{_;bHH=7qKJBMmz4NN)=}5Q_SJf$@u3YCTtM}St#b#%pUNKHa zLs!<ptRxs>eYto%iik|8%Y$&itNSO`&rUab#E6xY(6f%=Jw`vM8}hbF|u*V zk1Bixcwzf!(A-X>M!5C&g>BJg?;TvqwY`&1~m${!0{xsZEnbS$WQ`c&LycMos>U zt|}Ba$m2!yzMHbysQm8&_alQT6!Yd@wKM$W4~beonxI~hj^mC5TywjSbQH4Ibx7{z z;<3+Stxhho_q&irpWWzP2+BKX3`>aDbb2u2qDHj`Vu*h`)wnUmaOQ3b*+=TKoI;}G zqkq1xyP=|7c0LgVvmQOQvtW@_wc0iG7r&A|-(AyjVFs9LApRlaSLLg9iWT?ogKH*lAl!7?aM8E+kI(Yf3JDp zTS$&x@?BHV@~B2jvulq&^s4*mV?nEpUK(4$#cJM;zRMFY)#n<7lkBoqA1;atUa)&2 z^&O~d&^atv#;Ks;}y7yA#slK3cmwimBsZOFcV_* z;;5sBE@1wqqK8HvY+Sl!PXC>(P{Vb)-x0q~1I7H663! z@mBaFdK0FBJnad>-1;dZNeYqIWZqM;L8qo1__b~G?9IVF=>EsdDANuo#VssU?pMH! z*o~;prCO1wpu^KjnetNeWgP?X@;pUi+<$4ltDwb|E^#+HX2 zv7iWX{EOlhdkecG#)C`Q*GX=59!O@=M6vDi_DE#5px!~3mZr5J6@K}%;`*`nCt0gT TM122W@uPq3?F-np_vQZ=JtzV4 delta 20863 zcmV)7K*zu7wg|_z2#`JqI4xmiH)UZtfkw4P0h+~s*>U2`kSWR5nzGF$YT7H-O9j{s zHZj%RXai_c^wa*0{d@aMPG!~=KwYxBNy_U%RHIOtV^pC~s9R=AjdJEbD8}A3a-may z5{Y}&_AUQrH0`|DXxB(OW#1as3bYF-19D)cYDL-=lwrG9VSiF7k6n33O}Q*T|Iezz zTyQ>rtRjy-fReqcv)V$tuD1KuTD8T6x9y=-RBb_k^B{A6w+$9pjWZxdt=MH_*Tt&X z%*&(2>e>XFSzX79u0~U1O~+hXw-(S!tOPZVG8%KKG3Q!Ld>PIyy+p59*QAPpgS1WA z)+;`@#C63|Eijh+mbyb*r&U#C>pVtPxYq@LKUfbhv7+Ggu#&|U#ZoOTI=GU{3X6&* zA7y=zcL*!APKEQL5qmiJ<8dkvRpD0>emq{~p*s93y1A0edI3Kkq4Jh+Msv|ro0}Tc zT9=Pgk(NTJPV4gVD%Da7V!<*b*2j-!_ljS>|&e+qBCQ+I+@w{5CB@HYZ`WSsw9wZIW!^RVse^|ciYPbA3fJomcnZtDfDu{uGH zl}`Ot(Wu0VlxlH*P^-vqm{$e9_xYqP%T=t4ARt)oLpxd!3_D8HW>SZHeCI15>G1ozhv6GEv#q% zEV72{6J}4W+$^h&7~3rCdbui_avH5NTM~yCD9OL(Q*xbvid?F|(SrsLQT!2eeRwmC zlngzbypf3^hBMlKVR{%TXB%5 zPcWfuZ{g}Eez8|QRU24eg;Rf}>919yO53;AM#{ypogw1hb*NOCTd=C&CZAplfA3`C z2bo5y->!K$x~<#psD^HTUTliyfk6Vk$B-z}Flh@X*ye=Ox~TUUWR6FAp-#tOd{RID}M4RB6;U+ONo-T3wU9)WlPc z!y3h3wn3wZcuLu!Mxid1#ba8_f7HiO^%GJldpF9y%HBcTeVd~`d8^x^hb6U;>d0+! zBjj_Ux~aMcqf@3-{lm;AHKi>o?Z*!7y^1u+d$mY~V=C4tv}<=%isbrG6@95w3GEY3 z{=Bz+80 z!svqD@n|h_O{&d7TG&%`Jp(j9+gJSm#Dap+3!6-g^*-7vPViFAcC=J_P4v|FrgZ9# z++Enen$YrRGBMr@f+a|+e+)FOoS-)YHOMWOB zV*LD|5+31w=p*#Ho&osYPet@NN{?Ka3>6}=O0eHiXVngUEpS6 z`WccT)d>oT?&0z30TQmOO}TiI_Vg5$m6I6nQ=^c8njgRUS;N%7MiX=rqLjoMGauL# zm*u8xs(D+^n{xlk5aI{dN1qIZ{*OZ93MKZd0dvMlonQLWJ-34qr6A^P)uj~MuR_{I zE1tmK-}gste-uMHlpDF1e-Ho2Gb)GI8~y=r({SqJacT_z$Rj9e zb)||1Cr%EmrEV_v4CaqeM_h?oQ-7Qqf~9r%w#^gkM;7Yi{LtEw%_q^n{GeJ?t5I5* zRI_KL$QBdu5I-&iG?Wyl@1$xtDt28h8!ut1{v~Q9A5cuKq@!0{UcP0q89J@b%Fo~{ zf3oWS;`QdBnOt{oq>e|ezg=6Jb?d))CHlA~*W|z2(yX~BS10s_k35usp8hFD642ov zFpwCObkA|b-F&4jhxzYBLv${o?W6hcy^YSftTu>tu<-ExZN5q|CRc}^=f$?Tu2xm| z(3HF~tGV1qBfMGroXl3ou|u#9mKQqve+QGD@G2;I%lWbLecV1}@tk-LUT;jk-&8(Q zfwrsNvU-N+)WdP`xWxMjPXe5XbuysI)6;jLIXGQexe|(Uw|@Ep+^zkze_JPzh5x}h zKo#%D4&iIVI9Py*#g(`nVa;dHiyC(t&=Gtg0+NUWz>rcTbyPyzabKJCHDK!}f0tG$ z%%ylP#O2Z65j|SLE=9_#VBaR|_NbW8%eGzBOL2Y>K7u-_UGx8orTaNc1;;wKFhB{9 zf22^Zg~ZuF52}lz@GJ2Oliu^pKa0$V6%a>GI+gdKP37sxdUr}j|=}I4aCVLrA|IxJXZ7IL#{3FjGnuL zFtC(je~`HYW!!bQXnfc4%DCQjWoYj~7mEvFiIfmiXW>ZYS!Puju+cMTcJxvh>ZmevVWw00W73vEjU1cQOj~qFi-_PNBR2Iy1T-)Ve4I(YeJFaU#1&zJ{!?RZ8^j z2o{j=q@JpTgXb13V%kIee`)B~>0yQZi%UHd%6 z3*kEpDg^YIq6;H%c#)6HZ-{pJrE4G`(n+k4Bl8==bUs4;wAE(0Dox54uc0xa)H=FC z4JPFYYU!x_%&Wis%T-pccpX}2M{iIOvFe(TD~HFf4+G|+qMWxSe==Xy^TOw0^r;G< zF+=5eFES+-2TnCVYgq(fs334$+FuGjUWL76+;zOJ7Z2-Qhx-V@LMWl$fENC3u#?h)pD5QFO}$w!%iR+MVJE*Dk7 zseOU>!~yQ0_)g_YdsieaJ{+YiGdyz19N-R$mpWH;xxrVm4(ESpF7!y*QXZRX-(RAa z@zMb`HtCpq;Krp88Q-XkKjEE(5{U%d8F=c_^q_nt5e2}#2udkb4L$Mm7{ZKrs0To= z|GrAntNnPFT+;y_HO$W{g#w|}zc!EZOxtZakDt?dq;gwep%PxHY?QLVk`VOwx>tR& zf;f;yY~IT}y6fJn2v3)<(*ZF7kC)-o0V^C93DL8gk+M;bB|dF2U1t*`B_AQjjOKNnkI zj?{oodG?on)Bz)Z#7{p{@wmmrxZ*u%z-mLW6(Z;*r6rOTB4SQ#ql`#g;O&@&g`2c8 zGW@(~pR*MC7&cfeWdQkKi#D~K;+oQ-KSJpSV4gCG*FM^v`$|IS&L6QTgU;MnmRM(Q zS3Sl9U%(xHa;(diV$m~_LyAbIf)u8ZKZ&UgFa(D>B{OZx zW%(Jei_N0lUAJA;?Yff08fBAc`_P80K0rP!!>i5WN|)}|0T~en-njt_mEh<{%=Y2U zG+vfxml)RpD}U1tYy%0&joyx+|7`@-jE>D`0NMv&Y#>G2C9zAVc^Iyz{YGCNQ;K zpgs$0+Xt$@1hxm%l+uW`r{^h%#PXLxtZk*xUqM`mV)Z=y6_~#O??sN35~x?`d5WOX z{AEz@v6n&E0U8DY()?6z8kb?%0TwD0cl`8ji}rR~ud4Y&kQZcQA|HvHrB)&R#FH*D ziNfaSrdmPprmpD9Czqkv0Tc&RbhUm5*DII2*a0Gc$9z96g$m;xBt_m%E_QbE$UKxn zi^thXgvVp|4Z|KG+7GmRmI_hD?)VE4+}X#YAR;f5nMsU>01Eg@rvE0={S3SMQ@Od` zf#3TeJIu#KMiMhyu>$)DX+L1vn34AE49+00>!yo(sMc<&!2={nP`prU*VB+Tg0V@w zg1de(Z_-V914fV@-#7d}2LU5zCr$0VV@fJ}{RH+5uw(ecgN-mvY(xFbe0Q zrr-$7AX?>fm#o?WKmwoWm+smDPXRZVKHC9X0q2*R+W{R6w+#G#-c{xOak027m&4lu zC4W^_B1o=5D0L}2rRS|q2`X#!7dhxjuT(z!eO}@mP?Em7dQT^;D%tAUuZvaTTdoO= z+(_*~Rj+}|6 z`w9VGF3jm|yb#0dVt(ZAD+k1QM&tzoBrZH7se{CmnEW>%O_fhII!BCLVFbzGi40aP z{vu;dNhs%pg0#ge!}#~TinTH5slopaP*cB&Tm(w4QG%2@{u_?-Jj(zK;qdejaew3` z_-&ht59GV*>8V*rAyxozR2I$Zfx)gk3v4Gp4ml9Y5irN&KN2v&v8^x+_-DOBlKQG> z1v>1rQAy#k4xl$YSv_99AW97Yrd-% zgj(e<;T?RFdqT(Jd%nKb%~2(gDdS;SO(Mh?4DiF=Ib`ob5zWxdJr^X~TSb!p|5N`{NT7TbE9etVDURq8Wjeo-OumH>@ zV2GV8!Bgz#8QlQ_7{xl|Yl#v>+s6PSXe;G53ECbo@8ec6PJJD^Yvz(zU8QF|j(|;_RCXuap^Q0{k|CPW`;!$|j9;5N6;stn7Ufui|TOS7_ zHa_T!5-G*|J0~YZ8uKx;V!aF=+_|)Pnc4M76!?u2Lu9osvo7x3%p8)wn@9U%o<$GS z_-{ETV2E>5gkMra3xGmIC=ZhQbKhD;ro&oI<{GWl!C|(e zb+xJ1yS3_XTKXD`oUIvM$3tDXr8!6omu7GVE|=1#t8=f#h{73e);W@+wpwrF6BcoT zMC4^+kzAiC3Cx8BZ==ULR+3BLF;DyS-chkAwq11xi=~_Su781(*bhm^8(a!-VChH? z6~aEimn-4iW)&^Tx@yIoyN12kRbnv)(xME5#DoG!$y||%v0vfnuq&B5+0wG~gGgR2 zVyN%ANiUVskZTQ&Gfb}1-Sc67vA$juX~ofE$X{R43E%U1g1{v(R6Aa6wcL!%5C~KU z@{02yIf=a5^M6v5BBi)4HjDde(cLn?lX71P2C_I@=_iqyYdNt48)dz_xhd0*Vha)E zh`;F61XhI6(;NYG&>_7&^TmcwlIv|6At{8MgYdS z(8*~~0=p+SP4E!(4uJ*?>DynDTO;?8!b6z>pQep6X@8rWG$vd?I0|oW=u7}5!sHpv zGaL*f1K2*(eHI1!lzj^W&KvYX5SI$ZrK!{D*P3sgMzD@Cf*6*?uDks+iIkR-AvVPY znHogb6rcol5MeW!oC^PG!jiJ}&>>-rAp%DT-W0o4+7JLGaH9l%kO>K(7@LaEY}yAt zjasukK7aUQakZh-PoD;7MK)v_vLjuH-9dq&8qnCG;V+xVJ2-?x|#*O+4wk) z#ifPQAeGEp+O~AH@plt1fi|Z~<*o&3c3Ez|+BPf3wAzCEpJnRI+=$`koxUP-thTwr zjWGQ8eYeLKLQC!O4slu9=iyVeTCtGpHy=A&5D41WWSStP9l=GIu8PfaS0sJ}l#t#K zReuR&iW)XX4Z~n18?_^R%I%5Yfll=>tKD6=Bqt-F|}?TTr-f z>e2@+T!=hWY5&+2?QI%I6hU_Qi_S}6#Tb1V!(bQRkX^8g`RBAz6whx=6h6}jD1Z~8 z(iLed+nq3b>I(XLc2`7~vMJ}BdNinqo_a()=bnaMNseUPnRZ*wJNh!HKBd+dw|@fJ z$P@}+rA*l&D=q}~`g60O4>mP-Te$b_TF?fZnz#ObErNs2G3$h{?U?nRU)(Y4@K@J9 zd-(Z5@8~A;?u*N1Nh>9ZyRfLyTnOyDReL})?@42yU?(I|2h$Zw8`oCV7>bp(?F#2^ zsWwalad8%~mqcN&ZLU!i4aJcRfq%X+!Au&qY5oIwId$!&C6&NfjmHLb+GJd~v2y?4ZP34aKiVv6Sf zh)1&jQ}|Ted@9&=Q-4Y?lP!yDY1VZLdz8y12P)-t+1~C)7$o)J-z23oMhbDhdoHml z(vP&trl`{w+7$nFy;<(r$6KO!devN}P)0FwAHRNR|5)uw*N7rrbD2aJ#K?5Fb=xiZ z7k59#eQDjQ`^hQfl_Z(}-hb`-fYmZZHl-D(teI(wiOXi1S9s-2Q$o{}y`H-zYK&Zx zA2y27pEMd|t^e4QaZg$-|Jf-?yrm?Wy7ofxg35MKyJ?Qc;VnS1L+>)_f>hZp&uBiX4cexj{S(oPh=xbI)EZeXT~y*QmO;SNwN^ec^rZKUhStx ze4hIm~8dolsOR-$g z7?U2lLr0UVD(t#wAfqg*s<0=Gp-*(Qq^gQFg%qABzAoBw5r6J#{RABvlE~u0WRwpz zfq~<6nz8>tXkhanVma`k?lt-MJ1m!h|5zaGqRq5-ipAD*+|y%)1{4XwLU^5(%Q%55 zPGeHJKHuCNP#P``W#Iqq$sFK4eElBQkHEqOJh(ClV??4q(sA6=A$AU}glKX6{@8P+ zC`Wd&(zQAgO@IBVq7`Oz^g_srO(6J1XW!~LG>ASX8}h<_RIFC@z4!ty7=Fy7FY!p< zu|tWl_Z=m(_!d{leS3~7#Ya;{!);Sb%zxfh^{(B!v;zgQ;n_)tmQtzPvxIGN!(V}> zZd=y!4m0lPZCu8Y3Y^jp6AvvXD9lk|E{Tpys>4ZHUw_6}c}xf@BSYI?)zPP{ub_jJ zJ5+!jpP1t}RdF46yr4a^_f#FbR0-ct{C21W?g#^RB;#CD3l&U!)0ETbqpqn+b@kYO z=ZD0_r%*xX0IjY5O3j&2|L47{Be8}bwAD>&8)^xRsfkRCukEXyNKN4S}!&lI8lE(Y_ zUtvsU+@mLq$aYQjr9P}2I`(1JV|PALYjI(me_f0AmSAz~8^n(_$%)_~l~fWvmN)Xb z;(tdeRzlA=C`Ppvw!>*Oot>ONEZ5Ou6}518oWGXR$6Q*^l=$xuRe!9m960z%+hMwJ zmQ+#++mgp9iAz-QgY~o$Xxs6+a>0T~d0sT}B_}6klobBG;F#D!lm!bSK&i%q7+N)q0~ z*beR{_bA!?wx~8FzP7Zqq%qoJZ0&Gwa*Hw-&||LdU}{I@=WTRj#VvXaqT+P;HrN#0 zVbyX(cJQ=As_EbxumM6D)8S!4-31S&al3br#)1JX9l_Xi&<@yB@eJ^KLD`O$b$`9& zT~<>@t=M~V`08oVAU``kTP|dBfO9cZ5!#M+yLgy3ll>U7V+Hd6!(>#*wHyZuXm z_-pgIT)@O~8w}MokUKVF$9Gfl6*Ea;j)PEaIm+e8fI_(1zyK8XNXmFv#}V(5w5C70 zlU52Pu5URD`uhVeZ{BD@si#Bc!|QRi|{kqZ^MppsMwOr##V z5G4VTWR6k^J1Bv2VPas9-d!-gSw(y_{rvD;`YGw;_5N1)iI^~bhI1rH;OJ?{ z{8lN9hThJdNhUuNBVtPA2`w0mD0s9eNv`sYpztnFjox5oPS}TS%mC6 zTGUJ7Yin_M^ga``PsHrYx$;~p50Y3m#rC%1V~79~tImw%@jz0z3V%J?iX9zCHQb4|j*ScG3Zf-a7CETj;R zGH;NBP@5~Y0bptQP1Dqs7-W z*(n4p7;aWiMASYKh9;-2UddRz)4 zE37^434Wl*yt$m;34ipMFdt|%9seSx(^ZmkRE59OsA_BqQG#nN$H<1{Uu(?F?PSYA zYQCkp8g@`qqRKWfIx=<8oNv8ehaF`1dCF#Nm`IKISaq+H=KN_)bnN}fqdKh|dQ~;* z5NZ@InmQRD2`cj+H7^&b?4|&RPQh>LD1vlgIzHyRZ0o(ba(`KN4NK7nu>}SWrYan@ zbfFDaPF2*PUrR|J+CoZf9N|yf@5;6gHw>*z`qY*bqFmumB2oO~a1E145s^v!^qaxz zEg-dtgcaAJnnbet-O3e>D0|-jC358ws=wX8Y5_LHjo(jiq`>U4+9<|0_BK&FaQo3_ zw^}_`Ok6u?dw-6KBj9&0p9Gq#jR4;l>*h(IQE?Q!U1MDPry(GfM@ihRpNPQS+6m6T z4RTQgO(_a&)Nu+z?6@x`x1_CT&L~zcH0!p2`~9})vZ_VmFqYi4S$&h)T0XzN z%G%u)hD8PjzT3sJgma^6F=Bw!&tZY_>cd)^{+-tp|MyE)cJrfbT^1XTi^Z0*85{>~ z4e0$Xe}53BMjW91918RyD;n`=#Iu4DahO(BZ@KtKuO6;>-+5EX(q8YX)#CWN+7yl2 zE3!cQcNJjC;jon~%Pvzld+O?j^K09cN1wWFN`Os5(1&*R1LnB8dT(oOx1}^`8fnB^ zPHkQBi9h)|Th+Wuc`JsXVf-Z#dQ~e&lY!$!xqod+VZ1R!@n$rQ8lFKfTV`czv)f{` zSn(&}z%5lE9WNl{h`gXhoACyQs&Fjt0=fY+Lp$cG7tqe>1%^zS%;bVwm3QTeb@z1` zOJ^gLFvH9l==7FSZ~Wuwmo^MO$UNwy1TgGb^1dR2&NVZKX?XW^TkP6S_fLJz?WPJt z^M4H95?}rDF1z6n%IiVvLVyoay^EDl>w3{D8%B|G{eXC^;@xbM>cC-0qDanIi! zv)2bC;aCY+T30U390jY|cGOF7&Y11QBsxh_Y zW&y?I5-s#c7r*}U%bSajnGy+-l)~&}GJm5ovqeI5_-zt2OcADVP^RosDcGU`DnEVv zczHa{L_QKH%Pdz|h@>(bD+usYnKPZ)LvsBf%$(V#?$}Me+bp8A#v(l;>j!MheB@B` z?g(xVp+Dbj*ZO^)|ukc&oZ*pkMGAHE&S7Q0uH% z7S#q+JUGcW2iG4u8kMkXlG*i(P|HOn)BPSd_~K zPLt`6{{R!0TG35BDBU&VE-5N7;3)=>na-D~d;(vYln-H4y3qFa)qi4c{(7Sq4=_i# zlzuT~)f27ODpqEo{xG13e0%D~7T#r6Pw z<5)ygHbP2zzp*VFt=&$LKx}?1hg*5oigjzxn1RW! zJ`O6GT0{vHRV|}v6tE-%VQ`8}Ut3|gvNVmM^%tuUQwpZ9TYs_Mn5_re8?_;d&MSv- zTA1OiE=5HyBwnl8AgXh%ng!~?wQhzW+B`<;*>5e4k=JH=9QRnALDD?8ZkM^ptxRrn zjh4QQ!+a|-WagvQn=xIPB?q;kr5(#glGU5l1KVx+{H2WhsM7C=w!yb`NmiB_WDeSM zsr{|!&xCy>+<(Z#jTg&u!$(Br;Jm*|l~YUj$#ZrcE+3Vvt&6RFU$BuWIZFH}E;6`y zp`&f`ly2;GY;;?WdZB0jAZMx{AsrPldEPR9dK3Q&ZeojF3ZmkV;^Kv=4moVJYC=3w zp1#9fD|k;{z#>dB+}_4%^|8Dq5dX&O5ij+0NQtuEwtr7ZhrGzOz-3e7NUxLBUc->3 z$*Og+EL*J~za`Z*S-r}CA^#t!8_g#MI&tW!#~Dg#PwIWO*?m6#xtR0k*ueJH2kWi| zYWXney8c|Y5b~#6RW2}{=UC+FgX+7v)bIoT=RlYj^V`zt^?8VcIejpAQ@L@zfD1X6 za6#5h8-IbSo7Rg2(igvexC#KY%}oS==kNl6^u^VOw|;;{y@LU!<8bg0NaPgf+kAj_ z9^R7SLVTvPuW>he{qg-(R&LOag1>p+xClwZ1o7Il;mMs5D`}a~pBu=bSH9%~Y43yX zi{>2ZwMZF^>%Ue5J=8|veADj389wCg1OGy-iGPGSjEAsM$aG;_`?dzr*&A^7JRz@Ex`qq}ZmLD!JGV$Am`ezk+w;+cX2c&=Ge;k_s zU4N8IbQRWBcj(Y;KAP(ep`0Q_z^nRBK8KQJ{DscNg#&n|@)jn9CHp8-ztn-W$i zV7TQplpHc~%lH^U?p){A=8^AV&qXKeCHSx3f4KU_$rD1zEpR|0)~`Zih2wE{KRt*q zCvu8quIvY?1r`m)10f%|s(WDlC6ib!I)8zfi`lfMy9}AOH_fc1$9)Et9 zCVc*WYGep9qyW|)9!!um>#(ch0qRk$$$!m?wIhbBFspocjN5qt6$!IG%A9t^~=z z5S+6Sv-=^9-6ArOfrvmJV37S6{}5)ci>_EU#X7rGraNE%>-DAU!N4(QQrCdzo#%5O zG~Skaeh1_Ii~se%{N2NSdF|Ra8DXkyUR15;_wriokGSeoSiaJFqJCZ9C#yn~!*S29q;L9#9hppE|K~M5X2ya#0f=T{^MtWT<_>^HgFj z&`S-^I2E?v2r9pTq>H4;Zh!fju$W%qy-eTh)8$ir*WxJtHS2)RRpAiMLr!{gao5SB#^!iL!sB7_O^gCJnXvc?#pr`J3~5nNQt0xC2nTu zW=iIeePe{+DEoq=%o_pMnlvq2b{~K`I5c@qcgt4;`_tPe$%a zT)?hG3sxCwSmk+2_EnUPeI;8dPpTjc6l@BrTTUcxA{j%`8O;twSmBJUab{G`h-_^; zzICzT-yK*_CnLJWt-Tj>KQs3ul!$NlNWg*(y}3rtthf7y?6u^2q9eUNrO7_4V0FsgTP;tHo8h?nTA-XpA>A>Ej0ogC& zPVbwWM5W!AMcXIv_50cJ`;%{okGdZforG>epo`b};xf>g8?k;cT@sRso{LC)#veec zuVKZ!xUOLB_qdqDc~U-?!_y`)$dIfY+hAHI(+C85S84dYD*^wjokTzf8(?0I)x1rE zHga;LudfN94u2~;vXE>`dcDCRX6BrWhQC}Cq_K$Xh~-rjsJC^jc5o9=UGjngSTFb0 zg2{An9BL`Pt>JtZu8<_qv82>F_b!tOF+^TpDNvAIjbkc^BP_R_V=(4t0+{(J-xj0f z|LIq96rQKy`i~fYHz_0rqLPYCl>_=FMnF1XNWfLTAb$|HaIfh$WXYe>be=>D?__W%sj{lAJicF`RbL{XfwX*SFXjG_NTjeZ`+WP>j6;JQ+u@F2a=r8~6Up>1 z<;#qga(^m~LO~q-O|J?qCqKjLGJvTZFUhY?O+y??$?qfne*FUp4i2@Xf`N>Hf@3fw zn038bCKn2YL(EB?enY6d>SNfil^e4ZFaGb?nF`> z%;xMGaYF4Cl!u1Z`LF$A#6UN44m`blVe{;kYzvXUOaj7p)`3iktNTW zK(aR{JD@_xdV+Mv+4ttWcjb=MBrr~_W> ztbg5$PaB{S#+E^_?wTzb*oeq+J6>nM?N%E;E0ZfeA4=TjxNK@XxAOIGAMm(JK<&Tp z#n)|<`dJ(b3ReGd_3JNiiX{73e=0ZG*Z=YR$8YcxS@J33obZi=z+|aZEEzc*sCmu& zP_*SjEumeA?mmpofABGkjADcRS+U&IuzwImrs7v&+6vmT>XUDx`y6B@@szeVU`D-b z!IQzFzTZf{#eDwTcZ!}so~qYhBB};jG3)Cc#1OC9ZZp=GT)3dNK#_uSk5G`q9vNNr zzC}!baJ$W?4Hr)>ixl(olo_HNMy?`A0&?yvuOcArimqRIWc>L4Dy$XvFFV?fr+-t( z+VSiH+nqY%L9yf$wI!^8%vbQ(qdl)*Qtn%*Xc+yqLVK0;SD9Ab(9)R(|Gf>HsoIUiF+(+kRxMOTsQOO^fVWwG$=F z-PKr8dN_3AlK>M@msj*h0;d^$k?2G!|@{)+wO|`6W64UO@ z($*o@8f6N&p6IGg5$^|vDtQKstmg32e25;iDb~~e7Q;*=Gi<6wi7`hHp?{~97;cb+ zZv2uNQBF?AjhBqNx;G&yFPSK9=xrvgw`##-5NY2$ zu0eL9;7v$qgaKX}o89%$!R#L5quvsnwuxIwY>e08eF+&2g42JGoD&lWxovOBY}{tJ zqIy1P>)NYdAu8zl))$29PZNJn31HKJxNENDmjc# zo%N0csVSsOpiZbOXOA=k;P_ zqnQSW@v5e|R&{!$oX7Koa-OLh4iaDk%x_DNJ_nHiBVbF_>VGG1Rcu(08+6si{{hMN5+NvNpcXb0BH~kNl zCyQK!GAY^5#-hlUSyeu5($^OpmRmzHR}~-3$*69&ikp_*tcsXpBf#C=U>0{qD?qP1HK?tZ%*`T4GZWUB=r!yKJJBbr3dS znC-KuMhlhXVjt2bf>1c|Z8-G|d;P2gFy~v+SANq&HOeR`d z8NPE&_lMpt+VrJgnbaIpb0baYW`~Fj`qcI?++66$gD`TwbSRQi69yBn%3`!SVic+mMZ|HOmF6bj(zg?~tv!{NfAL_lvY^rp~J{)8WwjMlmj72IZi z{3oS7Cx3JGiI1(?;6@577wsc*l3H`n8l-|YLd4^r zr$>aYBhUGkj|rRVGef`G%vyyy0A8rfQDrVw21$U`(qr_Cpoh>aYPd)06E!TYN9>#V z?hkA9gyAe(%mcurmLQZpj@}2@wY6?S+*9y|XP)84DG>>*^AI+p~ke-I<44ch|ezhHQ?4Ej_KQiTJ7&2MRqOG~N z3!yjrwqY=YvQUu=6``p)8&Az$XyeJb4ax83Mh9uuK8oPvLLJVoZA@FSH&UM0$)kWmi}Dn&n^AZFXx5H#*;+07uKrsBe59) zzW_&QaQaqS@~_LTK(}WBB<-l{zFe&so{$~MOH@>@UB_$XSD3CH$p0+UUzmdnLw~o6 z`aHr{f74r0=%BM)yy$y^wwGoc@g%gb=NC~2(tb@oZ__bH(n5<68*3Noa<8{V@3=Y` zyj1hfQey`W>FKduN{)kuh&+GnDn4luzelV1o-B3@6Cq=kn8qUg_ySh+023P_j7d=7 zSa$thx2l__UT3}N31pTgguy-(2Y;~?g8WS0C^>w0=P87Qgp&k*4CfBaQTQ597NPY& zAz5T`=-NP#25_p|h83E8%GD1VyYXv>QvLzUN87W3YY=>JkXm5`-9@K(b@D1Ol;ybPnUS^dhpjqOAm# zsc0-B?8@-A=c_TuAiGYL_Fhb5%dAGpO6kR=>qg#hKk@F|a<_DqPg5)Pjd2}%*h z9LKL1#i$^LGKu1NqMyWLnm7WM1i@rH6FYK=Gbl+&iHTa$k51x?Iug?5^jH$mIdW6t zkNnXO?J)|}0b_@dBFvx$HINqrSc}{I6L(kfo(R|c;vqnSFz$dMu76k{j_3^`&7Iz~ z4@dk62Jwgh45EERqIr12PsA`(go=2)4o8TIka87)A{=YN6F?%Q*p4wViX!0+4iVJY ziy&b~i5TUqHnS-seqTqKrFt0%{u&MVL*HCSL<$d-BOb*t+=MHXL{Oy)H6db3SW6-f zDPy_eTqj}H>ce1>#eZ6V_zF{GkSxMt1XF<){MvhgE`)j&G(ifATFex4Rm%DSeTwH3T}(LOnID~2scGZmd1b1o9s z)w6P44Y=ylD1S7v2e#q8DDT;ZTehheVhb0- zEf07RzD#O(Oxn5B>18in3W?$6xCp^Y_Jl;;i!A861UDTEAiC-*I8p*sMexx!D>}It zwl2YAcz>~2fKzqB`|6i>5uT4-w-BLJs+7)`pfU?HLm&~8#xkB@_j?huRQ@J@mNu>b z`@jEB+k3v*J#V2p7?@fQ>C-iP-Q*9QwcG7tD>MeS!5suVb}Ns;#MrU~2A`6J=OFW= z&@j3~14~Kdu#pIf&psm$Y2bbE$4}|GKr^xjj(;adAo7NU)(dmt0I{Q`8zVQ^0<+_A z>*acX=xx1VKh!G@3Q(5uhlq^#Z52GEVs#jJZYg?Ye)jy;x*71!0e?IY{>8}z zkilYQZ-LSch-VDzwtN&VTzNQ@-5Z`6V<*g5lNz!_qlWn^A}yEYW<<8glJGKy zkRm2#?Ab{wBa}UB&Di%XOZFw%vLt?Uec$iD^SsYOl`GtbvoQCd)pmB5Dm2QWrYCYEaszU&f9Sw?Ve%%;>CP?)RYrs z>UY@ZiwHH7INAaY99}c7Q`@~|Vm*6&8cA)dQcvg(1EgZk_WCyTS`>eT*p9g#gypeR zjPz&>1+Tb?x_AY&Ebdr&skict*N@blYWVnyEzrcXkTs~b?yMW`H=kWpBW{vLuN9hj z!BY=RS>FHNQK|Ho?%OaL#`)Yb%gDAY4l5c-3$Kw#TAR?5@1Y%+zx|z6*i!Lr9l!Zi zEKV`b6uQZ9jM9#ge|zP==UTdzb1#1WZl-X#iJn6g7jlObe*}yoi{+TBrodtfUx_b2 z*)GalB6K~oXj=f?H%fYd`(zwh5VFlpy9LM$nN6s^udwz&$r-Zw@!JX+SPsAFu7{0! z=jcn?6s^3ge?0BopR>VoCEegjT|)Gy-mJ8}_7$PCQi(a98*#OU%N>$_`qRmFeCDsr zq$TU>!@Nd;G|ddd>*{ z<=mx8hbXvQb2Rfq$VF(TMNKXu&4kZDo@g44Q2r>~EFv&NCnzO)?p$S_PK)vK5Km(q za9#2jPowOKYP}qxpTENC@~qxlIR}5O>3iw$D#uZ6Mu_gzDPZmqg8{}h55Rh3Bgodg zrr7i5gxTUuD>k&rdGiDrvnV*{46zscYD!jMx*wyY?~%8$tM6Ry)!F;7vq)({u=CSE zM}q5`l)zkG#Aa`e6#P`1o{R?xzHuc|oJzU!M>1Fb2+yc_EBbUB^uY~wgF-eqr$ct6 z-$hwoQ@Q4T0-|;R>va>jLAGKmue8=l$XsO6)d9{~e=!3JSlDAr1SkJ3a0 zobl82LU|qbOe13hBq>o-H-)Is?!MIjMJeLL^OOq*0B>jWBsqlAQ9eLrsB~vQ-Dtq%wXG}TaV`Kq8cG<&wXVVaCFqvi+c6$p0imzg;`Pe?O}jcuZXbC={1t^rT2(HHFpT$7&3Yuu2x;N;}g6Ob_AV$r#a z@}%V7x1+bSgKkCacM|puo;S`VcZDm!0Vg)a)Fl>YarX@mBj^5DXZ&Z8RpwW-L=J@y zh?1rGScqaCK~J7{C9Q8ZjesfrONnq?zpW>GPR5J`=bR%RV9!mP%6%(}@OG_-Ns`fW zbPBUizFo8k^N+NchaL`TiE2O?c2i1VCXac;1USZcUdFasC$;BJE>rA`w_jO|@b^$G znFHe2L!J3R{&$?@P4D$04G&|ok>;0MV1`oL7Fnv-ALhspR^59ER|g1B6ND^BlTYhr3G9dz#%xvOz+HNaVDa z8r5JfQ9}mY&AwlyM(tcmv@ZY;%kG2Jshy*&4u1;P?!GrQZYkH=qbmm+eo2RV$!t6A z)0N{GV-~LhcGew=FD4VJ$_eUvWJD(?`Nc8s~KNA>l#S$Oi1!;Gdm1d#? zWdaECOxF0WrJGQC*=fXd9oJM{ZmIx^20=uPRf4^yGorprc${oNrh;JelA%|{PjaGY z*ifi4cc_8OJQKZ46yeYbA)~W3!Bix>_J4rUV5%GE{{eHD@K&P#0X{-Q@gryAy}FpH z7R;I$V_qm={Ko=%q2F|)A|v7Zpy{`<(aQ5)P6j4)ZZghU1CzlLUm(lEMlVxEh%+do zKkGoKZhYERGT?D|@k3J0PZ(U8G%rdTm*8kD3j1U<^EU6J|_Igsd#%C@URX+ zff2{6-Ova6Me@jKKvqLxKars-IVX`Qnh6j_41{T1drd*1XntYD$0r+9r9>~GXh#UP zu}TfE#*Al?dT2fSq#F=+e^^7hBv}qcyBP*m#71?&}^bVX|fxC_#NF22N=BdlF0 z1MY{X1T#^&ME_ORG`od~swbxXZ@*@9L#eT6w6hk#-9FjoP%4)&AuAZpB2+lcBW=wn zEjNs)!aijHa}y)uX8(gQTVcght^x5z&ftELj?)CINM1>LB50ny2Sc#R=acMaw6tsw zo?z7qlkBE~7Oe;Z%v7$^gofZmR-yNYttLouG7h4NNrxKA{4>Ue7roJ~p@mBi(!G>y zUy%;wRpbBBKsr-X7)#dpme^sNHWe=6LO6`6v*IRyFkWyF%0g| zj0AZFGLE20xW?I#AvKIg(awcKgHD6g3hq2W(e%QJe-uOiC^EJVR*|BTQ|pk1Nn?3s zGA@Yk`dX@8l~If4i5PPhDqZ3K@)R^8nYdxA|A18T?s%YR_ri%PoKu~P6{qNmGl=O7 zsF&g5ARtCpTwxe2UWV}1Yh>J_CPa-BlxZxlLB<_uVy^LIWx(ArWE>X;5-b7=*7*o? zXe~Jl9YwEtX~bkmc>HXP4Zg`%>il{-6z+Ue+fS4(Y)#~Sk}r7T`iFIWbc21XqXa{J z3L}~u8zaSx&gT(t$k+&HGIEpT(~}?r3XzB@{lfrw_Dd+*f}7#H%kZr}h%XT%D;Tng z|K{R973IjCa=)S>wh;pMnQ2Sw<`Nm6AbL1YL94^1cy4bq`Q1|f=`jPVGBZqTlw{ODDu&#v_b>;MaXI;}so-^8_EdZoMHzw|Cf#Lgsl$LZZ~&k41B@lSP6Z0KfgkIHI=3r?zsN5Ihb-)t0bgoYKUO9GMXuLY z)d$;~pO?5)Rxyr5NDC9kQ!wRa-ok>b`FlSYHv-p_bTihKY z^4kwVelMiTzY!hm^rZQUIGzw$x2E(o@(j$W-nPB+$!zGXbdG?x>yGrFcZcp1<3b(T zHnYk469Z|UW3O5y-guU+=LJmM^B<^b{#71=~+|rw4Q3e)K%ZM>yufn#7JV(jM*9V_kwVC9_5f#M?!dxWiaMv2^>HFNUDKWJV zRThJw^1QCrwbK=I8z<}hap7_GHlK`ZMVRMr)Y@)cSngPmzh29FhIRcJsUu*+%dKT*k-7$yK5&UY8)iE)-IQG0RINGe zs;gzX4{fj3HKEKJp2S?FhVBpLe=hoV<2SWA(0Bhr)X=X2o8In{637aZWIT3&MB7C3 zwcLAkE?xuUQXL8Wy^r#<9XUX?@bCwJ`W;YYgZLmb69}tc^$LddrDLa$nTeSb9o6rD zJ~Zf)WiAJ-vP#In%*4Q9Uw}r}hBX)d*{6Mf6E5ulZ0FX(x$Whd%(nVp>Q;@clEr^$ zAlOq~6zVpb=Xz?ge%qMpI$2k3e2GKk&#+3BCvCs9F}`E2MpmV%!G!eShNqg)${L4b z))m}-Fn;dqqYP0seE1?wI4C*Xy}9i;;%;~D=WYO#wbPXPCeBi*&~{>3*108nf)l~o zc$+2#ECoahj;s`?Yjsbvu0GpGmGe-~-ENA83us5xjs23d+tM@+!};8IRwVBDJ_Z?h zkZiu%{c_#H7h}+s!Hb(Od`36EvH^kenWdXnv*CNSMm?>&IXV0bNi#uNd(Y{KR*%a< zG61zZ*QBe&SmX0yea3GhUSuvm`QA)(t1DKbluRV~?7b)-`Rm@ZRQWzaZg!nsIuan< zsF!od6=8X*ol{y03)jn@1|Nmo@hBsxH>>+eeeZr*VKtv^%<+H*&*m2m)xB^WqCG~S3KzBN*R+n0 JYq$y8{{tAuUj+aF diff --git a/docs/inventories/v1.24/objects.inv b/docs/inventories/v1.24/objects.inv index 241dac7fd3b39c76e26221ca305dd57a66794eac..1aac3cc4bd58599bd73f63a83a75cb2e018020b6 100644 GIT binary patch delta 24108 zcmV)2K+M18(Fmr}2#`JqH!Wf_GGa1jfkw4P0j|Y=-Ig4;l{Mk}ehNl-$AlfTS!GFe zXs?*~ceJbBcJI};Yjw4kckCMkOJb?GS`uZFs)MiQHRko^Nd}oXk_5m6rK&Ff7u_X- z$oxJeK@g`zx=7N?_z^DEK_f9y><4TX->N*DrNZrp$3C%*hR-T^p2W5+S9RVryn9~1 zU^AtEJ+5QyhNop*axGU<%Yh8Jt$4TmVch)ta~iS5bCdxuQNxH?cAD|VIop0I6AruL z(hNDmMK{KrY5{*;R8?7R%T+Od{A-ywb(P>iW>?GkKcntT)P0~wfj@7u-Ku%X=C)Xl zGAaIi2l8=dE9al{X8nil=ps_aeX?8Xgp*5u3t+6>q!!!V#ikim!jCd##7Ep=tuZa7 z!%BO&0=bMc)FL{J65aT@jWN#8_IXS)>{{zF%m}Aik9qcTA$uMxk)6mq#v1QM=CRqR zqYIwLR)Yr^cx)7ZhCy9yyXS_9iHvcFqG@&JNHj5Nv){RBTy6JMG%;N>d^DQ3fFn+S zM>D#bD$|PQRi5!|aiKx`2IDHVwLtyers7Jpo!T6ui+F+JF#$tqCKEdWU%1UA$9=Ws@7#0UCfVC!QrZ# zq4c6FT#x1+raM@rt9RG+yeh6=c&+Py(TpzWmC|sydS{V((G{=Hb06y+tm^UK@(h)6 zURuv&y&lczf_|zpdCbcQ7px)Mz3lh4&vOs!9q!GwX}#!I*FkhI10L+<-E9{ye)!eq z7+u8QQ2cO}PSJVM6|O^aw9)peXnMe!&2RTGl#b}874K~-JxWDpQH8kxk$nw+va6eN zUHq9ZvK!pB`1mrd)2A6{DlPv;${*Zr`KGDpUFL~G-?^CZ=StoxcjCVTt?~_Bz_smV~lajNY7)E;bkO`VMZ(?dCardGSc%{ ziM)*DG1mBHB#+HTEh9aTtp=BWkvulqXBkOPNfaZGl6jWq*|3W$)KxL&d>E&3_T2CJ zTvm$M@#!?zK3hJ0c6)ey!Pa_s9Al%0#{*dgevIAn`K(q-ZK9(r20upa^{h7qlYU$1 z(Yh(*RkXp3B8+No=^bmYO_53!=xl1J(aUqSDWpSLv{e?5n`~Xone+*NG`kB#=8?0b zKE=vrZqz+2C>1K#Mzgu1RgAH=5MO{16R6GHRAsmr)s$%^7$0-K=ljVO80%eHr7m3lm# zvs=Eym`xmT@u~b-rajl=85MN>ullS8W3?aP!NrGhKg3fUb27(&U+(D*9qh5sQ;R-W0=YBTj>iSQ!bJ&tXk+mQ@^pC=R@)RwT16)#@8EnP%M+42e%jk6{HH*#tTjHRVVf91eT@NalBsmGQ+ zi_{b47x+8AdbCPxbnaW~4k1bzep`qIzt@YmFTq;rKPn@;#`PyvZbwmEo#v!|AXBZ%`JbK*dM#Rl1GRA zA=5C4UhtkaacEcVQ?;`CVkQ6l4ZJw=~TQ6>72^dn%@A8uA!P>9V+$(> zKzcrZAt7-VlL+AgH_cY99O?=K&m_lBGn;314ysfFT3tZVPCY-Q)h_|^&~L-?j&X| zcfZH8tBP&b+-~4~TNQueTFw72o>Hh_2YXn*RmCL-`NakyKeMtbmc=Gpv0}@S8WH12 z`o=Or36MpCxQ=MEoZ63D+Tr1};bNYQ0!Rx!AXNa;T~|S(D%FR;!8qF$8{V6LgQ^Q7hl29iHusyN)6@UHij^G^3& zQSNF>u}a#{*d4`k>|rPEZDgg-GO&|-tN|`%(jI25vl~8+smi)$H6Jssv4w$uL`)M& zV=dd$jUHp&#J#Nx8dKDm3XSM`=PW|JDXw_p1*BbJ62Zow^GAl8tzjxIXV_dxA3V|S ztIgQFPf!tUt92e5o#NXRV>+%SMJ+JJn5U>ky4Cra5lZ%sUtyD~wt!!OJi%1B1F6_7 z`NwU&E;p>$YFqsslp zXi=lm_mF8%b1wMn!F#fxw%ERKLpr!dmG!NWe}8#k<88~T`Rka^upMiEz)P;-AST4G zRbB3?dH(V%po46j0a3vfDmY9P6pPH8Y@NSw+jWUXl~SyL|H6e2Vza+SZQG-s9khm_ zuD?M0)+IW^Vuxzg1eNA-oBt(jW{8fk*(WZ12%9xkwyELi{YAQU9==g!{-kv5-KDGR zmE0^`_&ifhi=p!f@WzJ#?^}5)OwOX;oiMQ>(CMvcd zzS>ZRYvWVs&kz60NR>imrN7eND$)TGtlPN;HD%M;VQ& z)R=NDCKwUt7GI)&x2tQCVPMyFmDlBpk1cUtF;^3eIlrau(B^5CiEN$6s0!D*;0J5r zMH&jWnk!kHQOwoUqJt~BtT3sV^Iq0Rd517V>r^<;DzS!xKMrF&RE1wj_;EPnp*s93 zy1A0edI3KUV7w)4=v`FB=B5O-*5zRgX(8!kE2OXR)_Zj) zXgb)jB|bhdOkhpE-mbDHm%|2ii5X}W6BLRs=&`L3JqoOA@r9$B43vuRCyW_Vr9_S) zTr6ot}6W6KwH(%Gpe}% z$eIym{9{+l<#SA*JA3MU;}f; z8(ZXm5G!yi_BB6jE4hL!(}v98R&)%Cjxpz!+=ByEsW>6dQk8-VxlXJ&(F^icr-jXN zoz0K*Fm|^UH4L`V!Ynv`EM;1;g%p`}&^TP}u^y3k)#B%}lcvVfWSemWGgepHV9*ZQ z@Vn_G9#!_=WZL^JEJXb*vWD#m(;h6}EQ?KlzSp!`OVJ@_f>2>96;h|rhKfU*IBRQx zAyru}SG+efjk5v_mQa%SucqV54Pzk<9-{ao=KA=t{5W~9fAQ7hh8~-bha(hqt(!%Hvnq0OnHwCslH^oD~0JnsH zt=Nyqui%1ef{walqYmt0DJNS+epHmk6m`Hh@2%LD*d`cJws&y#6TcW`r(y%EMX-lC zHy<&TsM5w-t*2aULhK@rwxLpGZowR>n|!<#-pj;~GL2;4rg}WOE$il}gl{2li?ZsG zK?1(VkSNkHX$$*t=LFva)O!pv$34A&VADPr*`{1^3L+`!NQDkez~;$IIHJ(M^27^F z`w5aRm5wM&!mrlpzK{xuDver4+lFkZ)s}3fM()|}S`>fTzWg5IF=Ypff-RM^e_YMf zo9p!xQYoV=Wm{#-tzc|*)P4lIE_zr}3#l9kqAMXE9+p+nJQ|%crD(fmnADhmx~Q}t zJG8e9X_80PNQGkzYZThn?Uf?h9?H;{I+ajA!Huq)RmNiN`zGU*Cau6Q=8wPH?Rvy3w4;5DoG zzBAVx#0HOB(_5wF?09{Yts3cnWY!G&H}|BEK}xug(HkDEMb5C-DM$}UIi z|BWmtxbU&b#2j3MUh5OQR6{^56>o`_`qnsC-H~ghZ z*F+wMEVuqFfi8oCnrivT-9C6wgwkEk9VJyt zRMbSFaeS%g;$)QC?N&Zeig6EGRGN8SHN_3i3&~Tacz$_A?+Cky_d+-%>|qY^LIk-{ z3U^@?{8N{g*8v|I*Tp7ZJV||e3S*@t#$zlB38?w-)z9jtzKoa9*8w0kqmZ~liE$P% z+r89Qt}oqkI~Y+4Vm6#zO0jVUsTZ|)0;9k02dxxcI+P+@EoxCE_W8KKjF%Cq?9^cFRZG_UYa&Q=Ya_Cegi_tZ_a zn-s}XLh{Y3f%8WwoASJ>pwnPFNnhF|9X6B#K*iJdw(zI;w>* zPz|b0g#sC{#z}FgXumDC(gp8#OnWEfM%idtm)GzGI*gIHHiqgBY$i8Lf}KKstjq5~ zFv3u37FisYEC;04K-u_;Z_9Fp1sk-N(kV`amWqV$EdFGc13G9W+e;}Hn<}I>Yas0+ zODe1!R$plOOP7h+0d;?GxV(JJhDPW#o8=F1@@>()aJxBZM%(TOspFvb+ts<5w|?P9 z^l^>0#r39aFQyAsgSKSfUh8vH#v5}lHc>_^XL;id8 z5gJ@pyB9iGc=-NvzKS^-Lv4DVXWQ($SQX7t5EMKx4H+i_c)Jb=O0PcE%cm`d?fh|8_LJ$kf)U5b=9!M07>_9&ar z^SWM@OR;|tK7u-_Uh}`%(*2yJf@7UqIG}|4KT;^wLSk>A2h~MU9JcWWlRogwKa0%! z6%cz)I+eGfRpDvYpS;wmn@WT!j(+;|=~AcuAT;c0pJIQ(nG`pLXWi=Qtzr|1vBi5W zx2FYozwm`r5GRuun|!)BRQ2Fpt}Snjp1XlCu#{qdl(_?C+%>o8eAjSeT<`FJFYnre zE*2NU5-B03_QH|M$E7j72if8DMrH7>Bn*`T8#j64#1|-0+JYY_9S&XPF&HCCy3L0g zsHZ8Sz#Swi+hrbXg?xqjf{%{;*vO9z1DD5?2)aDzDTW$iO4Ekw@VJ1N9^3&JO0%$c z0d_dO#Vq1QrlDC{Dz>z2e40FY?iCsqvHk9-Zi+65G;feYIS%aA+PJQA{7S5uUrD|wQ(K9 zzULW!mKo?pA$2aEpqE_T0U<(j7RE+(kou9|zQWtILfq!rBCo#0+tNjLkoyU<#Jw>> zZT%(cSr6JzF%$@;{sIy23gSTOf3f)>^XR5~R1u!I-SgrN7iXv? zT$S0v95e=3ctL47j3|_9RnbZXQsyYMKA@A=2Y!Xeh!3_P-trI{ivt&^(>8cZG}H{N zbJ)gSo9$ULfwFf~W`**|pIR-YY*NN-(dt6?DT>WmKZ}IHy_*ANqi#!l+G4u)CJvOC zv@OwBe|S=5eWD`ws;#e}!}S*8r(yUsahR*TptLMHGoIc`+)${6!~v~cI2Wmp{Ne*# z3^psivJf8BK?~D)IKI-1eKbPOiqS`yJbr@mrF{aV0$VZph9;N?ETB`Kz33J)t;{+l zP}m)Bi`EzbO8yKNOXR^qHN?(}l)*A7h(7v9fAQ1LR6K4mFfMot8t~dsY=sD(l2Q}N z3K20Two!T{F7Wn{hlQK8ax#2a)X!Opv=19BmU4jnuSFeOPH|0X&<{|$4w$D*;lat ze-BES9#03eOYBM=WzRDW_G3fEp>|@QsAR6*Yp9T%&a(O6qULd% zYrpz=S$;0u{2`w{7eM3-l7{@+@n7m~VO?O_03irFkEc-Gl-1h5{@o%6GOR5`XX*fK z5xV5}Keg^hm*A-6Ey$NCOKPA8N_F(Bf6BQ>`S{?eTITgCt8YEdWY1IBw8ev5@}r+Y z>`I93V}=c+t%^T$$W`6UXWzX(`;z8x@}ZmM7bOn4KEAsEagTXa%xyf{dw0DG9uA77 zml?nSo89_)AZ7`cS&E3`x^0jyg|~Rud!T1GyXDsfB>s4M3TCgPe+m)8 z?Q*o-$>dfXxc46NMio#lus(kY`@n%B$`#)DGK#O?7kG*uV0Rfs=CRLFNPme3=|c{c za>S#A?esD22L@}4DvpiNMTgD!wF2P#yndoe^MKCphWngy%KqcOSdT(aUW2gKdK})Ko|TMevkYd zzyvW&A%7H88=wmgHYGEy@@4*j*V$%K@2=~nXm(9bVvVvWO)wQK5+G_>Yk0r*cz?b3e1yHU}s4CP+x;AyMjUR+#Vfol9}YPwzIX ze{Z+ts+d0pc|kTN@{zb%Y8BE?Jn0gXC~S^yiWLNJYO*GOB4()#(Pw$|4sXLQj-13; zoFPzbpNg*5@8Eic_ug*;3_{{()M{Eh$+vw^0V4Z?1Q6?tPFQ=3^owiJ7fffqjCs?=h{_ zPJjdn$|=`&;+Bfz4Zhtx+Xhx)}J4CJz`9vK!_ z9!$$t<#OqBxfcfSoRX{0v$da~0!bA}Dr_oprwSyc(m68u8T|ngayB7oe;=6#d#U0l316!Y!Hf8gpYunCGk!rDnDSjsCZSB0Ni_(aBwd?aUE5YLXv_u$=P z5oC^!P)9iqSH-fjo;wpgJ0dmQByPb3I7zEu-;Zj+P2Sv>)#qLeJpPr=f0ou}i~cPV ze+h&AMk;k3{O2hGKv9V5;6)V1u%iLCP3KcHYz1rwmusmK%) z*&-x9(#>0Agy=wHHqUt0xNk5>;O7E<&`3e6C6I>rlvS}T;B=b;Vh}f?O@AR@3oM2r zm=@dA5F9ENPV3z4R!q$4GW|}GYJvJ>&{--xIftBDcx0+FxOY_Lf9MK}jRNho5v5R6 z0^Bi99aJ3NJQNjwEc`>ue?hLGjd67bXH`lO2O+B6R;Z~$O)b;_Te+=sDf{YjvuK;= z!=mo*s8gFY2;3Bg>ZD5y3M;G#j{UB@xK(+aRr6a0$Ccg{`TcRWxXbuJd~vL#^$%tS z^Mf4mc?(2{N0i_uf7ayPZZnsmV+T2-Fp~y)DpMjzszE5Wl#SB!np1*Gi~b@zJ@H26 zgZAeo&H*LqtE&%m!pg{2%YL1$GT$sypyx)a56Zj-E>&>pFv-QHW^fGf>MkD~O3LxM zJklDz8~41!0fUGKaS}1tMgI^h-drlx3<8XCBdLE!(m|R~f4~sWe~?L)W3>Ax=Auvn zZ2s64b5K~`u)sOqj#>tSEMk5I@}~$S7hDEeL4TUp;(7JBZAumxsUC`*089m7Iy^Fk z0|*$NK)|en8J_N$0$wVPNU-rj46n2Kk^4i^A;vQzFAyMc;TcI^58PeLfAg+p{!~L3 z#7GrJkPPm!e__Sqg>=Lvp_CH}(iX1_;|*vrRL9^m24^Y(HT9dwMWCb_B}l2^zhR5k zv-D084tInRyQPBPwkr52rzxJEnuQc%1pr5RR;?Zx?6hEkEdqxj2SOsT8WE{G z4Bvi4e@Bb*eiPSo8awZ+u~UhiTG&yXgR2tztvU8dQdS^f`h1%~eun*N@*KK7?;g_| zg}HM9m`cD9JGD6T1okqz0|YQKdFcEsN?>#!1B{BUl-nd|>-xNnTMcn)>(EUzmqc^r zp23a`?05WrLM0=z+9 z&HNc_9|t2=K4^;~DGj%GPELr_=VM02dW|5stZVTyqwAh1@arXp$ZB0?(ZIRV+a-NB ze;;g%c^2JGY zYeZfqCdu`No4{CD@HTp^JvXTY?(=MIYs?newkhslvUD@wRj|AMF>3FJO96H)9qFM$ zwDvkG;nZd|Sdw*7i!pZz3)QR0VsxZM89Ip(1(1@dA~VE(g}uWj*E>b!3U6E~d|E_T z-*b~*Dy1RS8XOzET&26`-TZ8Qf4#`!ilfDlzrLaqzUSivflFYhR=nzBx#_u<5U6(K z73WTJ6nVAgr6>)Q;yT+b?u$ip%lrq=`%2J}#qLTwiOgKfi51vE*1MaVJZ^iv5J8Uk zi%v~o4KR9|{gDnjq_=0j*ziGexs4+vg^-i{O{cZkq=-1g4CcM-qRFHEe>qS}xpUwJ zoty?GuzPZM6AwXe7pTLKzWgP*RdO9E+?DC@sp>%{b#)WRgbRqn->Vxs6F?bY@{BGw z4u%6A*f!HKlLBqZ-VI)^8}vdDrwaO|DdXvvnr|6LupD9paY?gXbNgo$DJ><#riu$P zHHerhKnd(1YHBn&71Gs&f0||M?je&y3=w#M;7zt$#dQHt0vAf)2bqun8e&uMkxl)` zhf!-b;)CB8R|`7rR61!cnhH^3r(ikXVqJ>c8?U8smO@>SX>y5FfDW+(w;K30Vb@i$ zSjK&ld(yO#XPYYc>7b5OGN4nga`4H|_oUv|vR@ITu9%|ytL8Uxf7yIqE+XrJ0jg4q3xz-K;fZIhqaBVhw!oyYyU(xVxp5)wbH;ziaRbBwX40{zDOvD@A|J=T zxOD&Oq>_0{$DgJ&XBZ8aK>M1-a@T@1r!_ZU?YK6?wAjLtPRrPlxe>#~JAFmwSZq^; zJHYT?_sxhegq9lde-3e3+UMbOv0Aavaeh8_wjdC+ugNq)NFVMuz;u;umb)zSBcO!z zf~ZO$Q`E3S^dLGe3&-~JvTVzWT@@-Ra*hQBtw?`Ew0ChW6VH$IcJ#IfAO={JP* zM1%-#(1u@RHWF%x(1yWj?VCj$Cnbfn)CHXyoalg{L+q4We{5aF4Kr*(;WP?-LYWpsO@lsr4(H6zc%&inMO(m8J4g-uSJF{9msoS(# zoIE?Vx+l#X$>>M*HlH{2iCev|t}SkbQ<5nZzDk*LD_2|y?DYq7LGK=G?zV92YrCLz zku`7q#a#ru$$i!l{ewR1k$r?d>u^88-Yxm?sMnyQe|fjX^atB$zXz1T{ku-7j1D2jgnNQOY)IE_Yi zG&TQ@yc}$|(xOUW4gnbIg6b0h#0e$A5VwI6bj0Ik3-lQP;=B^gfxvoqeO<<_ZE40v z6c>rie`OS95F1>>QgrkAZ5hiGWN@{J|BYdWSOiLV5bCO0l6OV(ZC;HtWJerbf4tgBF`TvNphbBr6S?x$Z|3wem}u?jYhlpD z4K9VrL&eAM=W*6mJ^g;beLk`hNA2{Hr89k*gOqBIev&MNIi-(4epLNw5g)UXs+bN{ zpxt7NM|_HFDorkRY^2oDp5lta4o<&;V?8Bmke)fO01xSjX+MCb7E-76klPFUe|URw zUXQQ2Q0G-2k>5TgF&w_LOHN4!-ycn=%Ku>ZMRUum0sCW@51*%EnDs1;vQ5r4{W#E5CG6#4MUw;UKBk*#9Q+yeO zCnM4C={Rob5Nokk!eDXy0Si*4C`EQdrK@#>sq%9{D@=8(3aPA2t2Dd;UMJ>{9y-4=lE5TT@RlWqT?*I zR2{oiiNnhH?NABa@j~oy>Zzs{Dwy~Ht5eR8nySdv)uE$Ebcu`awt`LpT3v-Gw?l^< z-9B#lpLw<+KY_M_W352XQRo!X187GXNe!Q&GVoS;yIsLM0y(G~f2%wdNe!HNX8BXq zR&p75vupT-Hdl3vvB>L&_inHl#r8`Uz+%F?T9MS@(A<|o9=Qx5C-$p7e~cRq>@htQ>mi z!>ar4e4tk20t5fLX7w$>Mcmhk?`x6+!A>fvB)TteKIhC+2d5n@cMFl@tODlo4 z9e6JlEQplnSv5T6Fr3L@MYKE4rh9k=B1ZD}bC6ab?jw!$B(-mQf9 zFt+1I%GG%`zs-sbiLWg!UEhqhA+~lzNx3+m3g|vpe|Nmbqx@kzSWn!d#~>=)Ikv%< z*B!YrMPvt0J8GkG;)Pv9gkCsqqtzeHu5`bl(qh(nx zSMU-I9+c6N!^TpJ2Km_`AX6ce0^ATY@)T*C*vN^|HqFp1 z{x{4;f5o0m5y8cw6L*ro+zqYggUavMpo=}P*-7&otTs<%B9GFLb%w#zy{Xd z@<4psAn%?Eu?Epej4Oo_*SDO7afH}BDxnSHRVI`7>fr)Gup`KN+>=Eg>26}UB#tYT}uSaHc6&o=W9G63Z&v-tx|Bh>2x0BYE7B6ekw8 zf0Bq)l@y8Z?IZ7)6c0~eF{L>OFLkSsazq*(DIy^$l`5KG;E!n1TQ_T%23fE zUgezM+0h~so9q+@mhv!zaM(^42`xe85sJyFsMMby@3Yls-+X$PhsjPMVJY`Bc&IX( z4Im>06&3pN{w%hiJ##T#M3I$(DU~`=I{}@u)o>5E-{OLl4DmBm4?8iP5syL)e;2DK z6Pn*hpJ7Y9HL;@Z$hrME`na`qGVywp^vj|5E(Swq2g}k^(TWPH65|S@_dPm;Z2JmR zW%tCx^!ueSvci4qo^XPGpEsBDAD`bR%=;l#!@oxHbd{v+bm8wfsv4U@lyE4aqvJz5 znXu2yeLSIq)O<@jq}@SHi7IE`f5DEcgXVneJ+IwCcFa>YUB^gj#K)>zow!5j`b7Ju z(0NqHl|xS!4<1a19(Wk{o{1$A>R7!esQmfVuwA6Gn+y(?3w~44mk(8?LMSj=@AF;O z%A3U2^6>U5sdroG78w}$ZfDCJHZZHnhz?RahXuyX zhgO>YJGT^n|MyFhH}j)poo5@4i^Z0*3G7R*4e0$Xe-OqZ4$yWE1^Sp|m3TDbSwV?7 zOs&kDFaD!thil$;-W0O5*Sli1IKD16S*4beEYS8{1(65Nz~1jy0YT@>U!nT zr*4}PV3iQ`p`Lw@Ij*ig*jnptE`95I8u6A>Ta$f%=1;y%RwXwnuf-5-B?901= zZouEwj#ljjv~zlaAyXnVx!_j$UA|(?eHq5m*$5>}FmnPry`|K<1aMPN9R}}Y?zB+? z7?w1Dd0UY{T~LcDpp&31L8+o!hXdQ*gA&+kQrtO;gKWL807kzembc&=Ms z$D5KZcIz!}c46w9Ti%GzdY4OtsG-H%9LSb`;I+x;5S`4u13rHUP#b|ce^lF)VC@Vhrz!TUddC9fj5 z?dspv+XNH#wfb5B{frOtd4=MIS|`~uD>k6w(Mi6_moOXz)o{$f0P{}n&PQwweOAE< zsiH!l!{0I3dM~8;XQjd?CO4TZ@?`}Z;dFnbgNd`h7(oM+?y7#1G$_#FDf+aD&X=fs z0$&}s(1lUyLfgAH|G?b*^~U3Wc)(Q$KCd?!Y?-ayU*ce9!xEBp3-TXM82%OKBJ; zVfu=vreY#2l|qqoP7Fu<%uEKf0K3xxh=+-(kcgNbV6Cx-Jy+ctdA0BZV=tuAx^Mdz zYLyo{O^?hRWDdnZ>lqptdhl6yf2s0NgnEmW$tRVNTwwyT>dx5&kh`-s2yV+q&f>6O zAF4$prWYk>i5#iSk!sbET`ZRO!rKRCmk=5y;-;CH@+-5u)Ga_?ry3EJYnYNA*U@IZ z*{wj4d-Gd@2pry~G*BkIn7gCW0v_(FAU401!mYHZ#k{_!1H)t&1betle@!w4imIlW zGzvHnf-sx{PoH~YIP*3gLTgXvA*K|JpVwop4rB|o)rCY9omciJwJ^gWW{iqdNW4`; zQ&i`gRSej{HLr*u+TTd(88_F+$P+~^jw7aukTlP&>t!l(t8Z7T(cyoT2@7ENxr&9Yn(VhvnSh$gh z8?To6hIh~Mj=O|PmE&Ca$#XUh&SmARt&6RFU$BuWDN6h(E)uwSMVk`!JQ>=g=vT50 zf^6t3sMjsa8>{6Tm^Re`r#W!bKgY0lTaFdr6Y^1x?>|90Dq?cuf9&u!T>M`a1NTZu zK~yX+UYX}6JKNSH6IG9=?{N7MUg%dab(MAZDd1$pvAiV^|BXLzywrn8CCXascOf0} zBGm$yRgV4RMpAnVeXA-d*4Z+zwSN5Q71w0-D*uK2+fmmWg>`gdhh2Awl+u>ehhnpP zIQ}`C^XFK>I^;*|f360qd4KJ?e8_7E#@4I~7nsg-Eb_EL_1#oz_#Xc_5a!wZHg}%K z-^Ib4J{Y{I4%okd3)$&$L6%iL0997C7YU>GMM%1Mh}WKVw_x>H zNgt2ixDNV=>6Z8Sy$`xAnsTJKBBe6``C9czQ5%8tO+AJ)G_ds}|CFtXggK0duuxt6YWog22dgz#tn?F3r#FDt6}zZbMjt9hJ&?MVA1^Nw@!N&^ zXEE?@K@KktNdL(Hb7=lomM`(}!KS#wBNFGmL3tO-DdNKxnNyKDRr>!EQxlP@*1Y}f zVrbF$bcpyFV9d@iFlO=L3HiU{{~}hY0|ci$svPEPf1vZ}^t$b$x@CMcAa|;BYhT+B zupFk7_3FsiA3k1vJiW5mSqy4#{ypyPqC}7Zo|h3}o`X)B;l|{oa<3T+uwT z_5{+96`evso;<9kO+csh_(Xe{V z;sX%!e?LE7{o{ilB;X6#1#3G&dO+Lye&tu6^p*bwRiQ`21XL~`w$^efegq5d30~@~ zTPCv)+GxXClzddzF63S~(k>{Tx#U9v>5nDv=Y5Z7;lG)~+JqQ)@GjHWAHRQd@lBHA z0L=xC;X)~Wb`eR(#{$ELN;MxbgpSwJJ|EXEf1u^u>Or+MU74T$YSr_Q>Oz7Wlm`oB z%>?`^djxx_m7%YBxB445Anz>I+Z=pl2BA}(H9*2-N}w#JYhf8&Q|oaT0N2okB#bjANqCVxvVsvIViKIf0y zm&yYIHGjmu^8kbFZ~U8|yv>?yS!L_wQu(xf{oC71*Moum{iG%U9RcRvrU-9KJ-@@D z7Z?BYfBKt&`~2FqVAjJ_*}SM)&+qxQSQvBJ)a7U1#gY!@9w4dVzr`PaEAuDE7{K9o zy(l34-~2zVxhcyPeO^oC{)GQ`CY|t>8Y5%h2!A&QVR~D?+-ioNm>_iQ7IGk~u`%|g zFej$N_Xw7{ajO<4Cy?BsCbro%Vrh^{%`N1jCQdwR#JZxP_7QjQi8)j+HQZ5`+2^WI z`2{3hBt>@XEEbFD72Zqqy*6F`y?4!i58SK=eRZ^;Nf=NZLnu2!BOqG%Gt{g)_3onNc|-vb8q{t+NgP?!Y<( z8PP3nty!7-iMc;OiTHN+1T5Iln``9Ed?g9S%vuT%Ijx+?K=gW=GNi-dEJ6~nU}cIQ zXt#y^v^mXyDzgQklAH+E$7x7^_d7QH6%%Rv$*qe`xhj{BGzd#T0DUqEA)H%2@_)cp z_;tC+S46C~ki=7o2`0V@ueMw749+bO>O)XiZm~3Pnz}K-qZ5Uj1aC#%7CU%ai^9 zHi54{%#J^td_#QHaEJt{Wp{|k*MIrqGSHYCv37WP5|W7rk4SuEA3!RvVG$s^E?_$Q zIKvr2K9<8xD?^YWS=qM{GmbKiK%jLMhu^vq@W0wg1axqM=VmPCbsV&jlRbTXO#pSc zk&=aEUDERn4ly(4TvYt!q9FA}WJ4^k27y{j?rP;b0@WoiD1iHZTP>U!5P$54T8eK= z*uaM~BoTBhDdknsBr;(Lk=rW<3bLzy4FA0H!Tg_@w+lUt~v|_;XX=nLZM;vwf65hzRos_`(n}Dl6zT$ z3IwC?ReHFi)zJnH8kG!6zJDsNtKP8{V|A$RpT5OA4)PX-*K$=}%cZfghH|Npa~XIi zfjd#gDz?%;BSRn3NIo5D`O;p>bs&*QVPO^x$6CgrLH6x%c`@29?aN3q?MwMGqhF>< zqfih#yW>rv<>W0~byyQn_Z}k$(t;8a6Bsd&lukkEk>Y3&Mz=@_LpnxDjgSx=CEcQ+ zbPP~Z0qO46k-z=E@B8mQ&wb8&&OPUS?;rO$=Q_NwAQ7G%czIl5I7ZWEaC{llu@C|a z3m;ahwdQzOfV94(0U<2hGk!*@a7qi1D8{O7oA-TqU9~q@M6V%dYDNSmYj{jq`kp{s z-F!8%M{H=_)?zNUnDO0kp5k)oi~z5sSw-+$kKN$6;di@RDdOK~Wf7i}zvexhcvHWq z!*iQ=MR}g_?1%8|yAQEwK}`jI^usXJTtZxqexFyJ1QD|AH`BxWoy&xS-)=TjSZ8Iz zptJC2N2>#uW2*zI8FCIp15;>M>hSY3qOJ4#q;M*1Flft^Z~lkRaT2$KB7wZQ$Ee)$`dS8Qg|@<5H2R^va1ypMBy` zdG|G+75*}N=&l7Sy`KL1@3Wh&V!Va&_ie@MkZa=C@m%?Xqc|Sig(C9ylrjYDqoA5g z-=0#lC6p1z{jiO&T_?!K^(OqUja?eVb+m2`6A{ZR%Fsb>Xz~LNKk%dEL}{DBqC@80>3DQS&2#e(S4V+7 zT~3#tdJxoaJDL6WJL-uHVM0->?=RV11FqkB|54$F+yhIa*kyR3ntCJkzPc6n0r&2d z`5>zmq(Opv_p<#Ym}f}So-urZ%$V6^Y}t^@kmC~TdKYQMhOq3uruUz-=Jif2IoZu6 z=9=p|=65qb-OtMi+sLN$=wW%6$)h*3oABe6J&v&2Feq97xO^>brZ=Tl2vpH6E?DO@ z`Rp~kgyLcf1&VvgucsHEeZJ=p?kP>3*GK-{oD$D7)=H3Ip?7H@pOt?owZLV4S(hiFHpmB{29bHlaKkro7ro7ai58ia* zj$+|jDopu_;+2I&uToMUXx6~g7OjWnC;H)b>e{btSsyXspA|Uy9%b>m`RD4HyC99b zQuK;_Pi!dbXd_oXL_FPG?5X{2;g=-F#yRgIxD=!0B@mY&{W=KBS)}ZFaWKC%N2KEJ zXDiCm>`*H_`fN}5&7{|TD#x{M6E#&oqxH<{+PjvkItuLWQQY=YuiUBpwmh_91_^nq zwYB`xDApO3q0t{vSR3?xXOEiI(F)~hp4#quif{CUXRsrq?CPs*Rm z?55jINWsn+cH}8Lx}!sLd|ZRbnaoP83i;An!e00#WKt1U@n~Mwts?l_bI%2&pMTQd z@|N#K4WhnEW>(6N;%5kg!W|s!{nZVsf*y$~+-VbZCPPe5PY-Z3TpM!nD8{Om?c6-} zVV~Y@`J-U>{JmXIy~sxV^wH*ruYp!J>l1TYk)87_TaR5x$K8Hab1)?e4o&d+QyKnM zwg2%}P2tf!`A3IRQg?nm)@dvse~4r#4~49rsQ&Rx23T2znB_$@+yi-@HyRBm{*X7# zU&vO!HA-DeKeGAW{J7fO=IJac|28J%tGNK5azC$@THPIH=RG}(aE8F>EjM;U-zRQF z@Lm5)-o(kBUCCDgQm-|loVevplJhFley8=d2t zH(So(AM8a)A-PAL4vEi=5ak;f?JerhfHNq#9izm5L7Ss&tH|fGQf(;R7$hay;PzVV z0={N>gw&*0v4%x-+97N6`7`?-xZ~5;wLFT25{1U9jIim{!?n z)=qSU$;+J7wO5rlQ4vdm9J5!=Pl?IC^$d5H6#-;|^kt$ZzC zjL?Q#zSZNOLo#G6p4fAod7l)Fv^F}oD!gh!-mRB#Av*eYh=KCo?=gLQNl9U+ZwqZB zZ&DO9=TE6)W3?L^MBi+n-KoFoG&pH>h?enTFi*38U|-WRHI$u&w3H!#q>$LG`a4;1 zedM{rW62=r?+cUjq5?X9P^sH_)-_v(cbTKtQ)`@JW$t3`B*QSrF* zz}NcJX27xXW$B{m$x?or0&`A9(i_(vl6wX(+DobXHJo*nU6RQ!X!bzRS41}5W7&>F zYIlmh(g^zUl|Ky6%%>(pT7<9>&9i;b=&Z*?vlN@n6IaHOMs3uHxs16`+ZBqcrI#5o zaS`YmMDvPma;Cw%9+ZNnY8psd_KfA&rZz!c@T&ZvPz^z4-nLl~``g>q3Y@qbwF}ef4lhh()b{v=+#j1~X_@5^qzoqv ze0;!$+`m88JaKYG`0!0s-(|dVo5X}4X9LOQ<2!8EkBo{{)Es$2&m4f}j@PXKmUjy} z2~=ZT%8%r-C>mwa%_43AfA23uS@V}(w!e^(d2`%$W%H0q0(Sl-jIv<$!jO{CBmQOL z_o>-+kiTO;Dor#g7oUbeHs1R)1uVgFQ?YK#PAfZ)iD%b3x zKKAgjx)E#*3`&o=tlnmwI*=*3u*o)U|9+Y239Yl^2G_@KzRcX92C`UCq$WUOU8_H7 z>=M>)5p;z4Ok3Ai%SlY{9!O%t7DjyIVH+TlRHN`Ptc6@G-8C%s@ zh1h+*q`=EAlUs~Q`{CT^o1^mJj3#;dRkF)M*ieW>rWFm4>F&Cl#DSp1>o=JhY?nks zA+GmQHj*+?G(ZReCbNMCC_q4(xkv*vAn?u;GbOv8cN|hti1{q}zujRGb?)aS%&x;& ztPSz0S-i9SwXK_=c-v%OYLMD_bb2`7TiMX{7aa?P2A`H!o=U6;UNv`z9glfIl~G{u z=_#*u9bqowwF)|UO%C`3@}v}No{eTdakslcD;(NY#C%7zAqzD z<;_)9;js33+jG*)!_n^Vr>-L2q)%NEvuW{HO#a|ISNE*N@-!!0V1?i*XkMcPU2+*E z^7r~;LB-e8a~w(GJ7m~S#FqPrh1tuCe>HZU*fGseH3zRA}&|WvU<6RP8UIH;zd&mRVwkyzQ)(9s@?O%Nbp%L-r;>-ilQDxrk+*(wznl;ba`QmBVlzr8P)-r4Q^Bb=L9PE zoTPR<-U~^6%haHZE`G21-tzpoh3Dtj-FuvlcCZ(}_3f0Wxmn-V0X|e_lyv>iJG=@;}s8fcggRt73@eDDme_dGAbf2A~81OXfQU;A5mM@dhYs zMEX9ljqsOGV)xeOfvO_M5I-Snvt{mnH+wXM4Exm(P3^YSexQP%K}>&L}= zF`q^~pAvTbe%`|`t=74KYz4)4Qkj|{#3CS^Ze0<+Pj&8BUgft(WOq!|NtPYJRNlrfC`;<$_H07Dh~fCs>=xZ7wByT>mtoYZG)w(!>>tfqm~`Q5+zlrlKX-N za;S1>)8j(k$%pUnRgp?3`=HMfCRL=83#V~7u2d-D$Tg%L5sdJ;)DpcGiV&l}r7rAu zT~E)6Y@qD$mMKEWcTEY+?uy=g%?r_5l)D<-LNN|T7-^-irjd&@a$wR-%ZUas`EW8Z z13}|=5Th7+ie+SkfmZtBfBhv?ee62O8F5_7?ODk+^coS&^{L3>hVMx*tTjd+&wSyg z-|f?exTnz{-O@y15%asL$xTa=W)2g%uvdyajttyVg5lr(ju3l|h z|1QOFOU;QtoP{i<6HCbStEa>N($1^+zJnNSY&o?zVJ~7~Wm1L|4$MhnMax)dE>j~9 z-x59lU~HuKSeZtuFiES>d(-=;8gR61+ zE;52#v{q?XKqZ`wR+-&@ZBvV0%HE#OiV-p->l31HYkchhabITOk zYC};Pti5S*B8|crulr*Bz1U6;V~+n>g9UzgDjGX9D2?ozFhkBt3?;FCc%su}4Nr_8 zsLcF-xTP9M2cA3!&Nkiw5QD9)vUW_bNu3yfKNgtP!;EYJZ=sNksQuZ2$3cLVv^Xzk zo6^{U10b2yJ*iWbc~3`}ElH*6R>kL+qdx1e8{OFBtM)Ba_F?g4QXZWY zdkP!|O#|U{^u`SNu@hlG{}gOV6XRRF(eV3XgE<96uu?JzV_4uc59Dr)A%Axnu~&Kd z79a64irfpKocp`$YMS%@(IoSmMbVr977P+EdXI2?qF_NQZ-rNu{FEIUgn81)i-`w% z(7)Eun_)3B;w(yq_lk|_#2T{ntoxJvo!^!^h1BXP*i|zz*dmm%6SJPWZ~v!YeVP)Z z7(vJwxk$EypW=P-eCK(sKq5~r_5kc-rEd|FyZ0YyKCX$#W0y)rVD7X-K8Cdk73oGR5FM6@VI3PLJzm|73iHzKk!ltm(a2Zg#$}Tq z&4l?QD?5;Bl0hwJKbQnj2Qo#n)V;BILMkJ~D*H6>gxo08e zwLdkxKUK~$(8n+ird>?&Eip0JBMKWZJ(h;n{nQuTVbX8P)^HVD2Tz$?F zXpGp5S&8as+5HX1Ug@pMuD`yUvGLl-i-a5;17*8Ae-2oN{v;^rcxOFpzU!FM_&lM_LC_5)SG#(WpOS!1~(A*Qf7W4#| zM;z*)TEdD5SV>lb0Sh4Fk`EBLjbvdMA-Y7h?*SrUH(6;ESl}RzG+{&F4wHpP3DFxO zBl6heE)cj`-pjmS>}YI^^%Z(8>q+N(B2S2FCLUlrS*e8J2j6gPG<;r`3kwJMJbkre zlc7irI^_Zcs`#ARWJFVgPQ*h4m{%^kp!w`{$8Z1++RGFJw^ZUu*bKYZ*>U^46|R-)XRmb zqw4|ds7?NEC_5+JrUu|NVqLBqTFptf?gTgy?|qL4R^?i)2OKl~`5pt$Q{$RD0qjdp zuEoITm6a!&kjWHhMqesRikr{Uj+>g~qvp`5_n+Y>3j=6Ul1m6yolq{c^>5i43j8M95Msj+9oZV~>@=g?U7JFvk%M?6D@;CXYlIHn6XWJ%%F5 zJDY;z&1*J3fXB@f(Q7WM|4BEkF5XB1OaLSNW>{`USAUhFURn>>My;22L#esx)-?bp z5xbx9F&1&S&W`pIUz$$v#$D3ZUnQ!SrUJGjw()VoxahXULuEdG<`o%|jE#_?C>U|n z)xsV>$NMJ|hV{E^V~?$n;O}|F)}vt79Q0cFT_X1&&eqTl`bP9(EIXnY|pUX65#K;z?xJtb}oc8drD{t!J<=y36idUE=B!YJ>U?vUD6FD z#kV$T0L~)TTf3o|_;cbbo3=UB4TbQ~ZNdSksLem!&}RHP0C1YSdC(1=F2)N^)v<6Y9j*yD!1w9NE8C2Cde8}1Xn@LTux-XT zJrZ;(4hYmdjk3*PxeGdl0srr40(t*~}1nb=lCdelZ9o-6ff%X0dUTUXWY)?GI+Gc1m zg8sDu4*iU*1QD&q#QL*|#zIE5DdEc=J<-U1<*qyN0XSkKy|-cmv^H%+x~9q6@xYAV-4p%)J(AEMwRW_Sr*W>6TrvU3LdiVVGvkB|i|>TZTu zV!aRXNhfSE{Je3m-n?DCr5>;k?YxJA>soUyrUDK@ld>qdyA9XGIAELarxgmGXv_7Z z9g9Lm{VtQ9qXsM$i2cp_ zg~YPH(4G-koCYR0o^*mY9Pm;tT2a}3k`IUXxGP~m6+|@T_mkpwb#7oYEJ6|sx_Fsmg zrD-B2!q2u9z1Z(|?(#FP?z!wdIm*x6G|L8wY`t*dvyd3r( zLR}*}$eiNKAF6ehby5nz{P_p6kjvlpDe+SrZ$F9{$%UP{lnlv@&+kUCrNPepItQ*5 zT-KZ8y+O5N@braMyFzt@*g1fzjjnip$w&&nLiO@rsjS zplyRae8uyO=pQMxyY)k)~=d6fyot*|lcFLPKvW zMIL|1k{{id6ZUF0Mw;8^!6ckk)?W5(9e;LTksxE2%min$4(le(*onjq4ihQyh|&0d z@9O5m7h}3>YtHGTB3C}`PUUZpTYO}d{+RwD@=pkM{Pnn^&R3pjVRQ%R=lDwbL>ouK zyeaGqF=MZ(=!^VlD-vCZoN2vKe;~-o(Zs)T|5Yy0uaP~{eMd^VMM{O*mgiG|hL5AY zZfwoCBj$F$fXF08t5*H5OvLMIt(GS|+tJa2&${VN6zp==gv{0lrtc}f`Nxy;!Ivgj zzpCE$$wTW62G%O!>_8y5v8S^5ok$DUrZx;l`JBkqKAPIuuTT?4cv{s+ss4iF!iw0S`z8R!Bt0~Ia?d-*^EIbdmY3?3O-ISC3 zGJ1~hG=+zJ`2wg(`1h})j)K{qLXgN*@y7EEeTX~U_0GC0gk zVV+4tdkdx>nV3HqWZ83K*Um4MRXo4$^Dl8?;M#3!v$=+Z{@4#;;wFc(E>2NY;$#du z4pWXDBMNu)5A9f3&*563K1#Akl$==6p3lggd-SH^m@w0W7pMTNp2)1`1v0ZPoOljxBQ}$G)`@pRAeny2%N8KO^z4ST$52G>B{Qv*} delta 24032 zcmV(|K+(UZ(g@|z2#`JqHZ5f~GBjc^fkw4P0j|Y=U6&lUkv8G~eHD!G#D*O^tg@s! zv?n&ciFP0Cey45MbhXD5?;iw9VyQSSi84vm!N2x4_V)Hl7MVDb1i%EPsxE&&x=RF+ z`8-I1AWn&Nk))ULBV4M3Mq;Aa57;ceRe3T?h1(C0ePSC8pH=WYiEUY~>bz-q_q=|_ zW=ea1T*ufAPs_OAS}vuQ0~vB#@oxG3xcT>|G-8YAC<9)gh7q&uG~ zQvCT2grJGgQWT zX+4qkdNiX8`l-s~F)t%ru!d~+yx-eC&poVnxHs3P^{iiA2hqI@c(9juw_QB@;a8hu zbP<0=@xxU*Mdw*pxDLtDM%$~R=>cmtzuCi3I-;9aytk?JC>5DS73MXF>}!yJUEP%H z;*WfhUE{9Bhv#XXKFv5&Y57-D{@`}YH%;|tF`nSMrv=}Fl0D6c8y)5M=vmsXMKG#p zxM5R%Cik?`sAZ(SdG?y&?we+C8L3~8_+=!IF~%(;J&j3*mytY%8L^DyG0$GhNKa!W z@-mXgSmT$GJT@D(jPx|N8eB$y^4MsfWh6Z%QH(rF=1H1o!!D{&m&KU#VVuU!y%b(FQY$FsiwwcdWfOMJiRGv#Fs*FVE$skPc@EK+*e9ZZt?ZWKD*LZ(?dwshW~W zZut&lHgUkkr}8J6_Ee8&RM7Q5>$4h+)qa2n7azv`5KnQ;$sB)wx~DgEu*W`6E&ALK zC)i<}pPdipz#h@g=Me8EJ#2+>zSAwe`-Zq~+4E2_u3H9-33BZz+K7GJElp$HZCb|J z*AJj&6pE`&w+QhoBu*oYTOWCRQw*<-I1MslWh7vhL-<-gm8oLADg)-*+lkU?(!G|N z9^Z=OWhSTL#xF8|Ic+s+iRtleG`PIvv`zftQdf<*9=E#GwMt9(fsE7Q7g4THcs*i) z$!(xLSD2p8QW0xRZjF1poCwIS(N39j*TKPlBG7%K3rG2(ASW;ONvyIH=81$#xoNogGqzFc zB09!yM>2eWo=|j9Tf#n8ym;=mbP*k8%S%)=&X)Y&$f+eTmX;p;-uzv0QG9$We> zQcsj$;P3eA(JHagxo@dEgeYbBbs-i2_j%xL*rS`QzKNSDdNPxw7Polo9R0|pDU(5E zwayxN&Fp0a@*<~_KDJj?qSsQCmew+fb0N~nmbSKks70&%cXn4aH~eK{zwh!&9v$|( zOv5C4!F$@opVO#gbmWkNNMviL`%0QqWX4dA4e9*!(7+f5vqW zIAu(quQp_wj>iEvy^> z>G_0zgv41)B7_UvG+VWDs4EOSlN>+IY@XFQs8R`Vt@F`79RP9MgE($MfQ8zaR6{}F z54oypl&4Bp3c5fv#D9c2E@Z-Me*I46BYcHy{kPew;Ehzt^$U?jjtBplWpAA$-q1$* zaZ9c(gI!kn@CxPL2Coyxm z`#qjrRcy27W&`irs`vxfYW{!mltKkN*u(m@DlRz4&o&78nUz(sEH>GS6nQBLs=#@Vje@ZKDMR9zT3RJ`UMWvf+r#~|`-zF_h=Z>}7ZMXE@o zy`w$-#}55qO-C}5A48&r6;!IAQq-9WovA$68)Q928Do}Lko*Bt#o^wBcfGHice-zj za#vf5RnmUO?kJXH4?AgZBP)HDft}oA4R9fo_AqOmUGs5FRn|4D`IvEyEes@oVwy-A zYuTP|^cd?V?rmMrn4-p1XhhdLXA$B}amf>3L)s-K5p4W9e_*)T8m8iMhRv1q!4vJi z+KkQn1QpS?TIaFRDZWiHrsG;t)B;nCd5T)3Tb-X7p=9s)6*j4A3-~3-6HJ9Wkc!Qc zf85sVa>I(vcGqARwy4StM7ZRCcvX>It>g!uVk@QexTW;ir4&qvx2YCXx>KP$6}nSP zcdB%YkA2sKktH5)L$!VhpQY_~Rm`!6uf~3uX`~_5f-TQ1e=5xVjHs4tQOLtMs@$)P z7Bwn;51IBf=Yqc;yeA84i|sQvq=RczS>G7>_ZJ5?-nOioKacqg+pz|Jyx^Qu5EWdag2PlnvB(ya@EXEbd%~6^dxioJ!^<}o%*CzGu>OU|;`}~a3Vn!~{rI6a! zEW4_Bo>j#rMlQ{-RS{QJP@LftrksyZ{qsvs8NVZg4{c&Htobc#^ z###_@QiYQmjj7ZqNA8_sjJC*Yo%*9l9A)hr{>`ZBdA8B#Xz7%Hu@=?}^m$)HZZiAGTUD5Eiz z8dI*t1S8_y;!E^@W_3j}4D7nD^158{u_ewc=4yg5=eN`y+B~f?k*)I>RpD9}{9rA- zNJGI^b0v#2in*FvbZ{k?6($vP-pl$Z?+|8aoeJk!CDw58$6<_zs_-ibKMrR+REJ+h zH&=35U&D_B7;gz1dKXo(xh_Gib$J*=S_;9O*5%=hYAFSOv0xb@?eRn2yyBPd`QtAlf1<90SI*swN?2Yz=ljFvqTt8N`%d(kYfxdWkRNFv!xc-GT%p$g<`;} zxP_9ysw^+XZLz>HNcFI7%Jr2Mt5YP%>^%26Z*Iy3(V#AnW2IB~V>Bw&oI)+GNNcsJ z(qe`1404Y*#%oOY>FPpc|407t6Dk~S?%%UUE9bv|4{2nsPTLn?_A)J3psniX8CBeW zX3YpQ{=O^b^0~0Osqzfnk41u5@DQmq%YnAzmNxAEwA2&r2C8I*ONAu%cznbLizKR= zScbX8RH}72;qsb^d1kTAiC>G$2&>py#GWQgYm7}$P)>V`ol9LhLj|~iYccgruz@+_ zjV*G2h!waM`?>6$Cz_V?!keoRGbiJsY*eGTqjnX=mmMB)57Ms z%H~IU7`xkw8V1{FVHO-emNKo_LW;~fXdJHgSdYlNYVmW~NmFBKvduVx8LKO8FlYyD z_}z38k1G4GGVT2q7NULjjwh2a*?HyeG$S+3Oso2135$s{k z%|}cnsnRtT5W9$@ZKzb4TQEoJCLeEw_cHOLOe5L1sUD7Q%DOo!;akYtqO5vg zkbv(oB#Jal+QNR^Il=b;^&W%FaZfLQ*t8EuwkcPff=CKFQlSGAuzB(ljwtjmJn=QA z{Rl}HN=K9>;aBT)Ur2>Sl}4?jZ9}%yYD>0KBlm1~EsDQvUw#kqn6iUK!IsL|Kdxr# z&Gq^Tsg%)`vaPb^Rxq|YYCi&97d8(<7cDz2yR*iIjGHV9?n|sp7ASGPL=naq7B4^m^6r_cBSkrPv_OpG( z|3(%RT=>{zVh*lBuk{IDsv)43inl~deQTVn?#Q*$@n%9l#>vDS+!HK8oH5YJeu4-2 zsIjl!5*KM`?E$4YiEge)JbND6p5%aTJ2~HaUH49A!5uZY7yo6 znB9Gk7m-%s{m-gtR@fg?ziFefzG0L+?qXJYM;k?Hz@6P~X5c2d+PZ$8M1!mi|IZjJAJ(JifFk;0Cw-b1+1o6VV>>xYbW6$g9Rkv*_97 z=P|r-(4mw4Ntgro9t#qinRS%PaT-9mYsp8$)#mHj|qr!A?Ry)aCae z7-1+ii!2UHmIG32plp1_w`IA)f(_bB=@chIOGUzW7Jo9!0UflG?WGipO%+m`HIR0Z zB^6c&PbGtccM%(VK)NxSz?dsgjTR(Fn z`nX10^4B^ybMDdRgx2uDT?uIEAEPG$4gMY-iB3sJ_9JfREA6Ale@7mx!6mexA^$!4 z2n{Z)-3uKoJbeE(U&S1ap*B6wvu$=&tcvEL%DFL%xm-shyjg!^c4n*X*dbU4%hx*l zN0XiKDkypDD0g*|d;OTjbK*I8y*2s%rSg#qv|a6%#S=WI7LJ3*1>TQ%5^yA9BSTv~ zK79w8gVUvzE21d3>&Gv^?b=WKU+W06@E1-@D2CgyefZii4i?~IaVc&Ou;!!ZMTsj7 z7}g>ap{c|MU`T(-q8cip?YON?9>8GhCzn}##O2oB9z9yYE=0c|ule6>>3+^q!LiOQ98kjjA1Rb-A+a~mgX*Fv4%>KzNgsITpG4;U z3Wz-?oyyzLs_-=Hk6!B3O(ntp4m-R46L z)Z-LU;0`2}?II7hLcT(M!AD1aY~)9Vfy-k`1YI8V6hjR$rD;QTc)W&}9^3&JN|Ug6 z0d_dO!7SoMrJy5!(IsjyDUV>Kqx_zm zzkYVcN)@ktZFck)6%kEWg^qGKv^@-%i;6nDEt2`FoM*mcS0AeY8Z%Um4> zZT%(cSr6JWtLi=dQ3S?H0UM;E5Y zLp=bx{o5)@ulC_e(0iMGC5d^Xt>;VmyzEmN7gUdrRh0(6C81K3)e}Ff7z%_^|56>~ znbzBUKD|zOK2ni;)z(+g;d%@4(=dFRILuXEP+At98BcE|ZYb13;(*pJoQu>)e(?b= z2AdUMSqKm6poQr?9A9b1J{los#pokU9zR0)(mnxFfvuQ)Llevc7SJhAUUUnYR%V?N zDC~~6MQaQIC4YvCCGudQ8e(Tf%3zrkL?8X5fB5NVDjv5O7#F++4R~!Rwn79?NvVls zg@_mv+bBH}ukrSfhlQK8ax%PM)K6K8v=19BmU4jn_o9w1r?{px=m#iW2h39@@!AhI z=f08Jwa`2qx48z~`S{?eTITgCt8YBcWKUDrw8ev5@}r+X z>{5vBV}=c+t%^T#$W>j>XWzX%`;z8x@}ZmMXC)4~KE8Vm;vVy;nA>=?_wH&HJRB5D zFEfAvHoNuJK+F;>vlJ1>b=x3a3UBbP_dw5XcFV6z?u3DR#b&!}Nc{2i6wF>ne-$Ex z+vRAvlgX_(aPK|jjVhpAV14=$_JIRMlq z<%mZK+v#K44-D28RU8|kiw>Ld$xR0+g40(v%NW8!BARg<$i@2-bCB7qXm>H(bu8l; z_FbSZ5fv71EOY!U*ZC4ojd`+Cf220(K#A}}dnxh`mu^!o;y$1}e^g0qfG+sY{2uu^ zfC*xlLjEYGHb55~Y)WQY<;(mYud~gf-d)vA(d?R>#2RIzX#3E*tUf|MEW@kK;!@{} zDGt%{`;@_9dqX_V`22cVmP`Ic$fr-hJ2haT5*+=2**?CV4wvOgn)`55mp$SECx5sd zn-)YDAA`=4pMkCFXtP_b@b?q&P7PS71V=yfZ1~$(7WIv{`U{`)hd>ErrpOM0%xU)! z0H()5bqW5FKg2O6z5s`0?|22=C84|lu2L17#gh~TaQyuF-AK}EBkNOSHCy+Th@1dZk|gL;oWh0t^LRkm3M z8Xb%*&T|YC)f4kVqe2nW@gE`0PvyE2=YD3ZZ4OT8O^}exL!#8ztT5l{I~U@PpWaPY zf8T7&RWW}E@`7wk<^7b^(f)MQQmNX$|jqR;Z^9o~jr965=x zI76V=J{DcA-@)|~@4ep!h&IhOup3bGgNbyx8HB{osMWN1l5hK-%2RBb+4PX_r=?I~ zyoaQLx08#VogA2lQfTowI~m|{=(eHTeU8q4{w4JOC3&kQ-U3E)T~>+s;wQU0Pv%(-`xE-|2(T#UA@vc;p?)z41Gy}#2ZqI! z2h*}uxm@~O?uEfSr{wDMZ0#qgKvD&g3Y&`DsRBu!q=)p@FKqB=lp@;j9-uj<~`PVkM%7Y z9TZPtE=xd#X~F6?w$bW$IT+YhK69~;$B4-|mhy_qRpF-=K9TVvAIaGk#IvLFJ$ScR z1exO_)KQMZRk5tB=gvgWjz|qRiCZuMPSPsa_oG^HlQ(x|^|==VkAJ1Jf2FnAqJN9T zU&3I&kxE?$|9Ol6P!ys%coBs$>}Y^3`Jv~p@}Qrh{LP@Al^fV!9z{^&5xB{RJ7h=U zkIK-680tiGQx(_fBqM%B@l%)tdJtsusm%uwK}jaWjf72@n0l}E$~bJe;L=7Z66a5NNtc1pm9?6%%v1Outj4TA)4|be0NF&LO839+|2P?j2P*f4ahAqd+@tL@5-N z0C$X22Nj1m4@Jcv3;)paUyv(kV_coVS(Q@6L5OO%6>6$bQwue~R&MKD%D%eXEZXLI zzo`2=>eOZp0yl-BI_VOF!U`*bW4|jeZdD#<)%=FRaizCKes`QLZZkd*UmPoG{ezjo z{2)hs-U1Qg5hb{Ze>Hix+stL?*g?)H%%p)H%ajO`Y7mMoWux@8=9Hk)qCd+{PrOn2 zp#5oyb3jS@^6~?nurjjMvR`GZ%s0yv=(&;VgEFsyOBGx?OmeZQ85{$=y3NT$N!~F$ z203w!<4AoKMVX7G8azN2_mKKGBORpm1Pt-$2bokk2D*P_e=Z6oz~=W|F$aa^H47Z! z?WkZN$RegzAb*NLa;;^M6?CO}ZJ1XN+ooiJLF%E{3BXhUro)3$ID3HM;RDP%li~56 zDd45zbOakO#PBMcAGyCF9b!Bq@&W-87oL&y;lPcx{5S7q=8rXML5x&k1j*nA8&)iy zNgrGiN;#n*e{J!~Fy8MehUyr6!QeP0pr(Ekxd@b0qXa26{5R~*dXkPQ!r^8xVvkht z+g1f1;55bKQ?rmltN`FB&#Kh}gH0AJush%|JRK zI*PYZN#V?MrNj;e_JMW2PlYXyZvlt>o8M+5k?aN zbh|aH`e>e44bzj|LzK~Bcniy`B9b~!S=DmgVoe+dbBPpC%@q3(M_9?>ns18*!K(Zv zyoGOaOXyg9&(~MFI;sRRW!&4WNrV`~c?#j?Ss(F$pA8{EBO-N&;oFbsXi?s6;(AVF zr+hVbe=4z43pwNtjSk7e9#{v*|mkSu`8E# zc*({|7!;+3#UN`=Gq0*hw&Kl`u4(*N1V4#K;Z=Ih0C+53fH%minLlCe<6y+f2W?R# zrQ!C@$qAAAe9Wj=FAoG)bS++HblnpLe!avHS*^>g7dRJryQFXDgKaU-qPuDQx9k%z zf5f@Y!Y`?z1wbJplm|(Dat77&(9{;EqingX@}+OfwFRvtjVweV`0=rDl+X_HJa<7RXckVkJiPeSnt-V zy{YMQEONAFbRBnf;ZkoWEnJ$;mc~>{o2Je^7b6O1xLNyaj_P8)9UicVoi!pae-o4B zdaq4jEG&2%J=S)cR08*TwstgTi)`Bzw=h|{neQstSpN{U?Zc%2JC=_0P$62|9F=ft zvl=YPx~RpNyM#6BRb(+b(xMEV#E1e&$yAXUV!y)PVUz1kqBV=(pykyfy851*^inAe zsn+1w+vO_V9q;C6>#Ic;R~#*dfBf|oo$wtWCkR{uL$%^n7t2l0tAs$cBd<7jlB3A0 zH7`YJpcGfxW^q?6nj7XnbKX~ijx2Up+DT;QT28FM4zk`|U*~Z<>xBq%#9wr30&9TL z<7|y|&>_7&^Tmb_lFMxzAt{8MA!ab|T^CIr?aF~t%98^xf9T{iD1qIh zdzyF%db>a!hVNa3zbhfh@xGO4TUI3`>`ocmr~)0qIu0Fx*5ws9~V=)ks_ zj+qo_Q}#^^IB(DkL7Xb+m!^!TUuwQ(9Kmvk5ribocFoNnQKYn#jF>7e$kZTMssJUh zgMg{g*W7>A3fy z9riN*9U#|!@W6;;Ht6Hwq5~*HOkk6FT-Q*D=xy!vj-bREU{vSLxXz0$C{E*Q;yWx{ zh}>1Ff8S;GO&muQL3a3y&P!koG5RtN(=NUte`y!<`?y{d&u?55?&%#Az!{*@WN|aw zjc|KvGWvXWf0GSPWvlJG48}`wjYV4&H#4_V$TXE$DmV-?ZeFNAwT+tVi|{`mDqK1besS{exbEj^^DKmlLU0e@Ya0;RB?(5ZE`XdXH+}lEzld zPKcro&Nh@fuKnXN7BH*p6^=Gzy{Vj(Le{ncTFuHA^1Re3X*#dnAfH^rq5gd}lCBX&y5=&9E{KuoZp*q^@{Ynd#%*a`tJ}#j)%H*Zb^%I ze^xssino*`Q`g6RJo?EYZo>Ki5enB$!9vm5LTrsFPIH>eD6*SEhijwJ-D``UP2Rgx z&wDerV6mI_R58H;cvP{pAkIU~M^qkg2PV_BPHB1k6JI=ht%F|(lw&4X71h`|Rz+=K znpIKyqvT!De4AI}4A~J!7q50w3}-Dme`rzO%0#Yw^_w}p3?^DT`dSz?af3@?@;veJ z`+1yoRZqVkaG#H?#8EqaWa&&_<{+ioqn{)TVNU5IkRMflTExe!q$;LE6==8E9+`7F zAMb3>{-N=3tfxc`(lh53;2}LR?FZ1*Lh7_0a(iK)Y%k91@iiCfsOlr~+ovRke^Kl-*6;WMH- zGRk$W2p?I}J(M3^){1ORA%$DFud+H{gqvDFL5GGYvbX~rT%*}xS2^re+M=1Vg53fCksSUwaK=2GTC~LTY9W$pCTbx3hzPw zGW4eky&6@n`|ImHO2eh04E)bX<^b>E>yJKg1YS;XiZ6r6WF-1M9mg#lVlCE67%YxI zRza#1rO0lmbhQpHRemmLh3Rd-5wc=g4Svzt*E+NmqEpF6yaqk8rs@;xe|(Rux1NW> zDI8_1Re2|R5(|c%qQS= zhRS0?;29a(^16|-0jb_3rf%1f%wfe{_AK=gAfQjy#Q8%s(Z8>67;F`mGR zvYw>(H0G21BJ3x^f3>hpw$8ugt1Y9(APoZP=g=C(G((c`mu~+hWn$AWNkip)2BeV& zf%NX`Kl~%{B^rDMJxtOZ*Z&;)Wad2oh#uLlD!$Z*l|v7GSasi>57cU0VBlZZtiB<* zi2FM6eNA#8*hwXoMEB*5e5&|SiWSlFb&64Jg|mAcO=lzLe~+(sv{(%u$>7+2FQt#U zv>qw(-yy30pnEBB@RPRV?cpdXrxLa$k5Lk*sNe@{X(iCM1Mj7R1(EVRtA?kXoRm>g z97qS-?e?N9SP&^MVJ&4*6`SjF6md&lY^Tkq2_SA*SVS*%LYp4X<7yg+JJ9=WER5AyxSTb|CU!xR`^5JyOr=B#&-NjxjN70H(9YE@wKI; z>zmOw#MX`|DHrEc0o~{7j@NjU-){%&iCgp-M1?!Ye>V8?x+6EHi0t5LM{P7tys&GC z&?9qfqrXqiT6{%2>3Mx^Bz(ne%C{Yp+ zIqnOit9LJzure1Y7bXVw(6|ex7eR@Srkx+2e@iQ%!=x`1h11 z6H#)HpaJI19w`#9%o~IkYEz{)04)8cf69i2t+SKhr9V{}Dmp}`obx+7S{!1Nox;FU z9%c{@(&-|hC8#_?5jYi<`t!qGw)*UwJ@4``*(oF}<$ebDRYtP`WTc>?LO!MK5)6;;)0Y6=rdFgI}w}_k3tNWrY94cmq?#sOT0CWf1>Wl znfy5VxRG@-@%oSS%b|BF1_NaW%hFTPiUFz;;|ij8IXXjX`wCNKcf_Of`=v0l!ddH% za9)0&H<$Atmft7L`yo}szee$Nm89%+;qN%A8k<6ta0H>F<3l=!u+PkW7@>pId`ml$ z-9b%>D%ak@j;n*_eCs`(-9dKDe^WMH$4F|#$EsVMxFhHKMEhsXc~r-hLr)D49z%!z zb{OTJi3JhrSiL8xy!X_wU8J(>433ctepAtN4^^c?C@@;@^Ig{EXs%qAUBP_%UTlGZ zgQX_kW2PeD10|c?%hmrc8Eqo-{Xk!u7nmc>i~!U6;qH z-;PPF5%L;zBoN?;--01I4oEoj1f0v0EZO9DiIz2B#b#dR;1iUsTxiy90r%TY)+9xZ z#Gx;_tCR9Nv9;X4xlHQa7P>_S2EN-rhny@;(z~sPV#1c zl&te?!*Q|LQZ|8I$h85zyWtPQSi}L^&Y?gblB^PsMm#Gh5r?UjdGp2JTXwkOZRd3% zOMA5|R*U1SVv|*BDaitD-&KG)hr?R3%$r2rY^f_N&abXl9)0SzDFIdqK_BYb_n71I z@`J6l-saMiuBQ=iIkh#}XMg_W%Vbq@lk!>&!4>&SBJ`?Sj-D#Vi+o$~C( zH9Uh>w#3R-CpXz zr<-h7H@balYpyp%7@BABmiY4L_sKPfP=}V(E(G`>mAj!5s$51Z34d;ak=fuIgmtS+2^#pK7An}uwRDS&Q z>Ed{rh(Rm5OE7R}kQ*GABB-hvfP}IQ+9s*|6(!w|`j-(&~$}h^!y*_2QkG zlDCj>y9<4Py=Av8$9^YVE1RRN#fR~nkTK}EC6NeP$i}V7z#@qzFrOX!aX8lb< z@%9r|-&5!YWrCH}_r?!bCOvp8Mvg3k&3o)U@#Z8;v@9<@)Qv=*ezoSGgCtvEuBH@_ zt1??ij|1VKvVWQHsw&^iADDDru)u?v+x?r(|NYWXc%O)+Q94X?*H z}&P40Qwmp z($K_P^==NCy*Ve=%MLDBV^4CTUQh!&CHW6P+(n`2@Z?T%il2(uKBnul|X- z`Rk2`?|*<}4t!p(GuR+oyT8Q2%!VZ->lWniP8j|b=i{OI%{r?-)3Evy*60tq8&?ox zatrMPe=Ifj^kQ|V@Hh`mQZCc<~OR!DqUa;_?ImH0XBcv88lx2)_dCg!^>NS%hx~k zG3%UvM2hAigr@R1W=)iIm9KI*2}?$eM;tgDh275NHmmk@mve^cE1sH)iLg`(Mb0@f z9Pu+V8PEdkP6r?!CZ<9nVtRnJ#vb-ub!+6+!ViqSkV@;m?M3 z(7@1x&$@q0m4_nKTdYhzsf6Sb6OdJR&L)7|owY%5TRw6YhXwmkEg~_!C_zi)NM(*x ztB&kqvAh?|J}|q4&?pf%&BT-!ncbys0s1=Ch^QRFl=QfcHtWr91&Z98-x5UNs5Yg6 zGU3JC9hDaFa90Jf`K=Ugr9~~~^*tRJCc7Zm!)1SJk||JBHO-_^z+Di8;mmjX+!Mo@ zx9JdCdom9(rC|KL9&2?VTcE8jB%9jYQmdvCKETdzN?HB~+>$ z=fY2(vuSWHD_?D0Z0-AkjZ8^V;sok8*tf3DQv!lN*0$hqvM4|FRgkS3(M+VtMhxJU7|dwjPb5@uOEvVrbe?08rwyv_rc%TA_{V`T&*nF|^Yr~L4(9a1;7xU${%g39 zogNotS=9qjWmS8TK>GDBA1?y{b#*-ez;k#3K>GFN$9H~!MY)5Xuw$2Z7f9q38x?(k zb{<}n;X?E$lCN=F!TQsO%Ou~RzYKqW^S*Hrk}e+NwP)QeSUpzK$D=o{gFa%qdJiL2@$S$W&W*;-mmKvS!aF43F_LHgjqo#kn3|2-j z3CJQ}g?I2L&X>eQrGh1#p^`;cA@@J47^*A!;1sbKk)w? zn*WpKOFU|@DQ@vV#CdN}-i30C_^?IhRAf$-{{O<%M5L-UZ$G;jS~NZ#B7O!Kp)(AO zS$udx{vY_ih)n7L!6}a_hxvaR=zKc8Zo8;%86OSEo$B1$*Y*P}hv{U!I`Z|0kC)## zc|yF<1@?Nx)MBVZa@^1Erw74AMNTmTnS3v`z!XZqx8);OG!Lvjfiz@Ar%;e5&o1m~ zFh0oi{NM)( z_(FEU+D?!j(6+u`_|+$U<$pm{pwTb^mCJ{%wOoiF!Ge22l{)K|$?StR+OQTSAJw%B zxfgu23yLQ!`H(>RW6Aq@-{Z;muja5eA;ulN%k=ff?_a(CCP{IC<^spypp-tlh@0bM zfx$whnvWPT$7^Yyk86Jy&~k3|pjw(P%};-|>Ul_YA;AsGg9Wl?0)CY}fW6ep(AT_M z{f!%tcb4jH4!$yj(5cQEAYn2k(3gL&uRh3G7&)WjVbhy0b@9+MrfvqH1F$dgur=Jo z3nP4a+I;f$<*PZDw0-^Sn+w;2f&KlYCIB4)=H8|VZ%aMD!J!wg|L6boHv@P1m21JQ zhpDo8QMI1m@++}0=CY~F&%BEz9n3vIQp10XKmJzc4~{W_!|!@gK>EM=e_C@>mMi+a zmdO1H|L;sX;Vm^r#=a4MZVbZowtl(Q3_USH=-4gfKvrX8>`P%zOo#6gEOp~nElf@z zxkF8CvuniCAeEY1$VE+@c+`k>MMLc)?%oq~s9tKgqb{@0RiW|=NV-Uh?ABQ<7Sl_- zm*{(Ky8L_Zn*GNA&HO39SnTF<+BFacM*V0{;}QhKH18{UH2~6oz~rco=|qCjA`(bn zhoSIPtokN{9b@b&Z|?GZBO5(Bcu0w$b17~ncnq7&A=|(Kg1xvqWL7;%LFEVOA^4L_ zeGwn_XFLTRG)Tk4{*_n?YGr8;9y;P`n~Yp`dJW4?HMk|I?vv*!*;j*X>?>JIc~Swf z0%6rx-EtynAK@W?6rItm?1UB0$Qoxx<&4PI-W;^fHvGE->kwo_x45-tW$q{D{s1N7 z+uakeU_)=Nku&p^Bp5SmDL~}3aw5ad>uJi64u`V{Nx*`YDSn{c7WUKTGy|&47Jy1} zB3K`%A^qL&*zi|Or12-WE;i+=Tt3hsECB)Z$s~kuZurQ516SeK5L1DSY(!6Qv#sH5_%xx096=PfM;Qjg13|vEk>KFCx_Dc$2J!}rW z>U||6a65oxB;Hk=k3a*lI7HX(4LY!Q??>{pxYPRPMp3E9vZ&hxzWy*f{&4aQ@lnGe z5~P;hAtGOY=hqj3#@vXtgTa%KO!Rj|;v@S2Qh5c70NGUm)7i%v&Jgmk9Bx_}f(*&Z zzKxi1lxYM4t*bcv)|G(&#ZDrigA+VAV==GeppBgD>Fa9(sKbqvEF|lao^No7nK9?0 z;x88ksV^cMVtF+P)LL>^E8h{QE_p!#-1pmR;mm-4U_aDSd{e>(KAa(mpkqlXuZkv- z2}6k7UNKORUG-!5=amoU|H!;O8B5Omi~ut~<=bME{ExPg2jO`d&i{!1cNIgTBPyxL zRN0}gh6ui^(ROKHMv`e?%9k1aGF2Ldg4o#|Zwf6ZKf~%W zfT>O%lPz3zR8(KI9!63G#u*qoMan@^xE=Z^|v`7WyoV0GBa9z)z46FSg!b?a6Og4S@80F)w#SzDmZL5iBhA} zLFY)Pjp3p7suO%6TH;H52`f@%aEPW21ZCDHJbUqVb!O_J_3V&D6sOiAibIQ|VpaT+ zN_DrjA5CQ$r%Fv`fbhm>_g-=2YX-a3b&k@XI}u@#@?VGr$&@%04fAwDD!pS8hLkxC zzOLvIy$$(k!__eV?j@lJTgC4$(|s+T-^rL0`)D)2ia&KXg+Jccy1cSLGP}M~liNe? zcTPJ(u8nu^-#57PxEvTRf5mm-N7n}r_4|gWzLk+oR+Y*h24D0Yyb}|BgKGg;ZTsba zGymm2NIv7+v<2HYLKA@jLnsICBxd1Hawxpd1muO?&jGG$D|s#dOS6kq*)f8Y2?hQ{kUz3`jf zB>$}H5Zai_`%Krb2X;^iJ5U~RAKti*i|mi>IrwW*uQn>9@LbwrB#`IZ@$n;Lf!RKW zx!D17KJYfboF*znGE+g;n=c1k_79i@FxVTN!cfbuWjl4ojOX2Zy?tvTXQ~dCtNV7` z-lemL292yfcEJ3>MPd!Zq~FI9RYbIp%S<$j9T{kM;>y+3$JtU10<2LU^;yn?=q#}U zAVF1;-S01)y3rC_j7X1gCyhVGRTD;-(L?D5y{CG67~0&3B-|xf7m(gpgxyYz`)5Id zA*fxHhR6c>d;OyaG*iQ_>X$OU*c_vm*OJi~@L)L)RJ?ojS9pejRjoXjiH>$KqQc!x zW%2N^6#p8IPd>Gv_&H5EtW$}z`){01@dpFrnmWh<%&U*PA=xu^srjLRE zgpIT8U;&)z0I5+rSWJP`s2F53HsZ{=K)?T*MGV=>P+VS$b-0Ao)Lq2rOLukbNtKDp z_YQQVI2zbW)L;Ew&j<`Tx>CNADrxpUs4O|dd({?z{Jku__sbE;o?`$(6YL$) z?z4>a#AR01y8Wi!3;x}LB387Sg$3uwku$zn^bHp{zqBD08(@n0<9Qup>=8#b+?~c} z-jn<3Y&#${Sb{?I-uw?n^%~2TU~92zm|o$MxcVH} zN9LlU)T>jsl01CYTo_fK;M<}qMGrEAO~!(n0F_Cys)yD;|Lo<88I)mukE|`v%$i+H zTI~J{9a%zr^7y&Tu<9iFlJa1tO!YkD=ustA1X7G*dRD4NU;glFg%0X8$IPPz6(_OL zjmwOF5zC9=@t<3zwJSGvBI`1xY82w^pqt|XFP z`t|#0ZEMd;<0(@Ly;FO`eIPrWkI*Y=%riDDvtmQt-HTj2M1X{^y+M}9(|2y{LP3o2 z0@;{c-_STnXr1XL9A=H3P^Ad?w~zJJoR~ZK;oQvPYo!M=8=9ACamYb=fJBzid&hNO zTrWyqKTR*DDF^C=ngUYx&5VOmSaIJUg?)?$fr$P%wS0JTRWSJA%R~^ zzNv4=lLBpY_0%;!3uhb+aRG)TBqd&1StrmODUL z#E|)NCW`E7Ub(*xb4>h1O7Ij5)d}ky#AuVaJtF_vXwoN|=YFgv+q&=v7j*CoWH`<)ihu@!9P zZnpn9+?}9^`@w8xN(7N2BQJWf9QiB1Nz|oA-h_^j=)_@B`gTDe&$)QxlIZ!xgRke* zA3g1h4iw&!ojKA-U0D>Vghq|119pb-Ye%l!@kH2LQt24;RD;dE1*zY8@jJhnJK7g$ zCToLhp9lPW|NdfC^s?hQ!@G=K9|C&0zSBZF!bNBg!^e^uN!N{(_peLw@{f@>0zENf zgz@Kk`Gt~ml?6mZ>tr@1b{fpz*1f&0#@{$i;(ss5HJ{jZCvP*Ne^L8g6U%JhVVyrj zzItD`lx6b%ZLH$10c{9yVxaq3%kb2^lM)$Qkb{0=RLzmIyWuTwoxgi>#5LbG!_8WDt-OuO8I~0zSdx zMD2lqx_B-qxre~dcr2))hrq9Rjwo|5&F9%+gC|6|!{;Y%uk88#V8Cd} zW6Du@18d|zDc(G^s;Q{`ldPkh!1wD0rZ!&?57Oxj#lh2)@or`wwc914!WDi~%Q^`! zrj}D2`;~9;r5+OAl9tnIJ>IS=)P86W)taLt>758eI=H)xOq%;%`+wB#Yiadpe=oXG zS|SkOZzO1muzTE^gQ;D~vg93qZ)k3U-aZU&8YrRK!ccU2QFEFQ$u^lbt&!JKP+$@! zjG5joS0#l5T}lYph*GBhm`J*@Sset+#$IWU6dM$op+fJK$>@?j|IMQpF*^r2^`LwL z?+)KU_W4fl3JMS$K7;-AzxSp3a=ku;ckNgoZ$EMBOgQbdMw6OhX&^65aZEE`lh4n` zG=9_k>=Q`1g7e2NS4Gwi@Qa)=U{qhP^_Dh^2Ut6$I@Q}(>Oo%k)AkbqgAt#%#qCXC z&_=c+sg8NRnol_ujgZhIe6YY)#KinmC6T~OqqUaK;e~=zOwyT|LuhwvO3TNf9?ngWdUU3C)-kuSx2Gd>NZ2Ujwvig!^{JAIfA@)0 z-to;Mvuysxv-(YG7nB_p@EP6{R3;VfoA~tp5JWKD-RM~fsZ>TgSraTp`h)yOo;`}t zGQzu!b7{-k;JcfBo#sCJ+pRf>9M40flw5gq@sJMV*SyN!7A6=bOpHl&v-R$A)T1cx z)=$RYs)8fu-_6N4NZ<~3Pqy)JtnaR$m5 zuzZ3!7gF>Wx;D4kZ+eTeh635>h9)SRZgT%ENh%LkEBi{uYwg#f>N444oNZBAP@o{4 zOG}Z*E+mN1rr02f&Zf3Y^>_J@>{?3H4iRb>3Z$V&ZC~v;o7a-dk9nu34&94wj*BcF z%OY2ao906?Qv+#{g1<#BucBJ0;&zDO4pd-H9w zp+KI*wTRN`tG{=kU@T%YkbcZlY06eC_k)GwcdNuSw&jC`5M@iEk|!A2_V}1;%XW!e zj%b71vR~V*$?C)*Uwaccp`9V~)-|j52CM~hgVc*z^hm9dDgOTk8o9G88XtU6@48b2 z38o7C6V7v-Ig!XMc%YDS02LOfjulVNF(a`dFVGuLW)?7|ed_Wn7k zjl#ElH8-_Tv9OT=4ckhWp%vGm zVzCYAP~9Xp?#EeN)X|uNkKJM%$=RC%h-iN#@&ZY$h!AoON zT!&z3Lfi8NA;ZO78_@o`Z-z%^$FI|(?!rO1*hxC2|q|YJSrROCeGGKS4;2 z6V4x5_|`<8D$PezlqPSq-sXJKat_6^`i*R! z1wzQ<6qe^$Te=;ca*!~U%qs2?dX!cBls3r#lR*E8YO%@N}1#+x-!Gy0zX+el>R)KrKPe) ze85_LmNkvV3FP_to^`5pkgHmkSQY<7=c*STei|u*Z?-arSwI1dnPN*V zso%Qo7hINHAdxB!SVYnPSJOqHy_dYAMsiZRwmb*4H?PJ-1Yan>U7Z)rT~%$rMN5T1 zv~9|4o= zW?;-L_F9C*jy~}B`{x?oW7{vZNYcLc{@_GH`$KLxxVE^d5bxSHokmX5f(3G|u&`*_ zH4W`xPIw4}RKCeR9&*2vn`&huSWJKNBmwC9JX8$BGPG-XI{=Elcty}S}*3R%_&=@?HW?VLIn|LO~=Dx}`xq zi`O($ZzOy>{r{RmCaUoDd%@8MF!TYO*);G{ibm+ay(*>6SY_Tts7UBtw~)}ngX&;4 zH-%8N5opTA{W%gZSy#RQ8c{WD=cXyo`U;82p4p+&yWjtx{wug4i(IAMm|*Yey*Hs%vHku9YslV^ik&LYt6UL zAz&=S{+%ObYf!pXQ zwb(ZS+EN}Xa|(w<%8$I+EB5NBl{FCWLaN&Ka);CM(X||F^?WVZxro;sa~^ko)M$@( z4JY|f&0xil1;Kw@-^V|Y`#`>DUhO#X zG!^3VM@*0{-V*I-a;UL6jC~{?ut_r56lxsE%Z?oftaFbphZ<)*Wgo!+HfaY5!i<~w z*s;feb)RUx2bdef;9f@xDD{zgW2JBp6yZ}D6lVr(A&TRnakAIDZ8OsRBFH{>wlb$P8LOX^8u}5*NmYP5`BMq9I%EiY zz$_TRN3bBNT*)o|0m<+XKX9vbcKsgzfJJzSFP)rpzW5RUzy?Re>{-35WkzI*x?8tp z5;ZxACpaiwsTR4xKfo7`FGU8>+9H?49!;<=4*_e1zL8R3{Jm6)IPoSRDS8x`MjsV) zAs$w*B9#Iob{hgzcdYm@E@`8x|yy zD}KR0fC&%r0k=xAHr(KR_HOQ5;#XchKERyWoZ_d zN@I_4G$$SwQgKazxd~J$x5b;m5QnoMF8A?2%r8nlT{=+W~)gC9v!f@4FStB#68RipPQH`Qh4Z@T5d4wxtt z_z(t=%Dp=0ANYym87wEA?SqSL5*H^fvyZO;0t1)f_`Xnp^dnyafdR${d|xJjH0KqG zz(77H@to(j>xZb**EwZK3QU^6N^Lw~pS}G$+}KI>|7@t85?LiZ0+Dz?<T2fC`IRbh(fQaD4vb+ysnqh_ ztL1wnm4aUDR|4KX1>7)B_Cy-*tH@7OE5?!PJFI41a0?8`NB)m9WuG7zauu?8 zr@4soE~dyO)dl`6QwG~#iit0E4yYaZTXpMmhs?McxUoc~KN6n%Kj;~@GnUmS-TPE);jayvBuQcJ(ZQsw2a%2?hakS4R*B&qmQf%$`FKm!572a_ zwSe515_0DdwX;Yvyeq!`;e;5{;n+j}-IjIFEW=`#Yx4d<%>QldE4n6&LlY~j_a^Vg z0!(>?dW_?y#G9^K+?n0`oK%m`*J~9~N5#hvln_Il>?N(VTD-@M!$p4XQlyUTQ(pK}_r@(BU#7ru4+r6+y za{P8BWGGGR^kpHv_&C#I2vdUAsaN~o>!`M)0z0#}R_(7oJBGuVb~816*$EA{21Q{C$2 z7oyO|HL2m1Man13a72f8{y5y~2I+YmLs=F50_OhuV)Tz4TgA<#o9H`-I9JkJ?{F7wM~+wC4MQrrKQ$F4mQgpyUu4*ZST~N z8^86X+UUVf)<-!J*Is#*;pvNwROnQr5!{j{KKJWuRmnuucYT>2?e@!pV#5AELwf)Z zdk@Icv$JN)`5*mLryzgXG9>9j_9(%i&>H>Qtd62|hk({nChDZ99s00$8{3aODno*1 zzNNhZ*n?k7)-c!RK2r%2z35yqF}^pFBtu{0Gqc`ff3dhmC3EY)_WTq@Q7fpig(Xq-HjlMZkQ8kp+T{_;7njLC_5IVid}EaC zT3Q&whRTSU^#lAHkHz}A5@o!8jvSCA--+k;G`DTW$7ZjCQg&o%;Mq#P+lL3z52xcV ze8P`2(fjcGNNSK^fGEOJ8rcY9 z)?cI{Ov-^j@%76rsd@4Rske6d8~j8o=h*aE2G@J{@*#i5N5@WN&t0h52-mB(tRq){ z_trQo3BDQDnDiT07rnsCW4U-&x5~%JH_GOtH*p+Ce~veAwMgdJ0Xx$7T2dka=NyPG ztlBSf_m?}6@BR317tmzSjGj^-M&{a5gs?BiEMc0ELoZnh@h;hs4It|(O?20KtvwnRI30G2ECILH{SQyd Bpt1k} diff --git a/docs/inventories/v1.25/objects.inv b/docs/inventories/v1.25/objects.inv index e4a500a0de209a582a653d37b2a4b6e0a1a93493..0b74444cb7e1a02972fb26136e59c2526b681f78 100644 GIT binary patch delta 3804 zcmV<24kPjZj|s?-36MSrHZ5j2Ib|?qfkuT!0kuW~Uparb=0=Ss4Bzjs;4--&_8b5? zH>uRFQ{Pq&wY8hn=B|rK*jB|l1W7*p`)^b>McP-$1zw^b-aF^GC~wQLTfQH*97jc)CIYlC78!}McNog)m3GAv_Q zWNQx%Gn9XjF+%813R;OAMdS>TpM@w;M8Ob6x1-idLMZ<_2k63v{-&gC^Zl(4>;a%9!8?=;dzBlxNebImV)%k#2@qAzC0~tdh8&Ka6UJb?9 zL-i}-w;?k_!B=FoCc~+XLNM@GtQjQy% zXV33F`as6GdU19)!La^yj?je-dk-sRo8NzX^nrcFmY>2aagY-lE0m%bMX_FI5Q;`A zno%^`>q29LQVgSveimCG82F2}3}dlg7n)`$BV&wgbWiUbB1aK9L*!Vm3r!1@Q7}e< z^}5Wz&Jnt>q1RjN1@q5UB=)+j|YiVhp9LJ$T;7>0l7 zZ;q7*exa#F8J00D)E7cacq_^nBE$MZXquypoH6njzpBI;?Xhd_tMEa6KQPmYW;;O$ z3xyj0TlaG}(wq3HyY1_9U#1W0tF+UJW~&Tgq008?IJ(XM=AgB_c3YOA{*=P_}3s?|jQ}>FphkbpOoO0SfLcshBj&0>%uHZrpu|SZUW1^y0L?M2jhN2{ zF$;lNoFlpsbJ9fe*h-6$jqkLWtVYaIgSblIDo|e|=B+_cEkHFUv=KAeAf^$R22|XL zxor^C3Q&tlZ^Udjh?xn@>>Le_nEwWmbAg2;}YXkPR;05i{-}t`)czRq=>9cn~!cs2L{b5%cIo z=Gdf*F>}8qLxLSKvkt;40jthX?ufZ|5LgRfjZ1jM>^q2S1g=4~JYqf`M709dVxk@~ zLl0tR0yBfvJz|!fh#h-&F>K*CWK6#!=G{SDC2-XlLLM;_4+4K{0jyC)kC>YWQH?+~ zn6yXC)`OT;kC?d!F|EL~u+m4&<%6)ffX&a4{D}E@B6w`%#lTm7D+WY8Vul_BRRUDO>K-v? z55gJ&YtE4Qh*^Jp5ZDS}i|T#EygrDU3DgV|{%A1tB;?4^ivay)2*BJUCfSL=v1S)T z`b%1veMd~YgRn}#sxy2%Vj>;{)&f|cVdxQ4^B}Mhzy@{ph{<{o)e2OLS$xFwJ&2hJ z%nbDUhzWfVG#8*bX!;RT{2*u{Knu|QBc|zz(6Oi&gZh8#!JxlKOx%OGO5mz9j6Py2 z9|YC{Sfh>~F}V+-8i8ss>yMc32QjU{w4et_OaO$SnE=f|Gmw}P2tjiJnxEkc64Uxb z@L24NfxBZY;`pQ$a3O#T+({%R zBSPGTz+IeSDH78YA@G#|zCyi4Vv3ViF`oy%MNbs2xd6WE81ml`%&3w~v5^BQdoR;wpiwaOaVj>yX0@$LaBr!!2qGkd$1Kmkt(j)}U1!#`il*BYjh*}8L z0`-3@iK&nxbu2^1sQyM0(2*o2M?zdBa20M%64NCit`@i&^eBl5ln~SiP=lJ4#FR>i zY6YqVT}xt;B?Qd`XolLC#I#F@nhVq%^)YD>C?)7fp^PB?n0#KWq``!ga3dEog7b%0 z$dIH#X_SZ~sWC$II|fX3q`^&;kRv-Wg7km40wK;K4U(Y*9BGCTpuc4UkPm6l4gsE; z<|$A|q`@DQfFpx20`zwfKwcpYBA`SZselopzuF55f;6as08>pf6{ZZ*gTjrzDD0uR zxDb1q9z8%}+3=jZSZWo0_#F)orGxam$?TW$ht<*W3^~Y7o6N7`&z7U%L1~anGns#1 z#~+|Z!&9N4d+#UnoA}eBXm~aabg%Gaej9&g4GquZfbQR%%%8=d(?P>C6relnCiCa< zXDiS!@jY~()nxu6J|R90Q=&unz)a>}#HUK9VJd6reu&BZSMe#X=>fXVH%xULORl5y zAJZ^5F?6MNGW%tG&SDzoVuh|}P3C`B@i|&)nB5ZkWp*;Zj?a2Y!?bbGuPT%IO?)~z z8YY&4&i*Fz+xUc2G)$objgTkvXYr|Z=mEJH%w{HuB{$LOO6UQ(=F6rNiKVvDIY#IK zxw6Y90Ewl}qO*d~19JVA&Dar3okyqfpni%CUdodxwX zO#mabsnl0~0*c4-xV&5KZkuO3Q(fIub+uhq(o?EKi~bLf2>(l)=poV9?(KJv)%`vF zyWKr*Y4yAH3hiVQClEJ_cQ<9pmfWbFw}p*G)vyMYNALztD*f`6*uMdV8$uZF%4D_Pgz&I=|l6 zDTJy+a60g!wTHN*k9)BKr$fVjy4lmNro-~C+Eh~IP2j_!3yjp)rb~ZaRJyCvF5RfG z-M(tJDOcKVoBW;rva27e)CNA z?sTkZyF1d&?zCM2ANcD=qgB{Er|sg@Elz#TuJ+Ay(k@BdlB8Yo)4Qit^PKv+x^SJc zJRR?L_4@ey!@gY&HsODY5auUak@9@k{`38awOp@i88bw@Ca~7gWi?~z?XSqEY}@(r+Qh* zsBzS5(u*t-i0*gG(j9d$;5d33{35GZfZO4NtII+KG|R`i$LD{2b+KuVcU9ef{ndS( zIvMDh?{}Mx+eKYJ?H#e54e7UXdwVKxtK_@)*S~$AbWIN;eYM+^>n&CQkM%WtscN^^ zOPVzeIFZ&I%iM8fe>xs8i)zCqhOEuYrDQ# zSF7arj`j1Wk0O7scHeS^(O9TweQw@>)~c>Zn`xYK^s z^xL=9)g@_5Y4)P-%+vMG#J@cOcISMXrvA5)LzaG}yEn#Aw^y#p=54yI_sh_o_HWpR z`;0(olQA7<{1O)W-P35{xUl2#+jfF!)9}wfe;a2zZQr<~;0gZf|87>~rrvF)b`+;k ST_i&mz|Vz>q1CKVp&`Ql3M=#H^5-&kp_*I)`gbIo_-o61{e^0W-k5XT=p(( zEcH^#_7j|MtoM=+L8U|4&{m-k#USd%-Lg51MlqUUG`hDBtqqDX3^R;Db&fDB%CL-K zk-fcdn4y1!j1l_&q@az+QAExV`B{hpMHCECbO&m!B!u#hbAT>v>~BiSHs9Zdz@cCj zjH5MB^G|Tqicu8Y6Z+N~#b|~ZhC#YO7zSk+#xO`v=o?y;U>TwBYsw;s3`JxNk@dT2 zh9GhjkuyZjFY^}?#-FujKkzPW>7`!|4 zO)!6qf1D$9VdG)YNZIBOgG1m@u^BHn#iL-~QlSXN5bSc(AP|j0G=s3;IjI7n!0*0c z6n?)#BNU5LETdSoOZ1I1l#($@#=8WB$x%$sF!|6W48kZ-M!^^bbPDWSE>OgRAr`~= zPwd&&Kh6=lu(4-bDcgL{9s-AoEx){0;sAf7Z>&&?Vid)Cgh418rD#UcXpiU{8^-V3xu#91${j_fh??rzZ zLuAl?+8*=g-XcXCF3ZAFmROkiNam zH#dOuLBqSSm`*hNF7#o2oeMs!vO=NleWbZ-NFOv5dNrMBw$MJTFZ3$t0OFq2HB}fv zjfCMlAw4NJM0Bz#LPRE;lgDVZy{09#XTH~SK`mA3WqNsr1xBN`B|!smD*}I2Efmzb zh?!R+G<2_GSiiA|l10p(5}~0#6@zvg3uslud@2a41gL^VD`JKfgw+C8!|D|=OG?Cs zo>UC0{acLbQ^dR}h^qvy!i6egCKbfh0#~1*SP^rpAg~d@29&Od*;WwL3Q&t_Sj7A* zh?xn@3>3478CeiC7oa(&Y7u{Pv>;|7FpG2KEn=RPNDfV_7}@wviwRi7%qxhi1g-*= zEMhJe1l0mmW0DpzI}2hOfoVW(iM`T1*ZjW`#k_Okif`=wie?F^HTC9_h#6uKSqo%+jygun8H311ARAmJBW967 zTq|%bs+SS-${=bcP&0o_I3wnUiOkRri!pP*CqvR0FM$9*Zs79a~OhhAQpg~M4FfFX65wpldZ0LWH#jp#%M8H7~=R-NIP5tGXxuol4j4C{=TZU%vk z05+(HMod71s8*m_%uFMuq(RI~U}m7JModzJpt%6eL3@puwgy290a}1Q8!`1vgog52 z4C?O(gH9ST84cnpfve82)QIV65LgRfje2Xugf)n21gd|*Og3T)8^p8%(}HdrF{urL zW&$(=Z8u_?8wAY-XnuzOMod)`!J(`c19#_G#9<>QuR&BLP!(pi5!cx;sI;x>5*6Ku z8*Ug_+*}2_Z^WcG2&@IJ25mTE8XN>Q0@Q$h95FEtf?5G;L1T`XDhEL`0h+-M9Wi+h z!sY@tKf`~jBc{_q;6eZwxMxR9u!Fb@fx9@vyd$RELEtL^e1*Dr#I!k)8j5oK^Fpaf%<=a#8fiQ9r{2*#3P&3f}Bc}a9&|HA# zs1Ha?-4m&yyceVTA6I}*A2FE^;wpiwaLbRF-Uo5Dz}2AlM@;yGphkci)C44^076tN zP%VGx1`?A3A!sH*Gt?HOQSXzWf#4TG`Vr&2_D7?qC*cB9FM{)@9LV3JQMi+cfpQli z`Ywuzcr==G5;Ab-B1r#36~w5cQGt_yfdm%;`X3YkY8;JH9Kci4JO#>fG}>(vF!0+V zK!1N5vfgOa)Ffgcszr$YUL>fm(I~C~Of`SaRG834_X_vTqOkkstMAzQWpoc|PvgpB zvD7NM+!&3^fI;%uWcJJW@?bQs+6B2&llfJAH7^>Mrh-h5$^1ILWEG9;@Ib!|pUiLK z>+{gK8VK}D*2(-fzDfvE}YDt#aBk4an^e1cVCnF^Z0D`G)^@R{R(I@e-VG5 zf}F-lsG(nIOy*z2C#R-y@?hw0{bc^D_@u&g58Z_tCzp*S*U=egX`Brfx{o-S{W3mF zERC~YLU&sx^Q-u*m^98O2z_rjnP11}8KiNdF6f(v$^0fhfftR_1wq$gllg6Y8X+1d zUx7x*llimwq%3rg+^S^rX2g=4=wyEybdTJTWRqRQQrqYZ7<7-^cVyF3#8PL`IVRwBJYhFv<``3~`y4T)3+&GyO?d<)pq-ot`B}K`Lqjb7oOH>$M|~t zl&lZk>hfJgyO{KaPJGBO@k)Q!+hbL4%e#iR-)#@o`Srd`AynN5rz0;~dmon!anCm3 zbZFR**L&L4bXeY2n@Xy@3A{gafsy*!bg7FNihR?@q^xw!0%;?@oW)74U|?Y!qz5 z<|%DoPTk9?&)L68B2dzVgeP$5pd3cwJq%PFbFgx4U|MeEMnMZU&nUL!U@h0HTE>hKFA1!5H2LoCu3EMW!&lp5J+nRW=m<2PHC}+*V;_&SFY9u1 zn8Uw}jlm-sV`vvtZJU1sv`l>#_7eC!PmI;C(k`RIsF$Q?StJnM?Uv0#^cXOVo(4b5 zDi+`l_~7QUPyx;IaqjVHUtMgP^IcW9AAfZhr%ncX=DXcy;|@_D9`}ye&eijKxxG1+ zH&yb(yAQwrm~>4KB7L>nlf+P3(c5>w zUHs`DlF|)G`omw+Alu)51BsZTwy=#A9TYvI@!#DJ-_hM+&Wc~c% z%U|t#(WdWT3X)T~I(!!(Z5qD4xpAG&ZP6-uzdkH??#+E_kHu~n8e&$}-SZ146y%$J z+`6}X;|4`#yKRx|bU6KcSk~)*t5g{}Ou}N4Fea>a7!%yO>z(f`tk0499RJ%rxLv$T zmUnAcOmf&)%jP_CNFHv>NxYVa6~0i~jHwyciaCJbvFn wFl`$C_1EuVw$t{N2MV6x@8R!eL$2%HX6it33L;+#Zc>EkUqkf&0fOCSF}_S0&j0`b diff --git a/docs/inventories/v1.26/objects.inv b/docs/inventories/v1.26/objects.inv new file mode 100644 index 0000000000000000000000000000000000000000..5f223e1c9578861177d8b283bf259d6edabcbed6 GIT binary patch literal 153489 zcmV)1K+V4+AX9K?X>NERX>N99Zgg*Qc_4OWa&u{KZXhxWBOp+6Z)#;@bUGkqZgy{Z z3L_v^WpZMd?av*PJAarPHb0B7E zY-J#6b0A}HZE$jBb8}^6Aa!$TZf78RY-wUH3V7Pwz3X!1Mv^Z0UrzzE=^m5z49Jx< zHg>G-%xPnhEQu{v6`RSDdioCxnE;b$B7qcuB#T;Gvwe+yy?v4m_wWb=fVd(r>T`aO z2?WCZ`#b;y;_7}LJ$2poPwTvkzK!Z_-lSboSM7P4ZHnsbsoRwQ!mh<}6n}NT%A2nB z%+K?JP`PWnyz!P3*%i@0Ki17A?f&y0ZCwSb8Hj?)x_T_uNw(W;eKjc(Lj`{>Dh89~ zNuIUg$_COS#hbK!{?GsG|DK@~CCBQjbdx7dzRthQQ>w_1RX2HC@0wLUPqCD?sQuUZ zg(0>4t6D5wUV*5;J|O_&+ugRQAB!??xcs(m%?jWm)1b&O6ZN`=YxCx%SmkH*%AUP^e|E_a zuZnh6zvRu^f@e8jbWNUaE(bLV(ct%e$Eloy)Ju`So`1>OB;6J&sZi8-ggPOf2?0=Y zv0k$`CGGOez`oCLPmkwe4Dng^=-V2F_}h!!oyl z#~Sdd89P?hZ+I#2LsPl)tB0ZQ1dWHF6WVRQYI1yA7I*85GpDC!Ap)?cdR|uc4^Zk ztjt^g=8R4f6uz}xQ+!GC7uMEETD*0>8g-&fWl2n`KruQ|7uyIIOZA*>RE$3U6%P?078cWBCqyFAQb}O&+N0q z-iQCh-h4=r4+VR5ylP8VUGb8~AQPr^Dn3>{SEp84K=nP9(<%GcCsxbXtlzvzmb=w6 zYwG;xYPZ>yDHt>t$*$Cs279#r0Ue_Ql8`F36n1T{j1J5IlnTb`=T$ixX|v*g?eZ_s zGDYW zPpkcgz+S$OrLMt<`v4^0b4+J|lB?r11DJBpv6lhNp!Gh0Y4#a!7+^GF@F0%yiDCaU z*4kukoV0t2+AjOkYLnGl95Ws~U+nI4rSiH?0qQNw`ZeBF&sF_e#g9c^W^Jfuort`M2c+=*qK>bEF zfT27ku^2!LQ?Jtinu~P>_d`^t*;|!v=GM%1U4T~BD`uAX@D$sb#u(LbjCIUgtHD~O z;80JTx2trUXS4N51w!wf??5S~XH?Wu40LSjEDw}N5s=-c4Sb5)r(Ku9O`cV=7?gni zCjUpi>JIZf+I4X>{zl~6di4~nU6loGG)p1cwt@P_(yGQP{EiJoRb9Wa#>7lrD2lpj z=qb&4kW@)U(~L)2l|)TA4%MZh6?-oST38c8J}C2Ak~$5bd0thg)NFxYls|w>87`C1 z5}|YGFY;kn(2>>bQ}k^#y^1QL|54RZS9E3mKYIn;YHLv-uTB)COV-y(SpI=vpUMR; z;5_P{@`$DFHmpKHv%X@TC=@HxCA5^8;I=rIJ*z)N7k4+1Ck9Q}>tem#u3pvU08E#G z<9@kXMEBQ=hbXPGXnlWo8HwwtmUaKId&U2cbq=qRA#ga9We3xaunJ^%Efi;o|Mwzm5!%Q-Bm<1+B0G}Iyy42$dl9{;ThjjYbP;EC-K-FFJ z@;%gdQIkLB(BIF6*s`uW=qGObBaIqOF0S^SI2HOtHPJ7;oobup8P+CJRlV8NRkC4C z`*fYR<8z&2yM%bPAu$q(o@mLJYz87oX;GuE|8}+brpJcb<~&`k*k#s2?~k;Qrp)ve z!P5wOy@SFWNzs_F4*+EytB>SZNJKeYU^74u01mH#EF#mJ<{>q>k!jDQd)rRGu%xel zan!fS#lj{vdhZOl^k$u_JrFI_DC6KwH0XqsqFL?OH*MME=`)jZ*yEBURDLOMS0x6<4u|92*fe+I53LGtIrw2Wr}zCo5#`NS_q z@}h|gKK=A#fmO4q?~PvFbN-g#@{+z`<}}P9$w7b0-_G$eQ&cznKdXMDZMj<)4jVUV zw|e45M~9xAOoDXNE7mB_v*-iAgkNPlj~(yNB+ag5Ul`t~QBr(ad8y$Xf6UX2Q4<*= z%wT4@h0(Wy-*0hj<)agjPl-sF;J}YfE#7?8axDkxwMyWLDYBG33+*(;g<)!l?>}SnQUUke?5@-ue2GZa!b*3spf6l)tmmn186Kl zka+fC$-3Re3$(CVWdL39mm)J7mzDVE#-xlK>556P3l&w3%d|{c6WWCF#M%Q*yGmtE z;6|H>(0eG7BT9B6;`*p0uUPGthrp_GV{E?WieEDj_%-d|&y7eKrz1Vu+-l^9bB@U5AHY!0CX@F z6{9!3p}^?N=GR;$RqJNEN?^HGo#A#kUkZ9T&IAWM`LzhH`A`tMH$vbCG72=MyB-j} zZlwJ~JXCgM>VoLE3jBGkP6xQIl`2u!S7lm}*z zNmlw|(3wdwd})M$r_0-&k=qg zVJma^FycQahRmsaJ*}$6q?tT$LQEuv6K!@lT8u4zonWKC&{dcBx+&=sm;_D%f zEEv-MF3@m!txJ*~3Q{ zzbvAgiw}k;N23%LMkdf(&7e<`W@(qspk~x!pvmgNoOQoj7_z1O2Fhx;0qUc{shOiy zY})adJ~kr-jC;|tHfrLUDc_PO9H~y^NVOeQE2Dfa@Otz#9R1dTlZyULb^uTEE1(2ttQAxRKEu@d z{8K>V@H8r@n_^v5X_*w;e#XcFs+CcI#r5dsuAgYqf#ZUS$6d)f|BlH7ZFaT$OqZfo zJE?#Rtbs8+?hPX}O)+D82#b|IJUNm&w{V0jvqpqu3xluSkj{%-ud!W@K0$Xk{kKm4 zBCBfVK&T~9pi>smEASGZ&=QwDIQ_aSR-XM_s~G3-*UvYX{VlO}bX9H8b5^Vi zC1VuVG>Pn-px47ELyBshh}zneZebUuNa7Y{EihBon)!}Gx+I6I)x|<^e9a0;K&n>RF=*s zM4x{$jAef;n`OEZGmaLU2H6PW-NmK2Kdnvc`HmL3$OBKDso-eQ%m)iP{Bz*Izup#| z5-3>6>37(NH(uRX}3R`In;JwS#gPCc4Oas1Q?RJk3&w zBVEkuks`0+^V6auH{4`eSt2sL8GL8!^?Z$1+qD&{uH? zYREYHDNS*Zk0nb4{i3o!5iWE2%_T)q-m9@ z!%$7;elG>BXZ5iQc6{gjzXhz%q?96_PpV~}7o3HpU7ERdqf?Btoz}H7(1x2)YXB^Xt<#uO4W7ydNK21i`a<6Pn1-O(kFlVp*)8I@ZL&3q|iuD6_tvBvigg#g-^%7G9kXgqe-(a;G0zQ)J@gKwmPYRyhZ+-s$ z?&i_L+PCTDpWj~7^4Vj>gi(a*q35?>c&ewR{QdU@RdQ?ApJ|mk@V_*v|2^(A(`ex@ zi|B9hefq{ryBRIdb3p%AY*|5hgR9v&{NFLfSs2Tv*241EiJD z41L*dSy9R6>c#Ei8-H%-j9fOp`C`MhdlG9ug}T=L*I>{$szu&5P|F8rv#{wC&RVZR z4iHpX+GG(A`p}NH!29#WMijh?(2fe{I29M|Cf0KEdTIKN-*PE|I&(Wo^dUJ<=V;}^ ztzq4Of+M{;q$0SF&lc@41=L&=*Q1rs-fbO(7{_?5n^*SaB)BDEE?+-ogQ1d$^74&p z^X;JZTHo`7WDa|q3U%q1Xw_c&a^(At)Ap@efu~`<>9{{lHhezWi)@fCdP?9be6Pg3 z^(u_G@vcIPY?ZRsX8?0wW!8tpV~}!+jPt3ifjrn!mbiAb&(bsAq9&FAI5`d{v&BgO zIh%sPuEwUH8aI!pPxbq?guUd3n=DxaFZbKhH&iZn&@Wl0vXDtN*6BRalc&Eb+KP%P za;JXXgfY$X19G(~8?e1nGJ5h||5j{Ag#k&SOs#5Y%`rOb{<}esCaDrY=&k<3P$(LfAq;~H{d+8fGWP|(>zwARB7o4YAx`hxViQ+(?yNZ~mhuAPvx$qv!uR{Y} zdiD1 z@Y>Q!$OXUXYsoULvezQ(o)X!$7$pT;;Hy$NeW6R#WAWv^dwk?$JNzh|rB(rP!W3UB z^3|C4w&}!`?>c{NoP)hXl^KL~ta-PLj>CBwm8k5lKE}7~LMTD#mbL{AeuN*tZ`CV0|%u2rJCaRetDZKnzNVO9H@FQtb_c07-pR*Ayxmwf3> z3g%D(=3gAb%AAWQfss=aT0}YUa}r9+SEb%(IJq42ozv7IceQlUqkt=^b4~J8tQrp zStl8Fa8iHygugiI2K{oR3{3))OT2$u%B>-BT*u<^B#4t;i3KxHwM5_ zFafPRtpbG?!kb5xWcenoxHl1T5>CgWhFDe66(&YkA~L!+N8zTbZLn9Uo`*AaDkHwa z_tm0V1%5yJr!@I>SMyKSI4zjBzf#4g$SuAOe}!uyE#JrlPlzT|w|eE171 zu2FmmkbBMVUCuef6xcj-nl!a}H)s^8=fSkx%0z&?Zf>ip{dJeNPpBM^!Z~Uc5HC#e zrZVzBC)_p-e9{He^H$pLl2<+5i3Yf@4PW{J2X-7zS0_)*iglG|d{XTC%j$`>2to(i z#g3dbymK`cH3m>LDG>WL6X-$X8$V15X5^=ExIOd|F8DGI-NxV$RyHxX3COJlxG}$| zVBJ#3Cp3mBA;bceDwD7`;)?$R*E@N(H5-G!V!>)BjZyyf4Yxx!NiyD>P|s&s;8r+l z^IWxi@!qqY7NBsSwwi<)hPMa^ld->2HHwA|W{SGL2qY>+UEzmH0)nr|0#UR&WV zeN`41p{FgeqtCbD43a|lhiVlYz~K^F5t``|AsAr0W#J#C3nVFHM~`HNwQ5=iIzO zx7{7pEIm_pSe)Kl)W%5p>Iqi^!0#NUQ+b+PX6GR^_FhcjqiGmW!mzF&fHlSTpc~b% zzYbr@9injongk5fAq;_v;W_r(D&=$zZ@=#xjMetQ4}qK-6DfybK2kSkAHcLK62-fuaypBsLmTn@cq5$(1rPIrPUE%{9p?c7_g~*>C9_T6Fo4yF zjVAeE_J=}>=tp?@2t{zf)?~Q;J(`#UutSU`c_^Iur-fuuGzh|bh!5i!OFcgX z_o+UO3I25FE~qfe2d$|PYJRKh=RDuCs!SR>N}5B?S1Wlis_UQdlJ^%3;w^)hI|{!{ zKJO^hcmQ9wo{4W{KXMQcasKOL)6fT0=OK0dtQZ`}c41}pW%(vv&xVGiEov6G2F%SF^^h0G{v zwza;h`*$4I*UYy~8-e{mn->oNAUtevr*_(K@Jb!+uyqpj9RlcqoAsQ%L3{<%Ai>+l zd5UGEKD-dYg-PsrPm>hSkr<>XAzzG2xyme|R)yj~zU_Fs#eDYtZ4Z2yJT(pWXYavYMhXz0i1V~8 z>sKB)fFeh7zwFcG#J|yTFc{%)k>%n6?Rfx8)=dx)k$~uh2P~Kd4_#up0=I3J${g|kF`FUsj&|iiHV}!ZFjS)laEHfR_l|2k`=p0ndlpzwTxE*b{A#ICW&S0<8 z-Ub+S4Tvumw{u8iP7#v^;~ccO|E$CJklfKsQ-h4z3qx+HqN@>I^$3+?BJlJEfdN-( zB^QOrlzr@Ss-<3qNZi8$?)=KnZ0!uCO0zAFSBLp|RxYLS?AV81s5TmJDwpa~ zZ<6r$oh3I)Rp1D{t;-h(fUw>28V(N<^;(D}VLG`zcopa2#MMyptF+<$9XvP;d8bZJ zeig&zz1Xi|3MymY|Hqo|yO5`cu_Q=8`LpZugqp@9Unkh7dZc*WBKFGoT8K&EdWWO4 z6yjUv@qS&lY4^k{lJi|tBwfDQmK1DIJ?E8+k3};S(rjx}L=SY8Z-9!NvpWdZ#Xo?q52z_Rj>7~fj32wYfX-mEsZ)YzQ1b+groQWa3 zk4XoIBf~%6T&kbTo2y9Xa}V)zQh2wMKBnv-!QVCyH40~(cVQGUQ^YfQ5@uKHM^-ED zniWm&^yl7m-AWX)croG^JxOx=*wdRo=1r4l34C$#w&gS3Us!@EN5H7zA(5u{M&^$w zzuR=hXX}gwO1E z;k-2B!e2O~M)%j>S*h?4N=0_Q+Z8x&4ZnUgETYs%_{{2Q8#u3O64v`K@X62W*GhWD z5|usLsK4UJ^DrpX5S2NU&rus>B(JQVa_*z7KbvQaU7J`?#8P4YcpF5N$M=JIw!Hs< zOcLkO4$&>blJkp^xM;NnqTD~#ZFgQn?aJQAw0TP)0q^r9(a(rkO3A$x)Y_MLG5QfM z!knFRc4`a>M}tNF)6{^^>b}|)%OF#~3KJE3O}{A=EVK88s zL(&%2b9=sS((M!LnDT%G8Tb1?Nb=!2Lm^S6OaE|cqJnErzx`HO!7f^o;tG9+=kW|wKlIPpM+}(gXo3%Np{QoI7`-OS#A5cKk{TIBa5znw=#ZJqX+rMEco2OEBoqh z1D{K}C98by>pfB6{bWv|=MeW^^W~*tFf9)bO?_usjN03L#af$t5mZ$ONAi%;?$2)a z`#hHKo!RykIZav3qs3ar1*>KI)$ak_hxvM`c<*lBzrT(y*^|lI=GY(Vm(`oi!7YX< zPUYdOP$Tvoig`Ysnmne^>+ssfGE1ehw=t!Pq3DBC%`srz*2xlvA`fYTU1@K`4~tI` z<_k85T0$c38272C`UX_yU$(VhD{ZFIp<6%2%kW)oZJUkY@Snz|-c0sxbu_liuFogU zzGLEZwm)T;GOCDHd+-$nB3jo;z4ab*4N|4}K9tS$Pg~cxY7YTrapTWSacpJdI7R-y>9Y#3X@2E0J(*{?N{d8KF}&p!pMxo2AwQ3pfTKjG;{QLzLGdDSOB50BGC@G<=ZUTX-lvnf` z@E7l02g}RMN9~Zf2LVlPb*NAG)m~V^c_teme_sd$PbRHSc-?n=kkrcvP4+>ED4s!A zl$SPZJjVG=S^FVV(YT`2@3z3?92BsxZE1aD3MdR>o)wvIPA9yNl#MT2~&#QH0{2qu`UX( zXx;GVlo>Q`@pkAp^oxL-Mq^sx$wh?lV&?C3fx&wA(B#v$D4A1coWT zaPWkBgY}t}=hqVmSEPC)DY=D5v$`_BuHDCW(Ez+|N>%I&Pjt zWZ;+SAzf=Bpw4|COxACzg(Eh zs#mqEiY@@T&)%c*uFwNC>OWV7u@u{UlR_xdgtb&#XXD5yAX?b`T13}Opk4i(SIKe* za{x}jX=Z9|LVCp0bwDluZ_rFvNfjIlSvgt{1B^?UUw4f3+XXD6`OigJwo%GdG9y@0 zlc!n4ARwKc=3?Y&JjPFx`ADTqmwBm>;g^JWBEbF^ekCR$Ie63#wIy;a-KMu6aF<6K zkhs@%AqU{s{eb@jQt^9eL+>5($|Ea!tf=jY^sW)dd|*(%DVC0cg)HJ%2QRc-(r<1m zaoATS-5uCK2OF?-kdPsknsBF;I0MXc_BVQPA&ExK?`}k%q}^>{bSuxq+udr!G`Z;Y z7wOm$64y{U1y+8{*%zakyp-e1^v>k$HTbEjUn|kGeoxoGvT#+4(8d-^P+p6r4wDhh z>M4Ql{7aF)(u14elgoW^53vh&=j-!<%z3?po^iS?N?7gRYgj|OkuzfXW09Yp;us%e z!cd=Ko@D05KBz^VvAXz}#3aa_IOr$bEZlHyy2znW8A}C_NlDLOsNy1RaWq|D=ukHHS!*by(tu8C1{3a5K8ZjgAc?}~KbEH@z(R{x=$JKkhuiKjG*01n_ zyd9Hg{Td?#ji0jy-)dN%I(mfQ`8Av_X0x?hlp34sSMIO)5K%@ng?o8ArqQHD z<@IXT+nld>ZO?^}7F;O2qJuFYMXS_}3%OtiqO|^2%YkPU!$@DQ#@mLVyDV~B`6a8Y zJq5=Ya4xSFb~0IBcIipzaHK2zk1OB4xVYsfDk#9Yc9AJhLTW;!zuc>*xx9uHKFCU=bxOE&eT_875^cZ84T7=6dDg%fX7}DGk1U zGt0Ma7?obmzymYHs917gXL*(7)oR)>rE!tf6RV%wLx1{c@Pm}os0TkA(O8%D5~452 zHG9F0_=yI?CyD2aAs;nye!!Hdk?ZiL-zoBB_0ntu?p80P<$+gYP)Q!Ke%_VfIgk&C zf$vHP&3xC?>jwND{Ct1+jgeK7=SLtGGp*NOm=uqQ)B7Sf^Qf2L45@g@+hg!iw11L^ zW-`sLJPr@8_#Xb7(Q#U=OExw8d>(@Y*B^8J4|o#c$(T~JU4_@MAf+V$+Mp4jK}nWP zk*)pOj{$J8REv!dai$|cOZeIY6Od|=Kr5GH4hTG@+wEJDzvZkl2^(kg>NTSJSgQXK zPNQpaN|QfhN=lDAk;CNP#6}%as-B6Uo$h7C!+vyXjyU6Iprs=HXI{ju&MIv#^}-S&Go ztlYBxy%q|Upku15p{-T?p=sEQBQw#BO3Gy7r_#=IW6mU5e-u58ke$ z=shmnld^O(k(N-2MD%04s-C27vfOVGKrn|`>ZUHQIJfb^jZ`uM@G)2+LM#!88*P=R zR2@SRoH*UG_H_rn{Wg|h;!}>1k72J6UC@!0E&Ah8>>-etJk`-bSEdQ~ z4}L^S2onpKd+z3zJ0%|s)oCfR2O#i0AQ3YiQYn2g7@mE)lu(dw5efVVRALShONc8T zic_4oHhNFFXRkJ#mQXE%H`2L{jo7?!1U5qTJ>9>TB8|w!jEsP0qb6JeAA_6JeJ2h) z9xU=Ce0Vlo@jW>z=8MOz&k(QP+ZAh^AB7Lr?Ogpm*64WbX4GLw!J{$a>Nz7k9x*Da zqASu;I`#+Ga$4k^&GUZpTIoxJA7`f1+>Vx-N3ypV_mKG<3Vt^d8gQ1~o;NVKEY13-}*eWT2Y!Ltp`tz*YBNFst--$ zPQ#Tw0x%m(QRhz0JVx|bp#LIkK}e1$*~f}ax@OPO$>|g92mJBjDLZW$GmISsu9qHH zbW)pFJ;yoA?wd+M_TH4lx%P;MiALW43Ct4@7!A*Bt>v-S8aa>&84pwfgWa3|e8n-? zl|65(Lx&M@n$$(y$2YJXGV`e$A^LLgMVUl4K-pL)JApbP zbb}X7%>4(dEo_AnD`kl!3zVo{^7Z^hWgG?F)YW>%whk22azcF{;QwS#N0oQ4b@P0z zL1$7g$DBX%bM^*)e}4P#-x!9TS?y95OmUWXv}VV@&LCMG8M+krYCrLmNT#@mBx$*Z ze&f^TNR;9p;QAlLQYJas(K^zE+^VJP{8%&zDOQl)(?ZufndIc_x4M2FL(^LODU0y6 z-9bczdU-!>i@M`xATe8{>@| z8C%$YMIIRjEB1YN#fB_H-okz?^6fBKyiZ_vutQ6@oyc)ly1Iy(mJ^LFU_HNeGiTd{ zkknEYTJPv!gbMx<4(Cn*wcXcc9=%IjaH7SX3!yKxy3|1XFT@F;QQtlM*CEO05~47&PNDC5-!OdnKoSbA|BbDT$OGEZxCs4NHq!lxexD678|*4S zIIn^BOUtT0F?MnUe@$ZCLs2bOuB9gbP2{{ksG21iEd^319RJ(aH&Q1mQ9Rmv*QFC# z9U00LXA}SbGQ~wH!Efm6Q z(Ml;eHHU`eV-e(0#KH+Ry}!9zMBo1V{~3R$!Kzm+y4c^AUSkhFprA6ADnDH=?w|tM zYW&Ip0fw<#mf+8|uDgSAAg!^~`oI$+kuYWgHC)?aEVD4Dx4op6ORsk!4@cM(QP*dG zU)ZoGQ%pPiASf?C(pDba`@2`+^E4^IU1Mm$rOpR=q;qXC)5V4=W`y_ftU-Y|Lhm|> zX^~0e1OL_IiTJE={GW5Lx+{T8Kdl(H@(T;!X5NAS2d?YjU1D3CCdv@A%S zo(EqD7cp7EWmlBbnh95;or`N6<`2<3I{!-N@PFW%U+41wUwC3Hr7R%;Rv6J~I9jBy zN*>c+&xeIIcx}0&;lPlrL4K@XqpXf@WNGISUHY?;0c-f6WqcN-ost>Ny-!8##CYad z>X#g$0ahYGo<(>QX}!7PPQT*#oMVA=&wus&TuU{kH+Cvvp`}@np}F0^P1a~&X`-v* zx`|D#mLadnF7xwV#b6}+4{94lH*qo4t8pOshvA@R)Z~vj?3c>F8U$TQCV^gtnoVH^ z&e5NRC{-ogN<518f|bA+Q4WCfLCU&m56(f3e)kZUJSF-hQZ&S}{4w2?9R;E}JS);q z5ow|;ej2^!r#x1ihSss@kY0l4F4#=yy|Z=^f$d0iddKYo=#Q7TpBE3;_sNIrhl{I= zhl{U2UVOM-++AE=|LLde|N1|^`P1M3XyM&&EG@F*k?wsxGn7AGe){ffsF;MN}rYchntTVczngm|K@gKu_r4~tHyQw0oe~g=;6Im z(k%3PXFJ;*X5wI=<|WCi<2@LPCmw+0Lpd)oK>qRS(=Q(v5BJv>ACkLk_IzJn-#;Wj zC)XcskBq?v&3=UATfsq1b}Sw)?jQUt;ahqkYMNNH!jju@SL`?jNo=`_g|Jn-9_}wb zF77@tyWX=`;Pd@;!jc!CKHALF&R!nU1_AKLtIzlNhT7PWBcr2N-@>I)UwvQw7pt#t zF6UL>=GE#4S$>xA3)LkMG0v*Nowx-rFfQ`L9P~2D<-vfBo05|Mutqa`<>eRB;9UHnq1R_RJ0-wCD+Oj4hQeAQ#{2!VLa|&4Kg+) zVT$RckMSkqi#Xzp>1j;EpZHSAS{pnMcS002%S$d{d5MXVSf8i{eaoA6uOb^Ovy&HV z7uAq*z+e)#&tbyTYA@1HD=(T1W$Xys$qQ#0t-iXC#TBO^$Bw*VLh>oh23TU9{^qVs zoXn?{iitcdv`&AciUw9yi{}}~Wre3y8IFEf4VmoFuB8k#VX(H+VZ>;$r6)bt?w8{e z_Sz`pX$%g{YlrbqU*qBB15 zQ3$6qUruhxLBB+y7TG^SoK2WHVhm>L;P1$$h%p%izS`%Ga32#BZ$(Gu&r!ig5qEAe z^ehr=#9^z{P-8r*(_o9lHU&AJ(iXqvr+r<-b_O}t!YPO?4fn!X@PZ|W+b3KjochU= zo)O^Tlg<(LeEzu4y;GaQQMse90eegjHe%wE@yPVaSz|a)_UUJ@T(aC{>-@A|?wGJ+ zf&IYGBYqxkN_k@X4DaD~Vdfb`M~f^wtG#Oe^TO9OyXH^&ZB7%pY08iQ|50%Gk_oYz z8si!FoWwLOy*5c^e|tyXAXx4(xR`(?g)u0+Vt-1IVBwBa(8zYnWvyQnr$bjlJgS1j zvjHuA$%aJ7Lru)So} z6x*(D5?-?fEXnDW;@}%hZ)o{pEHlU#3veFGAnw5-=mIV#>gq^j;t>v_J`)6B5C*Bk z0=_TTNM?LK;H-B3F}#P;@mFr~NQG{ntS1>|8>XJZV9%@pzI$ql$8Hur7qt=~?GOBi zO5~DXm5k6qdCFwm>$`Y1`WYf)7#gmH2cf}Z4G27MtO@tV>cx$c?u)HRn+U**#ad{X zM@a{#B&>yo$i= zbUlM27xds26T}bv$3ajV=g3qiiQL084JUhSBRvJ5DPj#3?KHEQlAIcr?f*+A7hUmG zD>3fNf#qJEh9Y<8fOzP*0go~pQF5raO54KY85`YFRn|g|yy6iChSK`>>|3a)z<1y! zCR^FHDFki!Z8pPR-px|MQsWHu$u_3#EPkz%)i~x+Rcv=6;9iR9V({jz%kP|#56R2sDi!E<@ zfny5{L6>G3Y~MQ~X9J+=f*=3Kx?$F$%}1xMVrkO+2#c71M`tU6#tD8rdnZHChnj?q zIRfD+qVbB$1PU~rXQJ|iHMB=GK#ylg_b2~YHzy$tIHU6*l6|~Pey2`Fx(}d9xN#Wm zu7S>SmeAJy<5z)0_5?4U4FMlu#F5|25zsinpXbyg=aG0K&^RH0C(dY{geW3SQe0H) z4ja?L3Cj_wj*%uQE~;Yu?WpY_1w=j3bE7y0`L+ygS}KwjS`+{Q^;7HvJl@n<@0!cS z%wvcAAqa{M^D0kD3Ztv$K04_Em`WX=$;yX&lLb!`WwvJ#^zk^>nk@?V$`*wYjXugY z9gFx1q3j02HllstZ90~SoWG3Z?j+2byN|4tla}?5x-OjaMBbA&$;CbH15b$-PsJl7 zoG-+s8CE3ilVf)Nn6|_mU0BqTMbR?vUh}*{|3zYTKs(DiIQsppLv(4Kyo2egsad0E zvn#t|Tjov^-8&t0FNtP}{fNANM}rnzv=`AkWAG54Hy6<^6fnZc(-7Uxl;X%9n<)j5 z9Wx!G+nHAI@hOYn@&ty_kH%EO#0VqR(2_W@D{jcf&Kz02kkPK)|;0;WYz(xNdRD{poJ22?y@-u zOTMKB7np1h!1w#p5dNMoB>H|X=UsX=T%N9#9gV9i^#+BM+o?=6go zPJ?dYv&t(LLa_>aL5Z-GT_hSvZ;BQigrgVUrY=JnkHkIbl0+;3FGPwAPNXIJ+9qYX zS!QV67b;3>9uEqI!i@REIV zUyF>j95kUcU1UeXPNFhD#nXHz&UrXZAA({1r2`SQcBUlJ-Q%P0r&sbq88;oS`W^mg z!Tr{t5B-~hh!dJC#+By6|FjlAgF&=<8%Tkq+$=H}He>29uRg}Na6nX>D9}2sv zh&9DxUAixL5HW;yt<*zY*s?wD?7_Vdv-+`9*@Gt``o}agXS| zDKAB;?%!#>Y?;-7pJ#_pvwci&#!E7foORGqvQsvj&go!G<=s4$-KPYp?mV@0Le{G- zRu+M;krZ=MV>x3E@WbWZIO3b7#p1cqbjS*2+?3P}rh4$XEo~Hply808*=qY5+tqSQ z;?mr}R$!X^g{9>CvK^}NV~vtAN$PE&i%wfc4g!wjVqj_k`WsMSfqeXz#ix%^pJ*mM zquS}^{0MRgEK^6VgPLFSBASVWBwR_@RzJhzpkbP<6!+%vnnMu$Ar{2L3Fxq>63QrRW_t zj8ZS>Bvfo;0daE|J){F)v;#8#c2KvcLTDF#42f89f(eDMp6FNU4%e zUXWP~CPzQ96ETYHBsIU0I-B62p^8C7^8x2^166ASe7JXM=;uyIIr&QUw{)N?>ncn&d ztyz7Q{f+bKx0JiU&kYlBGhs?^cc}?bbRk}wbh{0G%RTX_&k+j@-rtBW?ruC;CF!;R z*HX#yt;^d}aEuuu_>+nWG4I~N!Pnmy87oV8j#`$AD>H$kxG<}i&=<89EwxdON|G8i z0)v`f9^_=R>8SJsiXB`!ci^M&^rlZh9ciN$#Kn}pwIK29va+?VLZ?{)tZ(jwx7bMyNyV`D859z-kI4+&n<-eO>0!JJvACM3`xsJdD1+e|K^D zjcrI0QT8l0U!v2p63aD2(_T-}=}%e1$>>9Q$V%rzSp(jCAY?VxZj)z6WYfDJ6jF?1vcp>(`MKK4{Xe%QstS8||v2zKeQ5 zZK@A%SQp5c)lDn|44i_!4%Twb$CCWZmi55$EN*_4(I__UDU2~7+b{c=RBYH6p@24& zI{AkYGEZTb4c;Si=EGR_ef>95@5gxv zs7>|a%{mJZj0m>l<|nCQdqiycOp|?RpQu@*gVUsWF@|Re-89f#0yG;=@^l-J?U%z$ zf-n58#a+QCHfextzwB$$8(;i$nosr+P@C${`;?R?Z2gGJ(9{s`5$+B?Zl|DaK(=2F zGKsZBV5Suc}k;e z*m^b09{!UN?^E9=4ZKGlt%a~MWZw9qD(SQYmV-!`1#630hZfskcM!QUj1J1UiEz~m z{LzE63ORq=m8@v^Zu>AABWX7w5{L!df)XN1(3AWUI8)68+*Fv-#ovC=0(NQOW@aYf z1O%lI>DB~@P8TYgt#6J3k8T#KSU`Y_g({*mBQ5@F1uquxrPQm}&sm69JjCyg;4qYR z(vh_Zhx#mrhaoW5N=xoFZ8u?*K*C7`)&9c%2~g>g*vHAwD8l7Lc7Q8=4ol59!qfIG zr3ezzAC8AR6_K|+A4^RV26Ut|9DyN7jD4((web5=R1n>*vu{nda2qPIGUIow2gHen z8*yr+8M4;bR)`+UOec)J7ydd~yT`bX&D}UOKjAJ{T4e3q(pFV?u~l8X_X{?za>le8 zj1ZgII#!hUur(|#F|)NSFPRYK8{|d!*qKy^O6)8#Z!7`taI5_tof>I&WxIE+FwkKq z$+9YS0)%KbR|z7Nbilhb_0}aBh(OU?o2)iRfhHyRVL+WPVegsDqg~?)c@W1PnVns=Rxxo9D1*)ktW3K43uCWSiplzz%C#?GoEdg!X3E`0zFUI{A7mloKaN%qAhO5q8%=aqr1Mv4Q@~mjU z50}sIu$l2li4S2e(hmt=;dwM3`JK?+Etj?1Et7?6ufU5YWEb9;;CSeR$wF_RCtyBF z@B5ywp6YqzK+=a5B3!e7MhZSIqbxjZPK739JiSL5IR>pNJv3>{vRpG^amCxC)WKPc zD+d2kW9rZ8<8zuUoBG+ebwB2*YVe_o`C~%-2~MJS{N&ME+YfMfp3@ViVudeNkLAU9 zkD6&O!oGDA2ifH9(>^3Y-k&0(@kihT9Qr%}h_L_d4<|a~)+0z&JN7akfhx!RSSvf9 z#ti2TWsVv8j0TQ*dJkl}DDU%59fJ6HgY1~^rmog|wVM^8d39E1~sEpm;==n^NG)|D-=xE zCeEWU=UY7AU0g=j6$S~%dYaV~59oJ`@A%App6R{bzWE0DzJ1%e?O>1cJ}=-f5u11dpst4+u-}K?A!MV>pWOJE|{t9Q<8HD29;p=uK7a} zc5R?HGj4ZFW?Q%8K1MOxa5r}dY%Xt=L)>#Pv!DI(Fm14(aPx8oAf_Pg(=)+-0S-fd zejV22S+UorLs8(x7P8Yv;~tLfUyOnFLv%Uj)AjX?Z!>{MW89$EpM5I1`9|=FjmUti zJV!S~#rl0Hd)Lh9Q*QI?UQhEthZj?l5YIH8e)y~7u^z(gQ?7dw_AcGFDiY!5U=~wF zfe%fVJr;QVx@Jm1A0CU3H#V}tA$IK4y~FM34ADp3VFpB*E%MITtZud4#|$a%Lz`tL zny_5VkCrT?y&jZ#G-VkJZ1Dw@T=LRhQTMoeypJ=<`@V$=N7jzYyQEy6mM5ad3>BRJ zpkKr~VhtQjj_SwgiSrPBQsTo?icsnO zy{>=4b9(`fk@gur+@E#OFya{QFrAVE7%{u6GR7`hpArHK?@?PHz#L+s%yHFQ*o_s@ z1&!j^gP~^1h{6vHWY%F2#DJ2pq z7Ru0QBF}%Pq6;l10eKH+5g*f zsVTc#4oRz5do|uZGa*Qx!=oy zG91A`gk^wx6qC{ql32vuz(=1eRwb>T(q&N=-P?p!*OmJh2PlC8czC!K02AlUVsn%K zcJW@t0Cnu;S$GpsrlKP9Ha&bG)QPAgkkQZ>6L2p~Q7V7k4lctI6X8x3oht0Fo;znC zrwrgd3)^|~KiuL~Q)t4Jn{QE3%edjL?W>XHt|SXsVp z@|C*=_k&ZlQCuo|bc8!`9+FDr1OLhNsE>&$v0D#7ObP4CE&a>gBQDy-5ULHl zE-ia~2hASAqv^6k6cDPR8h+a3jlRBrP__pP?DlQ51lQc`8&$Ep_j-_;QYeipd_KTTX4kM!!8sQm5adaHD^PxX7kELp zr+T3j8bqn$o@^dd_Kd%*%7B&t%RS`6l*LjRxWtZrzPybVI334**uVve&7Pfv)yU5f z-!|Po$%X1xFQ@eq453Ecd9UMqq@!0qch2RFY+yhECM z#Nr*9vQ6J`XyGunXta44LsA-R?&ylQIeshR99GyDLygE=SH&6G1r6XIhpuJKnVGh8 zF#AYcO~cq@rp_oSE`CN=kz;PUpebF=R0=niJWMqtd47{o`dpPB5`fFXNhNn1kIIok z)Zsd49_<>E1*KiKo#{9YB&8#}{&>KNrB7X^fvc9%;5JQ1C&?-iEiIf`FY6>0v%GX1 zU42}PDiSR-NDP?T92Xgix_Ln&M}eXnOkvtt;iKlgb8269u|T860-hWL=D+r4XsRY& zc!(10)$ZM?C1tUqOoTwuboH0WDZY$;(Q5Mize^y)}Vr#!UCgEL(LH~C}U$*(oTNc8Ei*fiLxC$=&y8zg(k^KZ|-6*0*# zQ)N%GTUDD@MOXYblSnVE+*f`r>aY2dX=hMqr@-uDvE7yGD4b1=dKN}NJyW0S-nrLs z{fb2wd1=VERAwjd4FI0!TkStzTYC(WMMbEA$N(?jnNF@ z-SiI%k zcN2Pf3Vhx{fWwMD!;dv<>}dx4Ar`p#!57^m)=b^Pk~BbA?V1MHP%52i2QLpbqK(co zKtoDo@F2G-TCr9Iw%^TSrq>~8(N(d*&&}zzP0Dn$%+fhz#=^wnmqm1o$zJdj4mN3v zbp;FV=JC*L=?UFa(AfpQ?qs%1tL(MNx~IYTTnH<@2F~c6xHcFdwdmSCKFZ;QAQGw| zVPldL5*ykx5LG$qY(UD;7nnsntCF&=w{wt(q6B{RF}`Jq)M|3$^I-JdEYnA_qOV1% z!yz`i8i>nYH#0d5gnJS^*#jQibqR!JOO{Q+y1=t|n6$_qC|r-%_ji}kJG}0}tZ8Ra zY1$$7r~8E)tZg3Wfc0Vowzz-q24g+6dNcO@8icFWKosy=-){4K4k4889#5i1oYT2d zF1Sfpwk8I#acyywqg}e)iVnSXwwz0;7gbo)SMQv_V|St<$XZvVcBig(Wvr}LGkGYY zBN}(?3WTBZ_*(Pr^Wg+kIT}aOZhW_B=m(;b*k*x_mdvv{#hx^%ss*SVeua{@Pa(F0F*!vEv7oBb^s%X09) z{PHmR7CA{g7iB323Bq}3#5t9p*ab1T;3NUxu3b8-x*v(sC^#j$f%V>zg9un0ESo$& zTeR)w)DgWD!S38cOf@_{-^~CU2-3hmJapf=A*&?I(GSuNRto2HzECC|M5K!O)dg$j>< z^H=lzZguvLVjT*678<)?ld>X#bH&S14P+u<+9&v&70e@%2Kb}QM8${U@ zEO~xR-g(^iXt#w0;hja#6wKEar*~e|-Tml=wQ;yae z_~+0H9VGJdTtACQ@PLfg_vjp-$wr}rk@1e-! z9rLzb@tzx@=TPW(P*3)j>)p4c_(@L6daWFqkv)WIMLPe~-`4Ba7vKFdI7O}RhEdw_ zNZaKcPqt0sN5 zuLmVxBHrGZoW7Qv`uoEfC=hfzmsCGHc@RQV2GeaJ`Marw8FF zA}xzd-9@gXSm1;7s5{c5pJlRzTz}}rp+BTIM4o>&-|Ms=vX}2_lOK4q0}d~6k9LdI z-HsQC!Ap^o^q7;p)P`X~%24*Zv;;pFjU5R+)xoTeY=ZdWP9$kH93{(n z2|NDq)Ej(wO3x0Yjs2E7{r7r9j$Hnl9s8Oc1$tu%r-C-ufflzd-Z;{kjLvb4hhLZ= zpn)?Po8y?VD=t3wFr0~Sw!Q&Zh71dg$xRaIOoz*}Ox?h(mnm%_lYU1+jYkoXtk#I8^T<)Es5P zf~}88;fOn-37?Q|wx#u}I?gI)GWpq8uDA?dn$;K%j*t-(=>jGIvfZll15>lh=7>!B z${kk017jS7VH#xND@W)Q4-mg`3XvPOGO+}EUwxXHni%r@D|@g958Y~52xANkgGXnx zdY$WMKiAGW4S3)Oh6fT3&qj*HT1V-AN_1~lwpE2TDY1Bl&yQ%#yPTS5)<@^7qFL=& zZN6l=o~_khuqxuqQzJ~BK1xolPpdsBJ*}o2kY;djc4vJx3~!;nC+pPZ?BDuou=RSK zIvgXOe)5j77NZh(ed!dn)}LeP07HkbsnH^JuiSX=oFAS;!}P%qV1rZIGdQ4}GW+d| z$J9>;=lt_9E>KaSIQlRF9P~Pg79OnDTJyoO4hv{I#tCs$a&hV zo@7|19)&en%R+HUEL6s%G8QT?`)kn;{95#$U;7HYc*Gu_W8MBg{^bu!#Q;I9ha;-{ z=v80$wt0-bgOa%&i7@S>*r)(7uRx}%Wr2Xw*#$Dys}zXp4`eok$*IT~YFpNC811Zk z+vZ7D@0Mk*^IgrK+fjZ7qX=;QeU=!q#5{)xc23Qdt`;qpe}hFH6)gq$H{6-WC1}!d z%n)-25LEK(`C$ely=cqcaFybOkcfT5dhuE>$_8FjF+DvI^`@(puq6|gbyfN>OIW)H z9*H9}j0KE_skkJDg}+jQN)1?1MVdYS@33slgZP@z1 zLas$W{{o)j5h*65SR%z=&iG0kSmwl^k8Ro% zEnLEnOTD#y5~M6x&?~aqKpz2_o1T{eY|LQMARWg`nVC;MoPFAQS_ck%c!fvSSt{BC zTR4V})rwcg^#>-vYhJR_SPheOKLii=5#0wC6}*9bZt6@uF+PrDwbTVx_lkY2mbvl1 zPH4mv8fpm)Y~U^sZ!kZfIy+NO>`(PT(<^wK&RDsM10G3Up5`wwCVqnQ5PG(^DL8~l)dYfSz$nxJVsj%SWJqcS%f<`=8I#1z425|2D ztkQUljBSj9UbfSC%m_3VpaWBh;KjD!hH>8?aPIi7_LkYY`-e1U(AA&vH+Szxd3oa+ z16=doH>IT8ZC+)m`;jDLmT4Pi!vr|i0PC#dE&889kdFjV$^{Bucs_)DHqPCr6ZWaI zccFInKn8nQ23LwoX0rZ5v}b^iD;RN zVI6)q+h3TH&%F}g`--F8f)`y2cqd1$cR~t8gB;u5QuOE}jHim!`?Vc_ZPT{xo|<~M ze(K7X_6*P20R~UFnXgW*y9)k-4k-oD$4WiE>KQRv=k4)m(6A}(LjJ|~`s@XL9{_zvjAoEC)!h&(9 zOvn1TPe61wP$aS|k;%+mZpL9!d`NWllol0MDQCSGEZ&lGRd_-f7$%*aVQ8N!uR6bn zQ3hpZ*T#fmEg`i4RCD-%?IlQK^>)^8cH*s1dwr*>lE*l|HK!PYZI{PAq{&dRbBa%= z*{u@u8_?c?dGmFXZt_jh8KnKk0RqW%V?O9A}O&s z5UZF~TIR4Kr)fnpZzfS*yyUutLK5ti%6$wKnX%fjGl_d9Y6+;eEI`8pY7jded25Xt zHF(K>!@+G1egomx-Ef6$I6V*9eGG<-`2B-lO?y<}gG-(>S5tFHc9#_$?f(|-aZF>G z+5h7jGR2NZ*k&Uk-z}FFEpJS_6tb2AfE_AW0_=;!HxJSzi3~N)Yd)*#pagBl;GcmN zFbZKo2nuvtV?iM%3WE|5#9`A9dPSt0eFj5lS3sQHeR|lQ^RF9OE>49f!%3_r7Nm+nDH!0S@Tq)41Npx_b}8q0fL)AY%LU z@Uc8`Sr|w-i$QGi9=;kLCGf&y+dkE8ch;3H^bDB-SZe6{f*;6wg+Qz!ptYO=@W)t@Gd0^?haN86Z+pil!(pY5XQ!y+bKpHnta+!2$|uYOjy29)Z!zna ztkC*;n^&v6YIm)A1!C^zU+ktpK5TGF4SoiJ9GWyjjt9sC^90#XOym+pM2iZpTyhZ) zV}C2Q&qXDBUnQTw5IfjBM#rSlTvM3ADtc^0*BbtJ6eXiCh3GYjng`3U2{KqA1EZBP z0hTnrVUo>uP%>*t54JuA3woyLv5JniPdS6jd+y|jwbQPVGbDwWExrSfWx8(9c2!DF zeDjQ5O55XGJ|GaQ1Y#m!FkXCw-XrtRD)W03^s=8-hK%r*bzoqTF*`fa4Ikjf3a-K0 zhug(n&y$@wZn82dXZggULCTyV?bAuJyd$^axQ58XIAsP={hE%s7{8d~jKn0TD4fR6 zpq*Q%wL}HA znnQcX3^vJ4`t2FDJo^4BwA!BJX-mClt%c)qc#20x+6L`(z%2rXHq(p3_d`jkTvS4UJ@&5g#xmF!2!wqQAC0H^8 z%HpiwlLne8ku~EWkg0J@a#%`2}I~> ziPv0*VQAB!aLpQhBXxr$mZxMDL>W?WpkP#NsB=!Cf#9gf21$~it<*F|{s2jreV$uT za?oP4Zj<@vD7oRzn)!ui*>Yo0`Wq{hky1>WQ)nPKDm2tJr_exfRA`vwRZ<$G{tiig z#Up(AP!*Zwut`YqB}{rAF9Oo~?D!Xka*L(ppg0p)DOa$Iay zW3T$N{?OfY80o}_@;DC(2U%XGZ_43}vO{4gHm-SQiJ>sgMoKR6h?b+Uq+##j;0g`v z5BdKIZa#KM{^h zjM$kNjxR5z|m%Ksf`r#nfxKvtn!meV%b_7 z5{?zZ)`HZS{!S$E`I&3f6j-7V%7|IEqLot6(Jrbr&^cSy*;~tA{@}tCI%%yG=}b`R ztWsp@uzY1jE2tEb@qnbY098xxq!K8uvv*9%f+;y%*-=CMpS|=(H0PEd?>5XRLD;^R z{G+X_gngl_7MssvMd=93o23_3>2@TepbnX-Wc@8y(Y0MaY#QN z)3+A2saRn1;$K+uL!bJ&DL<3m1=AZ+_Rxkj^)}8xG^~uF8O2@Z-e4?^3HjBMVCahhIs-_-Um1YYrQRhABpz@h2r7^NG)U2|j9kN~B zFL>zNGAdDcWZh8GBtx9 z4G?A(8E6pY0|E-ic(hE`*UX>TIThHMZ4s?WUk!6=-{Eu&9tqC^>^S)O;u>&@29D;H z5m$m!Gzy_u60(*~Z_OtNRr(#hd~h82I1Pr8`;$FJ$p=L>87tqXD>fK1?@vMo&&{^? ztK?=6rh63ZHg9NAO8A7#sS8$c^_s8oDA>Ea!4*j1^Eg5|#brBFLK#!f4L3W4yvU?V zjf#xJd5Y%Hq9m0u>frl?sKeH?GZmY`GO>-r|D`r4YV)Q@OA2WG&sk1Wih+f-P!)ui z82Xh{qKpeyz_g&IzalO`TX8j~EDcE5&NGmhTgLc$_xfVeC)VjqjEZ}sT%~U*FWBkW zZ|(R%7hs9y>IC_IN+@{?V zY^~)3BUxLf?UQwz@BgAV(rV|yZ{{6CgT2Pr9BskTE&>|Uvu`MOAV==4VDyWxasA)y zc3P}F_|4=#_sHkh0PD!N5|*q&2UdnotJ2N?Rv$q!FdiCs@EiC*?*cc@zy@Cb5lAqzFKea#Dc zz&sYrT}5mthC52e!aZ9oZo>;#OSuj99F7UO%4x9a@VXZc zZ}eh-hQ4cqr~C{mm@NdV%scR0(8AjG%;K5KR$x5{_y62u&MrRC(f8?k-tm}stnhZ0 z>##I1WCefdpi;f7!P6m?#y!(&^inme!10hT80imZJ)fs6?w)EqVm-gG@er5ED2L(C zHU(be2y)y=JE=eosk%=KpPv%Nufm?2!xJ88wnf|_j!L&k4!UV)VaCRK&1}smRBAES z!joS3+}FYG&6`2bz~WS%G&Xe6)B1oXdPPsVdKbENt;+cr-QVDqM95L=S&HfalBVR6f3? ztS*7*wZ}qP49fUpX*4dj88^N!zJ+;53772ry~sy%y7Cr6?VB z@*KifY?yn;gWbJdQ^3G-1<#I{5fcu3_8D3KC%>~6v%?13_q*$#n*s*fKdLl@+oAa* z@TO=|;N3U@kH*+KQVO+ROd6CWz^m7mawBg zEjyUQK%b2;8JBu30y+94t7R7pcHRZJ;A$?)4rWk3kdcejV-<)|CwR3;!6za%*|?7j zv#-}v7TSPWtTOX1F!r3?^~S-O*!O)mQ8L+;W*dq~-u@U|vs) z))(*N}3QhR5>Ca ze>03<5cL^P=q>W1rsy5;;%}beQ-A~aKH-sJEcIY9k`dJ-QgV!-?{99e_o5C(d2;Fq z3aw!G_SQ($XDMq|XNQeP&DmR{QpX(j;_|2KtC_^JI&YJz?pR-_ zXuH!}0QfP(k2(I;r|U)X@zX zZN#p6uIkt7^d<=4&mVvK`01CA9`qf7+0WKZn(fl^Kw-4UHaNnx#{&Ha{(As>ip?g^ zio*p454EVrzY#~Xtdeox&tCrhWTr^an1R0h`@Y2gJpX4`aKNb6p#T%a^m$*QLT@| zyobXMK^{xEtNV+ak2fEGm_ZqyH{3#aI@$wB#{1#s;`Zk6^XQkwu^#aWZG^hL`T2SV z&25t>ze7ng*kNhk-CrkxC0=Z{WzGX4ongOoRqr5>g53lWK`Ds70>!(&0_yV9=Z_E9 z_jA~;)7?5h1y>IA!^P(x%nyk0z@WMY2fsLJQ?zgimNcE!+q3*vo45WsF3dT7;c!89 zWj@A_>ut>Num8HY73=)>wE}L({`aE%ovmQ_w_#R*BQD-aAS*=2eOWiA-qumf858;c z$$QuBwsEB2`&N7da8D;Q^E~^bVk?PvJGNh0?&<6=ShNVqoKU2Oq#UcS{zVlI1Sv`a zcmU;|wGxY@MAq+L1VJ1Ns&!d`Og`Puj@7)J#KkbVVkheM`59;wJVf$LZfDcU^>z9nNR5L{bN73Fyu)D3ZgvK zE*J~X;a(~vqLHq7!o9t@oAR`KHJx7fba!D502WeSLa|y%&yRQkMJiCACpSDrQ(khq zy1%>TMW_3_UL2rSu^L~sCb@w^*4^Z`Cw*9>qCvLW!i2zc(1}Du6W!eX&db5qbTa$$ z^lf@K??pE>Fyk6+;|qqiW`P3D%%6E7x}Rq^{V~2a&T#)*KWerGO_wwa5}yUugSxf_ zNtZ;>+&|6f{bxFx+IQLG-OG14lkS%iAbb}>FnFh<0H>#>GkyPrl%K< z?oVTHFTplHi-NFXA|=yo`gs2|yMnjv*U8gkU&UjwG8fPEa5(E3NUsAu-A$h6UwI~c zb2aHv^eca(LxAzK5UgRi*xz}vV)Ch{n%y>$%@N6&+*=dp>F(y+!|fDau)W>a!+ma^ zh1&JONpa6bt>6pKPvsoYYRtO>biP@pTmpjXX*+Cr+hJ=LET{pXNa+oXPPeL7*Cxr9v|-S z9;d^l4C#zBO&Zc0W!jMLXp=a8!Z81k=VNL^F3YTT#hQE&N7$bilx~de>|fKoRoXG;6MJ(=QV!(GH>g9Cf4Sf%IWy_|)#f2Kvdz_Lz*bIE zdE(jo)&1S)n=el@Y&yK3-(TI|_U)E(O5XXorDUtR<(emt$N6kJ`PRiPVU#b@SC-+z zXO}F@ed?|lCt!Sf{n%LquW9g_a7@EwL@^SA0L1P67hV;=rPp`e#id!aieT1Tm>1)z zQxpU!?q@e&Ztg(eb7ZxF$?lSEip8TJS0#JW*4{9$>W^L=CV%0 z8iJ+ltGeL*HQFJ)88gv zZl}L@(U1j6n65|BAWhFEch~pdI%!ZLdd88VVhvr)Aa#A6JbvvY1_lt&EPalWapZt~ zBJ9s#2m_tnf4aGAB`M!N=SbTKcwCu+40MTn>0#1XOSiyX{IBbj%d2c z-FG^@yMDO8xtkB>4;lK4iZ9R4BPv3gEqL<$LVlhNm8ngoS#dQlxVoS zd-|EzI6h6Te(TQ}Dw1o>DQcI-QGO07r5RKEh9IW$4!ym3oDc3Ib(o%40y^L+9R%1+ zQQxNX2@ZoDBx#9e#5CJGaNgOpq9n($edOP^9z45@-PVVma(r7q{$RWO{W%e+l?Q*D zet#Uzedm=C+PZJ>SFFAJ3W&}cCBEA$I=T;H9|>w0o6Mi#o4ZzR8!DJ;{H&rl8$nes z*V~N$!@3y#;L|^KtV*m`BCte2*bf)qo@V?XR@CT+-d*ta=DjLRUcsGa`thh5!N zM$zHl%8+C@WES?Zw<2UQFHXS-y`vIkR2>avh${(Ge8>UtdJ24@it`sHP+C`9vV?-H+NUJPuJ679Y9ZwW*yfF zINDV@g4&FF)$y|N8DF#FB@J=$Z~o*DSUafLBg)>@{oNhUV%kB6EM^7s#w^WUPj9F5 zX*=Q-^XFo)^<~=9weXhkMvBJZz1xmee9&!@Z`dMQM$8|%{4h;_o6f)9L;LIF!}RLr z^UbuSD*3+lCPmr1e}LdikL{p0kUM!p7SKU1q*+fvamLvO|I;-WM0pXlGP@Q(hOF!k|Q04M6Oy?V=L64inib{(dxw`HJbZ+ z^TqmM6Klf-VaO{Oxn9idvRG_vs4M3=fmiJ6nUMtS&aWOi(r?3Y=)vyOwIjW3A(o5u zVUT3}OP^20PE$I>sZ?I5ts2=LY;eff$$|r>)J^Fpo6u>N`e4u@vu!^eSYyc|bixqG zsY}!939XxqJx6D8Hx-G{`7oP)y0O2bJSSf+qeU2Ha`ttUSf|pKIb7W&Q6_KhoD$<6 zI?p-1vum68PxEf{eoE4N0!*_G#b@z&iNQJ|a1^daj1^^7uZ5B=cGzOYK1hzXkgE%V zHnmM5Xmpyeq;Sd7)!Dc#T@8&ip~1*i7_i#EX0;^hxeC{O(?9IOqq@5w57vxE8|Vm) z({yucEaaHp08|ofpl4L0K2E+(>CFRliQeAdKiCX}G<~GEv4>BNd^%b1ZE`|O<$7BK zxR!%Y$XTahwA`JXD?6+t0U>6~e!IWE`TX5##;jT$0lF0UMbj0mp}dX?T&hJMIVMY9 z!B}cRuWD*9PM(Q<2X&0{2(JV@&hNj>Cf_>A%D3xvknP45r63-kzI~g_zV{&rhY_C1 zJ(EZ{aIi#@VsW6L^}U3V8!+J(OQee+X3%Awoc-9a*+O?XlGem19%_gBA7=hU*tq@BMkOa7y{sAcsa z(TpkuqvKIzF$CfkrbRS>Qgc`Erm6aYz(bV4ch@$+P6Ob!u6BvPixdq1%GL_ zcrdKAdU^Ds+hb1}w@LIyq0*NZ!FqEhQUQvTP)~O^ ze^8jH@i?D+d$1P{8#1ZON!&?V;LD_v;?+uVRIX6WvMlxZ%$Jnp2&G=nPOgVc1f^Qs zjy_IjzfWg$I-A|kdiRjk>TjcXrNv@!&gx|oF|}NHX%s(5#>#q3kR)!drqp3wrbgeW zlvjIuyuf<%umHGxdi=jG4Bs>}62VwLJDTw;uN&RMs^RkSaes51b`T69r*cgBdAkWn zi6m`s*2CoSk_;4gsXsoFZ&-EGenlR{%RfDW@1QD`z4S9(3Yz4>-yv0~y0^##wpMv#hTg|0cz zF;9wsvGOQm!;%F{@@;N@!?)prm(NGzpx{rj<4<42MG%~gl4=OVL?9gD)5o6VEaG$< zQanWEvC;J*ll-w_y1Ko8x~4dj0XlhmyrB>^Njl&}d}%M@0Z0B8a3GwMlx-1l+EVhV zSG^aZ_V0K*QRkJ0tEsnLy8ScS{D=}g9yu0a3{~h>HPO0rgf{IDVXuuZOj2@))MuzR zM<3!4A}H0Ig`aSc5Ypg~AShW4p%?kjN3g&_#rV=bC>i_xDvLD3`T79%x8WgE?XQ1p zR0`MXuuwDh_EnZ?-bWz@3Xv$F^4Guhtnk4s%R z#B$hE&joCH%C;*Eq+}q1VSfAg`fG2J=vjy&QWot|*oU(a_CX5!(4Vl6XCdsP6!x(X zVe->|4s%eX`ZS4tpUxg{T(TA;{JQMB!nkCIr5Ul+(`^w)ETbVS8&9|ZkBw`>DK)SlWdz@Z9 z&2N66(trH%M{jahygvUk*wH1g%5O7vZqg+lQFx{szD(xRe@(vA&y$m>!<0G9-=F{F(p55&?C0B2} zso&+6SHc%#6k<7r?%)y`LUKd+Vppk^5XOdSi`6ksutU4SNbG1D+GN_Y!|0%VBzR#P z1|U7Q#Xl4jYk2rmVX;nuzhu#}7>ivPpMRa*e4aZikS_%>+G_!Y2H!Fa0wM_}S5P?V zq$Od+&ouinrWl2AH=Y0Me)d}*a_F(h{C;KWX#G+iJr%XzMkWk;961Rp1Q=j>ED=I0 zYLerGL^-6ToW2Ib@Lu$8t|l%W;gAHzv(?0PCr{N^k{`FSQb>Z z%>C`-*p4{DN|05z*>TqE+2r$_ss_8ieH_9;>2i6_J^{S@&z~)oZy-u1sCwp5aikBW zSW%J(Spt(HG7J)QEa+C%1M1Shh^zckUK*qBiAy{CLS2=J3y|`vaH>3L09yy?>gbnc zn!F>YAnoV-yTRPCg1BM_CbplYM-=dv>D_d8Wa4{AzZ@L;c9u>NL*tH(h&$kp)t}L( z=8VcA{+pN={o>1KmKW4G>f#%?V$CyXq`hC@^qN;HhZHseZ1d>Fr|BcaHv0B3n?Az3 zXIO{j8+M?l>`YxHfYC{h57Q(4XId-eKiR zl2__ivh`-*Dpp=V!`e!(sYKO*v-g|=)y~u8%5#r%7$flI`;cxEs{_1)s;PFDCjNYL zJD<)5shJw?d3#^Jc|*tiF|1c!w#(%@NDfEyI7{b9Y7y?^>(l4Y-zIm*qkZ(?sw`&j z$3p-+yPdlF<4-QI<_|oio$y@5!Jj*)#tFDU%@KIb_KtwlYmUG}TD_wDG{2qvzwaLg z@ItyN&fW*W_-TIsFdslaf6HaT*?T}xewy=F@N6=a7e2fXH+)cj=>KtO(JeWGA3F9$ zp7z2r$j{yf$awvE^4P!rcOV}T2NA!@7z=l2E?+`^nN6qH-+OlfymHUn0f-1ifK8(uL`ds@atIhDilBcHl0tdC-cdGjuWR*u`cTmwtH8O6Q{|gTmfEG>8V)F z^|$Hw0UalXVZAGNN#w87$@P(t=Z9T`B^g}35;j5>2Rma^CMS#bj+#^O65tPU{Z6hB zEn%=~rRE}4MAtSBWkL#S7I#PW2&ige4GmOANV;a@Y=?JXEtF7CNpP?cghYDWb=uR{2TEPc;Aa%_Bq! zyB^kE5UQ4)t_Pf}w-o+t`u{ym?Wzf#eT8x9RMeGs?l_EGr?p5gVxYR&^zi}aPY>f{}|39=$Ou=4qtdxn`+CYWvshub-=|DYnNj|+Y&jm;Jyqy74aqIt0~_zlH%RO{3%Hz< z!|D9jYMm#I-gGexSBq|>V>#^*Y)@}Baegp<*$4vguv$8@OV3U{OW!WV1UZO zL5*+$oC4+MR2T=-ZtzRheK6GJ;IKu!0J42*Q*XBr-lnMHQG3@yiG>HOCccfOU%*+< zZ?n?wI>5oJ_AcWZ+|=6+EFDLFt3jl04v@nQ&r`pGlfxkUS8p-yaUA6iV;n1rMd6$l zsqUjT`bh9pF%A3jImp>R|L!Dy|JpL@PTgV$$B3(uA}yCxFZ)akej@olO&+KJ{QGcj za&GoHD<`#Y1Ul^?5rS&FezxzPJ|hQ-s8xdg?&;enx1MSj7om*j?NVIry}C-N5QoG% z37?)m)88kzPlKhdBkLQ!1D8qq6AL09a#6IQMX*}&D}ZaFjoOi&QRtuNv~a5gmKO!D z7$!P+gTKJ!Rp2BIN(@kfpr7vNUk7o6TD#(1xI)9K1vUn8iCP=@UARQUswIn^UGgJ} zVZmm+Bsc~3)vj0@Il@&(-23m%D}z@mmOA(auA@7a`Dq3*(Zrp*>}~zhqGNe%$8_yW zPGRkr4;Zu=x8%zQwng3=5ee}OUFQ%W(ZWF+mdW+oTStu@$43ghpP;gxJ!Kbc_8eX1 z9*efCvutD3o}sr|g|JorlHsc_!|n5P3f<{%JK0r_O9YC!AMl;6W8?*YF)oNh@r~=Z zbB_wNjaMRgCD}!;*YLLoC*&-Pf;jp!&?{QJKwD*yd^lHU>=6@9{8etW@e8h; z%ZnX}T!>>GB*9AbUYwO@A1Ky$y%y3h;=)dXi zpm6p5#pK~(dUvgE%((aDzw~*jJSgY$gycaI715vUzvxbA$&Zt}oB7TEo1VuVpgAl4 z3-1B}+3HTv$LZY5-Pi=h%TH(v!tsnDj#^~7T;w)dZ(_EFRTc4Ka^2qrbsTZ^es0n+ zZ;+}ti1LOW;&t=w;dc5B)|i|_L!(>~`HHp}`Zdh${K43UZj;4JkgQnvUv~FEJSoNh zHl04ur@O1Kle;g|>+$c6Sk;m`_VE2h?u`&{bQZ#**0e>R;V}0`jrPX5z<=2y^wB${ zT!2%El?$HkrjJ*Xhbc}W_F1I1zgw_=XLe|v_WzRGrF~GjU$Ca>?AY4zecmvfl#+=K zC!9&alh7JpqpD^aEotzav*a&$rw)r(wbN)xqRy0wqJ+cprF8yvcK`I{tCu^%VHPleEZwjS zd6HnwF4+o3yZ`lWgH%f@YKc@k{Q}YDNAx;B^55ikCHLm8wc~#T%O3$`Nq}VrOjWYP2*5U&*Fs$T!_eP6Rn5r=^@Q}9_7(1 zdGrB}w(;&J%?r3ix~ExqSREEe^E@^qRk?bzYD0UpX1beK)x~uGv+!(12a!5d#3~iA zYH+_noiNJYzu%M&sMOxfTKO$|4lw=#ezJtWA{W>kg}t3TjTcd}b%++HTm_fIfiG~` zZXSV4;D8tStPv7}@aq0`@1PFJWucM=;xc+MW4e?I+dI8&(@#C^Ca zf)kgUV(2ME6@CYuzMKYwg|m0S{9OCj|^{| zc`KWa((+jANCC=|ruA~X_^`8cm9vFy>1{o*ym8iUU3ot-2Kv4QE>Q^nN;jZY{DR(` z-y*`nU&8s_Vk7kH{&vw5{yMjzx3~-a-WIpW41b;5(K@EXQTK0xU7QAC-T(6N(AiRh zbqprj05!lZkY?)lU{CwtSyPLf(Bo6^?ruWAPeOgN9ZV>q>PROCLx@Y$QjgxN&$|O? zeFUdNQOvKe?r(c{&~uao+3vk~M$&$|xtq+s_wEabpBA_eQX0z{3Kv!!pR^l6bkrSj z@J8QP^5DcOiQG>9paUh7;%8J89p9NAouY8(|GJsadw9=rci`BNO_~*D9;*+Xv0EzG zB9|?(F`J#wCU=hy_p^EbURq`@YaER39f|vS=DN6noj2T;qZ{2Dt(T&+^TsxLXQ@L2 zKi%1<5Hn(*7+5CC%Q9FxT?9E3nXNbg`}%h>oRHoJj5I;3GJuM00Ae$|Uy1Qc1w$-? zU~;*6(+uxd+92wZ1THBu>Vs0r5vgs$84vA!?w}37G@SO|a0i53ybLs^JWOZwYGM~Y z%)Ar)%ZOKqQtm);ro6Ldvx)wNyay5h#++{x*jl8S9&{hXar)E6{iXe1D)x=06aAA8 zcc3&w#F^4e*-BEANsh@RLgwW5_WoZZJ3QpcfZDyx-oM*Hta=q*^E6NG--iR<0=Fq! z@Bc1zo~708UTm`n=DZ}3H>%h;D(uscvsmB7jXUC;(I>Tzblcr#yZ*KBa_7mfBi+Nl zU>4i@9=_M?x6SO?zxJK%bu#Ta+*R9Iw*N)9wfm%7>^6H2;vj*^K@k90-m0t5f&nfSF&k85~BYZMxkF?32?MC+gw^u1DZ{IFw9kAf^U= zpHPD=ZP2fl>!x+^-FlnxfA}N%;k`Ofh!8*eaPjSF#{Z%7=!b#b_xAR#Q{CH}NN|7r zVV%}=10~JV#Sd2SBXN$afv~KAvUE@|naA+o!z>tnGCymZ7rZ` zP4N5E#nwIJzi{K=yG6QQ!?a;1$LXsPc{ReXhD(DSkwTiK5Fv$>QYKfp_~FyteD?G> zw<~w{>RjB}QT&$T*YDW9LX7y92r<7;XE&d}Q*ngB188jL!+>M5n=+h?d525|xN#@7MQEFhLd zAYxsINq#1|Tq7ruoCG+p?;Z#A1V0eP@4^>?boy;_b333vq74ng&?cn9SvV!y5QTg5 zKquGNwowsUx`%&K*x-dR*(!QA#j)d#;-5rzmY3joz&ofnetjq2_*Hu2*FnAU;hlKngY?FSLA~+uop|G;^v2<}wP37Ew7{h< zVVKxLESCz5Y9&0qiXVjS+2$&uNprnOUhWrlxnHDPU=>C@z>5iX?Z)y-;0735vk)tx znqZ^~Ml_fVV=^5;v}{)2#KGcx9UvRJ1*CF@Z0CNsom`!E%nRf1Wo%7c1e@SFilbtu z$2?=##!>O&bnGz*a~G}RL{zHn>cPv{e}L<1bmYg`&|?S?hM}|Jv7>}yG0GhPv-7Ld z{%>9(Og~&W^*H@;R6%v|uh>aM^D1BZvB4R@-2DA-{RnwwnOK=@TtOUz2n>jhNpEky z-OPItxJ`Z}>Ca?Tb&BEX?zg-9f8BM}Rerrq_@D4V6bujQ%CCWh0Lk@{SQhQ{y`~po6YW6a4y*pqI{oDPG>{11^@*yu|^CIM^W_xJVXOtD?6T8#qY5O zn7ft8WZ@o4WPA4IAtLNw-C{S{Q#-Y!zNDF3F;&*=)vU^1_S3-30l3opVOo36`a85A2IXG zE=)eoC%4nq)VpNiH-XO6};Od%nl@8MXb(s zfZj7qYKDn2{PpSXdUhlU+e=wTam;_F%z>xDgS#^lD-Dj7jaOxM?ee$F>w?A zipiUl|Lv%W`))U?Bmh7~*VDW2Zi0@sMeVcW&)aZYwKO%;9c)SJ8}D>GQ8VC?cA&ly z?OJ&bU>7CM|ji#CLNE=MwkajIB&6vjuO?`8^mX)fc2irgT#ys7+p#Vi05S#hM z!0;1A4_ul@mR23`N;5>8naB+%#u$QT?=l9x;lrcWtWCuJOzyjYKL_E8-=d!$mQnp2y(k{gbUheu2F$|%P^$PFh(o~nm` zn3R<(@#zA{)|+DJ=SJ9=!e1w}w+)P9g!20`h~mM$xNmP-9{G$%T=2y;LI%rSvScx| zllROC;%j}ti9OApBYv@DXlQNcv)XdQ+$Q1Rh+yM0{B=Q-FA7-cDBTX7jn<6GH`83~ zI#C~rS>GconE1l~xNw2lSC+w$y(BI2?Z!(P<&afqkNRC4ze|fpb=*ZzwBF`}^GhQ) z-=1T3sys?iYALU#<53n%uR$C8YmW8J>137r-*%vH}cpc9-G23 z7ZEW_R>g~4NpI-%QZsgoyED1#M4cY&-0YE77`ep{H@b8avVyxKA@!~1b)PA7g!m_HP{v} z{0_;<>YlRf@7%-{qJ8h+qRM^BF>OKCulF30ajH!?{BvMf;)cOy^o9 zKUqgF^x;#}80WnycO9wMX?S=xaj(eQ66gIUw_RWl%$?|u@%ZeB2BJ-n6;TlT$$@Le z9>i{oyjZV8lg2?dc>S?%?Lq9%)ASbh9l;@qdwPM_tv!hSHZN&iYCIa7u1EAed%Bo9 zlSiEJGwi60ytVf>7<@1exwrV57xa~UnoOE1S|ub{h+vfB>@mv^@Ll(!YzFaQp*fFV zAzwxw3R6uMXK)M}56fdtB1An_M04bCxH!x`WwggE%i8r<{O&V;*Cqp&Cj2y=*NjOC zEt<|@ZG*zkJGx?O20Ja@&pBf$zl}q;iE1;CjNQYA8zYXc-6p{+&x}1hstpDo#i6J2 zkc)zXtv6{FWIIpqvb!$BH_{R6;GwIYOvsKmcKL>wj zRZKk@J8rx(fB_^hc)<;_))eA9_EXIR!7np2s1${Cg)T^L`3 zEDDk$i1RdItCZJ+i^1E_n{ECsj!yW|JAJe&1&d} zeh1X$?+!8J#R)%$z&UtSokQ^2@aH7YA__As3t!5hc9hp%meJc`GC0ElN*hp7^N2K1 zwdVjTPM~ppAWSci;^NTjdQsXr4VT6-!$}QL^WI%XNgO4N;?5CBeU|v83vksS5eA7c zXj*THF1C*+(cbaJwnN%Ig3gmxy3H2;@wH*K(Mg6J>Ed05vMWEGHN@!xX7hOAR<`7) z1uF)}#Xw@Px&(Y21I1rc^>YVI!wivT#K%cQm3B@%f%<3mRO%Fm$4px}{Y-TMa({5N zwUbXt7cSRFL7ga*WBJBEjYriv?wIn-vJ7ycx15PZl{9*V=?8@uYHg8#MgG47^NE-4 zAk@O9-3#|1dF~fFzbH4Iq1juoD5%Whyl|sMm8o7->^sv(p%-LsCJLS5G{AmMeIQPq z)uF|TeP{Z7@xs=Au$3wNum*)!o>QKTC%99>X`OOov}^x_b-`3tT%RYjY4tF5p>unT z(30yj)CI50(*r*Wb>LgM7TWPLpZ>K2Z<85^0$?L09;9N&11%nuaXcWE6My$+6i;{t(OmEVf`5VHjjkcU(zF8G{*2nfU6Wv};WrrrYP3VcUc{ ziaS*9<);;>RufkZ`=&*&Dbc2@B%L*>9bPB2yg7YKQ#-J984@Uz-n8LPKAtjZUwl}r zijv~vhfxyYp5>m>-~mPclHdVh-BaKRWkV;x6S`WbzkmLHtn^p4++X@@Qsg!LH3{n_ z{T);0A^jcG)iV8U)K0Xis9|r5c$@Po^l3rSri~h9m{bju(y-p@uEsvq^(&@BVml6_mh>yQ4W!zqGY8GZh`_PSXu}W=d}>AXOw88YQtK9+LWIZu6-I^ zi*;sKo9YwonZ{d_7a3zd$f69zFj#4IRDmUh z-s&XLKI`gNl(y@fdo`)hzPNh`pmrgaEz@!T{R^`6O;!gK!qj~P7!#e&3;T1V?uak;$0DvC=o#?mrcyBGGk?NBG~3 zZ&9Kj9gh$&U&HAn&G%!7f_iw^5FWR=2QEjrDatn9k*;?^j$0iqg~=hbWfH3cIjiHf zuO#)C2k=^7l9m-eWx546Ze5CS=YGhA*QG`A(VE+1nOT7=ldk3CjIG$4+Fzw96umO` z!J?TNPHX{g8YIFX0fVLOcI>$030v60zt*rKjGDRBdYG<*C~;`C6eqNR-?L4`rOMiW zjis}y>fRXIs(VUdf6DBe*#?u=zLBlk>I|co?Cw(QR_AJqvo@$UT7qK{%p0|^hJRb~ zMyn*v7+nN8^F}Rf60~X9Z>iwW9H)WcrMVL`osJ{t!4h68R&R%%u)(B!NtD*x;zQI> zkLc1IejG;@L&IQb7Rzy@8ovNUucUD30I=J*Ffksiv_PZ<92$0EnsX%G1X(^Ps4yTn z4P$ng1+4QU?`O!C!8R_$yv|-(4J$-i5e*R+6`StJI9N{}8jKhR2Pz|`#gn|oK(9-d zgog>822xZN8iXXGQImo zUf0Dp3Z;MFC>R&t;MijRbZ!>~ZqscRABGYmk>$_r*$gA%7#LC3oDffBs6Frr3kNgN zAYAiW0jGLSQ(PcyBgSQi+x4bA);L2SPnB~l`56>XISz@Zz@Y|f<*^IBe+`W)i?5X%RUA*vW(NJJ~ zn+OA?EnB>h@d{g4u_@f3@lB;I&`oGDK|+BlZ_y*YRWSy#LgO4 zM`?_pv6^vL_H>bNsN5MA94rxK$u!#rdEfQJ!|&gLi}q>86u~q3D5T3QUHcP(HVN7+ z&AW=&1P2xu>PDQ7+u^gcB`jfOFib`h|6=j^-~x#zI#iM%hjh5J8}SadCjGt-zl_#(%ULQ=vrI zmsn?OsZoBTe&JuCN}UQoes+}8sx|6bkDujHgag5Tr1XT-8B7#8dPL zPnobx49gXgc0$H`q$>8iK2cRMBRx=6MZG;wBRxRfo){5nF$hBz($^?}pAJ)EKtlu# z5pD>6dQma!fV+sY#WpJFbH;)nryuT4#L^bgw(7=+|WJ^+SiAv&%59hemBl{zwORgBSrK8j0mWTSve;(X!L`+US5cZXwp&m`@!40}kuj~%oME~p z^)6V)9;wlsVcMB`!wX*pzC!{(d8M#Ol*KB5D}ET^LJagJEhv9^?2lO(MuRlV%d-Vh zW*bmis)1o$GTvJ5Q4NeN(`Fk`oYiD7`Wt%K3%2r-|DZ*pEmi^A;h*lg?GqiXm4;oB zS%N7LLib_ggAZ_FQ-ljdn{|kmB9;&|e}%5#uRmUCT6t6h#t9lH%J@O7mipG)g1u1~ zUFcUD+9k?v6O5~%WdwfV6`2=~SU|HF7w+DzB@a6MuCHIksL^ zCRCq}1UQQ4s#UZ}sCrklp^c&2hmk27PgFH9((RlVhoIcWt*BT<+K-$Sk=9DX?U5C3n+Rz{o1}o-W*;NOK zRX1us^r{m^o2(+VRgidk#&sVO+B1x|1^+Wmyn+jRhH+=`bn$}~)K{h|%!$0!Fx`^c z*E>s63v;DuXX+v-a#{plYCLF+2xHQ$EAKXEIyd#$`p(VjEMB$5H=vwQqem`?&Aj&+ z6vik8WkP-)C1Z=w99IFm{3r2{!+5GDahE8QrhQj#ahhC&!MKn|9VUV)%+%FMmXA?9 zr*yR&#B)lSj}bkml(ie)!;)~5MoG~$09AcQT_1Z_W%~>|(^B8*eBlsWTb1f7_2^YE zJA&jYP~8C+H9%+Cvfp^jdn>!O_ipV0w3ZUj=f*bibh@uF-syz}aq*&t<%tpJFAJ=b zrrV<5!6kKjYUKONf+%}?1a1Og%$YbK$8ilT00zB@19BAE#U6p900`Hw1}R2su|PWd zV;My8&}yV{951Pj!|g%i$JyL;!WsNZ@^*&|*G~^AM!OhTr7*41o-!?>Zd@GW>Y`94 z3IFTFVHGPTmiAI~`Al?#umq{*tD3WuAL4bLxO0*cOe29_c1l{w$V9zi<+syvgi zb$YVHt2Y^{c%`f!pgAe292O3pEd9fmqrvg|^3xV;;5UjIi(Gk+xV=M+ant<|Z#5Z} z0kvUEc5t-uV5kQNLmwXb>?CGydCJRCoeBPjxs@TZ6YU*a+J6J(DFI!C^vmR*2)#pp-GXMnxi3R>3;!WLcJG z-QnM5{9)GYSVZI!OC z1X{Con(c5(>*93ABY$lPBz{fGYwIl=q!$+O2PgWI4S)pKCU|Xa#Rh2;tQelK+*@q5 z7c6ysg4eHDbHdvu>tM4P4Qi1V{JGh@?D7a6py%Dm7-+3%(F{>$h%nP+ z+n!m>9i)q3@xlfp7Dk9LvTye|EljV$*0M`6C$^wJgQ%d3-6B5eWSp8KPPXON=7=;$ zz#Nh08W3#h<7mmCqu3X|vRC16g0K~kY}e3J&okxX^z6!R?A#PFP1Q^vCC*ZQou+0q z{9@?AA9d?xw-bD^H>9V+)Jhf}_c+?@kU)c@WtrPXq3N$N6`FVR!i4Iq&_{{fMq#@W z`kq2rs~qRqlXcwcqt`^#M$_mI-X2C@&)vShQ69T}eeIq)eobDCJNQ}KJI~isFrEkS^JPJ{LY%~>blMv&nKlfXl_jV!>4&4uUltd zw0rj+&>kf)JLVj+IYH8Vr;kv!7kyrN+##lZv$%uGN66t0D(;lQ9aMNWNqo5MjV&@( z90?6Cf6)7EwLQc2i15mAGfzuVx>wk{D&0F>lT{;Xr%QY>>;g>(rHL=v<~b)MIpr~Z zNwLgKPN0hq@9@ejesmJhswC5wjFu&qJxQ{1?(cz@O$4Pu(|#-Xah`6|c5e@NMUbaw zyCS6Rqg?^__GDMkT^?+h_KvgO<#cw6Wnnp?z4tlur9Cx$o?y9L*=vHZ6<`E#94sJ~ z!EeNn)G@Fc5NzoOi5p_z;`L9)2Qh#wM(K4UQU*?qx-*!z8|**|;#h>488m?kW6W&y~^f;)}s5|gBi-slz6a4>Cwj$VM>_1!nB2>|}Pley%XV_Ae7gw>g`Ng#vTQ0t7 z)u+@BjBl#0qSDY&v~2=oQg*!Lu;K+fDAMKX7?lUHgm@w)MoCmSl_VSNs$)bRB*JbZ zOuEhS;`Q=W{4IVp}f#Y2qB0ht)HfBEH3r-S)jQJY;lNtQnGfbt!x10No{rN(`7k{8 zf*Xz+P3yOx!Qrez+^S1x4;UlG2%7}jD5Uw4q^*`vW&~pjQVdE}jj63Pz2L-{+BAd? zrdOkCt8SHT7IHTAnEC*Ns8BBgW1p<8#ur|o;Pv+rsu#g(Oic=e0Em*; zMAusBHoP`PS#cli`-dPnlM(V7d3U` zDvqfNSlU{|(EwM2t1zq|j&;?fLsTy+bssMZOuKSNR&{L2RsBI#ae`v;LZc=9!FCi1 z+{<*EgkH2%NeGgVXh}E;RtW@2Ghi3Bo^Dl>JgC2kPg?6zG`v`!sYc=`2nan?-CkE>!|R^FBfbT2=Kdv2!umyZVEXTE^F&KQAx8RMVSn4 z^m47RW8;8_;g<`RX~Pu`TMgw7UepVIxg%`iUw2@=b!svB;rPof1EGn_U7d`4Hjk07FCCsxXi9is6NQ5S?_Uhy_h%0k1gT+`1 zWzJAy9O7%CG|6Gyej0DFBm8s|(-1PnimH-9wK6r5wGgNmn#j{FuXXs&QpyD|l>I%T z)}r(kcxIoEbnj{;{1uR^J6J;Xpm1?E3KylqMV|`evr!nA3gZq6S%O_vVU%w~vk^VK z#=~&x1O+K61LQ?LCOSkN|6wksqpgg_IMzf!xQ0t}0Cd&%I=|1#DC z^in=AFCCs2IqGMVv1Rn;TY?yWByCqn+=a-nBH}8(9|;-nL>aHLxNC`n9m^nGglHr4 zEJj*aEp!((8h1GdCtD7u;pqI!uwZFE<5d{qR0(n^k_ zOCz|6I~L%=pEONzINZf)StFHnWQ=BIy5wl9V#!nPi)g?&v?%4!Y7l`K2>V>!)TId> zqcPT0mnK9Z>~pnMCuhB@z%%f2%p&i_3n5dm=!iH5%MsGn!lUFzK8{RjSyQ-6kS}=8 z_{ysMY=Mn~&UB+7+|E&QKrs?dH(9=77qT%du7S{9EcCM%iw#|-8D20>(;wT7?<5BU z0XPNW6ew6x9H4WT`4bvm!qoRI)MnL2d-8lF^jx#IaQmBXqrlSs51vqU&z5IBZ@T4# zNR+bY?cx}xt!h~^mi4tuth{+IKUE^kpvDD~z-9Iz*>gtVSY;RF2)L61LRMUmr!r zf;D}Pl5oUFQ*jXBP)}zQoV8AV`E)HBAw?TxwRP0vL%K&&Z7LNH>ONd=N)HBy@~#v~ z_Tg-mVmFKEWx4d2R1LGb|IsNe%En{ab}V-N6kw{BiL@*}Z0QyDbCCNF zpNnSDj_SwCAL^R>$}aZwjg))r>FYH!@m3nGsIi$Jdzd*Z!hEcJ18mEDqr_;fV$6G> z?e}mujQirTe0)F--XgVnS>Zf9ry@?R);>;RQ;Ww->m`b<7S(2W>2t7}GO9-vV(aoqKcr=_B!lZI_vMB?VYWGsJ4Vq+R~0xD)sl*6swUK4^q-gRir$`e7Fop3My7x z(dTWrV#WCSkcLPa6RjVe;#!yDgUa|tXQEm_N$o1u(nS@uS1&+6k8m-+gjiK;u{$qk zK(~}g5+-6JcxTt(;52?+D1o0CS?k9U2t-Pt8la_Mxfwv5fg&hQj#M$V$#b1G$;|+Q z9&nqYExJPB248b>?ulQTCE9GA!%;=df@Hg~nkLbw3SqXK8d2wPRgvb*EGohT1g!eV zFzIp$7H{e}j#-k0NfKB)lhNE2jxD#VH^f-TQ=_7KRV+Tyq-o!gJYBEB14Xf+h_Xmy zn{Y{eDnSGS2t*-NR#H90*Gg&pY|)L?zqt7s7&N(kGEM9_S#k>kt zWJjJzPDUh|4SqNO@N43fL;idR`3uzT#z_FBUo;VsM3uC07`G=s$m{BFV4pq*agZz+ z&-`FC#n_IiA)=p%L?jZi0ec_^!2}sdl2Bg3Su*km=jJ7t;G;NkmXG2*B%qVO#M^R5 zH$f6D?7E?bbBcr^2xD|f9f0g4Tt-z)*iUq7PN7pj=2yv}46zu;0Wt<8hr%QP%PMjx zf>4A*oq+D-5S$GX6+8NqMXQ&Bg=6Qi3Wz8mBZEBx+sO%f6s~xAVO-jA)ay~g;$169 zn(#X5l}jhCS+-*IXOx8L&vE3+bsZ6eNRS4&PGbK|6Gnrd0e^xjz~BtWD2-OHH!WN#38zdG{ST#&nCtxMTW?)LM8qumAk5Z#(B|&v(LTfVRd1E3c z#FbzB6{q@Ku}8h+F}}XACU6;>L4b8 zY?GrITS5_@X8iT;Nf(4lkmiXrPtg1!E_Zbz;sU?eI*NB5WeK}P+T9~@7c$cA*}E;> zZiiC<<=&-q4V-*T)T_Tf6Embreo-NCgJ&?tUzS|F-VPS#=8Fmp@3S~-k(=^2%xX?W zzoIGxLZzXu4^?XFna6kOo-Q>jPB0iiZrUtXy6RDd{SsFVsOIX#zJSe@lJkoSfg^lT zY(9+D|AfZ(d!Cw^+;xp9t65b?QL-^n-fq=lld*N`uQyvZc*+E*dX^N!Q9?Jd@iVV+ z#L>c6!7P3z(w?^L44iMDvHQ&Dn}u@=v%00V#^I_$oGgE#m&m|_-Xxg}<@z0hp^DT= z-7<*dXBgDuIi*$PDF~JPYEYd7s@-TV^qYXvzClCjb`2OvRC z0KkChCaT)AC^}?Dl2`2j6&*$wjewmbt7V1hS!7;{W-rq^gxN{E7*eoKLpEMA7Q-EY z0aJ1nsgt^#Cly9(DP|mzh*D3|7ma}3B#XkT$___*6Uj2PK~ckU)W`y?_~0xb@156Z zT}?PL2Vpc*wU8#Sp*tdBO7JVwmwt|d+^5zW4iPxo1Blar*n{xJON!B+(rr?VsaS9j z!C^x6pfQirpD=bMD@I7hb)f)55JoScP{(tW3{iuFJdak%n)$nl5WU#2TBB$oL5#_S zZg#(g0T7~X>o$}oA~#X%fERyXz2}|6tdVKfIohbuk5YlF<#`;kQngRez8IQPqPQw5 zr>|ng#QX3Cb2cs3m<)m=bdEA*;nkNWWln9l(R%FiX*pOT&7xJ51ThUqC=e;*B&?K& znwi?^3XK;gw)I;^rXE10fg(C1%+S-aW>x4@^LONo(0@--c<@HyuW4EQ~E$a9YyZy zj*NwosNSB9Gc;0zD-PqjIzo1*i&h$cd6e$mFAeXrxM`7N=V0g@2Iu!ZP2I4MvHHvT z>J^&yqM&3xq9&*`X!9nEtD<}ZsTD@n6@@<`Xcb5Ifz=v?qXejt<1|j+#c;4JCY6?D z@CF)pIdFb;V(oI9_{E?y8vWQb+45HA>Dz zssJ&i*3lfLrrRW>S^6CL4~&4e3EDQF!zQ8@qVNi%Hy!Vz^xtp{r{meG4`j9C`Eu?r z)2RUv1EAdOrU~Y+$!=@MXk9&PgPC=*S6zYZvvt#IBnPh(YwJ>tgsM2|WNt!5;3_>0 z5f(`FBu#|S(n8U415Fz_D!RIeROVNZiukTsy}zAT(a!-yY<6Y_4up)3SG=QOnrRl&P!l2hCr+ zGc@HM->lV2<2_^dx5D^AfqE(Bv1XeHof>0*;>jsaH`T67%=W0Xk5y%lDH=yiT}M|5 zs`J2aDm3M&4h}_dgs!E`F=uyarOng4jx##y%^58g8w?#*&Jy<~G-rfc$B1U-to8ij z0F|4xpFvdM6@CWU8YYTGN#yJO47O1f*oG-=XuFZfPL!B7>QZ!)XSga5r@5Aqq_GyU zDY{{Hj6(rK$n*#@)zJkX(ctxo2Gc~FRvR5v?Dh%}Ufw?krS0}g5|ukw$Fx&lU-e$} zY_WgKw*b_bE%ok=0dQ7cXA!+ziUsTR7yEcEbwadAw%B4H=Re-w#uI6P5CExx^&oE* z6zpfPbBsGZuv7_EMxQF44gTbyMYel5ir1n1z7Ot~obv9B@Dnni~?C8nuA2 zsK-;^@Svy>7J}>)dVd~K(*a-8cspigq`@U>_M${P3E!VnT%*eS@kdE@47sHj97iS& zpjZ>Q4NTlq0%IF>71nLg#sI2d*rl#x>yaA`h<&IXw>_aE6;bx!`Vwie;Js(9L=qiB zZK*up#8H8bPZ`Y9suT-Ymf4337{CX}Mig7MSfkiZ;J=E~QsA@GR8D_Og{ViQ0e;Y8N%MYSeT8_wdy zT>D%IlRMsZnGi%l$WJFlYEynvh}!5?joMV72w>85lEYtNY*aZ2VA6Dv1MaxQQ`8r{ zpy59|scFQZpC%PiKfh3T{fZXz%TKlMdskgz>8;MX<#s;0QVpUuI!n_|x(t#~PT0ss z*Qu!pr&8Zs7=%6QPSWBGb#p4M+aT;wcbc7{>{NeoQS$XDT5na4pOFVDgoET}r8i{o z>vMF24A*U-)mVFXL%A;&dpna?dfg@EzQC04c$0ZR*F?!?Tb!{^1Rwq#&Rg*6&z`ePl$bu+7KR+m}CP!3}d+>xD zAaRz@i5$~$_(3@c-=Aa7?NO0FX?#|iko)t***z`N70r)JBXk#M;BL-}%WNC7d`$U@ zga{J++eMjfMIKX>Zq_avwYz9iZNy_rQcc#n!P-L5wg_8CT{O8 zIQK-MJz;=0!1wN#C>Ikt&fF_eE~chj?1A_0k$jgIY<=ebkdQU#qrS*+XmSvSkd!XP zFL8J`%|IuTd04|p^Je;mMM+*V)y zmGNV>xr%ig>={CLk1tXP!C<)c7U||?Rp*4u*6R27d*l?{FrFl}CC#s;S%^yK?)5zYq(?*epmiLu4 z1G$OfuU-bbK4fTHYQqgxHc#0sgwPYPY#-AWD*&$W=3VjwZzh+EDu50AHi|>}n!8-H zB7p3An74sxifB`6!sA013F%+0_(M8wGZSf2^KUHFEejd#9Ec# zcxIISCMY{$MV7{!I7s*d6SAa;`0uh|US6g#Vi+qoEr^X8BL8uerOBF~O<#j73ZCPE z>WfBI(J)H1(Iyf%Ez5|%S1fxmI`L~mYQv2)O5!MC6vI^w&MJ(FI9V5_N1{Xm1gypg8=?>-T#=< zkS&943~k~;#^38W%Hy^{Dm6RQ;s_d zHz_|zqv0ATnhXTwpCCa%6oKe7Z^E*Xs8&h4gaP=g zG}{f6ryGm3DLZPzi4w-W@$x`8G^MtqHk>GjD}0oME+ZvMGU%tL+bqt9#sfnH4QZ23 zpo6yjl$h?#KKk5ReSZ`0^qWD}PJTwk3xpkp1v)KDOC9PpjbH@9^t()+BEg6Pv!QS8 z8lM%`W*<~5t*-UeW>r;ZjUuahQ!lDSt%W(|5WQ<^oo!^Y%Aj7eL$}WFo7Bzo_U-Zw z+@5+y8t}XJkiZC?HRE|kh#{ZkSe;98$bN(u4)l>H$=Fd2x~dRr_z)3?Ki2Sfi^`ow1dl#jnw3X}%3N9Jma&+!^uv= zk#wbTz13QXIaqQGE#}b8%E-;uiZWJYJ1W+d9zzk@m+M+zm@tltR%r^M9sNEf!l-ym z41iYXB7MOW*+s^%Ejx&F-%>;(gPFS~M4hUeKI1}{fmmy%NPpP%!w#a=03rg22q2TL zKh4W5UDHjxT}9_18F+}~G1&SOET4=;y54LHMp?qI&dx%*e10Vvkz|d$y}Kg{*5~8{ zz$wXkSF6THpU;ynb*o|A0<4L)Z(nftR8i zgQ%BtIe8i3eA9(yG}A^C_&^@A@1phV70IPG22r0c((_9Y5(@qfBYOJ`E8m~-3q%r{ zuaJFc5576a?4s%$LAVD!h;PK@+1ug;=JhDObel!z<|E-7MXD=IKzdk22cPk4B`l^v zwi>C}E&vDum^9rrjou$4rB%9Q_SrfM1#exBeFMS1xQV^oJk%ssDUWbk1>YK7o(R|# z;oJ}4(PVwfmCHV#lVZ6nwL&WSI_qxeagd|J#+2UB-JqzdkiF{1P#NcvX_+oSuu3kr zhuW(n^875_z_BV$s9|<+!HDhG!gS|W=+X^ziERTrI$ocJ*+(v5j`qbi|0f{5D`Hn19%CdqrC0iz@G&3`CYKAp!@+EX`Dorh5!u_ zG$mMXS~pqr3L(+P>M^K^l{nZTV4(FTH%n9QLGH%<9Ml305i})O2dza=IE2LLUmdqk zw0+IeUQofyaI6@umMhGQs;ee(RHNc0xWs_zM~I5wpe@%?9F;hPpy)?{GH4%ULdJ0q zHgapAIak_@TH@+Lep(}nnv!Ln+ZIsNeZYua4r-!V*WS9p2{X1;G2=`zNy2KWp&mGe zHHQ_1j`LERB~<&y){CZ^2(3hjBfjXRHSy7k_XlGe%GR4=NBtygNe*@i+Lh_N4q~S! zE?yQdZ1LkVh<^qOVZ74-uhO3!+*egWFfy8XvVb_l$ zw2My}+O1+ZYQQ4dI1dn(r7Mrm9!v`{m7LLKnpc0KDp_Ky$zI1qwW&D_qpgpc#4)9+ zBhYUSSXWt@>1iT4suSBg%EFafRONnTN%I8Em%)h}p;`(nUSN(w8i)Qxm8SP#T*bf= zyyN7DxHNDjKiq?9A*S+5PD8e!i#UQ8hV4L#{=`+@%78)yiWo7m2iH+7d)u%DjMjoC zr)}=D99$QxP>4p^gX<_3R@53#A6kb$9aV@q>f`B<_Qgj>Eq;rHbtNv5O^>O_*n&l` zBjytdKz??XBi0d7%gh%k#O@wboml5Y@{l5h*d-UunEh+FS*qHze7#Kd0ocw?$Tw+{ zGu2I)M}Lm0U(l#7O{8f;?5w&F(Ob{pU8q^gm*GBMN4;h#oytY-$5U)pAc&@EKy*~C zYFqz~io&NpRqc~@s~FB2eAk1t#g?@lb=nAS7luo}S3tVSpc|Ggc)@!#IYa>*0CSX= zq_oI_Wij?}&cKL~6?My*d)?RTtB*2LtNV+;mSB8*!PI4CvP57+65yUlQ&5;x9WLU8tk z9OpXvrUBrNUM^F=NvK|r+N+RqJ^D>TN`xwrt1`W6r;JUC6&Sk6BL2FMJoZSHi;Bva zsR%+e$O%wK+1q5~J{%2D>%}<$)>__}${Iv<)-yuSR4J%5-T-7RNwkdQthn^qgrltz z!>WA~O;K!qRe6lg3Kt8uDJXVr{rWc>5$Xz#0ve1HVO;dIapP5%){5dd{YisoT!sZB zW&2hI07L;m(C&+Urf%vMFWck?3t^eSm0iC^@A@PHkqAv(Z8fTZ?a+mVtE`NZI#n2X zz`#XdR60Gx*H$k}u(g%H9i!NCG<1(Lg>|B>*J#>`5?uu_D+FA`U-X`33HwlWCDQ&L zQ8#J)I^r?_3p1V;ur~J3Gz;@lq-q`_cnI(i!Lvu$mvngju;n(E9<^+N9Z_m1S8JGBvr4@A0QUO?Z)rmxrQIWCUxL=fR43c7$S6)AEpwqzRF1BKJ~kPD9rVgPt11yliO|H=R^!ViSp1-c=JFFQDP8pHUL> zL@|O&)kA6AO#ve`A7}0A0hz&#uU_WMP0IHf+p1m%Z}7XtHp^h#p15%CSuuwoB?M6r z&1@Z&=4`Pocm}{i!7+Qa(~p%2{Q!feSnKK0G6z#_cAA+3ki27)mepX>Qp^KaihfULp`p zX91$vCy3ZaR~H|q+d>$Vt8q&+Be+VDsCB`mmoAP__9A?I!A(ND2|T0f&xiA?{(RW4 z{uD)&tny6&Ugy=$OJg~DzuwtHAqGmh*`HGD8N7pvbey-|DD89tv0TBlPo<~RxPY=@ zxAe@%=Grco;6-2H6TW4#l^o;VokLQ=~?YZke+i0QtvRy_C^yn5~ z(B12L@IWJ(rOf7*P+2VJ?^eDIKhGVaMzq)4+zKjd6$XroCUF9`r3<1E`U&g@u=J-F zEbEahCPqsovg)OpAHr=G_`iv=#;BS|G{gb84pLQez$GjEi&UvqnBN2HAWY6v5%n3j zk%#?CR3KOzGeC|&c91P|^YN0ceEC$8Yz@j%-ZqQWDymi^D&Tzu=QzvYC#mI9)k>)j zh%UqSIw7_gt|t*K!?AV410#0P)^t<7DyEzL3z&tw40CFieD@g4{1)FSn&A*!E2U#7 zYcsLtFuB9@n!W|JpJ1~uXYj(xz6|#V7^5CX#qKg=tHI)rDI-K05g#se`SzLqnI>#t zK5m8xGt!<5qX#}TVacTev0N^r7$#G>#EfxAR97d$S{WCnX59r`glF=LFXBVhMKqFF zNJ-L=UCOW)OBNKiA=%sofF%`DLtCm@wgoEjj$dhe8=rg0OkPck z`KFfuqy*SQve6Z7c;3Qqtqs1+P#wX!1%ejfk1djIdT5X2boaCLZKvva^1Drt=P(~A z-L76}5x?4YT)gdQyBcEy1OW&lAqYb3VYQU!keBo=N||=ZPRy%-2c!c)|D~8ZXd5-J zH!(E~Il~b#N}{gX_@D8W@jpVBqOnl*pDoLFUadgV8y&MyxgOM)zGTwat|wNhG~0j*=AzA3}kEigrNzYm!hF&e}&5U9_)QMcXGJ=v4w?au58>U@9v-<4kPh3`Aio8Pxl zQhfa2(e1)uKVFyczML-6+xt9Uys$MBV}>!LoCxdziwKIZp!Ne(9ezS!x#8++hdJa| zP+2oL_StO2Vx&CPAtFBq>qDtnQ{C?u^3$0%+Pwaq|6;wdi~iFCF<2zhqG8>c9J*>l ztT}rEM}t_SzbI`xV`Ej{I`ioP8=D}6!P2%nwtW12K`V1=!x@Y8r!b)QBwH# zdX%W=Q=50?j@c`VeKHG!_}7)y_Id0J)nzr-3Md98^GT`<28AnYu4av1R#9)zUKH2zB8w9EK~lJ^!)IPdMwX-UQf@L201*kKR^uz_x~SA4 z@ba_JXyBBaUG)7s*Q9455O`GlNrSD6#B?1M=JMp{aVLSwTtz*<*yHRX8rIM_r~3*# z>b@FmJxP=uPv@qTNUVKQ3*E(HqsJYTGqapVeY7e>n|90L5ao!EeU$cMIM_4GeM^y6 z>lp22KvRrw5ooiDU?V^+C?LuKm&1+iu32&__|+-RHSYb;iYZjmqG{fS9)7MTugEWL z9j(uELHHB#iwna+wAfXs%XW&6ZM?eUG z5Dr2NkOsJ3Ko20HM1D1@VPS=H%Oc_w+;t-~VdO68N&oQx{sp0-^ zs++Q!vcZyYlkx%@u*w;#5CJvF^JtZb0 zlQQ*-d@NZdfZpmP==5l?A=OChCx?XD>d-%g<&tCm@GVbF)E-`3`9nk^0;!3!5B-qs zaQxQ@iaRhvcNxHgJ+cexR76r8pmlQuG^6INvK)Y=8H2KUYr$x}ugs<@#ast_JW1vf z-4{)0Z6tEUr|xlQp5Wkyxbs?L)_Z7?ETDk!;3mBUBaPsfr0f z@wk7b3dD%1gdhh1Ew$=xiy=u(uX`MslP3I)oxAJP+x8=r8rjVS87&%yhSOoL@vK^j z8sOUL!z;+2je58CaQeA_roJ%Baq%1&S^LVYSMBzxSt1gFSY6)x2!40pzee) zLi4DN?h@?=!1)yEIzp~l=(u;T<9&ix*&1dDEySBqZ2PNY9N-{gThuZPrjF{wd?*>> zcNry74$)gM^0xTdsG7yhEm?Y1bRurJq&vyq(LbVuCH?L6s1D)~Q4W+zLE!8YT1t+e z^`6@B`oxy@^qj2g3eVW$f`(rDPAdlrmdU6yXo17;TnzRaFeWy!10r1lhXdvXQTGh z!5KZtwGZKVN|aidZgf!^L~U4#QcVN;O8Dk>tUGpkT{jZ6tS&aIK|J}A_t!GhtP7tt zSq2d-dvu+&y1otukl6N(&@4z4(ZupaUfU3x#fd_sSbM3~lvri#L3EVoTi-3G$5acR zy6hTd54v@oBuZd}<7(urNQl=}sVE^D>M(B)ZpgQb1-}xkXMZ+Hpk%5ig9chhAE?*b zIb_-umH{&Dx>mQM(I67r6^fb#jh<;&#SN8dH;EoI({55dM5f)Od-P1ZN&K*xc9R>% z&a|7n;4jnG9gXG)q)|PfyF|@_Qf8NEmGp@ERkm5sMaFOq-5>@h1V8gASj+EEui;a> z0#2;>$rE6(wo?n|lk$B0jX_|&z|ScqwnKx4)>9cAWH~6PLEAEl84O+xReeNtlHriq zQ$bJ~M8X9rUQsVc(^~KWI62T4L#C9Rz3PSm@6p@&N|$^U+I}36Gyc1Ro~kB~Bwx@G zL06IeHiHrtKa;$EkQ=8%c8e8X zAV~K|XnBf6q%4O(IO+oU4pA4gLRY`WFYBfq8ljF$u%Asol#;fS=_|T6k=}(LsvTQB z<~c1imDcfx+PZmJOC@40v6EBVnr?=3!>y@7u54yAy{!S)c2e6=*li}Z?aNra1W}@v z#lszn%PEYY2(2r1Vn?9t)3uhEkE|<~V)7OWMtVL3e!GW6fTu~w)4>D$tBBTUTX;cF4VE(|M-!|;Z85kL%%k)_F`Do6~huF}pYX*NEMeLjZrTBl;ec~dGt zN~pHF%2J0-2-e}{mBl2@vyW&gwrrKa-V7S2tKr*N%EkyZO2BAo-$looAvof@|KpzheBrMI7>B!s}@ZAy$C_`xj1Ku5DCgDbzWe47}1 z%38K~br55$Fee#QtJgug_A=#lR2S=8z{{2L*eH@KJOp`^7cMGv{YmEEhi9c3fDF+l zUklZ|nzUtkC%rPMHe8vbRgz|muHy7rE{aQW ztxAs?1OgEV__iei;Sig%MhH#<)sqnl?8R{kr0RZoRwz5av3eC=vq86 zlDuz$!Zs;w@6oi7ljNHqV@csxh_-5MyTU`1aiiPAbMqs&UTLra>kUNCqIiy;X3;8w zGHRInye;rvQKku)uCq95kpm*dFL)Zw7ZB@p=e0+x%0kMBUy`K`*G{6Wx5{8);g|~L zheoNPwbSQ)h+$1s+XCv?Lvm{}x(>~)Uast>-M|oj2?|dxnN+O!ECv5(8!hUKofu6PQAk(t)1XLQOT2Jpr^R?o!3|hu zTT2uYjn&Wx-$32P9ocSEr1XWqNb;B92Xk$+?B5XxL^M#@L3@C-yLdS%dh8S+gv*)R z>b^G+b`UH&?84DBD*N)YgOVtW$4{#Oq0^^H)Ir!LfHu4wh7$36jzfx!haW@lQ=C0B znrcZ1BcFf}K8@T935g)I*3!jaS(+}@p!AU+VPsK*SS3pXUk9~_tPZ+1IQOm>j_IMS zx)r*Sv4dtgUR|tJ$IAq2VCL9&TMnw%Txt}iy7MQvEj_XgH)>bKfKPuW@`X@Y70}iZ z4b5uq4%x#ZC6+M>46qLIwsrvCNyoH5kZ61YDhb~$z3+aRHI*g`12o39m zVkantZc?>es7DR3dMN^6Pp$@N=4zwxbVp&&ilo7l-Gc`43s1Z$2z=gRL4*}Wl&tbg zxjQKQ0oz2`mY>~(oUE)5CPP?0RT(!W>|n+9H%vPagWt*GTu*yJ<4r^ky8Kl^y4FAZ7R|er z%MDf-OJ=*!aG%sLysfgta2>=!iG0v6*&AFSk`u^=S0e2aZ5J!=7gdZFI->udy?0S= zROhyZUnMV~^x1o#+V#H;MU3LGg25l)BwkPdYhEP8%|=4p(x<9BU~Hr@213`VIrD9J zn0$#x8-@puQvL%DOVsPX9tX-3>Y}mkV)Zfjv9Lqc zf0d_HeGfKfKkD|U@!4jhw=)%(DMw+m0C>PJI1L$EUg|@0wMbrnY2lIN8cW?*Nd}>3t3>T=*Ke%&XcIFJ)-AvA(x@30SFxnvJJ?SmKr0f z5$#8W+lE!&XMJ9bKo|sFNZ!oY#N$%rse1m^7S!1C#6akQnn+IH$G5H2>~wN%YUaO|#;B_`m1zJYlW7QS1O2N+QV#KjqKHgs@9IKjjy2t=LFb9160kQ3asg+b}y zq-_8vY-NV3aB-Zldm1y+IlAG5u-U7$5)iZ-GtxP_;pD)^GKs8cAB%%Fx!M6oUIwwa z(V66cAP9x&-UJ3<-pVYx#*f`}1wpAY5}1+BaSF~NOHX&$|4tDO76OU6h$a_vHXUL^ zv|1FTosV_y0JKc=Q+kcFg>g37G^bNg>K|k8Hcgg7sk6bZEo8&lIYPWI4{vg(wt;o4 zKOlTD_@PJ^2iVH2^~oP}bA8E1V3>n-9rJEOapJ_$$4YrAYPsC-!ru-!^ z`WyWPr=S{n`Lu7>)vK3_k*%Mp&v-PQZz?7dyNQz8S_K8Vu%a2BWlM=OT~o<`(9ZR>GjL5KZD$sMKiW9A{ba z9i2*HnXdkv?2)&r9Zq1h(-$l8szl-uSqL8eckYy->p>Uwsmq$1qDa)U0 zQOS+oG|Jq(YHcizM0jXE!6)MaM0k=wXWqu*NCdh&@ss7o@C=~i1-WwWj+m>DI8h~X z8}TV~BYqpp6e5Z0F`)z4*WxP4{B{irCR@>DYtGzotBkMl`Ppn(aNJF{=4=vfocmL~ zRh54w*_7q2NbakkAIMx_=J3ZGK1>PTs?7+P?BOtl1XWSf4V(gjx3ir-HrJr3WzZYa zA;v7kVqC8?(#l`x{J9g-hT~3$A|r3ag@QCw<`xil7HR$^%gty|?##o6;!fo048S4; zCfVB!116UtTPaVuv+_0+FCvw~vmq16zM;~9ap!`zT3a!}%$bJ;#GOSIze8N&v)rtB z0Rr-yzoPlI;+($FHtK)((-|`-U$*<*tcXP~3@tS7Y_2f#HZ!gT4dTbn?d!7uPW(=XPbJ&ejsWhzvHPzi(T0pb}tz=+!fe%9Yr8%h9E z0VXze185lM4B{pt&6Z>>5Y6cfQu<@02WaZXWK)LvqtJ*1AsRtCh5LVQvbC{vjaNJZSCI9Ftu5|o@Of=Pg~eKBY1sn4QkCkG@i$>H*RPg z1K9jH-c^}SPtha;t;Gluh{k%Wi)s!voyMznD*ANC@6tRqBIEm%5Vi0HVqBfd^a`QJ zKIMHUlrMdyE>;E=OSJkHgXnnNdg!te6 z3h}>vA^tav5L*sioi-djrLcoB>SBY_J&j5i9FFd19JAZ|+$ zu!!_sXe=#gGila$ zE@Lob+zzHWyf}$HRN`$!3sl`5$~^bC)`IvmzC1gLa4cvcnC_sD*&!Rgr^!2gw_IW7 z4Zj9L3y}}0O=_Dd@k#~|Cus#+dDgEzIAf|L8W3*i@$b60DNFq8h7}wkmuVt5wmQ%- zLfaj5>Fr=Dpi4FMoIq8tw3V=-q_dp3i%^@P>7{_8s#moGZRal_|Hk68g~o?XU0PzWPYaI+A@J?8B_GJYq0>U+!v@{yZpa3`{2Dek8Xq<^S6|BSGF@jUXqOJHA7}t= zJRXE-d&hQsXupZhzlFw&jb&!L2^-7&_oao#gUtpO`yD=A>wFK3{4S?j4Fe;bu-JkR zyfQ$F{wV44F!BF@`11JKZJb>xLm{Y>bkd8{G*~#>^j!&&F^#M%HW+uR70}MqP zwr8w6hdY5ag0d!cEr`y%;byw`^}T?d9IKOT5C}#}6{#?mf%X<3*hlvsjvnOx}oDanf8^KPyH?OI~yRn06 zwuf#*2)oe@h9O{nf+ud32%f-XpB}-z-QM5TN1Q;J5F#NcbeM$H zV@$$96L%c*;4#eUbXQ%U!4s3C(?Se`ZH=s>x&ZsiHKDi0{Gu+v-1B7^)+}0JJ^Cd# zWyTOj!MX6`rNr%?8?S8k9^4?XdCb;-bbWH>*^R+-pa_3gWP{n^-{}}(E{b;|u7?yd z6(&;yU^?o$j8AEPUJtR(kdF>Xv9}6M4m+-2u|G}7vF{8`B0JJ=b)$R0CNtCSt*=*^ zC(%#zUXtK*=FuqhE7~*?z%adq*Go}G<5TX&`bgY`5!1ZNs zqBcbowrMNc28UoAV*s6r7GH<_REZn;5|m24J}|`-rQLjh+)^t)Gwu!-qi=?mcHKHHdg3%Q}_S7POkc z@Qg=Tcm`1hVcjG*F$U^d^c!HVF;ky%FFsSbDMtiAhuK8douC_I<1ukJO6X~L+f@i5 zif4gurxB{_YJbsJ5P78@MF zrz^!<9${r5=o4o;7zkyf@eD?1(E}O_h%*UmN^YZBDtn&L)cbu{vzTQm#2U;trKT|q z0h>jR_r9v?byI%4As1kbz;k7}`j#h)T5GG5^A;zYM1IO{iHyFK2xjBW>N5q5LVSH# zTZ5{XVQ}sgvotqONH(7%B{<_#rBC3xv+qPgiq;FF+NRwJ)TE=TjscucbeqPnPn2P&zKmaxZco`+OSu>d?#?WF)3-)|T_VFTgv{>rZ(B6>N zwchP{Z6;&h78qA5+)L&z^Cs;oci_1$S89WmV{HaZFoc0I0?(5rrJ@(Lwz1$EfHK#r z>jArs=FlFH3}gK@M7g}>(1(fM%WVd{L!=p#Fqpa^Ll6hES{YUG<(kQ453CW8(12R- z-1*|x-c33nyvvyKwt={k*l66Yv-KRM->&CUy_@`-)tdy)Ug+#Qa6H-YPuM(Wd2XXhCT5CyEk(-N3+nkT8Fuxb}#u0T`0Dg8c*D0TlRCk?rY`^Ctu-G(cSr z24T{`!TkqrK>AMV#9JJ_$|}(2e?Ax9@6R+uLd9tfF7jD zoamyS3mRMEaG>bJs|8|o9i|$!S@)%zp7WyQ1;-}LNR;Mu3Fl>;_F>H@PmpM8rGX?; zo^3IZCPJSn47e3mN#gSCmTG{rA%u-EBk!HFwxub$shoZ064s(eF9`(G0=T1)g~vi&~U3`^1k%F za?N)^f42?wwu)!2Nt?sj2qymR;b&u6gAuK%YjUH6eMLNu5YfML4c z)1BeShMwx|5}oP#AOz86oF6$Ww5hbeIg6 z0%@MW*<&U1Dy`EMJ-SewOvqd`nYXdH65+W_{IudS6f@z8Rh@eWjU$&$B9Ap>!bBWv z=zz`7$`mk5_~Qt5O_$pZTAi!wc$hG+y7`j;yYKXDpwc_B_BvaGTkxr6Bl^rx$tPBS2uW1mOE4xkTFn%Avo5Y1~poasIrtnm0sbTNH$Z6^pd`ckl88(gO?Xv0q5&ldd;f88`Yx z8_NK`(0rp!3+4c6Dy(Bjr_g5~r?00hRj)0XJ+9UbUd zKf?gC6_~9B!38KVm<3m%kpL!$#_3JaKbex-{Mc|BWf z1{t!_7dfj|#*^UEBs0(OuQ~J7&e>=7se^qjFD)J5POUxP3sd^5dHS^Sbo2CSgH^9V z>}0(W2CvSRkFKcKb`fJkpXDOPCZT;qa0q7g+gpwnT5Cpov6*RK77R!rAh0JhfSyA3 z)gZ0Rwp|Xpx7v1}AqCRPY%u1~fCT3WTwnIuVxlGMR@}8&N2?UPJcR1Xd&N~Wo1xCu zVcE<^y>dH*ki8qB?St^xG!_rVV^d#wPp-{##-7|3gUzD!w8(_E+IvEAy|6D&ScWj! z8{)db*5o0k0qym=EYng(b+!pJl{L(UzgRHb8Eh71sm&(1x#2UGTfKS<*&4R#uDkEH zO>oA(+ZKt{?7mv?T`j+C$QwpHvNlp6*-wK^~p5o+y|VzFK!-cJ2wZ5B*u%kUBQF?3F6Vkd9q~5 zY3C+XZd^W0z8;A0t0KM|#CK1`@X*YXFqb*bq6 z(c1vi7P5IHRbNN7(c7`q`47=-WAo+7Bi?OGFptQ`i|{DQnY1@6#GN^aA+kY=F{ia% zIcB261$hviGgbUp$yyC#v!zBw9dbHLL0RHoXWV&FulPJvZY;jZ*BNn9t_Zj&8L!Uu zIb1c$yWX@jS{SS9L19$>JhvevDw;y9NW! z2mZPu#5gric7udMY30RdQ_gWAC&OTu) z-!%9bJVSkB=PNj0LHV{&ICDtzs*dxMe7Z;!n1jxjR)<`)J%5ITx`IT;>%p@@!YsO*oHdt%SkOPVtJ`6@EKUD;Uef zR-y|!OW(@4yswK|XUfMZxQnY)m2h6&D$_cs#uj==6t`)DcO}E4Nz0K(7lhCt_$L~~ zFg7om#VJ06SA;l8N?BDa^W)rw&b@h_VM4f<1*vHmCg95$zFA~3bS+mf z8xh68x*Ea|E+$Tf24BBU2lR7Z>Nk z7LLlSVn(LE^m}hA^jk+B{ha4B0X4z!U=nP!P{2>62XyGjV^;9T72oURH#iK)Nn7KF zkoQWE{}(hsXO$WVqtXV}W$6a3_NI1F@dbDO;V zU=!un_0gxpq(t zv4Dlw*Z8CcCRJ9Pfrt6PRx=#;47`}D>hqH3bDBbD4%%2lUitn6k*qV^lm!g&r*G`sX*8pnbYy%*( zl=2MmY#p-V6lZ(jk3fcaw-C{-gsxVpR*~6-*LG@2U>CmEcb|u15MhE0H&t z1Y;76X#(!qf_#IkM%u}&UMZP=jm+tLpa|BzQ$xodsqsIBSiY5x#^6T*T^rSf`h8i|k=>i2 z-E^xnL-$K{ArMfj3r3s%UY;Qip~LN}ySBSfLBr8oIbR;54XQxBF9qty3Ir(wS-h`T z0q{I}6Grd`5TV=y^Kyu88I0p@-gP89`l=6N4N1S=75(}l{dpxktH3xCyq=8(m{^o= z1|gPB_H8tNjEpXiR_~VNWFDIa&_y(p*S52MhFyEkZy4Q7rW?=qHPughwr>RAY_d~B zdacBHD5%$Cv@;dh&0J?w@s7=OKE*s!iapG78Wo1-I3FPm$9v#-<#A-IHE zAbPO82_n2xkv+7iS(u;3V2oosaW~aA;oe3%gDiaEud&5C#vnx_{fn7J}&`hve(g)F?Wr0utQC)6`yi+Yhvb zFa+f^*_dbA#0pE35R5KT^>?;r#UR?4=CG?yAMzqMVx=?3!yFc7cA?c7TV)1hn3}jlGa)^!vlOVn=g4_O#uYe1K>!UF7e+KTnlLt4@v(L>jSdXD7JRdwZBGef?z4KI zpeKMaL~8X4dho)~Jr#-EoG5~5K?+B1wapY$NSdt_gAl`zUbm|^&bR68wSfg?xsx}w z+v6Jo8HTrxeXu!Y1EdFS7SZ9Rd*!*R9_!)0Mt`+w#zd9Q70pdw6Pd;LD9yp}c?H_7 z%mrpHOy+GYUTktuqbjl+ZW!aG$hBGJSb76;Ailv|r{(~%n9*!S`j z@k|-a@`x6>*M!ZC`_i!SfLh#p0&T#zMaZ=zRr@%V6pmp~?LE2HYPto~5r z$J6J^+z19+^4KuO@cD7QW%X7yEoR=-v~4UtY|{KI&e9~JTi2fbYm;Lvf^O~30X{_P zL!C;4x+d3t2dRKk&K3$E5_zGm@S^wPmaknN^Fmt%3d+8V$CFblB=}a$TJg#oegoJ% zSyD}OH>R)aQ)C3DzxpzO%99-JBHHND(QYR5$Sf>AY>F>gWLm0y4#qZi5`))He(QL_R#_1Z#1C*giZ#P9TMBGO61nn#VAU>3Kf1}6D z>XWMAlC4yL2v*u@+NK>46RBpEwTO3u<2#1sw(X#9=*MSyx;7Ruh(#DIbPP^1 zinDqh$1e_Sg#yK%pF1dn^H&K&y05n~-ID?gO8_kO--q@;@&Bk_=4V-6%QRoRG^^Z& zoqGq34BIXfVb*sKv;oN&>jx-(NP`E1Q+oXX-)gtC zUb}h%e+rbnpzOPNd^uI>OK7)Nu!-(7;>^Jwx{bw~O`bk>t-UaxJZ&6ock%dgT5H>8 zpN`JiLNQ8$7rQWK(yh)?SthckK9T%VeJ928@iYl1fbo(jfykr~Q4$jv=|wn*oHPVO zFbq2Qv(amNj|R<;u#v`?}gGc*`BioqEGxe z<+n?ON~tT==EM{l(V%Hz^QWjP2Gt&?`AL$St(~+-081b&@(re84WjjD4-;!u9bc}S z6K;^0vl+DR*X^5M1Ob2id6+4|v0$HT@-efQ0LDhGQpCtLZPssviyQ@D4Pv;F+H=ytWXJ0GmK$zp#hUq}@3GqgN|GN?Cmr zS+XN}+Wm`v^t!7-I+4*F(f0BN+49VKcZw1N6!>oz>;0@aD89w8{>DCaR&g@5hpvPV_Djfi0Ymj}5m-a*XN z?+wd~C@WyGENWiaEhyxLARJbO0yswSxf7i-PF{wZ)3~?~pWn8vx)-`02-2@GTsGa#-;T$7uH?S z>O@Dm)u1Cta9QM5t{S<(Tx|$m9H#Z+LUVBMzCGfgw|%Vp9P};}%GHA4!h!EIK4=$* zuTh%}^@$H^&3cu>qmk<)2bymX_%NvB@+|9WHSjt=K6*i}*DEEEm-_Dk6Ki!oUrk=D z*_%xH!~?y4DPGqs*~XF^GxaRd4f5ON?a_9xrLa@Bbnw7WFUGpfQY-5)ZBv^K&h~9U z1?L?xUxUD(#A6?`i{O9mTlP8XGJWE4dl}?9TYcis;9I8WkF}NOOzGf){VOf&Tbw1u zB~J6#IIq584q$~QtCg{!jIsS{F!=M(oxYV>m~8clKZ8|jpEi2;W=#I5x;95I7qIc; zNg$vQpmmPC5ruLbf-#xHXI^~gQFVKNPk)x~`Cwkc(Ne2t$v=TpB1q#R&p0!(!STt&Z_vG%qXKl88G0aevynxB_L#p#=zCgG~f zTMp*k$wIiA^v&5bT+MEv*NGwB7@>(d~#pm0#9h61v{l5rR(QOWZK=G|M77)~Fz{BsHP zLs_@uLQvLyG_#2<__jV|RHN(ZL6=dWK0fee4nnH)LxKY1$}!cuX$op%?@y>EC5q%MIkmd@cPuIHpn2 z=7~kMC~|Fxh}qD=1s@cx#nE9V@W&O|D}1>MJpA$n{Be%;-c6GI?TFy9^1o z7Jg_e`yBcT&2gCiC;slcjIUQ&Vm~_4EoFpXJtd&;oetm55_{4@XGveZ48U-wq2;0b zBNk{CIy!598anF-u$<`8<6rqXQKsD|H3-Fr%r;MpNsTk9jwVu!9l7iGXWlw6<;G5sZfp zAqf6cfveY@u>??>ss%6vU-f@?bgc*5zzackG?w)yG#`eycBKY(@=3JI(u>eqWw-~{ zS%DgkCz*i|@&}@a!7u3x>w>!oknkUsFs}Nl^k+x`NePt(+|tEpU3h6u-DEhI*Ap$k z!s0Z1GMZCdjaO>qqWA1peMD)l^8Jc4M@=AIbPknO-eN!+w0Jv$lpAS*NQFTFyVMDPp#j=xmr=rY+k7pu^mN~`Dz*n|20O+g zv|7;P8@0&XS{>-lAUKUOG*K8j;ZS^KX;PzX2lF`9m zNzKE4HwU=6X-Oj_TxzXVucHhw(@sf(N}RZ_K0K_0<=8g15Hs#A;>E4rEpZ!L2;$#y zg|y+=koKxqAPHO}gd}~*!p-p87M`Y-m{2{u6$LES$JbY!bA>J-xDSvyxtG)z%{6-_ z^7HK(XOsG$Uh2TTpXFpBOv*whB6K2fPZxJWR@HQm#8@+qKno2Xd=kX;y!5i`&mkC5f`MLKg zD|L%+)dUsf52=J4O0w9P)ZXDzPKJs zMoqwYjnD)h^9w8{?8v16%F}A(9mog!f+-v{L+)~zDp4tuw4!K->SVBPQe*pf@GLy; z4tPi#Ob@iclgM8Yyq{tqb`hFl!wB2{q;1$s$nhxH#k?p!1Ecw<~&_LX`)S+3Du@CbaN;0uN? z4E!NZq2%k090*Xvi1sX9Zv;Wmg@!&#D!g{xp4X%i#xA7m_9hYe7^U#SWkvCMyQ<7C zX?8z`Vgy+*qmGEMz+OfP(9@6eYDTHhWxIGf#H9B5l>J_AA z3lXf0WjBofG;txyQSR9Nk9V8?m7!tbJ{qke$0b^{~T(-nSajXA29 z5kk{cyTtYBBhAmWkC4CpPU{c+0{0T~MYj^B)BR(A8|g${CDN(UPQpIQ=Atf4UD$!t z;Nw0;(BSwOi1JTTMZ2c002=BR*lF}(tdOObvn$Xbr7?O2ee9Am}yMarEJyJ(}g7Mj4Jw({&Q(xwRk8!=jaAk*V z?=UEVaY#_EIHuVpTSA6^12M+WJL5{2@NXcN=31fpj%At{SJhQ?Rh`LOzB>(ih|T(r z0ADwVxW{2JwGJT=f}Ab``!9-6V-Y^vbzHsU}cr1|w$M{jC0 z@fp3PlO~+E5dwh<1S$|{7kMB-{&S17vFLYu&_UzlV<6VwX`U3{`z`^5ksiK<*Ij`` zhHp>oGy{oXCmFs?84*F(y$~&ny)VxH3J3FrsU7A^PMkzPHi8A>mv;8r*enSX*m;>to8xCuEzVBKJ z19Av?06{MDQk3_dmI(sK#lc1zfVgLKx(e_4%_w5(wjXmVziulrlX$m<2v-3P5(nFd zG=I;)fnEts$u9YWtp=kMgtCV_h#GT>{~9{Xe+B+kv^p<7UGjPX;MUH2kEwalhC4g& zJzbhv_(v(vY5mt>9<=oz(;-Q5+jm^vcPD@atvL-qS?1WA24jrR>UP;3Z^9=Gz7F01 z3ix#MR-tYIbMDSjah74Lw{E>YlXACS4_Czv5;w2Sw3YAmskZ>yAD+_3#e=UnOOv=R zU|U$4rfz2|%xDxJmXN-!e3j z!jnX1@m>Ao2R0(WxuYhS2u2aYWPO#uyYVG6pag;v1j+(9pDlR@-y2;*#e8ju^9f&N zc~Ye*KFj^kwJ?S%9v=w=wyn5>3D?kO5iKDgKQ<-=6RB*?>Qt(qDww zpbaAgzdJj8{``s)G8<+X;`@5d8`^VTb#Tr98Yl6!PQP$J?&(&Ry1m!e?aS6VX{%B_ zNeK?lz)~HV_^s971B9dXso#_1<|Eu5lS(4pRY) z5gVulnAjNOwPeOTPjPm-WoU9mxwjs~?V7-P^i(g9M|u{+>{}@j`^LHC17$XTW?($f zD^7pG(*iS!-uv}3sK%mC2wg)Ode@nfEzOZW5LMTqU${OIgw63r`q_XUnc`_RP z4Kj~WWW!mGd$5_%zozzoB()*AN{#+V?tjv{{^jMGDbisxp?S}rPUlxE&YK+#*ADm1 zo`DXfImtorp`U;bD@sgTycbo!iZ9npMkeG;#FIK{CJ(PSakIkd*xbf>yXk#NEA6KT za@V`HE8by?*D#lAN+{rtrk%t|&~A>tYH~VyjIXzO+sK85yg3aokl%dM=IYFvW16B1 z3{^_`55*~hb^5pXEuFIKac@e&NX!rsHjz<^;ngg!)%2JQds9$>w*xiURKhETwIgi3 zBcuDHX{F%{4Zlri0lmuO7ObkY%9TxI0hw+TFY#1@!+Tv6?DZhBDA=9F^j*@Lze^>dn z`B6HZ@;kuUN%aiF$<}E9S{J7x(;je*IjafiIr7V?F+JN-kEMo^kAm`)UZ*EadK_8C z*ua{gpJTsl;(k7n!Abq^D$3%^TN0C_EE2kO93Y!SP`f5cPqDr$K?#tzEYd|`QvYXj zOV$14yJC_bqt$sf_{fD(2k&hD*Z7_l5KSzuD%Hg-Zs(8PX}}nTjd1|CzZO?nsuXHz zm5ER3WiPU%8f}z+7G_r^^t@)3o^y8EtS)O8k$e$DS(Ne{>V9WYIyq={x*7Y+ zq$g@QCA_gqfx>cV3O1ib1VdYMFb&IYU>ZiYh}A)*Loz~>5t_^}$3^5U<=J_rb~o}d zcseX1PbJH*^duvft&uwui7C(qN}JCjfsc>H zJp5Cp*nb}Lc*~KbK!vT?gzj^8rzD4Xd=$(g$>CIQI#J)Gjt`U*L%KR)PsT|O@dGKU zg^dfpfyg8XFAPHZ5aBvb392)GUjWxr`51u)Wy5lx}prh77UyGj@cX*l*(shw5>`rEMOhA;VlXH z7&4ik>yHD;#QQ6-Kr#3}pHSlQsT%gZl@pxYZY7gG)iC?W0||0~GSeyzQ&pfW#r4TwYit4KXR34YT3^NEKPAM8w`c^?izna@ zQl@wrVW3cr@Cd5X5sTFz^3}>!SFr=Nh-1=i(NYeJdBhfUSkuj0))`BLt7#W*J#b71 z9WA;#R=Z$eZ`DA|TU=U6r9g{K&m=GpW7irqD9xFh3KaGNj@KefPj^~yi%z9XU^jo9 zx$>vL7MQRvXNq0XU}87fHl`lRAgsOC7Z+sQT2%B}O&nu-k@ZKB?1!nw(?S=7X;G%< z6oWlNTZ?AUyJ1b(1Ysk~?Uzy1LDaaG=@;xk%4Pjsl%LoasrFM07@lBwg5f>FK2sL? zbA?5m{k-`E$h`V(d^5??_ny7OCCJkErYqRSH!h~v1Q zhe&|TLu})7B3qU3NLC$nt(7DxM)=CON_2chRZ2)0QuZXQZbiZdlxRQ+4XTfGChLR4Wy{dh(bBgpLWWyEdTU8eqV~CHu@shdH`31H0uG1dvG*FLD#-7q(D$Q^dRBFNR z{X)YZq2;1oH$GwOb2tNlZqUM3Hh!{o0~fYF2R8)p&?!CVMX7pH1sJU7)q7E1e5Tv> zAsvwsc_9oLJ>xdcGnl~k2QQO>wMA@w(13?n1d}Mpdf(bIExPVEMXv5U8z*0~tkbGD zfqVoRyYH+a)151S8Kt?jzW$3~JJB@ITG$iqLd>VHFW92f>5~*Ya&cR9?KQdQ@g+S) zs&W2~%LJU`{M>s!gpyo4%kfccC<~NsFmF!{DJ{Wi2~4Z!CMe0ZTOj7qB^F+~D*lqu z?ON5Pj4zQIsg|x&eml|+$OuhF7&1c?gDBbYIAi-F@#-;es$|D8jqPPc_m~F=F0Nw+ zPRbm2kndMW&1Q(Ai1kT&BNaf?;pi5@E}NA^!y4iW@v;voAJ%sS z|2f0JlIs4!MbpIQPl{O-XOR%|7Syh$m z3hQ#BwV`T*0)i?u)IkEF)S2gXV>M+0yWx1G$TdEFMsc-gH2|WBD^{Wf*jokgBJgQM z!0lK+wRP)GlxV>e4AZ*xrH?$2B-+T_6uKh0NAzG^C$!g+N1Mx=H%CDh4dlm+yk%1# z=zO1JJq>TM@txuSRur}BA;j0FfkXOL^eOLqQ^SC5+G+wT0PI65^OZVfGA<*Wo}h`v zy{Ur$1=%kXz#ioJDkL($M=`9@;d{9^4FH~qYhA0nbl}eC{~o8=tt6+Jt^ z@k#Z^OCNLqA^Z)g)(|0-Rn-*oroP#?3Z~M08G%1v7Yf}%j*Isu28EhoY&OvOv?=1p zK50{g62@i&oliT>(d&71m8vkEz`Ec)J;n9zBw(BZa0(me7=duo)T;MUd4$+pQ56MI zXov%_=WAhlV|v&F9?dH?B9y!9ZkukU9<*6dAJCYuk*|N%6*LLpM|GU@4QIw-)vw$m z?;B3-Yj`cV^ZBFcRFW$OXxbQjD*8iY9< z)#&F|raL4~-~@s*I9+KgOed}#Ax>llpUJ2TxLpdT$Wk0 zII9Q}Y%XV91X#P}b?gZI=&1H;pIX4OGXoVG=m7YD${a$Kt|E)6ajUsKx@8^%ly$^l zjDj$>A%~NK&(O1mYe=-!v}iz55VQ{HaB9q%(pg1`KodKb0&v>+2-{&Z(gL_hE7F!= zpKnB4h2>rdIGsNfT~32=!_YC(e$LKTVm;Hxna#T?Gf-q+TgE!G3Vx?B22i05wQY8U zP1eQie`8$R-I#yvz9GgNgb$BoBAzmjosV^PW;O6pFeYca$YuC*oi_x_3G^bu9 z;S+t>y1W`BTu%K9Xopi|z1tCc=iU!byB5;E#9u~n2hqcNF?Ohj^$K_N7{DDw4}OJq zs0AbI=+MDCQ-efi@qHJZ2?_oU!6`~^rJiry&x1v1LPIl8b{KIG8TMGbNri6l9>hCW z1K!A#6yG^-$vtJa3hc0zlru^D zY&~0wtH$E7zifaRI%YoA>6eV~!ag{n+h{~37*!8vXx_W3^LqSEy*G+`rXFS(U&iRO z=aTNDgKs7s0GA+K16aee-(_1kphmA{oWNzKTE<}ow=27l#yHy0pt>lXNT;d+t?uf! z3-4rZcKE-^kN7Js%GbC1HForc(@z(b=;!7si%LjPLcBk_+4Zks3x-XH;G(?WvP|#( zKAbcNo-e-GOcs?Xit{9jv+wv`MQQ$zJk@ieRqstlq1XvxCx~4)Xt9$P!JounE>Zd( zPGTJR{o7=)d`fNSepMsi&$5sR{%{;ax>`^qP;j>M_ zL$<0eO0aL`mJW6a99%(Wq<_wgdMxMbN;NF&s4L_Bi)X_Y2DUy4EXze+iqiC>(2JMP zIDT`Th==|bdGNS@lwn{cJ4N`oQg+@?Tg!2d%ZjPmr<;sE((_039jk;carx=tBX4)w zA};~FgyAJLuZaq{D1Sa(aX(j1?mm-L<|p}jE8Sd)n^Zu(CP_ch)cv?cEGIvn1W5oS z_1~MD`he%Hrqnwe!g7R%aJIu8_=Y-oE(fl5yKWC$Sql6aVANp4N>IvLH4PNQZqG6x zPQf^B&|&4Mi%&He^@!52vK)w02Vk&4d&_}|;H%cEE2{Kgxf3C}2)UL(gbmnv0+5KW z;KO21tl7E{VZomuB){TJHK-ojxYi+y@bo4w8>;?hReR?~K(Zy4 zqPH?l&J-S*iv8_+E@N@7)yWWAmvP0gR6^ML{W2EOdMy7_!ii zA<6|p9^>MvRVl%dnY1)(4-0uY3p{%Hs@2XSq-vb9rCxii$jfyI-e$6lR>n*OI-6c2 zg&k;S>|ua}9!3l?TSi0hk@iujr0|ya5oi*}AF*8AWu7uUFS_5kq-^PFC78w1R|1lN65ld*ES&R0s{;X0ni2 zr&tvU)g^&+&}dTTw+jTpsg9@~un`s;0c?b3^N{OO`W89Ftd4v)Z0zR9s-kx%RDqP2u50MKVH@G2SDH%p<0YihIjTc}~G}8Ddr5s=ByrA|*Er?X1y_GBG=Rg*ev{^gnqX=ViLasjRaNicwwh+BIg{LPB5zN! zYhE-Z&zTUfams55u6Je2?<~z_lt?r>^qKeDliYgqIk6Hpt6{o}Nez!Qk=3b8uXRx# zP*o_d7J)9Km4u7l(){q^2_P}VY#|Y?xd%0#Q#r<0Rm)@toD;OBSA=FFkjW^=g*4Rz zxvw}&6Y}_S)FFZ%sWw#1cwwC7va&Y$>Gr9VxHD3{BneFS;GmCkJr!A&R!3I8$5s$q z!E9;dOSxKXr%1BVW75;e7g>K4$*+4ZdKR7O4U*ni`NPksPkjcx8TThbEc9I1|NZfF zZh-U2LIT~&tZA65t2`Y}DaD}abRdHsCui-@^;ge;MS^n}4mAzG@R&Hmgu|FyU{XE% z6u_v%WD~|7>QHK6uf;2p-t6Q;`14o3x``ip7e^SK- zX0qXDQ6|4+wk@{?jUY5Vd_kD1#$9!I&iCDrHGqat&Zg`-gS`+@1*19jCdK|MwlA#g zVuOI#sJy9ttG-n69lfQl9p=Ey2!}u%7A6;7I?fA6U_{rKc*$JRCvH?b=>qF+R(=BL zXf-;5B)G+F0TvBla~4z)W5jW2075zOKdN|w2?M5W?eZ!+qZ5atmd6qvBd2lG@cbV1))ewoWUkXiR zfZQ@FI`7ThjckbAbwloSVessj$`Ew?8%!Is&%!xo`b&?&kMk}8X2F>05LOP4&HbCp!I<55??0EL$W1KQv~u1Dyty((Zbl{y zzei{(z0h*laUOj~dTIDQLSgkP<)!#KSnb1@>EI|wR)0j`%JNY3n>gXh>u zvJ6cY-U6s31SY|=KpG``-$RrQ;9wGP-c(4+-ASSkCdkX{8R0KYfT_}-XR?w2dbEr9Y_Klm|M{*PzpnfZ81-Ra%K6co)Mi_4sFWIqt{q%{ z9&8^n1^Lp_u}iQLTZ2uXc(eZJ`?HFxdwzPC_37jFJx(*3M5V-!CZ_QoG-x9ggj6t6 z6O{R7gh4og)Efnrs>~fn@N3DUUVnqHg?5GnHJqz{WO}0YJBn2porYrvw&A@C*3T0Uc6~{k|4FBR;qA+ffhTJI}}m%h9%j9>8~2;Da{aSt?|4dZ`Nz z4uGGcy>uhH@C1IM-eiL`9DANY+%0`O9)yK;jsU)qFP5V`C+NipJMWMei?a+8?V`(C zw~i*z)n4jx4NNq^%vlh^R4LR&cY%{b^!-cp*Vd;)u;T9Dri5F9qOPE z3P30{!anFATJV6AeA2?cE4`$%H%WNJOh91^nE^UVA&fvp`5l+Z?&M%td<;o7y2kgc zh!Y<#{_8Opj8ZS{4}%Y*OIchZvZUBuO##@1#%2Qz$9$)41v1yzY@p$oqxXe<5@%=@ z%pkP6*SzC)fIEhE1JDN@M2oblZu_hL7!@6Y@NCA)9-)y|VB8Gt8DAH05EWGSD$XL9 zkKP-9M20ivFAc~boUf{#JVv9hkd8I{<0JxUL`m+rO4Nebb>aZ*Aaa07S1ot>o)9#8 zQ7o8qXj{|Aujt0Z%f@p#?)G#y;xk5^EwqbB5EW%aL$AGyAqqWwd>LU7N+4Ypqc~5Z zz0EG!`ZP05d1B2r4dV~4&W>hN2SuSkoHp7}5@?GDaNn{+1vET(e_m{?;rZwHUGvZH zFE-Xd{MBm(_6+t~>JDoMMk!h-?GJ&}D9QtfK_C~7gC@NZC)tCN1W}| z`MtCW9qxARz<$cwnEkcj_61^Nnx=MNOHzXngDy@Di?F+n!mtoa^cQI00^^ofw2O<- zFm4YTG{^)Z6NJn{8G+H#L)8b9DPzGEZYY6u6Wppdko$*jaBy5? z>FK^Rz6eb)G#lp(-lo++^K1-Kw-44WU=&=BV(0}UyT@j}%X_(YengFG$13<)=6|MH zrUL~6armr1CMvO$bB%7N2*g`C_|bO}>PTO35<=hfhm$oIJXO%Q z3r?@Oz%b2)moDxwYE-xDwJ2+um~^Y13Sy873uzy{lPtK(2QE)Bx);1Hc&WM4gkN%$ zrJXXoGtH%l()JVx7b#83a8nzzurXWbVTEC@^*_yzyv$d&)QR2k|5!iL6vo+wRgpK* zP`nLG42kJup)I?fj@^RDZXcmDzSbv;y;R3TZ5Jp$IjFc`EbJdb8)TcD4tS zmmR6K^?e!P6;`~ecJAtKD3wO*+RdFJL+!KwW@9mGVK5u~;f1+XhtlLmQCe<+`($K4hehT%rRD3x>Pb{DZ5B=n-Oy_s9Q1I!DsHN&i=171 zt29u9o3C>?bth6)2R&CnSOj7jz~{*{%`pV_rVI%KF|+V^GO{$OHlt)cECs?s@x<4Ij_+yLCR}?^8lG0RD_0;T@iX#fCa4G{5O!~ zGmkwRbSJRGGf#e$3jCJO7>WCyj4mf$--7d~Iu%#P+He_qY!3wIfdw7p#nK2yy@=Q~|>k_vjkeJCs7h z69CU>F$g-O7_Nq3>u3O|Y=}V^Q*Wr+6I~6-oCP3SrRQAr1hK^IhbyBJgvx>#fH6(_ z?5~lhnJI&-4RaRnrm=iIWht1CrO6(6H@F1h5(<|=7>qJ{pP<-Tdt;5zDAL!9jWaBF zO!9Say_=p62(#VI3D&(C(IicFuSc4&78>gkI^ee|m8-KE=(=Lyw?gA=po zy(S2U9f9(E^VVwNhv`~kV1tbQiXQixnCIlc8jLbqAia7O3cFCOjPYlh* z>BK#c5_yU91nK8|UL+?+k9;N_WNZ@kWStATABi(4L7?#OY=$b0X2=u4qG0V9n0|S} z3S%<{5*lL!ZH9sva^z*3?slSAbqPM2ezEApP-un?-9<%LUHE%s zjC=Lv!zv7P8XIOqXPD*U!s;B&#dCQ*MIKhHdy}|JOaL)$?`n};Ut#n~@Kz-E-=!}m zg|rO8jkNuZr5BT8Viw8u6%Mg`WcO+>xW&ZzDbNIV62%p)bfSBQH~A5NrNwbO)67XU zoP^=@fbSYQH`(f3A3cG{XMSXdPomHI(K9l0FB6p6=$XC8S@ss6KCK-JzwjApmR6cC z1I#yi2Bo~IsxpJ1n9aAp@F^fq0X(gj)5I6JJyhKev$`s~7rNk8X>4W6E4v(Ez!k8Bot+SRb=uv$>Rq`1Us zeo)yMJRKGkX@(<~{fHIJEsL*s9{w(NsNQ^!Espvv;z(pa!fxtT#Pv~R|EZ!%HEBm$ z@oTHC7zk!yu?eeLp;9c{+s)(qBdn2l~nYm|*#vl?XMR>E?yM?0ybpNP^j-$<_r zSuOt(vpq|+d`egj_6W215tNpNS^MyOPAi0VeuZdxeFB$}K)wc7;l8+2!4Zuhywa_d z(WOXa1ZxBPlZGJ*fQZW%FFnu!gs`eaX!+8U+@BIweXB9{VF%R$=ei?pRjShYh;LQx z(WP8b8=S*8wy~nY{G3*3?L1dz*A2eZ&bw6WzgsAt>hgw2{pvxc$>&dJM!u z6@*&k>!WT-nr)|bBWQi;n<^DPY)RSFnFs{_I8lOIOC&{S$vuxRDcPbrmRL*xnNd=n z-0)^(M`w&k5TX{oSxlk#1P)l_4;)mNy8Mry@zg@R8K2PjMEM`nGkAb~mL^KovW`w4 zsk*7U#`!LUKu`if2?o_hIg@gko**@O`t+N9T8uUT69`N+!1`$C(Z5|)MOallow+Lk z7#dH;8fb%v!-#-p_98xIx)cOb=;)X?pP9lWqRVL9%9L>US>LU#Y)ey;*#Lh)%0md_e!5EVER3~d;3C=qITxn^-K z_ojfsHGnl3caNd$8q^t9AL%UOrwo<{%U!V5Sok=Lms#@K(bF_*a%M83h0%$x8h^<5 zTNYKS-Be#AbH1i8@e#)=j&E`&b(Er;qo*jGjNoKYo;Gmv6$7_v1lDL?K#gr>3KN6F zMWpU;k0@66w=Z1`PU8KXP5fXuKfM)YbSvHZ*6KzRDi{*+)ZpiA0tCZ3KoI~%W17WF zmUa4`o}BBEWvE$btn*m}$rte{&LbF#TbJM5%xKnB)2kl z4v@MGe*;CI%;81K@G6$q#M?3?2^TNBnQ6B{v~%cR(x{ZDqD-PH{dW(q^CcAoNFbm# zzBx>oi51;bB1DcFR02&L~9D>zHaf6@dG$p;K8jGL_f@arNW0>dYpvM1clGgW~ zG`$)QHorDD7mjs&g{~MJPQ~S-`oA90w$$Cw&4NVysHr>Qhj1ViO{6UglMAo!xViu_ zJbF-q`+yVP??(!5gZ#j(aT(?o`93ICA^|T|?iBAd8Oz_PH z$_}0IdU=?`>_w}2kscj1@(IVg7m?;ho#YF+P8y79UT5DVM0PDgbtH!LIMft`Ljweq0VhD zlW2$5TJ_;@{oxv?<*P$ia}ToG*s)_ey+i7!kN@J@?&J!&{5Dz4Rt&s4`>Kv4Un{B~ zg747@tru9Wf{%EWHZ;eTgV(A(CO%9Gx9CgR07G?s}MY1`S)8=x8si8Q) zx89|1*`n%8uF7;$WMLphZ#g(5xp6Vj7^NDJ`oxv~?d`3gSu3EPlu5<>Be!sMhJBMD zFt^k9rw@tn@I&1LAT@sm*@7HE0<_=Z6(^iw*{cj>XvAV5&>r-ea==IWjpGv9no=|% zy%yOfaNlbou$UvnKu?@LK@r<4Y5If+;ZHcu_>H5f&PL?%8x6GuJeVApTUMvnOh$IS zvTs>5@FvE?u5Eog06bbsT3DLtK)A>;3EpVaw7B=5&T&nccukj;Hm8U8> zh?^9j*aPrx6U&M4Mk_0PB(0lO@#sj^V#h4An1bs@ThE5NJ4uEF5%4rvlD{2PE~j1#A?o{PWB z(MOJ!`WG0h4|M)k6m?aX@zuCb zija5ZvZqzcV6yl_zvKM8g-~CiozlN>hlVT+y30n4j@VC)$zTINs4kwyWZOuh!TK4JJr+WL z>tR;7jTrh{0BnH|D$1ON$8A&zwe0vRp~3Hok5vhz85YKtjyk zqOi6YVYe$hl4bU`g)O8BzlEVEQJ`g=o;ow9cYCS?T-uqUf#2fJ7AnkH;K3L&*4DI3 zoX2PC?qfoi;w{Z2{Hpnp{*7O&Jdz`5H3r68J@hhXZI|N&aiZc0yIEUh9;iAjC57E^ ziNCWmIfpeQS{TKrVYr@_o`F^lCT%w=%P9X9g?EgubTq{^y*}f`HBOR_8#ViOrvL-T z5Isq{u63;2iSGnW)_5$pN|urOl-IxR2>>-g)Y?oJ*ERc&?^OhGi*$^RG*J~=)PxV;*fn2XLPokgby(quqGLh=lE6QJ1;*q2H5r(5cjw9Tc zmYil@eYVE!T4kDDMp_NS<+Vg_*?6C6EPi8Wg`I@seWvBqL?5ZJ^b7dQgVN<40U1*455hYcbzO`dgm< zL{$M1yZ#34fv(VWZSG5}&oSBLHM{@DYK8;F0X<#A^cgR$F}Oe1132EP9DTn1DxJaO zyB@bPMWHATGhbY9S)OH7WpVZK%PMYUCN#4a&!t7pi}C_)AZ2uo>yKZhs-Y&3TAS(8 zx|;Bh=@n+u__kj<-EM+ZrdMIe+SBdN)L8myz!S%Kj*+ZR@l_`J^@?ae+OK*VKo&z> z3MQ*-7T$~Z@=T4zp9aL9E59Trilc&GI`^xI3Qg3W1b(E#@=pNl&i?EF{2#chPV<}m zWx9juouVfUy*}f0GzL4q+piqzHyfmGW|%*~c@>!{M;>ElsY+43P6WnhQ^@{xw+Ra- z%ufUf_cd7mIh68Jd`UYI@Dr@M0eJo!mstCchBUAfu{1EtualJ8>;e8&Ppb1sw$@%7 z$VTs3eEv%wfeDF4oClRy|ACP4GT0!>9BgO}9xwy_n}Fq>Td4aFLXjuq<%Dy;Mbe%+KK3r@Tmj?2qg zN2gm^;+BM8zB>bk*vxfshsF~vR(KSgJ7o@ebJnkL{@b3#7+HKMFYsqm#XsUS|9d1y zw)l5T(UZLTYa|D*Nxo^VQr+JEQNR9cBsYBNxTW;BGW~002d?V>;&Okc*XZZ{JtA)$ zze`@lO(SN}MD=g8Ix{jOKY{!#u4~B)JLqs5hRQU`WPbi- zd#|+2uz?WwqbQz&c@A)2OZMUmtfI5uU4t;mrV8n@UQ2C7{VPq}QgW`p}BRqt|Tp+ zKXZuV-zYGfJ!^)1Tvy)T=zgj#@4R4vGv(E<&ad+u-qu*x7)aCRytvvG?4-FAwbY)N zewWIX6hu!upnB;uURvX?H2zhh0w48%Y1v@KFu$o`cnOiDljx(k{;t9aOQedRV1fb( z4zUiaH2qwMS4`6C1mnT`fu(H)jus80F2zN3Oe^*mcxX4dEMZy+L&hSuh;RqDie<%o zV>eFI2)Y2t@%MVLR>($g@UW?c(+<79q0=|+K)w~pJtBkeg9_NCB!H4&N*28ZMZAFJ z&S3{RJpnZxK86!+cG%*bC>A!F@e{#j2Q=Q1XyM}=KN64j9j1x7{Wr81eO7C<8`?(! zRSDQ?99BP@2{snkN-U}sH$f3}v&;0b{x1f!^4&UjBKU>578O>r?Ljn}r{@;V#WbV$gM3nw7$?u9;$-shWWuXo4p4;o@-J9Dpt zIL0`*y3n7Z9Bw6U;;pi@s#`nAqxSd57JZm+YhStO@*US=k;hRTc`nE1vBYdnOQ&B} zA&_YO%3z&ZfUDW)Q{JPuTbdo$vZy5|eh`Y3Fr>OPmy{Fh@L!`NHdR*Rw6JmmHEsme z^>5O1pv4iy@rejaYOh`nem7p<%lLeed96RQ=Z~*AyPyAB z7;$#}h+p4}@;fe*G(VTOOup7}c?R!x8CXD;e!4*Ve>i>X(p+I64Z+EZHf%w#g@)Z` z5niO?nx1AAoh>8+(7FiO4Cc#&msspXC7Gx2n!mmkMO~?i@fwkz%4)w_4KomifzS+k zoEB4$uXtJIY%UyMq2|zLvYcG%_S;3};%9)fy<&`Co7!~vET)_40F*#$zd+V_rZT#@ zPy1}Ho53##e}~UqE8J00?Y)TMl?F1pta<5B zSV$comrEO~gOEEELaJk`K?Df{1^E(U$5Ri;{aDU(%E6VbpOF z*YUn(0LX$N3j=wONHCdZG%@Rn6Q)JE+^hd&mJsl9z7S2KD1?=VzYvg!C4>PQNQkQ% zhM!AW87-*0uP^gEgnhmU>BME_;i0Xajp_Gya`#qAr(=Yo{(p-F_hbd?$t?QBmXL2o zk=lc9D;6{zB3?a`Dm$W9hJXM9Ejml+Wd^YamCFpv0l!VgQLNU`p!tx<>QtuJx^SPQ zXblHK3!tZgK-P9l)CIzEJ=J&mFXP`KvDG+E6 zdKg6-4!F{qx^LH6OAJH~MJ}Uzhz{<~G2S}uuF&Y9b0JJcmTNIX(=%#vb zJ~nupN@PH{D1?y+mn8E1S`=9&>pIQP)$7~s{au#Pdv>cneBX9+v?&OpAdrIT z$qn&ZPQ@gzqEoCIFHfVgjTKbz1;W=hWnDhYDaZ4ny!M1mLv=9qc$Swb2%<2=YjG7n z;_9Q0&u5S6@_lG%6X@t&uKaRAQ%Vs+E$9;JD7Nl#_5a}gM>NOQq(djx_>L$h<_ zc5COd5LCfXTfhU$!LABUaV3@H6(m!qU-Ag@upUEEcW8Zxc!_II7#HWy0NuF3_Iis$gn;TlI}}J2bGD?YqeTj zmK6gpL;=gTfk{I~ccNhC+qR?NBy2W8Rk-SVyw^axsH|T?Dqv!bdEk}cr zPr~t^VGp;i1Yu0&3fQuK`H!UqU(vgdzhqWiy*$sf(?&_!x%sQY=~%+ps0E`Y$1^sC z!0OoX29BeS9Y&!QBr>o%b6%y*#98OHs>@j22|EXhw}0fRjkz;8_R_R+s-4_As)vLi>2RT3<_(7E*7&=#ymdoE_Ojh*=XO!(w3Z(!p2v zsse)4(NuL9)sXzrrS-nq6GZ&$$h=Cgk?A-Yp9-f^#_DG@O$p|8wMxQO0+aL~m9+7u z0NHqpppq1(i)1_~w9pETmX8bQ;o~o9BQX%Fhg+m!84{Vv;}nY)+C_>*TMrLUjT>p< z7m#OIgHqOID!&d>Dmr)YP28P&fAVTwYJU9$8jXrHfhsYU7>7rm&1ip1VJ`(`TONlBue|q!>7e$5bJBiicIe zh8k)3n5u?9NGh;?rqRYuW5gd1pqy#F5x6#}cxm%kNG$409Z)O+Zp3QySV%10s2)%* z4c!RV<+6}ursX}LW(Kh%k%gIt7OB3t|EU1R= zVjSIHo)~(EN)W{; z&7(~Bx6m#!+27V<*K@jUG3L+bn&nY&#r7H}Nu*LP?@?WlcQ$_^v_QOpvq=M z7cz=Em@o{Q4vR_iDesYwvE#<#DA;8AsPQi*5VnU0Q-?6)-^Fw>1GIx_W8v0kvo*r_ zojext46(D4DmKzt;dY*gaE(j#Qr5ezbXz(!V9nWVHD+uk=}R!vbhb}_?q-rPzFwP2 zR^#ixnPi0Tyv-zQ@eZ-sjdp{imQb&fRdr0^7v zmr5YHW_sILRiJ=AHXg{~>#1jLDdzg>c_e}QihsU)71UP-Y6Cl{7ACR;PMqjH9!HSI zM3zA0!XR%GSw^hlDlsVTR~jElDTXZownbqXjZ_<`pTa>ZU|TA%F3{76x*ORew3u*t zMRPmR>}^q-D5v2!o22@b0&3^)Gir66f_RFm@-#~BYIv8PqW7Zwj?1L+??6B1{bO^( zBaY?-ejmnR_)C|kf|C@SWaAdXW)?hh;&e@M=h|1C-Q@346Zhy5r-q)Rg7v9+L*WCR z$0yYYS5a0tmpzjnsM93wt%gdXVXyHC!Y2S<12r&n4u+cM>Mo`1-i6?Rt_JQXkn z_;C07Xqj z021p61ORnx2n%cw@@bMu-=6V0>;a+a{MvAX@JG{CDo#;c*Ya|$cUozp%`G7n2I&y| zV7gdA?V7-)QNpXD`&G$`(%0Y?2KRGnVHBL@J8~tp0=kSGRsmF%5O83h0zkPScm`-7 z+LYL9k(`psbHuoo1O!zeR6PW=aR*Uk#>~PxI)s@|7$&bys0X7uFnZ1DU}KsA3Y(2{ zhU7e8$9G85tvkc@O{JEl`FYoMmO*LelRL1(YQeKr>^2tmeZ@(#b7gooiZR3XHNf1a zu#8GYRr?tBY^0Ayt4|iuMnFwY8M2%Q(%BETtjvfZQTt-c$xq@Q*qin=kLle4ui0O|Jo#^rklOvB6+;1o9v2MNw(>@x};fN|Rayqyw z^--qpHKW#FQuhLS_PV5kj&!vy%(YK>#}(;ooR+ULKY0X}v$Ho}_Ol0Z2SE9&jM=i# z`b)|HhBxMQRq1tE3!AlI)^ylPdJlK+N7voYYkV-pO$82hQ+16`e_g>hR~rTTqHOhe zN_mz)WG$om`QK$)|8>s?B_P`&p(PlQWveIT;{@*LkHpuJTrknStmBTZI+6CWoSUIL z`#g)vTjp-He?ZBN4|C+(V!x*9DnHM?z(>?vv7NQ6c(&9<9$nI^O7rvIA~(dwWX28o z4P?KTvXXiI7s;*}H{{ory_A>YOa3jAGg~@@kop?m(J5a?YrbrWjmcLslc%~UUr!%t zmIM)f005{yge$DW80g+rRJ&&$vz39)`x0hwj7uEl&W3$&`m--WFp02cbTF6iM_KjUMnR5N#SeSQ4zS zf8ZI>gUG+wtf;g2A?HPpBmOdTqi*7B7d(Gk)uX);5pPR`5RU*{L1pPBtwH$agP*V8 zL6uFSpe=fyqAbEnYI1>Onh$gQ0lVn+9;ex@l+mR~WR%GKethYek^o8qDfLJ$C}_i0 zyWTp!qUeH+PBo`K%>^~B>iGN%WYjd&-v!0;bu+O$xr{3|KSva{Wayi;M4jmDloge< z{S&DE!R^ZDvMgtg)F>Aejq(d47$t@((HYXPJu#>rkES7r#&f1~gw=}rg%@dQro{@R zl@@76Xpv#2UV%kcVd@`LWRU5+kRod^4iZje0{19@;nXS;Xq%@t?tp^K+;6m=AVIek zJrJa66ssMp(RAP{tLIjQ)PZMNFDW!BsyVitfi3zbVh)0-&*NQSAFJlL&Wd~3% zSo02PRNivv3r6oH{pes~F??a*cliXAPxbH8YZPqaX<4xlY-hHphd-b=^LJgZr_BOe zuZNVGA3SXhO~M3Dn;?K+*tGQ}m=iK>f(&1AmO_~NUB4)8MKt8*ed&=1p_3h3ZI`Op zfz_%PPp2s3%+vf!mi2y#Gd?lx>3#yW__}VG8kC}Vm2H}ng+Bv3nk9H zkQ3`_5-R+}11)%`*QdN!WU*0xsjgdG$^A>id&mH~(9i`!A0-t~IB>Yl(+(;Vm0RO- zd;ijbLvamWTw!Ia0snwlK>4)s%r&kmb-lQqKlU%1w#TtS4-g3{(=pmL&}OODY-W(o z^_2(}rt^^^a~zMaE=V9JxhHuCEAxr}4Joht%aDoj;Id(Jo(ICPye1fD-=9 zWv-HVr8+07id%WwYj3VJu^9y8Y@rSzgIxhfJDxuPDeRVmCJ}n*XG#*xdO_&B zbb{+OW9+i6RP)9$x8#c9mU%i~>7qWh0tcV6ROYqgI9Db6r%4tRu`+QML#2k1lX((f z{k#KG5J>Y%*^mPZp+3~=b}LeKDXSv8VN|;#h}0s0cQgd-sMI}LiwcSFx6BiWE0n1M ze!Esjk`e1Gf-f}u0ZNNXh!5-w9HhBz{@8#5be?h36=jpcW<{ODUM~0zKUPY*fDunRgJZM z=Q1m^YPx2!y0SB}ra${`Faih&&a5&^BU*%&XXMNy(J+)fFyq0qc1W8gtYfs z3tUJ-+RDQW??4kiWXWwS)P1ZZdMrxlhlDoOFU_+wPu`|_gk96FPi8&k+}p3a3$(bU z%K!COVmZ#42hPgMf=(>3g@3)(SAKKh2h{ip9*cYHJ~d!Kvo*h*Y8Tz2$pCZAm)_kC zX>W6@+fC;k7S8NR8vwTJWIAwHlt3)@aELUyw+3Q8hg(oc2kP{#AxO&`7l3dwz$p>0 zBt{x%iBDfaXZvSK^PO)x;2O9vHafso41rJ2B5>eNY4ROu;&E?8YfH~Nl%%~B;y#$m!M!T^0m zQE<4Oz7gQ$w}&tb^T;9A+}`rwus(g|z!`ik;7K|&l07A1);tBgV&-KbL`)K=HFlWr zz@kmnvv7w6v9p90thFFJOxj^s96L*RVApo))1l9X?Nl0<^*DWEWTBlu;ZIF`%v4in zJQv{1lC_=gBNYZ^G*i+M#$0+0`p%P1i`jvMR&jBO{l*}2!6}kZF zm@Ip~aBQ`aJ(Rq|?005IAjk*=dx3ZvhGn3JOBk8blX=NeHX;kMX^Cws;5p30ll7qV zTM$t&l8uZm!L7yoM%jWSZ>m@B^D-Yr^lfPRO=Lr2jEEWO?xk>@{+Za#*?3z5121hh zN1v>fyqCHu&hQzlpnH;m(dF?Cfo!4jed z1IrQio2w3k6_+}&*^B*+DNW3nV5V65C<@)PDv26ROPM56&PAL)zRDGO>L%KSWHJid z^C^3qI*esvN^6>oT~drlshD8$i4~+KDQ}~)%JH^!RXJqHt1hjrwCmod1-0u^O`Pw8{L48yS{cVRr*$S0_gmojV?)8Wv}A-3(l8@;&AoP z)jqr`MIPAo8;xG>`seMoyULB5Qs_oTMUXZI=W3f>mC}+?M>g6Cu+)a9Z94Q&3%Egt zNPu&dPxQjoc>t3pAxt*;v<&iv!VVzZbs*p-9UV+=f(q$c^W88LxE04Z!?qkw7=>8j zDSp@!g8_EMOM!cBdWn}B1vWRR~aFwZ0oIHk?tZC~9 zl|xQ45ihovw0R0HIUGj?$L-R}95}!0IM=ETmBml&)`NIXg@?Ar=+RT}bD+mGdb0I7 zLw{T8dFk&0`JE&XfrZ|@;=oFM24a$xne6uZ_G3+jSysFR(bFY=PU0X-@{{E^m0Op* zAD;?|GC`CHq9*p6-UI$qYO0hstZ=ZIK7GFVM_iI3Gm?~%>ZaoI79@T;3vzh5Wa*#S zGWu7kNH{zEM#wtsRE084erw&qbH}lY@&}OcZIaYc2up41EUcgYWnQQg-05!Nn^tgV zg1da%&WUriyH-D^@RO}husly-`d7YFz?TWWOz|~{aNRpH{C)q+-;`mEcNmev=GLg` zb=*RnG>O>5h6Y3kA>1BHpmVv(1X~Kq9t?cdNdtpl70hi7>!7J%se|2+pApAh2M64h zE-<%SqGlIz5tR!$G$ZfV4{S(Pw3L)R82jT?nV`x-i1kTStwq`P-iU|SYL45@t1m7h z8I9Ps3RcNKVYCllMKn(N_Jt%)^!^o`hBWV)8#iLw*$Gm$Q82gQucgq9Qj3 z+JT!j6ytWFn1YK59IpH=GlXm6U?UNTnLJ)H{`0SEeCUeeImN*&I(i7}Bo-Gjvq-lR zsBis_%HpF%e%ut2_L#88$O=A3X0#&xK!3axX@7roG*ra8{u=1f{?xpFV*@?f-_;0K z>qty#E>=U!0e|r#%~E{6Xj={t1(PY267~14%!xJO?2l^T^UmNC#4;oY?P?o40i`qT zJ1)LH$*S0X1WTIM{~BklRGW=&69c4XqYt-llrQK&-&j>1?8xYH=@a0$*1()vGXc zCYH8`rxnyHhqo-CR_jR`ePW*Sox@X>d#E+CyK*Q@OX>bNO?l7l-uQU!5NO_f?|Tf?iG_+#}&uO_mywk zvGED99xu7o%*Y#(NkQ%qSXG)0-g5g2weJhhEaEx*mYy%?0Ix22`Ryy6fK69vtgugH zU76DSP0B>Db(#7^R+VY(Z+H8uwbwVhig(gsCdWgXi@5B0OlK>$z*-+7YP)i|Ts<(x z(3ptiF7R*LC>vF6wT|_5L|fjW#<0TNn;2jX(!9JhILAD#iu?>sy02lCLWeUra2l+@S#5&YgZ@p-w@x9a>OQ?{eZYswdScx>Zc<67m}`}q>emA ze0nWm4R#2x!7wYILM^3APRTd9GUZ%0uf75*lXt>No3dOpWr_*oS_(3d3r>)M#%43c zApSs>)b$&cOTm{1+X+>id7H&*q1ml!r7_TsH&ZMwnt(sTZ&WRg9)64m!D2I&()f)E z#DS;V)#T?()}*il+8b4;#jlPw)9Q~)nlx`zC&G^ZZ>A6a|3>v8?D+p?`dp&&jj994 z`M>RyX&(RATa^Z&!+%?elf2P3V^SZ)Ur%My68diZ|0s>CpbGPozZ7YSqeta(-f0O% z`2T>42~--wrDPrUnc9)8-}X{+B#d`fb6OKDzhoZc>_I*7a6I2UlLKHwV|Uzudc?t) zHE%ie)!I~_U%4IiWW%vdGt7Z6tQM;hs}*inCJt-dBvK8vr4;%m*T=9dX@D3H_$7Ho zZ7dvRmpBP7IZP+V=_iljnK$a}MbiDcFqo8*N#{U9?ns_u02xbL8LP7N)4@i3eemce*> z(hP1vXhcmV6zUEK8-=Rm1?)~~B(j)P3dA)lwM5=&y(Att#YCwo%{nZ)s>$L9Beyvf ztMn`6hS&le^w%`TZ%fr{*%art9VZh+cAlFePD)jaEDPc!3unKP82#Nz^QwiX0N4vj zhB3H-GR&kM)y$hF0#gh!Mnqap`$*VYdFvXsRt_3eFBSg?8zmq=-sth;mX&JYaiL(` zgwkph^ooy~rASL3uWy5%fNXwTjd*sT!IYhy0|PmXyw=c zkR|5y0(W;~?r0ypZxDTSB!_cUB=E<`8dKI(1VNnKGDD96Jz3(QbKOPRzx-<}`xjl= zzgQ|;!K0m}=d|HfK^;|TNx@COl^E8nP16o-e#~aOcVPDpbCwRRV=%_h7}Ja~Fal-T zH2n}jb}_FeMHyE0{G^~ONg%+{pVcU1eAe_a;E&$?Vvj2^$JnyzBNG^P#H;Dh@KQHQ zAECj2b7Xl(@TkVcFC{Dt4bKp$683n;evC*l47(v_$!iA z)piV{XNblVC1S6bS>=@7JFtgRfqi>f#0u#&W>64y7y@Gej3F?wwasAwbsYt097krw zg$v}Q4tamuVp+ysiqKG$$_h&_NFN1smddPRb#bY9{&SJ1SUbnXOD>+g_A3jv2HPvz z1ugQSh)`iRFfVBj$eq~^XgvbaL4ESYq%YQ5)IKd?8*zd(eqs`#I?M(Rv$8nf9%>^$ zC1I%==y0Ol1~_%t`+F#S>!NLxqr(LPqZJ5|TdIb43mb*RYrU<#dsLjoN^=Ne4bfr6 zCHz<|5@niWgx8^71244VqwjiwdeT^X2|jNdc{m4wh3}%TM_9#mETC-#N3aD=7>7*= zdO<2A$~5_?iBoz@&a1|~+<}?9p%>6=nCjf$Lm#+Mje#{flGdPr7tEjxlNUCqB4e7?ITvnD4PDLpZ1lC>tq zHg!f~jEJ$8SfA;ft`N?-I;tRdWfG-4Q*}HQS%|A6#6T?3vRmFpM(+kj@5rbc5EDM^ zwL-gXh(2kTk)d|1m9%N&3e2xybJRg>A|itWz7av>)+Z8mkHxwYW$8l|RJO0GKo?k+tG4xv$cEI zvkNtH-<7p0IDrk#GHHX64Ja~%s|G9W#wnfEfL0l{#USEg0Ix}&>HJp2fp#XMhY*fT zp*NFSTROYRk+&k1Kx;K$Qj_8(tTA#D{9>RYaD5* zi|9FNV7Jdk?6$%GK81N)KZVaw;U(4AG@9)u3=H0ZA=*mAW%0AJ!-O3M$(JYWwZEvGSbFq^?LX8=yF{B3nT|-x!!H+kpt8HYA#k!VZT{lm>DSLWS z;~4(}nrgI^z6TT6t@T}A%if%}>({YZnXCQMY5a+&$b_M)cI~vGmR(1kS5^#07=F?bnc2N$m?sli`S^&9s1_o;c5lXM(?h zRyX^Lhck(I_7UXz*B_3YN#tfitLsL4R|H8|Wv_wX8)tvX2sA^`*b_etXe<(H_(N4c zsvCZ|hCf5~644(l6BTE*Fpe5GfZI-pV>z6j@w)&$5c?-;FHtWD1As2&{Reei2Xzuv3&zZEK>MNQ zj1yx-Om_Wk*spg#Xq=&)XnJ$H(g2ADZ~%EVF(9r^R@~g91@Ucc##t@OKfwFcU(t3(-A;rS*t5Xf4PtMR`5CV}BCzv0{cExx#~ z{!Z4WRCU@(@WFY}{H9fPGoh3p$@Te6L*T3G5~nro2@e|t!Y)w!9@Jn67Uy5i<)Xp7 z4V4nOnZnHkZl>Y3A-AfUkC!Y9V7~->Ez|+`ROe}MPHU@30M`%_AT!+J0Un$nfz-0_70+s=7d6fV4`2C`(}Wzz%Lr}f z`~5hPRrQj;@+QT}RJ@&E-=a{m#v+)EU=E>gEzQj}DQv|uv@5LWZ6xS9DK5=zOD7<9 zA^Xa4O7f&iBa-$=f4u1nE6Ww=pkfjg{_!Q+hU}G5eUI`W@LL(U_VJ!X&B(%QKw?uc zy@?Y_QH=z*Qmu*lOu&-$S{LO)$~W>P6Vc@luzW48*A!fvZ%qDz|^L=IEQJzs5=ypkHK8pH53{{FJ^2>^c^l6 zNk!I|)rwbDiwLYxlUnISx8&iH$JyJ~i=H$=L}%r#A5HfSvj*;h44f_gUcb?7geN0) zAH?5vxz}NiQ=xB_Jjcffp8)DhA3EXfD@~tm$RQ~UGv+G5Isx!OzXmL*UF zWCs)Sl|;G0sl`1-Hk?e%|8PmO;&jZ4D9qGeRaRS)Y&=e`urb7@cD^EP2E5Bfun8c% z8*2n(5fa#fSCyO#{_DBTyG!7Dx7UPtfettaDrs;t%(Q%h_?nlrxrOzL9dL}mF#)av zbshFe6h8&grTTh%+yNRB&@`Cq@UGKS9yXVXKT_EVV*NjzZ%zP(k?9x>z}JxkI`Zf6 zTYA2n)rt&QP*>l?+Q6WEH^i!)1bP=1@&(5}WF(#7|IFvBpy5K>7l0S%&% zcV#8F8lVJkx#OwyPox`n!M`29eOw;?HFZ`&&pyP48f;{EW3#rwf5-VeKp z_lC7IOSg@3qHXb(6ca5ZvrdwyG%EwK=CShB6ZOaLzn*cq{X)W1W4{!DTIYQzJFVQJEfB09BbKlqLl$`$xYOeia zZ6XRa9}SdD<1DdV_16l;4apeeYsds)#^GDU03HK)YQ>+9b%wg!r8El5@FC68=Cw+= zw=@!QmuRRhD_$@Nq{x%bfARI(zf^8oI)R1>G^~Th1ksW?&y`k1eope{YgnZaL4g0J zinv{iQ@mIAyw(@QX{p(qOLGl-T5*6n%&615( zmwv@pc-`*+i7S<;h?AYrqbvB&9pi2lp$2woC8*$QFULT=9e1Lk`DU$eBO+H?c^Ewh z_=1ZQ@qzN3)_V~H?+jnQaUwpTqW2M0uH>pq4(;ik$VhMEuPazykEMH6LA>_lzR;s3 zO!zByq(AOa*=zosHpDgIt<(IJCC%2gjt*IKDUW7>H_=mE;1bG5{tgI2Y3|~S>|v%;?Oz>4Z2Ab`C|{GD5+~$^ML!D!U&8?s?YpycuHs>4HAqZ2vc{e9B>Y2 z=bFjizA8q0ml_e!Fc>0GSXN=BQhBSON|XA?4+A)Uy(-l{ETGN^mIAn#e;RW;DKA)E zjjINsJzS9(-pF+p^gW=?>);TZ^jQ}K)H7MO+ zg=0YFT;E)pCm%t0+{SyCBcXS%gtVAj9X%y;9kD%_ zNgViRIji)~Ti}E!_yhX z*PqH<)YET%lZaq3a=c$Cb*gQf@|&gR$TN)%XAFacly$%7V-Sq5z= zyzB8pBuo#=r-+V&T#0d=G`Bz(V2r@?ASw?6?KuhKcyFSn2N+*2ty!D0MSt`B$(!DlkywG<-^XO zj6_*qm90RLZs3YD5yDm9(9}yI5L5-nZ zSAdKvbdfEj3ER&Em+HEeG=N|LLZa~|CEmH$yl0(z zwd@wD76i1h;-LGiUzeW1)ca{3%uv@B;KsNQX|KTeMwqq?S7KJb)=hGj&5t0>AB*d9 zBtWvI#0bz31vTw0H1RiR-CwgoyFy!bx&?PFB7?Vr-`n6e+GHCw$FAKH zTd@(=r*e(Ty-%l8pZithgE}0(F;+zulv$Wx;qeoXoj}0|MF-OYZddwAEom)ZztyyE zgX!vRhHaV)g3NU*1wm+qT7f>ksF7j*`>UHOO!KBbhIt%l=X{6*p|CjW)84McG+Fd8AB%CB5MMM|64Pc4YFgHrD45W z8&P$IrCagW9kL_(Kn_A$eE(4itAL zO_S}7A6)W(?pUYx)7%iRbUU#N<|J;|h`K2%HN&l)?y@rz0Ypu8(|`=IdDFbYPeKy& zuZ*D3@CIF>F{%R^-l$GvbP}$g0#~?qNYv`aLF^LlC<4?l$M<|`F!0CxY-3rnGuaCg zlDaM`OZ2!ti76b+n>x}4&K3L;=N9Ub}H*k zVh^N5`|{}`L3&If>EQC@Mt8X?a7hrvPrWKXdi)R#MR`38@Okp2>6W)~7YR#yAA8t5 z8CuNv-Y?y|U%||Cco|%w0VG|!H+~f8B#(pQF`yuQS3=J*u;Nh!3OsZW zx$qHp^xBlDUFV0d@o`ws)aL-bp$tgKgZ)`Z&rCs(2ew4g^ErvrkmIplU%5Jy z0S5P9{*VUEBCD^>M8Xh%{rNznBDh9tX2=Pk$d-);%b(j-YJnjy&$?j9=QeJWSKCuS zNb{45rBJ`-QE;A{RJl5#0TPBt7$Rvw`4ba=uP{C`q}GYhssme2?khOP7~afN+j}?k0j@yxb{6ur1&yO#~tU{^W+Yd(xG_n2O9V zfQ=zuAi{05B-D`4icN_g!K>HGJ&r3jCVEV-Un%#fuG*aF2e!D82Y1y5#Ypgu+o8E; zlcG=dh92M58Wl4CZG3?1qgnC^j%Uq8j5f6ev`7pvfSTG0e)@(|5uo@K+wN*ZWx89a z%S80eTHj_!mdvtC4TBSTToy2iL|hTc^OM~cax;3D$uvhqhq8zhOc}ymyw!z(uIx`x zLSgEHD03JE=|r6p@PXL*R%)JCx{W#){YkI)C8PiLx<9$vo6hiN@5^cU7#f~GZ+5&Y z&UqzVH9=iO&q))UVXwzzXO0R5BNR-c7=!1{_fpaax8NbA#b9GQ;%>%OB4B_(?vZPP zWyeuI0wP78Zpf+YQJo&-ffsJ^TeTSdUlqLgBiwdC_|VYiaZyFO<6F#(MAh$um)ID@ zMZB-icfiw+pZ1HDgN|(*Z$nl?zgpY!&>niTn2qqUI)1CT9#fc$YUXJZch+unbZvfP z=+u)aODl9^uvgco)90|+dSW+iI-Mkm%mmk2le$bI+KE^R8wtE!rhd$g?wUbPmrV?%mKK&I@9UpbWrp6GjR#~ohYBM{bl zdyg@^#$D=-CRp98tO#RlhStO;jPXRZL`3z0iIZGfL%=7iZ23SrY!%nUsS7~WY6PGI zX92C*rpZQZtQshz*t;$Moo`3Ck*{t=y@Ux~uM+5Wr}-(cdbeyv46iFJ?fYf~;}Y6r zO?8KYQx)cyEUf0Li>;^=p0*XK>x+7^ZQOH`qYCE|q^ATfbZD$)hK zg(O1rneFGVq=tArI5-7!3k~p65Y}~i%FnzKx#8!7GPh9WgEB&4GqqYz@~73hhB;`k zZ%dZW>=$%R4f_OiO@RPxNP`4Mi0!AWW(|T?G|B1p?Wn3*Q{WARmZ^Dx_dhkkYgwVu z4+5{$*Og00Cu=myD$p4audYX%ek(P;L6$as{hEJTll*d)1Kc%qY>JoUrsd&y7ewRRzs%%yttj2@usOhYq|odB>Lu zXITfAyh(A7#N;OEOI!?bF~Vg6yQUaa{Z|Rp18IJ`1sy1F3#kWj3452loi}2;WP9fs zdazmzzZzVp))MZiveL0EpwwQTohiNzsrIr}Y)ET5s#RPj75{Btr2=j*t9Cm_!w}5? z-zu!HP80zwrt>nt2yQDPFJi@;z_5fycO9h7M%5dU>Mqc1x2iVbTSVA>YwoQWueLk) z3}Eb-6vS<)ZLQU|pL%d|#g{2{U0o?%m}DaB^x53#~B zgtrFiEadOIOUb(dGvk+=(ghSuqR=6&L2>ZzeiVtCnb;MPD z4g5?w??CZk)NakY%X8FqE1xBo=cr7oD>P7buglw`Jr1<^Vf$OqBS9-LY-|dpJCpz!sqJa@m`S^edV4aCk5g+idJhL+tiJPIUwx1o+=FiG!=~Am3d& z20T8j_N}h0LlHMs7$xo8c|3Jp6M%&PHy$|7n8z~heEALl_r(B~bmm(jdvLOAmrPQs z%P_C6L`3+8|4sOzJB4BUX`h?oIh`2kgUol386SVjJPi@MZBT;tuz?cOJyR2&G8|d3 zx(*&R5C7TX8QZ{d9XJzvHFm%NVvM2lSX5`)Glo}z^a=fVPO1}sjIyvE3;5Fk3$}uu zF$d&BC`>_N2!%-~9bn5i=8%@9eD7a(WdH2CmQS2Ngj*y35r*`YMz|7xUu+$e%uK=FhGwHs?VnYXh zDq;jc?3@MH(!9JhG7{(X`&6R^wgN0#JF=QUAtP`i3JsAXIsRYuiu;~w1Tm3zL*0*{ z2?@hQ+O_qdp>iZCqL*I#77k!6zlSuebM+_G3}Y>N*S~!uIF=BE!3KF+H)aplPRC_zy`XUkSLSr9bpAq7Ig&P z^C;d*2m0d!InBQ&RkJ5#;gSCMK%VZ_tq*vv)l;dyfY4MIjo z(3&5&-(HoRl5cn(=5c*_sGGF8G>LqlSm8w*j(|t}%6E!-v{K( z4Nvx=L9XzB8<^cTVN~Qzm~Po7?6wD$KY8E|*?y>M1-yfl}+~iRp*WEi~hq0ACk%K>hyLIi1I=^JQJ}2N$pl)3|qlS?~n5woTaN$y5?*MRy z@BIB{F?x?rHgZry8yewN=1h$(GCGfnh`jo+2zNAJPmh3{c@wUT(&Ak>fz$3)xB+DA zm!S!Kn#Xd!=T4|`iy+}q{B?)?RMgFmc+nmF=Z1$Zc-)T6{2V(Ybow-ToqHiZk*;CHxnSTzI-3O7H04|{PK68RDT>UNxRmwj z5StQBnm0duzkb!$nzgCfG!W}nY#I#9dX1O(%YSU5hcIiYhXk>@9ukJ79YpY2@p1z-0XY<3kqa5azwQWXX&^qNv(Cz!KuXClfvy`qTlqLGMH -N@Q3N+2 z(TR+Rfw+d@5fvyuV%!8+00TEdRztM605`$8rRmA6DV~!&nAqfM10v{C?rfz4n1Ws> z=P+;5NJQ8fh=hUyXTLgKPLR?&YeJ~`M#jRp8~BpcL7~ae-~U2 zEjcPDzXNi7_{7iy11;5TdLK-QQGd=X@oA2rrJA*yTz1^rVueY9cwTP_6y_!hFRepc zh?E;^N}#wUPI85HHM;7#HX(k2CnGL!Od)(IDzUYY2>$RA^14X|AvYqIK(u1+G{US<}j2jI><bpF(l! z!FD%iay}$4+sJ;JOCA6`*SUwq(8J&nNF3}PliOQEU z9X>&(!)Kg^@s4)Ge2*__`F37iNf?ET>bv;z@IdccI9Nl@3BxOEN}d=xz*jpSh&MYN z<4|KHLJ3MGAB(Cd`|8^2D;m{2%f-TuSzTFGeO;xRu{rjv{z^Hvepqp460IS^qc!9- z+gw>pGL2&ibVR^Wyb2#hk^fL`^C)t9orE=%GuvEQIc}nxV0s9*kPC0rN-}A-)|ZE( z5RU&gSN}~uhmX%;@KEvYF8*%xb3ei6EiRS~O)@eG7CPqy`oAIR@FOOzr_(vXmOC|M z+oUBzO`2RuDKMxS%UY5y)vw92o`Pg`ey#t=+Fy%y>Zj72y~Prt#r7AdpAz6-U+@3^ zalN{{Eb8V|CAH#2x4xU~r1Stk zv2@a?UhQ{MWP<_{fBpM!E$ zrQcem->0d`x~f!76)H;nc&SK6b1TZjZ7OZxnBac-{BB)UIyZT#CI(|GD$;pU)UaB) z*H29i6Uvme)an`BK(RKZubPKKBHY{Q4nw}Fy&vwLUQ7Nb3UftFrPwLuZP-~c>4{O) zI_ZfiyGpq>g(&=0o*zUZ9|hImYLVv&j=u#DVHW05au;V0iX}g`6VBD1vtDYDRykn8 z0pmuMSa2)`#IlD&oB1FzSG~b859!J({R(^BKPRv7oB5i;riyn_^;$N?xuq%-?DX?8 z>MWx0!f+j@^unZ3C8A0Y!`*oL>%&15hpC?jS$sVO{|Fn!lOHe9&1h;RwYWVc6dV)G zzrXvknmibABuMg8nkOsNX^!TcWkUN**mtVGe6*osZE|Uni4p{O@+Q+85O4PQL zJI*L48-1$&8f6MMKMrc_6wo}F$1t*NX4zSB)`FW-eBqePGG$h<|DX3C4|M6EE?HOK zq(~T_Q~c7!YknDL)=1ivaf}zGU2cF;sHmIM%Dv2F2cS?Ccr{W73q&ZGavJ<

qtl>7H)el=`vxLX<%ORR@{&!3j)?FAyNnK}RrlmZYfM-!|8n37+ zRT-KiB2Q$cc|P;kJ0%}@I&MZ=+md=H3Y@}@8?&gBc~Vp5tVpRckfl8#$3Mz*TN>iE zjc=#4iY~mKf2fjh`s<@*St~lSwF>VyGQxxrd7mitAW~}|W_0MrNH62eL>zY7LXPdq zAtV|Bb7G+vaU~&^jgq_HOWKZ1tRMBc3%qxd@!m)fCrvUp(agHED=A@yDKmoKZDC0X8o7hP0bJhJw(~6L&6KqxhemJ&0gY%aMe%((dA*bC`ihGFhq<_AcH#YLN1jg8 zu+A2aY0=tUXbgg+wyU0?nv*V3rkDj?LixgIiZ(ZtkN2Mz1OQ&_~0dH zKFL>{GP-e2TI0(wqBe%8^K!hncqHkl8bz-G#(>#;f5Vn%J8BRH!O=0 z;vath{L8@=Jm!vy%pQEVgcEpKR8Arjr-!1X4MW|rf5Q)G?hFRyTzSYT0Ub8q@Dmzw za^%7rC`I8*GS(<9SNsrL5SA;p#p4P@OlPgVNZsL7z;<0s?q z=BJoCTd`s(#@xu9nma1YIZht!u*H^SR3$O2Bn>m{xWCjv8lLAl&Xu)dhHcT-2xfe# z88;#)BJd1{_bNkuc=wO{!!PfCUBD1CLB}@^x*@$C*^b)rWsaH9U>gf`M1`Mu*29mG z6+W<009s2$^oBQoD!I|L=FcV_%nM;r+={>6fX5BTcyazriv!uH;m5YTpp^lCj)S+LMps_(;rhoV zjjUT`lLei*frW01?C-z-^1giz-5O;&HfS|!_rRUpr~zk^(D>ADVz2q>$yLl?&?Z3G z57Nw)wPGlzP-wF$x&tkdw?1(^4|=?m z6>wPuzYNM_@j+0W9)1n;@MK?^tN3=<>(4BNHD*R8`;A|QN2Os@E+3-e}(kJCuomHHj~UcWXdY}niiM3WztQt6~)eugB-GWt3@wG z4g^!8w+he=w^_8FLt_SQb#KkWPw9}MVW4(*UshK`g)Q8icY2Kmd+BjDL>!>K9 z=5f41!(winsQmDDQ99CsAz-ij*3kSxnjbRH$&lTOOGC2^#Wa+5;wL3|La)LI^@X8~ z*k&H6xr7}7D2j2csHj}HM`s7TT)88M5SP5uRx~c)dxaeFQ)yN-dYDaKQ)Suf0otkI z18+h7Rwnru+Wy3s8-DGivJT`1rQF!zcTy@YpEAM16@HeUo*Ld2EFHl2%D6#Sg`FmB zDBrv)$fG2$FSP^Uejzu+ZE4vS;0f7fa3G-Ncf(Ex zUO(M$Rp|1px7pngLOD6O{Ez#?|7M@VS9(V5Lj;-${F#>g9yw6Z3%HS>l7hzn!4um@ z12<)LUP3ZkY8pR&k(pdRf)lCQ#Ur&HkEsdm0Zs#)<78{Wg5Tt{c$$+qHaVU%KEcl$ zbJ=n?%m=gNJX6w!t7$fw&apG6%!gi_zVlx6Z!4Y zs)B6`g3kSLzf@saPB#BXbK7GU#k5BOaf;;m-<%dgfNJ3YezXvo12W04_I5D{W(`c! z(!vgqB_=FUCi%Tzj^Xm3rncyC5l;-c6eSr&707(q8qF-=jBxDF5Jxp7mh6gc@VEX`gE;8wK~o3i8vFO&{AuxSSij~GWg`_AeKTnS+iG!x=&~a zv!E*NDHCSpj|nb7gHz67i599e`shP1HVS#?*-0K-bFTV~{*s{2jCQAgOFmz>hU`?J zPn~<)1PVCyQz@muE(SyFD0OO@5#U< zu_S{ZuJBYK?EXz$puo5(e+~N2LSKb7o`XL*(cbNYBzF}i4S8t*#e2^ICW+5h{BVWm z0AcrU-U0>2P5Eoke-8R8_&*2uLGf#trE%C4yrXbV^YnZ<2dd`)v*~+jw!-+^c2Ekx zYqn09FxYlhU?zV*O-$vdWc%2t8oMN{vey8|-JyM)JcgI73EI@4EV4AR4A$>t%vu{P zPh zw4!m`GWuA%Gim*tmVwWfr}h?og^~TRc860i-QDI?=!97hbj3>_+#ooN!?H=gVhp3` zQdRPaH>G>*KkRPaSvmY-EA27O9z_~6q>C{UaS29_P2{Qq=T<)=Kviy#%y3 z>?@rx#)L7_t)D1G6Y1?T9bd&V9N%sd72!pU{LtsQk&8YGE^y?js5Gi#&&a_i{9rhC zQ7A00tGGM$;XPdUSNf$6vhe&6hjIZwo%v`vY06Wrmzt3k85%&GwNu|dV!|Q;M1v!! z(^D=6)C|dim)y>#do_oYn(P54*d^~%5d7uD4x z;^KdOXbp;c(&FWj7W@^o?mz$RPwP@Q#Tk~Im;4$%a@`~!vX(|YB|uY5nnE>tNkSM* zg=mnNy0o(Ik0R^m9wN<~^OL7PFx@#9-}F7lrn~q@XC{~~_z2|Tz*Dl($q>d1QxhY6 z)#T|xcxj&g6cZTe#6z_yLw@S;bcRO6LYz?M1e-GCCl5D*btrzdXivl>g7=>ea1OyOqh*yTb|v8GyEQr}v(`Mt zfNlwz9qXK6Zv?vSci$bqJN$^5*3hI%zVhT`Wo@RdF4jnxx6|q~WnM+If7V4F;3~R^ zSMilvjE85wl35sCpcq(Ztv$62kj`4c*ni#s_P0ZOX_-(M-f#KSWHhrt42n8vd|u#E zS~oTuQ%*vDNgmXW9Q@dqxzWr5=G=zb=>v{6=Kn|@{3*i>GpG4!GsQhA!u&JCJTzZg zPq;%?Aa}?TYs&Q`53;sK{Ujh$?S@~phP{(y-u;WL_|NY(acl||09&J}rTWWfX?l@} zP0@ziZgtdCbt&1Gf~k<(3)}X3(oX^SYhv-G5l#6G$Dpqz@>HS%tMN4 zQ+_h=%0^hhrFrd{Sd$0w04P9&uK@piY1($T_q%k|iXUxmu5G$^-lYHak1sd|vO>H4 zIn6TpNgQD7=N=NwoAc8G?5J<*yjbU4?5oL>4&v*&w;Wj!g&eTH#HCI0{;qCq)lVF> z)m=-|CUw57e}rT`9MVWqej`j;@)xADma_?gUBb?}2++cl z*6IerU{vyW;Wf5c!6i1^(9?X>u{P_elCZ^uExFN0C#~J{1CkFSSq?+^Dkejtg+9Yj z{lb3iWso(s4a1kve+24MQuLny24k^$>ZW+Gv@_|)U0Pyj3Af?tridMQMB;l$^VY^a zV9`u*Kh9&piacnk074iaJI!GoS3@&mx+!jS9nCt+pkffwXo$3?0wcC5+O*SOWa5HK zQW2Z%zfJA#f^2V z^=^V_=6t+grBct6=B22f;2YG#wBA17ULQI!k)|0lk=&uWy7WL5T{cIC{ zm_Xc~F4x0kZxuCQ{;_r{9ur6rHS~Id%FgbNus9$=L_?{ zeo_oAcuN2#aI!J{71L??;Ua!3%rLjH<)7X6TCBk5TP%7W@gX_uWUK!)VBD zd!jw{WIJQepF0LxZd)nZsUCHCaZ|0%(?HwNNIUftWX_(qAl#C^Uy^<*K79P`7Zrqh zd0!cTk1i1$z38u*udhXVj;>4`zu0OWueCx^%Fsk*OIw}zNjhcATQY8^m8d-tgEj9z zE&nw(PgVNZ2;}P9$|h5PS9IwwDEbyMs!u>yoGLsiN*TVwKCgXM49P3r30J4<-V5QT ze=|(UVsKYn+2MAq+G2C}u&&U1wCy?^*BQC9Fu9ISn~mLd7%UsBnlQDF2))s&FL-U6 z;i^6t{d^sDh89;7sdH}7YJo2PGbF|>c=KLyo`gjAXHh&SNy*>x>PGdF{8-!C_0~dJ zYafGtk_ew&6#tE>{tebkT;c{PbS}3`p))sm<}Eo3!cF2XD|9aRToGr3jY>V)^M8w~ z%qbgQjNzZLlKlO{Cvl@Ob@^(%0d#e;P=cPFXja{WNc2+1S^xN{dS^f?5}k2Wh}hn=1JucuAWl{)h#CT#`x* z2K5qM)pD`wp&z%zge8WSXrQT|XsM`-BrggdurTLZIj+*vS5gi)GP9G=#)h?E`VvgT zUTFGg+ya`^W@#Hnenibyys3(YRHsSRHl|$IwqZy0STOWqN=73!U$GQosRMhB+PJ5B z$eC_9)R8OwfHZ&QmoQMOdo&u2aGjn@o~6sxWs-`EEgzl8Xg^}CK0W0gb0|3|daP)o z)J;)^r{qv(ms6U%Yt)MtC_dkbQ?b#8_Fb=g!9~Co9674ni%TfJ8>%pcwA)AfL9utC zGYM8s4EjX7e?ZmV4{*3td8jXO_ERfGFKL0=w#Jt!EKO?K_LJK80nWHuQBT?)rXTEL zt%w>cxNy^!_ND2E(y)%U@qxCt5j)pB^m)d+ z>u1_cRUuaEgGrMNOwz1eJwZ;&rL+U2Ni~%bUlFyfps1wSlSq4lR2Fv~-K~2`sckhy z`Q*k*?W3u($jMZY@BjOM{@?l_^2*pS|06SseyFpsYG!*{a=H!8pplg!q0!>uQi6lB z{^s4JRg>s(%RAvm*0xgAA9@L$u)E?Bm4+ZBh%&U`*KZc!+apF-=6@Dz**CR@Zrm7@ zg+B!EfB*dR>dFk3I7UVBoQn1#963m1Y0FKU!FotCMQu$ylC#6%%DI9Bt4&Ob%SSGOIbV zmR5Rf8-i$>mU_fA7D=0g8bBVd{& zy`+V@J33gPw=}D1-B0T1{7GEgT;n=?~`+?<@a#)9xtw?_jB?J?YoN@@xkiKb>zdwhP7JU7(-)p9gPh}I{rI(y~9k0 zHWLnUBnt*Qa4N_{n1y)+!MCv|nkH(WM8`eQ-K<&5gY!a^StiZO9T!LTO7DKJfo^p3 zQt-yj;S7?2&4aQjw=Qj+t|4NUgmt4wIO`-VlhnrcVu8`79!jm7Aclb;O$Af?2)h2L zbp7PKCF~gCb9l;GKQa(=t#^o$CK@}slq%i&NSBuj+SX-wzgOs zGDfqM@JP;3f|CukfwQNuenLX>WT8S6Hka!JXJ8MdQYS(!5a+M_d_%#COAZa&hcp-8 z*X8L#(Pr!}7Y-~kv?#vit5Y@SE06jAJY~6jxOp#xE~R%*kmlv3!SSTDD)MuZHvxPq z4j(dXSK(zl0sSpoODJ8)7bm%3nhPdfU>H@n+>TJmJO`QQfEiX$Pfq&bw8WvfvPiiZ zt4mdr6Nz|Pc_^-Q+9TrHL0Tcv>J?P0YmCXj|E!_bh_$6EaeLU}=YLG<+~0p(MHws_ zh)uYm<;gQZw4!F8fbaEyL!C>gc;ml?F|6F0+s}|JBc_-nNs_!=$x#V4Es^ z#%(ca3#YK^=afavWv$Y~7jc1i%=v|b`y47HRV6PK?2J4~OJs7;%H6D+UZUu#BaP|q80IwhDy00KSd-2F?^h*_qNj2@cgBSq~zs zhx%H;`#7niDlKV((zlZ8D*o3_T5Chrx@pTwHH!=^QB0(=8m>2Oa~>drm;~S=2*d+X zJwz24IW2>-sG3f&WRWKOla@Lyv~MpTL?KaqQx%07x~JS|Co;246S?)+B{C26W?_#D z$fvrs?qB|@EFwp<1pdy`KRyQ zDB?FM{vN-(TvGfz3x93tO%!&ZMxV;~3mr=1y@tP{UG+PEi#rjbx;;1-d5S;kxOmA` zz;M|%z`J3-V#|FUKMpd9mzi|PTC-Za2p^7-E22xJE^2M1dYQ)bCVEOJB7J-2tL3!4 z1)9;@3rw}|GCY&3M|c8Ee=L{jk>WPeF!?EIn>d3+Uv*rpFVIR_Y%k$=j((yTKNRT) zneHZFQ@!3j6?Jo0z&kk#^RRjiAOU|sgG5{9yv#aC#_cmG$-opNQ<#81!(pNbX$|lG z_n3b8E2P7$Dfc-#eR^=ziDNQMb!vYuwePJX9_b5E4XK@WCJGAI+m)th({JvR##>FQ zQ_oB{#aT!@gFecsS$XR5>!tMO=PiEl$|^0Yw0UhAh?KN9A+b&Nc>I=nag&@pYZ+)1*N)x)HkE`r^P`o5*!=7#%I2q-_N??= zB;as25B0$K>=v@v!2%hF8;9u94R;Vx0`|5NS)?ZpWpa}6H&B;-#hbFWKq(yhrz8{!aq6IzcU~7fnw?yW!0LaS73Iq+yePGKg?|Cnyxa%b19o&WF zbtB8klQp%&Eu7RFFkeua_bn{v+*aIho7qM%suE#tK#j66YdwkL#<#noy-~m3}t}EBH+2w+9`wZ=mOZK1l zza22ka`8l8O$0W;$xhTY5{LR!t+KG~u6mla7f;%!F%e;uX2Q~;HqQ&}%b`NLs&7H~ zsc`F>_Nd{1hE1HjwVF5Rs}lZa*#1e=2SU89C=PEa%68})(#)Cj6$To#qFrm2KyGS zu=MenHRUNxyZMUg(GmNwct1JrpJ^6+n_1z5%nQ7x&oD7g!xD5>kLIMH{e zsi%3m$TVe9mZ`GzB}zV>r_Z0>AL!bFUp$EHhktwb@&4cKxm+4tv>rB1l|Eb=4DZr# zwQWdEm$hWajo@Wzgb5?^dz(x@h;(OaiP(t~_x^OCWhm!HNgAL8L8r4DsJdsCl(g;8 zqS)rSGZnBSp>2@HOWKVdNw0BQN5xkWWjwS3YA#|2Rh7j&i@>TASIE7B9>Cj^&M=lr zKD_(%3;zwA%OX#5i^b3qW|GHck>)smIkCDI#u9dnWKn$lUGGHl6_!R<2@Gh6CDXm6 zKxDNc;b_c~ADCdu1k&s!$*gYFzv2EhQK#O41YWyOE^cx$U`dgx-l$W9DN8cwgWTO% zT!1y?u9ca3rMF?S0+-*PMR{w+WJn9K)&DhG-bV_1k3Ai$pPC`R9cB?}q8LodwZg#2GU^jjR#{;+m$5_(k!IA#rL3w| z&=)Cv#6?Pfhs%e1N$>N=zdPK)?k<0sExTb6dsgjE ze_1_Ta#goHTxdWrOT=UK>mO#fgpYeG0d&{jz4BN)BCLHO*6tWDd6QBI&8o3OX)0lf zlZv94jY9gFQcW*}&tF{L2=s*!1+-#f$Zy9(e98pDUG;`q`9hzTEkPOzSfa#|*jQ*` zzRyw!s6*FGlE1%qk=$w~$yjDP8iV;EmP$932zY@94)3U`f|igg5eS10y6>hhkY@d6 zg6+E0u=H%$@zLDVxS^ODrS60d$6;wo!H%61TnO7hQvqpTh3O$VyP^(6Flt2ChT5~2 zw?DshuGIGQ#M)J{zY=_|#E{{0)Fz8r(2`4QX|p95^T6`$7E82s5WH1wgGTjn;b)K8 zP;9fM+{obVGKE&dXU{UES|5|ec74-D+&J_$<5L?!``(DWh+UqiHoF^aC?_EQjUW2M zy__E=XUa|$aDj1L!up8)U49w2H?UE-p8Bz^mnV15E4M<+e`z*c&o6 zGlGaeT)|uQ&4yepL)@^yoyWvW%rTWKp$Y5$|b(N&j0?T_U&p z&=zopeNk&_?(xJMHb#(lK&|C>15NSl$g09+Cn{tOqv(o46|kMmHv0T2aeyR7;2^LWKR&%$;WHPq1uF;rWa;4`<~?0W z9qxMz>XN686N|QhD+4I0p{Fs+FEC{FJHKL$y-vWMq~U{Ph=cp z#)~j0f{C(-AT;II4sDdi)f~_^j}1o1ZGfjavP4#7nV8GSEBtC!PIG=M#Hu92k|y+K zFt$u^4pLBtRd}xFc?@&gu@Ez$u$+;D0frp|^|5;nd@rL7?)F3a5OUwI9onhjQRDe+ zu=XN4O*nN!8o&P;82ThG80gzn>Z<*%cI*Tk)wJ57gahks-EbL$Zt!G8^a1nXTW=|q<8^p3ACgB8#?WO| zs~v0kY|xIWt8|vfI8Qk?B83nOYW-y`pHl2pTXIeF=7;b1xSvBuMm)bjHjjHYFn_!X zDdxfO=tuW2|FILpo})2RfBBC!cpR@nig_^fx^1O3vx%XrQCj6RgHN7d`H|aQS%RJQ zoJ~{VM;^BU*0j-Zi8ZB}QJFDXLr+FPA9*$!vT-9~4S#NoaSZwvFDSD)Zl*h% z;voORjqvAld`y!p-s`65=yYfV^fFtG;P_GtHqx#iBf zAl#K`j}|`stuC`F;-y zto1fa=16-Re*WouOIoz$D2(1&-;|lj0%Meho{WHK76A+zy4|0F`6J=nofhl*wIg}_ zfQ1|>@6t`M=ptBm4`{el+rZW9V{Qj2jgV?4JKo+WS^Htv1{o}V5WS)m!Rc0q`0y%e zI0I1|c=Y|q+{dSz?D*2gD`!x(BTZ|8(kym7wc}Iski;m;ijbtmJ|r>rG6M-eLeRJH zg`eU3&~(ckm-tnKmUs=ph4GnJcRbfaQNXhqmh@pHzMM8r~JesKb9q=g)x2FWwlcJOqY%8?e^+zw8hG+~<6TcDDrP2fJzH7AV6BC5;n z@iN`b1oq=^Jf9PfO;DRLP9mZ(hjVB|E=xetW(!87SOD?(1qbz0$h(kk;7q?`3DfRp zJTP>OUwI=cjl`9`@Ux!V2|C9QlvP}DV+ZT$_9xf%I>qfCbJ|m*<$)VK_hh&a^&XR_ za24lsn8z@GN5o;C21m{IAk7~OOMF8IEl9fdkzW&$o+puR^p-XLC~ba>Nfd8hbM$;D z-#_tm50H3D0ar}ZMYciB22Mw4;3>s^8JkMd@)AK$x4#1hM=gGN(e7RhP08w8UfT4o z4Ba!c`ze01I+vGx?v$w3uSMGEeUdX(^VXDMJ8TLTx_*T}u&unpe%lm|bhg^Z{kk`S zW&$+}M+PLFL8G$S!D+23M;~R94VbNK7L%hPQmdN9U`p03^3rY`Sl{s!Cs~`wOUpP? zz_4o(CGRe5;fIkpQl`pl|GV1rJ4utOOP70UQ5X6C`g;|UuKi@CbWYAif1Z6G6Xmf3 zB8n!@$QgARyn&MxI?Ht>G38q~R`|6NoT0>*n?1z>D(#8LwE?hgTD-6|Xx8;MPC-43c)^Jf)mS_6={TF9Lsg>%8yJIwA-I*t>NLX|N=W;Kr%fSB z&t;a-?geuVCfv)w5|3`SMTDueDZ&({%fmmDs0lONG^H`D;#%pQejm7-MI7|nlOD3SzLIimB70Z>3cMx zQmc<$sGO=YqJg2^xC~sHaQ!FGL<3VUY@YZlC8>5}(s5_P^Hj1)d(`;qB; zG@<$;^UP3D(@e3=2o3)!0xCBUXp349P?>4VVDMu?@D8rnfmJ6v0B(Jmkf__ z*IgO*T#j@56*KydGJl55J`C%uc&VGP+Vf4{_c_k(p69%WobJS$D$MJ$sG7QfjfV|^ zFgIdnbUaS+i_CtH*+)WT4$IF^ywDg?W=BHZi58mcnb{Ru&a4RIhcFBCi2n$8sK-tI zhNwRBTtCWO_O-Xc`0+ZNmOT2Q^| zZ5nzTB7*rz&sv}&tFh8c2`%REELt0!{8Zecwdjm{9U}Z)sAo`|kv%;qlJ-X=W*_oHB8ho1kW+&}W%Gw&!Xu1Apod(_h-3 z-sdCZ$RMDo$m%HMzhs+nFTqO|bfPrvwNFjv{v>n1hurfcUp5rSE+=_Yy?S$A+9AdwE};8fdLVwJ%lG2)Jm@r!Z3d9Nt82|cA1c2xI~FWZUeYZ7npE{{JHqct z=-BVWglqpp*Jgo6X6&s~()6AkQx%k8K-TMPAzCF>j6cp~?&P_;#!7 z%pS)GIMsf*(IFY!_;DFJ8WN~PMPvJ@Jf^(I_*yr~nKxZC`r-|xD7 z?}7bU<$tg8-)s3D*j|C79Wy6B5SHwmG*3l5|FMF2K#72g1Y#WDno7@G4+}&;`xURw zd|fa>$FR7mT_vy{1tz^tQo$&?sYOkILE;>Fi2SY`CV+dOrI#Kg?1Y=O4#yeLyAKka zxX{MYdy>&SPOza(su$2XMq!pkRT3w8lZIKnKRwn*U`6-%o|k&xmRgCvxRi$Ho6>|C zVR;E9F8%WE*A3K@F%-45$ErA&+2xexHz4CQK0iHWP)IH9XKwQo+FVOmktK|x;*vN2 zd}%1G+ig9h79jm37Fphr;s5%_7m9tn!2?`3^}a9LNj+OIp1L))&x>2v=PAtdBnzw4 zRURDpxJd)Q z6TCh{Uftn%NcFmroSl_n=Nzo|!@8tDo zEVBw}AW|k(#2augMHScA6oK`zc!+ZRgf`xD8~*?sRw8}BCw|>&f>W86TTPcm z(|vqtX@RLKzP-vHs{J%cEEf3uz)cSOz#)sQKP@LqcFdMxT{llvaXCFT+1GkULVeFu z%s%nFUnuXFkN-1YkNt5;s#mABi0kjN=b`8S?{L+=UGc{yjdpu&n4Jv>q!GM6-Z{NK z!1PW{3zI7Y2iK-Wg{$4qY2$JiysOQjw;r6&xNe?ju)YOVaxR)=+ezmGAAQt#-C+Vc z>><6|UAty_iw|OgUiNO>5{CZ88#OFvyVP&6WfPzD8)Me4i+vEt%~pOapg> zdr9-=hwnE;D;$`?{{9PGoW)v4Psv$q-i8q`X;{j#^7DCPPI&BLyT_a~*!888bDkCk z;oNpjLf1zhJ6^P1i9n|8zdcj_fC{){Hh+I6$d;d;bwhRWHL0r9uH(51^w#49hx#{> zX`T#!%{FO+E4}C$s~d<+Y{f?_8&KhZN;{DoL(u1^<)gvhd(1aWv5896*xu;(&~edo z|0OEbD9}9CZsgr9YS6xmKHP3F-+9EZWfJ$+S=qo=MVzpkob?Gnp5B_SSr(VLBsuH} z@FIcuMY{y;BqDu(JvhI|CCP6KUxY8+abS7MWuJ}6>NC)co2(^qoL0C)#3s%AfsY=B zf6(<0^7<&~B#+C2w@kDk-Y{_?tn2iYpJ&={vM~NQq#+9yGOrfEyXVw^PDJp9Sj+EJ zMt@!wL8LdM*NY@Q%W^K5w1Mnw=w+Q8pbM%5s!&gUo!H9;KGSiR$M7HHswjEYD7G%y zn-A-|pW%1>K>5YiDRzA2vFn4#ZJo;^=TArN%2ByA!1+lWJ7L$S!py&oHe$nuASU?f zvt(x8SSH5sgq;)Kn2YxMp^bT-;0Tq-ub9>T=a)TP;ephg8X$dqaC#`^7sJ+Qy(ble zcM~U(t>tESdt?(v8Q53}+q{7#%tv7_>LRQ`_g-^udg+=eRIVPuC8B4x zuo9q|K-%%7JBDYR0Yq{55lX0qrqd$mGd*!DtvIdulR*blyt@VOFh75c>A;3A8DPQy zO;+a5FC83a;Nf)TQBoOh%u1n3VUYW4QiU(Pn=8}j@2boxJJ<5up?fWh;H7a82FUL) z_vi41cfCJ~^qJe(Va$7vG_Y?m@TwD4MqJqq9GXa}Omm~L9WG@mPiBFRl38G+BQ zfwEYU5%*IX$asJr?Gt&t``fYSc!Hk8+p=r4t?sP0D%g~u3Ck?!wl1FE%D_nmt9=dj z)?={Mfs-)v`yS}#9A`u7Qb1A9p2x(*NFAKZ7B0(6Ea`mk=l#3!~#9 zKKgth13uaRG*DP4IzG2T_cFwQzX?akTmO_Nf5&b=Vl-v3J>VLMdt3kYG0*Mq*GIjF z$Wt2IO5B8}x=f<t3_{du`A zW5Gb?rEL#}E)B?j#B4uPwwa%%UJm}&Ps#j;$u+oiof*);ch!5I;nU$?7kSE`!L3>J-5C&l z7Fm9!ELJS)D9nDQ_^;%t$o6`k2mTD0zQ{Z-PxW5-Ct%N@RVxR3ixw?AAF2dy zgOXV_b5xn$%0)xBqk~Rc!)Vj>s2;*de#V`C!9q^N4<&2}@O<*yZ2K~m{$^gFGFzEc z=d`Xjf~HyT`TeA>I6yqkjJ_?0$+V%gA=Km4P?>@|JA2m~W@m1$f;T+_SHY^%JW9(j z+XR_1-UlZy*yB5fm*xrPgTM%7yeV4Nbv1_l_)gE}z+|h(RpB>1t;)O2!wP@5fQWW` zzCQo>?!6Tt{shEnlNF~^nx8C2nwNlNCSL~N{2Ac+K2pB90cP2_4n*tv_?Ui6;@~L> zOF3zck|4$6?bb7Q*P4^_?zAK;1%yQ=EHW$(Gnz^{1CR9q{{P&)-E!nOvL<-nPZ4G_ zw$^PkjM%C^c5L?Cj47qes_f3JR4Y_f$IcCn3Q{4K7-S^L&`(~?YwYXolPugF06`KU z0gw@jtcy$qLAd+pA4m|t4hP>pK+?}FW$fG7)s^OdBpddQh2oA>Dej0H9<240OY-_= zeK-_anmd9@19*29KJq)FYc|SnUA`_0)>a8OtSef+h7UzmhY=WX#7sC(7nNO5ax4LxfcHxN~E+X5=86=Hc z1yeoVN}I=(##RqLyEWE$6M17+XRuz#@VL-Ch#ON~5Le=k1zz0V$LcSJph_(IKJ!4= zS?zg&nQnNdv2K!(q8a#8InuJ$g+kN$w!lU3{n6Q&d62iwb-{>$|HG3q3if!Loa#+g zjrDP}d$#YYSd0YqEIewba=>7>3r{S3y5ZSu-D*b&JJ|!JH`@i*4p3(OfXRWKVl91N)C9Jr>cpXQ8GhqB+)NzYox9d?BiC|X^v}x`IrCnegzs3W3I9fqLkrn5g+eg z^JtfDY@&f4%k>U|alc6bcIaFU1rs$py@mBm5V!C34g6J%kGlF3<^5;!>fxnwRLkh% z^&|P(gD~o`a(vA2l6JG-yQq4E-Y$gAv)ICOB-eE)M!o-xLKN|DY@g2n0dMy*+rPyV zUf=dd`Ar{iBa~79+iiwP5Bys{>W&Zj)W@5LouInqZTxKkuy7djQ`P3kv(`!5 z{;0ncr_`fOViSRFZWp^T*feg9MqnFo$${l zTXzfW!p_cxy%&U50X`muR)@NI_Mp|llJBC;V{w0fQf~qihygE~rw>{_EcqRi`Eq~n z=sg7ArW*Og9xH|iy@Y)932^o7%?i`H1Sb?*O_MBkGaScpv2B0UQJBYYn8xV2yD!tC z961)`NS;d5qfTD)oEHAUj}!QUW_q~chb6|7eVG(vTD;cZq(;&5A*vou_IYaT28!JQ zUzR_!Y}Owwty{__XF5!*PLt}bd=0c<>j|^wsLFKKk>DGb8mw-5NqGXSK5gQ>j+3~r`~6*y;}wZhkX65aHXe)+6E2z49|! z?ZR7QMxTn%WMXN%8en$k0^L#olY>jEwE#m-F7L}|J9dJ>alH*B1Q)lLAGp-G|8*H{ zMozw&=iL;Bx&Uq|eH`i2LtGuHC4s8D_I`VaE#Cg8w5$(NcCs1Aj=dgex@|q9$koBP z?#ZyDF#~WIF&;<#oTS@_I*Culde>kV(C_@dHSZmTVg)}XpWL6IZGOZ%v7VojjlR~f z(MW_6N#ZEprTJLcmKt_S!PRbOcSceT8=*w-5YJ28z^zwF(G159(f+ zM=kvEaE@GzRlxg$#mJE__?G^*=`VO|WXz`p^H>$7qFo2krh=rwT-M4w>PJXpr43^( zGFX3xYMX7xzIf~ImSJx6c62{5wB`B6!<4l>EFgjac9I@npQG&2^0Z78qm71&oi2z= zylD#=)@a4W)&Vn@a01-6(#3E^cda4Q3bVq6Xf6{LwHxk^sUC8=|HYba)=AAbNnRZ) z2k?!M6ZoF3E|@PY!zE^LfnO(C0>SiN9X*h8&Q5eK>ioDBd2C&2UcKYLuRU8`Fzd(H z-zkGFeEeY@XsB_(KDMbuuQAm{1EUs+U0|V>2O~o*5;nBnLcA)Sp(~>-ExbdcEM1V; zL_GykY^R#wVk>nV^Gg}Iyql2S396#MYx+NLBI_5hQM0TP9G<&RlWcfUYYxys+nU*h z<(e{nz>MP#eM|T34R&Cj1f1Mx+2{iJFT82uACJR|HtbcdGWPk#*k1gC+Dcl?AhnVi z@thy2VhA|M1u`qe`)XV3$k(uy+uW{AXgBu|2t;Ssl%zbct+e5^O8tr+_D`j~DnJ*tH| zt4{Xfv~6h|__TI=O5UMTPDd+iiRjVPK=npqG3Xf z^)ALQl3@PHh;@ zX!|y9_E8Ku`+;&_%iL}*R|kAKO4HQFv7J@z54pLfd^g%T5E3j(9*7K=aK?15?W=Ku zpFCqi*q2uh!L{~?$x=A7zQGovE1uzJ%AlX=Z%L3UL=(kA@o~M-n{1&X{FjjBA{O4* zdxGVC$WQG}$E?XCe9C?m*&MtYsq1HUMycC1J`C0AedxTRIT?6i&jg=WWg`ycMONfn z_}=l??&vSL)?y3OFU2R!A>VZKD<0=`{#jnt(Oot`KFfjA8F>|A&T+gfE7nIswQBDw zuWljH4%Lze>M$+_p9f)}n$e+M@(3XM_263M>_?Ih9CkAka@C9$Is5URXRt@khu)72 zTv##T=A--0u#f$;bS#a)cY9{IVsVb+Y21+3MqClcMmoDwN98s-QJHXCBd*+HE_+A# z%E`Fcna)a|&ylt*lOue+<`w2LI3iX67V$tk$A0w!0u~GYK&NU17Gn*=Xoj&z!PI4x zSNo!@t??+zRh!M2_%JpY?G$Smadu-AB1l-H>#J9d!z7znAc^yuj#!sF_H7#MCeDI6 z5M~CkFF#YRaUW>84p+{n$mVxJYo+VKnrGs(%7&%;s#I0L(WY*XjQdlqv!bF{<9&x` zhZK%<7AC?~ItMM7L*0TtJk|p*s-eO3IZy`Nhym>K5^vjMW>3+v5u*tc-Wzt)9_@3P z*2$6Kk_{8y-{GzeV6VEu7x`$FK`yvg8mREOjP`rOo(UB_&u5LjrCha&_!)9e_cZusXhT6)3D|W~3e0eizpfr$5 zdrM^PN$k7cT1Pv9RQh7mPJ=ipOW)5G>7hfwQKKJYBurZp+Hh=KSfh;Br>n(7aj4IU zhZY=59+-?556@ABD&Vg#2Ox{>Fwo`aKZxRP&lZev*uS&Cc7ER#1vFP#Lr)xP62hP) zY==`%SP^Ms6i9Weo;skSZ^M!osaKx85(pLQfh-cHy@%qEmQOrwkS%ds)-u{|%VZn1 z4)3z7aD|GMRDrEJK3i-_hfoqoCCXu5{c2PXn{NWPs4S(t*pqT!l+RHahYaq^t3L%= zbyMiz3Inb%K!tYH7+v$A3APDVB2aYY#|?_&HgWh7`Z9vjB8GG|PJ@IFL8U7P-)ZzW zs$O%(A?rDz61;?~U6e;#YYtG?O3{UlHhCFxZPsLo)60xqF%_?hHp zCI`<#kK%n{n7la?o&p7?CuMlTdK6z5x(T7aPU8z2=M`&+KO~#9+#J&SjwyXK9hT5% z&sYb+91QApq|aCfnQ%I|dX-m~pyHBLyyGh7q0i;rd;``@uje>f)-NdT{f1pjIBLW1 zKBx7=P;j$8IYP}!y&G~Jj=Tt6G-9KHH0;HDarbv7VvK&)*U=pAz?5XgCdy<=oK%}K z-J4Fx?9(A7@oOd(yYbMR^uV$i0gX8F3FKr5{wpcx%*Z*B;laa^C|H`8_`eYm0U>gy zV`QIdukztGn`;UiQ zqySg)_NhJ7n>V7Rfou7du=7@o{x9zs!Yq|Z1$L+@(ZXY)W)~go&ZE+ z5B|SclAH5g`n=rlJ^_;m=sp3X6m(s%AOU;$Pf?b}aE^VoWZ%+sci4q;qzR&YQJP4S zxyRYTs05AnRaZMu)U#+!7Y=YV+5@hs8C8R@VJ7U(EPSRkKGs(KCUOrFoJ3{zN^3)K zuh4yT$ZAZ4w1Gni6dD^H*mEZxVUrs2Lo6(LLqS?mg1d>dkc=)|QU3Rk2+@^H#}X(C z7rs%1=cNkcvZeX{P)Au- zJR3JroDqA02J#d_F)Z9i(02j)^gB1q2CxFe*A;%mNfx=*T_}jl=;sExl=?hkuyhP%7vfmgdr5uz1=F@4mn)aid0cx#dIHX&3;^Qo#Nzi zjJhb{xp=i^Virqrg&y1j6UEUUB1=F_xXqy~#dl4lE6KK0f!>u)FHTGA>^brV1_Gfd zP!!RaZXdX-^A|k*vPquKC~T$X!Efub$THhy5a&Uw5xpx7KXFBuQjx`I^CW_oN}M;}=(}Pv=z^cb?2?B_y5!o9%rx%I_*2jxv^<%$4Ddw3Qxyn^{y7Sy<;lDS0$Zy* z3M9kD?sXE1kR4*Ro*X3hX`7m~(q1c*Nw0Dc+5)L9KyA%9@>uh9hyvImclyADVP_uh#l`!Oz3g^@d&Q~KP9K@sGG)OgW=qRkP0 zi*cwFZp7b7@k+nFvl5A|)1qRrcV2#*92V)f9G72W|F5q6!WvfM>jB#?zHdv%d6*Ri zEv1Ze`03*_*Me>fjb5I|c>u!bd<%|^1DA3BiT!lu0FDH$QogxRT(}V+r1*MLuFs1C zAkzg4E8uFvJxb($uWMYOb6qX_k>G zDt$?5mXRrl>|D5%SJO%0vm6?n#wIr zy@&ejo}mGRb}sf_H?JoO1Du91cpTm_;qA1+K#jhYJV?%yG}K73mq?@aXrIA5zj3}z zK?}CwuOlb4A{`a@Q9z5%(efC)%c89OoU&UsemvLuYs8HXIxG(-*{ZxpH`!kwH<=i^ z32MCDWP<2sV}A_lCHw2_=Ao*KT_|c6^ca&wiksJMZAd`JfEO32LsoxCOB~XCo`;kfI#?&A%CVz0MBvoU)*!7jbhu8- zt7FG&kl3l6uMuXacfd|fty@QIgxiUoF=J#E@5Z4yY>d{F7r&-JGFI(Hqsx9RMx(k3 zbER*|q2*+%(QV&(Ra98u3y#0s2`eN+g+C<)zOaN~9O@PRvUifDhj6^=H=Hu&=d9vl zJznEFc`dc^C`4SfkMAr!KxxdZ17_vYg(}G_{(0E(Xxg5Lo{{$K5I?RB+LkVwcdU}S zPV;Sb7R+=YS-C-zrfV+a$pxO3c2-oh5m`Bb2g^+)f?eS6v*192v4Z~orH0qz+eFX3 zhDndjz~~66mI?Ra(UuxK*=FFBwo{X#;<0i7Pu(#c@HV7@LEVS|e-QW=uHAe6LX)5Y zwfeFE!Xm(58_YnK?0atkOW)w6CM#27kdwY$ZJ2(pQqN~@cPIh5;z2#l< z5Iv=iAIQE!ag0;Pj29_zq!X#>t;v*V-CU)v#yc)NFLBGe*L_q~-xq%Y2R((x@BH?Zs>mnt`T-QVuA_D(H_GEEdyZb&Cld~phwVv8US~AU9orZSGlOJE|UdCy9{0ojAt-DrSwHP)C&jQh=0 z>2=dX@O@C)!s_NGJ)5qSZ8v)yj>>690)j6%4yH*B7}Hm(^WlU^<6v4&lMoz?74!?9 z+tYJ)fxo?9zhkkW1!m>aPn$iAvF3iSQcG1`JTh)Qw{9JanI0%BS8hSdYg1RB5$eRB z4;dVfJr6XR;6OoY2Fc92c9T zK4r=I*qfsB#rb_{l$6O6OqkEHD&MGeKAN~**A@DM{(63ZZ7KAth2EOV@oLJJQjeMo zRR2*Gd03ZGUTun)TF{UV$xW&|ryaFGr((pLQcxQ-iYOji53Q@i5D@7fEcsTZ9)m>Y zb(+`39##ZZ-xs4n3;J|=m)D^03+X!w*L*P>Kdk|_V||yr8{L?p4EUTj35aBAorCIqpddZ}f;Kqf_ePK^bd( zWN+tX(1T**w%mfuvV8(6dmd;MA>1A6NQCC5B$498MMoRZ(ly}25a|en4qRH$2I=4) z4TF5&XsNK>IRy^)7+gI`cDL<30_0 zM$&y)N82qAnVIhSg%cZ%4&Xx3=)(kSHF0ss-2k$4;N$fzT})T6|el#jth^)x~soTPJuDwgg8S&sqz*Nr}=5Y z-pufJ4L}?dAFb2Z6!r2(V9w z$j2G4Nxa#d8l9DEoO7RkE1>H95ya`2lZzw*Ug9q=KS>AV~X0hy8Nyx7SDl173A7%S*s@+xKQ*U9jab0M!152DZocV0cPvW7k<4i zH};G)4b$-Jp`dAV*;>SuyTj%&sl7rIc9jEFJ%c`Uff9bAAu7Qxb@UtO0=z1|1@k_e=7{4RMrz?mhuG;xil}gZZ=~zXy#KlU- zFE7?MdKE!zzwc@)(7G~P`CKvWB4qCgDzxNfpR$%~RKa<>Flgr$qU(GL$L!B+CG?(5 zTu2k{p}yuTre&ENsw9i5y1DmfE4o1E`uYF|*CRBNlO9{Apf7tt+z?*e}I# zUp?=wWN%miR@=?SnfBX)HcqJsy!*Y-De?MKDZ&Q3(J06N{iZfrlNuAJGg=cF<8v4- z^d1uzdhfl){g2PW=!Aft4fn}`@-D(*1NiI40TV#?L4lX!Ob~qr%VY3v-d$Y!IptTt zb2>L(mWq=eC5XXH&|UoH<|vAwtE9%uSrkQg3Bfp2iooLUn96poUMy)_LzbYbj0LQKqT@idrgth++y{_K$U4aBVbHUk_3BfT02L zmz(n_f^L%CZ-9`?955!j~gzzZX+z=_4;N)Rj z7dYF|?&G&m5ugf?Xalr&Ong2sA;F%Z=cw9&IQ(^@0|bBsRN&(}2tq!zJO=M~_SecO zScjeDtT_SVVVYO_WK+}On~)+CY<(5_R&9N_tF>*xkprQaZ)X&0#7PnjP&X&x*rksm z!Hu!m)&ok;2G<07^&TZ_UdsZ9T=iHQDAmd7=$O`}itD&?=S|sHXUJ|9g3&S8l?YN1p1~R-{7sSZO${%vBEgxZwmyEu zlWM`2DLQOcsNvfj6rgTSUPH#~pH?Flt)valx7 zdc%d(SZ7uHtIQJoTOKWK*vM|F_vMXgX*WAsAf4R1%__GxN1oKrMftegWNDJuVO3?g z)ddG_Fo~r=f-P5E->Rbzx9xOHIMT&62c#xoYH%kH4&}6vx_NntN$d3|Smo+QS&`>- zTzn|oE=`(KJ593IQOS2ZwaxhQCiMieq1J`#=c_=%5#2HDA*hrflHdIpG*v1ohL5@ z&5;&t5KUoam(&kMyo(&70vTw*7kMT^R8;o9d`79yk!^IKaY~l^sEpWkQsO1Fq|*Cz z@|$%essa;QFAA5}LK|MLg|fvZ8$pobTCRHwRxRFvmCa@&%*neRK9yC|OZHuF==$7YZ=)4ILMd zY<+nDf+7CCIkB*Xp`dib9~Y{XCRUv-l`GN#^l;^#+;FY@WguM+wW-NZ-)gn352tS{ zdh=xO0Tca(CE65{cB)upmtr0sNj0x$MBhRM?o9!9IB*u*V<@Ao8f!f2^(*byG~XZU zX~c5Hz^Sk?3#(jNr@>8teY{rM*&jj4JY2GbeNxu1-w~S{$UdH5b2W1_S|#OEx=Fqx zLi>n){9comS?HWPa@nSEuCdEDfi9z$Cht7PFWai%AQ(#U8m4OV1ZTmtw9mp591x>I zbn)#cN^l>tGO^?_+Iw1#Mn-;34Ba@60QdAv?-KTSn$+p(@#EGl?D;pbV~C5hlnJa| zlt){7hf&S0xJ$DHay6NF9#rsOI$4GK?=akZK&iP#W9@9=GSjM z;{wzBH)oN3g6 z?q{6TQJVdkD#al$=PbN+QeToCT7Y%Q>|BwZ;}&CGMjMB7qrQV86CTZ3Jh!CtxJ7Z; zZ_6l7{&dw|+pjr`*4 z+=`<5-kGna6Mc+FyrPT8GHcR%s}h)@k@9~N1)~bYGOuQ5mQC)@65on(QAz;Y_$3~1e4#>efFlw3(Avk(a>XU!b~^q}i=Qn_t^m@SHu zpy>z)D+;o_f+!6*>JdXpXxkO}R^7n|mtM;g6-BuIRnhCo5~@)4?KRNs{rL=+?n@GP zuhrK~VFfPzSZNd?D~euURzQWqN(0T_pUUiWnWbFwoEFih~8I`>_t#pr>u}V zraZY4S|nXmF|`4^JABQP-5Hz~6kTvOQ$%pVl6`1&ts8#zj!!6X>Hma)+0yBefZ$!6 z)-l1QU=pX5vSt@9MXV#w)pd$WbYHRH%L1rS*6r#*+WU9|mm++V&l)%bXYRGi$(S0h zKMef1RvIQ&FiOw9C^FY0sNR64poi6mH;h4a&9pe3B{fduzGThv7r*iMQme!WcMmj|@|#`|fUL8p!b5ur1X!o3&}sX4;Nzrnb9Vn}%z~ zw`?07^w_iA=&H*|D%zAu#IjLK!=EswQnCf=CsW0gNS@jlon5>{ao(7kkuq8dc;=jO zklv(`{0ywcNtT@ZgVx6A?BW%Q&-b7SIA{H!bwGNPHrs<%CUu!6PiOw9bx}IIdBYXV z_qa8X&iaAt!JI(dY!6*^xT`j0dUxheT@R+SqidmIzUNK>b=D7FALkVIW_$G5Zyr0z zZ*&08laO1obBa3?BlcVezh?= zyZ9@L&-c&?IA{I9bwGNPHt9i=9oA^`Fo7s}!l+a{nJU>!MPyFST!P3{LJn13?9y*Z z9Npup#nrw`MODk*=#eK$ibB zl7Ta?X{6I|H9C!C;AqM;k{psqJwwTyq&sW323!ShHf^BU`?JndzM;&SmH0J}cIhVMdjgx)1(exybkw9+aHh99fkCcNY;9~7 zSG3&l<$}Hd9s{WLrg@a%!mBEiO7PU-&BXVUDQMV{kZgUp-FVbpS{9Z?9<%039-ko{ zfyCy@%koQpg_1K6#C}YTzbTFN1V?QRvOExfg$nzv)Ukkr#>?G`UAi6Ur6wFmU zEuWKg`%ovb=O^+MtaTwev-Uez;PctEVr|L)Qu24eVbBIVmC+^tOUZ8hK9uQw%}t1# z6jvjxi*T3JQ5@9~hA+j`x)5p3JS(Q;&Aus&H_`{-TO4SVmx#DY>VvungnEo`+VyW9 z+9*WGZSk3>ovL`2kI5^BM%hPc`DZVypyljw3U2w&(ZO& z+c!_>6m9X$98X1C7Vbe-##!)-ywcJ;_3+LXpL3nv*&@*8=uU%c9=pI=ntV6ww#b>f zi?BtC%|_j>3-0SHv?Tbgt~fpw<0~t)gm|p07@%d2NDZf=$~7j#0DY4~FlZ^Av{2>v zELty8(GussL`BP$*8-K}b2Pd*Mazx*(v+?my7pU}(q*NkdnR&P_#FJY#cPk{DSCn? zu1`4~Qw}Rs^ps6mqGIIk<@Kx?dwKQZ>1A5gU3;9*auQlbPG0HpQO46Alc|$x$E=)P zX45)uO`1m=rfA{}+6f7Bm^vG#Z^ES6b5!k+ME2K}E06(>P=_Z)AdA$1eG8nwv%gLb zek+Q{B-tlr!=6m?s(m{Ofq z$|dZA!HaX}QU}zF%QY(U>clEqmZLr6V6mH z6#-WfpbCBpYYU~le>ch3-zn{1kTgiuLlR~6L%4ZJHjg;(0WnHtbk;FyI?@z=RmNep zC4#gBLJN<;Dq^oLY|vt8RVPQOzt9XbxVlPGO()vdS>6GX;|U8Z%ruBF>~LN0X{$Zd zBS~1jvi{F5q_eOvnYJP%bOX;^hh)3224r7h{*_)kFk`T5%M_j)1`<$>CIKl@lW7xz z!2wq)Sfe9Z;|)9{9+E9SK8orp-R+t3wpL2?C%2~*c;z27cOFSwheAC>2Orm%=*}8$ zW)DSB3US^JzYOVfsGR#w>2_amVz6ysr8S@&H`KB0L@%0>XF7|6hr}gmRE~dd`Ql9s zKLnoAEgAq??NEo7$);BHilhl#ceILcf@)|Aze1P7YzMT5+c!MPmdPY4v)7O>Oxzu6 zoMKvbujMjNkz&QCR`;rV-7M*p=3H1sBVyI2(9pY$wr671GJ(s7S~gt>=2SMy1EDJbUARbS0KW$v*F52pU4Iby zY-?n$|0&8+*2XTFeiDfjjC$WZ$})#L(T(y|OS$qa%Pc-zH~Iq-;rg4$xC`Ec1$huC zPy{I|2-|8)ulnWw9u%@alL=)yfJV?bojj%mgc@8Ndy)7*j zYsDn@hb*j;szS1h;xWaqF#Wn?-{6?v#LQL{b=9s&8TI?nfD1=LTfnsiqyrx+14suy zFY(tqP_d?)Ysl=uUGs#}ubH@O-cgU2v&UWYLYhE2_(|`e8+U2WT5~ye2C1vEjIykF z4zskXlQ`sOSuy39OsSM;_m1}TuQd9BO-*v>HYy=ZHC;K>8IaC^A8G}%23LG&xq#$P zm`W#GS-k6g<*d>Dlop3dQ7n?Ck=;?u`yOi2WRitGgH@)lI# z4x76ui}FpHZ)s2{Wsmlz*zIWSf`O=@=E!T9tYew7J3GbCl1pG zXHH%Zsgs|yF9fGa=AezXsN~V^+*#`ab@Gz-&-}uKXW{Jocu6O`o^!sK1mipSEOdc- zFgDl6fS}Z``{cXetO4r5*)0>EgR`3Te=8`x@1!+s!0Y7mt{k4*7pSSpZb6*WdCk}4I+Gm9FuUe5k3QsyRaSti@$<@=&AAk5Q< zT;6~ZFFMmbJlE;om$h_{^ljV7Wt3>Hg}J%vO6ncAZA0ER7J-6mN*=nh;iA3 z-4b`~hPMSps?HUv!lj1rBW&4oBnkpm2w5D6O~i<-2CKENFczDzR@M!_V==I;Rk(>t zT0)0E<{=MN;a3uV%ts!o!>^(qtF^C}@M8w@wh#^=F4O$J0JX~HJVYuAfllReK2jB> zAU5cOBOgB{^#!~9$S(cJu8PfLQir%g@jlulG~kUJI_Q?XV@q#WBgUa2jbYE4BdsJK zNO>CY)}poa^@^m|cxnk%P%pysKv8}+REe^%nnh;vpA5(C5CAemR*SB zGDFn0(b~GJ(04;wYzQpVC-4Lo#WF9R(ir<7MK{>t|HxP50NRmde@hIZ0;fB)?#0 z617Z$WYvt*$=!cNwFiw~59vm2jizz9P4c9~-HrH}VK31x%~_8*-8A{qSmv_h6zr@F zcAzp4yA%~`p$V2H^I%eKo~LmjbG0tj(o<#61?gCvr!KZ@u~rhD3F z{j%pZrbrv(Sg&XWim*xqg_Z<@s(=n$QB(z>g0D9bnCu<7l4-+q+(nzE9M?JQ>Gg(f zy)Z3=tw)}g?jeO|9VLgUUCAM;u3BtAcGAS)k~L?{LNj|fbmzBS;ZP16&GOW=DK?pUJ>Yo54x&iO0Uw6T!O=8kg1mYR;4 zmEH^vf#?BPv&dTj!cx%aQyQZ92n{*p$Ksqo1-*F>g2dBTV#7j*38IrDr4ksqzSL@V5NQFd^X8Vk`iEOEr zOtw-T`vF}>@s}E+*Z|zccVHB_6fun5o=h2ZL6(q6@qQ@VDqG=;{UqU{M29&l= ziog=i58=K{i!!ZW8=X8QZHMe=QeB=!p?#&%-XgL|_Ie_Porc&(p~`O03uitA(bqE7 z=J}+vANeUQVYN>->3s?lQAHNvsuw=?Y)37fYuVGbB5kt&z?It)Ne5es6J?~6vd6cGl zG(ccadIeGxgUP;zmRk56X%&Dpg;%OuvHZ+vFW6s4M=?(H4?i>q7hu@OAMjF)Sx6~& zCR*xS12f_d@2L3Ggn__*_@OztVbB9NGB8Tz4|tG+7_5JL!j3{3a* zQB6NustSz+x4yb#0YuZha&WP zlOi=cq$`a_!TrgG@GQX(|a61<6+ua zettxw2QTzqxP@EO&_OT6AT}H4E*k~ENd^4aNKd-ZGkQ5EQ8cO65d3t(SaE_vR6Wec z10>v~c@iJxc{&DJffZsOMj-(;&;R%-&D8UF1hs%{JW<-r6CXuf@|bR_WK$;oWRUn| zKH6Yx;QvGtYn1R~pjogh7URBk&8Rmz*BEiq?a&Vu~a$vKk^t z*{mWps)9yC(@1)2N;+t49Dv44BjQa0`%q@Z_Dx%I4GK%Y*;o2gvGsO&AC>Isfyi%X zQ0xjAOYg6xgf|2H{GU378;tvNAfk`Sk`I1f(JczHN5hjAIl6ov-5U)Y8!zNQw_Vrf z5bwi%ggcFKerY%HuGt;C;X~EM^Rq+l49f%U6$aLBm?~p1T zvw3gkoA$!L=zJX&qQAXL+}}S$uxY zapZ?Pb-S6$4u5QSP$Kfv*ZOrzNht>^QM96%-wsgV^;nSRnm-GpE-#ptj^GL_*u_<^}Ga3}I!(>gj zM^M+Bhq%Cr7CaU=qND9;{WGLQP8AhtTeY?EXf>*SYi~(mz{LX+=PR3g@a!DBrj(1p zx{Y(Yj5eF3so+ypNv&d z$8nMDj`?<-4Aw#>r{1vWtc%UR$eKCVw{TFkKIY@?r!}AC9d~o{kZc}Z3JzL!xE~NT zxcJmI)xmKIH?0eYtFo!n=Yrj>$lJJWD{l6YPm`D`nvfsSukp5>6AwugC*@nbEnUkRa6e-fw>O5=R&Vi~^-LRGs@$G5=upEOc~WPj zO`R9)Ce#4-@Hsc9_CT67IKhj<-VbQqWN+X|W7puxEze-*O1j%6af)YWo&G)17}r4j zC~~>ID;ySUj@k?}%f)E0gF&8jpjv=nV(ieLA{xJmdy*eJ?2Edy(;H zyyM^zCE(1!vAt=TGI>G{;9P`_7lNT9mq*u34~HlKd;Z%htzPZP+PvN&>Y{s3rDW^jP}Oe8LJ7iB8r=Ea&UzgHZ?~h3SMQUomU{$1ib4T)Z+TY zt}s~qXbocY$D*>B*`ZF|VDxCL8rUVUTEA*mq45NpC8|6t_``onwKl$SnPw$g*K{8u ztwH-t38PC?M|`tgeR*6rbxSZuJhobWX>#7`VLg(|y~oz%==gK19P9Y?P|%?_cL&^v z6A!x96~sur*;l*QQ8|GUg>k4@S{;D)4l{j>-5i26Ou`halAGN<6NUlUH!STFB>f9g z{K`^qmKQOt$v+1kf;J#$Quu>GdkSV}Xc^r_d0gaY%2&{VH;ujJ1!+4Lmv}3xs&tFn z!OoRa1I&rs;;jp*bvnlf_>h$eM7gQnlEgL+YnFZmu^SSbiw6FKg~#~xM|!wWgn@5y{7^#iD0O^$`UMz(2%!kr3c~?Z;$O0)q@t1B3|O^lwJgV zvx9DlC>H-2SFV22wM~USgLfTc2A}A)AFa+!o>)}pGrH2L%uT+GRpyP{J}FP|w6(4D zKwKxK#j{kG5uLv*PYl{5Rs&X;6GNWB52qmwlxPMtIySq&*5W)OmT7dDYRDbiF{&eZ zrW>3Za?)yKO6#xH`!lzfz1K+VoB1uOU-yX&jE-AN3l||KqVN3jJoDEtytUX_T2C+4XO`#!&v@HFWK(6wB^Y{PffBpMg(xJs; zPs=Yt9CF=!xCC(nJv??gozHT(%LXI7qxU=m5Re~scOwVSq0d}|&RkKAbRk^fU1L=C zZua{$gCN%I%kBi5#Be4naRRTg*M;-}vp~zp-VJ4wJzuX2_tJcSsIC0LNPd{Ta&l+L zZWp7nx1TQXN=cFCkJL25b)XW+lH z>%LyU#XD@WJmjgzTMV09UA~)mK(>=Rs)ep~*cxm7i3CLu@-g4K0451%#BD=5+-uBX zjb5BE?t)#&IHB(v>H-l*k^3UYR&v`n_i3tmTgTpjMJI;{_NVuc!IonV`+by3|w zKJ?)tkJmDvi#wLgPw_#+eFNN$e0IGpiY@yhBtM=2chDFc@?iNBX1jU+DLZUxN5o9R6|rJOgP7g*L+o6+Pengiv9~;NtJblda@>YZ%S6}yZ*bGP3{+Li z{E%h%`vkay#@LVt%U@VF{B80@eB+h&!ejC}-~rF%*g)Vp)fNJn>0uB}f`2BjZip>k zfH~1?UID!)HgAB=t2B>K;tJR?<7amrL5po=rvfkbmJK^0HkY|N^H5av$vi=0!*2Xs zRIX3aU?|IDxFPHfdt#hPA91D_Zj;XJO3}1QjgA!N_9)C4Ldymw0NL|b?CNN12G4Wj zG$7^t9rx!IS)u}}*#1zph-5(3C)UUz1#(N+413>F~Kd^XL;)8*7 zzIMwBU!zvSVo9dewjxiO*MI+U&i7MMC~SO$q>;CiiT(S(&5T1RRB-(Jzm4FSyKd;# z2&46YTvH684(nw5I>HCf9g6H$sQN~PdAVkJ&F?rM6)ojz4~MJol0%->CqW!TX zP3MZ0#VyPIYn~32N0sa%I9dCNRZxQqqi`i1^Ngt6s-Unjl-xXy&dQ6c#n+T%hgM`gdMDqM0Nb8wVL0rIH z^N~Vjat|S543RuW$n|M^V0p=q^8ar9fW~-S2FKP-m-&(;sJ~;lBr=>u-X9YXz|++ zUROCtjh8tsgKWeIr?|4kD(nY}eM`h~;)z)m#K^kI1l%nYgUGQ6S(&6U#0nmM6pKKJk|`Jb5my4Nh2B0{fVTN?qi^*e(2b2v1j$!=@Z3fYP1?HeRG->`{!K5W^|~ zi}gd9-s8?f`sIXA*tCFmi1*>C>0>tSD8$v*^)N8;Hp}%1_`ycWP%~$3#3pSG01_ur zGlefu>9Fdla(%U1I@IH>NvRE1afUTY1RRDn?4icYDREo0ZXi;0xG59dk`!fDwK~e^ z_7xdXF@ZoeG6d)ll`(7g^BzQXq*j+O69_W{kUABI4MnPm6&@aHOxhTN1D}jd#Im}z zbU*?>%J9Pu6nrJf(kLDpryks)W<=JORffIgc5!>OC>Tf_W4M#3%w9v9rw9iloS(ie zfes-+K9LegfMaeIWx7ov;BC4O5$%=)2eAq9@HOMixL1kmq zwb>%92EuBfSRFu%5&iC>EXrY4i{=5+!J1=7+0IKJ?G%X`m!Iei=3OI-EW z9E;^_7O;d3(QW27Y*67-s@W01%Azb*QMq{t;dqs&G;VgsmCp{nkcn%r%IbL#^fu-KVJ;X48Rr$WUW)7~86Di=BWC;vF8HWDv&L!D z4FFyCjW&UsNd*$Y0AgHfZ(g%GOZ^TusQgEfl*x0M*2$5v2rC0ti_DJY?>bDxx~=@CY83}d3FKq`#}!sq~6C77HN!|Ql&ixLD+wKs(DF4`>h zpXd%SmJxA*fE5;&(dxU5J3H8K*274SHGqIN0%2ni3{F#rhLF3b?|*SXta2HKO)q%J zK27P^GX@O=XdweM@!Jv$SqY?-K}a?G4atXI;7pb7r;P(k1rm6p(gr)rZ9f2v@$kbc%^wG!S0&gvMbNhb zf-(6?_W%T%ewoIZs-v0GaUhKZ?0KuRYPm_uI+RnDV~{ag4#aq6Z8TVT!&0(}&pPic z$8(rO09Km`(lLxM56?6|rLhI8tVVf)21ZM0MNO&QoqQZcf;D8^DI`G*A%mD}WE}~^zJm+qWR^O>RX*=My1I4xz=E3RYX~STBkT#RZFax@eAb*J6 zF6;i2I(6#OyoTvdG8k}!L+H2%#~3zjbPoV{j;h^Y+*sueg}f_VamQqA5I{y5F;)d( zw;+qvaoHHAwhmb}GfAW?F$}odEzJr5z$k8(IdO-X@Mv0z13Q!l#cJvwZlVE^>r4Uf*dkp`|{ari(Am?3>mAaaVFYRKTF6gJtXkE;*$YvV*&D#Va+PRO0U(yB^7#ad%Q;@bL*H|?a znpmK^jJDe{*_t*msRQa6xZedQR}yd~)O{JkoW#}m zG>>Sth?v4*oK_pS`#J&#w2Oo97E(w=4qp8A{nh$!W^C-`O*iZnit}S)lTc2)@SVY{w z8vhL8DAP@S(s3-9_nMn2n3k`>40y(!D3gAlhLVwOL|?yn!1|Uv3d;msw}54y=A-;5=Cc(GIJq z%@Qf(4cu`0vWiH%tU6;IWwov*uYCk9zlq{7aypY+? zOe-R1K^iC)CV?)s0c$2;M1?o+*SEnx)?a>E>={nnhd79=c@mYi$4F?%8Kvn+n^2j%_n3&i2Hd_GC{h~55qrFzU^X$|97@<~`|7kNR;K_5D>Ugg@R4dE9H^*9 zJ=vudjpG*3(##y)=tz?fokRx*xMU!}jG@nP#Gy`P?NNp9n+c3sX!yu*)7~qgA&?r3 z?{3?>=PJyi-CZ2Hxg*68zr3;s{LDI*WY+?rSP}v!h4%KDTR`o|3(TElC-7oXh=4TW z1)L)LoW}J-Xg1To?-duU{91krJ}@o3;! zy*-;a53{1!y8+@th>8EUhZW}mS{y?LbE#EYC(fI@QA+IX#tVCJ(x?TxqwhU805o<$ zItX#87)}a=cV!xHU7763+qG>vcu@=O2jnP5f!nVI;YdrqsA9MW)B#pjqe!aq-VF&8 zpc4SgdwV2cWemw1?6uQD9oTB8L4Cq-2O6*=z4bOypK<**WaCnP-3@yuEJyqM-W2Ze z3yHUI16Yh9VmHI+P(OTg0wtwZl!-2^p|P6*8MQ#ily02Cfe_zj2y>CJDQ4F_aA6D~ z8Xf_7A00Bc78goj#~OZE10q9XXq3LXNBgQT-W>pNY~gnb%SQV>Q#q~435Eqh5-Ef!KXRVllEv#U?T!`ST-dTqh$hN+LDRT z+ghf%z;(H}@<+vP;(|PgPf@yi(NVz9jOABuPJpUF!D53SR|mVJMloNq$)*ju=j6^E z8xxWNGT3i7Y+!K&nP;2=1lW<~Q7Y6VYq&$v2+XMKA{-2TJOW%1SFkIouJG%k364DO zfeh#2id_oY#Rbvhm9as_kfwWx6}@#GRcgd=F_XNq#+dE{VRr=KAJ4VN=T=I2y!lMl z?pynqW?49NPJjuWDhL48YipPwBu@<(K?g0DT-ZZ{(1d7o44qh>Rh4diHdPQYD%bWf#p*4vK7v%)yZiM}IDQ4Y!Xogp znr4k!+0{!(ERegu$*p(T zAVtNl-J4oW$hwhLeLY0ggBwN^L3H@b9v9gfL-aN=oOF1HLOAJ=zqqykSpH_S>5F~S z3hazPsUx@fq?&f)*OC1!QXP#>Ac^&#c#5-IWziPJZE39(YnVce;z^DnwpnA}!fCtU zzJBN;A@R7CZX4DRCvp#Ci_E{hGa2rkCx%2WW*RD9@Zj1970VTx= zJMBl8SS--jSvBEBZAoK2tR^@?hmGGhZ>fyym8D5uhgFqfPv3PxTSuI>VYHHL(95V_ zD2k!V5R3)UH| z$`bYA?yhi~q?L#pQJn2;E}bBQ*yv=efSYU{3Rj#UgVUbuuM0A?WdR9KQ`r@Fsa(C< z3oZ0k!HW!zQA9yR+*(@#A_B36#u!F}i<@!n?9~boWV>#Q|gG+66tz1ssDdi$WtY6sOYxOnrgxiQB+~(2=E{Ivf-4|88Weo*C z#BFK$RJW5|zza!h{NB&?gn?B7*5xU78D5nT9YZE^HLD zFKLXzu>0zXxFt;^#cGNZZZ2sJU51`JdK5c7U>VHL25X**-H*;_8&MpnZ7!X_=Bd~~ zX$g1IT2mFgamlIirmJJFt51cS5RAslEh&iO5F3w&2SU?W!&D*$9zXh^0k0B-&;qi{ z5V3&jI!Y{{jSLhENZ$(96ixOROWA|SjyP`2ZYQ~5mZGDKvRcM-A7xX`$%#Cw?d)gs zpo$xv%;RU2`LCtYKC5y9Va7F9{2_K)WhG8vi(c_akGb8`=*4l_ezJpuT-wdUrnNSU zV-s3}+>2%@fCYH3H>3sJXSQ1)9eYI%@gC1zple&0Ols9giSrid$_5?8nuN0!xYa>; zO>{5}iRn^90!%3+Vq;p@i0XNO6dlu4A&L`@mewzzkXWg&meumeDBUf3Y(iBikIlW2#Pi3MCNAQ*knx>F2koO;^m9ciL*YB^v%1Y38GA&pn- zP`D9tenAkke>q0}A(s+(lIq4}T-f05uw%z~UAYlplI< zJEsON`|%hD<5a2Q@~H7Hw9cu4rZ9|MDE;$t=ZDfe09OyiDymuc%w9khhq~2CMPd+Zu3@MGZ8~-cvC`%P)t{7U+dc>Fh++YEp)5+cyS1((FzZ zgSX;Rb0)!nm(5{#{{D4zh{sn+a`ve?EpN~p?HQM@dr4@UqfXProh9Z|E-m(%m=C)Y zWk@=#ywmDgwe%?ar72<5Gwnf_q9i#uC%<0q?nciAZlBE?A@XH=O*AQ^) zwqPAzgC5xC(YJi&dti;v`OtBr%P7l=XX-M=A@=@8TM9jWD~%G0QPfKg#}}&%ol_d1 zO6q0B0f+18G5h(H7Kh5)+l~VMmZ(Wny%g#^J*=X8_P&-yRfQF6a*Q8%(D$*Y0!O*K ziBtUqg#juIIMBYQa7@ZmlpPYKQ!?g{lu^1Nb}Oae5EYbEIv^TQhBI-OSrS#X?d~>% zl2uCyHqO+9A@qg=bOTf{u9DzOLGRos(=L?ynX`By! z`9S6f93aEhgT=NCBWj@lfr|?-?0AH*e%FQ(esRd5dcPY#L(J$vqk@lGic2piKs+Eo zeK+Khyb+y+%Ll*U)Xd(7-s#8lc24ExHhH@!;5G+9wvM%iHH2wdy(c<4c>k6TC`edW&JjAC^>r@b5sL+UYIO4BOW&A#YFF|@ zPVLywSrgXb&26KuDjed5Ee{@Ob7=S4yreWufw5SUx8#>_P>0h3HqO_C`=~UX_Exis zYQ?K9WtW;OEVL7FJL9lK;8yA;9~GI1^D0c)gR9GkbvsUjPTI18$H#?FLLMJ@bzzUg zI}Lptp96pV9J;t>i~Iqhu!Ovc%JB&T zEo79W2=cJcWfEjUVF`H)i)G?6&F>2@;EFu$W0aTEoKSJ~UD-$V1HE`y=hew>pRwLX z>q0*|q&Z)m>7qAtrHbQnzMi@vKR^+^omd+R!S;oD+z5Rt6FeJxz$>1^&MlsK*fC*q z%r%~Gz&VMn@7>o!x>?d0^&vlEmztM&XZ*Yc$hB(8|M%eYTxK|msdRv(q^|7Tzt%1`xXaF0$Wy*_tK1d^m z6E+Xg_&_~+s7mt)8;!Ox+?ZK3EBOclnKoeMeWZPv+$Uw3#357{Nd?0oFXJ%Lj>XLW4r8IQCsui(1sw+7BW4|$6yKG&c+O=gFj4(g2cf|&v;S@upu{GP?r#nh}I>#8S{$s0_x-CojlrzI@?X{B0xF$es@#DR7FddK7{E7vk=3$_+)d6te`p*37I`!X$9 z3p!Q7H)(br@cgeUjfR#hO|o_2G$M^h3Oe?3uVn@izAifu2n?7@ain<&!P3(E6qUvt-PXZxLVh?*DH3fvn zd^iZqz=c(sKf*_GONTVXL;G^r8&;;9N<>8+e9j?5fXdmoY#VUpgCMqLw0~gz!yzOJ zgN@{>BT45uw2~~L3MnhR53jJ`=H&>C3C)A%xo+%)q=t`XA4W&Y5)hXPgsnkfM|+TL z3dLrBz4GMPZmm~P7axbSRir>rc?5Y)P+#w5+~F!Wdv!C@a+j5`+!n zuTkv>mI+Fn?6(_4u&O~%~6FH|)-#vO>^gw?UvNA4X6oSpv1pt2GEIVT+?NVTMvYzCK6UqiG7Oi=)YI z142=5M(|Qbb{s%P8>^_$Z0}pTe>4mTb}dp?8=G9I)@r-%gtM|M_ZkyWVIMb>GoN|b2h7?bWzD`u?r?=9Z*<{rH>lFK z4N!|C1T+mFt8)bU$n=K^Xh3cDc1PH0pb1eC*lM)ls)4!L8&9*-fb}Wmrs?Pi&9Yc! zt>YGOy$`e>aowQW%?~-OKN^G`W3faa71FGv?Vv+XP(*eS;}4V?-bWBbeej!#QC(0~ zKtcn<4t>1as_@3jWWzg{3z>u#5NMF{G7v{IQaqDUs6vXL%}7#e@MC4pb6*TU7*ZKO ztC2$(#oCXuF$xs_)5JA)glLGpp}{BLjIh{S&0TZ=3#?r=j$q}x6$luk=z9@5MzJyp zQQ!JgtB=~zU}N{)A5uUz*E0&=1)E0{fby7%o)3V+r(f{{R34#mtnhJ2r99+yx=WV% zAshWj*p9`({eUYDQ1OWIr~h>n(@gD%;pb>q9>p{&9yMO=;Kz{waOI;Qo_0q9@U+t! zXWukFnu43PX^Z4I1QJ`d#rQH6C1n^NDG9g;-=4`(-bY_o%qU;U^YnLw6T`9bKPSA%Il12THESR2)>yQ7t(JuYb>X&y_3rPh!7I2U? zy9GlD*pXt$7Z5OQ5iRoMc_C#RV`;uA6WE>^WjbrhZ8G=!LsT!)3V}l#_)sm1`-Q6I zmUYEI+ofw6l07&Wa-EM$(UmQepw{-X1UlUQO@k5+gx@5 zTNtXw=<@?Bh=vi3(RL1D^mP%HR5Vmp;1M%Z33`+Gf46wJV|D(0%Dufiq*=VWOY^7{ zb9su<_FZ8xVdn5D*(UWu+-#{UU`KFOWd?oXcH;p)5(FQfvv-(dz5b+Xt@a6DLeo=#iCtvB(NWDSL`H#ct1D64EQ4atcNiC;KwlG$D=@E5?k?Lk(rn%C=ClRf~rx zk26*h8oP-&Sl(z5a*Vv7QMF(X7^=cPh|bVG1TtYsby;M?2GyKK!;opgGwIA_$y1Vr z^>Z<7OJ=k1aDkZ@pwsB3T&97`#;RfCoy?s&iVVY?GOH^VP`NIeRcNewdWa5HE$gSY z<|)9- zqi6Q+V9c3OQ~mr#tB`epSr{k107%fz@G8C2;;+UkE@MB@y#vn>9gL&;vm|< zVs#E=Tj07Ro(;Z1_-u6{K?R>yq$2#c|NGw{6aa7Ai^wUJ8I_H6Q%R=c`L2>Y2IiSW z1#mCF{PN5D%iDzz83f6N!+-nX@3zQPks&zjwv}n%Ym>(!47 z%4da1iA6*kEnSiI6%61LnHMs%0p#wLVmpO-QHS>sVPq6mU!(E&H_oXYhfJ7D(dY8k)w80#;K-F}zv-!A-IH6IKlb z&5w%0tWc~UzBndMwk51YlIYUV6}ID~02G*l z<~hyd;<d(56YL??33Qny*v5(1mWPVQ*%6 z$ib|I{X-r`?ZtB@1g)He;WC2i3yk$jn|*j$&9J`w+_zaJ>?4xauS3>E9?htUkS-Gl zz=z#S_E?i|5T`19`a549)*kNCjxhZlhmBhutaGxd8PhSr@k= zVX?*pMAaR$8L-?PwL#;myyGZNIM{}20dZ+X5iH6P@En0u?byT$<=D~yF>-VXO{0X{ zFjFWtWvWf;mO)da8ZC-_l|tIDp&j-{w+3A9jc*YUh{`K8*bI1u`i4r&@OV=NZL?be z+zQf)#`O&in2qsGVe}X( zLeMO?d@cjZExuiWwB|mh6q<&GtdaMIlKOCYmH^ZajWa3x%EKbhUcmc^f$B2aPYyR0bmK4EBxg;tq_f3Cr1Cy4{A4-n8ppAcOnVB6 zdY{nnlmI1uqKgGw{KalK)KDsR1ET+p*FTUAH)_f`%i>+OD^2NU6ONa4)?gp;!;DV@_U;L#hH{Q@zii{pRmVUsgM|he40>f5O zcZ~uLM6CENviRRwf#ap@Knl^YHFr0pgI@$v;JQq(ZCrC`SJ1917wInACY98WAH4L= zTD{1B$^T;OyHVHcfpv6ZcG(ybN?TH&(){qU`W$UoIhHU7d86EwhAP%pyDMIj3Jy!J zvs7nH=2=mmwo!RE;2PfHAB}}gw0TIhr^$8gU`~@6yeW={yMzng;L({Z%4)<^A*vva zC6K=SaHWXf;B7CyM`zyEv_^fRP(SW87`p*rfbuycfVYZ;r`2p9}HrHk_B`a#1mqE#B1OZw2tMCDS z)0?2Ns@pZ(HOTh@8*Fh6KhYnfdw96sU77a!qdjX)Mx#kMW{>OI-dtZU=(o=5w{+ws zh8#vaApH~jPow#dDA{5#)H;2_kgc0^j7P1A$u?yUD03k6{{vGOlqx3H&FEZU+2QE` zF(Y`4?jNIJ!CNT24(-T0fC}v-TX8^+`Ua~U6Zg=&0q^;o4JN8dWBLGlVE>ky@7;$GV1_{Ces4CE+6I9CqB%?2apVCdPBoH9CC12j{tOl?KK7r#P!NBpux{L zS^RbV%g4W&&4&!;{)ah01f*%r{$Kp?AB)Q}f$_7)vdUdTU7}&~SRT1HyhmEm9#M@lY?+};d@_+mfb7l0L-09{9dp1QjqgAQr z_vDV|0Ck+|LIHGb@fZLGLtMju(I0ay^NraUvBT~ft$_4@vHzszx+pUHVLZx>qo%Yw zpT@gsviop%$m>H}vlt`5CL1n_!t}O5zRV7H&w|iOFQkDY#s;v7hdD7FK9exbkIU3B zc>&2CX!OCVX`MF`1?d{^ zE#$p6T-FTHjsD~R&G<39hz}b+>^yK9nfTEj#>dARjjAU}P=4aOIzRK& zzqp@1=MO=iT_?@M+8tL2VlHh04xyq0hEKa)!n9ijTe3uZ%1}z`)hHVEia(_+DTO1k zVa8kB(pr-Hy2q@@jLpo}u)@DvYy5koa`kshxa!KNU6e;#oQSbDsTWr!?V9_G=KcsI z%h$UnpqL$cbsM>6z}Cu4Gi*yn__#VWVPQGD9H$-9q1h>8C7_t)DP}{5JuD_p>%nb+L=%U$k1(*Fl6U}^+j+A!mfyu%+gj{h}#p1 z3rzfr+fsuu!y}6nnm*;wQ{PwOVYg$@+7fFvZ9J@jn2p8a4hC_fyA*~aR}qfzutqPg zZWuE=B+EYnG-AZlEC95w-0Zim$ov;I$-+A9l*XePDmT-O9B&fIYYU^Ay@u98{1uWt zHH}!)IFC#A<+ria7vbye`PHbQ*4(U^B6YB;lP3tU$Gfc-HkJ>5uNB1)1+4PIzNG^k zMM^BauNORF%p&8<#S~;0^?h2Wtk?4`GS>KtT=UbxSo4#=Esc`@t4;E#d6tG9WooCC zE+9Ih912h64WDuh01s2)U{^%UKyx=-+`?&Cp({6mtnQ*beoo{1fz~_Pyj)`n2+=!{ zz7Xj)a9|Zb9B6_oQq#M#7u&#B2j7O#++tr9e#-{eHY@J<)P^rnIu*Rrhj$im$BC?T z&lmAWxY5uNPe+=+wAXwDof2(OG?PBQ%xE<54?j*1IrGxKbOh7B)Gwn(9np#U@3a(ikrp{yxSThRp zRk>mT*@e1*Y!m`*&X1G5&0v{PumgdmCV9-NO!htb4_Jik0Haw};tZr&Vq$`VIUK<7 zWsA6K(GQcVVGx>{(E%Zh(xMD3r1iwu)-i~(oz)8`OP&goC0n9oIUPW@`eYLb?dutY z+t2Qe*_+&+8VAIVB-amam0d@47#QSm08_(}X}D*_3y>5a1SNCX1FO97R0A-EZSyns zp31$o;YNhF+wpqw+aZH<4SZ@Uxh(K8v<_ z0RvIiP|PZtX9ah(%C$jMeReWi;pCfJU^6uwrWA+DQGTrd+YWi}0Nz%w*&|yGc5W7T zObKigJLG+R&dv>L8*ph*ZU#rD6`8$iUl-jA?vOv`Ox)EME@sBbGmK*BMX#_D;C)|y zHDaX7?DdPdmA9YP!&-5kr?#6J(h4avL-oM4kUY;K+Om$?7Dhn$v20^j&60fHLL@IP zDS6obP&6qgK|apb32885oS$iDIreIj zgUuX+L5z>y{IX`v50N%(mLhD0BEpun-HsfePH2bg8n)aGse!?-#@8&3$#V*XHgyT9 zy|UzmJyR3F2zgO+a&3NKluL{GT01SgYW@FvJGbSwkt7Y@>nRZ7xo}4eY~e_%V`C>e z-0q#;neOqnJ<}6=Q$drE#I;DOd62BTeT{v+eUe3;6F^nshp6ovS_ZQ6OM+E66zb1( zd7VfKy4d?w!10t0#CoyQzC9($6n945@~C7f&yT9(hRjCG>)E4~ymS-zO$v0kU*D|N zL(JjXQBogfuF-6f2`G;1?PB~q(0ADysG4}Q8cU39`>D#w78%|^qi7st{@bP^3_V%+ zeeNeT|GyZff*Lj-B#P?hhGJLj@~d<2*4HWv)WYT!8#)-6*ae%{D(%xk<@aga{gf$d zB5v0QSvzj%uV{NdZ*!3eY8AvuBtaS!+yJ=StfQz zAPA}3Yq2c`XQSbGMFpGwHqL z^1Y4xns@8=_RYnb&`RF}l_jTi43}%8^vuVGHw6rqG6p3>H8Km@f{QKSC4bH5}yCTvv| z5EoXpKlC3qcC)l(1-9FH0g7l5Vn}xzlhlF&Sr*G^YGZQQDY{pdJ*JmsRj>nZUQHe2 zsZaBcMgH0Gv0q$Y%V~^u-1TO#;#N+EAU7svBk?NKm=o@N7(GS%bLlI;HC0E$P6@~6Ieyt-oY&`sqF#Kz{4eD)I=Aj8{%q^5dbJ~()!_wW zGG0^Zl?_Fyn$84kA1p?{?9=CRU3%v%B5{{0cQAbS+SXKY@V7cP@Z_?aB5;Y zO!??w29NC_?}G<-3h8(leV`Z1e&+7LBYnqxYd`XYFH~lF{m@Px^+SZt!#>5Rto7PO zs$Fn10b)245TXs|0w}WGn|dG6B*CDnR*2evn5RtIKKzVN5yI9YKbPf-nkT=SKPYkR z^hD)8Ii$f#d-}N%IscrW4joSY~CfYSo>4IXq zXx2f$QqjGQCb3^AngRGirp%Jdk4-9r zO>Ad(q8?-t~6#YWbiNsNObRH$M_Lv@}V_ z56wn6EK8zoemkw(XC(di;H5#tgjPw>ZTyVZut~Rq0jW>mt1s|Oz(HV^_gmMiuJ^l5 z(y7k?N9l_sJSW6nwFWjmZ#x>C8oSFW0qGXOAk@Wh8DI_ywKGR%Ov^}2 zQMYy^ei?$$mKy#q+g);8AMcufy$F}d;Q*wNj;)sytbxP%@I`XERd7NTZdgCI0#2vY zO{k0w>&2G2>D05~YSeJv2DNOUVl~gGJ;Me8maYNI(oC%~Gd;BaIS;5l6D=yk(Yuvr zCRCRhDHp>rLzQJ>bz_)fFf5|K;E7g+3Dwefy}~a{H5b-lujosstV^h53+v35 zbkVh2`nw}npq_y_4cm2NST!J;wOR1CM?VpKA|N}g4IhnC%(K%w~Pr}nS^sV zEMXEUQ4%XX!Z{CSs$rsbbwDEiTon+w+()p42aD(>yNQNMi*9iK`UO1V6*>lWH{^XE z=UW-r8&YYj3c~B%GV}bpy^5&$+IU3|)_u3`N2s(%pcn(^@#xlb@C7{Bnj9m5Uh$5R z;vA7`5@QToaALgtMxbH@=iC;Q;7dnD>OpXhX~~E|QHW@*2i9Y!AOtP?FtA*2i(G*B zLIY#tzCE(L$3ya2jd||=h!mKJAV_(MeieyOg$UM_tstRGMUd&LKr1gR3eI5(GBebh z03&lluhbS(R1BCX61H4I+eMQfq(rfkeiGf z^32kf^RedHPjg3Tf#n8gAoZdHwXV*~t-nnODjH7UjmfhD(S8OYsLcU5cFeo8RROkf zkQ zGY3&H5%;E5CWqQD{u_^zkfzKzhodH?i&uyL_sB^FqYpi#@Slvn? zEZ8&q@UudGW*@VA@G&`O>mU%ufH{!I`QMekiuAPn+WssEy=!?f8S&i>xckxL$-ue)kqc=XcS zx+m(zbK>xCzIt!7I=s%?EC$zkpB11vKcQD$L87|5Xn&EzOQBQe&?&98-q_`&lTY8) zh>L6!DLDG9sFbtz@jTg_j;Dp50(dRo2z?`U!Oix1Cs)8P@77WcN>mfj!8Pil)yBR{HuNbI7b@pm)c$T<#-7)wM$p$>L}xyEctX zXTMmjWX69T_s5@q9EtO#kF6{)YFwJkVDext{1XBcn9)rQR$-kH7;*Uz{bxhN`rrTl zpF`*OO&;4G>MDBK`H*h7ejNgQp(861LeXlE%o2q_01*k(OGM&jP0Y=wi6XMd{6)g> zzPTGq6UcdYB2YZfx$`hu|FbWC*3UboD12&p<_^T{FlaY1$Jc!wOx^f)y;q8~T(+Bz zk5|fe6Y_n(dJzXQP2VSq_T09XKhomvJa|eRedqt|BvTGY(4I`rWh)o*BZ7!ePELUI zCM!D=lwqJ+|G7IfF9J@jGaG=3$83%S61_Q`z}G(B-N}SZ{^nf1m_4jjHjOa)W_>&p zEIrt7`)})55B{&;&cw71{_D4h^KEl%FI{H>j1&D&1SH>-+C;R^Cs{A%U&9KNmS*J8 z#8p1e~E?5f1n9BWJWYyD`&SwmqkYtd{Ad} zVYDnQU1wMm$QE^}Ap{5zdP{;x04bq_qLe`BNM9+^1Pm-45m1z3h%rb0u_uVry=iW0vrrlYSU75J~o^|S+@IX)fx9}Uak{|~s zaN{GoZ#B2y!H}Ly=stXdhUGR=QuZSy_<{cb*UW8!5|uuEIm%Q3B=gS8e?(>di4Ry{VIq zUG`xqNp-7snHlx3{xH}pf2sBO^{Tf#^>Y^IwtajWZ%QuD zZr0`w+Ya65BKuy;!O^}_rQ;i?g@o-#61vU3gX{(2bG^(_1SPM??31HwyV<@X;Gbez zv=r#eot%P@Dbx1T3ShG|!>ddK_dmv8l8k;?H>hn^qu&Sa9hBNH*hIxG6_#?kpsvD% zM8KPSqtOI>R_0~Ac&_Hp7Xut+x`*?cmJuGWF)FP0aUJUUOp@XDaG zuNu^A9!B78OY!%>@a=y>kC>MI#WT5A{M#4{|@dbsC|H^Sp12KRGZa65r+uO zcTYJepN^1~Ex4sqZlKM&0P(kZnQEDB&u1~YVlFb0aA0pN{Gj#Ki}Ylq;WPcdLou~^ zr@-Oq>z0^uq#0)<3GPMQ6=jKj(%oP4T<*w_WC1la}5s7+P2|ybpP`_ffGM|;n#xNhL4w?y3`yM3#OhHkho{9Riqm(q z=5SZbRX_eA%{TGY6`vgO9vWkwZL59}nN1-&wHOl@jkk<)<$wu-K7_dC-9U9DTaVewD_4qL`Y#=v4-& ze;7e&>DJHlZ%ohY2^PRE`MrNk6y z|04Fb!Bt2WKfr~aJp=~c+Tt`U{h@FbZ%@k3vbDHoHqk3!Q3)s`88LNH5l_zKOmQVK zT^_XUO;N#8WGIaVWTG~DaCVzVWOC7u$IndIsja*PK&BS^u=so(k(8kaAAVl|xp&LN z;zuU(#izB5WGsMWz{`qm>UnXPS{&?3O36~O@IO28Fig1$KG`Rsx9z#Dt@IQRLVB)olZ@%zu}=L{E1b zS2Yh#cu`E0-s>>z_JLUGXGc$^+WODrGwz%#Z}cf;PWAU_C!Z5d9m|{4+P2`&qJrq2lPR6lEsK(xAO zg+|?Nk4Uyt({)y3AZg#CPb1ZK|5zDh2urijh&~%e3){HxTpt_c#3QXIE-|?t))Zc# z^5J5M#5d1Vd2$E@0D@`hGi+II17D1H!BAz1$7y$>1g^s>e!Waku>l)PvJ46PvsrGl zj4q)k3Oc_&zitCAlt^BcUp1U?lk&DokR@(#f5w6|-^$#;`L9DmhQl^F;JRq)4RDj! z8&+Of+z|0 z#)8zC@8EC<7MTCAL4=hKf`|os3xeuS_26+80GR)Pfu)rPf>;Cgz6h$zgKI%B{{S)r z32uvg789o67kYt6g5HNbCGC+bZ8$O&I7b4^|tPNy*w*|Ms@I%5u zC^Qj4Wa4F_gS7g}l5qZfu#nC$PRD8l(Sn!s5Y*63OU5k;!mvFm3eFS+aVMiEYWyzj zyH&KWRW^bs%KJM$`A`Q4rWkNLf~|nuWsHU-gB!dmSkGL6VBP|dmIRls&nD<=#Iuc0 zF`&!zW>XlGF*-X5tl?83BJTphJO$WA5c0@y6EyYzj)4%&$bXK8rf5hSxDoq&o8=oB zDsL<}GcHGX_ z!kzJNSecbniWjUfPAMOLFzNn-Gg8AN$Yq{D7o^bQw6?hEYFeT<8sh}}qpggg=WF~q@ z$*%HoiKsbfFm4uf(FO?5$G_2jJA~FPgBn@@TvaGcj?O^%Gkl2*vm$z^4C-wOa79vp z7cT?h9r))OiO*!0R9Sg_4L^x67cE#^K%dS7M-PY5`Rns>L=^0&0CT(HcmWQe>=NJ< zBSKofBTj@nfaT8o5pAG4E3cp7Z_zddIDlWD=~fgDxxJ#r${S`lleO@$>7@hMT((_%Vr<4vG=vF`tZ2>xdB@vD=d7?Um3Uur2c zmp$qVl+pGwRTMe3JkFOvu)jVkZTh`mk{NS^6@wT7l@nQVTnch8Z7cCmqo1VcZv=G`#7>7PL_3U=&*R zdk`*9?~L?vwzVYhk94`j+!+Z44jGyr(HC0FyJWfX*bLHfa+o1{!66xK(` zPxI`O^4k?B@&#_@d)?|3iR;vO32SF*$qb>D)9)<%Z}3_j!6h?x>8CX;`y)jLKQbDx zXs2^6`XdDfpEDZK3#Sjv`y(Z+lHhL5e4hk0zQza+dc)ng!Jn||Ut=V#xZrLdxIb~L zeBHN@-MsE5{^-@y+0<&d(#}w1U1;Mw!qdc%OHKOfUB}7~KFp2QVmEtyBBEv6gQC+f zy@;7oLamfdsQ2{*gu8{wFy1rWHD2-Ce)vx)lRuaJl?>5D>R)leyE5^a2U$P1+i9|O zKR&PTm{;?VCiT7-b(c=o_zFb1(9S>FUdWGw?X2{KhB^(=$A&Hh?nc$WNLt;IbCawWQJ>YS*_Uzj$>k#qi%*f&!O@O$!?;0;whJ*dS8@+pS_>e(^%8Fy}Bkg zSw2OHnx++h<7qis^k)2o_U-P`T2tVnu6TZr{G_|=eBRfV*p25Ns7i5NPNAc(=HH{s zeWV*($W3#FjS8O zg0xpHNBetQUp->EvRVnqv7|sP{n~N;`D@PawgX0dT6;1KT-ghMtgYphiEjp`m&jNJ8_M|xKNQQ(5#D(o( zR&9P2mA?boK3<8nWH5f*JTX@wsCi*yfA*jw?tab__J{3L!GbG_f~OkBk+*_&R$Y`c z=c%8a^I9J^e9Sx;TTg(obLq^xd+cRNje7{TwDR?y^*ymzL6dqvBe8Et&G?@Uh$5I|F{vldto!!Fo@4DVs@@B_K0z$ zxcTmGG+*Pm&w~FmfW~-)1lsWUh$K=xOFklyQBSx2sYVmYrVV< z8YQIVoAMKViRPuRrK#VjdMWiCbEUCv!Nu_6t@)=W(fNLoo-0~1X=6o;yS-;%DD#g` z#GbXtHU$?)7m4I-X9h^c&*?V0x0*kUUP0y1!sgF@iKJzmpHoT;fk@I|L4AY&bfjWV z7pv#hZdg_ir&#J`JKJo)T#pkv=1|kVqsPRKLeYk|#{~nsEvTmA>GB7=FU?OkjU8E} zO>cfywRS{ooSo9U(MVPs&6obPlN&0%sGlCx)}$Pax>maC^4g_;=rcCHqyFu;;}pit zq9^ZtrndJ+*Qn*?!^0t7eT$pcxXPftZ)2V Date: Tue, 2 May 2023 08:36:05 -0400 Subject: [PATCH 071/740] mobile: improve compile time workflow (#27085) turning up signal tracing, and consistently using macos CI settings Signed-off-by: Alyssa Wilk --- .github/workflows/mobile-compile_time_options.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index 309cb69b4cea..b9ee5e056dda 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -36,6 +36,7 @@ jobs: run: | cd mobile && ./bazelw test \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-linux") \ + --define=signal_trace=disabled \ --define=admin_html=enabled \ --define=envoy_mobile_request_compression=disabled \ --define=envoy_enable_http_datagrams=disabled \ @@ -67,6 +68,7 @@ jobs: ./bazelw build \ --config=ios \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ + --define=signal_trace=disabled \ --define=admin_html=enabled \ --define=envoy_mobile_request_compression=disabled \ --define=envoy_mobile_stats_reporting=disabled \ @@ -107,6 +109,7 @@ jobs: && ./bazelw build \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --fat_apk_cpu=x86_64 \ + --define=signal_trace=disabled \ --define=admin_html=enabled \ --define=envoy_mobile_request_compression=disabled \ --define=envoy_enable_http_datagrams=disabled \ @@ -115,8 +118,10 @@ jobs: --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ //:android_dist \ && ./bazelw test \ + $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --config=ci \ --fat_apk_cpu=x86_64 \ + --define=signal_trace=disabled \ --define=admin_html=enabled \ --define=envoy_mobile_request_compression=disabled \ --define=envoy_enable_http_datagrams=disabled \ From 522f0551adf70a9fa27edf7088de0c6d2d14d63f Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 2 May 2023 09:45:28 -0400 Subject: [PATCH 072/740] config: moving grpc subscriptions to extensions (#26742) Risk Level: low Testing: n/a Docs Changes: inline Release Notes: n/a Signed-off-by: Alyssa Wilk --- CODEOWNERS | 1 + api/envoy/config/core/v3/config_source.proto | 12 -- changelogs/current.yaml | 4 + docs/root/api-docs/xds_protocol.rst | 8 + envoy/config/subscription_factory.h | 38 +++- .../integration/xds_integration_test.cc | 9 + source/common/config/BUILD | 14 -- .../config/subscription_factory_impl.cc | 190 ++++++------------ .../filesystem/filesystem_subscription_impl.h | 38 +--- .../extensions/config_subscription/grpc/BUILD | 77 +++++++ .../grpc_collection_subscription_factory.cc | 63 ++++++ .../grpc_collection_subscription_factory.h | 34 ++++ .../grpc/grpc_subscription_factory.cc | 105 ++++++++++ .../grpc/grpc_subscription_factory.h | 32 +++ .../grpc}/grpc_subscription_impl.cc | 2 +- .../grpc}/grpc_subscription_impl.h | 0 .../rest/http_subscription_impl.h | 24 +-- source/extensions/extensions_build_config.bzl | 6 + source/extensions/extensions_metadata.yaml | 36 +++- test/common/config/BUILD | 8 +- .../config/delta_subscription_test_harness.h | 2 +- .../config/grpc_subscription_test_harness.h | 2 +- test/common/router/BUILD | 1 + test/common/upstream/BUILD | 2 + test/extensions/clusters/eds/BUILD | 2 +- .../extensions/clusters/eds/eds_speed_test.cc | 2 +- .../config_subscription/common/BUILD | 2 + test/integration/BUILD | 4 +- test/per_file_coverage.sh | 2 +- tools/code_format/config.yaml | 1 + 30 files changed, 500 insertions(+), 221 deletions(-) create mode 100644 source/extensions/config_subscription/grpc/BUILD create mode 100644 source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc create mode 100644 source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h create mode 100644 source/extensions/config_subscription/grpc/grpc_subscription_factory.cc create mode 100644 source/extensions/config_subscription/grpc/grpc_subscription_factory.h rename source/{common/config => extensions/config_subscription/grpc}/grpc_subscription_impl.cc (98%) rename source/{common/config => extensions/config_subscription/grpc}/grpc_subscription_impl.h (100%) diff --git a/CODEOWNERS b/CODEOWNERS index e4663e4a2f8a..28d0a8b456fa 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -325,6 +325,7 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/source/extensions/listener_managers/listener_manager @UNOWNED @UNOWNED /*/source/extensions/listener_managers/validation_listener_manager @UNOWNED @UNOWNED /*/source/extensions/config_subscription/ @UNOWNED @UNOWNED +/*/source/extensions/config_subscription/grpc @adisuissa @UNOWNED # URL Pattern Match and Rewrite Library /*/extensions/path/uri_template_lib @alyssawilk @yanjunxiang-google diff --git a/api/envoy/config/core/v3/config_source.proto b/api/envoy/config/core/v3/config_source.proto index d7b68a9a222d..c12930135a1b 100644 --- a/api/envoy/config/core/v3/config_source.proto +++ b/api/envoy/config/core/v3/config_source.proto @@ -281,15 +281,3 @@ message ExtensionConfigSource { // if they do not match any type URL in the set. repeated string type_urls = 4 [(validate.rules).repeated = {min_items: 1}]; } - -// A placeholder proto for the registration for the REST subscription implementation. -message RestSubscription { -} - -// A placeholder proto for the registration for the filesystem subscription implementation. -message FilesystemSubscription { -} - -// A placeholder proto for the registration for the filesystem collection subscription implementation. -message FilesystemCollectionSubscription { -} diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 3cf56cbeb5ed..a0984d789730 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -6,6 +6,10 @@ behavior_changes: change: | Moved the subset and maglev LB code into extensions. If you use these load balancers and override extensions_build_config.bzl you will need to include them explicitly. +- area: build + change: | + Moved xDS code extensions. If you use the xDS and override extensions_build_config.bzl you will + need to include the new config_subscriptions explicitly. minor_behavior_changes: # *Changes that may cause incompatibilities for some users, but should not for most* diff --git a/docs/root/api-docs/xds_protocol.rst b/docs/root/api-docs/xds_protocol.rst index 3d726488937a..f2e1cf8a594b 100644 --- a/docs/root/api-docs/xds_protocol.rst +++ b/docs/root/api-docs/xds_protocol.rst @@ -88,6 +88,9 @@ occurs. .. _xds_protocol_streaming_grpc_subscriptions: +.. _extension_envoy.config_subscription.grpc: +.. _extension_envoy.config_subscription.aggregated_grpc_collection: + Streaming gRPC subscriptions ---------------------------- @@ -771,6 +774,9 @@ This feature is gated by the *xds.config.supports-resource-in-sotw* client featu Aggregated Discovery Service ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _extension_envoy.config_subscription.ads: +.. _extension_envoy.config_subscription.ads_collection: + It's challenging to provide the above guarantees on sequencing to avoid traffic drop when management servers are distributed. ADS allow a single @@ -792,6 +798,8 @@ An example minimal ``bootstrap.yaml`` fragment for ADS configuration is: .. literalinclude:: ../_include/ads.yaml .. _xds_protocol_delta: +.. _extension_envoy.config_subscription.delta_grpc: +.. _extension_envoy.config_subscription.aggregated_delta_grpc_collection: Incremental xDS ~~~~~~~~~~~~~~~ diff --git a/envoy/config/subscription_factory.h b/envoy/config/subscription_factory.h index e3ee0a425908..e2042fa8322b 100644 --- a/envoy/config/subscription_factory.h +++ b/envoy/config/subscription_factory.h @@ -11,7 +11,14 @@ #include "xds/core/v3/resource_locator.pb.h" namespace Envoy { + +namespace Server { +class Instance; +} // namespace Server + namespace Config { +class XdsResourcesDelegate; +class XdsConfigTracker; class SubscriptionFactory { public: @@ -71,17 +78,30 @@ class SubscriptionFactory { }; // A factory class that individual config subscriptions can subclass to be factory-created. -// TODO(alyssawilk) rename once https://github.com/envoyproxy/envoy/pull/26468 lands -class ConfigSubscriptionFactory : public Config::TypedFactory { +class ConfigSubscriptionFactory : public Config::UntypedFactory { public: + struct SubscriptionData { + const LocalInfo::LocalInfo& local_info_; + Event::Dispatcher& dispatcher_; + Upstream::ClusterManager& cm_; + ProtobufMessage::ValidationVisitor& validation_visitor_; + Api::Api& api_; + const Server::Instance& server_; + OptRef xds_resources_delegate_; + OptRef xds_config_tracker_; + + const envoy::config::core::v3::ConfigSource& config_; + absl::string_view type_url_; + Stats::Scope& scope_; + SubscriptionCallbacks& callbacks_; + OpaqueResourceDecoderSharedPtr resource_decoder_; + const SubscriptionOptions& options_; + OptRef collection_locator_; + SubscriptionStats stats_; + }; + std::string category() const override { return "envoy.config_subscription"; } - virtual SubscriptionPtr create(const LocalInfo::LocalInfo& local_info, - Upstream::ClusterManager& cm, Event::Dispatcher& dispatcher, - Api::Api& api, const envoy::config::core::v3::ConfigSource& config, - absl::string_view type_url, SubscriptionCallbacks& callbacks, - OpaqueResourceDecoderSharedPtr resource_decoder, - SubscriptionStats stats, - ProtobufMessage::ValidationVisitor& validation_visitor) PURE; + virtual SubscriptionPtr create(SubscriptionData& data) PURE; }; } // namespace Config diff --git a/mobile/test/common/integration/xds_integration_test.cc b/mobile/test/common/integration/xds_integration_test.cc index 4936d131f108..232bbfe09a88 100644 --- a/mobile/test/common/integration/xds_integration_test.cc +++ b/mobile/test/common/integration/xds_integration_test.cc @@ -4,6 +4,8 @@ #include "envoy/config/cluster/v3/cluster.pb.h" #include "source/common/grpc/google_grpc_creds_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h" +#include "source/extensions/config_subscription/grpc/grpc_subscription_factory.h" #include "test/common/grpc/grpc_client_integration.h" #include "test/common/integration/base_client_integration_test.h" @@ -20,6 +22,13 @@ using ::testing::AssertionSuccess; XdsIntegrationTest::XdsIntegrationTest() : BaseClientIntegrationTest(ipVersion()) { Grpc::forceRegisterDefaultGoogleGrpcCredentialsFactory(); + Config::forceRegisterAdsConfigSubscriptionFactory(); + Config::forceRegisterGrpcConfigSubscriptionFactory(); + Config::forceRegisterDeltaGrpcConfigSubscriptionFactory(); + Config::forceRegisterDeltaGrpcCollectionConfigSubscriptionFactory(); + Config::forceRegisterAggregatedGrpcCollectionConfigSubscriptionFactory(); + Config::forceRegisterAdsCollectionConfigSubscriptionFactory(); + override_builder_config_ = false; expect_dns_ = false; // doesn't use DFP. create_xds_upstream_ = true; diff --git a/source/common/config/BUILD b/source/common/config/BUILD index a24dc4b14d0f..6d62b4c7f491 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -161,19 +161,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "grpc_subscription_lib", - srcs = ["grpc_subscription_impl.cc"], - hdrs = ["grpc_subscription_impl.h"], - deps = [ - ":grpc_mux_lib", - ":xds_resource_lib", - "//envoy/config:subscription_interface", - "//envoy/event:dispatcher_interface", - "//envoy/grpc:async_client_interface", - ], -) - envoy_cc_library( name = "new_grpc_mux_lib", srcs = ["new_grpc_mux_impl.cc"], @@ -280,7 +267,6 @@ envoy_cc_library( hdrs = ["subscription_factory_impl.h"], deps = [ ":custom_config_validators_lib", - ":grpc_subscription_lib", ":new_grpc_mux_lib", ":type_to_endpoint_lib", ":utility_lib", diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 81fa75e3609d..38b744147185 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -4,8 +4,6 @@ #include "envoy/config/xds_resources_delegate.h" #include "source/common/config/custom_config_validators_impl.h" -#include "source/common/config/grpc_mux_impl.h" -#include "source/common/config/grpc_subscription_impl.h" #include "source/common/config/new_grpc_mux_impl.h" #include "source/common/config/type_to_endpoint.h" #include "source/common/config/utility.h" @@ -35,6 +33,23 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( SubscriptionStats stats = Utility::generateStats(scope); std::string subscription_type = ""; + ConfigSubscriptionFactory::SubscriptionData data{local_info_, + dispatcher_, + cm_, + validation_visitor_, + api_, + server_, + xds_resources_delegate_, + xds_config_tracker_, + config, + type_url, + scope, + callbacks, + resource_decoder, + options, + absl::nullopt, + stats}; + switch (config.config_source_specifier_case()) { case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kPath: { Utility::checkFilesystemSubscriptionBackingPath(config.path(), api_); @@ -65,78 +80,12 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( case envoy::config::core::v3::ApiConfigSource::REST: subscription_type = "envoy.config_subscription.rest"; break; - case envoy::config::core::v3::ApiConfigSource::GRPC: { - GrpcMuxSharedPtr mux; - CustomConfigValidatorsPtr custom_config_validators = - std::make_unique(validation_visitor_, server_, - api_config_source.config_validators()); - - JitteredExponentialBackOffStrategyPtr backoff_strategy = - Utility::prepareJitteredExponentialBackOffStrategy( - api_config_source, api_.randomGenerator(), RetryInitialDelayMs, RetryMaxDelayMs); - - const std::string control_plane_id = - Utility::getGrpcControlPlane(api_config_source).value_or(""); - - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { - mux = std::make_shared( - Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), api_config_source, - scope, true) - ->createUncachedRawAsyncClient(), - dispatcher_, sotwGrpcMethod(type_url), scope, - Utility::parseRateLimitSettings(api_config_source), local_info_, - api_config_source.set_node_on_first_message_only(), std::move(custom_config_validators), - std::move(backoff_strategy), xds_config_tracker_, xds_resources_delegate_, - control_plane_id); - } else { - mux = std::make_shared( - local_info_, - Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), api_config_source, - scope, true) - ->createUncachedRawAsyncClient(), - dispatcher_, sotwGrpcMethod(type_url), scope, - Utility::parseRateLimitSettings(api_config_source), - api_config_source.set_node_on_first_message_only(), std::move(custom_config_validators), - std::move(backoff_strategy), xds_config_tracker_, xds_resources_delegate_, - control_plane_id); - } - return std::make_unique( - std::move(mux), callbacks, resource_decoder, stats, type_url, dispatcher_, - Utility::configSourceInitialFetchTimeout(config), - /*is_aggregated*/ false, options); - } - case envoy::config::core::v3::ApiConfigSource::DELTA_GRPC: { - GrpcMuxSharedPtr mux; - CustomConfigValidatorsPtr custom_config_validators = - std::make_unique(validation_visitor_, server_, - api_config_source.config_validators()); - - JitteredExponentialBackOffStrategyPtr backoff_strategy = - Utility::prepareJitteredExponentialBackOffStrategy( - api_config_source, api_.randomGenerator(), RetryInitialDelayMs, RetryMaxDelayMs); - - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { - mux = std::make_shared( - Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), api_config_source, - scope, true) - ->createUncachedRawAsyncClient(), - dispatcher_, deltaGrpcMethod(type_url), scope, - Utility::parseRateLimitSettings(api_config_source), local_info_, - api_config_source.set_node_on_first_message_only(), std::move(custom_config_validators), - std::move(backoff_strategy), xds_config_tracker_); - } else { - mux = std::make_shared( - Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), - api_config_source, scope, true) - ->createUncachedRawAsyncClient(), - dispatcher_, deltaGrpcMethod(type_url), scope, - Utility::parseRateLimitSettings(api_config_source), local_info_, - std::move(custom_config_validators), std::move(backoff_strategy), xds_config_tracker_); - } - return std::make_unique( - std::move(mux), callbacks, resource_decoder, stats, type_url, dispatcher_, - Utility::configSourceInitialFetchTimeout(config), /*is_aggregated*/ false, options); - } + case envoy::config::core::v3::ApiConfigSource::GRPC: + subscription_type = "envoy.config_subscription.grpc"; + break; + case envoy::config::core::v3::ApiConfigSource::DELTA_GRPC: + subscription_type = "envoy.config_subscription.delta_grpc"; + break; } if (subscription_type.empty()) { throw EnvoyException("Invalid API config source API type"); @@ -144,9 +93,8 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( break; } case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kAds: { - return std::make_unique( - cm_.adsMux(), callbacks, resource_decoder, stats, type_url, dispatcher_, - Utility::configSourceInitialFetchTimeout(config), true, options); + subscription_type = "envoy.config_subscription.ads"; + break; } default: throw EnvoyException( @@ -159,8 +107,19 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( "Didn't find a registered config subscription factory implementation for name: '{}'", subscription_type)); } - return factory->create(local_info_, cm_, dispatcher_, api_, config, type_url, callbacks, - resource_decoder, stats, validation_visitor_); + return factory->create(data); +} + +SubscriptionPtr createFromFactoryOrThrow(ConfigSubscriptionFactory::SubscriptionData& data, + absl::string_view subscription_type) { + ConfigSubscriptionFactory* factory = + Registry::FactoryRegistry::getFactory(subscription_type); + if (factory == nullptr) { + throw EnvoyException(fmt::format( + "Didn't find a registered config subscription factory implementation for name: '{}'", + subscription_type)); + } + return factory->create(data); } SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( @@ -169,23 +128,30 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( Stats::Scope& scope, SubscriptionCallbacks& callbacks, OpaqueResourceDecoderSharedPtr resource_decoder) { SubscriptionStats stats = Utility::generateStats(scope); - + SubscriptionOptions options; + envoy::config::core::v3::ConfigSource factory_config = config; + ConfigSubscriptionFactory::SubscriptionData data{local_info_, + dispatcher_, + cm_, + validation_visitor_, + api_, + server_, + xds_resources_delegate_, + xds_config_tracker_, + factory_config, + "", + scope, + callbacks, + resource_decoder, + options, + {collection_locator}, + stats}; switch (collection_locator.scheme()) { case xds::core::v3::ResourceLocator::FILE: { const std::string path = Http::Utility::localPathFromFilePath(collection_locator.id()); Utility::checkFilesystemSubscriptionBackingPath(path, api_); - envoy::config::core::v3::ConfigSource factory_config; factory_config.set_path(path); - const std::string subscription_type = "envoy.config_subscription.filesystem_collection"; - ConfigSubscriptionFactory* factory = - Registry::FactoryRegistry::getFactory(subscription_type); - if (factory == nullptr) { - throw EnvoyException(fmt::format( - "Didn't find a registered config subscription factory implementation for name: '{}'", - subscription_type)); - } - return factory->create(local_info_, cm_, dispatcher_, api_, factory_config, "", callbacks, - resource_decoder, stats, validation_visitor_); + return createFromFactoryOrThrow(data, "envoy.config_subscription.filesystem_collection"); } case xds::core::v3::ResourceLocator::XDSTP: { if (resource_type != collection_locator.resource_type()) { @@ -199,43 +165,20 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( config.api_config_source(); Utility::checkApiConfigSourceSubscriptionBackingCluster(cm_.primaryClusters(), api_config_source); - CustomConfigValidatorsPtr custom_config_validators = - std::make_unique(validation_visitor_, server_, - api_config_source.config_validators()); - - JitteredExponentialBackOffStrategyPtr backoff_strategy = - Utility::prepareJitteredExponentialBackOffStrategy( - api_config_source, api_.randomGenerator(), RetryInitialDelayMs, RetryMaxDelayMs); - - SubscriptionOptions options; // All Envoy collections currently are xDS resource graph roots and require node context // parameters. options.add_xdstp_node_context_params_ = true; switch (api_config_source.api_type()) { case envoy::config::core::v3::ApiConfigSource::DELTA_GRPC: { - const std::string type_url = TypeUtil::descriptorFullNameToTypeUrl(resource_type); - return std::make_unique( - collection_locator, - std::make_shared( - Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), - api_config_source, scope, true) - ->createUncachedRawAsyncClient(), - dispatcher_, deltaGrpcMethod(type_url), scope, - Utility::parseRateLimitSettings(api_config_source), local_info_, - std::move(custom_config_validators), std::move(backoff_strategy), - xds_config_tracker_), - callbacks, resource_decoder, stats, dispatcher_, - Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/false, options); - } - case envoy::config::core::v3::ApiConfigSource::AGGREGATED_GRPC: { - return std::make_unique( - collection_locator, cm_.adsMux(), callbacks, resource_decoder, stats, dispatcher_, - Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/true, options); + std::string type_url = TypeUtil::descriptorFullNameToTypeUrl(resource_type); + data.type_url_ = type_url; + return createFromFactoryOrThrow(data, "envoy.config_subscription.delta_grpc_collection"); } + case envoy::config::core::v3::ApiConfigSource::AGGREGATED_GRPC: + FALLTHRU; case envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC: { - return std::make_unique( - collection_locator, cm_.adsMux(), callbacks, resource_decoder, stats, dispatcher_, - Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/true, options); + return createFromFactoryOrThrow(data, + "envoy.config_subscription.aggregated_grpc_collection"); } default: throw EnvoyException(fmt::format("Unknown xdstp:// transport API type in {}", @@ -244,13 +187,10 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( } case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kAds: { // TODO(adisuissa): verify that the ADS is set up in delta-xDS mode. - SubscriptionOptions options; // All Envoy collections currently are xDS resource graph roots and require node context // parameters. options.add_xdstp_node_context_params_ = true; - return std::make_unique( - collection_locator, cm_.adsMux(), callbacks, resource_decoder, stats, dispatcher_, - Utility::configSourceInitialFetchTimeout(config), true, options); + return createFromFactoryOrThrow(data, "envoy.config_subscription.ads_collection"); } default: throw EnvoyException("Missing or not supported config source specifier in " diff --git a/source/extensions/config_subscription/filesystem/filesystem_subscription_impl.h b/source/extensions/config_subscription/filesystem/filesystem_subscription_impl.h index 13dd9948ea28..258d8e7b028b 100644 --- a/source/extensions/config_subscription/filesystem/filesystem_subscription_impl.h +++ b/source/extensions/config_subscription/filesystem/filesystem_subscription_impl.h @@ -74,48 +74,30 @@ class FilesystemSubscriptionFactory : public ConfigSubscriptionFactory { public: std::string name() const override { return "envoy.config_subscription.filesystem"; } - SubscriptionPtr create(const LocalInfo::LocalInfo&, Upstream::ClusterManager&, - Event::Dispatcher& dispatcher, Api::Api& api, - const envoy::config::core::v3::ConfigSource& config, absl::string_view, - SubscriptionCallbacks& callbacks, - OpaqueResourceDecoderSharedPtr resource_decoder, SubscriptionStats stats, - ProtobufMessage::ValidationVisitor& validation_visitor) override { - if (config.config_source_specifier_case() == + SubscriptionPtr create(SubscriptionData& data) override { + if (data.config_.config_source_specifier_case() == envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kPath) { return std::make_unique( - dispatcher, makePathConfigSource(config.path()), callbacks, resource_decoder, stats, - validation_visitor, api); + data.dispatcher_, makePathConfigSource(data.config_.path()), data.callbacks_, + data.resource_decoder_, data.stats_, data.validation_visitor_, data.api_); } else { - ASSERT(config.config_source_specifier_case() == + ASSERT(data.config_.config_source_specifier_case() == envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kPathConfigSource); return std::make_unique( - dispatcher, config.path_config_source(), callbacks, resource_decoder, stats, - validation_visitor, api); + data.dispatcher_, data.config_.path_config_source(), data.callbacks_, + data.resource_decoder_, data.stats_, data.validation_visitor_, data.api_); } } - - ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return std::make_unique(); - } }; class FilesystemCollectionSubscriptionFactory : public ConfigSubscriptionFactory { public: std::string name() const override { return "envoy.config_subscription.filesystem_collection"; } - SubscriptionPtr create(const LocalInfo::LocalInfo&, Upstream::ClusterManager&, - Event::Dispatcher& dispatcher, Api::Api& api, - const envoy::config::core::v3::ConfigSource& config, absl::string_view, - SubscriptionCallbacks& callbacks, - OpaqueResourceDecoderSharedPtr resource_decoder, SubscriptionStats stats, - ProtobufMessage::ValidationVisitor& validation_visitor) override { + SubscriptionPtr create(SubscriptionData& data) override { return std::make_unique( - dispatcher, makePathConfigSource(config.path()), callbacks, resource_decoder, stats, - validation_visitor, api); - } - - ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return std::make_unique(); + data.dispatcher_, makePathConfigSource(data.config_.path()), data.callbacks_, + data.resource_decoder_, data.stats_, data.validation_visitor_, data.api_); } }; diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD new file mode 100644 index 000000000000..19d01c1deded --- /dev/null +++ b/source/extensions/config_subscription/grpc/BUILD @@ -0,0 +1,77 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "grpc_subscription_impl_lib", + srcs = ["grpc_subscription_impl.cc"], + hdrs = ["grpc_subscription_impl.h"], + deps = [ + "//envoy/config:subscription_interface", + "//envoy/event:dispatcher_interface", + "//envoy/grpc:async_client_interface", + "//source/common/config:custom_config_validators_lib", + "//source/common/config:grpc_mux_lib", + "//source/common/config:new_grpc_mux_lib", + "//source/common/config:type_to_endpoint_lib", + "//source/common/config:xds_resource_lib", + "//source/common/config/xds_mux:grpc_mux_lib", + ], +) + +envoy_cc_extension( + name = "grpc_collection_subscription_lib", + srcs = ["grpc_collection_subscription_factory.cc"], + hdrs = ["grpc_collection_subscription_factory.h"], + extra_visibility = [ + # previously considered core code. + "//test:__subpackages__", + ], + deps = [ + ":grpc_subscription_impl_lib", + "//envoy/config:subscription_interface", + "//envoy/event:dispatcher_interface", + "//source/common/common:minimal_logger_lib", + "//source/common/common:utility_lib", + "//source/common/config:custom_config_validators_lib", + "//source/common/config:decoded_resource_lib", + "//source/common/config:type_to_endpoint_lib", + "//source/common/config:utility_lib", + "//source/common/config:watched_directory_lib", + "//source/common/protobuf", + "//source/common/protobuf:message_validator_lib", + "//source/common/protobuf:utility_lib", + ], +) + +envoy_cc_extension( + name = "grpc_subscription_lib", + srcs = ["grpc_subscription_factory.cc"], + hdrs = ["grpc_subscription_factory.h"], + extra_visibility = [ + # previously considered core code. + "//test:__subpackages__", + ], + deps = [ + ":grpc_subscription_impl_lib", + "//envoy/config:subscription_interface", + "//envoy/event:dispatcher_interface", + "//source/common/common:minimal_logger_lib", + "//source/common/common:utility_lib", + "//source/common/config:custom_config_validators_lib", + "//source/common/config:decoded_resource_lib", + "//source/common/config:type_to_endpoint_lib", + "//source/common/config:utility_lib", + "//source/common/config:watched_directory_lib", + "//source/common/protobuf", + "//source/common/protobuf:message_validator_lib", + "//source/common/protobuf:utility_lib", + ], +) diff --git a/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc new file mode 100644 index 000000000000..7531b1c3f994 --- /dev/null +++ b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc @@ -0,0 +1,63 @@ +#include "source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h" + +#include "source/common/config/custom_config_validators_impl.h" +#include "source/common/config/grpc_mux_impl.h" +#include "source/common/config/new_grpc_mux_impl.h" +#include "source/common/config/type_to_endpoint.h" +#include "source/common/config/xds_mux/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" + +namespace Envoy { +namespace Config { + +SubscriptionPtr DeltaGrpcCollectionConfigSubscriptionFactory::create( + ConfigSubscriptionFactory::SubscriptionData& data) { + const envoy::config::core::v3::ApiConfigSource& api_config_source = + data.config_.api_config_source(); + CustomConfigValidatorsPtr custom_config_validators = std::make_unique( + data.validation_visitor_, data.server_, api_config_source.config_validators()); + + JitteredExponentialBackOffStrategyPtr backoff_strategy = + Utility::prepareJitteredExponentialBackOffStrategy( + api_config_source, data.api_.randomGenerator(), SubscriptionFactory::RetryInitialDelayMs, + SubscriptionFactory::RetryMaxDelayMs); + + return std::make_unique( + data.collection_locator_.value(), + std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(data.cm_.grpcAsyncClientManager(), + api_config_source, data.scope_, true) + ->createUncachedRawAsyncClient(), + data.dispatcher_, deltaGrpcMethod(data.type_url_), data.scope_, + Utility::parseRateLimitSettings(api_config_source), data.local_info_, + std::move(custom_config_validators), std::move(backoff_strategy), + data.xds_config_tracker_), + data.callbacks_, data.resource_decoder_, data.stats_, data.dispatcher_, + Utility::configSourceInitialFetchTimeout(data.config_), /*is_aggregated=*/false, + data.options_); +} + +SubscriptionPtr AggregatedGrpcCollectionConfigSubscriptionFactory::create( + ConfigSubscriptionFactory::SubscriptionData& data) { + return std::make_unique( + data.collection_locator_.value(), data.cm_.adsMux(), data.callbacks_, data.resource_decoder_, + data.stats_, data.dispatcher_, Utility::configSourceInitialFetchTimeout(data.config_), + /*is_aggregated=*/true, data.options_); +} + +SubscriptionPtr +AdsCollectionConfigSubscriptionFactory::create(ConfigSubscriptionFactory::SubscriptionData& data) { + // All Envoy collections currently are xDS resource graph roots and require node context + // parameters. + return std::make_unique( + data.collection_locator_.value(), data.cm_.adsMux(), data.callbacks_, data.resource_decoder_, + data.stats_, data.dispatcher_, Utility::configSourceInitialFetchTimeout(data.config_), true, + data.options_); +} + +REGISTER_FACTORY(DeltaGrpcCollectionConfigSubscriptionFactory, ConfigSubscriptionFactory); +REGISTER_FACTORY(AggregatedGrpcCollectionConfigSubscriptionFactory, ConfigSubscriptionFactory); +REGISTER_FACTORY(AdsCollectionConfigSubscriptionFactory, ConfigSubscriptionFactory); + +} // namespace Config +} // namespace Envoy diff --git a/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h new file mode 100644 index 000000000000..469313be97e2 --- /dev/null +++ b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h @@ -0,0 +1,34 @@ +#pragma once + +#include "envoy/config/subscription_factory.h" +#include "envoy/registry/registry.h" + +namespace Envoy { +namespace Config { + +class DeltaGrpcCollectionConfigSubscriptionFactory : public ConfigSubscriptionFactory { +public: + std::string name() const override { return "envoy.config_subscription.delta_grpc_collection"; } + SubscriptionPtr create(SubscriptionData& data) override; +}; + +class AggregatedGrpcCollectionConfigSubscriptionFactory : public ConfigSubscriptionFactory { +public: + std::string name() const override { + return "envoy.config_subscription.aggregated_grpc_collection"; + } + SubscriptionPtr create(SubscriptionData& data) override; +}; + +class AdsCollectionConfigSubscriptionFactory : public ConfigSubscriptionFactory { +public: + std::string name() const override { return "envoy.config_subscription.ads_collection"; } + SubscriptionPtr create(SubscriptionData& data) override; +}; + +DECLARE_FACTORY(DeltaGrpcCollectionConfigSubscriptionFactory); +DECLARE_FACTORY(AggregatedGrpcCollectionConfigSubscriptionFactory); +DECLARE_FACTORY(AdsCollectionConfigSubscriptionFactory); + +} // namespace Config +} // namespace Envoy diff --git a/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc b/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc new file mode 100644 index 000000000000..2d703316cd37 --- /dev/null +++ b/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc @@ -0,0 +1,105 @@ +#include "source/extensions/config_subscription/grpc/grpc_subscription_factory.h" + +#include "source/common/config/custom_config_validators_impl.h" +#include "source/common/config/grpc_mux_impl.h" +#include "source/common/config/new_grpc_mux_impl.h" +#include "source/common/config/type_to_endpoint.h" +#include "source/common/config/xds_mux/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" + +namespace Envoy { +namespace Config { + +SubscriptionPtr +GrpcConfigSubscriptionFactory::create(ConfigSubscriptionFactory::SubscriptionData& data) { + GrpcMuxSharedPtr mux; + const envoy::config::core::v3::ApiConfigSource& api_config_source = + data.config_.api_config_source(); + CustomConfigValidatorsPtr custom_config_validators = std::make_unique( + data.validation_visitor_, data.server_, api_config_source.config_validators()); + const std::string control_plane_id = Utility::getGrpcControlPlane(api_config_source).value_or(""); + + JitteredExponentialBackOffStrategyPtr backoff_strategy = + Utility::prepareJitteredExponentialBackOffStrategy( + api_config_source, data.api_.randomGenerator(), SubscriptionFactory::RetryInitialDelayMs, + SubscriptionFactory::RetryMaxDelayMs); + + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { + mux = std::make_shared( + Utility::factoryForGrpcApiConfigSource(data.cm_.grpcAsyncClientManager(), api_config_source, + data.scope_, true) + ->createUncachedRawAsyncClient(), + data.dispatcher_, sotwGrpcMethod(data.type_url_), data.scope_, + Utility::parseRateLimitSettings(api_config_source), data.local_info_, + api_config_source.set_node_on_first_message_only(), std::move(custom_config_validators), + std::move(backoff_strategy), data.xds_config_tracker_, data.xds_resources_delegate_, + control_plane_id); + } else { + mux = std::make_shared( + data.local_info_, + Utility::factoryForGrpcApiConfigSource(data.cm_.grpcAsyncClientManager(), api_config_source, + data.scope_, true) + ->createUncachedRawAsyncClient(), + data.dispatcher_, sotwGrpcMethod(data.type_url_), data.scope_, + Utility::parseRateLimitSettings(api_config_source), + api_config_source.set_node_on_first_message_only(), std::move(custom_config_validators), + std::move(backoff_strategy), data.xds_config_tracker_, data.xds_resources_delegate_, + control_plane_id); + } + return std::make_unique( + std::move(mux), data.callbacks_, data.resource_decoder_, data.stats_, data.type_url_, + data.dispatcher_, Utility::configSourceInitialFetchTimeout(data.config_), + /*is_aggregated*/ false, data.options_); +} + +SubscriptionPtr +DeltaGrpcConfigSubscriptionFactory::create(ConfigSubscriptionFactory::SubscriptionData& data) { + GrpcMuxSharedPtr mux; + const envoy::config::core::v3::ApiConfigSource& api_config_source = + data.config_.api_config_source(); + CustomConfigValidatorsPtr custom_config_validators = std::make_unique( + data.validation_visitor_, data.server_, api_config_source.config_validators()); + + JitteredExponentialBackOffStrategyPtr backoff_strategy = + Utility::prepareJitteredExponentialBackOffStrategy( + api_config_source, data.api_.randomGenerator(), SubscriptionFactory::RetryInitialDelayMs, + SubscriptionFactory::RetryMaxDelayMs); + + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { + mux = std::make_shared( + Utility::factoryForGrpcApiConfigSource(data.cm_.grpcAsyncClientManager(), api_config_source, + data.scope_, true) + ->createUncachedRawAsyncClient(), + data.dispatcher_, deltaGrpcMethod(data.type_url_), data.scope_, + Utility::parseRateLimitSettings(api_config_source), data.local_info_, + api_config_source.set_node_on_first_message_only(), std::move(custom_config_validators), + std::move(backoff_strategy), data.xds_config_tracker_); + } else { + mux = std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(data.cm_.grpcAsyncClientManager(), + api_config_source, data.scope_, true) + ->createUncachedRawAsyncClient(), + data.dispatcher_, deltaGrpcMethod(data.type_url_), data.scope_, + Utility::parseRateLimitSettings(api_config_source), data.local_info_, + std::move(custom_config_validators), std::move(backoff_strategy), data.xds_config_tracker_); + } + return std::make_unique( + std::move(mux), data.callbacks_, data.resource_decoder_, data.stats_, data.type_url_, + data.dispatcher_, Utility::configSourceInitialFetchTimeout(data.config_), + /*is_aggregated*/ false, data.options_); +} + +SubscriptionPtr +AdsConfigSubscriptionFactory::create(ConfigSubscriptionFactory::SubscriptionData& data) { + return std::make_unique( + data.cm_.adsMux(), data.callbacks_, data.resource_decoder_, data.stats_, data.type_url_, + data.dispatcher_, Utility::configSourceInitialFetchTimeout(data.config_), true, + data.options_); +} + +REGISTER_FACTORY(GrpcConfigSubscriptionFactory, ConfigSubscriptionFactory); +REGISTER_FACTORY(DeltaGrpcConfigSubscriptionFactory, ConfigSubscriptionFactory); +REGISTER_FACTORY(AdsConfigSubscriptionFactory, ConfigSubscriptionFactory); + +} // namespace Config +} // namespace Envoy diff --git a/source/extensions/config_subscription/grpc/grpc_subscription_factory.h b/source/extensions/config_subscription/grpc/grpc_subscription_factory.h new file mode 100644 index 000000000000..0f62cbd85e92 --- /dev/null +++ b/source/extensions/config_subscription/grpc/grpc_subscription_factory.h @@ -0,0 +1,32 @@ +#pragma once + +#include "envoy/config/subscription_factory.h" +#include "envoy/registry/registry.h" + +namespace Envoy { +namespace Config { + +class GrpcConfigSubscriptionFactory : public ConfigSubscriptionFactory { +public: + std::string name() const override { return "envoy.config_subscription.grpc"; } + SubscriptionPtr create(SubscriptionData& data) override; +}; + +class DeltaGrpcConfigSubscriptionFactory : public ConfigSubscriptionFactory { +public: + std::string name() const override { return "envoy.config_subscription.delta_grpc"; } + SubscriptionPtr create(SubscriptionData& data) override; +}; + +class AdsConfigSubscriptionFactory : public ConfigSubscriptionFactory { +public: + std::string name() const override { return "envoy.config_subscription.ads"; } + SubscriptionPtr create(SubscriptionData& data) override; +}; + +DECLARE_FACTORY(GrpcConfigSubscriptionFactory); +DECLARE_FACTORY(DeltaGrpcConfigSubscriptionFactory); +DECLARE_FACTORY(AdsConfigSubscriptionFactory); + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/grpc_subscription_impl.cc b/source/extensions/config_subscription/grpc/grpc_subscription_impl.cc similarity index 98% rename from source/common/config/grpc_subscription_impl.cc rename to source/extensions/config_subscription/grpc/grpc_subscription_impl.cc index 0c38d8d4cd0e..d32adfa13160 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/extensions/config_subscription/grpc/grpc_subscription_impl.cc @@ -1,4 +1,4 @@ -#include "source/common/config/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include diff --git a/source/common/config/grpc_subscription_impl.h b/source/extensions/config_subscription/grpc/grpc_subscription_impl.h similarity index 100% rename from source/common/config/grpc_subscription_impl.h rename to source/extensions/config_subscription/grpc/grpc_subscription_impl.h diff --git a/source/extensions/config_subscription/rest/http_subscription_impl.h b/source/extensions/config_subscription/rest/http_subscription_impl.h index 0fbb12007e9b..cd42c9e2074e 100644 --- a/source/extensions/config_subscription/rest/http_subscription_impl.h +++ b/source/extensions/config_subscription/rest/http_subscription_impl.h @@ -67,23 +67,15 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher, class HttpSubscriptionFactory : public ConfigSubscriptionFactory { public: std::string name() const override { return "envoy.config_subscription.rest"; } - SubscriptionPtr create(const LocalInfo::LocalInfo& local_info, Upstream::ClusterManager& cm, - Event::Dispatcher& dispatcher, Api::Api& api, - const envoy::config::core::v3::ConfigSource& config, - absl::string_view type_url, SubscriptionCallbacks& callbacks, - OpaqueResourceDecoderSharedPtr resource_decoder, SubscriptionStats stats, - ProtobufMessage::ValidationVisitor& validation_visitor) override { - - const envoy::config::core::v3::ApiConfigSource& api_config_source = config.api_config_source(); + SubscriptionPtr create(SubscriptionData& data) override { + const envoy::config::core::v3::ApiConfigSource& api_config_source = + data.config_.api_config_source(); return std::make_unique( - local_info, cm, api_config_source.cluster_names()[0], dispatcher, api.randomGenerator(), - Utility::apiConfigSourceRefreshDelay(api_config_source), - Utility::apiConfigSourceRequestTimeout(api_config_source), restMethod(type_url), type_url, - callbacks, resource_decoder, stats, Utility::configSourceInitialFetchTimeout(config), - validation_visitor); - } - ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return std::make_unique(); + data.local_info_, data.cm_, api_config_source.cluster_names()[0], data.dispatcher_, + data.api_.randomGenerator(), Utility::apiConfigSourceRefreshDelay(api_config_source), + Utility::apiConfigSourceRequestTimeout(api_config_source), restMethod(data.type_url_), + data.type_url_, data.callbacks_, data.resource_decoder_, data.stats_, + Utility::configSourceInitialFetchTimeout(data.config_), data.validation_visitor_); } }; diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index c8b5431bc579..6655438230f0 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -439,6 +439,12 @@ EXTENSIONS = { "envoy.config_subscription.rest": "//source/extensions/config_subscription/rest:http_subscription_lib", "envoy.config_subscription.filesystem": "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib", "envoy.config_subscription.filesystem_collection": "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib", + "envoy.config_subscription.grpc": "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "envoy.config_subscription.delta_grpc": "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "envoy.config_subscription.ads": "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "envoy.config_subscription.aggregated_grpc_collection": "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", + "envoy.config_subscription.aggregated_delta_grpc_collection": "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", + "envoy.config_subscription.ads_collection": "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", } # These can be changed to ["//visibility:public"], for downstream builds which diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 15fbfcab0466..3da991ca9960 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -1523,19 +1523,43 @@ envoy.config_subscription.rest: - envoy.config_subscription security_posture: unknown status: stable - type_urls: - - envoy.config.core.v3.RestSubscription envoy.config_subscription.filesystem: categories: - envoy.config_subscription security_posture: unknown status: stable - type_urls: - - envoy.config.core.v3.FilesystemSubscription +envoy.config_subscription.ads_collection: + categories: + - envoy.config_subscription + security_posture: unknown + status: stable +envoy.config_subscription.aggregated_delta_grpc_collection: + categories: + - envoy.config_subscription + security_posture: unknown + status: stable +envoy.config_subscription.aggregated_grpc_collection: + categories: + - envoy.config_subscription + security_posture: unknown + status: stable +envoy.config_subscription.ads: + categories: + - envoy.config_subscription + security_posture: unknown + status: stable +envoy.config_subscription.delta_grpc: + categories: + - envoy.config_subscription + security_posture: unknown + status: stable +envoy.config_subscription.grpc: + categories: + - envoy.config_subscription + security_posture: unknown + status: stable envoy.config_subscription.filesystem_collection: categories: - envoy.config_subscription security_posture: unknown status: stable - type_urls: - - envoy.config.core.v3.FilesystemCollectionSubscription diff --git a/test/common/config/BUILD b/test/common/config/BUILD index d35a44ae1c8d..65295b3142fd 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -38,9 +38,9 @@ envoy_cc_test( ":delta_subscription_test_harness", "//envoy/config:xds_config_tracker_interface", "//source/common/config:api_version_lib", - "//source/common/config:grpc_subscription_lib", "//source/common/config:new_grpc_mux_lib", "//source/common/stats:isolated_store_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//test/mocks:common_lib", "//test/mocks/config:config_mocks", "//test/mocks/event:event_mocks", @@ -59,10 +59,10 @@ envoy_cc_test( srcs = ["delta_subscription_state_test.cc"], deps = [ "//source/common/config:delta_subscription_state_lib", - "//source/common/config:grpc_subscription_lib", "//source/common/config:new_grpc_mux_lib", "//source/common/config/xds_mux:delta_subscription_state_lib", "//source/common/stats:isolated_store_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//test/mocks:common_lib", "//test/mocks/config:config_mocks", "//test/mocks/event:event_mocks", @@ -81,9 +81,9 @@ envoy_cc_test( srcs = ["delta_subscription_state_old_test.cc"], deps = [ "//source/common/config:delta_subscription_state_lib", - "//source/common/config:grpc_subscription_lib", "//source/common/config:new_grpc_mux_lib", "//source/common/stats:isolated_store_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//test/mocks:common_lib", "//test/mocks/config:config_mocks", "//test/mocks/event:event_mocks", @@ -236,8 +236,8 @@ envoy_cc_test_library( "//source/common/common:hash_lib", "//source/common/config:api_version_lib", "//source/common/config:grpc_mux_lib", - "//source/common/config:grpc_subscription_lib", "//source/common/config/xds_mux:grpc_mux_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//test/mocks/config:config_mocks", "//test/mocks/config:custom_config_validators_mocks", "//test/mocks/event:event_mocks", diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index bf1dfa0df637..9a2422dd50c2 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -8,10 +8,10 @@ #include "envoy/config/xds_config_tracker.h" #include "envoy/service/discovery/v3/discovery.pb.h" -#include "source/common/config/grpc_subscription_impl.h" #include "source/common/config/new_grpc_mux_impl.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/grpc/common.h" +#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "test/common/config/subscription_test_harness.h" #include "test/mocks/common.h" diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 0b0da0302bc8..5f90f6cae80b 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -12,8 +12,8 @@ #include "source/common/common/hash.h" #include "source/common/config/api_version.h" #include "source/common/config/grpc_mux_impl.h" -#include "source/common/config/grpc_subscription_impl.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "test/common/config/subscription_test_harness.h" #include "test/mocks/config/custom_config_validators.h" diff --git a/test/common/router/BUILD b/test/common/router/BUILD index ab545edf30fa..1e2f7ddca4fa 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -156,6 +156,7 @@ envoy_cc_test( "//envoy/config:subscription_interface", "//envoy/init:manager_interface", "//source/common/config:api_version_lib", + "//source/common/config:grpc_mux_lib", "//source/common/config:utility_lib", "//source/common/http:message_lib", "//source/common/json:json_loader_lib", diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 9b6e8408af00..511388c4f87a 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -77,6 +77,8 @@ envoy_cc_test( "//source/extensions/clusters/original_dst:original_dst_cluster_lib", "//source/extensions/clusters/static:static_cluster_lib", "//source/extensions/clusters/strict_dns:strict_dns_cluster_lib", + "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//source/extensions/health_checkers/http:health_checker_lib", "//source/extensions/health_checkers/tcp:health_checker_lib", "//source/extensions/load_balancing_policies/maglev:config", diff --git a/test/extensions/clusters/eds/BUILD b/test/extensions/clusters/eds/BUILD index 29a3e3ddb2b6..89070a08523c 100644 --- a/test/extensions/clusters/eds/BUILD +++ b/test/extensions/clusters/eds/BUILD @@ -48,11 +48,11 @@ envoy_cc_benchmark_binary( deps = [ "//envoy/config:xds_resources_delegate_interface", "//source/common/config:grpc_mux_lib", - "//source/common/config:grpc_subscription_lib", "//source/common/config:protobuf_link_hacks", "//source/common/config:utility_lib", "//source/common/config/xds_mux:grpc_mux_lib", "//source/extensions/clusters/eds:eds_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/upstream:utility_lib", diff --git a/test/extensions/clusters/eds/eds_speed_test.cc b/test/extensions/clusters/eds/eds_speed_test.cc index fc5af2ad8916..602792695bfc 100644 --- a/test/extensions/clusters/eds/eds_speed_test.cc +++ b/test/extensions/clusters/eds/eds_speed_test.cc @@ -11,12 +11,12 @@ #include "envoy/stats/scope.h" #include "source/common/config/grpc_mux_impl.h" -#include "source/common/config/grpc_subscription_impl.h" #include "source/common/config/protobuf_link_hacks.h" #include "source/common/config/utility.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/singleton/manager_impl.h" #include "source/extensions/clusters/eds/eds.h" +#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/server/transport_socket_config_impl.h" #include "test/benchmark/main.h" diff --git a/test/extensions/config_subscription/common/BUILD b/test/extensions/config_subscription/common/BUILD index 2e726d6d5110..6807a30b3676 100644 --- a/test/extensions/config_subscription/common/BUILD +++ b/test/extensions/config_subscription/common/BUILD @@ -17,6 +17,8 @@ envoy_cc_test( "//source/common/config:subscription_factory_lib", "//source/common/config:xds_resource_lib", "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib", + "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//source/extensions/config_subscription/rest:http_subscription_lib", "//test/config:v2_link_hacks", "//test/mocks/config:config_mocks", diff --git a/test/integration/BUILD b/test/integration/BUILD index 3f02e24f4b08..7c05d4a84a8f 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -952,8 +952,10 @@ envoy_cc_test_library( "base_integration_test.h", ], deps = [ - "//source/extensions/load_balancing_policies/maglev:config", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib", + "//source/extensions/load_balancing_policies/maglev:config", "//source/server:proto_descriptors_lib", "//source/extensions/request_id/uuid:config", ":autonomous_upstream_lib", diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 661903315d4c..685c13b430d0 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -7,7 +7,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/api:82.4" "source/common/api/posix:81.3" "source/common/common/posix:92.7" -"source/common/config:96.2" +"source/common/config:96.1" "source/common/crypto:88.1" "source/common/event:95.1" # Emulated edge events guards don't report LCOV "source/common/filesystem/posix:96.2" # FileReadToEndNotReadable fails in some env; createPath can't test all failure branches. diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index f95619d87c4a..d36f14206e1a 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -319,4 +319,5 @@ visibility_excludes: - source/extensions/health_checkers/BUILD - source/extensions/config_subscription/rest/BUILD - source/extensions/config_subscription/filesystem/BUILD +- source/extensions/config_subscription/grpc/BUILD - source/extensions/load_balancing_policies/subset/BUILD From 82690b1e8461e4294e188d44f9eba7252722addf Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Tue, 2 May 2023 16:54:17 +0300 Subject: [PATCH 073/740] access_log: remove leftover default access log type parameter (#27096) Signed-off-by: ohadvano --- .../extensions/access_loggers/open_telemetry/access_log_impl.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/extensions/access_loggers/open_telemetry/access_log_impl.h b/source/extensions/access_loggers/open_telemetry/access_log_impl.h index fa029aa9b1d2..be5456d08c8d 100644 --- a/source/extensions/access_loggers/open_telemetry/access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/access_log_impl.h @@ -53,8 +53,7 @@ class AccessLog : public Common::ImplBase { const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - Envoy::AccessLog::AccessLogType access_log_type = - Envoy::AccessLog::AccessLogType::NotSet) override; + Envoy::AccessLog::AccessLogType access_log_type) override; const ThreadLocal::SlotPtr tls_slot_; const GrpcAccessLoggerCacheSharedPtr access_logger_cache_; From 686f270a7f6704728e8e9f219d5753eadb7a1234 Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Tue, 2 May 2023 16:55:29 +0300 Subject: [PATCH 074/740] access_log: set access log type for deferred quic logging (#27089) * Update protocol_integration_test.cc Signed-off-by: ohadvano <49730675+ohadvano@users.noreply.github.com> --- source/common/quic/quic_stats_gatherer.cc | 2 +- test/integration/protocol_integration_test.cc | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/source/common/quic/quic_stats_gatherer.cc b/source/common/quic/quic_stats_gatherer.cc index 32552993751f..e632062c5c70 100644 --- a/source/common/quic/quic_stats_gatherer.cc +++ b/source/common/quic/quic_stats_gatherer.cc @@ -26,7 +26,7 @@ void QuicStatsGatherer::maybeDoDeferredLog(bool record_ack_timing) { const Http::ResponseTrailerMap* response_trailers = response_trailer_map_.get(); for (const AccessLog::InstanceSharedPtr& log_handler : access_log_handlers_) { log_handler->log(request_headers, response_headers, response_trailers, *stream_info_, - AccessLog::AccessLogType::NotSet); + AccessLog::AccessLogType::DownstreamEnd); } } diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index f742e4cad2d5..96ca214e296d 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -431,10 +431,8 @@ TEST_P(ProtocolIntegrationTest, AccessLogTest) { EXPECT_TRUE(response->complete()); EXPECT_EQ("200", response->headers().getStatusValue()); - if (downstream_protocol_ != Http::CodecType::HTTP3) { - EXPECT_EQ(absl::StrCat("200 ", AccessLogType_Name(AccessLog::AccessLogType::DownstreamEnd)), - waitForAccessLog(access_log_name_, 1, true)); - } + EXPECT_EQ(absl::StrCat("200 ", AccessLogType_Name(AccessLog::AccessLogType::DownstreamEnd)), + waitForAccessLog(access_log_name_, 1, true)); } TEST_P(ProtocolIntegrationTest, PeriodicAccessLog) { From 6e706d90b3c5a2947017c77d5bd209f16115dc8a Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Tue, 2 May 2023 09:56:02 -0400 Subject: [PATCH 075/740] test: Fix flaky KeyValueStoreXdsDelegateIntegrationTest (#27082) This commit addresses a few aspects of the tests flakiness: 1. Ensure that the Envoy server is shutdown before restarting it to load xDS from the KV store. 2. Don't try to re-establish the xDS FakeUpstreams after shutdown. We need to use the same ports, otherwise we can't create a bootstrap config passing in the FakeUpstream's port without first having a running FakeUpstream, and if we try to re-use the same port, we ocassionally get port "Address already in use" errors. Re-checking connection establishment doesn't add a huge amount of value to the KV store test, so we just remove that part of the test to avoid flaky behavior. Signed-off-by: Ali Beyad --- .../kv_store_xds_delegate_integration_test.cc | 32 ++++++------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/contrib/config/test/kv_store_xds_delegate_integration_test.cc b/contrib/config/test/kv_store_xds_delegate_integration_test.cc index fa2f9eb5e669..27c54162935f 100644 --- a/contrib/config/test/kv_store_xds_delegate_integration_test.cc +++ b/contrib/config/test/kv_store_xds_delegate_integration_test.cc @@ -233,8 +233,18 @@ class KeyValueStoreXdsDelegateIntegrationTest } void shutdownAndRestartTestServer() { + // Shutdown the Envoy server. + absl::Notification shutdown_done; + Server::Instance& server = test_server_->server(); + server.dispatcher().post([&server, &shutdown_done]() { + server.shutdown(); + shutdown_done.Notify(); + }); + ASSERT_TRUE(shutdown_done.WaitForNotificationWithTimeout(absl::Seconds(5))); // Reset the test server. test_server_.reset(); + // Reset the server init function, as we don't want xDS responses sent on restart (we want to + // use the values in the KV store instead). on_server_init_function_ = nullptr; // Set up a new Envoy, using the previous Envoy's configuration, and create the test server. @@ -365,28 +375,6 @@ TEST_P(KeyValueStoreXdsDelegateIntegrationTest, BasicSuccess) { EXPECT_EQ("whatevs", getRuntimeKey("foo")); EXPECT_EQ("yar", getRuntimeKey("bar")); EXPECT_EQ("saz", getRuntimeKey("baz")); - - // Reset the RTDS upstream to a FakeUpstream again, and re-establish the connection. - getRtdsUpstream() = std::make_unique(rtds_upstream_port_, version_, - configWithType(Http::CodecType::HTTP2)); - - // Send v2 of the RTDS layer. - initXdsStream(*getRtdsUpstream(), rtds_connection_, rtds_stream_); - auto rtds_resource_v2 = TestUtility::parseYaml(R"EOF( - name: some_rtds_layer - layer: - foo: zoo - baz: jazz - )EOF"); - sendSotwDiscoveryResponse( - Config::TypeUrl::get().Runtime, {rtds_resource_v2}, /*version=*/"2", rtds_stream_.get()); - - test_server_->waitForCounterGe("runtime.load_success", 3); - - // Verify that the values from the xDS response are used instead of from the persisted xDS once - // connectivity is re-established. - EXPECT_EQ("zoo", getRuntimeKey("foo")); - EXPECT_EQ("jazz", getRuntimeKey("baz")); } // A KeyValueStore implementation that returns an invalid proto field value for a Cluster resource. From 37cdc12cf3a98d9d4888872766a3910046d197b9 Mon Sep 17 00:00:00 2001 From: code Date: Tue, 2 May 2023 23:44:01 +0800 Subject: [PATCH 076/740] tracing: expose stream info to the extended tracing driver (#27062) * tracing: expose stream info to the extended tracing driver Signed-off-by: wbpcode --- envoy/tracing/BUILD | 1 + envoy/tracing/trace_driver.h | 6 +- envoy/tracing/tracer.h | 7 +- source/common/tracing/tracer_impl.cc | 4 +- .../common/ot/opentracing_driver_impl.cc | 4 +- .../common/ot/opentracing_driver_impl.h | 5 +- source/extensions/tracers/datadog/tracer.cc | 5 +- source/extensions/tracers/datadog/tracer.h | 9 +- .../opencensus/opencensus_tracer_impl.cc | 7 +- .../opencensus/opencensus_tracer_impl.h | 7 +- .../opentelemetry_tracer_impl.cc | 8 +- .../opentelemetry/opentelemetry_tracer_impl.h | 7 +- .../skywalking/skywalking_tracer_impl.cc | 2 +- .../skywalking/skywalking_tracer_impl.h | 6 +- .../tracers/xray/xray_tracer_impl.cc | 5 +- .../tracers/xray/xray_tracer_impl.h | 4 +- .../tracers/zipkin/zipkin_tracer_impl.cc | 12 +- .../tracers/zipkin/zipkin_tracer_impl.h | 5 +- test/common/tracing/tracer_impl_test.cc | 12 +- .../tracing/tracer_manager_impl_test.cc | 4 +- test/extensions/tracers/common/ot/BUILD | 1 + .../common/ot/opentracing_driver_impl_test.cc | 49 +++---- test/extensions/tracers/datadog/BUILD | 1 + .../extensions/tracers/datadog/config_test.cc | 6 +- .../extensions/tracers/datadog/tracer_test.cc | 16 ++- test/extensions/tracers/dynamic_ot/BUILD | 1 + .../dynamic_opentracing_driver_impl_test.cc | 9 +- test/extensions/tracers/opencensus/BUILD | 1 + .../tracers/opencensus/tracer_test.cc | 14 +- test/extensions/tracers/opentelemetry/BUILD | 1 + .../opentelemetry_tracer_impl_test.cc | 63 ++++----- test/extensions/tracers/skywalking/BUILD | 1 + .../skywalking/skywalking_tracer_impl_test.cc | 33 +++-- test/extensions/tracers/xray/BUILD | 1 + .../tracers/xray/xray_tracer_impl_test.cc | 40 ++++-- test/extensions/tracers/zipkin/BUILD | 1 + .../tracers/zipkin/zipkin_tracer_impl_test.cc | 130 +++++++++--------- test/mocks/tracing/mocks.h | 10 +- 38 files changed, 267 insertions(+), 231 deletions(-) diff --git a/envoy/tracing/BUILD b/envoy/tracing/BUILD index 0e65f6da5923..29c2776504ba 100644 --- a/envoy/tracing/BUILD +++ b/envoy/tracing/BUILD @@ -36,6 +36,7 @@ envoy_cc_library( hdrs = ["trace_driver.h"], deps = [ ":trace_config_interface", + "//envoy/stream_info:stream_info_interface", ], ) diff --git a/envoy/tracing/trace_driver.h b/envoy/tracing/trace_driver.h index 60557ef41f67..111500063e14 100644 --- a/envoy/tracing/trace_driver.h +++ b/envoy/tracing/trace_driver.h @@ -5,6 +5,7 @@ #include #include "envoy/common/pure.h" +#include "envoy/stream_info/stream_info.h" #include "envoy/tracing/trace_config.h" namespace Envoy { @@ -107,8 +108,9 @@ class Driver { /** * Start driver specific span. */ - virtual SpanPtr startSpan(const Config& config, TraceContext& trace_conext, - const std::string& operation_name, SystemTime start_time, + virtual SpanPtr startSpan(const Config& config, TraceContext& trace_context, + const StreamInfo::StreamInfo& stream_info, + const std::string& operation_name, const Tracing::Decision tracing_decision) PURE; }; diff --git a/envoy/tracing/tracer.h b/envoy/tracing/tracer.h index 08c83ab6e8c5..43aa84907e67 100644 --- a/envoy/tracing/tracer.h +++ b/envoy/tracing/tracer.h @@ -1,17 +1,12 @@ #pragma once #include -#include -#include -#include "envoy/access_log/access_log.h" #include "envoy/common/pure.h" +#include "envoy/tracing/trace_context.h" #include "envoy/tracing/trace_driver.h" #include "envoy/tracing/trace_reason.h" -#include "trace_context.h" -#include "tracer.h" - namespace Envoy { namespace Tracing { diff --git a/source/common/tracing/tracer_impl.cc b/source/common/tracing/tracer_impl.cc index 0d493d1a56b2..01ee929bbd35 100644 --- a/source/common/tracing/tracer_impl.cc +++ b/source/common/tracing/tracer_impl.cc @@ -149,8 +149,8 @@ SpanPtr TracerImpl::startSpan(const Config& config, TraceContext& trace_context, span_name.append(std::string(trace_context.host())); } - SpanPtr active_span = driver_->startSpan(config, trace_context, span_name, - stream_info.startTime(), tracing_decision); + SpanPtr active_span = + driver_->startSpan(config, trace_context, stream_info, span_name, tracing_decision); // Set tags related to the local environment if (active_span) { diff --git a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc index 4f0399f3ddf1..387be89f9d2b 100644 --- a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc +++ b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc @@ -153,8 +153,8 @@ OpenTracingDriver::OpenTracingDriver(Stats::Scope& scope) Tracing::SpanPtr OpenTracingDriver::startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, + const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - SystemTime start_time, const Tracing::Decision tracing_decision) { const PropagationMode propagation_mode = this->propagationMode(); const opentracing::Tracer& tracer = this->tracer(); @@ -196,7 +196,7 @@ Tracing::SpanPtr OpenTracingDriver::startSpan(const Tracing::Config& config, opentracing::StartSpanOptions options; options.references.emplace_back(opentracing::SpanReferenceType::ChildOfRef, parent_span_ctx.get()); - options.start_system_timestamp = start_time; + options.start_system_timestamp = stream_info.startTime(); if (!tracing_decision.traced) { options.tags.emplace_back(opentracing::ext::sampling_priority, 0); } diff --git a/source/extensions/tracers/common/ot/opentracing_driver_impl.h b/source/extensions/tracers/common/ot/opentracing_driver_impl.h index 07982592e932..94c58b44cf4f 100644 --- a/source/extensions/tracers/common/ot/opentracing_driver_impl.h +++ b/source/extensions/tracers/common/ot/opentracing_driver_impl.h @@ -64,9 +64,10 @@ class OpenTracingDriver : public Tracing::Driver, protected Logger::Loggable tracer; }; - // Tracer::TracingDriver + // Tracing::Driver /** * Create a Datadog span from the specified \p trace_context, and having the - * specified \p operation_name, \p start_time and \p tracing_decision. + * specified \p operation_name, \p stream_info and \p tracing_decision. * If this tracer encountered an error during initialization, then return a * \c Tracing::NullSpan instead. * @param config this parameter is ignored * @param trace_context possibly contains information about an existing trace * that the returned span will be a part of; otherwise, the returned span is * the root of a new trace + * @param stream_info contains information about the stream. * @param operation_name the name of the operation representation by this * span, e.g. "handle.request" - * @param start_time when the span begins * @param tracing_decision the sampling decision made in advance by Envoy for * this trace. If the decision is to drop the trace, then this tracer will * honor that decision. If the decision is to keep the trace, then this tracer * will apply its own sampling logic, which might keep or drop the trace. */ Tracing::SpanPtr startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, - const std::string& operation_name, SystemTime start_time, + const StreamInfo::StreamInfo& stream_info, + const std::string& operation_name, const Tracing::Decision tracing_decision) override; private: diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 85458a34de24..436b82666732 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -375,10 +375,11 @@ void Driver::applyTraceConfig(const opencensus::proto::trace::v1::TraceConfig& c Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, - const std::string& operation_name, SystemTime start_time, + const StreamInfo::StreamInfo& stream_info, + const std::string& operation_name, const Tracing::Decision tracing_decision) { - return std::make_unique(config, oc_config_, trace_context, operation_name, start_time, - tracing_decision); + return std::make_unique(config, oc_config_, trace_context, operation_name, + stream_info.startTime(), tracing_decision); } } // namespace OpenCensus diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.h b/source/extensions/tracers/opencensus/opencensus_tracer_impl.h index 0fdb17c4c683..1a392ce9f59d 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.h +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.h @@ -20,11 +20,10 @@ class Driver : public Tracing::Driver, Logger::Loggable { Driver(const envoy::config::trace::v3::OpenCensusConfig& oc_config, const LocalInfo::LocalInfo& localinfo, Api::Api& api); - /** - * Implements the abstract Driver's startSpan operation. - */ + // Tracing::Driver Tracing::SpanPtr startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, - const std::string& operation_name, SystemTime start_time, + const StreamInfo::StreamInfo& stream_info, + const std::string& operation_name, const Tracing::Decision tracing_decision) override; private: diff --git a/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.cc b/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.cc index 6e84d474e00c..a3119c187e83 100644 --- a/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.cc +++ b/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.cc @@ -47,7 +47,8 @@ Driver::Driver(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetr Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, - const std::string& operation_name, SystemTime start_time, + const StreamInfo::StreamInfo& stream_info, + const std::string& operation_name, const Tracing::Decision tracing_decision) { // Get tracer from TLS and start span. auto& tracer = tls_slot_ptr_->getTyped().tracer(); @@ -55,14 +56,15 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, if (!extractor.propagationHeaderPresent()) { // No propagation header, so we can create a fresh span with the given decision. Tracing::SpanPtr new_open_telemetry_span = - tracer.startSpan(config, operation_name, start_time, tracing_decision); + tracer.startSpan(config, operation_name, stream_info.startTime(), tracing_decision); new_open_telemetry_span->setSampled(tracing_decision.traced); return new_open_telemetry_span; } else { // Try to extract the span context. If we can't, just return a null span. absl::StatusOr span_context = extractor.extractSpanContext(); if (span_context.ok()) { - return tracer.startSpan(config, operation_name, start_time, span_context.value()); + return tracer.startSpan(config, operation_name, stream_info.startTime(), + span_context.value()); } else { ENVOY_LOG(trace, "Unable to extract span context: ", span_context.status()); return std::make_unique(); diff --git a/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h b/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h index f464b760e4d9..35f734c87b82 100644 --- a/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h +++ b/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h @@ -32,11 +32,10 @@ class Driver : Logger::Loggable, public Tracing::Driver { Driver(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, Server::Configuration::TracerFactoryContext& context); - /** - * Implements the abstract Driver's startSpan operation. - */ + // Tracing::Driver Tracing::SpanPtr startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, - const std::string& operation_name, SystemTime start_time, + const StreamInfo::StreamInfo& stream_info, + const std::string& operation_name, const Tracing::Decision tracing_decision) override; private: diff --git a/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc b/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc index 7fda11c78c4d..f727f3ead197 100644 --- a/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc +++ b/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc @@ -45,7 +45,7 @@ Driver::Driver(const envoy::config::trace::v3::SkyWalkingConfig& proto_config, } Tracing::SpanPtr Driver::startSpan(const Tracing::Config&, Tracing::TraceContext& trace_context, - const std::string&, Envoy::SystemTime, + const StreamInfo::StreamInfo&, const std::string&, const Tracing::Decision decision) { auto& tracer = tls_slot_ptr_->getTyped().tracer(); TracingContextPtr tracing_context; diff --git a/source/extensions/tracers/skywalking/skywalking_tracer_impl.h b/source/extensions/tracers/skywalking/skywalking_tracer_impl.h index 58849667b13a..0a97052c624e 100644 --- a/source/extensions/tracers/skywalking/skywalking_tracer_impl.h +++ b/source/extensions/tracers/skywalking/skywalking_tracer_impl.h @@ -24,9 +24,11 @@ class Driver : public Tracing::Driver, public Logger::LoggablegetTyped().tracer_.get(); if (should_trace.value()) { - return tracer->startSpan(config, operation_name, start_time, + return tracer->startSpan(config, operation_name, stream_info.startTime(), header.has_value() ? absl::optional(xray_header) : absl::nullopt, trace_context.getByKey(XForwardedForHeader)); diff --git a/source/extensions/tracers/xray/xray_tracer_impl.h b/source/extensions/tracers/xray/xray_tracer_impl.h index 775f11a836ac..4e3b9705e5cb 100644 --- a/source/extensions/tracers/xray/xray_tracer_impl.h +++ b/source/extensions/tracers/xray/xray_tracer_impl.h @@ -17,8 +17,10 @@ class Driver : public Tracing::Driver, public Logger::LoggablegetTyped().tracer_; SpanPtr new_zipkin_span; @@ -122,13 +122,13 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, auto ret_span_context = extractor.extractSpanContext(sampled); if (!ret_span_context.second) { // Create a root Zipkin span. No context was found in the headers. - new_zipkin_span = tracer.startSpan(config, std::string(trace_context.host()), start_time); + new_zipkin_span = + tracer.startSpan(config, std::string(trace_context.host()), stream_info.startTime()); new_zipkin_span->setSampled(sampled); } else { - new_zipkin_span = tracer.startSpan(config, std::string(trace_context.host()), start_time, - ret_span_context.first); + new_zipkin_span = tracer.startSpan(config, std::string(trace_context.host()), + stream_info.startTime(), ret_span_context.first); } - } catch (const ExtractorException& e) { return std::make_unique(); } diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h index 1f2aad8d3497..98dc6810eaeb 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h @@ -123,8 +123,9 @@ class Driver : public Tracing::Driver { * Thus, this implementation of the virtual function startSpan() ignores the operation name * ("ingress" or "egress") passed by the caller. */ - Tracing::SpanPtr startSpan(const Tracing::Config&, Tracing::TraceContext& trace_context, - const std::string&, SystemTime start_time, + Tracing::SpanPtr startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, + const StreamInfo::StreamInfo& stream_info, + const std::string& operation_name, const Tracing::Decision tracing_decision) override; // Getters to return the ZipkinDriver's key members. diff --git a/test/common/tracing/tracer_impl_test.cc b/test/common/tracing/tracer_impl_test.cc index 2905928c367c..6535d3756b99 100644 --- a/test/common/tracing/tracer_impl_test.cc +++ b/test/common/tracing/tracer_impl_test.cc @@ -301,22 +301,18 @@ class TracerImplTest : public testing::Test { TEST_F(TracerImplTest, BasicFunctionalityNullSpan) { EXPECT_CALL(config_, operationName()).Times(2); - EXPECT_CALL(stream_info_, startTime()); const std::string operation_name = "ingress"; - EXPECT_CALL(*driver_, startSpan_(_, _, operation_name, stream_info_.start_time_, _)) - .WillOnce(Return(nullptr)); + EXPECT_CALL(*driver_, startSpan_(_, _, _, operation_name, _)).WillOnce(Return(nullptr)); tracer_->startSpan(config_, request_headers_, stream_info_, {Reason::Sampling, true}); } TEST_F(TracerImplTest, BasicFunctionalityNodeSet) { - EXPECT_CALL(stream_info_, startTime()); EXPECT_CALL(local_info_, nodeName()); EXPECT_CALL(config_, operationName()).Times(2).WillRepeatedly(Return(OperationName::Egress)); NiceMock* span = new NiceMock(); const std::string operation_name = "egress test"; - EXPECT_CALL(*driver_, startSpan_(_, _, operation_name, stream_info_.start_time_, _)) - .WillOnce(Return(span)); + EXPECT_CALL(*driver_, startSpan_(_, _, _, operation_name, _)).WillOnce(Return(span)); EXPECT_CALL(*span, setTag(_, _)).Times(testing::AnyNumber()); EXPECT_CALL(*span, setTag(Eq(Tracing::Tags::get().NodeId), Eq("node_name"))); @@ -324,14 +320,12 @@ TEST_F(TracerImplTest, BasicFunctionalityNodeSet) { } TEST_F(TracerImplTest, ChildGrpcUpstreamSpanTest) { - EXPECT_CALL(stream_info_, startTime()); EXPECT_CALL(local_info_, nodeName()); EXPECT_CALL(config_, operationName()).Times(2).WillRepeatedly(Return(OperationName::Egress)); NiceMock* span = new NiceMock(); const std::string operation_name = "egress test"; - EXPECT_CALL(*driver_, startSpan_(_, _, operation_name, stream_info_.start_time_, _)) - .WillOnce(Return(span)); + EXPECT_CALL(*driver_, startSpan_(_, _, _, operation_name, _)).WillOnce(Return(span)); EXPECT_CALL(*span, setTag(_, _)).Times(testing::AnyNumber()); EXPECT_CALL(*span, setTag(Eq(Tracing::Tags::get().NodeId), Eq("node_name"))); diff --git a/test/common/tracing/tracer_manager_impl_test.cc b/test/common/tracing/tracer_manager_impl_test.cc index b28a3091179b..75864ff587fb 100644 --- a/test/common/tracing/tracer_manager_impl_test.cc +++ b/test/common/tracing/tracer_manager_impl_test.cc @@ -23,8 +23,8 @@ namespace { class SampleDriver : public Driver { public: - SpanPtr startSpan(const Config&, Tracing::TraceContext&, const std::string&, SystemTime, - const Tracing::Decision) override { + SpanPtr startSpan(const Config&, Tracing::TraceContext&, const StreamInfo::StreamInfo&, + const std::string&, const Tracing::Decision) override { return nullptr; } }; diff --git a/test/extensions/tracers/common/ot/BUILD b/test/extensions/tracers/common/ot/BUILD index f58cf9093984..e27a825efa2a 100644 --- a/test/extensions/tracers/common/ot/BUILD +++ b/test/extensions/tracers/common/ot/BUILD @@ -23,6 +23,7 @@ envoy_extension_cc_test( "//source/extensions/tracers/dynamic_ot:dynamic_opentracing_driver_lib", "//test/mocks/http:http_mocks", "//test/mocks/stats:stats_mocks", + "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/tracing:tracing_mocks", "@io_opentracing_cpp//mocktracer", ], diff --git a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc index 7e43c3fff9b9..07bd22ec9955 100644 --- a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc +++ b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc @@ -6,6 +6,7 @@ #include "test/mocks/http/mocks.h" #include "test/mocks/stats/mocks.h" +#include "test/mocks/stream_info/mocks.h" #include "test/mocks/tracing/mocks.h" #include "gmock/gmock.h" @@ -185,7 +186,7 @@ class OpenTracingDriverTest : public testing::Test { Http::TestRequestHeaderMapImpl request_headers_{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; const Http::TestResponseHeaderMapImpl response_headers_{{":status", "500"}}; - SystemTime start_time_; + NiceMock stream_info_; std::unique_ptr driver_; Stats::TestUtil::TestStore stats_; @@ -196,8 +197,8 @@ class OpenTracingDriverTest : public testing::Test { TEST_F(OpenTracingDriverTest, FlushSpanWithTag) { setupValidDriver(); - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); first_span->setTag("abc", "123"); first_span->finishSpan(); @@ -212,8 +213,8 @@ TEST_F(OpenTracingDriverTest, FlushSpanWithTag) { TEST_F(OpenTracingDriverTest, FlushSpanWithLog) { setupValidDriver(); - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); const auto timestamp = SystemTime{std::chrono::duration_cast(std::chrono::hours{123})}; first_span->log(timestamp, "abc"); @@ -229,8 +230,8 @@ TEST_F(OpenTracingDriverTest, FlushSpanWithLog) { TEST_F(OpenTracingDriverTest, FlushSpanWithBaggage) { setupValidDriver(); - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); first_span->setBaggage("abc", "123"); first_span->finishSpan(); @@ -243,8 +244,8 @@ TEST_F(OpenTracingDriverTest, FlushSpanWithBaggage) { TEST_F(OpenTracingDriverTest, TagSamplingFalseByDecision) { setupValidDriver(OpenTracingDriver::PropagationMode::TracerNative, {}); - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, false}); + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, false}); first_span->finishSpan(); const std::map expected_tags = { @@ -258,8 +259,8 @@ TEST_F(OpenTracingDriverTest, TagSamplingFalseByDecision) { TEST_F(OpenTracingDriverTest, TagSamplingFalseByFlag) { setupValidDriver(OpenTracingDriver::PropagationMode::TracerNative, {}); - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); first_span->setSampled(false); first_span->finishSpan(); @@ -276,8 +277,8 @@ TEST_F(OpenTracingDriverTest, TagSpanKindClient) { ON_CALL(config_, operationName()).WillByDefault(testing::Return(Tracing::OperationName::Egress)); - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); first_span->finishSpan(); const std::map expected_tags = { @@ -292,8 +293,8 @@ TEST_F(OpenTracingDriverTest, TagSpanKindServer) { ON_CALL(config_, operationName()).WillByDefault(testing::Return(Tracing::OperationName::Ingress)); - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); first_span->finishSpan(); const std::map expected_tags = { @@ -311,8 +312,8 @@ TEST_F(OpenTracingDriverTest, InjectFailure) { propagation_options.inject_error_code = std::make_error_code(std::errc::bad_message); setupValidDriver(propagation_mode, propagation_options); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); const auto span_context_injection_error_count = stats_.counter("tracing.opentracing.span_context_injection_error").value(); @@ -329,12 +330,12 @@ TEST_F(OpenTracingDriverTest, ExtractWithUnindexedHeader) { propagation_options.propagation_key = "unindexed-header"; setupValidDriver(OpenTracingDriver::PropagationMode::TracerNative, propagation_options); - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); first_span->injectContext(request_headers_, nullptr); - Tracing::SpanPtr second_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr second_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); second_span->finishSpan(); first_span->finishSpan(); @@ -345,8 +346,8 @@ TEST_F(OpenTracingDriverTest, ExtractWithUnindexedHeader) { TEST_F(OpenTracingDriverTest, GetTraceId) { setupValidDriver(); - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); first_span->setTag("abc", "123"); first_span->finishSpan(); @@ -360,7 +361,7 @@ TEST_F(OpenTracingDriverTest, ExtractUsingForeach) { // Starting a new span, given the `request_headers_`, will visit the headers // using "for each." We can immediately discard the span. - driver_->startSpan(config_, request_headers_, operation_name_, start_time_, + driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); for (const auto& [key, value] : extracted_headers) { diff --git a/test/extensions/tracers/datadog/BUILD b/test/extensions/tracers/datadog/BUILD index bbd5b05870ac..d292aa9c0f39 100644 --- a/test/extensions/tracers/datadog/BUILD +++ b/test/extensions/tracers/datadog/BUILD @@ -49,6 +49,7 @@ envoy_extension_cc_test( "//test/mocks/server:tracer_factory_context_mocks", "//test/mocks/server:tracer_factory_mocks", "//test/mocks/stats:stats_mocks", + "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/tracing:tracing_mocks", "//test/mocks/upstream:cluster_manager_mocks", diff --git a/test/extensions/tracers/datadog/config_test.cc b/test/extensions/tracers/datadog/config_test.cc index 1ac54fa8e5dc..1a412108253e 100644 --- a/test/extensions/tracers/datadog/config_test.cc +++ b/test/extensions/tracers/datadog/config_test.cc @@ -82,9 +82,9 @@ class DatadogConfigTest : public testing::Test { const std::string operation_name_{"test"}; Http::TestRequestHeaderMapImpl request_headers_{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; - SystemTime start_time_; NiceMock tls_; + NiceMock stream_info_; NiceMock* timer_; Stats::TestUtil::TestStore stats_; NiceMock cm_; @@ -173,8 +173,8 @@ TEST_F(DatadogConfigTest, CollectorHostname) { return &request; })); - Tracing::SpanPtr span = tracer_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = tracer_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); span->finishSpan(); // Timer should be re-enabled. diff --git a/test/extensions/tracers/datadog/tracer_test.cc b/test/extensions/tracers/datadog/tracer_test.cc index a1e0d3e7cde7..1247b8a44f28 100644 --- a/test/extensions/tracers/datadog/tracer_test.cc +++ b/test/extensions/tracers/datadog/tracer_test.cc @@ -4,6 +4,7 @@ #include "source/extensions/tracers/datadog/span.h" #include "source/extensions/tracers/datadog/tracer.h" +#include "test/mocks/stream_info/mocks.h" #include "test/mocks/thread_local/mocks.h" #include "test/mocks/upstream/cluster_manager.h" #include "test/test_common/simulated_time_system.h" @@ -35,6 +36,7 @@ class DatadogTracerTest : public testing::Test { Stats::TestUtil::TestStore store_; NiceMock thread_local_slot_allocator_; Event::SimulatedTimeSystem time_; + NiceMock stream_info_; }; TEST_F(DatadogTracerTest, Breathing) { @@ -71,9 +73,10 @@ TEST_F(DatadogTracerTest, NoOpMode) { const std::string operation_name = "do.thing"; const SystemTime start = time_.timeSystem().systemTime(); + ON_CALL(stream_info_, startTime()).WillByDefault(testing::Return(start)); const Tracing::SpanPtr span = - tracer.startSpan(Tracing::MockConfig{}, context, operation_name, start, decision); + tracer.startSpan(Tracing::MockConfig{}, context, stream_info_, operation_name, decision); ASSERT_TRUE(span); const auto as_null_span = dynamic_cast(span.get()); EXPECT_NE(nullptr, as_null_span); @@ -100,9 +103,10 @@ TEST_F(DatadogTracerTest, SpanProperties) { const std::string operation_name = "do.thing"; const SystemTime start = time_.timeSystem().systemTime(); + ON_CALL(stream_info_, startTime()).WillByDefault(testing::Return(start)); const Tracing::SpanPtr span = - tracer.startSpan(Tracing::MockConfig{}, context, operation_name, start, decision); + tracer.startSpan(Tracing::MockConfig{}, context, stream_info_, operation_name, decision); ASSERT_TRUE(span); const auto as_dd_span_wrapper = dynamic_cast(span.get()); EXPECT_NE(nullptr, as_dd_span_wrapper); @@ -139,12 +143,14 @@ TEST_F(DatadogTracerTest, ExtractionSuccess) { const std::string operation_name = "do.thing"; const SystemTime start = time_.timeSystem().systemTime(); + ON_CALL(stream_info_, startTime()).WillByDefault(testing::Return(start)); + // trace context in the Datadog style Tracing::TestTraceContextImpl context{{"x-datadog-trace-id", "1234"}, {"x-datadog-parent-id", "5678"}}; const Tracing::SpanPtr span = - tracer.startSpan(Tracing::MockConfig{}, context, operation_name, start, decision); + tracer.startSpan(Tracing::MockConfig{}, context, stream_info_, operation_name, decision); ASSERT_TRUE(span); const auto as_dd_span_wrapper = dynamic_cast(span.get()); EXPECT_NE(nullptr, as_dd_span_wrapper); @@ -176,12 +182,14 @@ TEST_F(DatadogTracerTest, ExtractionFailure) { const std::string operation_name = "do.thing"; const SystemTime start = time_.timeSystem().systemTime(); + ON_CALL(stream_info_, startTime()).WillByDefault(testing::Return(start)); + // invalid trace context in the Datadog style Tracing::TestTraceContextImpl context{{"x-datadog-trace-id", "nope"}, {"x-datadog-parent-id", "nice try"}}; const Tracing::SpanPtr span = - tracer.startSpan(Tracing::MockConfig{}, context, operation_name, start, decision); + tracer.startSpan(Tracing::MockConfig{}, context, stream_info_, operation_name, decision); ASSERT_TRUE(span); const auto as_dd_span_wrapper = dynamic_cast(span.get()); EXPECT_NE(nullptr, as_dd_span_wrapper); diff --git a/test/extensions/tracers/dynamic_ot/BUILD b/test/extensions/tracers/dynamic_ot/BUILD index 97b99f2a8d4a..b982d22efa6a 100644 --- a/test/extensions/tracers/dynamic_ot/BUILD +++ b/test/extensions/tracers/dynamic_ot/BUILD @@ -27,6 +27,7 @@ envoy_extension_cc_test( "//source/extensions/tracers/dynamic_ot:dynamic_opentracing_driver_lib", "//test/mocks/http:http_mocks", "//test/mocks/stats:stats_mocks", + "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/tracing:tracing_mocks", "//test/test_common:environment_lib", ], diff --git a/test/extensions/tracers/dynamic_ot/dynamic_opentracing_driver_impl_test.cc b/test/extensions/tracers/dynamic_ot/dynamic_opentracing_driver_impl_test.cc index e6f4a9604dab..dc16e9434c45 100644 --- a/test/extensions/tracers/dynamic_ot/dynamic_opentracing_driver_impl_test.cc +++ b/test/extensions/tracers/dynamic_ot/dynamic_opentracing_driver_impl_test.cc @@ -5,6 +5,7 @@ #include "test/mocks/http/mocks.h" #include "test/mocks/stats/mocks.h" +#include "test/mocks/stream_info/mocks.h" #include "test/mocks/tracing/mocks.h" #include "test/test_common/environment.h" @@ -42,8 +43,9 @@ class DynamicOpenTracingDriverTest : public testing::Test { const std::string operation_name_{"test"}; Http::TestRequestHeaderMapImpl request_headers_{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; - SystemTime start_time_; + NiceMock config_; + NiceMock stream_info_; }; TEST_F(DynamicOpenTracingDriverTest, FormatErrorMessage) { @@ -77,8 +79,9 @@ TEST_F(DynamicOpenTracingDriverTest, FlushSpans) { setupValidDriver(); { - Tracing::SpanPtr first_span = driver_->startSpan( - config_, request_headers_, operation_name_, start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = + driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, + {Tracing::Reason::Sampling, true}); first_span->finishSpan(); driver_->tracer().Close(); } diff --git a/test/extensions/tracers/opencensus/BUILD b/test/extensions/tracers/opencensus/BUILD index 297918723790..64e7311cec58 100644 --- a/test/extensions/tracers/opencensus/BUILD +++ b/test/extensions/tracers/opencensus/BUILD @@ -21,6 +21,7 @@ envoy_extension_cc_test( "//source/extensions/tracers/opencensus:opencensus_tracer_impl", "//test/mocks/http:http_mocks", "//test/mocks/local_info:local_info_mocks", + "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/tracing:tracing_mocks", "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", ], diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index 10befb5373db..13eeffd2e7c7 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -12,6 +12,7 @@ #include "test/mocks/http/mocks.h" #include "test/mocks/local_info/mocks.h" +#include "test/mocks/stream_info/mocks.h" #include "test/mocks/tracing/mocks.h" #include "gmock/gmock.h" @@ -109,16 +110,17 @@ TEST(OpenCensusTracerTest, Span) { Http::TestRequestHeaderMapImpl request_headers{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; const std::string operation_name{"my_operation_1"}; - SystemTime start_time; + SystemTime fake_system_time; + NiceMock stream_info; { - Tracing::SpanPtr span = driver->startSpan(config, request_headers, operation_name, start_time, + Tracing::SpanPtr span = driver->startSpan(config, request_headers, stream_info, operation_name, {Tracing::Reason::Sampling, true}); span->setOperation("different_name"); span->setTag("my_key", "my_value"); - span->log(start_time, "my annotation"); + span->log(fake_system_time, "my annotation"); // injectContext is tested in another unit test. - Tracing::SpanPtr child = span->spawnChild(config, "child_span", start_time); + Tracing::SpanPtr child = span->spawnChild(config, "child_span", fake_system_time); child->finishSpan(); span->setSampled(false); // Abandon tracer. span->finishSpan(); @@ -211,10 +213,10 @@ void testIncomingHeaders( } const std::string operation_name{"my_operation_2"}; - SystemTime start_time; + NiceMock stream_info; Http::TestRequestHeaderMapImpl injected_headers; { - Tracing::SpanPtr span = driver->startSpan(config, request_headers, operation_name, start_time, + Tracing::SpanPtr span = driver->startSpan(config, request_headers, stream_info, operation_name, {Tracing::Reason::Sampling, false}); span->injectContext(injected_headers, nullptr); span->finishSpan(); diff --git a/test/extensions/tracers/opentelemetry/BUILD b/test/extensions/tracers/opentelemetry/BUILD index 46683827e06c..c1ac977443d6 100644 --- a/test/extensions/tracers/opentelemetry/BUILD +++ b/test/extensions/tracers/opentelemetry/BUILD @@ -34,6 +34,7 @@ envoy_extension_cc_test( "//test/mocks/runtime:runtime_mocks", "//test/mocks/server:tracer_factory_context_mocks", "//test/mocks/stats:stats_mocks", + "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/tracing:tracing_mocks", "//test/mocks/upstream:cluster_manager_mocks", diff --git a/test/extensions/tracers/opentelemetry/opentelemetry_tracer_impl_test.cc b/test/extensions/tracers/opentelemetry/opentelemetry_tracer_impl_test.cc index 87fb0672db0e..7a427a500955 100644 --- a/test/extensions/tracers/opentelemetry/opentelemetry_tracer_impl_test.cc +++ b/test/extensions/tracers/opentelemetry/opentelemetry_tracer_impl_test.cc @@ -8,6 +8,7 @@ #include "test/mocks/common.h" #include "test/mocks/server/tracer_factory_context.h" #include "test/mocks/stats/mocks.h" +#include "test/mocks/stream_info/mocks.h" #include "test/mocks/tracing/mocks.h" #include "test/test_common/utility.h" @@ -63,6 +64,7 @@ class OpenTelemetryDriverTest : public testing::Test { const std::string operation_name_{"test"}; NiceMock context_; NiceMock mock_tracing_config_; + NiceMock stream_info_; Event::SimulatedTimeSystem time_system_; std::unique_ptr> mock_stream_ptr_{nullptr}; envoy::config::trace::v3::OpenTelemetryConfig config_; @@ -107,9 +109,8 @@ TEST_F(OpenTelemetryDriverTest, ParseSpanContextFromHeadersTest) { context_.server_factory_context_.api_.random_; ON_CALL(mock_random_generator_, random()).WillByDefault(Return(new_span_id)); - Tracing::SpanPtr span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_EQ(span->getTraceIdAsHex(), trace_id_hex); @@ -146,6 +147,8 @@ TEST_F(OpenTelemetryDriverTest, ParseSpanContextFromHeadersTest) { )"; opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest request_proto; SystemTime timestamp = time_system_.systemTime(); + ON_CALL(stream_info_, startTime()).WillByDefault(Return(timestamp)); + int64_t timestamp_ns = std::chrono::nanoseconds(timestamp.time_since_epoch()).count(); TestUtility::loadFromYaml(fmt::format(request_yaml, timestamp_ns, timestamp_ns), request_proto); auto* expected_span = @@ -187,9 +190,8 @@ TEST_F(OpenTelemetryDriverTest, GenerateSpanContextWithoutHeadersTest) { EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(new_span_id)); } - Tracing::SpanPtr span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); // Remove headers, then inject context into header from the span. request_headers.remove(OpenTelemetryConstants::get().TRACE_PARENT); @@ -212,9 +214,8 @@ TEST_F(OpenTelemetryDriverTest, NullSpanWithPropagationHeaderError) { request_headers.addReferenceKey(OpenTelemetryConstants::get().TRACE_PARENT, "invalid00-0000000000000003-01"); - Tracing::SpanPtr span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); auto& null_span = *span; EXPECT_EQ(typeid(null_span).name(), typeid(Tracing::NullSpan).name()); @@ -226,9 +227,8 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpan) { Http::TestRequestHeaderMapImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; - Tracing::SpanPtr span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(span.get(), nullptr); // Test baggage noop and other noop calls. @@ -254,9 +254,8 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithBuffer) { Http::TestRequestHeaderMapImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; - Tracing::SpanPtr span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(span.get(), nullptr); // Flush after two spans. @@ -267,8 +266,8 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithBuffer) { span->finishSpan(); // Once we create a Tracing::SpanPtr second_span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, operation_name_, + {Tracing::Reason::Sampling, true}); EXPECT_NE(second_span.get(), nullptr); // Only now should we see the span exported. EXPECT_CALL(*mock_stream_ptr_, sendMessageRaw_(_, _)); @@ -287,9 +286,8 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithFlushTimeout) { Http::TestRequestHeaderMapImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; - Tracing::SpanPtr span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(span.get(), nullptr); // Set it to flush after 2 spans so that the span will only be flushed by timeout. @@ -329,9 +327,8 @@ TEST_F(OpenTelemetryDriverTest, SpawnChildSpan) { EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_span_id)); } - Tracing::SpanPtr span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(span.get(), nullptr); // The child should only generate a span ID for itself; the trace id should come from the parent.. @@ -359,9 +356,10 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithAttributes) { int64_t generated_int = 1; EXPECT_CALL(mock_random_generator_, random()).Times(3).WillRepeatedly(Return(generated_int)); SystemTime timestamp = time_system_.systemTime(); + ON_CALL(stream_info_, startTime()).WillByDefault(Return(timestamp)); - Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - timestamp, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(span.get(), nullptr); span->setTag("first_tag_name", "first_tag_value"); @@ -418,9 +416,8 @@ TEST_F(OpenTelemetryDriverTest, IgnoreNotSampledSpan) { setupValidDriver(); Http::TestRequestHeaderMapImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; - Tracing::SpanPtr span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(span.get(), nullptr); span->setSampled(false); @@ -440,9 +437,8 @@ TEST_F(OpenTelemetryDriverTest, NoExportWithoutGrpcService) { Http::TestRequestHeaderMapImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; - Tracing::SpanPtr span = - driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - time_system_.systemTime(), {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(span.get(), nullptr); // Flush after a single span. @@ -474,9 +470,10 @@ TEST_F(OpenTelemetryDriverTest, ExportSpanWithCustomServiceName) { int64_t generated_int = 1; EXPECT_CALL(mock_random_generator_, random()).Times(3).WillRepeatedly(Return(generated_int)); SystemTime timestamp = time_system_.systemTime(); + ON_CALL(stream_info_, startTime()).WillByDefault(Return(timestamp)); - Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, operation_name_, - timestamp, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(span.get(), nullptr); const std::string request_yaml = R"( diff --git a/test/extensions/tracers/skywalking/BUILD b/test/extensions/tracers/skywalking/BUILD index 555463226ebf..fcebfe3bee30 100644 --- a/test/extensions/tracers/skywalking/BUILD +++ b/test/extensions/tracers/skywalking/BUILD @@ -85,6 +85,7 @@ envoy_extension_cc_test( "//test/mocks/grpc:grpc_mocks", "//test/mocks/server:tracer_factory_context_mocks", "//test/mocks/stats:stats_mocks", + "//test/mocks/stream_info:stream_info_mocks", "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc b/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc index 7044f9b746ea..c6687af8e9cd 100644 --- a/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc +++ b/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc @@ -3,6 +3,7 @@ #include "test/extensions/tracers/skywalking/skywalking_test_helper.h" #include "test/mocks/common.h" #include "test/mocks/server/tracer_factory_context.h" +#include "test/mocks/stream_info/mocks.h" #include "test/mocks/tracing/mocks.h" #include "test/test_common/utility.h" @@ -49,7 +50,7 @@ class SkyWalkingDriverTest : public testing::Test { protected: NiceMock context_; NiceMock mock_tracing_config_; - Event::SimulatedTimeSystem time_system_; + NiceMock stream_info_; std::unique_ptr> mock_stream_ptr_{nullptr}; envoy::config::trace::v3::SkyWalkingConfig config_; std::string test_string = "ABCDEFGHIJKLMN"; @@ -83,8 +84,8 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { ON_CALL(mock_tracing_config_, operationName()) .WillByDefault(Return(Tracing::OperationName::Ingress)); - Tracing::SpanPtr org_span = driver_->startSpan(mock_tracing_config_, request_headers, "TEST_OP", - time_system_.systemTime(), decision); + Tracing::SpanPtr org_span = driver_->startSpan(mock_tracing_config_, request_headers, + stream_info_, "TEST_OP", decision); EXPECT_NE(nullptr, org_span.get()); Span* span = dynamic_cast(org_span.get()); @@ -112,8 +113,8 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { Http::TestRequestHeaderMapImpl new_request_headers{ {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}}; - Tracing::SpanPtr org_span = driver_->startSpan(mock_tracing_config_, new_request_headers, "", - time_system_.systemTime(), decision); + Tracing::SpanPtr org_span = + driver_->startSpan(mock_tracing_config_, new_request_headers, stream_info_, "", decision); Span* span = dynamic_cast(org_span.get()); ASSERT(span); @@ -137,7 +138,7 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { {":authority", "test.com"}, {"sw8", "xxxxxx-error-propagation-header"}}; Tracing::SpanPtr org_span = driver_->startSpan(mock_tracing_config_, error_request_headers, - "TEST_OP", time_system_.systemTime(), decision); + stream_info_, "TEST_OP", decision); Span* span = dynamic_cast(org_span.get()); ASSERT(span); @@ -160,7 +161,7 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { {":authority", "test.com"}, {"sw8", "1-x-x-x-x-x-x-x"}}; // The first field is legal and fourth field is illegal. Tracing::SpanPtr org_span = driver_->startSpan(mock_tracing_config_, error_request_headers, - "TEST_OP", time_system_.systemTime(), decision); + stream_info_, "TEST_OP", decision); Span* span = dynamic_cast(org_span.get()); ASSERT(span); @@ -181,8 +182,8 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { decision.traced = false; Http::TestRequestHeaderMapImpl request_headers{ {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}}; - Tracing::SpanPtr org_null_span = driver_->startSpan( - mock_tracing_config_, request_headers, "TEST_OP", time_system_.systemTime(), decision); + Tracing::SpanPtr org_null_span = driver_->startSpan(mock_tracing_config_, request_headers, + stream_info_, "TEST_OP", decision); EXPECT_EQ(nullptr, dynamic_cast(org_null_span.get())); @@ -198,9 +199,8 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { {":method", "GET"}, {":authority", "test.com"}, {"sw8", "xxxxxx-error-propagation-header"}}; - Tracing::SpanPtr org_null_span = - driver_->startSpan(mock_tracing_config_, error_request_headers, "TEST_OP", - time_system_.systemTime(), decision); + Tracing::SpanPtr org_null_span = driver_->startSpan(mock_tracing_config_, error_request_headers, + stream_info_, "TEST_OP", decision); EXPECT_EQ(nullptr, dynamic_cast(org_null_span.get())); @@ -216,9 +216,8 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { {":method", "GET"}, {":authority", "test.com"}, {"sw8", "1-x-x-x-x-x-x-x"}}; // The first field is legal and fourth field is illegal. - Tracing::SpanPtr org_null_span = - driver_->startSpan(mock_tracing_config_, error_request_headers, "TEST_OP", - time_system_.systemTime(), decision); + Tracing::SpanPtr org_null_span = driver_->startSpan(mock_tracing_config_, error_request_headers, + stream_info_, "TEST_OP", decision); EXPECT_EQ(nullptr, dynamic_cast(org_null_span.get())); @@ -243,8 +242,8 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestNoClientConfig) { Http::TestRequestHeaderMapImpl request_headers{ {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}}; - Tracing::SpanPtr org_span = driver_->startSpan(mock_tracing_config_, request_headers, "TEST_OP", - time_system_.systemTime(), decision); + Tracing::SpanPtr org_span = + driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, "TEST_OP", decision); EXPECT_NE(nullptr, org_span.get()); Span* span = dynamic_cast(org_span.get()); diff --git a/test/extensions/tracers/xray/BUILD b/test/extensions/tracers/xray/BUILD index f224ec02978a..fc87f0baa3c0 100644 --- a/test/extensions/tracers/xray/BUILD +++ b/test/extensions/tracers/xray/BUILD @@ -30,6 +30,7 @@ envoy_extension_cc_test( "//test/mocks/server:instance_mocks", "//test/mocks/server:tracer_factory_context_mocks", "//test/mocks/stats:stats_mocks", + "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/tracing:tracing_mocks", "//test/test_common:environment_lib", diff --git a/test/extensions/tracers/xray/xray_tracer_impl_test.cc b/test/extensions/tracers/xray/xray_tracer_impl_test.cc index e08f347c2eac..a4300d367107 100644 --- a/test/extensions/tracers/xray/xray_tracer_impl_test.cc +++ b/test/extensions/tracers/xray/xray_tracer_impl_test.cc @@ -5,6 +5,7 @@ #include "source/extensions/tracers/xray/xray_tracer_impl.h" #include "test/mocks/server/tracer_factory_context.h" +#include "test/mocks/stream_info/mocks.h" #include "test/mocks/thread_local/mocks.h" #include "test/mocks/tracing/mocks.h" #include "test/test_common/utility.h" @@ -21,7 +22,18 @@ namespace { class XRayDriverTest : public ::testing::Test { public: + XRayDriverTest() { + // To ensure the start_time_ is set to zero for each test to avoid flakiness. + stream_info_.start_time_ = Envoy::SystemTime{}; + // To ensure the monotonicTime is set to value larger than 1s for each test + // to avoid flakiness. + stream_info_.ts_.setMonotonicTime(std::chrono::seconds(2048)); + } + const std::string operation_name_ = "test_operation_name"; + // The MockStreamInfo will register the singleton time system to SimulatedTimeSystem and ignore + // the TestRealTimeSystem in the MockTracerFactoryContext. + NiceMock stream_info_; absl::flat_hash_map aws_metadata_; NiceMock context_; NiceMock tls_; @@ -38,8 +50,8 @@ TEST_F(XRayDriverTest, XRayTraceHeaderNotSampled) { Driver driver(config, context_); Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; - Envoy::SystemTime start_time; - auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + + auto span = driver.startSpan(tracing_config_, request_headers_, stream_info_, operation_name_, tracing_decision); ASSERT_NE(span, nullptr); auto* xray_span = static_cast(span.get()); @@ -54,8 +66,8 @@ TEST_F(XRayDriverTest, XRayTraceHeaderSampled) { Driver driver(config, context_); Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; - Envoy::SystemTime start_time; - auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + + auto span = driver.startSpan(tracing_config_, request_headers_, stream_info_, operation_name_, tracing_decision); ASSERT_NE(span, nullptr); } @@ -68,8 +80,8 @@ TEST_F(XRayDriverTest, XRayTraceHeaderSamplingUnknown) { Driver driver(config, context_); Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; - Envoy::SystemTime start_time; - auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + + auto span = driver.startSpan(tracing_config_, request_headers_, stream_info_, operation_name_, tracing_decision); // sampling should fall back to the default manifest since: // a) there is no valid sampling decision in the X-Ray header @@ -94,8 +106,8 @@ TEST_F(XRayDriverTest, XRayTraceHeaderWithoutSamplingDecision) { Driver driver(config, context_); Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; - Envoy::SystemTime start_time; - auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + + auto span = driver.startSpan(tracing_config_, request_headers_, stream_info_, operation_name_, tracing_decision); // sampling will not be done since: // a) there is no sampling decision in the X-Ray header @@ -111,8 +123,8 @@ TEST_F(XRayDriverTest, NoXRayTracerHeader) { Driver driver(config, context_); Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; - Envoy::SystemTime start_time; - auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + + auto span = driver.startSpan(tracing_config_, request_headers_, stream_info_, operation_name_, tracing_decision); // sampling should fall back to the default manifest since: // a) there is no X-Ray header to determine the sampling decision @@ -128,8 +140,8 @@ TEST_F(XRayDriverTest, XForwardedForHeaderSet) { Driver driver(config, context_); Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; - Envoy::SystemTime start_time; - auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + + auto span = driver.startSpan(tracing_config_, request_headers_, stream_info_, operation_name_, tracing_decision); ASSERT_NE(span, nullptr); @@ -144,8 +156,8 @@ TEST_F(XRayDriverTest, XForwardedForHeaderNotSet) { Driver driver(config, context_); Tracing::Decision tracing_decision{Tracing::Reason::Sampling, false /*sampled*/}; - Envoy::SystemTime start_time; - auto span = driver.startSpan(tracing_config_, request_headers_, operation_name_, start_time, + + auto span = driver.startSpan(tracing_config_, request_headers_, stream_info_, operation_name_, tracing_decision); ASSERT_NE(span, nullptr); diff --git a/test/extensions/tracers/zipkin/BUILD b/test/extensions/tracers/zipkin/BUILD index 1794401823df..68e5a6d8aad3 100644 --- a/test/extensions/tracers/zipkin/BUILD +++ b/test/extensions/tracers/zipkin/BUILD @@ -37,6 +37,7 @@ envoy_extension_cc_test( "//test/mocks/local_info:local_info_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/stats:stats_mocks", + "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/tracing:tracing_mocks", "//test/mocks/upstream:cluster_manager_mocks", diff --git a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc index c055e5afa58a..6dd8a6b1b093 100644 --- a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc @@ -17,6 +17,7 @@ #include "test/mocks/local_info/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/stats/mocks.h" +#include "test/mocks/stream_info/mocks.h" #include "test/mocks/thread_local/mocks.h" #include "test/mocks/tracing/mocks.h" #include "test/mocks/upstream/cluster_manager.h" @@ -112,12 +113,14 @@ class ZipkinDriverTest : public testing::Test { EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.request_timeout", 5000U)) .WillOnce(Return(5000U)); - Tracing::SpanPtr first_span = driver_->startSpan( - config_, request_headers_, operation_name_, start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr first_span = + driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, + {Tracing::Reason::Sampling, true}); first_span->finishSpan(); - Tracing::SpanPtr second_span = driver_->startSpan( - config_, request_headers_, operation_name_, start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr second_span = + driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, + {Tracing::Reason::Sampling, true}); second_span->finishSpan(); Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl( @@ -151,8 +154,7 @@ class ZipkinDriverTest : public testing::Test { const std::string operation_name_{"test"}; Http::TestRequestHeaderMapImpl request_headers_{ {":authority", "api.lyft.com"}, {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; - SystemTime start_time_; - StreamInfo::MockStreamInfo stream_info_; + NiceMock stream_info_; NiceMock tls_; std::unique_ptr driver_; @@ -256,8 +258,8 @@ TEST_F(ZipkinDriverTest, FlushOneSpanReportFailure) { EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.request_timeout", 5000U)) .WillOnce(Return(5000U)); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); span->finishSpan(); Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl( @@ -298,7 +300,7 @@ TEST_F(ZipkinDriverTest, SkipReportIfCollectorClusterHasBeenRemoved) { // Trigger flush of a span. driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, + ->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}) ->finishSpan(); @@ -322,7 +324,7 @@ TEST_F(ZipkinDriverTest, SkipReportIfCollectorClusterHasBeenRemoved) { // Trigger flush of a span. driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, + ->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}) ->finishSpan(); @@ -348,7 +350,7 @@ TEST_F(ZipkinDriverTest, SkipReportIfCollectorClusterHasBeenRemoved) { // Trigger flush of a span. driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, + ->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}) ->finishSpan(); @@ -377,7 +379,7 @@ TEST_F(ZipkinDriverTest, SkipReportIfCollectorClusterHasBeenRemoved) { // Trigger flush of a span. driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, + ->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}) ->finishSpan(); @@ -421,22 +423,22 @@ TEST_F(ZipkinDriverTest, CancelInflightRequestsOnDestruction) { // Trigger 1st report request. driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, + ->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}) ->finishSpan(); // Trigger 2nd report request. driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, + ->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}) ->finishSpan(); // Trigger 3rd report request. driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, + ->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}) ->finishSpan(); // Trigger 4th report request. driver_ - ->startSpan(config_, request_headers_, operation_name_, start_time_, + ->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}) ->finishSpan(); @@ -467,8 +469,8 @@ TEST_F(ZipkinDriverTest, FlushSpansTimer) { EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.min_flush_spans", 5)) .WillOnce(Return(5)); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); span->finishSpan(); // Timer should be re-enabled. @@ -491,8 +493,8 @@ TEST_F(ZipkinDriverTest, NoB3ContextSampledTrue) { EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID).empty()); EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED).empty()); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); ZipkinSpanPtr zipkin_span(dynamic_cast(span.release())); EXPECT_TRUE(zipkin_span->span().sampled()); @@ -505,8 +507,8 @@ TEST_F(ZipkinDriverTest, NoB3ContextSampledFalse) { EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID).empty()); EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED).empty()); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, false}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, false}); ZipkinSpanPtr zipkin_span(dynamic_cast(span.release())); EXPECT_FALSE(zipkin_span->span().sampled()); @@ -521,8 +523,8 @@ TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleTrue) { Hex::uint64ToHex(generateRandom64())); EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED).empty()); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); ZipkinSpanPtr zipkin_span(dynamic_cast(span.release())); EXPECT_TRUE(zipkin_span->span().sampled()); @@ -537,8 +539,8 @@ TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleFalse) { Hex::uint64ToHex(generateRandom64())); EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED).empty()); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, false}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, false}); ZipkinSpanPtr zipkin_span(dynamic_cast(span.release())); EXPECT_FALSE(zipkin_span->span().sampled()); @@ -552,8 +554,8 @@ TEST_F(ZipkinDriverTest, PropagateB3NotSampled) { // Only context header set is B3 sampled to indicate trace should not be sampled request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, NOT_SAMPLED); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED); @@ -575,8 +577,8 @@ TEST_F(ZipkinDriverTest, PropagateB3NotSampledWithFalse) { // 'false' value) const std::string sampled = "false"; request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, sampled); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED); @@ -597,8 +599,8 @@ TEST_F(ZipkinDriverTest, PropagateB3SampledWithTrue) { // 'true' value) const std::string sampled = "true"; request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, sampled); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, false}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, false}); request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED); @@ -618,8 +620,8 @@ TEST_F(ZipkinDriverTest, PropagateB3SampleFalse) { Hex::uint64ToHex(generateRandom64())); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, NOT_SAMPLED); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); ZipkinSpanPtr zipkin_span(dynamic_cast(span.release())); EXPECT_FALSE(zipkin_span->span().sampled()); @@ -635,8 +637,8 @@ TEST_F(ZipkinDriverTest, ZipkinSpanTest) { request_headers_.remove(Http::CustomHeaders::get().OtSpanContext); // New span will have a CS annotation - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); ZipkinSpanPtr zipkin_span(dynamic_cast(span.release())); zipkin_span->setTag("key", "value"); @@ -658,8 +660,8 @@ TEST_F(ZipkinDriverTest, ZipkinSpanTest) { request_headers_.setCopy(Http::CustomHeaders::get().OtSpanContext, context); // New span will have an SR annotation - Tracing::SpanPtr span2 = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span2 = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); ZipkinSpanPtr zipkin_span2(dynamic_cast(span2.release())); zipkin_span2->setTag("key2", "value2"); @@ -672,8 +674,8 @@ TEST_F(ZipkinDriverTest, ZipkinSpanTest) { // ==== // Test setTag() with empty annotations vector // ==== - Tracing::SpanPtr span3 = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span3 = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); ZipkinSpanPtr zipkin_span3(dynamic_cast(span3.release())); Span& zipkin_zipkin_span3 = zipkin_span3->span(); @@ -689,8 +691,8 @@ TEST_F(ZipkinDriverTest, ZipkinSpanTest) { // Test effective log() // ==== - Tracing::SpanPtr span4 = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span4 = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); const auto timestamp = SystemTime{std::chrono::duration_cast(std::chrono::hours{123})}; const auto timestamp_count = @@ -706,16 +708,16 @@ TEST_F(ZipkinDriverTest, ZipkinSpanTest) { // ==== // Test baggage noop // ==== - Tracing::SpanPtr span5 = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span5 = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); span5->setBaggage("baggage_key", "baggage_value"); EXPECT_EQ("", span5->getBaggage("baggage_key")); // ==== // Test trace id noop // ==== - Tracing::SpanPtr span6 = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span6 = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_EQ(span6->getTraceIdAsHex(), ""); } @@ -732,8 +734,8 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersTest) { // New span will have an SR annotation - so its span and parent ids will be // the same as the supplied span context (i.e. shared context) - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); ZipkinSpanPtr zipkin_span(dynamic_cast(span.release())); @@ -756,8 +758,8 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersEmptyParentSpanTest) { const std::string parent_span_id = ""; request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, parent_span_id); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); ZipkinSpanPtr zipkin_span(dynamic_cast(span.release())); EXPECT_TRUE(zipkin_span->span().sampled()); @@ -778,8 +780,8 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3Headers128TraceIdTest) { // New span will have an SR annotation - so its span and parent ids will be // the same as the supplied span context (i.e. shared context) - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); ZipkinSpanPtr zipkin_span(dynamic_cast(span.release())); @@ -800,8 +802,8 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidTraceIdB3HeadersTest) { request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, Hex::uint64ToHex(generateRandom64())); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(nullptr, dynamic_cast(span.get())); } @@ -814,8 +816,8 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidSpanIdB3HeadersTest) { request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, Hex::uint64ToHex(generateRandom64())); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(nullptr, dynamic_cast(span.get())); } @@ -829,16 +831,16 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidParentIdB3HeadersTest) { request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, std::string("xyz")); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_NE(nullptr, dynamic_cast(span.get())); } TEST_F(ZipkinDriverTest, ExplicitlySetSampledFalse) { setupValidDriver("HTTP_JSON"); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); span->setSampled(false); @@ -854,8 +856,8 @@ TEST_F(ZipkinDriverTest, ExplicitlySetSampledFalse) { TEST_F(ZipkinDriverTest, ExplicitlySetSampledTrue) { setupValidDriver("HTTP_JSON"); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, false}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, false}); span->setSampled(true); @@ -876,8 +878,8 @@ TEST_F(ZipkinDriverTest, DuplicatedHeader) { Hex::uint64ToHex(generateRandom64())); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, Hex::uint64ToHex(generateRandom64())); - Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, false}); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, + operation_name_, {Tracing::Reason::Sampling, false}); using DupCallback = std::function; DupCallback dup_callback = [](absl::string_view key) -> bool { diff --git a/test/mocks/tracing/mocks.h b/test/mocks/tracing/mocks.h index 26b310c5c1bb..73e170248bb9 100644 --- a/test/mocks/tracing/mocks.h +++ b/test/mocks/tracing/mocks.h @@ -76,14 +76,16 @@ class MockDriver : public Driver { ~MockDriver() override; SpanPtr startSpan(const Config& config, TraceContext& trace_context, - const std::string& operation_name, SystemTime start_time, + const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, const Tracing::Decision tracing_decision) override { - return SpanPtr{startSpan_(config, trace_context, operation_name, start_time, tracing_decision)}; + return SpanPtr{ + startSpan_(config, trace_context, stream_info, operation_name, tracing_decision)}; } MOCK_METHOD(Span*, startSpan_, - (const Config& config, TraceContext& trace_context, const std::string& operation_name, - SystemTime start_time, const Tracing::Decision tracing_decision)); + (const Config& config, TraceContext& trace_context, + const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, + const Tracing::Decision tracing_decision)); }; class MockTracerManager : public TracerManager { From 43831ce58771c4e88ebb436e0d11ae9315432a87 Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Tue, 2 May 2023 13:30:41 -0400 Subject: [PATCH 077/740] mobile: Fixes the StatFlushIntegrationTest test (#27095) This commit makes sure the CountDownLatch is set up *before* flushStats(), but await() is called *after* flushStats(), so that we don't miss any updates from the TestStatsdServer and we don't block on a condition that never gets triggered. Fixes #24657 Signed-off-by: Ali Beyad --- .../kotlin/integration/StatFlushIntegrationTest.kt | 13 ++++++------- mobile/test/kotlin/integration/TestStatsdServer.kt | 14 +++++++++++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt b/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt index 733cd816272c..ad449a434d7c 100644 --- a/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt +++ b/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt @@ -40,9 +40,6 @@ class StatFlushIntegrationTest { assertThat(countDownLatch.await(30, TimeUnit.SECONDS)).isTrue() } - // TODO(abeyad): Fix the test and re-enable. - // See https://github.com/envoyproxy/envoy/issues/24657. - @Ignore @Test fun `flush flushes to stats sink`() { val countDownLatch = CountDownLatch(1) @@ -56,18 +53,20 @@ class StatFlushIntegrationTest { assertThat(countDownLatch.await(30, TimeUnit.SECONDS)).isTrue() - engine!!.pulseClient().counter(Element("foo"), Element("bar")).increment(1) - val statsdServer1 = TestStatsdServer() statsdServer1.runAsync(8125) val statsdServer2 = TestStatsdServer() statsdServer2.runAsync(5000) + statsdServer1.setStatMatching { s -> s == "envoy.pulse.foo.bar:1|c" } + statsdServer2.setStatMatching { s -> s == "envoy.pulse.foo.bar:1|c" } + + engine!!.pulseClient().counter(Element("foo"), Element("bar")).increment(1) engine!!.flushStats() - statsdServer1.awaitStatMatching { s -> s == "envoy.pulse.foo.bar:1|c" } - statsdServer2.awaitStatMatching { s -> s == "envoy.pulse.foo.bar:1|c" } + statsdServer1.awaitStatMatching() + statsdServer2.awaitStatMatching() } private fun statsdSinkConfig(port: Int): String { diff --git a/mobile/test/kotlin/integration/TestStatsdServer.kt b/mobile/test/kotlin/integration/TestStatsdServer.kt index 573be70aa245..9e380e4e4c14 100644 --- a/mobile/test/kotlin/integration/TestStatsdServer.kt +++ b/mobile/test/kotlin/integration/TestStatsdServer.kt @@ -13,6 +13,7 @@ class TestStatsdServer { private val matchCriteria: AtomicReference<(String) -> Boolean> = AtomicReference() private var thread: Thread? = null + private var latch: CountDownLatch? = null @Throws(IOException::class) fun runAsync(port: Int) { @@ -48,11 +49,18 @@ class TestStatsdServer { thread!!.start() } - fun awaitStatMatching(predicate: (String) -> Boolean) { - val latch = CountDownLatch(1) + // Creates the notification condition based on the input predicate. + // Callers of this method must call awaitStatMatching() to wait for the condition to be met on + // the server. + fun setStatMatching(predicate: (String) -> Boolean) { + latch = CountDownLatch(1) awaitNextStat.set(latch) matchCriteria.set(predicate) - latch.await() + } + + // setStatMatching() must be called before calling this method. + fun awaitStatMatching() { + latch?.await() } @Throws(InterruptedException::class) From 80a2d90d32db0acb0488ee8174325c3269e42ff6 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 2 May 2023 19:22:59 +0100 Subject: [PATCH 078/740] deps: Bump `sphinx` -> 7.0.0 (#27077) Signed-off-by: Ryan Northey --- tools/base/requirements.in | 1 + tools/base/requirements.txt | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 66ca2c322174..4e3cc6c11406 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -27,6 +27,7 @@ pyreadline pyyaml setuptools slackclient +sphinx>=7 thrift verboselogs yapf diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index fa6c98056653..60a6c8567166 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -1007,10 +1007,11 @@ snowballstemmer==2.2.0 \ --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a # via sphinx -sphinx==6.2.0 \ - --hash=sha256:9ef22c2941bc3d0ff080d25a797f7521fc317e857395c712ddde97a19d5bb440 \ - --hash=sha256:ff1c2a1167bef9cdcd8ec71339e85fe10f26d4e9ef9382ef10b2687c876c936b +sphinx==7.0.0 \ + --hash=sha256:283c44aa28922bb4223777b44ac0d59af50a279ac7690dfe945bb2b9575dc41b \ + --hash=sha256:3cfc1c6756ef1b132687b813ec6ea2214cb7a7e5d1dcb2772006cb895a0fa469 # via + # -r requirements.in # envoy-docs-sphinx-runner # sphinx-copybutton # sphinxcontrib-httpdomain From 4db0b0ceb66002a0855cd7ef8fbc04f56b8d80b7 Mon Sep 17 00:00:00 2001 From: danzh Date: Tue, 2 May 2023 14:54:14 -0400 Subject: [PATCH 079/740] mobile: install Android application context at start up (#27093) Signed-off-by: Dan Zhang Co-authored-by: Dan Zhang --- .../io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java | 4 ++++ .../io/envoyproxy/envoymobile/utilities/FakeX509Util.java | 5 +++++ .../envoymobile/utilities/CertificateVerificationTest.java | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java index 10c27be03f0b..3fa756ad184e 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java @@ -8,6 +8,7 @@ import io.envoyproxy.envoymobile.engine.types.EnvoyOnEngineRunning; import io.envoyproxy.envoymobile.engine.types.EnvoyStringAccessor; import io.envoyproxy.envoymobile.engine.types.EnvoyStatus; +import io.envoyproxy.envoymobile.utilities.ContextUtils; import java.util.Map; @@ -22,6 +23,9 @@ public AndroidEngineImpl(Context context, EnvoyOnEngineRunning runningCallback, EnvoyLogger logger, EnvoyEventTracker eventTracker, Boolean enableProxying) { this.envoyEngine = new EnvoyEngineImpl(runningCallback, logger, eventTracker); + if (ContextUtils.getApplicationContext() == null) { + ContextUtils.initApplicationContext(context.getApplicationContext()); + } AndroidJniLibrary.load(context); AndroidNetworkMonitor.load(context, envoyEngine); if (enableProxying) { diff --git a/mobile/library/java/io/envoyproxy/envoymobile/utilities/FakeX509Util.java b/mobile/library/java/io/envoyproxy/envoymobile/utilities/FakeX509Util.java index b04c9910edce..42db7a5e4f66 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/utilities/FakeX509Util.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/utilities/FakeX509Util.java @@ -45,6 +45,11 @@ public static AndroidCertVerifyResult verifyServerCertificates(byte[][] certChai + "chain passed as |certChain|. |certChain|=" + Arrays.deepToString(certChain)); } + if (ContextUtils.getApplicationContext() == null) { + throw new NullPointerException( + "ContextUtils is not initialized with a proper context. Call initApplicationContext() during startup."); + } + for (byte[] cert : certChain) { String fakeCert = new String(cert); if (!validFakeCerts.contains(fakeCert)) { diff --git a/mobile/test/java/io/envoyproxy/envoymobile/utilities/CertificateVerificationTest.java b/mobile/test/java/io/envoyproxy/envoymobile/utilities/CertificateVerificationTest.java index d26b97e233cd..5bf7b4137eb6 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/utilities/CertificateVerificationTest.java +++ b/mobile/test/java/io/envoyproxy/envoymobile/utilities/CertificateVerificationTest.java @@ -5,6 +5,7 @@ import android.content.Context; import io.envoyproxy.envoymobile.engine.AndroidJniLibrary; import io.envoyproxy.envoymobile.engine.JniLibrary; +import io.envoyproxy.envoymobile.utilities.ContextUtils; import androidx.test.platform.app.InstrumentationRegistry; import java.nio.charset.StandardCharsets; @@ -25,6 +26,9 @@ public final class CertificateVerificationTest { static { AndroidJniLibrary.loadTestLibrary(); Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + if (ContextUtils.getApplicationContext() == null) { + ContextUtils.initApplicationContext(context.getApplicationContext()); + } AndroidJniLibrary.load(context.getApplicationContext()); } From ffa156e21ac933cfa4d5c27ac1393d55692d36e5 Mon Sep 17 00:00:00 2001 From: Ryan Hamilton Date: Tue, 2 May 2023 12:09:04 -0700 Subject: [PATCH 080/740] Update QUICHE from d2d9e7bcc to 32d62793d (#27061) * Update QUICHE from d2d9e7bcc to 32d62793d https://github.com/google/quiche/compare/d2d9e7bcc..32d62793d ``` $ git log d2d9e7bcc..32d62793d --date=short --no-merges --format="%ad %al %s" 2023-04-27 quiche-dev Add ability to set a non-zero send alarm delay for QUIC connection. 2023-04-27 bnc Add comment to QPACK encoder. 2023-04-27 vasilvv More build fixes for QUICHE on Windows. 2023-04-26 vasilvv Add a Windows implementation of socket.h 2023-04-26 vasilvv Basic fixes to make QUICHE build on Windows. 2023-04-25 haoyuewang Deprecate --gfe2_reloadable_flag_quic_remove_connection_migration_connection_option_v2 2023-04-25 bnc Remove extra warnings from internal QUICHE platform implementation. 2023-04-24 renjietang Add stats on QuicConnection path degrading and recovering. 2023-04-24 wub Deprecate --gfe2_reloadable_flag_quic_dont_close_connection_in_tls_alert_callback. 2023-04-22 quiche-dev When a region override is in effect, enable debug mode. Otherwise, Beryllium will reject it when it does the country mismatch check. 2023-04-21 danzh Deprecate --gfe2_reloadable_flag_quic_verify_request_headers_2 ``` Signed-off-by: Ryan Hamilton --- bazel/external/quiche.BUILD | 8 ++++---- bazel/repository_locations.bzl | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index af486ef5e6ab..df09b393695c 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -3484,13 +3484,13 @@ envoy_cc_library( envoy_cc_library( name = "quic_core_io_socket_lib", - srcs = select({ - "@envoy//bazel:windows_x86_64": [], - "//conditions:default": ["quiche/quic/core/io/socket_posix.cc"], - }), + srcs = ["quiche/quic/core/io/socket.cc"], hdrs = [ "quiche/quic/core/connecting_client_socket.h", "quiche/quic/core/io/socket.h", + "quiche/quic/core/io/socket_internal.h", + "quiche/quic/core/io/socket_posix.inc", + "quiche/quic/core/io/socket_win.inc", "quiche/quic/core/socket_factory.h", ], copts = quiche_copts, diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 94c00e973cbb..2bf42e8df83a 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1054,12 +1054,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "d2d9e7bcc0d196773d736bd022fcc9d7cbf1b316", - sha256 = "66cfc69edaf4aa3c5819a922f0ac61688ef5ad6358d99c06e67a1f5611e35f63", + version = "32d62793deb3e7d3c77df57615ecd0ab4ab42859", + sha256 = "2aabe1824f7821cbf3ac6e05e23a571836ca141a243d6a2ab41e6f0037d6f370", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2023-04-20", + release_date = "2023-04-27", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", From 0c8fc2b585a4e3abbe9685fb774c362cfef196f7 Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Tue, 2 May 2023 15:44:27 -0400 Subject: [PATCH 081/740] Cleanup: Cleanup clang tidy errors in fuzzer (#27124) * Fuzzer should follow style guide for members and abide clang tidy. Signed-off-by: Kevin Baichoo --- .../http/http2/http2_connection_fuzz_test.cc | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/test/common/http/http2/http2_connection_fuzz_test.cc b/test/common/http/http2/http2_connection_fuzz_test.cc index 3d6e8a8a5105..603585e01b2b 100644 --- a/test/common/http/http2/http2_connection_fuzz_test.cc +++ b/test/common/http/http2/http2_connection_fuzz_test.cc @@ -23,8 +23,8 @@ namespace { static const uint32_t MAX_HEADERS = 32; static const size_t MAX_HD_TABLE_BUF_SIZE = 4096; -void deflate_headers(nghttp2_hd_deflater* deflater, const test::fuzz::Headers& fragment, - std::string& payload) { +void deflateHeaders(nghttp2_hd_deflater* deflater, const test::fuzz::Headers& fragment, + std::string& payload) { #define MAX_HDRS 32 nghttp2_nv h2hdrs[MAX_HEADERS]; uint8_t buf[MAX_HD_TABLE_BUF_SIZE]; @@ -56,8 +56,8 @@ void deflate_headers(nghttp2_hd_deflater* deflater, const test::fuzz::Headers& f } } -Http2Frame pb_to_h2_frame(nghttp2_hd_deflater* deflater, - const test::common::http::http2::Http2FrameOrJunk& pb_frame) { +Http2Frame pbToH2Frame(nghttp2_hd_deflater* deflater, + const test::common::http::http2::Http2FrameOrJunk& pb_frame) { if (pb_frame.has_junk()) { // Junk frame const auto& junk = pb_frame.junk(); @@ -91,7 +91,7 @@ Http2Frame pb_to_h2_frame(nghttp2_hd_deflater* deflater, payload.append(reinterpret_cast(&weight), sizeof(weight)); } - deflate_headers(deflater, f.headers(), payload); + deflateHeaders(deflater, f.headers(), payload); if (use_padding) { payload.append(f.padding().data(), padding_len); } @@ -124,7 +124,7 @@ Http2Frame pb_to_h2_frame(nghttp2_hd_deflater* deflater, } uint32_t stream_id = htonl(f.stream_id()); payload.append(reinterpret_cast(&stream_id), sizeof(stream_id)); - deflate_headers(deflater, f.headers(), payload); + deflateHeaders(deflater, f.headers(), payload); if (use_padding) { payload.append(f.padding().data(), padding_len); } @@ -149,15 +149,15 @@ Http2Frame pb_to_h2_frame(nghttp2_hd_deflater* deflater, } else if (h2frame.has_continuation()) { type = Http2Frame::Type::Continuation; const auto& f = h2frame.continuation(); - deflate_headers(deflater, f.headers(), payload); + deflateHeaders(deflater, f.headers(), payload); } else { // Empty frame - return Http2Frame(); + return {}; } return Http2Frame::makeRawFrame(type, flags, streamid, payload); } else { // Empty frame - return Http2Frame(); + return {}; } } @@ -173,23 +173,23 @@ envoy::config::core::v3::Http2ProtocolOptions http2Settings() { class Http2Harness { public: - Http2Harness() : server_settings(http2Settings()) { - ON_CALL(mock_server_callbacks, newStream(_, _)) + Http2Harness() : server_settings_(http2Settings()) { + ON_CALL(mock_server_callbacks_, newStream(_, _)) .WillByDefault(Invoke( - [&](ResponseEncoder&, bool) -> RequestDecoder& { return orphan_request_decoder; })); + [&](ResponseEncoder&, bool) -> RequestDecoder& { return orphan_request_decoder_; })); } - void fuzz_request(std::vector& frames, bool use_oghttp2) { + void fuzzRequest(std::vector& frames, bool use_oghttp2) { TestScopedRuntime scoped_runtime; if (use_oghttp2) { scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "true"}}); } else { scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "false"}}); } - Stats::Scope& scope = *stats_store.rootScope(); + Stats::Scope& scope = *stats_store_.rootScope(); server_ = std::make_unique( - mock_server_connection, mock_server_callbacks, - Http2::CodecStats::atomicGet(http2_stats, scope), random, server_settings, + mock_server_connection_, mock_server_callbacks_, + Http2::CodecStats::atomicGet(http2_stats_, scope), random_, server_settings_, Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT, envoy::config::core::v3::HttpProtocolOptions::ALLOW); @@ -204,20 +204,20 @@ class Http2Harness { } private: - const envoy::config::core::v3::Http2ProtocolOptions server_settings; - Stats::IsolatedStoreImpl stats_store; - Http2::CodecStats::AtomicPtr http2_stats; + const envoy::config::core::v3::Http2ProtocolOptions server_settings_; + Stats::IsolatedStoreImpl stats_store_; + Http2::CodecStats::AtomicPtr http2_stats_; - NiceMock orphan_request_decoder; - NiceMock mock_server_connection; - NiceMock mock_server_callbacks; - NiceMock random; + NiceMock orphan_request_decoder_; + NiceMock mock_server_connection_; + NiceMock mock_server_callbacks_; + NiceMock random_; ServerConnectionPtr server_; }; static std::unique_ptr harness; -static void reset_harness() { harness = nullptr; } +static void resetHarness() { harness = nullptr; } // HTTP/2 fuzzer // @@ -226,7 +226,7 @@ static void reset_harness() { harness = nullptr; } DEFINE_PROTO_FUZZER(const test::common::http::http2::Http2ConnectionFuzzCase& input) { if (harness == nullptr) { harness = std::make_unique(); - atexit(reset_harness); + atexit(resetHarness); } // Convert input to Http2Frame wire format @@ -240,13 +240,13 @@ DEFINE_PROTO_FUZZER(const test::common::http::http2::Http2ConnectionFuzzCase& in return; } for (auto& pbframe : input.frames()) { - Http2Frame frame = pb_to_h2_frame(deflater, pbframe); + Http2Frame frame = pbToH2Frame(deflater, pbframe); frames.push_back(frame); } nghttp2_hd_deflate_del(deflater); - harness->fuzz_request(frames, true /* use oghttp2 rather than nghttp2 */); - harness->fuzz_request(frames, false); + harness->fuzzRequest(frames, true /* use oghttp2 rather than nghttp2 */); + harness->fuzzRequest(frames, false); } } // namespace From 30e37ba34bd5d565ffbb75a600e7cdd4ae33d8ed Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 2 May 2023 16:09:57 -0400 Subject: [PATCH 082/740] test: hopefully fixing a tsan flake (#27119) Signed-off-by: Alyssa Wilk --- test/integration/idle_timeout_integration_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/idle_timeout_integration_test.cc b/test/integration/idle_timeout_integration_test.cc index a11a90e41a75..3bd8c0371282 100644 --- a/test/integration/idle_timeout_integration_test.cc +++ b/test/integration/idle_timeout_integration_test.cc @@ -118,7 +118,7 @@ class IdleTimeoutIntegrationTest : public HttpProtocolIntegrationTest { // TODO(htuch): This might require scaling for TSAN/ASAN/Valgrind/etc. Bump if // this is the cause of flakes. - static constexpr uint64_t IdleTimeoutMs = 400; + static constexpr uint64_t IdleTimeoutMs = 300 * TSAN_TIMEOUT_FACTOR; static constexpr uint64_t RequestTimeoutMs = 200; bool enable_global_idle_timeout_{false}; bool enable_per_stream_idle_timeout_{false}; From 49965dfd9eb884c35b9047c59945383b75568ded Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Tue, 2 May 2023 16:32:32 -0400 Subject: [PATCH 083/740] Harmonize UHV interface names (#27121) Signed-off-by: Yan Avlasov --- envoy/http/header_validator.h | 18 ++++++++---------- source/common/http/conn_manager_config.h | 2 +- source/common/http/conn_manager_impl.cc | 2 +- source/common/http/conn_manager_impl.h | 2 +- .../network/http_connection_manager/config.h | 3 ++- .../envoy_default/header_validator.cc | 6 +++--- .../envoy_default/header_validator.h | 2 +- .../envoy_default/header_validator_factory.cc | 2 +- .../envoy_default/header_validator_factory.h | 2 +- .../envoy_default/http1_header_validator.cc | 6 +++--- .../envoy_default/http1_header_validator.h | 10 +++++----- .../envoy_default/http2_header_validator.cc | 8 ++++---- .../envoy_default/http2_header_validator.h | 10 +++++----- source/server/admin/admin.h | 2 +- test/common/http/codec_client_test.cc | 8 ++++---- .../common/http/conn_manager_impl_fuzz_test.cc | 2 +- test/common/http/conn_manager_impl_test_2.cc | 18 ++++++++++-------- .../common/http/conn_manager_impl_test_base.cc | 17 +++++++++-------- test/common/http/conn_manager_impl_test_base.h | 8 ++++---- test/common/http/http1/codec_impl_test.cc | 8 ++++---- test/common/http/http2/codec_impl_test.cc | 6 +++--- test/common/quic/envoy_quic_utils_test.cc | 12 ++++++------ .../http_connection_manager/config_test.cc | 4 ++-- .../header_validator_factory_test.cc | 2 +- .../envoy_default/header_validator_test.cc | 4 ++-- .../http1_header_validator_test.cc | 5 +++-- .../http2_header_validator_test.cc | 7 ++++--- .../http_common_validation_test.cc | 2 +- test/integration/fake_upstream.cc | 2 +- test/integration/fake_upstream.h | 4 ++-- test/mocks/http/header_validator.h | 6 +++--- test/mocks/http/mocks.h | 2 +- 32 files changed, 98 insertions(+), 94 deletions(-) diff --git a/envoy/http/header_validator.h b/envoy/http/header_validator.h index b79380c513a9..1d082d8d83ef 100644 --- a/envoy/http/header_validator.h +++ b/envoy/http/header_validator.h @@ -11,11 +11,10 @@ namespace Http { /** * Common interface for server and client header validators. - * TODO(yanavlasov): rename interfaces in the next PR to `HeaderValidator` */ -class HeaderValidatorBase { +class HeaderValidator { public: - virtual ~HeaderValidatorBase() = default; + virtual ~HeaderValidator() = default; // A class that holds either success condition or an error condition with tuple of // action and error details. @@ -78,11 +77,10 @@ class HeaderValidatorBase { /** * Interface for server header validators. - * TODO(yanavlasov): rename interfaces in the next PR to `ServerHeaderValidator` */ -class HeaderValidator : public HeaderValidatorBase { +class ServerHeaderValidator : public HeaderValidator { public: - ~HeaderValidator() override = default; + ~ServerHeaderValidator() override = default; /** * Transform the entire request header map. @@ -127,7 +125,7 @@ class HeaderValidator : public HeaderValidatorBase { /** * Interface for server header validators. */ -class ClientHeaderValidator : public HeaderValidatorBase { +class ClientHeaderValidator : public HeaderValidator { public: ~ClientHeaderValidator() override = default; @@ -161,7 +159,7 @@ class ClientHeaderValidator : public HeaderValidatorBase { virtual TransformationResult transformResponseHeaders(ResponseHeaderMap& header_map) PURE; }; -using HeaderValidatorPtr = std::unique_ptr; +using ServerHeaderValidatorPtr = std::unique_ptr; using ClientHeaderValidatorPtr = std::unique_ptr; /** @@ -187,8 +185,8 @@ class HeaderValidatorFactory { /** * Create a new header validator for the specified protocol. */ - virtual HeaderValidatorPtr createServerHeaderValidator(Protocol protocol, - HeaderValidatorStats& stats) PURE; + virtual ServerHeaderValidatorPtr createServerHeaderValidator(Protocol protocol, + HeaderValidatorStats& stats) PURE; virtual ClientHeaderValidatorPtr createClientHeaderValidator(Protocol protocol, HeaderValidatorStats& stats) PURE; }; diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 2f63a913485a..d6d23cf2a419 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -513,7 +513,7 @@ class ConnectionManagerConfig { * @return pointer to the header validator. * If nullptr, header validation will not be done. */ - virtual HeaderValidatorPtr makeHeaderValidator(Protocol protocol) PURE; + virtual ServerHeaderValidatorPtr makeHeaderValidator(Protocol protocol) PURE; /** * @return whether to append the x-forwarded-port header. diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index d9b968ae239c..f33b7b5d3d2c 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -948,7 +948,7 @@ bool ConnectionManagerImpl::ActiveStream::validateHeaders() { auto transformation_result = header_validator_->transformRequestHeaders(*request_headers_); failure = !transformation_result.ok(); redirect = transformation_result.action() == - Http::HeaderValidator::RequestHeadersTransformationResult::Action::Redirect; + Http::ServerHeaderValidator::RequestHeadersTransformationResult::Action::Redirect; failure_details = std::string(transformation_result.details()); } if (failure) { diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 2a7b19cd112c..8250bd6cb1be 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -491,7 +491,7 @@ class ConnectionManagerImpl : Logger::Loggable, const std::string* decorated_operation_{nullptr}; std::unique_ptr route_config_update_requester_; std::unique_ptr tracing_custom_tags_{nullptr}; - Http::HeaderValidatorPtr header_validator_; + Http::ServerHeaderValidatorPtr header_validator_; friend FilterManager; diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index ec5e43cd0279..0f3e014dd8d9 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -243,7 +243,8 @@ class HttpConnectionManagerConfig : Logger::Loggable, const HttpConnectionManagerProto::ProxyStatusConfig* proxyStatusConfig() const override { return proxy_status_config_.get(); } - Http::HeaderValidatorPtr makeHeaderValidator([[maybe_unused]] Http::Protocol protocol) override { + Http::ServerHeaderValidatorPtr + makeHeaderValidator([[maybe_unused]] Http::Protocol protocol) override { #ifdef ENVOY_ENABLE_UHV return header_validator_factory_ ? header_validator_factory_->createServerHeaderValidator( protocol, getHeaderValidatorStats(protocol)) diff --git a/source/extensions/http/header_validators/envoy_default/header_validator.cc b/source/extensions/http/header_validators/envoy_default/header_validator.cc index e9a43ae62f40..d1f3e62976b9 100644 --- a/source/extensions/http/header_validators/envoy_default/header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/header_validator.cc @@ -518,7 +518,7 @@ HeaderValidator::HeaderEntryValidationResult HeaderValidator::validateGenericReq // For H/1 the codec will never produce H/2 pseudo headers and per // https://www.rfc-editor.org/rfc/rfc9110#section-6.5 there are no other prohibitions. // As a result this common function can cover trailer validation for all protocols. -::Envoy::Http::HeaderValidatorBase::ValidationResult +::Envoy::Http::HeaderValidator::ValidationResult HeaderValidator::validateTrailers(const ::Envoy::Http::HeaderMap& trailers) { std::string reject_details; trailers.iterate([this, &reject_details](const ::Envoy::Http::HeaderEntry& header_entry) @@ -541,10 +541,10 @@ HeaderValidator::validateTrailers(const ::Envoy::Http::HeaderMap& trailers) { }); if (!reject_details.empty()) { - return {::Envoy::Http::HeaderValidatorBase::ValidationResult::Action::Reject, reject_details}; + return {::Envoy::Http::HeaderValidator::ValidationResult::Action::Reject, reject_details}; } - return ::Envoy::Http::HeaderValidatorBase::ValidationResult::success(); + return ::Envoy::Http::HeaderValidator::ValidationResult::success(); } void HeaderValidator::sanitizeHeadersWithUnderscores(::Envoy::Http::HeaderMap& header_map) { diff --git a/source/extensions/http/header_validators/envoy_default/header_validator.h b/source/extensions/http/header_validators/envoy_default/header_validator.h index 0861574a3c48..949931fb99ab 100644 --- a/source/extensions/http/header_validators/envoy_default/header_validator.h +++ b/source/extensions/http/header_validators/envoy_default/header_validator.h @@ -89,7 +89,7 @@ class HeaderValidator { */ class HostHeaderValidationResult { public: - using RejectAction = ::Envoy::Http::HeaderValidatorBase::RejectAction; + using RejectAction = ::Envoy::Http::HeaderValidator::RejectAction; HostHeaderValidationResult(RejectAction action, absl::string_view details, absl::string_view address, absl::string_view port) : result_(action, details, address, port) { diff --git a/source/extensions/http/header_validators/envoy_default/header_validator_factory.cc b/source/extensions/http/header_validators/envoy_default/header_validator_factory.cc index 6ab000b85358..7b0d3e78d684 100644 --- a/source/extensions/http/header_validators/envoy_default/header_validator_factory.cc +++ b/source/extensions/http/header_validators/envoy_default/header_validator_factory.cc @@ -15,7 +15,7 @@ using ::Envoy::Http::Protocol; HeaderValidatorFactory::HeaderValidatorFactory(const HeaderValidatorConfig& config) : config_(config) {} -::Envoy::Http::HeaderValidatorPtr +::Envoy::Http::ServerHeaderValidatorPtr HeaderValidatorFactory::createServerHeaderValidator(Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats) { switch (protocol) { diff --git a/source/extensions/http/header_validators/envoy_default/header_validator_factory.h b/source/extensions/http/header_validators/envoy_default/header_validator_factory.h index 2fc2ebf72349..b14a07ae378f 100644 --- a/source/extensions/http/header_validators/envoy_default/header_validator_factory.h +++ b/source/extensions/http/header_validators/envoy_default/header_validator_factory.h @@ -15,7 +15,7 @@ class HeaderValidatorFactory : public ::Envoy::Http::HeaderValidatorFactory { const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& config); - ::Envoy::Http::HeaderValidatorPtr + ::Envoy::Http::ServerHeaderValidatorPtr createServerHeaderValidator(::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats) override; diff --git a/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc b/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc index c8d821926994..bc512e77a064 100644 --- a/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc @@ -22,7 +22,7 @@ using ::Envoy::Http::LowerCaseString; using ::Envoy::Http::Protocol; using ::Envoy::Http::RequestHeaderMap; using ::Envoy::Http::UhvResponseCodeDetail; -using ValidationResult = ::Envoy::Http::HeaderValidatorBase::ValidationResult; +using ValidationResult = ::Envoy::Http::HeaderValidator::ValidationResult; struct Http1ResponseCodeDetailValues { const std::string InvalidTransferEncoding = "uhv.http1.invalid_transfer_encoding"; @@ -295,7 +295,7 @@ void Http1HeaderValidator::sanitizeContentLength(::Envoy::Http::RequestHeaderMap } } -::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult +::Envoy::Http::ServerHeaderValidator::RequestHeadersTransformationResult ServerHttp1HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) { sanitizeContentLength(header_map); sanitizeHeadersWithUnderscores(header_map); @@ -305,7 +305,7 @@ ServerHttp1HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeader return path_result; } } - return ::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult::success(); + return ::Envoy::Http::ServerHeaderValidator::RequestHeadersTransformationResult::success(); } ValidationResult diff --git a/source/extensions/http/header_validators/envoy_default/http1_header_validator.h b/source/extensions/http/header_validators/envoy_default/http1_header_validator.h index 183666f0d108..9134619d0800 100644 --- a/source/extensions/http/header_validators/envoy_default/http1_header_validator.h +++ b/source/extensions/http/header_validators/envoy_default/http1_header_validator.h @@ -15,16 +15,16 @@ class Http1HeaderValidator : public HeaderValidator { config, ::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats); - ::Envoy::Http::HeaderValidatorBase::ValidationResult + ::Envoy::Http::HeaderValidator::ValidationResult validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map); - ::Envoy::Http::HeaderValidatorBase::ValidationResult + ::Envoy::Http::HeaderValidator::ValidationResult validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map); - ::Envoy::Http::HeaderValidatorBase::ValidationResult + ::Envoy::Http::HeaderValidator::ValidationResult validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map); - ::Envoy::Http::HeaderValidatorBase::ValidationResult + ::Envoy::Http::HeaderValidator::ValidationResult validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map); protected: @@ -56,7 +56,7 @@ class Http1HeaderValidator : public HeaderValidator { }; class ServerHttp1HeaderValidator : public Http1HeaderValidator, - public ::Envoy::Http::HeaderValidator { + public ::Envoy::Http::ServerHeaderValidator { public: ServerHttp1HeaderValidator( const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& diff --git a/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc b/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc index 49922360ff83..6643b98b2a0d 100644 --- a/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc @@ -26,7 +26,7 @@ using ::Envoy::Http::HeaderUtility; using ::Envoy::Http::Protocol; using ::Envoy::Http::testCharInTable; using ::Envoy::Http::UhvResponseCodeDetail; -using ValidationResult = ::Envoy::Http::HeaderValidatorBase::ValidationResult; +using ValidationResult = ::Envoy::Http::HeaderValidator::ValidationResult; struct Http2ResponseCodeDetailValues { const std::string InvalidTE = "uhv.http2.invalid_te"; @@ -425,7 +425,7 @@ ValidationResult Http2HeaderValidator::validateResponseTrailers( return result; } -::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult +::Envoy::Http::ServerHeaderValidator::RequestHeadersTransformationResult ServerHttp2HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeaderMap& header_map) { sanitizeHeadersWithUnderscores(header_map); if (!config_.uri_path_normalization_options().skip_path_normalization()) { @@ -440,10 +440,10 @@ ServerHttp2HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeader if (::Envoy::Http::Utility::isH2UpgradeRequest(header_map)) { ::Envoy::Http::Utility::transformUpgradeRequestFromH2toH1(header_map); } - return ::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult::success(); + return ::Envoy::Http::ServerHeaderValidator::RequestHeadersTransformationResult::success(); } -::Envoy::Http::HeaderValidator::ResponseHeadersTransformationResult +::Envoy::Http::ServerHeaderValidator::ResponseHeadersTransformationResult ServerHttp2HeaderValidator::transformResponseHeaders( const ::Envoy::Http::ResponseHeaderMap& header_map) { // Check if the response is for the the H/1 UPGRADE and transform it to the H/2 extended CONNECT diff --git a/source/extensions/http/header_validators/envoy_default/http2_header_validator.h b/source/extensions/http/header_validators/envoy_default/http2_header_validator.h index 8f32f10c13e2..361b200c920d 100644 --- a/source/extensions/http/header_validators/envoy_default/http2_header_validator.h +++ b/source/extensions/http/header_validators/envoy_default/http2_header_validator.h @@ -15,16 +15,16 @@ class Http2HeaderValidator : public HeaderValidator { config, ::Envoy::Http::Protocol protocol, ::Envoy::Http::HeaderValidatorStats& stats); - ::Envoy::Http::HeaderValidatorBase::ValidationResult + ::Envoy::Http::HeaderValidator::ValidationResult validateRequestHeaders(const ::Envoy::Http::RequestHeaderMap& header_map); - ::Envoy::Http::HeaderValidatorBase::ValidationResult + ::Envoy::Http::HeaderValidator::ValidationResult validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map); - ::Envoy::Http::HeaderValidatorBase::ValidationResult + ::Envoy::Http::HeaderValidator::ValidationResult validateRequestTrailers(const ::Envoy::Http::RequestTrailerMap& trailer_map); - ::Envoy::Http::HeaderValidatorBase::ValidationResult + ::Envoy::Http::HeaderValidator::ValidationResult validateResponseTrailers(const ::Envoy::Http::ResponseTrailerMap& trailer_map); private: @@ -56,7 +56,7 @@ class Http2HeaderValidator : public HeaderValidator { }; class ServerHttp2HeaderValidator : public Http2HeaderValidator, - public ::Envoy::Http::HeaderValidator { + public ::Envoy::Http::ServerHeaderValidator { public: ServerHttp2HeaderValidator( const envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig& diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 3045a916767f..00589f2526c5 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -217,7 +217,7 @@ class AdminImpl : public Admin, const HttpConnectionManagerProto::ProxyStatusConfig* proxyStatusConfig() const override { return proxy_status_config_.get(); } - Http::HeaderValidatorPtr makeHeaderValidator(Http::Protocol) override { + Http::ServerHeaderValidatorPtr makeHeaderValidator(Http::Protocol) override { // TODO(yanavlasov): admin interface should use the default validator return nullptr; } diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index ffb60f2a19e1..2c37efa2fac5 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -389,8 +389,8 @@ TEST_F(CodecClientTest, ResponseHeaderValidationFails) { EXPECT_OK(request_encoder.encodeHeaders(request_headers, true)); ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; EXPECT_CALL(*header_validator_, validateResponseHeaders(_)) - .WillOnce(Return(HeaderValidatorBase::ValidationResult{ - HeaderValidatorBase::ValidationResult::Action::Reject, "some error"})); + .WillOnce(Return(HeaderValidator::ValidationResult{ + HeaderValidator::ValidationResult::Action::Reject, "some error"})); // Invalid response should cause stream reset EXPECT_CALL(inner_encoder.stream_, resetStream(StreamResetReason::ProtocolError)); inner_decoder->decodeHeaders(std::move(response_headers), true); @@ -444,8 +444,8 @@ TEST_F(CodecClientTest, ResponseHeaderValidationFailsWithConnectionClosure) { EXPECT_OK(request_encoder.encodeHeaders(request_headers, true)); ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; EXPECT_CALL(*header_validator_, validateResponseHeaders(_)) - .WillOnce(Return(HeaderValidatorBase::ValidationResult{ - HeaderValidatorBase::ValidationResult::Action::Reject, "some error"})); + .WillOnce(Return(HeaderValidator::ValidationResult{ + HeaderValidator::ValidationResult::Action::Reject, "some error"})); // By default H/2 and H/3 connections are disconnected on protocol errors EXPECT_CALL(*connection_, close(_)); inner_decoder->decodeHeaders(std::move(response_headers), true); diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index a4e7de439763..f88dd9c62881 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -225,7 +225,7 @@ class FuzzConfig : public ConnectionManagerConfig { const HttpConnectionManagerProto::ProxyStatusConfig* proxyStatusConfig() const override { return proxy_status_config_.get(); } - Http::HeaderValidatorPtr makeHeaderValidator(Protocol) override { + Http::ServerHeaderValidatorPtr makeHeaderValidator(Protocol) override { // TODO(yanavlasov): fuzz test interface should use the default validator, although this could // be changed too return nullptr; diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index 18ed7b9799e3..ca9bfa2d1699 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -3355,7 +3355,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectHttp1) { setup(false, ""); expectUhvHeaderCheck(HeaderValidator::ValidationResult( HeaderValidator::ValidationResult::Action::Reject, "bad_header_map"), - HeaderValidator::RequestHeadersTransformationResult::success()); + ServerHeaderValidator::RequestHeadersTransformationResult::success()); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3401,7 +3401,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectHttp2) { setup(false, ""); expectUhvHeaderCheck(HeaderValidator::ValidationResult( HeaderValidator::ValidationResult::Action::Reject, "bad_header_map"), - HeaderValidator::RequestHeadersTransformationResult::success()); + ServerHeaderValidator::RequestHeadersTransformationResult::success()); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3430,7 +3430,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectGrpcRequest) { setup(false, ""); expectUhvHeaderCheck(HeaderValidator::ValidationResult( HeaderValidator::ValidationResult::Action::Reject, "bad_header_map"), - HeaderValidator::RequestHeadersTransformationResult::success()); + ServerHeaderValidator::RequestHeadersTransformationResult::success()); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3460,8 +3460,9 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRedirect) { setup(false, ""); expectUhvHeaderCheck( HeaderValidator::ValidationResult::success(), - HeaderValidator::RequestHeadersTransformationResult( - HeaderValidator::RequestHeadersTransformationResult::Action::Redirect, "bad_header_map")); + ServerHeaderValidator::RequestHeadersTransformationResult( + ServerHeaderValidator::RequestHeadersTransformationResult::Action::Redirect, + "bad_header_map")); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3489,8 +3490,9 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRedirectGrpcRequest) { setup(false, ""); expectUhvHeaderCheck( HeaderValidator::ValidationResult::success(), - HeaderValidator::RequestHeadersTransformationResult( - HeaderValidator::RequestHeadersTransformationResult::Action::Redirect, "bad_header_map")); + ServerHeaderValidator::RequestHeadersTransformationResult( + ServerHeaderValidator::RequestHeadersTransformationResult::Action::Redirect, + "bad_header_map")); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); @@ -3644,7 +3646,7 @@ TEST_F(HttpConnectionManagerImplTest, HeaderValidatorRejectTrailersAfterResponse TEST_F(HttpConnectionManagerImplTest, HeaderValidatorAccept) { setup(false, ""); expectUhvHeaderCheck(HeaderValidator::ValidationResult::success(), - HeaderValidator::RequestHeadersTransformationResult::success()); + ServerHeaderValidator::RequestHeadersTransformationResult::success()); // Store the basic request encoder during filter chain setup. std::shared_ptr filter(new NiceMock()); diff --git a/test/common/http/conn_manager_impl_test_base.cc b/test/common/http/conn_manager_impl_test_base.cc index bbbcd836b855..5d203bc53db0 100644 --- a/test/common/http/conn_manager_impl_test_base.cc +++ b/test/common/http/conn_manager_impl_test_base.cc @@ -311,10 +311,10 @@ void HttpConnectionManagerImplMixin::testPathNormalization( void HttpConnectionManagerImplMixin::expectUhvHeaderCheck( HeaderValidator::ValidationResult validation_result, - HeaderValidator::RequestHeadersTransformationResult transformation_result) { + ServerHeaderValidator::RequestHeadersTransformationResult transformation_result) { EXPECT_CALL(header_validator_factory_, createServerHeaderValidator(codec_->protocol_, _)) .WillOnce(InvokeWithoutArgs([validation_result, transformation_result]() { - auto header_validator = std::make_unique>(); + auto header_validator = std::make_unique>(); EXPECT_CALL(*header_validator, validateRequestHeaders(_)) .WillOnce(InvokeWithoutArgs([validation_result]() { return validation_result; })); @@ -322,7 +322,7 @@ void HttpConnectionManagerImplMixin::expectUhvHeaderCheck( EXPECT_CALL(*header_validator, transformRequestHeaders(_)) .WillOnce(Invoke([transformation_result](RequestHeaderMap& headers) { if (transformation_result.action() == - HeaderValidator::RequestHeadersTransformationResult::Action::Redirect) { + ServerHeaderValidator::RequestHeadersTransformationResult::Action::Redirect) { headers.setPath("/some/new/path"); } return transformation_result; @@ -330,8 +330,9 @@ void HttpConnectionManagerImplMixin::expectUhvHeaderCheck( } EXPECT_CALL(*header_validator, transformResponseHeaders(_)) - .WillOnce(InvokeWithoutArgs( - []() { return HeaderValidator::ResponseHeadersTransformationResult::success(); })); + .WillOnce(InvokeWithoutArgs([]() { + return ServerHeaderValidator::ResponseHeadersTransformationResult::success(); + })); return header_validator; })); @@ -342,13 +343,13 @@ void HttpConnectionManagerImplMixin::expectUhvTrailerCheck( HeaderValidator::TransformationResult transformation_result, bool expect_response) { EXPECT_CALL(header_validator_factory_, createServerHeaderValidator(codec_->protocol_, _)) .WillOnce(InvokeWithoutArgs([validation_result, transformation_result, expect_response]() { - auto header_validator = std::make_unique>(); + auto header_validator = std::make_unique>(); EXPECT_CALL(*header_validator, validateRequestHeaders(_)).WillOnce(InvokeWithoutArgs([]() { return HeaderValidator::ValidationResult::success(); })); EXPECT_CALL(*header_validator, transformRequestHeaders(_)).WillOnce(InvokeWithoutArgs([]() { - return HeaderValidator::RequestHeadersTransformationResult::success(); + return ServerHeaderValidator::RequestHeadersTransformationResult::success(); })); EXPECT_CALL(*header_validator, validateRequestTrailers(_)) @@ -361,7 +362,7 @@ void HttpConnectionManagerImplMixin::expectUhvTrailerCheck( if (expect_response) { EXPECT_CALL(*header_validator, transformResponseHeaders(_)) .WillOnce(InvokeWithoutArgs([]() { - return HeaderValidator::ResponseHeadersTransformationResult::success(); + return ServerHeaderValidator::ResponseHeadersTransformationResult::success(); })); } return header_validator; diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index f69ea07afbf3..92f0da9eeb64 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -159,7 +159,7 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { const HttpConnectionManagerProto::ProxyStatusConfig* proxyStatusConfig() const override { return proxy_status_config_.get(); } - HeaderValidatorPtr makeHeaderValidator(Protocol protocol) override { + ServerHeaderValidatorPtr makeHeaderValidator(Protocol protocol) override { return header_validator_factory_.createServerHeaderValidator(protocol, header_validator_stats_); } bool appendXForwardedPort() const override { return false; } @@ -186,9 +186,9 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { callbacks.addAccessLogHandler(handler); }; } - void - expectUhvHeaderCheck(HeaderValidatorBase::ValidationResult validation_result, - HeaderValidator::RequestHeadersTransformationResult transformation_result); + void expectUhvHeaderCheck( + HeaderValidator::ValidationResult validation_result, + ServerHeaderValidator::RequestHeadersTransformationResult transformation_result); void expectUhvTrailerCheck(HeaderValidator::ValidationResult validation_result, HeaderValidator::TransformationResult transformation_result, bool expect_response = true); diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 227b498f1c9a..06d24d86b8b6 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -93,14 +93,14 @@ struct HTTPStringTestCase { // validation fails it calls the `sendLocalReply` on the decoder, indicating validation error. class MockRequestDecoderShimWithUhv : public Http::MockRequestDecoder { public: - MockRequestDecoderShimWithUhv(Http::HeaderValidator* header_validator, + MockRequestDecoderShimWithUhv(Http::ServerHeaderValidator* header_validator, Network::MockConnection& connection) : header_validator_(header_validator), connection_(connection) {} void setResponseEncoder(Http::ResponseEncoder* response_encoder) { response_encoder_ = response_encoder; } - void setHeaderValidator(Http::HeaderValidator* header_validator) { + void setHeaderValidator(Http::ServerHeaderValidator* header_validator) { header_validator_ = header_validator; } void decodeHeaders(Http::RequestHeaderMapSharedPtr&& headers, bool end_stream) override { @@ -129,7 +129,7 @@ class MockRequestDecoderShimWithUhv : public Http::MockRequestDecoder { } private: - Http::HeaderValidator* header_validator_; + Http::ServerHeaderValidator* header_validator_; Network::MockConnection& connection_; Http::ResponseEncoder* response_encoder_{nullptr}; }; @@ -232,7 +232,7 @@ class Http1ServerConnectionImplTest : public Http1CodecTestBase { headers_with_underscores_action_{envoy::config::core::v3::HttpProtocolOptions::ALLOW}; envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig header_validator_config_; - HeaderValidatorPtr header_validator_; + ServerHeaderValidatorPtr header_validator_; }; void Http1ServerConnectionImplTest::expect400(Buffer::OwnedImpl& buffer, diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index b610485321a3..778dbddae095 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -75,7 +75,7 @@ class MockRequestDecoderShimWithUhv : public Http::MockRequestDecoder { void setResponseEncoder(Http::ResponseEncoder* response_encoder) { response_encoder_ = response_encoder; } - void setHeaderValidator(Http::HeaderValidator* header_validator) { + void setHeaderValidator(Http::ServerHeaderValidator* header_validator) { header_validator_ = header_validator; } void decodeHeaders(Http::RequestHeaderMapSharedPtr&& headers, bool end_stream) override { @@ -106,7 +106,7 @@ class MockRequestDecoderShimWithUhv : public Http::MockRequestDecoder { } private: - Http::HeaderValidator* header_validator_{nullptr}; + Http::ServerHeaderValidator* header_validator_{nullptr}; Http::ResponseEncoder* response_encoder_{nullptr}; }; } // namespace @@ -457,7 +457,7 @@ class Http2CodecImplTestFixture { headers_with_underscores_action_{envoy::config::core::v3::HttpProtocolOptions::ALLOW}; envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig header_validator_config_; - HeaderValidatorPtr header_validator_; + ServerHeaderValidatorPtr header_validator_; }; class Http2CodecImplTest : public ::testing::TestWithParam, diff --git a/test/common/quic/envoy_quic_utils_test.cc b/test/common/quic/envoy_quic_utils_test.cc index a2bb050f3e50..ea634b27a489 100644 --- a/test/common/quic/envoy_quic_utils_test.cc +++ b/test/common/quic/envoy_quic_utils_test.cc @@ -36,9 +36,9 @@ TEST(EnvoyQuicUtilsTest, ConversionBetweenQuicAddressAndEnvoyAddress) { EXPECT_FALSE(envoyIpAddressToQuicSocketAddress(nullptr).IsInitialized()); } -class MockHeaderValidator : public HeaderValidator { +class MockServerHeaderValidator : public HeaderValidator { public: - ~MockHeaderValidator() override = default; + ~MockServerHeaderValidator() override = default; MOCK_METHOD(Http::HeaderUtility::HeaderValidationResult, validateHeader, (absl::string_view header_name, absl::string_view header_value)); }; @@ -55,7 +55,7 @@ TEST(EnvoyQuicUtilsTest, HeadersConversion) { headers_block.AppendValueOrAddHeader("key1", "value1"); headers_block.AppendValueOrAddHeader("key1", ""); headers_block.AppendValueOrAddHeader("key1", "value2"); - NiceMock validator; + NiceMock validator; absl::string_view details; quic::QuicRstStreamErrorCode rst = quic::QUIC_REFUSED_STREAM; auto envoy_headers = http2HeaderBlockToEnvoyTrailers( @@ -126,7 +126,7 @@ TEST(EnvoyQuicUtilsTest, HeadersSizeBounds) { headers_block["foo"] = std::string("bar\0eep\0baz", 11); absl::string_view details; // 6 headers are allowed. - NiceMock validator; + NiceMock validator; quic::QuicRstStreamErrorCode rst = quic::QUIC_REFUSED_STREAM; EXPECT_NE(nullptr, http2HeaderBlockToEnvoyTrailers( headers_block, 6, validator, details, rst)); @@ -146,7 +146,7 @@ TEST(EnvoyQuicUtilsTest, TrailersSizeBounds) { headers_block[":scheme"] = "https"; headers_block["foo"] = std::string("bar\0eep\0baz", 11); absl::string_view details; - NiceMock validator; + NiceMock validator; quic::QuicRstStreamErrorCode rst = quic::QUIC_REFUSED_STREAM; EXPECT_NE(nullptr, http2HeaderBlockToEnvoyTrailers( headers_block, 6, validator, details, rst)); @@ -164,7 +164,7 @@ TEST(EnvoyQuicUtilsTest, TrailerCharacters) { headers_block[":path"] = "/index.hml"; headers_block[":scheme"] = "https"; absl::string_view details; - NiceMock validator; + NiceMock validator; EXPECT_CALL(validator, validateHeader(_, _)) .WillRepeatedly(Return(Http::HeaderUtility::HeaderValidationResult::REJECT)); quic::QuicRstStreamErrorCode rst = quic::QUIC_REFUSED_STREAM; diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 8e547b6a69e7..f3a2dfb4874c 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -2967,7 +2967,7 @@ class TestHeaderValidatorFactoryConfig : public Http::HeaderValidatorFactoryConf auto header_validator = std::make_unique>(); EXPECT_CALL(*header_validator, createServerHeaderValidator(Http::Protocol::Http2, _)) .WillOnce(InvokeWithoutArgs( - []() { return std::make_unique>(); })); + []() { return std::make_unique>(); })); return header_validator; } }; @@ -3002,7 +3002,7 @@ class DefaultHeaderValidatorFactoryConfigOverride : public Http::HeaderValidator auto header_validator = std::make_unique>(); EXPECT_CALL(*header_validator, createServerHeaderValidator(Http::Protocol::Http2, _)) .WillOnce(InvokeWithoutArgs( - []() { return std::make_unique>(); })); + []() { return std::make_unique>(); })); return header_validator; } diff --git a/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc b/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc index e7b3cc7796fc..9831a345cf72 100644 --- a/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc +++ b/test/extensions/http/header_validators/envoy_default/header_validator_factory_test.cc @@ -23,7 +23,7 @@ using ::testing::NiceMock; class HeaderValidatorFactoryTest : public testing::Test { protected: - ::Envoy::Http::HeaderValidatorPtr create(absl::string_view config_yaml, Protocol protocol) { + ::Envoy::Http::ServerHeaderValidatorPtr create(absl::string_view config_yaml, Protocol protocol) { auto* factory = Registry::FactoryRegistry::getFactory( "envoy.http.header_validators.envoy_default"); diff --git a/test/extensions/http/header_validators/envoy_default/header_validator_test.cc b/test/extensions/http/header_validators/envoy_default/header_validator_test.cc index c0da2124c55f..331569056e68 100644 --- a/test/extensions/http/header_validators/envoy_default/header_validator_test.cc +++ b/test/extensions/http/header_validators/envoy_default/header_validator_test.cc @@ -18,11 +18,11 @@ using ::Envoy::Http::Protocol; using ::Envoy::Http::testCharInTable; using ::Envoy::Http::UhvResponseCodeDetail; -using HeaderValidatorPtr = std::unique_ptr; +using ServerHeaderValidatorPtr = std::unique_ptr; class BaseHeaderValidatorTest : public HeaderValidatorTest, public testing::Test { protected: - HeaderValidatorPtr createBase(absl::string_view config_yaml) { + ServerHeaderValidatorPtr createBase(absl::string_view config_yaml) { envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig typed_config; TestUtility::loadFromYaml(std::string(config_yaml), typed_config); diff --git a/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc b/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc index 9e0f19a27dd7..ba3a38119dad 100644 --- a/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc @@ -595,8 +595,9 @@ TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderMapRedirectPath) { auto uhv = createH1(redirect_encoded_slash_config); EXPECT_TRUE(uhv->validateRequestHeaders(headers).ok()); auto result = uhv->transformRequestHeaders(headers); - EXPECT_EQ(result.action(), - ::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult::Action::Redirect); + EXPECT_EQ( + result.action(), + ::Envoy::Http::ServerHeaderValidator::RequestHeadersTransformationResult::Action::Redirect); EXPECT_EQ(result.details(), "uhv.path_noramlization_redirect"); EXPECT_EQ(headers.path(), "/dir1/dir2"); } diff --git a/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc b/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc index ca63ff94f2bd..2d8caf231d2b 100644 --- a/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc @@ -25,7 +25,7 @@ using ::Envoy::Http::UhvResponseCodeDetail; class Http2HeaderValidatorTest : public HeaderValidatorTest, public testing::Test { protected: - ::Envoy::Http::HeaderValidatorPtr createH2ServerUhv(absl::string_view config_yaml) { + ::Envoy::Http::ServerHeaderValidatorPtr createH2ServerUhv(absl::string_view config_yaml) { envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig typed_config; TestUtility::loadFromYaml(std::string(config_yaml), typed_config); @@ -689,8 +689,9 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapRedirectPath) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); // Path normalization should result in redirect auto result = uhv->transformRequestHeaders(headers); - EXPECT_EQ(result.action(), - ::Envoy::Http::HeaderValidator::RequestHeadersTransformationResult::Action::Redirect); + EXPECT_EQ( + result.action(), + ::Envoy::Http::ServerHeaderValidator::RequestHeadersTransformationResult::Action::Redirect); EXPECT_EQ(result.details(), "uhv.path_noramlization_redirect"); EXPECT_EQ(headers.path(), "/dir1/dir2"); } diff --git a/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc b/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc index 33d384d0920c..bf74898df8aa 100644 --- a/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc @@ -21,7 +21,7 @@ using ::Envoy::Http::TestRequestHeaderMapImpl; class HttpCommonValidationTest : public HeaderValidatorTest, public testing::TestWithParam { protected: - ::Envoy::Http::HeaderValidatorPtr createUhv(absl::string_view config_yaml) { + ::Envoy::Http::ServerHeaderValidatorPtr createUhv(absl::string_view config_yaml) { envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig typed_config; TestUtility::loadFromYaml(std::string(config_yaml), typed_config); diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 82f38fd2f591..35c73f6d24a7 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -441,7 +441,7 @@ Http::Protocol codeTypeToProtocol(Http::CodecType codec_type) { } } // namespace -Http::HeaderValidatorPtr FakeHttpConnection::makeHeaderValidator() { +Http::ServerHeaderValidatorPtr FakeHttpConnection::makeHeaderValidator() { return header_validator_factory_ ? header_validator_factory_->createServerHeaderValidator( codeTypeToProtocol(type_), header_validator_stats_) : nullptr; diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index c5b915f64bb9..0b2c2cc15715 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -267,7 +267,7 @@ class FakeStream : public Http::RequestDecoder, std::list access_log_handlers_; bool received_data_{false}; bool grpc_stream_started_{false}; - Http::HeaderValidatorPtr header_validator_; + Http::ServerHeaderValidatorPtr header_validator_; }; using FakeStreamPtr = std::unique_ptr; @@ -496,7 +496,7 @@ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeCo void writeRawData(absl::string_view data); ABSL_MUST_USE_RESULT AssertionResult postWriteRawData(std::string data); - Http::HeaderValidatorPtr makeHeaderValidator(); + Http::ServerHeaderValidatorPtr makeHeaderValidator(); private: struct ReadFilter : public Network::ReadFilterBaseImpl { diff --git a/test/mocks/http/header_validator.h b/test/mocks/http/header_validator.h index 2162e8b6d0ac..a83c4c05f7c7 100644 --- a/test/mocks/http/header_validator.h +++ b/test/mocks/http/header_validator.h @@ -7,9 +7,9 @@ namespace Envoy { namespace Http { -class MockHeaderValidator : public HeaderValidator { +class MockServerHeaderValidator : public ServerHeaderValidator { public: - ~MockHeaderValidator() override = default; + ~MockServerHeaderValidator() override = default; MOCK_METHOD(ValidationResult, validateRequestHeaders, (const RequestHeaderMap& header_map)); MOCK_METHOD(ValidationResult, validateResponseHeaders, (const ResponseHeaderMap& header_map)); MOCK_METHOD(ValidationResult, validateRequestTrailers, (const RequestTrailerMap& header_map)); @@ -42,7 +42,7 @@ class MockHeaderValidatorStats : public HeaderValidatorStats { class MockHeaderValidatorFactory : public HeaderValidatorFactory { public: - MOCK_METHOD(HeaderValidatorPtr, createServerHeaderValidator, + MOCK_METHOD(ServerHeaderValidatorPtr, createServerHeaderValidator, (Protocol protocol, HeaderValidatorStats& stats)); MOCK_METHOD(ClientHeaderValidatorPtr, createClientHeaderValidator, (Protocol protocol, HeaderValidatorStats& stats)); diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 820eb28ee7f4..aa99fae6f823 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -660,7 +660,7 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { } MOCK_METHOD(uint64_t, maxRequestsPerConnection, (), (const)); MOCK_METHOD(const HttpConnectionManagerProto::ProxyStatusConfig*, proxyStatusConfig, (), (const)); - MOCK_METHOD(HeaderValidatorPtr, makeHeaderValidator, (Protocol protocol)); + MOCK_METHOD(ServerHeaderValidatorPtr, makeHeaderValidator, (Protocol protocol)); MOCK_METHOD(bool, appendXForwardedPort, (), (const)); MOCK_METHOD(bool, addProxyProtocolConnectionState, (), (const)); From ce7d4e04638920490a78d67a5a4976c013e97e68 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Tue, 2 May 2023 14:13:18 -0700 Subject: [PATCH 084/740] test: fix shadow test flake (#27126) Commit Message: test: fix shadow test flakes Additional Description: The second time we call sendRequestAndValidateResponse(), we have to wait for the counter to be g.e. 2. Risk Level: none Signed-off-by: Paul Gallagher --- test/integration/shadow_policy_integration_test.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/integration/shadow_policy_integration_test.cc b/test/integration/shadow_policy_integration_test.cc index a779094ad014..d34796163a67 100644 --- a/test/integration/shadow_policy_integration_test.cc +++ b/test/integration/shadow_policy_integration_test.cc @@ -69,7 +69,7 @@ class ShadowPolicyIntegrationTest }); } - void sendRequestAndValidateResponse() { + void sendRequestAndValidateResponse(int times_called = 1) { codec_client_ = makeHttpConnection(lookupPort("http")); IntegrationStreamDecoderPtr response = @@ -80,7 +80,8 @@ class ShadowPolicyIntegrationTest if (filter_name_ != "add-body-filter") { EXPECT_EQ(10U, response->body().size()); } - test_server_->waitForCounterGe("cluster.cluster_1.internal.upstream_rq_completed", 1); + test_server_->waitForCounterGe("cluster.cluster_1.internal.upstream_rq_completed", + times_called); upstream_headers_ = reinterpret_cast(fake_upstreams_[0].get())->lastRequestHeaders(); @@ -113,8 +114,8 @@ TEST_P(ShadowPolicyIntegrationTest, Basic) { initialConfigSetup("cluster_1", ""); initialize(); - sendRequestAndValidateResponse(); - sendRequestAndValidateResponse(); + sendRequestAndValidateResponse(1); + sendRequestAndValidateResponse(2); test_server_->waitForCounterEq("cluster.cluster_1.upstream_rq_200", 2); EXPECT_EQ(1, test_server_->counter("cluster.cluster_0.upstream_cx_total")->value()); @@ -133,8 +134,8 @@ TEST_P(ShadowPolicyIntegrationTest, BasicWithLimits) { }); initialize(); - sendRequestAndValidateResponse(); - sendRequestAndValidateResponse(); + sendRequestAndValidateResponse(1); + sendRequestAndValidateResponse(2); test_server_->waitForCounterEq("cluster.cluster_1.upstream_rq_200", 2); EXPECT_EQ(2, test_server_->counter("cluster.cluster_0.upstream_cx_total")->value()); From f95f0e776e002014016953792f516ee9322815d5 Mon Sep 17 00:00:00 2001 From: RenjieTang Date: Tue, 2 May 2023 17:08:23 -0700 Subject: [PATCH 085/740] [mobile]Rename a bunch of Cronvoy impl java classes (#27122) Rename some Cronvoy impl java classes so that they don't get messed up with original Cronet. Additional Description: Most of this change is noop, as the classes in net/impl are not exposed to Cronet customers. There is a one change to the Cronvoy API though: The original CronetProvider is changed to CronvoyProvider. The user needs to ask CronvoyProvider instead of CronetProvider to provide a Cronet engine. This name specification prevents the app from keeping going to original Cronet when Cronvoy is desired. Risk Level: Low Testing: Existing tests Docs Changes: n/a Release Notes: n/a Platform Specific Features: Android only Signed-off-by: Renjie Tang --- .../java/org/chromium/net/CronetEngine.java | 26 ++-- ...onetProvider.java => CronvoyProvider.java} | 48 +++---- .../library/java/org/chromium/net/impl/BUILD | 50 ++++---- .../net/impl/CallbackExceptionImpl.java | 8 -- .../net/impl/CronetExceptionImpl.java | 9 -- ...te.java => CronvoyBidirectionalState.java} | 4 +- ...m.java => CronvoyBidirectionalStream.java} | 120 +++++++++--------- ...ronvoyBidirectionalStreamBuilderImpl.java} | 19 +-- ...yBidirectionalStreamNetworkException.java} | 8 +- .../impl/CronvoyCallbackExceptionImpl.java | 8 ++ ...EngineBase.java => CronvoyEngineBase.java} | 9 +- ...mpl.java => CronvoyEngineBuilderImpl.java} | 42 +++--- .../net/impl/CronvoyExceptionImpl.java | 9 ++ ...plVersion.java => CronvoyImplVersion.java} | 4 +- ...CronetMetrics.java => CronvoyMetrics.java} | 12 +- ....java => CronvoyNetworkExceptionImpl.java} | 4 +- ...ditions.java => CronvoyPreconditions.java} | 4 +- ...mpl.java => CronvoyQuicExceptionImpl.java} | 10 +- ...va => CronvoyRequestFinishedInfoImpl.java} | 12 +- ...ream.java => CronvoyUploadDataStream.java} | 20 +-- ...UrlRequest.java => CronvoyUrlRequest.java} | 106 ++++++++-------- ...stBase.java => CronvoyUrlRequestBase.java} | 4 +- ...java => CronvoyUrlRequestBuilderImpl.java} | 44 ++++--- ...ext.java => CronvoyUrlRequestContext.java} | 48 +++---- ...l.java => CronvoyUrlResponseInfoImpl.java} | 13 +- .../{UserAgent.java => CronvoyUserAgent.java} | 6 +- ....java => CronvoyVersionSafeCallbacks.java} | 2 +- .../java/org/chromium/net/impl/Errors.java | 2 +- ...netEngineBuilderWithLibraryLoaderImpl.java | 32 ----- ...va => NativeCronvoyEngineBuilderImpl.java} | 12 +- ...voyEngineBuilderWithLibraryLoaderImpl.java | 33 +++++ ...ovider.java => NativeCronvoyProvider.java} | 20 +-- .../java/org/chromium/net/urlconnection/BUILD | 18 +-- ....java => CronvoyBufferedOutputStream.java} | 18 +-- ...m.java => CronvoyChunkedOutputStream.java} | 16 +-- ...java => CronvoyFixedModeOutputStream.java} | 20 +-- ...ion.java => CronvoyHttpURLConnection.java} | 26 ++-- ....java => CronvoyHttpURLStreamHandler.java} | 8 +- ...putStream.java => CronvoyInputStream.java} | 16 +-- ...ssageLoop.java => CronvoyMessageLoop.java} | 8 +- ...utStream.java => CronvoyOutputStream.java} | 10 +- ...va => CronvoyURLStreamHandlerFactory.java} | 10 +- .../chromium/net/BidirectionalStreamTest.java | 44 +++---- .../chromium/net/CronetEngineBuilderTest.java | 26 ++-- .../net/CronetUrlRequestContextTest.java | 16 +-- .../chromium/net/CronetUrlRequestTest.java | 17 +-- .../chromium/net/RequestFinishedInfoTest.java | 8 +- .../org/chromium/net/UrlResponseInfoTest.java | 8 +- .../impl/CronetBidirectionalStateTest.java | 7 +- .../chromium/net/impl/CronvoyEngineTest.java | 8 +- .../chromium/net/testing/CronetTestRule.java | 8 +- .../net/testing/CronetTestRuleTest.java | 4 +- .../chromium/net/testing/CronetTestUtil.java | 6 +- .../CronetFixedModeOutputStreamTest.java | 6 +- .../CronetHttpURLStreamHandlerTest.java | 16 +-- .../urlconnection/CronetInputStreamTest.java | 8 +- .../CronetURLStreamHandlerFactoryTest.java | 2 +- .../net/urlconnection/MessageLoopTest.java | 6 +- 58 files changed, 547 insertions(+), 541 deletions(-) rename mobile/library/java/org/chromium/net/{CronetProvider.java => CronvoyProvider.java} (73%) delete mode 100644 mobile/library/java/org/chromium/net/impl/CallbackExceptionImpl.java delete mode 100644 mobile/library/java/org/chromium/net/impl/CronetExceptionImpl.java rename mobile/library/java/org/chromium/net/impl/{CronetBidirectionalState.java => CronvoyBidirectionalState.java} (99%) rename mobile/library/java/org/chromium/net/impl/{CronetBidirectionalStream.java => CronvoyBidirectionalStream.java} (89%) rename mobile/library/java/org/chromium/net/impl/{BidirectionalStreamBuilderImpl.java => CronvoyBidirectionalStreamBuilderImpl.java} (85%) rename mobile/library/java/org/chromium/net/impl/{BidirectionalStreamNetworkException.java => CronvoyBidirectionalStreamNetworkException.java} (65%) create mode 100644 mobile/library/java/org/chromium/net/impl/CronvoyCallbackExceptionImpl.java rename mobile/library/java/org/chromium/net/impl/{CronetEngineBase.java => CronvoyEngineBase.java} (95%) rename mobile/library/java/org/chromium/net/impl/{CronetEngineBuilderImpl.java => CronvoyEngineBuilderImpl.java} (86%) create mode 100644 mobile/library/java/org/chromium/net/impl/CronvoyExceptionImpl.java rename mobile/library/java/org/chromium/net/impl/{ImplVersion.java => CronvoyImplVersion.java} (92%) rename mobile/library/java/org/chromium/net/impl/{CronetMetrics.java => CronvoyMetrics.java} (90%) rename mobile/library/java/org/chromium/net/impl/{NetworkExceptionImpl.java => CronvoyNetworkExceptionImpl.java} (91%) rename mobile/library/java/org/chromium/net/impl/{Preconditions.java => CronvoyPreconditions.java} (85%) rename mobile/library/java/org/chromium/net/impl/{QuicExceptionImpl.java => CronvoyQuicExceptionImpl.java} (79%) rename mobile/library/java/org/chromium/net/impl/{RequestFinishedInfoImpl.java => CronvoyRequestFinishedInfoImpl.java} (77%) rename mobile/library/java/org/chromium/net/impl/{CronetUploadDataStream.java => CronvoyUploadDataStream.java} (92%) rename mobile/library/java/org/chromium/net/impl/{CronetUrlRequest.java => CronvoyUrlRequest.java} (90%) rename mobile/library/java/org/chromium/net/impl/{UrlRequestBase.java => CronvoyUrlRequestBase.java} (93%) rename mobile/library/java/org/chromium/net/impl/{UrlRequestBuilderImpl.java => CronvoyUrlRequestBuilderImpl.java} (78%) rename mobile/library/java/org/chromium/net/impl/{CronetUrlRequestContext.java => CronvoyUrlRequestContext.java} (86%) rename mobile/library/java/org/chromium/net/impl/{UrlResponseInfoImpl.java => CronvoyUrlResponseInfoImpl.java} (92%) rename mobile/library/java/org/chromium/net/impl/{UserAgent.java => CronvoyUserAgent.java} (95%) rename mobile/library/java/org/chromium/net/impl/{VersionSafeCallbacks.java => CronvoyVersionSafeCallbacks.java} (99%) delete mode 100644 mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderWithLibraryLoaderImpl.java rename mobile/library/java/org/chromium/net/impl/{NativeCronetEngineBuilderImpl.java => NativeCronvoyEngineBuilderImpl.java} (94%) create mode 100644 mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderWithLibraryLoaderImpl.java rename mobile/library/java/org/chromium/net/impl/{NativeCronetProvider.java => NativeCronvoyProvider.java} (54%) rename mobile/library/java/org/chromium/net/urlconnection/{CronetBufferedOutputStream.java => CronvoyBufferedOutputStream.java} (89%) rename mobile/library/java/org/chromium/net/urlconnection/{CronetChunkedOutputStream.java => CronvoyChunkedOutputStream.java} (88%) rename mobile/library/java/org/chromium/net/urlconnection/{CronetFixedModeOutputStream.java => CronvoyFixedModeOutputStream.java} (90%) rename mobile/library/java/org/chromium/net/urlconnection/{CronetHttpURLConnection.java => CronvoyHttpURLConnection.java} (96%) rename mobile/library/java/org/chromium/net/urlconnection/{CronetHttpURLStreamHandler.java => CronvoyHttpURLStreamHandler.java} (83%) rename mobile/library/java/org/chromium/net/urlconnection/{CronetInputStream.java => CronvoyInputStream.java} (83%) rename mobile/library/java/org/chromium/net/urlconnection/{MessageLoop.java => CronvoyMessageLoop.java} (94%) rename mobile/library/java/org/chromium/net/urlconnection/{CronetOutputStream.java => CronvoyOutputStream.java} (84%) rename mobile/library/java/org/chromium/net/urlconnection/{CronetURLStreamHandlerFactory.java => CronvoyURLStreamHandlerFactory.java} (85%) diff --git a/mobile/library/java/org/chromium/net/CronetEngine.java b/mobile/library/java/org/chromium/net/CronetEngine.java index 931acf9f8699..156383d96955 100644 --- a/mobile/library/java/org/chromium/net/CronetEngine.java +++ b/mobile/library/java/org/chromium/net/CronetEngine.java @@ -282,15 +282,15 @@ public Builder enablePublicKeyPinningBypassForLocalTrustAnchors(boolean value) { /** * Creates an implementation of {@link ICronetEngineBuilder} that can be used to delegate the - * builder calls to. The method uses {@link CronetProvider} to obtain the list of available + * builder calls to. The method uses {@link CronvoyProvider} to obtain the list of available * providers. * * @param context Android Context to use. * @return the created {@code ICronetEngineBuilder}. */ private static ICronetEngineBuilder createBuilderDelegate(Context context) { - List providers = new ArrayList<>(CronetProvider.getAllProviders(context)); - CronetProvider provider = getEnabledCronetProviders(context, providers).get(0); + List providers = new ArrayList<>(CronvoyProvider.getAllProviders(context)); + CronvoyProvider provider = getEnabledCronvoyProviders(context, providers).get(0); if (Log.isLoggable(TAG, Log.DEBUG)) { Log.d(TAG, String.format("Using '%s' provider for creating CronetEngine.Builder.", provider)); @@ -299,8 +299,8 @@ private static ICronetEngineBuilder createBuilderDelegate(Context context) { } /** - * Returns the list of available and enabled {@link CronetProvider}. The returned list is sorted - * based on the provider versions and types. + * Returns the list of available and enabled {@link CronvoyProvider}. The returned list is + * sorted based on the provider versions and types. * * @param context Android Context to use. * @param providers the list of enabled and disabled providers to filter out and sort. @@ -309,8 +309,8 @@ private static ICronetEngineBuilder createBuilderDelegate(Context context) { * disabled. */ @VisibleForTesting - static List getEnabledCronetProviders(Context context, - List providers) { + static List getEnabledCronvoyProviders(Context context, + List providers) { // Check that there is at least one available provider. if (providers.size() == 0) { throw new RuntimeException("Unable to find any Cronet provider." @@ -318,8 +318,8 @@ static List getEnabledCronetProviders(Context context, } // Exclude disabled providers from the list. - for (Iterator i = providers.iterator(); i.hasNext();) { - CronetProvider provider = i.next(); + for (Iterator i = providers.iterator(); i.hasNext();) { + CronvoyProvider provider = i.next(); if (!provider.isEnabled()) { i.remove(); } @@ -332,14 +332,14 @@ static List getEnabledCronetProviders(Context context, } // Sort providers based on version and type. - Collections.sort(providers, new Comparator() { + Collections.sort(providers, new Comparator() { @Override - public int compare(CronetProvider p1, CronetProvider p2) { + public int compare(CronvoyProvider p1, CronvoyProvider p2) { // The fallback provider should always be at the end of the list. - if (CronetProvider.PROVIDER_NAME_FALLBACK.equals(p1.getName())) { + if (CronvoyProvider.PROVIDER_NAME_FALLBACK.equals(p1.getName())) { return 1; } - if (CronetProvider.PROVIDER_NAME_FALLBACK.equals(p2.getName())) { + if (CronvoyProvider.PROVIDER_NAME_FALLBACK.equals(p2.getName())) { return -1; } // A provider with higher version should go first. diff --git a/mobile/library/java/org/chromium/net/CronetProvider.java b/mobile/library/java/org/chromium/net/CronvoyProvider.java similarity index 73% rename from mobile/library/java/org/chromium/net/CronetProvider.java rename to mobile/library/java/org/chromium/net/CronvoyProvider.java index 62c2249a0507..693357765828 100644 --- a/mobile/library/java/org/chromium/net/CronetProvider.java +++ b/mobile/library/java/org/chromium/net/CronvoyProvider.java @@ -13,7 +13,7 @@ /** * Provides a factory method to create {@link CronetEngine.Builder} instances. A {@code * CronetEngine.Builder} instance can be used to create a specific {@link CronetEngine} - * implementation. To get the list of available {@link CronetProvider}s call {@link + * implementation. To get the list of available {@link CronvoyProvider}s call {@link * #getAllProviders(Context)}. * *

NOTE: This class is for advanced users that want to select a particular Cronet @@ -22,9 +22,9 @@ * *

{@hide} */ -public abstract class CronetProvider { +public abstract class CronvoyProvider { /** - * String returned by {@link CronetProvider#getName} for {@link CronetProvider} that provides + * String returned by {@link CronvoyProvider#getName} for {@link CronvoyProvider} that provides * native Cronet implementation packaged inside an application. This implementation offers * significantly higher performance relative to the fallback Cronet implementations (see {@link * #PROVIDER_NAME_FALLBACK}). @@ -32,24 +32,18 @@ public abstract class CronetProvider { public static final String PROVIDER_NAME_APP_PACKAGED = "App-Packaged-Cronet-Provider"; /** - * String returned by {@link CronetProvider#getName} for {@link CronetProvider} that provides + * String returned by {@link CronvoyProvider#getName} for {@link CronvoyProvider} that provides * Cronet implementation based on the system's {@link java.net.HttpURLConnection} implementation. * This implementation offers significantly degraded performance relative to native Cronet * implementations (see {@link #PROVIDER_NAME_APP_PACKAGED}). */ public static final String PROVIDER_NAME_FALLBACK = "Fallback-Cronet-Provider"; - /** - * The name of an optional key in the app string resource file that contains the class name of an - * alternative {@code CronetProvider} implementation. - */ - private static final String RES_KEY_CRONET_IMPL_CLASS = "CronetProviderClassName"; - - private static final String TAG = CronetProvider.class.getSimpleName(); + private static final String TAG = CronvoyProvider.class.getSimpleName(); protected final Context mContext; - protected CronetProvider(Context context) { + protected CronvoyProvider(Context context) { if (context == null) { throw new IllegalArgumentException("Context must not be null"); } @@ -106,21 +100,21 @@ public String toString() { + "enabled=" + isEnabled() + "]"; } - /** Name of the native {@link CronetProvider} class. */ - private static final String NATIVE_CRONET_PROVIDER_CLASS = - "org.chromium.net.impl.NativeCronetProvider"; + /** Name of the native {@link CronvoyProvider} class. */ + private static final String NATIVE_CRONVOY_PROVIDER_CLASS = + "org.chromium.net.impl.NativeCronvoyProvider"; /** - * Returns an unmodifiable list of all available {@link CronetProvider}s. The providers are + * Returns an unmodifiable list of all available {@link CronvoyProvider}s. The providers are * returned in no particular order. Some of the returned providers may be in a disabled state and - * should be enabled by the invoker. See {@link CronetProvider#isEnabled()}. + * should be enabled by the invoker. See {@link CronvoyProvider#isEnabled()}. * * @return the list of available providers. */ - public static List getAllProviders(Context context) { + public static List getAllProviders(Context context) { // Use LinkedHashSet to preserve the order and eliminate duplicate providers. - Set providers = new LinkedHashSet<>(); - addCronetProviderImplByClassName(context, NATIVE_CRONET_PROVIDER_CLASS, providers, false); + Set providers = new LinkedHashSet<>(); + addCronvoyProviderImplByClassName(context, NATIVE_CRONVOY_PROVIDER_CLASS, providers, false); return Collections.unmodifiableList(new ArrayList<>(providers)); } @@ -132,14 +126,14 @@ public static List getAllProviders(Context context) { * @return {@code true} if the provider was added to the set; {@code false} if the provider * couldn't be instantiated. */ - private static boolean addCronetProviderImplByClassName(Context context, String className, - Set providers, - boolean logError) { + private static boolean addCronvoyProviderImplByClassName(Context context, String className, + Set providers, + boolean logError) { ClassLoader loader = context.getClassLoader(); try { - Class providerClass = - loader.loadClass(className).asSubclass(CronetProvider.class); - Constructor ctor = providerClass.getConstructor(Context.class); + Class providerClass = + loader.loadClass(className).asSubclass(CronvoyProvider.class); + Constructor ctor = providerClass.getConstructor(Context.class); providers.add(ctor.newInstance(context)); return true; } catch (InstantiationException e) { @@ -157,7 +151,7 @@ private static boolean addCronetProviderImplByClassName(Context context, String } /** - * De-duplicates exception handling logic in {@link #addCronetProviderImplByClassName}. It should + * De-duplicates exception handling logic in {@link #addCronvoyProviderImplByClassName}. It should * be removed when support of API Levels lower than 19 is deprecated. */ private static void logReflectiveOperationException(String className, boolean logError, diff --git a/mobile/library/java/org/chromium/net/impl/BUILD b/mobile/library/java/org/chromium/net/impl/BUILD index d6d8ec07ad23..2c370697ba43 100644 --- a/mobile/library/java/org/chromium/net/impl/BUILD +++ b/mobile/library/java/org/chromium/net/impl/BUILD @@ -11,35 +11,35 @@ android_library( srcs = [ "Annotations.java", "AtomicCombinatoryState.java", - "BidirectionalStreamBuilderImpl.java", - "BidirectionalStreamNetworkException.java", - "CallbackExceptionImpl.java", "CancelProofEnvoyStream.java", - "CronetBidirectionalState.java", - "CronetBidirectionalStream.java", - "CronetEngineBase.java", - "CronetEngineBuilderImpl.java", - "CronetExceptionImpl.java", - "CronetMetrics.java", - "CronetUploadDataStream.java", - "CronetUrlRequest.java", - "CronetUrlRequestContext.java", + "CronvoyBidirectionalState.java", + "CronvoyBidirectionalStream.java", + "CronvoyBidirectionalStreamBuilderImpl.java", + "CronvoyBidirectionalStreamNetworkException.java", + "CronvoyCallbackExceptionImpl.java", + "CronvoyEngineBase.java", + "CronvoyEngineBuilderImpl.java", + "CronvoyExceptionImpl.java", + "CronvoyImplVersion.java", + "CronvoyMetrics.java", + "CronvoyNetworkExceptionImpl.java", + "CronvoyPreconditions.java", + "CronvoyQuicExceptionImpl.java", + "CronvoyRequestFinishedInfoImpl.java", + "CronvoyUploadDataStream.java", + "CronvoyUrlRequest.java", + "CronvoyUrlRequestBase.java", + "CronvoyUrlRequestBuilderImpl.java", + "CronvoyUrlRequestContext.java", + "CronvoyUrlResponseInfoImpl.java", + "CronvoyUserAgent.java", + "CronvoyVersionSafeCallbacks.java", "Errors.java", "Executors.java", "HttpReason.java", - "ImplVersion.java", - "NativeCronetEngineBuilderImpl.java", - "NativeCronetEngineBuilderWithLibraryLoaderImpl.java", - "NativeCronetProvider.java", - "NetworkExceptionImpl.java", - "Preconditions.java", - "QuicExceptionImpl.java", - "RequestFinishedInfoImpl.java", - "UrlRequestBase.java", - "UrlRequestBuilderImpl.java", - "UrlResponseInfoImpl.java", - "UserAgent.java", - "VersionSafeCallbacks.java", + "NativeCronvoyEngineBuilderImpl.java", + "NativeCronvoyEngineBuilderWithLibraryLoaderImpl.java", + "NativeCronvoyProvider.java", ], manifest = "CronvoyManifest.xml", visibility = ["//visibility:public"], diff --git a/mobile/library/java/org/chromium/net/impl/CallbackExceptionImpl.java b/mobile/library/java/org/chromium/net/impl/CallbackExceptionImpl.java deleted file mode 100644 index 76bbcfe6665c..000000000000 --- a/mobile/library/java/org/chromium/net/impl/CallbackExceptionImpl.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.chromium.net.impl; - -import org.chromium.net.CallbackException; - -/** An implementation of {@link CallbackException}. */ -public final class CallbackExceptionImpl extends CallbackException { - CallbackExceptionImpl(String message, Throwable cause) { super(message, cause); } -} diff --git a/mobile/library/java/org/chromium/net/impl/CronetExceptionImpl.java b/mobile/library/java/org/chromium/net/impl/CronetExceptionImpl.java deleted file mode 100644 index 4677f1011162..000000000000 --- a/mobile/library/java/org/chromium/net/impl/CronetExceptionImpl.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.chromium.net.impl; - -import org.chromium.net.CronetException; - -/** Implements {@link CronetException}. */ -final class CronetExceptionImpl extends CronetException { - - CronetExceptionImpl(String message, Throwable cause) { super(message, cause); } -} diff --git a/mobile/library/java/org/chromium/net/impl/CronetBidirectionalState.java b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalState.java similarity index 99% rename from mobile/library/java/org/chromium/net/impl/CronetBidirectionalState.java rename to mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalState.java index 715fa65a2daf..7ec6d7d039df 100644 --- a/mobile/library/java/org/chromium/net/impl/CronetBidirectionalState.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalState.java @@ -3,7 +3,7 @@ import androidx.annotation.IntDef; import org.chromium.net.RequestFinishedInfo; -import org.chromium.net.impl.RequestFinishedInfoImpl.FinishedReason; +import org.chromium.net.impl.CronvoyRequestFinishedInfoImpl.FinishedReason; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -161,7 +161,7 @@ * -------------------------------------------- STREAM_READY_CALLBACK_DONE -- * */ -final class CronetBidirectionalState { +final class CronvoyBidirectionalState { /** * Enum of the events altering the global state. There are 3 types of events: User induced diff --git a/mobile/library/java/org/chromium/net/impl/CronetBidirectionalStream.java b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java similarity index 89% rename from mobile/library/java/org/chromium/net/impl/CronetBidirectionalStream.java rename to mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java index 7b64ce5dc70e..01cb9ad32dc9 100644 --- a/mobile/library/java/org/chromium/net/impl/CronetBidirectionalStream.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java @@ -17,10 +17,10 @@ import org.chromium.net.RequestFinishedInfo; import org.chromium.net.UrlResponseInfo; import org.chromium.net.impl.Annotations.RequestPriority; -import org.chromium.net.impl.CronetBidirectionalState.Event; -import org.chromium.net.impl.CronetBidirectionalState.NextAction; +import org.chromium.net.impl.CronvoyBidirectionalState.Event; +import org.chromium.net.impl.CronvoyBidirectionalState.NextAction; import org.chromium.net.impl.Errors.NetError; -import org.chromium.net.impl.UrlResponseInfoImpl.HeaderBlockImpl; +import org.chromium.net.impl.CronvoyUrlResponseInfoImpl.HeaderBlockImpl; import java.net.MalformedURLException; import java.net.URL; @@ -76,7 +76,7 @@ *
Implementation wise, the most noticeable difference between the Cronet implementation and * this one is the avoidance of any java "synchronized". This implementation is based on "Compare * And Swap" logic to guarantee correctness. The State of the Stream is kept in a single atomic - * Integer owned by {@link CronetBidirectionalState}. That state is a set of bits. Technically it + * Integer owned by {@link CronvoyBidirectionalState}. That state is a set of bits. Technically it * could have been the conjunction of Enums held inside a single Integer. Using bits turned out * to avoid more complex "if" logic. Still, the most important point here is the fact that the whole * state is a single Atomic Integer: it eases the avoidance of race conditions, especially when @@ -111,7 +111,7 @@ * a dedicated class handles this business: {@link CancelProofEnvoyStream}. * */ -public final class CronetBidirectionalStream +public final class CronvoyBidirectionalStream extends ExperimentalBidirectionalStream implements EnvoyHTTPCallbacks { private static final String X_ENVOY = "x-envoy"; @@ -119,9 +119,9 @@ public final class CronetBidirectionalStream private static final String USER_AGENT = "User-Agent"; private static final Executor DIRECT_EXECUTOR = new DirectExecutor(); - private final CronetUrlRequestContext mRequestContext; + private final CronvoyUrlRequestContext mRequestContext; private final Executor mExecutor; - private final VersionSafeCallbacks.BidirectionalStreamCallback mCallback; + private final CronvoyVersionSafeCallbacks.BidirectionalStreamCallback mCallback; private final String mInitialUrl; // TODO(https://github.com/envoyproxy/envoy-mobile/issues/1641): Priority? What should we do. private final int mInitialPriority; @@ -137,7 +137,7 @@ public final class CronetBidirectionalStream private final int mTrafficStatsUid; private final String mUserAgent; private final CancelProofEnvoyStream mStream = new CancelProofEnvoyStream(); - private final CronetBidirectionalState mState = new CronetBidirectionalState(); + private final CronvoyBidirectionalState mState = new CronvoyBidirectionalState(); private final AtomicInteger mUserflushConcurrentInvocationCount = new AtomicInteger(); private final AtomicInteger mFlushConcurrentInvocationCount = new AtomicInteger(); private final AtomicReference mException = new AtomicReference<>(); @@ -159,7 +159,7 @@ public final class CronetBidirectionalStream private final AtomicReference mLatestBufferRead = new AtomicReference<>(); // Only modified on the network thread. - private volatile UrlResponseInfoImpl mResponseInfo; + private volatile CronvoyUrlResponseInfoImpl mResponseInfo; private Runnable mOnDestroyedCallbackForTesting; @@ -183,7 +183,7 @@ public void run() { switch ( mState.nextAction(mEndOfStream ? Event.LAST_READ_COMPLETED : Event.READ_COMPLETED)) { case NextAction.NOTIFY_USER_READ_COMPLETED: - mCallback.onReadCompleted(CronetBidirectionalStream.this, mResponseInfo, buffer, + mCallback.onReadCompleted(CronvoyBidirectionalStream.this, mResponseInfo, buffer, mEndOfStream); break; case NextAction.TAKE_NO_MORE_ACTIONS: @@ -235,7 +235,7 @@ public void run() { switch ( mState.nextAction(mEndOfStream ? Event.LAST_WRITE_COMPLETED : Event.WRITE_COMPLETED)) { case NextAction.NOTIFY_USER_WRITE_COMPLETED: - mCallback.onWriteCompleted(CronetBidirectionalStream.this, mResponseInfo, buffer, + mCallback.onWriteCompleted(CronvoyBidirectionalStream.this, mResponseInfo, buffer, mEndOfStream); break; case NextAction.TAKE_NO_MORE_ACTIONS: @@ -264,17 +264,17 @@ public void run() { } } - CronetBidirectionalStream(CronetUrlRequestContext requestContext, String url, - @CronetEngineBase.StreamPriority int priority, Callback callback, - Executor executor, String userAgent, String httpMethod, - List> requestHeaders, - boolean delayRequestHeadersUntilNextFlush, - Collection requestAnnotations, boolean trafficStatsTagSet, - int trafficStatsTag, boolean trafficStatsUidSet, int trafficStatsUid) { + CronvoyBidirectionalStream(CronvoyUrlRequestContext requestContext, String url, + @CronvoyEngineBase.StreamPriority int priority, Callback callback, + Executor executor, String userAgent, String httpMethod, + List> requestHeaders, + boolean delayRequestHeadersUntilNextFlush, + Collection requestAnnotations, boolean trafficStatsTagSet, + int trafficStatsTag, boolean trafficStatsUidSet, int trafficStatsUid) { mRequestContext = requestContext; mInitialUrl = url; mInitialPriority = convertStreamPriority(priority); - mCallback = new VersionSafeCallbacks.BidirectionalStreamCallback(callback); + mCallback = new CronvoyVersionSafeCallbacks.BidirectionalStreamCallback(callback); mExecutor = executor; mUserAgent = userAgent; mMethod = httpMethod; @@ -319,14 +319,14 @@ public void start() { public void run() { try { mStream.setStream(mRequestContext.getEnvoyEngine().startStream( - CronetBidirectionalStream.this, /* explicitFlowCrontrol= */ true)); + CronvoyBidirectionalStream.this, /* explicitFlowCrontrol= */ true)); if (!mDelayRequestHeadersUntilFirstFlush) { mStream.sendHeaders(mEnvoyRequestHeaders, mReadOnly); } onStreamReady(); } catch (RuntimeException e) { // Will be reported when "onCancel" gets invoked. - reportException(new CronetExceptionImpl("Startup failure", e)); + reportException(new CronvoyExceptionImpl("Startup failure", e)); } } }; @@ -356,17 +356,17 @@ public void run() { @Nullable private static CronetException engineSimulatedError(Map> requestHeaders) { if (requestHeaders.get(":scheme").get(0).equals("http")) { - return new BidirectionalStreamNetworkException("Exception in BidirectionalStream: " - + "net::ERR_DISALLOWED_URL_SCHEME", - 11, -301); + return new CronvoyBidirectionalStreamNetworkException("Exception in BidirectionalStream: " + + "net::ERR_DISALLOWED_URL_SCHEME", + 11, -301); } return null; } @Override public void read(ByteBuffer buffer) { - Preconditions.checkHasRemaining(buffer); - Preconditions.checkDirect(buffer); + CronvoyPreconditions.checkHasRemaining(buffer); + CronvoyPreconditions.checkDirect(buffer); mLatestBufferRead.compareAndSet(null, new ReadBuffer(buffer)); attemptToRead(Event.USER_READ); // Read might not occur right now. If so, it is postponed. } @@ -393,7 +393,7 @@ private void attemptToRead(@Event int readEvent) { @Override public void write(ByteBuffer buffer, boolean endOfStream) { - Preconditions.checkDirect(buffer); + CronvoyPreconditions.checkDirect(buffer); if (!buffer.hasRemaining() && !endOfStream) { throw new IllegalArgumentException("Empty buffer before end of stream."); } @@ -542,9 +542,9 @@ public void run() { private void onSucceededOnExecutor() { cleanup(); try { - mCallback.onSucceeded(CronetBidirectionalStream.this, mResponseInfo); + mCallback.onSucceeded(CronvoyBidirectionalStream.this, mResponseInfo); } catch (Exception e) { - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception in onSucceeded method", e); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception in onSucceeded method", e); } } @@ -556,14 +556,14 @@ public void run() { if (mState.isTerminating()) { return; } - mCallback.onStreamReady(CronetBidirectionalStream.this); + mCallback.onStreamReady(CronvoyBidirectionalStream.this); // Under duress, or due to user long logic, the response headers might have been received // already. In that case mCallback.onResponseHeadersReceived was purposely not called, and // therefore this is done here. This guarantees correct ordering: mCallback.onStreamReady // must finish before invoking mCallback.onResponseHeadersReceived. switch (mState.nextAction(Event.STREAM_READY_CALLBACK_DONE)) { case NextAction.NOTIFY_USER_HEADERS_RECEIVED: - mCallback.onResponseHeadersReceived(CronetBidirectionalStream.this, mResponseInfo); + mCallback.onResponseHeadersReceived(CronvoyBidirectionalStream.this, mResponseInfo); break; case NextAction.CARRY_ON: break; // Response headers have not been received yet - most common outcome. @@ -594,7 +594,7 @@ public void run() { if (mState.isTerminating()) { return; } - mCallback.onResponseHeadersReceived(CronetBidirectionalStream.this, mResponseInfo); + mCallback.onResponseHeadersReceived(CronvoyBidirectionalStream.this, mResponseInfo); } catch (Exception e) { onCallbackException(e); } @@ -607,11 +607,11 @@ private void onReadCompleted(ReadBuffer readBuffer, int bytesRead) { int initialPosition = readBuffer.mInitialPosition; int initialLimit = readBuffer.mInitialLimit; if (byteBuffer.position() != initialPosition || byteBuffer.limit() != initialLimit) { - reportException(new CronetExceptionImpl("ByteBuffer modified externally during read", null)); + reportException(new CronvoyExceptionImpl("ByteBuffer modified externally during read", null)); return; } if (bytesRead < 0 || initialPosition + bytesRead > initialLimit) { - reportException(new CronetExceptionImpl("Invalid number of bytes read", null)); + reportException(new CronvoyExceptionImpl("Invalid number of bytes read", null)); return; } byteBuffer.position(initialPosition + bytesRead); @@ -622,7 +622,8 @@ private void onWriteCompleted(WriteBuffer writeBuffer) { ByteBuffer buffer = writeBuffer.mByteBuffer; if (buffer.position() != writeBuffer.mInitialPosition || buffer.limit() != writeBuffer.mInitialLimit) { - reportException(new CronetExceptionImpl("ByteBuffer modified externally during write", null)); + reportException( + new CronvoyExceptionImpl("ByteBuffer modified externally during write", null)); return; } // Current implementation always writes the complete buffer. @@ -639,7 +640,7 @@ public void run() { if (mState.isTerminating()) { return; } - mCallback.onResponseTrailersReceived(CronetBidirectionalStream.this, mResponseInfo, + mCallback.onResponseTrailersReceived(CronvoyBidirectionalStream.this, mResponseInfo, trailersBlock); } catch (Exception e) { onCallbackException(e); @@ -657,11 +658,11 @@ private void onErrorReceived(int errorCode, EnvoyFinalStreamIntel finalStreamInt int javaError = mapNetErrorToCronetApiErrorCode(netError); if (isQuicException(javaError)) { - mException.set(new QuicExceptionImpl("Exception in BidirectionalStream: " + netError, - javaError, netError.getErrorCode(), - Errors.QUIC_INTERNAL_ERROR)); + mException.set(new CronvoyQuicExceptionImpl("Exception in BidirectionalStream: " + netError, + javaError, netError.getErrorCode(), + Errors.QUIC_INTERNAL_ERROR)); } else { - mException.set(new BidirectionalStreamNetworkException( + mException.set(new CronvoyBidirectionalStreamNetworkException( "Exception in BidirectionalStream: " + netError, javaError, netError.getErrorCode())); } @@ -677,9 +678,9 @@ private void onCanceledReceived() { @Override public void run() { try { - mCallback.onCanceled(CronetBidirectionalStream.this, mResponseInfo); + mCallback.onCanceled(CronvoyBidirectionalStream.this, mResponseInfo); } catch (Exception e) { - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception in onCanceled method", e); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception in onCanceled method", e); } } }); @@ -695,13 +696,13 @@ private void onMetricsCollected(long requestStartMs, long dnsStartMs, long dnsEn long requestEndMs, boolean socketReused, long sentByteCount, long receivedByteCount) { // Metrics information. Obtained when request succeeds, fails or is canceled. - RequestFinishedInfo.Metrics mMetrics = new CronetMetrics( + RequestFinishedInfo.Metrics mMetrics = new CronvoyMetrics( requestStartMs, dnsStartMs, dnsEndMs, connectStartMs, connectEndMs, sslStartMs, sslEndMs, sendingStartMs, sendingEndMs, pushStartMs, pushEndMs, responseStartMs, requestEndMs, socketReused, sentByteCount, receivedByteCount); - final RequestFinishedInfo requestFinishedInfo = - new RequestFinishedInfoImpl(mInitialUrl, mRequestAnnotations, mMetrics, - mState.getFinishedReason(), mResponseInfo, mException.get()); + final RequestFinishedInfo requestFinishedInfo = new CronvoyRequestFinishedInfoImpl( + mInitialUrl, mRequestAnnotations, mMetrics, mState.getFinishedReason(), mResponseInfo, + mException.get()); mRequestContext.reportRequestFinished(requestFinishedInfo); } @@ -714,7 +715,7 @@ private static boolean doesMethodAllowWriteData(String methodName) { return !methodName.equals("GET") && !methodName.equals("HEAD"); } - private static int convertStreamPriority(@CronetEngineBase.StreamPriority int priority) { + private static int convertStreamPriority(@CronvoyEngineBase.StreamPriority int priority) { switch (priority) { case Builder.STREAM_PRIORITY_IDLE: return RequestPriority.IDLE; @@ -739,13 +740,14 @@ private void postTaskToExecutor(Runnable task) { try { mExecutor.execute(task); } catch (RejectedExecutionException failException) { - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception posting task to executor", failException); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception posting task to executor", failException); // If already in a failed state this invocation is a no-op. - reportException(new CronetExceptionImpl("Exception posting task to executor", failException)); + reportException( + new CronvoyExceptionImpl("Exception posting task to executor", failException)); } } - private UrlResponseInfoImpl + private CronvoyUrlResponseInfoImpl prepareResponseInfoOnNetworkThread(int httpStatusCode, String negotiatedProtocol, Map> responseHeaders, long receivedByteCount) { @@ -762,9 +764,9 @@ private void postTaskToExecutor(Runnable task) { } } // proxy and caching are not supported. - UrlResponseInfoImpl responseInfo = - new UrlResponseInfoImpl(Arrays.asList(mInitialUrl), httpStatusCode, "", headers, false, - negotiatedProtocol, null, receivedByteCount); + CronvoyUrlResponseInfoImpl responseInfo = + new CronvoyUrlResponseInfoImpl(Arrays.asList(mInitialUrl), httpStatusCode, "", headers, + false, negotiatedProtocol, null, receivedByteCount); return responseInfo; } @@ -788,9 +790,9 @@ private void failWithException() { @Override public void run() { try { - mCallback.onFailed(CronetBidirectionalStream.this, mResponseInfo, mException.get()); + mCallback.onFailed(CronvoyBidirectionalStream.this, mResponseInfo, mException.get()); } catch (Exception failException) { - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception notifying of failed request", + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception notifying of failed request", failException); } } @@ -803,8 +805,8 @@ public void run() { */ private void onCallbackException(Exception e) { CallbackException streamError = - new CallbackExceptionImpl("CalledByNative method has thrown an exception", e); - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception in CalledByNative method", e); + new CronvoyCallbackExceptionImpl("CalledByNative method has thrown an exception", e); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception in CalledByNative method", e); reportException(streamError); } @@ -823,7 +825,7 @@ private void reportException(CronetException exception) { failWithException(); break; case NextAction.TAKE_NO_MORE_ACTIONS: - Log.e(CronetUrlRequestContext.LOG_TAG, + Log.e(CronvoyUrlRequestContext.LOG_TAG, "An exception has already been previously recorded. This one is ignored.", exception); return; default: @@ -962,7 +964,7 @@ public void onHeaders(Map> headers, boolean endStream, mResponseInfo = prepareResponseInfoOnNetworkThread( httpStatusCode, negotiatedProtocol, headers, streamIntel.getConsumedBytesFromResponse()); } catch (Exception e) { - reportException(new CronetExceptionImpl("Cannot prepare ResponseInfo", null)); + reportException(new CronvoyExceptionImpl("Cannot prepare ResponseInfo", null)); return; } diff --git a/mobile/library/java/org/chromium/net/impl/BidirectionalStreamBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStreamBuilderImpl.java similarity index 85% rename from mobile/library/java/org/chromium/net/impl/BidirectionalStreamBuilderImpl.java rename to mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStreamBuilderImpl.java index 47cdb337296a..e5846eebe1a7 100644 --- a/mobile/library/java/org/chromium/net/impl/BidirectionalStreamBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStreamBuilderImpl.java @@ -11,12 +11,12 @@ import org.chromium.net.ExperimentalBidirectionalStream; /** Implementation of {@link ExperimentalBidirectionalStream.Builder}. */ -final class BidirectionalStreamBuilderImpl extends ExperimentalBidirectionalStream.Builder { +final class CronvoyBidirectionalStreamBuilderImpl extends ExperimentalBidirectionalStream.Builder { // All fields are temporary storage of ExperimentalBidirectionalStream configuration to be // copied to CronetBidirectionalStream. // CronetEngine to create the stream. - private final CronetEngineBase mCronetEngine; + private final CronvoyEngineBase mCronetEngine; // URL to request. private final String mUrl; // Callback to receive progress callbacks. @@ -29,7 +29,7 @@ final class BidirectionalStreamBuilderImpl extends ExperimentalBidirectionalStre // HTTP method for the request. Default to POST. private String mHttpMethod = "POST"; // Priority of the stream. Default is medium. - @CronetEngineBase.StreamPriority private int mPriority = STREAM_PRIORITY_MEDIUM; + @CronvoyEngineBase.StreamPriority private int mPriority = STREAM_PRIORITY_MEDIUM; private boolean mDelayRequestHeadersUntilFirstFlush; @@ -53,8 +53,8 @@ final class BidirectionalStreamBuilderImpl extends ExperimentalBidirectionalStre * @param executor the {@link Executor} on which {@code callback} methods will be invoked * @param cronetEngine the {@link CronetEngine} used to create the stream */ - BidirectionalStreamBuilderImpl(String url, BidirectionalStream.Callback callback, - Executor executor, CronetEngineBase cronetEngine) { + CronvoyBidirectionalStreamBuilderImpl(String url, BidirectionalStream.Callback callback, + Executor executor, CronvoyEngineBase cronetEngine) { super(); if (url == null) { throw new NullPointerException("URL is required."); @@ -75,7 +75,7 @@ final class BidirectionalStreamBuilderImpl extends ExperimentalBidirectionalStre } @Override - public BidirectionalStreamBuilderImpl setHttpMethod(String method) { + public CronvoyBidirectionalStreamBuilderImpl setHttpMethod(String method) { if (method == null) { throw new NullPointerException("Method is required."); } @@ -84,7 +84,7 @@ public BidirectionalStreamBuilderImpl setHttpMethod(String method) { } @Override - public BidirectionalStreamBuilderImpl addHeader(String header, String value) { + public CronvoyBidirectionalStreamBuilderImpl addHeader(String header, String value) { if (header == null) { throw new NullPointerException("Invalid header name."); } @@ -96,13 +96,14 @@ public BidirectionalStreamBuilderImpl addHeader(String header, String value) { } @Override - public BidirectionalStreamBuilderImpl setPriority(@CronetEngineBase.StreamPriority int priority) { + public CronvoyBidirectionalStreamBuilderImpl + setPriority(@CronvoyEngineBase.StreamPriority int priority) { mPriority = priority; return this; } @Override - public BidirectionalStreamBuilderImpl + public CronvoyBidirectionalStreamBuilderImpl delayRequestHeadersUntilFirstFlush(boolean delayRequestHeadersUntilFirstFlush) { mDelayRequestHeadersUntilFirstFlush = delayRequestHeadersUntilFirstFlush; return this; diff --git a/mobile/library/java/org/chromium/net/impl/BidirectionalStreamNetworkException.java b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStreamNetworkException.java similarity index 65% rename from mobile/library/java/org/chromium/net/impl/BidirectionalStreamNetworkException.java rename to mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStreamNetworkException.java index 0fe8e8d2dbb8..6a8f7cbb703a 100644 --- a/mobile/library/java/org/chromium/net/impl/BidirectionalStreamNetworkException.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStreamNetworkException.java @@ -3,11 +3,11 @@ import org.chromium.net.impl.Errors.NetError; /** - * Used in {@link CronetBidirectionalStream}. Implements {@link NetworkExceptionImpl}. + * Used in {@link CronetBidirectionalStream}. Implements {@link CronvoyNetworkExceptionImpl}. */ -public final class BidirectionalStreamNetworkException extends NetworkExceptionImpl { - public BidirectionalStreamNetworkException(String message, int errorCode, - int cronetInternalErrorCode) { +public final class CronvoyBidirectionalStreamNetworkException extends CronvoyNetworkExceptionImpl { + public CronvoyBidirectionalStreamNetworkException(String message, int errorCode, + int cronetInternalErrorCode) { super(message, errorCode, cronetInternalErrorCode); } diff --git a/mobile/library/java/org/chromium/net/impl/CronvoyCallbackExceptionImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyCallbackExceptionImpl.java new file mode 100644 index 000000000000..bd7f63208dfe --- /dev/null +++ b/mobile/library/java/org/chromium/net/impl/CronvoyCallbackExceptionImpl.java @@ -0,0 +1,8 @@ +package org.chromium.net.impl; + +import org.chromium.net.CallbackException; + +/** An implementation of {@link CallbackException}. */ +public final class CronvoyCallbackExceptionImpl extends CallbackException { + CronvoyCallbackExceptionImpl(String message, Throwable cause) { super(message, cause); } +} diff --git a/mobile/library/java/org/chromium/net/impl/CronetEngineBase.java b/mobile/library/java/org/chromium/net/impl/CronvoyEngineBase.java similarity index 95% rename from mobile/library/java/org/chromium/net/impl/CronetEngineBase.java rename to mobile/library/java/org/chromium/net/impl/CronvoyEngineBase.java index 59c26b4523b0..fbff50dcdf0f 100644 --- a/mobile/library/java/org/chromium/net/impl/CronetEngineBase.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyEngineBase.java @@ -17,10 +17,9 @@ import org.chromium.net.UrlRequest; /** - * Base class of {@link CronetUrlRequestContext} and {@link JavaCronetEngine} that contains - * shared logic. + * Base class of {@link CronvoyUrlRequestContext} */ -abstract class CronetEngineBase extends ExperimentalCronetEngine { +abstract class CronvoyEngineBase extends ExperimentalCronetEngine { /** * Creates a {@link UrlRequest} object. All callbacks will be called on {@code executor}'s thread. @@ -52,7 +51,7 @@ abstract class CronetEngineBase extends ExperimentalCronetEngine { * ExperimentalUrlRequest.Builder#DEFAULT_IDEMPOTENCY IDEMPOTENT NOT_IDEMPOTENT} values. * @return new request. */ - abstract UrlRequestBase createRequest( + abstract CronvoyUrlRequestBase createRequest( String url, UrlRequest.Callback callback, Executor executor, @RequestPriority int priority, Collection requestAnnotations, boolean disableCache, boolean disableConnectionMigration, boolean allowDirectExecutor, boolean trafficStatsTagSet, @@ -93,7 +92,7 @@ abstract ExperimentalBidirectionalStream createBidirectionalStream( @Override public ExperimentalUrlRequest.Builder newUrlRequestBuilder(String url, UrlRequest.Callback callback, Executor executor) { - return new UrlRequestBuilderImpl(url, callback, executor, this); + return new CronvoyUrlRequestBuilderImpl(url, callback, executor, this); } @IntDef({UrlRequest.Builder.REQUEST_PRIORITY_IDLE, UrlRequest.Builder.REQUEST_PRIORITY_LOWEST, diff --git a/mobile/library/java/org/chromium/net/impl/CronetEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java similarity index 86% rename from mobile/library/java/org/chromium/net/impl/CronetEngineBuilderImpl.java rename to mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java index 23d20b39c220..41aa4ddcf644 100644 --- a/mobile/library/java/org/chromium/net/impl/CronetEngineBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java @@ -21,7 +21,7 @@ import org.chromium.net.impl.Annotations.HttpCacheType; /** Implementation of {@link ICronetEngineBuilder} that builds Envoy-Mobile based Cronet engine. */ -public abstract class CronetEngineBuilderImpl extends ICronetEngineBuilder { +public abstract class CronvoyEngineBuilderImpl extends ICronetEngineBuilder { /** A hint that a host supports QUIC. */ final static class QuicHint { @@ -86,7 +86,7 @@ final static class Pkp { * * @param context Android {@link Context} for engine to use. */ - CronetEngineBuilderImpl(Context context) { + CronvoyEngineBuilderImpl(Context context) { mApplicationContext = context.getApplicationContext(); enableQuic(true); enableHttp2(true); @@ -98,11 +98,11 @@ final static class Pkp { @Override public String getDefaultUserAgent() { - return UserAgent.from(mApplicationContext); + return CronvoyUserAgent.from(mApplicationContext); } @Override - public CronetEngineBuilderImpl setUserAgent(String userAgent) { + public CronvoyEngineBuilderImpl setUserAgent(String userAgent) { mUserAgent = userAgent; return this; } @@ -110,7 +110,7 @@ public CronetEngineBuilderImpl setUserAgent(String userAgent) { String getUserAgent() { return mUserAgent; } @Override - public CronetEngineBuilderImpl setStoragePath(String value) { + public CronvoyEngineBuilderImpl setStoragePath(String value) { if (!new File(value).isDirectory()) { throw new IllegalArgumentException("Storage path must be set to existing directory"); } @@ -121,7 +121,7 @@ public CronetEngineBuilderImpl setStoragePath(String value) { String storagePath() { return mStoragePath; } @Override - public CronetEngineBuilderImpl setLibraryLoader(CronetEngine.Builder.LibraryLoader loader) { + public CronvoyEngineBuilderImpl setLibraryLoader(CronetEngine.Builder.LibraryLoader loader) { // |CronvoyEngineBuilderImpl| is an abstract class that is used by concrete builder // implementations, including the Java Cronet engine builder; therefore, the implementation // of this method should be "no-op". Subclasses that care about the library loader @@ -132,10 +132,10 @@ public CronetEngineBuilderImpl setLibraryLoader(CronetEngine.Builder.LibraryLoad /** * Default implementation of the method that returns {@code null}. */ - VersionSafeCallbacks.LibraryLoader libraryLoader() { return null; } + CronvoyVersionSafeCallbacks.LibraryLoader libraryLoader() { return null; } @Override - public CronetEngineBuilderImpl enableQuic(boolean value) { + public CronvoyEngineBuilderImpl enableQuic(boolean value) { mQuicEnabled = value; return this; } @@ -149,11 +149,11 @@ public CronetEngineBuilderImpl enableQuic(boolean value) { * @return QUIC User Agent ID string. */ String getDefaultQuicUserAgentId() { - return mQuicEnabled ? UserAgent.getQuicUserAgentIdFrom(mApplicationContext) : ""; + return mQuicEnabled ? CronvoyUserAgent.getQuicUserAgentIdFrom(mApplicationContext) : ""; } @Override - public CronetEngineBuilderImpl enableHttp2(boolean value) { + public CronvoyEngineBuilderImpl enableHttp2(boolean value) { mHttp2Enabled = value; return this; } @@ -161,12 +161,12 @@ public CronetEngineBuilderImpl enableHttp2(boolean value) { boolean http2Enabled() { return mHttp2Enabled; } @Override - public CronetEngineBuilderImpl enableSdch(boolean value) { + public CronvoyEngineBuilderImpl enableSdch(boolean value) { return this; } @Override - public CronetEngineBuilderImpl enableBrotli(boolean value) { + public CronvoyEngineBuilderImpl enableBrotli(boolean value) { mBrotiEnabled = value; return this; } @@ -179,7 +179,7 @@ public CronetEngineBuilderImpl enableBrotli(boolean value) { public @interface HttpCacheSetting {} @Override - public CronetEngineBuilderImpl enableHttpCache(@HttpCacheSetting int cacheMode, long maxSize) { + public CronvoyEngineBuilderImpl enableHttpCache(@HttpCacheSetting int cacheMode, long maxSize) { if (cacheMode == CronetEngine.Builder.HTTP_CACHE_DISK || cacheMode == CronetEngine.Builder.HTTP_CACHE_DISK_NO_HTTP) { if (storagePath() == null) { @@ -218,7 +218,7 @@ public CronetEngineBuilderImpl enableHttpCache(@HttpCacheSetting int cacheMode, int httpCacheMode() { return mHttpCacheMode; } @Override - public CronetEngineBuilderImpl addQuicHint(String host, int port, int alternatePort) { + public CronvoyEngineBuilderImpl addQuicHint(String host, int port, int alternatePort) { if (host.contains("/")) { throw new IllegalArgumentException("Illegal QUIC Hint Host: " + host); } @@ -229,8 +229,8 @@ public CronetEngineBuilderImpl addQuicHint(String host, int port, int alternateP List quicHints() { return mQuicHints; } @Override - public CronetEngineBuilderImpl addPublicKeyPins(String hostName, Set pinsSha256, - boolean includeSubdomains, Date expirationDate) { + public CronvoyEngineBuilderImpl addPublicKeyPins(String hostName, Set pinsSha256, + boolean includeSubdomains, Date expirationDate) { if (hostName == null) { throw new NullPointerException("The hostname cannot be null"); } @@ -263,7 +263,7 @@ public CronetEngineBuilderImpl addPublicKeyPins(String hostName, Set pin List publicKeyPins() { return mPkps; } @Override - public CronetEngineBuilderImpl enablePublicKeyPinningBypassForLocalTrustAnchors(boolean value) { + public CronvoyEngineBuilderImpl enablePublicKeyPinningBypassForLocalTrustAnchors(boolean value) { mPublicKeyPinningBypassForLocalTrustAnchorsEnabled = value; return this; } @@ -309,7 +309,7 @@ private static String validateHostNameForPinningAndConvert(String hostName) } @Override - public CronetEngineBuilderImpl setExperimentalOptions(String options) { + public CronvoyEngineBuilderImpl setExperimentalOptions(String options) { mExperimentalOptions = options; return this; } @@ -323,13 +323,13 @@ public CronetEngineBuilderImpl setExperimentalOptions(String options) { boolean networkQualityEstimatorEnabled() { return mNetworkQualityEstimatorEnabled; } @Override - public CronetEngineBuilderImpl enableNetworkQualityEstimator(boolean value) { + public CronvoyEngineBuilderImpl enableNetworkQualityEstimator(boolean value) { mNetworkQualityEstimatorEnabled = value; return this; } @Override - public CronetEngineBuilderImpl setThreadPriority(int priority) { + public CronvoyEngineBuilderImpl setThreadPriority(int priority) { if (priority > THREAD_PRIORITY_LOWEST || priority < -20) { throw new IllegalArgumentException("Thread priority invalid"); } @@ -344,7 +344,7 @@ int threadPriority(int defaultThreadPriority) { return mThreadPriority == INVALID_THREAD_PRIORITY ? defaultThreadPriority : mThreadPriority; } - public CronetEngineBuilderImpl setLogLevel(String logLevel) { + public CronvoyEngineBuilderImpl setLogLevel(String logLevel) { mLogLevel = logLevel; return this; } diff --git a/mobile/library/java/org/chromium/net/impl/CronvoyExceptionImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyExceptionImpl.java new file mode 100644 index 000000000000..8c4dd9451714 --- /dev/null +++ b/mobile/library/java/org/chromium/net/impl/CronvoyExceptionImpl.java @@ -0,0 +1,9 @@ +package org.chromium.net.impl; + +import org.chromium.net.CronetException; + +/** Implements {@link CronetException}. */ +final class CronvoyExceptionImpl extends CronetException { + + CronvoyExceptionImpl(String message, Throwable cause) { super(message, cause); } +} diff --git a/mobile/library/java/org/chromium/net/impl/ImplVersion.java b/mobile/library/java/org/chromium/net/impl/CronvoyImplVersion.java similarity index 92% rename from mobile/library/java/org/chromium/net/impl/ImplVersion.java rename to mobile/library/java/org/chromium/net/impl/CronvoyImplVersion.java index f4f70673f02d..887bd16a0c8e 100644 --- a/mobile/library/java/org/chromium/net/impl/ImplVersion.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyImplVersion.java @@ -1,7 +1,7 @@ package org.chromium.net.impl; // Version based on chrome/VERSION. -public final class ImplVersion { +public final class CronvoyImplVersion { // TODO(carloseltuerto) make this class a template and use "@MAJOR@.@MINOR@.@BUILD@.@PATCH@" private static final String CRONET_VERSION = "99.0.4512.7"; // TODO(carloseltuerto) make this class a template and use @API_LEVEL@; @@ -12,7 +12,7 @@ public final class ImplVersion { /** * Private constructor. All members of this class should be static. */ - private ImplVersion() {} + private CronvoyImplVersion() {} public static String getCronetVersionWithLastChange() { return CRONET_VERSION + "@" + LAST_CHANGE.substring(0, 8); diff --git a/mobile/library/java/org/chromium/net/impl/CronetMetrics.java b/mobile/library/java/org/chromium/net/impl/CronvoyMetrics.java similarity index 90% rename from mobile/library/java/org/chromium/net/impl/CronetMetrics.java rename to mobile/library/java/org/chromium/net/impl/CronvoyMetrics.java index c34b22894b91..bbada569073d 100644 --- a/mobile/library/java/org/chromium/net/impl/CronetMetrics.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyMetrics.java @@ -7,7 +7,7 @@ /** * Implementation of {@link RequestFinishedInfo.Metrics}. */ -public final class CronetMetrics extends RequestFinishedInfo.Metrics { +public final class CronvoyMetrics extends RequestFinishedInfo.Metrics { private final long mRequestStartMs; private final long mDnsStartMs; private final long mDnsEndMs; @@ -43,11 +43,11 @@ private static boolean checkOrder(long start, long end) { } /** New-style constructor */ - public CronetMetrics(long requestStartMs, long dnsStartMs, long dnsEndMs, long connectStartMs, - long connectEndMs, long sslStartMs, long sslEndMs, long sendingStartMs, - long sendingEndMs, long pushStartMs, long pushEndMs, long responseStartMs, - long requestEndMs, boolean socketReused, long sentByteCount, - long receivedByteCount) { + public CronvoyMetrics(long requestStartMs, long dnsStartMs, long dnsEndMs, long connectStartMs, + long connectEndMs, long sslStartMs, long sslEndMs, long sendingStartMs, + long sendingEndMs, long pushStartMs, long pushEndMs, long responseStartMs, + long requestEndMs, boolean socketReused, long sentByteCount, + long receivedByteCount) { // Check that no end times are before corresponding start times, // or exist when start time doesn't. assert checkOrder(dnsStartMs, dnsEndMs); diff --git a/mobile/library/java/org/chromium/net/impl/NetworkExceptionImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyNetworkExceptionImpl.java similarity index 91% rename from mobile/library/java/org/chromium/net/impl/NetworkExceptionImpl.java rename to mobile/library/java/org/chromium/net/impl/CronvoyNetworkExceptionImpl.java index f5bb618a73f0..a0888d9fd473 100644 --- a/mobile/library/java/org/chromium/net/impl/NetworkExceptionImpl.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyNetworkExceptionImpl.java @@ -5,7 +5,7 @@ /** * Implements {@link NetworkException}. */ -public class NetworkExceptionImpl extends NetworkException { +public class CronvoyNetworkExceptionImpl extends NetworkException { // Error code, one of ERROR_* protected final int mErrorCode; // Cronet internal error code. @@ -20,7 +20,7 @@ public class NetworkExceptionImpl extends NetworkException { * * these. */ - public NetworkExceptionImpl(String message, int errorCode, int cronetInternalErrorCode) { + public CronvoyNetworkExceptionImpl(String message, int errorCode, int cronetInternalErrorCode) { super(message, null); assert errorCode > 0 && errorCode < 12; assert cronetInternalErrorCode < 0; diff --git a/mobile/library/java/org/chromium/net/impl/Preconditions.java b/mobile/library/java/org/chromium/net/impl/CronvoyPreconditions.java similarity index 85% rename from mobile/library/java/org/chromium/net/impl/Preconditions.java rename to mobile/library/java/org/chromium/net/impl/CronvoyPreconditions.java index e24a61f7c629..5232c5f8233f 100644 --- a/mobile/library/java/org/chromium/net/impl/Preconditions.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyPreconditions.java @@ -5,7 +5,7 @@ /** * Utility class to check preconditions. */ -public final class Preconditions { +public final class CronvoyPreconditions { public static void checkDirect(ByteBuffer buffer) { if (!buffer.isDirect()) { @@ -19,5 +19,5 @@ public static void checkHasRemaining(ByteBuffer buffer) { } } - private Preconditions() {} + private CronvoyPreconditions() {} } diff --git a/mobile/library/java/org/chromium/net/impl/QuicExceptionImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyQuicExceptionImpl.java similarity index 79% rename from mobile/library/java/org/chromium/net/impl/QuicExceptionImpl.java rename to mobile/library/java/org/chromium/net/impl/CronvoyQuicExceptionImpl.java index 113550dcaeb0..80510998be20 100644 --- a/mobile/library/java/org/chromium/net/impl/QuicExceptionImpl.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyQuicExceptionImpl.java @@ -5,9 +5,9 @@ /** * Implements {@link QuicException}. */ -public class QuicExceptionImpl extends QuicException { +public class CronvoyQuicExceptionImpl extends QuicException { private final int mQuicDetailedErrorCode; - private final NetworkExceptionImpl mNetworkException; + private final CronvoyNetworkExceptionImpl mNetworkException; /** * Constructs an exception with a specific error. @@ -21,10 +21,10 @@ public class QuicExceptionImpl extends QuicException { * href="https://cs.chromium.org/search/?q=symbol:%5CbQuicErrorCode%5Cb"> * QuicErrorCode. */ - public QuicExceptionImpl(String message, int errorCode, int netErrorCode, - int quicDetailedErrorCode) { + public CronvoyQuicExceptionImpl(String message, int errorCode, int netErrorCode, + int quicDetailedErrorCode) { super(message, null); - mNetworkException = new NetworkExceptionImpl(message, errorCode, netErrorCode); + mNetworkException = new CronvoyNetworkExceptionImpl(message, errorCode, netErrorCode); mQuicDetailedErrorCode = quicDetailedErrorCode; } diff --git a/mobile/library/java/org/chromium/net/impl/RequestFinishedInfoImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyRequestFinishedInfoImpl.java similarity index 77% rename from mobile/library/java/org/chromium/net/impl/RequestFinishedInfoImpl.java rename to mobile/library/java/org/chromium/net/impl/CronvoyRequestFinishedInfoImpl.java index 8099a5777725..f091ba791627 100644 --- a/mobile/library/java/org/chromium/net/impl/RequestFinishedInfoImpl.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyRequestFinishedInfoImpl.java @@ -13,7 +13,7 @@ /** * Implements information about a finished request. Passed to {@link RequestFinishedInfo.Listener}. */ -public class RequestFinishedInfoImpl extends RequestFinishedInfo { +public class CronvoyRequestFinishedInfoImpl extends RequestFinishedInfo { private final String mUrl; private final Collection mAnnotations; private final RequestFinishedInfo.Metrics mMetrics; @@ -27,11 +27,11 @@ public class RequestFinishedInfoImpl extends RequestFinishedInfo { @Retention(RetentionPolicy.SOURCE) public @interface FinishedReason {} - public RequestFinishedInfoImpl(String url, Collection annotations, - RequestFinishedInfo.Metrics metrics, - @FinishedReason int finishedReason, - @Nullable UrlResponseInfo responseInfo, - @Nullable CronetException exception) { + public CronvoyRequestFinishedInfoImpl(String url, Collection annotations, + RequestFinishedInfo.Metrics metrics, + @FinishedReason int finishedReason, + @Nullable UrlResponseInfo responseInfo, + @Nullable CronetException exception) { mUrl = url; mAnnotations = annotations; mMetrics = metrics; diff --git a/mobile/library/java/org/chromium/net/impl/CronetUploadDataStream.java b/mobile/library/java/org/chromium/net/impl/CronvoyUploadDataStream.java similarity index 92% rename from mobile/library/java/org/chromium/net/impl/CronetUploadDataStream.java rename to mobile/library/java/org/chromium/net/impl/CronvoyUploadDataStream.java index a6943ff2b942..1f9141b634c2 100644 --- a/mobile/library/java/org/chromium/net/impl/CronetUploadDataStream.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyUploadDataStream.java @@ -13,19 +13,19 @@ import org.chromium.net.impl.Executors.DirectPreventingExecutor; /** - * CronetUploadDataStream handles communication between an upload body + * CronvoyUploadDataStream handles communication between an upload body * encapsulated in the embedder's {@link UploadDataSink}. */ -public final class CronetUploadDataStream extends UploadDataSink { +public final class CronvoyUploadDataStream extends UploadDataSink { - private static final String TAG = CronetUploadDataStream.class.getSimpleName(); + private static final String TAG = CronvoyUploadDataStream.class.getSimpleName(); private static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.allocateDirect(0); private final int BYTE_BUFFER_SIZE = 65535; // H2 initial_stream_window_size // These are never changed, once a request starts. private final Executor mExecutor; - private final VersionSafeCallbacks.UploadDataProviderWrapper mDataProvider; - private final CronetUrlRequest mRequest; + private final CronvoyVersionSafeCallbacks.UploadDataProviderWrapper mDataProvider; + private final CronvoyUrlRequest mRequest; private long mLength; private long mRemainingLength; @@ -67,10 +67,10 @@ public void run() { * @param dataProvider the UploadDataProvider to read data from. * @param executor the Executor to execute UploadDataProvider tasks. */ - public CronetUploadDataStream(UploadDataProvider dataProvider, Executor executor, - CronetUrlRequest request) { + public CronvoyUploadDataStream(UploadDataProvider dataProvider, Executor executor, + CronvoyUrlRequest request) { mExecutor = executor; - mDataProvider = new VersionSafeCallbacks.UploadDataProviderWrapper(dataProvider); + mDataProvider = new CronvoyVersionSafeCallbacks.UploadDataProviderWrapper(dataProvider); mRequest = request; } @@ -107,7 +107,7 @@ public void run() { mInWhichUserCallback = UserCallback.REWIND; } try { - mDataProvider.rewind(CronetUploadDataStream.this); + mDataProvider.rewind(CronvoyUploadDataStream.this); } catch (Exception exception) { onError(exception); } @@ -140,7 +140,7 @@ private void read() { mByteBuffer = mRemainingLength < 0 || mRemainingLength > BYTE_BUFFER_SIZE ? ByteBuffer.allocateDirect(BYTE_BUFFER_SIZE) : ByteBuffer.allocate(mByteBufferLimit); - mDataProvider.read(CronetUploadDataStream.this, mByteBuffer); + mDataProvider.read(CronvoyUploadDataStream.this, mByteBuffer); } catch (Exception exception) { onError(exception); } diff --git a/mobile/library/java/org/chromium/net/impl/CronetUrlRequest.java b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java similarity index 90% rename from mobile/library/java/org/chromium/net/impl/CronetUrlRequest.java rename to mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java index 538b5fb24ba5..e89ee7716c08 100644 --- a/mobile/library/java/org/chromium/net/impl/CronetUrlRequest.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java @@ -41,7 +41,7 @@ import org.chromium.net.impl.Errors.NetError; /** UrlRequest, backed by Envoy-Mobile. */ -public final class CronetUrlRequest extends UrlRequestBase { +public final class CronvoyUrlRequest extends CronvoyUrlRequestBase { /** * State interface for keeping track of the internal state of a {@link UrlRequestBase}. @@ -101,7 +101,7 @@ public final class CronetUrlRequest extends UrlRequestBase { private static final String X_ENVOY = "x-envoy"; private static final String X_ENVOY_UPSTREAM_ALPN = "x-envoy-upstream-alpn"; - private static final String TAG = CronetUrlRequest.class.getSimpleName(); + private static final String TAG = CronvoyUrlRequest.class.getSimpleName(); private static final String USER_AGENT = "User-Agent"; private static final String CONTENT_TYPE = "Content-Type"; private static final Executor DIRECT_EXECUTOR = new DirectExecutor(); @@ -109,7 +109,7 @@ public final class CronetUrlRequest extends UrlRequestBase { private final String mUserAgent; private final HeadersList mRequestHeaders = new HeadersList(); private final Collection mRequestAnnotations; - private final CronetUrlRequestContext mRequestContext; + private final CronvoyUrlRequestContext mRequestContext; private final AtomicBoolean mWaitingOnRedirect = new AtomicBoolean(false); private final AtomicBoolean mWaitingOnRead = new AtomicBoolean(false); private volatile ByteBuffer mUserCurrentReadBuffer = null; @@ -136,10 +136,10 @@ public final class CronetUrlRequest extends UrlRequestBase { private final AtomicCombinatoryState mSucceededState = new AtomicCombinatoryState(SucceededState.SUCCESS_READY); /** - * Ensures that the CronetMetrics will be posted after observing the required events, and that it + * Ensures that the CronvoyMetrics will be posted after observing the required events, and that it * will be done only once. * - *

At the end of a request, mRequestFinishedListener is used to post the CronetMetrics. Before + *

At the end of a request, mRequestFinishedListener is used to post the CronvoyMetrics. Before * doing so, two events must have occurred first: the "final user callback" and the "final Network * callback". The Thread involved with the last of these two events is in charge of the posting - * this is intrinsically racy. @@ -152,12 +152,12 @@ public final class CronetUrlRequest extends UrlRequestBase { /* These don't change with redirects */ private String mInitialMethod; private final Executor mUserExecutor; - private final VersionSafeCallbacks.UrlRequestCallback mCallback; + private final CronvoyVersionSafeCallbacks.UrlRequestCallback mCallback; private final String mInitialUrl; - private final VersionSafeCallbacks.RequestFinishedInfoListener mRequestFinishedListener; + private final CronvoyVersionSafeCallbacks.RequestFinishedInfoListener mRequestFinishedListener; private final ConditionVariable mStartBlock = new ConditionVariable(); - private CronetUploadDataStream mUploadDataStream; + private CronvoyUploadDataStream mUploadDataStream; private volatile CronetException mException; @@ -181,17 +181,17 @@ public final class CronetUrlRequest extends UrlRequestBase { private long mBytesReceivedFromLastRedirect = 0; private CronvoyHttpCallbacks mCronvoyCallbacks; private String mCurrentUrl; - private volatile UrlResponseInfoImpl mUrlResponseInfo; + private volatile CronvoyUrlResponseInfoImpl mUrlResponseInfo; private String mPendingRedirectUrl; /** * @param executor The executor for orchestrating tasks between envoy-mobile callbacks */ - CronetUrlRequest(CronetUrlRequestContext cronvoyEngine, String url, Callback callback, - Executor executor, String userAgent, boolean allowDirectExecutor, - Collection connectionAnnotations, boolean trafficStatsTagSet, - int trafficStatsTag, boolean trafficStatsUidSet, int trafficStatsUid, - RequestFinishedInfo.Listener requestFinishedListener) { + CronvoyUrlRequest(CronvoyUrlRequestContext cronvoyEngine, String url, Callback callback, + Executor executor, String userAgent, boolean allowDirectExecutor, + Collection connectionAnnotations, boolean trafficStatsTagSet, + int trafficStatsTag, boolean trafficStatsUidSet, int trafficStatsUid, + RequestFinishedInfo.Listener requestFinishedListener) { if (url == null) { throw new NullPointerException("URL is required"); } @@ -201,10 +201,10 @@ public final class CronetUrlRequest extends UrlRequestBase { if (executor == null) { throw new NullPointerException("Executor is required"); } - mCallback = new VersionSafeCallbacks.UrlRequestCallback(callback); + mCallback = new CronvoyVersionSafeCallbacks.UrlRequestCallback(callback); mRequestFinishedListener = requestFinishedListener != null - ? new VersionSafeCallbacks.RequestFinishedInfoListener(requestFinishedListener) + ? new CronvoyVersionSafeCallbacks.RequestFinishedInfoListener(requestFinishedListener) : null; mRequestContext = cronvoyEngine; mAllowDirectExecutor = allowDirectExecutor; @@ -286,7 +286,7 @@ public void setUploadDataProvider(UploadDataProvider uploadDataProvider, Executo if (mInitialMethod == null) { mInitialMethod = "POST"; } - mUploadDataStream = new CronetUploadDataStream(uploadDataProvider, executor, this); + mUploadDataStream = new CronvoyUploadDataStream(uploadDataProvider, executor, this); } @Override @@ -302,8 +302,8 @@ public void start() { @Override public void read(final ByteBuffer buffer) { - Preconditions.checkDirect(buffer); - Preconditions.checkHasRemaining(buffer); + CronvoyPreconditions.checkDirect(buffer); + CronvoyPreconditions.checkHasRemaining(buffer); if (!mWaitingOnRead.compareAndSet(true, false)) { throw new IllegalStateException("Unexpected read attempt."); } @@ -386,7 +386,7 @@ public void getStatus(StatusListener listener) { throw new IllegalStateException("Switch is exhaustive: " + state); } - sendStatus(new VersionSafeCallbacks.UrlRequestStatusListener(listener), status); + sendStatus(new CronvoyVersionSafeCallbacks.UrlRequestStatusListener(listener), status); } @State @@ -496,7 +496,7 @@ private static boolean isTerminalState(@State int state) { } private void enterCronetErrorState(final Throwable error) { - enterErrorState(new CronetExceptionImpl("System error", error)); + enterErrorState(new CronvoyExceptionImpl("System error", error)); } /** @@ -550,7 +550,7 @@ private void fireOpenConnection() { private static Map> buildEnvoyRequestHeaders(String initialMethod, HeadersList headersList, - CronetUploadDataStream mUploadDataStream, String userAgent, + CronvoyUploadDataStream mUploadDataStream, String userAgent, String currentUrl, boolean isQuicEnabled) { Map> headers = new LinkedHashMap<>(); final URL url; @@ -592,8 +592,8 @@ private void fireOpenConnection() { */ private void onCallbackException(Throwable t) { CallbackException requestError = - new CallbackExceptionImpl("Exception received from UrlRequest.Callback", t); - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception in CalledByNative method", t); + new CronvoyCallbackExceptionImpl("Exception received from UrlRequest.Callback", t); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception in CalledByNative method", t); enterErrorState(requestError); } @@ -602,13 +602,14 @@ private void onCallbackException(Throwable t) { */ void onUploadException(Exception t) { CallbackException uploadError = - new CallbackExceptionImpl("Exception received from UploadDataProvider", t); - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception in upload method", t); + new CronvoyCallbackExceptionImpl("Exception received from UploadDataProvider", t); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception in upload method", t); enterErrorState(uploadError); } /** This wrapper ensures that callbacks are always called on the correct executor */ - void sendStatus(final VersionSafeCallbacks.UrlRequestStatusListener listener, final int status) { + void sendStatus(final CronvoyVersionSafeCallbacks.UrlRequestStatusListener listener, + final int status) { mUserExecutor.execute(() -> listener.onStatus(status)); } @@ -616,7 +617,7 @@ void execute(Runnable runnable) { try { mUserExecutor.execute(runnable); } catch (RejectedExecutionException e) { - enterErrorState(new CronetExceptionImpl("Exception posting task to executor", e)); + enterErrorState(new CronvoyExceptionImpl("Exception posting task to executor", e)); } } @@ -625,10 +626,10 @@ void onCanceled() { @Override public void run() { try { - mCallback.onCanceled(CronetUrlRequest.this, mUrlResponseInfo); + mCallback.onCanceled(CronvoyUrlRequest.this, mUrlResponseInfo); maybeReportMetrics(ReportState.USER_FINAL_CALLBACK_DONE); } catch (Exception exception) { - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception in onCanceled method", exception); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception in onCanceled method", exception); } } }; @@ -640,10 +641,10 @@ void onSucceeded() { @Override public void run() { try { - mCallback.onSucceeded(CronetUrlRequest.this, mUrlResponseInfo); + mCallback.onSucceeded(CronvoyUrlRequest.this, mUrlResponseInfo); maybeReportMetrics(ReportState.USER_FINAL_CALLBACK_DONE); } catch (Exception exception) { - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception in onSucceeded method", exception); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception in onSucceeded method", exception); } } }; @@ -655,10 +656,10 @@ void onFailed() { @Override public void run() { try { - mCallback.onFailed(CronetUrlRequest.this, mUrlResponseInfo, mException); + mCallback.onFailed(CronvoyUrlRequest.this, mUrlResponseInfo, mException); maybeReportMetrics(ReportState.USER_FINAL_CALLBACK_DONE); } catch (Exception exception) { - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception in onFailed method", exception); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception in onFailed method", exception); } } }; @@ -713,8 +714,8 @@ private void maybeReportMetrics(@ReportState int reportStateEvent) { } Metrics metrics = getMetrics(mEnvoyFinalStreamIntel, mBytesReceivedFromRedirects); final RequestFinishedInfo requestInfo = - new RequestFinishedInfoImpl(mInitialUrl, mRequestAnnotations, metrics, getFinishedReason(), - mUrlResponseInfo, mException); + new CronvoyRequestFinishedInfoImpl(mInitialUrl, mRequestAnnotations, metrics, + getFinishedReason(), mUrlResponseInfo, mException); mRequestContext.reportRequestFinished(requestInfo); if (mRequestFinishedListener != null) { try { @@ -725,13 +726,14 @@ public void run() { } }); } catch (RejectedExecutionException failException) { - Log.e(CronetUrlRequestContext.LOG_TAG, "Exception posting task to executor", failException); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception posting task to executor", + failException); } } } private static Metrics getMetrics(EnvoyFinalStreamIntel intel, long bytesReceivedFromRedirects) { - return new CronetMetrics( + return new CronvoyMetrics( intel.getStreamStartMs(), intel.getDnsStartMs(), intel.getDnsEndMs(), intel.getConnectStartMs(), intel.getConnectEndMs(), intel.getSslStartMs(), intel.getSslEndMs(), intel.getSendingStartMs(), intel.getSendingEndMs(), @@ -740,15 +742,15 @@ private static Metrics getMetrics(EnvoyFinalStreamIntel intel, long bytesReceive intel.getReceivedByteCount() + bytesReceivedFromRedirects); } - @RequestFinishedInfoImpl.FinishedReason + @CronvoyRequestFinishedInfoImpl.FinishedReason private int getFinishedReason() { switch (mState.get()) { case State.COMPLETE: - return RequestFinishedInfoImpl.SUCCEEDED; + return CronvoyRequestFinishedInfoImpl.SUCCEEDED; case State.CANCELLED: - return RequestFinishedInfoImpl.CANCELED; + return CronvoyRequestFinishedInfoImpl.CANCELED; default: - return RequestFinishedInfoImpl.FAILED; + return CronvoyRequestFinishedInfoImpl.FAILED; } } @@ -786,7 +788,7 @@ public Executor getExecutor() { @Override public void onHeaders(Map> headers, boolean endStream, EnvoyStreamIntel streamIntel) { - mUrlResponseInfo = new UrlResponseInfoImpl(); + mUrlResponseInfo = new CronvoyUrlResponseInfoImpl(); recordEnvoyStreamIntel(streamIntel); mEndStream = endStream; List statuses = headers.get(":status"); @@ -834,7 +836,7 @@ public void run() { mStream.set(null); mPendingRedirectUrl = URI.create(mCurrentUrl).resolve(locationField).toString(); mWaitingOnRedirect.set(true); - mCallback.onRedirectReceived(CronetUrlRequest.this, mUrlResponseInfo, + mCallback.onRedirectReceived(CronvoyUrlRequest.this, mUrlResponseInfo, mPendingRedirectUrl); } else { if (responseCode < 300 || responseCode >= 400) { @@ -842,7 +844,7 @@ public void run() { } fireCloseUploadDataProvider(); // Idempotent mWaitingOnRead.set(true); - mCallback.onResponseStarted(CronetUrlRequest.this, mUrlResponseInfo); + mCallback.onResponseStarted(CronvoyUrlRequest.this, mUrlResponseInfo); } } catch (Throwable t) { onCallbackException(t); @@ -881,7 +883,7 @@ public void run() { mUserCurrentReadBuffer = null; // Avoid the reference to a potentially large buffer. userBuffer.put(data); // NPE ==> BUG, BufferOverflowException ==> User not behaving. mWaitingOnRead.set(true); - mCallback.onReadCompleted(CronetUrlRequest.this, mUrlResponseInfo, userBuffer); + mCallback.onReadCompleted(CronvoyUrlRequest.this, mUrlResponseInfo, userBuffer); } catch (Throwable t) { onCallbackException(t); } @@ -933,14 +935,14 @@ public void onError(int errorCode, String message, int attemptCount, int javaError = mapNetErrorToCronetApiErrorCode(netError); if (isQuicException(javaError)) { - enterErrorState(new QuicExceptionImpl("Exception in CronetUrlRequest: " + netError, - javaError, netError.getErrorCode(), - Errors.QUIC_INTERNAL_ERROR)); + enterErrorState(new CronvoyQuicExceptionImpl("Exception in CronvoyUrlRequest: " + netError, + javaError, netError.getErrorCode(), + Errors.QUIC_INTERNAL_ERROR)); return; } - enterErrorState(new NetworkExceptionImpl("Exception in CronetUrlRequest: " + netError, - javaError, netError.getErrorCode())); + enterErrorState(new CronvoyNetworkExceptionImpl("Exception in CronvoyUrlRequest: " + netError, + javaError, netError.getErrorCode())); } @Override @@ -960,7 +962,7 @@ public void onCancel(EnvoyStreamIntel streamIntel, EnvoyFinalStreamIntel finalSt return; } - CronetException exception = new CronetExceptionImpl("Cancelled", /* cause= */ null); + CronetException exception = new CronvoyExceptionImpl("Cancelled", /* cause= */ null); enterErrorState(exception); } diff --git a/mobile/library/java/org/chromium/net/impl/UrlRequestBase.java b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequestBase.java similarity index 93% rename from mobile/library/java/org/chromium/net/impl/UrlRequestBase.java rename to mobile/library/java/org/chromium/net/impl/CronvoyUrlRequestBase.java index e1bd3379d3e0..2af3d957cfec 100644 --- a/mobile/library/java/org/chromium/net/impl/UrlRequestBase.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequestBase.java @@ -10,9 +10,9 @@ /** * Base class for classes that implement {@link UrlRequest} including experimental - * features. {@link CronetUrlRequest} and {@link JavaUrlRequest} extends this class. + * features. {@link CronvoyUrlRequest} and {@link JavaUrlRequest} extends this class. */ -abstract class UrlRequestBase extends ExperimentalUrlRequest { +abstract class CronvoyUrlRequestBase extends ExperimentalUrlRequest { /** * Sets the HTTP method verb to use for this request. Must be done before * request has started. diff --git a/mobile/library/java/org/chromium/net/impl/UrlRequestBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequestBuilderImpl.java similarity index 78% rename from mobile/library/java/org/chromium/net/impl/UrlRequestBuilderImpl.java rename to mobile/library/java/org/chromium/net/impl/CronvoyUrlRequestBuilderImpl.java index 6ec92900212b..883d17f6d6f2 100644 --- a/mobile/library/java/org/chromium/net/impl/UrlRequestBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequestBuilderImpl.java @@ -14,15 +14,15 @@ /** * Implements {@link org.chromium.net.ExperimentalUrlRequest.Builder}. */ -public class UrlRequestBuilderImpl extends ExperimentalUrlRequest.Builder { +public class CronvoyUrlRequestBuilderImpl extends ExperimentalUrlRequest.Builder { private static final String ACCEPT_ENCODING = "Accept-Encoding"; - private static final String TAG = UrlRequestBuilderImpl.class.getSimpleName(); + private static final String TAG = CronvoyUrlRequestBuilderImpl.class.getSimpleName(); // All fields are temporary storage of ExperimentalUrlRequest configuration to be // copied to built ExperimentalUrlRequest. // CronetEngineBase to execute request. - private final CronetEngineBase mCronetEngine; + private final CronvoyEngineBase mCronetEngine; // URL to request. private final String mUrl; // Callback to receive progress callbacks. @@ -39,7 +39,7 @@ public class UrlRequestBuilderImpl extends ExperimentalUrlRequest.Builder { // Disable connection migration for just this request. private boolean mDisableConnectionMigration; // Priority of request. Default is medium. - @CronetEngineBase.RequestPriority private int mPriority = REQUEST_PRIORITY_MEDIUM; + @CronvoyEngineBase.RequestPriority private int mPriority = REQUEST_PRIORITY_MEDIUM; // Request reporting annotations. Avoid extra object creation if no annotations added. private Collection mRequestAnnotations; // If request is an upload, this provides the request body data. @@ -53,7 +53,7 @@ public class UrlRequestBuilderImpl extends ExperimentalUrlRequest.Builder { private int mTrafficStatsUid; private RequestFinishedInfo.Listener mRequestFinishedListener; // Idempotency of the request. - @CronetEngineBase.Idempotency private int mIdempotency = DEFAULT_IDEMPOTENCY; + @CronvoyEngineBase.Idempotency private int mIdempotency = DEFAULT_IDEMPOTENCY; /** * Creates a builder for {@link UrlRequest} objects. All callbacks for @@ -67,8 +67,8 @@ public class UrlRequestBuilderImpl extends ExperimentalUrlRequest.Builder { * @param executor {@link Executor} on which all callbacks will be invoked. * @param cronetEngine {@link CronetEngine} used to execute this request. */ - UrlRequestBuilderImpl(String url, UrlRequest.Callback callback, Executor executor, - CronetEngineBase cronetEngine) { + CronvoyUrlRequestBuilderImpl(String url, UrlRequest.Callback callback, Executor executor, + CronvoyEngineBase cronetEngine) { super(); if (url == null) { throw new NullPointerException("URL is required."); @@ -98,7 +98,7 @@ public ExperimentalUrlRequest.Builder setHttpMethod(String method) { } @Override - public UrlRequestBuilderImpl addHeader(String header, String value) { + public CronvoyUrlRequestBuilderImpl addHeader(String header, String value) { if (header == null) { throw new NullPointerException("Invalid header name."); } @@ -118,32 +118,33 @@ public UrlRequestBuilderImpl addHeader(String header, String value) { } @Override - public UrlRequestBuilderImpl disableCache() { + public CronvoyUrlRequestBuilderImpl disableCache() { mDisableCache = true; return this; } @Override - public UrlRequestBuilderImpl disableConnectionMigration() { + public CronvoyUrlRequestBuilderImpl disableConnectionMigration() { mDisableConnectionMigration = true; return this; } @Override - public UrlRequestBuilderImpl setPriority(@CronetEngineBase.RequestPriority int priority) { + public CronvoyUrlRequestBuilderImpl setPriority(@CronvoyEngineBase.RequestPriority int priority) { mPriority = priority; return this; } @Override - public UrlRequestBuilderImpl setIdempotency(@CronetEngineBase.Idempotency int idempotency) { + public CronvoyUrlRequestBuilderImpl + setIdempotency(@CronvoyEngineBase.Idempotency int idempotency) { mIdempotency = idempotency; return this; } @Override - public UrlRequestBuilderImpl setUploadDataProvider(UploadDataProvider uploadDataProvider, - Executor executor) { + public CronvoyUrlRequestBuilderImpl setUploadDataProvider(UploadDataProvider uploadDataProvider, + Executor executor) { if (uploadDataProvider == null) { throw new NullPointerException("Invalid UploadDataProvider."); } @@ -159,13 +160,13 @@ public UrlRequestBuilderImpl setUploadDataProvider(UploadDataProvider uploadData } @Override - public UrlRequestBuilderImpl allowDirectExecutor() { + public CronvoyUrlRequestBuilderImpl allowDirectExecutor() { mAllowDirectExecutor = true; return this; } @Override - public UrlRequestBuilderImpl addRequestAnnotation(Object annotation) { + public CronvoyUrlRequestBuilderImpl addRequestAnnotation(Object annotation) { if (annotation == null) { throw new NullPointerException("Invalid metrics annotation."); } @@ -177,28 +178,29 @@ public UrlRequestBuilderImpl addRequestAnnotation(Object annotation) { } @Override - public UrlRequestBuilderImpl setTrafficStatsTag(int tag) { + public CronvoyUrlRequestBuilderImpl setTrafficStatsTag(int tag) { mTrafficStatsTagSet = true; mTrafficStatsTag = tag; return this; } @Override - public UrlRequestBuilderImpl setTrafficStatsUid(int uid) { + public CronvoyUrlRequestBuilderImpl setTrafficStatsUid(int uid) { mTrafficStatsUidSet = true; mTrafficStatsUid = uid; return this; } @Override - public UrlRequestBuilderImpl setRequestFinishedListener(RequestFinishedInfo.Listener listener) { + public CronvoyUrlRequestBuilderImpl + setRequestFinishedListener(RequestFinishedInfo.Listener listener) { mRequestFinishedListener = listener; return this; } @Override - public UrlRequestBase build() { - final UrlRequestBase request = mCronetEngine.createRequest( + public CronvoyUrlRequestBase build() { + final CronvoyUrlRequestBase request = mCronetEngine.createRequest( mUrl, mCallback, mExecutor, mPriority, mRequestAnnotations, mDisableCache, mDisableConnectionMigration, mAllowDirectExecutor, mTrafficStatsTagSet, mTrafficStatsTag, mTrafficStatsUidSet, mTrafficStatsUid, mRequestFinishedListener, mIdempotency); diff --git a/mobile/library/java/org/chromium/net/impl/CronetUrlRequestContext.java b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequestContext.java similarity index 86% rename from mobile/library/java/org/chromium/net/impl/CronetUrlRequestContext.java rename to mobile/library/java/org/chromium/net/impl/CronvoyUrlRequestContext.java index 279042dd0799..3abb0927dfad 100644 --- a/mobile/library/java/org/chromium/net/impl/CronetUrlRequestContext.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequestContext.java @@ -27,18 +27,18 @@ import org.chromium.net.NetworkQualityThroughputListener; import org.chromium.net.RequestFinishedInfo; import org.chromium.net.UrlRequest; -import org.chromium.net.impl.VersionSafeCallbacks.RequestFinishedInfoListener; -import org.chromium.net.urlconnection.CronetHttpURLConnection; -import org.chromium.net.urlconnection.CronetURLStreamHandlerFactory; +import org.chromium.net.impl.CronvoyVersionSafeCallbacks.RequestFinishedInfoListener; +import org.chromium.net.urlconnection.CronvoyHttpURLConnection; +import org.chromium.net.urlconnection.CronvoyURLStreamHandlerFactory; /** * Cronvoy engine shim. * *

Does not support yet netlogs, transferred data measurement, bidistream, cache, or priority. */ -public final class CronetUrlRequestContext extends CronetEngineBase { +public final class CronvoyUrlRequestContext extends CronvoyEngineBase { - static final String LOG_TAG = CronetUrlRequestContext.class.getSimpleName(); + static final String LOG_TAG = CronvoyUrlRequestContext.class.getSimpleName(); /** * Synchronize access to mUrlRequestContextAdapter and shutdown routine. @@ -58,7 +58,7 @@ public final class CronetUrlRequestContext extends CronetEngineBase { private Thread mNetworkThread; private final String mUserAgent; - private final CronetEngineBuilderImpl mBuilder; + private final CronvoyEngineBuilderImpl mBuilder; private final AtomicReference mInitializationCompleter = new AtomicReference<>(); /** @@ -68,10 +68,11 @@ public final class CronetUrlRequestContext extends CronetEngineBase { */ private final Object mFinishedListenerLock = new Object(); @GuardedBy("mFinishedListenerLock") - private final Map - mFinishedListenerMap = new HashMap<>(); + private final Map mFinishedListenerMap = + new HashMap<>(); - public CronetUrlRequestContext(NativeCronetEngineBuilderImpl builder) { + public CronvoyUrlRequestContext(NativeCronvoyEngineBuilderImpl builder) { mBuilder = builder; // On android, all background threads (and all threads that are part // of background processes) are put in a cgroup that is allowed to @@ -106,7 +107,7 @@ public EnvoyEngine getEnvoyEngine() { } } - CronetEngineBuilderImpl getBuilder() { return mBuilder; } + CronvoyEngineBuilderImpl getBuilder() { return mBuilder; } void setTaskToExecuteWhenInitializationIsCompleted(Runnable runnable) { if (!mInitializationCompleter.compareAndSet(null, runnable)) { @@ -118,7 +119,7 @@ void setTaskToExecuteWhenInitializationIsCompleted(Runnable runnable) { } @Override - public UrlRequestBase + public CronvoyUrlRequestBase createRequest(String url, UrlRequest.Callback callback, Executor executor, int priority, Collection requestAnnotations, boolean disableCache, boolean disableConnectionMigration, boolean allowDirectExecutor, @@ -127,9 +128,9 @@ void setTaskToExecuteWhenInitializationIsCompleted(Runnable runnable) { int idempotency) { synchronized (mLock) { checkHaveAdapter(); - return new CronetUrlRequest(this, url, callback, executor, mUserAgent, allowDirectExecutor, - requestAnnotations, trafficStatsTagSet, trafficStatsTag, - trafficStatsUidSet, trafficStatsUid, requestFinishedListener); + return new CronvoyUrlRequest(this, url, callback, executor, mUserAgent, allowDirectExecutor, + requestAnnotations, trafficStatsTagSet, trafficStatsTag, + trafficStatsUidSet, trafficStatsUid, requestFinishedListener); } } @@ -143,7 +144,7 @@ void setTaskToExecuteWhenInitializationIsCompleted(Runnable runnable) { int trafficStatsTag, boolean trafficStatsUidSet, int trafficStatsUid) { synchronized (mLock) { checkHaveAdapter(); - return new CronetBidirectionalStream( + return new CronvoyBidirectionalStream( this, url, priority, callback, executor, mUserAgent, httpMethod, requestHeaders, delayRequestHeadersUntilFirstFlush, requestAnnotations, trafficStatsTagSet, trafficStatsTag, trafficStatsUidSet, trafficStatsUid); @@ -154,12 +155,12 @@ void setTaskToExecuteWhenInitializationIsCompleted(Runnable runnable) { public ExperimentalBidirectionalStream.Builder newBidirectionalStreamBuilder(String url, BidirectionalStream.Callback callback, Executor executor) { - return new BidirectionalStreamBuilderImpl(url, callback, executor, this); + return new CronvoyBidirectionalStreamBuilderImpl(url, callback, executor, this); } @Override public String getVersionString() { - return "Cronet/" + ImplVersion.getCronetVersionWithLastChange(); + return "Cronet/" + CronvoyImplVersion.getCronetVersionWithLastChange(); } @Override @@ -246,8 +247,8 @@ public void removeThroughputListener(NetworkQualityThroughputListener listener) @Override public void addRequestFinishedListener(RequestFinishedInfo.Listener listener) { synchronized (mFinishedListenerLock) { - mFinishedListenerMap.put(listener, - new VersionSafeCallbacks.RequestFinishedInfoListener(listener)); + mFinishedListenerMap.put( + listener, new CronvoyVersionSafeCallbacks.RequestFinishedInfoListener(listener)); } } @@ -272,14 +273,14 @@ public URLConnection openConnection(URL url, Proxy proxy) { } String protocol = url.getProtocol(); if ("http".equals(protocol) || "https".equals(protocol)) { - return new CronetHttpURLConnection(url, this); + return new CronvoyHttpURLConnection(url, this); } throw new UnsupportedOperationException("Unexpected protocol:" + protocol); } @Override public URLStreamHandlerFactory createURLStreamHandlerFactory() { - return new CronetURLStreamHandlerFactory(this); + return new CronvoyURLStreamHandlerFactory(this); } /** @@ -321,7 +322,8 @@ void reportRequestFinished(final RequestFinishedInfo requestInfo) { } currentListeners = new ArrayList<>(mFinishedListenerMap.values()); } - for (final VersionSafeCallbacks.RequestFinishedInfoListener listener : currentListeners) { + for (final CronvoyVersionSafeCallbacks.RequestFinishedInfoListener listener : + currentListeners) { Runnable task = new Runnable() { @Override public void run() { @@ -337,7 +339,7 @@ private static void postObservationTaskToExecutor(Executor executor, Runnable ta executor.execute(task); } catch (RejectedExecutionException failException) { // TODO(https://github.com/envoyproxy/envoy-mobile/issues/2262): go with Cronet ways for logs. - android.util.Log.e(CronetUrlRequestContext.LOG_TAG, "Exception posting task to executor", + android.util.Log.e(CronvoyUrlRequestContext.LOG_TAG, "Exception posting task to executor", failException); } } diff --git a/mobile/library/java/org/chromium/net/impl/UrlResponseInfoImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyUrlResponseInfoImpl.java similarity index 92% rename from mobile/library/java/org/chromium/net/impl/UrlResponseInfoImpl.java rename to mobile/library/java/org/chromium/net/impl/CronvoyUrlResponseInfoImpl.java index 2c1df9e31bba..1f85c47fc00b 100644 --- a/mobile/library/java/org/chromium/net/impl/UrlResponseInfoImpl.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyUrlResponseInfoImpl.java @@ -16,7 +16,7 @@ * callback gets a different copy of {@code UrlResponseInfo} describing a particular * redirect response. */ -public final class UrlResponseInfoImpl extends UrlResponseInfo { +public final class CronvoyUrlResponseInfoImpl extends UrlResponseInfo { private List mResponseInfoUrlChain; private int mHttpStatusCode; private String mHttpStatusText; @@ -77,10 +77,11 @@ public Map> getAsMap() { * @param receivedByteCount minimum count of bytes received from the network to process this * request. */ - public UrlResponseInfoImpl(List urlChain, int httpStatusCode, String httpStatusText, - List> allHeadersList, boolean wasCached, - String negotiatedProtocol, String proxyServer, - long receivedByteCount) { + public CronvoyUrlResponseInfoImpl(List urlChain, int httpStatusCode, + String httpStatusText, + List> allHeadersList, + boolean wasCached, String negotiatedProtocol, + String proxyServer, long receivedByteCount) { mResponseInfoUrlChain = Collections.unmodifiableList(urlChain); mHttpStatusCode = httpStatusCode; mHttpStatusText = httpStatusText; @@ -94,7 +95,7 @@ public UrlResponseInfoImpl(List urlChain, int httpStatusCode, String htt /** * Creates an empty implementation of {@link UrlResponseInfo}. */ - public UrlResponseInfoImpl() {} + public CronvoyUrlResponseInfoImpl() {} /** * Sets response values. diff --git a/mobile/library/java/org/chromium/net/impl/UserAgent.java b/mobile/library/java/org/chromium/net/impl/CronvoyUserAgent.java similarity index 95% rename from mobile/library/java/org/chromium/net/impl/UserAgent.java rename to mobile/library/java/org/chromium/net/impl/CronvoyUserAgent.java index a714f5d3a9d3..83f0ec1b34c4 100644 --- a/mobile/library/java/org/chromium/net/impl/UserAgent.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyUserAgent.java @@ -10,7 +10,7 @@ /** * Constructs a User-Agent string. */ -public final class UserAgent { +public final class CronvoyUserAgent { private static final Object sLock = new Object(); private static final int VERSION_CODE_UNINITIALIZED = 0; @@ -91,8 +91,8 @@ private static int versionFromContext(Context context) { private static void appendCronetVersion(StringBuilder builder) { builder.append(" Cronet/"); - builder.append(ImplVersion.getCronetVersion()); + builder.append(CronvoyImplVersion.getCronetVersion()); } - private UserAgent() {} + private CronvoyUserAgent() {} } diff --git a/mobile/library/java/org/chromium/net/impl/VersionSafeCallbacks.java b/mobile/library/java/org/chromium/net/impl/CronvoyVersionSafeCallbacks.java similarity index 99% rename from mobile/library/java/org/chromium/net/impl/VersionSafeCallbacks.java rename to mobile/library/java/org/chromium/net/impl/CronvoyVersionSafeCallbacks.java index e073cbb3bf24..dd845b7679fa 100644 --- a/mobile/library/java/org/chromium/net/impl/VersionSafeCallbacks.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyVersionSafeCallbacks.java @@ -24,7 +24,7 @@ * When adding any callback wrapping here, be sure you add the proper version check. Only callbacks * supported in all versions of the API should forgo a version check. */ -public class VersionSafeCallbacks { +public class CronvoyVersionSafeCallbacks { /** * Wrap a {@link UrlRequest.Callback} in a version safe manner. */ diff --git a/mobile/library/java/org/chromium/net/impl/Errors.java b/mobile/library/java/org/chromium/net/impl/Errors.java index f820635253e5..fde22f1e9492 100644 --- a/mobile/library/java/org/chromium/net/impl/Errors.java +++ b/mobile/library/java/org/chromium/net/impl/Errors.java @@ -114,7 +114,7 @@ public static int mapNetErrorToCronetApiErrorCode(NetError netError) { case ERR_QUIC_PROTOCOL_ERROR: return NetworkException.ERROR_QUIC_PROTOCOL_FAILED; } - Log.e(CronetUrlRequestContext.LOG_TAG, "Unknown error code: " + netError); + Log.e(CronvoyUrlRequestContext.LOG_TAG, "Unknown error code: " + netError); return NetworkException.ERROR_OTHER; } diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderWithLibraryLoaderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderWithLibraryLoaderImpl.java deleted file mode 100644 index 7b64822559fd..000000000000 --- a/mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderWithLibraryLoaderImpl.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.chromium.net.impl; - -import android.content.Context; -import org.chromium.net.CronetEngine.Builder.LibraryLoader; -import org.chromium.net.ICronetEngineBuilder; - -/** - * An extension of {@link NativeCronetEngineBuilderImpl} that implements - * {@link ICronetEngineBuilder#setLibraryLoader}. - */ -public class NativeCronetEngineBuilderWithLibraryLoaderImpl extends NativeCronetEngineBuilderImpl { - private VersionSafeCallbacks.LibraryLoader mLibraryLoader; - - /** - * Constructs a builder for Native Cronet Engine. - * Default config enables SPDY, disables QUIC and HTTP cache. - * - * @param context Android {@link Context} for engine to use. - */ - public NativeCronetEngineBuilderWithLibraryLoaderImpl(Context context) { super(context); } - - @Override - public CronetEngineBuilderImpl setLibraryLoader(LibraryLoader loader) { - mLibraryLoader = new VersionSafeCallbacks.LibraryLoader(loader); - return this; - } - - @Override - VersionSafeCallbacks.LibraryLoader libraryLoader() { - return mLibraryLoader; - } -} diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java similarity index 94% rename from mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderImpl.java rename to mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java index 32dfba0c769d..404e6addbb58 100644 --- a/mobile/library/java/org/chromium/net/impl/NativeCronetEngineBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java @@ -25,9 +25,9 @@ import org.chromium.net.ICronetEngineBuilder; /** - * Implementation of {@link ICronetEngineBuilder} that builds native Cronet engine. + * Implementation of {@link ICronetEngineBuilder} that builds native Cronvoy engine. */ -public class NativeCronetEngineBuilderImpl extends CronetEngineBuilderImpl { +public class NativeCronvoyEngineBuilderImpl extends CronvoyEngineBuilderImpl { // TODO(refactor) move unshared variables into their specific methods. private final List nativeFilterChain = new ArrayList<>(); @@ -84,7 +84,7 @@ public class NativeCronetEngineBuilderImpl extends CronetEngineBuilderImpl { * * @param context Android {@link Context} for engine to use. */ - public NativeCronetEngineBuilderImpl(Context context) { super(context); } + public NativeCronvoyEngineBuilderImpl(Context context) { super(context); } /** * Indicates to skip the TLS certificate verification. @@ -92,7 +92,7 @@ public class NativeCronetEngineBuilderImpl extends CronetEngineBuilderImpl { * @return the builder to facilitate chaining. */ @VisibleForTesting - public CronetEngineBuilderImpl setMockCertVerifierForTesting() { + public CronvoyEngineBuilderImpl setMockCertVerifierForTesting() { mTrustChainVerification = TrustChainVerification.ACCEPT_UNTRUSTED; return this; } @@ -103,7 +103,7 @@ public CronetEngineBuilderImpl setMockCertVerifierForTesting() { * @return the builder to facilitate chaining. */ @VisibleForTesting - public CronetEngineBuilderImpl addUrlInterceptorsForTesting() { + public CronvoyEngineBuilderImpl addUrlInterceptorsForTesting() { nativeFilterChain.add(new EnvoyNativeFilterConfig( "envoy.filters.http.test_read", "{\"@type\": type.googleapis.com/envoymobile.test.integration.filters.http.test_read.TestRead}")); @@ -115,7 +115,7 @@ public ExperimentalCronetEngine build() { if (getUserAgent() == null) { setUserAgent(getDefaultUserAgent()); } - return new CronetUrlRequestContext(this); + return new CronvoyUrlRequestContext(this); } EnvoyEngine createEngine(EnvoyOnEngineRunning onEngineRunning) { diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderWithLibraryLoaderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderWithLibraryLoaderImpl.java new file mode 100644 index 000000000000..ae8c3d42f22b --- /dev/null +++ b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderWithLibraryLoaderImpl.java @@ -0,0 +1,33 @@ +package org.chromium.net.impl; + +import android.content.Context; +import org.chromium.net.CronetEngine.Builder.LibraryLoader; +import org.chromium.net.ICronetEngineBuilder; + +/** + * An extension of {@link NativeCronvoyEngineBuilderImpl} that implements + * {@link ICronetEngineBuilder#setLibraryLoader}. + */ +public class NativeCronvoyEngineBuilderWithLibraryLoaderImpl + extends NativeCronvoyEngineBuilderImpl { + private CronvoyVersionSafeCallbacks.LibraryLoader mLibraryLoader; + + /** + * Constructs a builder for Native Cronet Engine. + * Default config enables SPDY, disables QUIC and HTTP cache. + * + * @param context Android {@link Context} for engine to use. + */ + public NativeCronvoyEngineBuilderWithLibraryLoaderImpl(Context context) { super(context); } + + @Override + public CronvoyEngineBuilderImpl setLibraryLoader(LibraryLoader loader) { + mLibraryLoader = new CronvoyVersionSafeCallbacks.LibraryLoader(loader); + return this; + } + + @Override + CronvoyVersionSafeCallbacks.LibraryLoader libraryLoader() { + return mLibraryLoader; + } +} diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronetProvider.java b/mobile/library/java/org/chromium/net/impl/NativeCronvoyProvider.java similarity index 54% rename from mobile/library/java/org/chromium/net/impl/NativeCronetProvider.java rename to mobile/library/java/org/chromium/net/impl/NativeCronvoyProvider.java index 33f1adf8899f..42285e15c5ff 100644 --- a/mobile/library/java/org/chromium/net/impl/NativeCronetProvider.java +++ b/mobile/library/java/org/chromium/net/impl/NativeCronvoyProvider.java @@ -3,37 +3,37 @@ import android.content.Context; import java.util.Arrays; import org.chromium.net.CronetEngine; -import org.chromium.net.CronetProvider; +import org.chromium.net.CronvoyProvider; import org.chromium.net.ExperimentalCronetEngine; import org.chromium.net.ICronetEngineBuilder; /** - * Implementation of {@link CronetProvider} that creates {@link CronetEngine.Builder} + * Implementation of {@link CronvoyProvider} that creates {@link CronetEngine.Builder} * for building the native implementation of {@link CronetEngine}. */ -public class NativeCronetProvider extends CronetProvider { +public class NativeCronvoyProvider extends CronvoyProvider { /** * Constructor. * * @param context Android context to use. */ // TODO(carloseltuerto) find something similar to @UsedByReflection("CronetProvider.java") - public NativeCronetProvider(Context context) { super(context); } + public NativeCronvoyProvider(Context context) { super(context); } @Override public CronetEngine.Builder createBuilder() { - ICronetEngineBuilder impl = new NativeCronetEngineBuilderWithLibraryLoaderImpl(mContext); + ICronetEngineBuilder impl = new NativeCronvoyEngineBuilderWithLibraryLoaderImpl(mContext); return new ExperimentalCronetEngine.Builder(impl); } @Override public String getName() { - return CronetProvider.PROVIDER_NAME_APP_PACKAGED; + return CronvoyProvider.PROVIDER_NAME_APP_PACKAGED; } @Override public String getVersion() { - return ImplVersion.getCronetVersion(); + return CronvoyImplVersion.getCronetVersion(); } @Override @@ -43,12 +43,12 @@ public boolean isEnabled() { @Override public int hashCode() { - return Arrays.hashCode(new Object[] {NativeCronetProvider.class, mContext}); + return Arrays.hashCode(new Object[] {NativeCronvoyProvider.class, mContext}); } @Override public boolean equals(Object other) { - return other == this || (other instanceof NativeCronetProvider && - this.mContext.equals(((NativeCronetProvider)other).mContext)); + return other == this || (other instanceof NativeCronvoyProvider && + this.mContext.equals(((NativeCronvoyProvider)other).mContext)); } } diff --git a/mobile/library/java/org/chromium/net/urlconnection/BUILD b/mobile/library/java/org/chromium/net/urlconnection/BUILD index 78565d33bf8a..2340d91645a7 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/BUILD +++ b/mobile/library/java/org/chromium/net/urlconnection/BUILD @@ -10,15 +10,15 @@ licenses(["notice"]) # Apache 2 android_library( name = "urlconnection", srcs = [ - "CronetBufferedOutputStream.java", - "CronetChunkedOutputStream.java", - "CronetFixedModeOutputStream.java", - "CronetHttpURLConnection.java", - "CronetHttpURLStreamHandler.java", - "CronetInputStream.java", - "CronetOutputStream.java", - "CronetURLStreamHandlerFactory.java", - "MessageLoop.java", + "CronvoyBufferedOutputStream.java", + "CronvoyChunkedOutputStream.java", + "CronvoyFixedModeOutputStream.java", + "CronvoyHttpURLConnection.java", + "CronvoyHttpURLStreamHandler.java", + "CronvoyInputStream.java", + "CronvoyMessageLoop.java", + "CronvoyOutputStream.java", + "CronvoyURLStreamHandlerFactory.java", ], manifest = "URLConnectionManifest.xml", visibility = ["//visibility:public"], diff --git a/mobile/library/java/org/chromium/net/urlconnection/CronetBufferedOutputStream.java b/mobile/library/java/org/chromium/net/urlconnection/CronvoyBufferedOutputStream.java similarity index 89% rename from mobile/library/java/org/chromium/net/urlconnection/CronetBufferedOutputStream.java rename to mobile/library/java/org/chromium/net/urlconnection/CronvoyBufferedOutputStream.java index 92b6ffacc502..c6203baa7fa7 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/CronetBufferedOutputStream.java +++ b/mobile/library/java/org/chromium/net/urlconnection/CronvoyBufferedOutputStream.java @@ -9,17 +9,17 @@ /** * An implementation of {@link java.io.OutputStream} that buffers entire request * body in memory. This is used when neither - * {@link CronetHttpURLConnection#setFixedLengthStreamingMode} - * nor {@link CronetHttpURLConnection#setChunkedStreamingMode} is set. + * {@link CronvoyHttpURLConnection#setFixedLengthStreamingMode} + * nor {@link CronvoyHttpURLConnection#setChunkedStreamingMode} is set. */ -final class CronetBufferedOutputStream extends CronetOutputStream { +final class CronvoyBufferedOutputStream extends CronvoyOutputStream { // QUIC uses a read buffer of 14520 bytes, SPDY uses 2852 bytes, and normal // stream uses 16384 bytes. Therefore, use 16384 for now to avoid growing // the buffer too many times. private static final int INITIAL_BUFFER_SIZE = 16384; // If content length is not passed in the constructor, this is -1. private final int mInitialContentLength; - private final CronetHttpURLConnection mConnection; + private final CronvoyHttpURLConnection mConnection; private final UploadDataProvider mUploadDataProvider = new UploadDataProviderImpl(); // Internal buffer that is used to buffer the request body. private ByteBuffer mBuffer; @@ -27,11 +27,11 @@ final class CronetBufferedOutputStream extends CronetOutputStream { /** * Package protected constructor. - * @param connection The CronetHttpURLConnection object. + * @param connection The CronvoyHttpURLConnection object. * @param contentLength The content length of the request body. It must not * be smaller than 0 or bigger than {@link Integer.MAX_VALUE}. */ - CronetBufferedOutputStream(final CronetHttpURLConnection connection, final long contentLength) { + CronvoyBufferedOutputStream(final CronvoyHttpURLConnection connection, final long contentLength) { if (connection == null) { throw new NullPointerException("Argument connection cannot be null."); } @@ -51,9 +51,9 @@ final class CronetBufferedOutputStream extends CronetOutputStream { /** * Package protected constructor used when content length is not known. - * @param connection The CronetHttpURLConnection object. + * @param connection The CronvoyHttpURLConnection object. */ - CronetBufferedOutputStream(final CronetHttpURLConnection connection) { + CronvoyBufferedOutputStream(final CronvoyHttpURLConnection connection) { if (connection == null) { throw new NullPointerException(); } @@ -106,7 +106,7 @@ private void ensureCanWrite(int count) throws IOException { mBuffer = newByteBuffer; } - // Below are CronetOutputStream implementations: + // Below are CronvoyOutputStream implementations: /** * Sets {@link #mConnected} to {@code true}. diff --git a/mobile/library/java/org/chromium/net/urlconnection/CronetChunkedOutputStream.java b/mobile/library/java/org/chromium/net/urlconnection/CronvoyChunkedOutputStream.java similarity index 88% rename from mobile/library/java/org/chromium/net/urlconnection/CronetChunkedOutputStream.java rename to mobile/library/java/org/chromium/net/urlconnection/CronvoyChunkedOutputStream.java index 470f02acf4a5..4b68d2986186 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/CronetChunkedOutputStream.java +++ b/mobile/library/java/org/chromium/net/urlconnection/CronvoyChunkedOutputStream.java @@ -8,26 +8,26 @@ /** * An implementation of {@link java.io.OutputStream} to send data to a server, - * when {@link CronetHttpURLConnection#setChunkedStreamingMode} is used. + * when {@link CronvoyHttpURLConnection#setChunkedStreamingMode} is used. * This implementation does not buffer the entire request body in memory. * It does not support rewind. Note that {@link #write} should only be called * from the thread on which the {@link #mConnection} is created. */ -final class CronetChunkedOutputStream extends CronetOutputStream { - private final CronetHttpURLConnection mConnection; - private final MessageLoop mMessageLoop; +final class CronvoyChunkedOutputStream extends CronvoyOutputStream { + private final CronvoyHttpURLConnection mConnection; + private final CronvoyMessageLoop mMessageLoop; private final ByteBuffer mBuffer; private final UploadDataProvider mUploadDataProvider = new UploadDataProviderImpl(); private boolean mLastChunk; /** * Package protected constructor. - * @param connection The CronetHttpURLConnection object. + * @param connection The CronvoyHttpURLConnection object. * @param chunkLength The chunk length of the request body in bytes. It must * be a positive number. */ - CronetChunkedOutputStream(CronetHttpURLConnection connection, int chunkLength, - MessageLoop messageLoop) { + CronvoyChunkedOutputStream(CronvoyHttpURLConnection connection, int chunkLength, + CronvoyMessageLoop messageLoop) { if (connection == null) { throw new NullPointerException(); } @@ -72,7 +72,7 @@ public void close() throws IOException { } } - // Below are CronetOutputStream implementations: + // Below are CronvoyOutputStream implementations: @Override void setConnected() throws IOException { diff --git a/mobile/library/java/org/chromium/net/urlconnection/CronetFixedModeOutputStream.java b/mobile/library/java/org/chromium/net/urlconnection/CronvoyFixedModeOutputStream.java similarity index 90% rename from mobile/library/java/org/chromium/net/urlconnection/CronetFixedModeOutputStream.java rename to mobile/library/java/org/chromium/net/urlconnection/CronvoyFixedModeOutputStream.java index 316ac5b7d6ea..6b8fdf4e484f 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/CronetFixedModeOutputStream.java +++ b/mobile/library/java/org/chromium/net/urlconnection/CronvoyFixedModeOutputStream.java @@ -10,23 +10,23 @@ /** * An implementation of {@link java.io.OutputStream} to send data to a server, - * when {@link CronetHttpURLConnection#setFixedLengthStreamingMode} is used. + * when {@link CronvoyHttpURLConnection#setFixedLengthStreamingMode} is used. * This implementation does not buffer the entire request body in memory. * It does not support rewind. Note that {@link #write} should only be called * from the thread on which the {@link #mConnection} is created. */ -final class CronetFixedModeOutputStream extends CronetOutputStream { - // CronetFixedModeOutputStream buffers up to this value and wait for UploadDataStream +final class CronvoyFixedModeOutputStream extends CronvoyOutputStream { + // CronvoyFixedModeOutputStream buffers up to this value and wait for UploadDataStream // to consume the data. This field is non-final, so it can be changed for tests. // Using 16384 bytes is because the internal read buffer is 14520 for QUIC, // 16384 for SPDY, and 16384 for normal HTTP/1.1 stream. @VisibleForTesting private static int sDefaultBufferLength = 16384; - private final CronetHttpURLConnection mConnection; - private final MessageLoop mMessageLoop; + private final CronvoyHttpURLConnection mConnection; + private final CronvoyMessageLoop mMessageLoop; private final long mContentLength; // Internal buffer for holding bytes from the client until the bytes are // copied to the UploadDataSink in UploadDataProvider.read(). - // CronetFixedModeOutputStream allows client to provide up to + // CronvoyFixedModeOutputStream allows client to provide up to // sDefaultBufferLength bytes, and wait for UploadDataProvider.read() to be // called after which point mBuffer is cleared so client can fill in again. // While the client is filling the buffer (via {@code write()}), the buffer's @@ -44,12 +44,12 @@ final class CronetFixedModeOutputStream extends CronetOutputStream { /** * Package protected constructor. - * @param connection The CronetHttpURLConnection object. + * @param connection The CronvoyHttpURLConnection object. * @param contentLength The content length of the request body. Non-zero for * non-chunked upload. */ - CronetFixedModeOutputStream(CronetHttpURLConnection connection, long contentLength, - MessageLoop messageLoop) { + CronvoyFixedModeOutputStream(CronvoyHttpURLConnection connection, long contentLength, + CronvoyMessageLoop messageLoop) { if (connection == null) { throw new NullPointerException(); } @@ -139,7 +139,7 @@ private void checkNotExceedContentLength(int numBytes) throws ProtocolException } } - // Below are CronetOutputStream implementations: + // Below are CronvoyOutputStream implementations: @Override void setConnected() throws IOException { diff --git a/mobile/library/java/org/chromium/net/urlconnection/CronetHttpURLConnection.java b/mobile/library/java/org/chromium/net/urlconnection/CronvoyHttpURLConnection.java similarity index 96% rename from mobile/library/java/org/chromium/net/urlconnection/CronetHttpURLConnection.java rename to mobile/library/java/org/chromium/net/urlconnection/CronvoyHttpURLConnection.java index 20b1109e7d4f..381a8ece6bc1 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/CronetHttpURLConnection.java +++ b/mobile/library/java/org/chromium/net/urlconnection/CronvoyHttpURLConnection.java @@ -32,11 +32,11 @@ * requests and receive responses. * {@hide} */ -public final class CronetHttpURLConnection extends HttpURLConnection { - private static final String TAG = CronetHttpURLConnection.class.getSimpleName(); +public final class CronvoyHttpURLConnection extends HttpURLConnection { + private static final String TAG = CronvoyHttpURLConnection.class.getSimpleName(); private static final String CONTENT_LENGTH = "Content-Length"; private final CronetEngine mCronetEngine; - private final MessageLoop mMessageLoop; + private final CronvoyMessageLoop mMessageLoop; private UrlRequest mRequest; private final List> mRequestHeaders; private boolean mTrafficStatsTagSet; @@ -44,8 +44,8 @@ public final class CronetHttpURLConnection extends HttpURLConnection { private boolean mTrafficStatsUidSet; private int mTrafficStatsUid; - private CronetInputStream mInputStream; - private CronetOutputStream mOutputStream; + private CronvoyInputStream mInputStream; + private CronvoyOutputStream mOutputStream; private UrlResponseInfo mResponseInfo; private IOException mException; private boolean mOnRedirectCalled; @@ -54,11 +54,11 @@ public final class CronetHttpURLConnection extends HttpURLConnection { private List> mResponseHeadersList; private Map> mResponseHeadersMap; - public CronetHttpURLConnection(URL url, CronetEngine cronetEngine) { + public CronvoyHttpURLConnection(URL url, CronetEngine cronetEngine) { super(url); mCronetEngine = cronetEngine; - mMessageLoop = new MessageLoop(); - mInputStream = new CronetInputStream(this); + mMessageLoop = new CronvoyMessageLoop(); + mInputStream = new CronvoyInputStream(this); mRequestHeaders = new ArrayList>(); } @@ -198,14 +198,14 @@ public OutputStream getOutputStream() throws IOException { throw new ProtocolException("Cannot write to OutputStream after receiving response."); } if (isChunkedUpload()) { - mOutputStream = new CronetChunkedOutputStream(this, chunkLength, mMessageLoop); + mOutputStream = new CronvoyChunkedOutputStream(this, chunkLength, mMessageLoop); // Start the request now since all headers can be sent. startRequest(); } else { long fixedStreamingModeContentLength = getStreamingModeContentLength(); if (fixedStreamingModeContentLength != -1) { mOutputStream = - new CronetFixedModeOutputStream(this, fixedStreamingModeContentLength, mMessageLoop); + new CronvoyFixedModeOutputStream(this, fixedStreamingModeContentLength, mMessageLoop); // Start the request now since all headers can be sent. startRequest(); } else { @@ -215,10 +215,10 @@ public OutputStream getOutputStream() throws IOException { Log.d(TAG, "Outputstream is being buffered in memory."); String length = getRequestProperty(CONTENT_LENGTH); if (length == null) { - mOutputStream = new CronetBufferedOutputStream(this); + mOutputStream = new CronvoyBufferedOutputStream(this); } else { long lengthParsed = Long.parseLong(length); - mOutputStream = new CronetBufferedOutputStream(this, lengthParsed); + mOutputStream = new CronvoyBufferedOutputStream(this, lengthParsed); } } } @@ -439,7 +439,7 @@ public void setConnectTimeout(int timeout) { // Sockets are assigned to requests according to request priorities // when sockets are connected. This requires requests with the same host, // domain and port to have same timeout. - Log.d(TAG, "setConnectTimeout is not supported by CronetHttpURLConnection"); + Log.d(TAG, "setConnectTimeout is not supported by CronvoyHttpURLConnection"); } /** diff --git a/mobile/library/java/org/chromium/net/urlconnection/CronetHttpURLStreamHandler.java b/mobile/library/java/org/chromium/net/urlconnection/CronvoyHttpURLStreamHandler.java similarity index 83% rename from mobile/library/java/org/chromium/net/urlconnection/CronetHttpURLStreamHandler.java rename to mobile/library/java/org/chromium/net/urlconnection/CronvoyHttpURLStreamHandler.java index 2abadbc23c8f..12ae47b642d0 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/CronetHttpURLStreamHandler.java +++ b/mobile/library/java/org/chromium/net/urlconnection/CronvoyHttpURLStreamHandler.java @@ -11,16 +11,16 @@ * A {@link URLStreamHandler} that handles HTTP and HTTPS connections. One can use this class to * create {@link java.net.HttpURLConnection} instances implemented by Cronet; for example: 
  *
- * CronetHttpURLStreamHandler streamHandler = new CronetHttpURLStreamHandler(myContext);
+ * CronvoyHttpURLStreamHandler streamHandler = new CronvoyHttpURLStreamHandler(myContext);
  * HttpURLConnection connection = (HttpURLConnection)streamHandler.openConnection(
  *         new URL("http://chromium.org"));
* Note: Cronet's {@code HttpURLConnection} implementation is subject to some limitations - * listed {@link CronetURLStreamHandlerFactory here}. + * listed {@link CronvoyURLStreamHandlerFactory here}. */ -final class CronetHttpURLStreamHandler extends URLStreamHandler { +final class CronvoyHttpURLStreamHandler extends URLStreamHandler { private final ExperimentalCronetEngine mCronetEngine; - public CronetHttpURLStreamHandler(ExperimentalCronetEngine cronetEngine) { + public CronvoyHttpURLStreamHandler(ExperimentalCronetEngine cronetEngine) { mCronetEngine = cronetEngine; } diff --git a/mobile/library/java/org/chromium/net/urlconnection/CronetInputStream.java b/mobile/library/java/org/chromium/net/urlconnection/CronvoyInputStream.java similarity index 83% rename from mobile/library/java/org/chromium/net/urlconnection/CronetInputStream.java rename to mobile/library/java/org/chromium/net/urlconnection/CronvoyInputStream.java index 31a2f37b2469..c154caecc963 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/CronetInputStream.java +++ b/mobile/library/java/org/chromium/net/urlconnection/CronvoyInputStream.java @@ -5,11 +5,11 @@ import java.nio.ByteBuffer; /** - * An InputStream that is used by {@link CronetHttpURLConnection} to request + * An InputStream that is used by {@link CronvoyHttpURLConnection} to request * data from the network stack as needed. */ -final class CronetInputStream extends InputStream { - private final CronetHttpURLConnection mHttpURLConnection; +final class CronvoyInputStream extends InputStream { + private final CronvoyHttpURLConnection mHttpURLConnection; // Indicates whether listener's onSucceeded or onFailed callback is invoked. private boolean mResponseDataCompleted; private ByteBuffer mBuffer; @@ -18,11 +18,11 @@ final class CronetInputStream extends InputStream { private static final int READ_BUFFER_SIZE = 32 * 1024; /** - * Constructs a CronetInputStream. - * @param httpURLConnection the CronetHttpURLConnection that is associated + * Constructs a CronvoyInputStream. + * @param httpURLConnection the CronvoyHttpURLConnection that is associated * with this InputStream. */ - public CronetInputStream(CronetHttpURLConnection httpURLConnection) { + public CronvoyInputStream(CronvoyHttpURLConnection httpURLConnection) { mHttpURLConnection = httpURLConnection; } @@ -68,7 +68,7 @@ public int available() throws IOException { } /** - * Called by {@link CronetHttpURLConnection} to notify that the entire + * Called by {@link CronvoyHttpURLConnection} to notify that the entire * response body has been read. * @param exception if not {@code null}, it is the exception to throw when caller * tries to read more data. @@ -94,7 +94,7 @@ private void getMoreDataIfNeeded() throws IOException { } mBuffer.clear(); - // Requests more data from CronetHttpURLConnection. + // Requests more data from CronvoyHttpURLConnection. mHttpURLConnection.getMoreData(mBuffer); if (mException != null) { throw mException; diff --git a/mobile/library/java/org/chromium/net/urlconnection/MessageLoop.java b/mobile/library/java/org/chromium/net/urlconnection/CronvoyMessageLoop.java similarity index 94% rename from mobile/library/java/org/chromium/net/urlconnection/MessageLoop.java rename to mobile/library/java/org/chromium/net/urlconnection/CronvoyMessageLoop.java index 3740fbab801e..09989a397958 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/MessageLoop.java +++ b/mobile/library/java/org/chromium/net/urlconnection/CronvoyMessageLoop.java @@ -12,7 +12,7 @@ /** * A MessageLoop class for use in {@link CronetHttpURLConnection}. */ -final class MessageLoop implements Executor { +final class CronvoyMessageLoop implements Executor { private final BlockingQueue mQueue; // Indicates whether this message loop is currently running. @@ -34,7 +34,7 @@ final class MessageLoop implements Executor { private static final long INVALID_THREAD_ID = -1; private long mThreadId = INVALID_THREAD_ID; - MessageLoop() { mQueue = new LinkedBlockingQueue(); } + CronvoyMessageLoop() { mQueue = new LinkedBlockingQueue(); } private boolean calledOnValidThread() { if (mThreadId == INVALID_THREAD_ID) { @@ -74,7 +74,7 @@ private Runnable take(boolean useTimeout, long timeoutNano) throws InterruptedIO } /** - * Runs the message loop. Be sure to call {@link MessageLoop#quit()} + * Runs the message loop. Be sure to call {@link CronvoyMessageLoop#quit()} * to end the loop. If an interruptedException occurs, the loop cannot be * started again (see {@link #mLoopFailed}). * @throws IOException @@ -82,7 +82,7 @@ private Runnable take(boolean useTimeout, long timeoutNano) throws InterruptedIO public void loop() throws IOException { loop(0); } /** - * Runs the message loop. Be sure to call {@link MessageLoop#quit()} + * Runs the message loop. Be sure to call {@link CronvoyMessageLoop#quit()} * to end the loop. If an interruptedException occurs, the loop cannot be * started again (see {@link #mLoopFailed}). * @param timeoutMilli Timeout, in milliseconds, or 0 for no timeout. diff --git a/mobile/library/java/org/chromium/net/urlconnection/CronetOutputStream.java b/mobile/library/java/org/chromium/net/urlconnection/CronvoyOutputStream.java similarity index 84% rename from mobile/library/java/org/chromium/net/urlconnection/CronetOutputStream.java rename to mobile/library/java/org/chromium/net/urlconnection/CronvoyOutputStream.java index 16a43450fcca..30f0b56290a4 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/CronetOutputStream.java +++ b/mobile/library/java/org/chromium/net/urlconnection/CronvoyOutputStream.java @@ -6,9 +6,9 @@ /** * An abstract class of {@link OutputStream} that concrete implementations must - * extend in order to be used in {@link CronetHttpURLConnection}. + * extend in order to be used in {@link CronvoyHttpURLConnection}. */ -abstract class CronetOutputStream extends OutputStream { +abstract class CronvoyOutputStream extends OutputStream { private IOException mException; private boolean mClosed; private boolean mRequestCompleted; @@ -20,13 +20,13 @@ public void close() throws IOException { /** * Tells the underlying implementation that connection has been established. - * Used in {@link CronetHttpURLConnection}. + * Used in {@link CronvoyHttpURLConnection}. */ abstract void setConnected() throws IOException; /** * Checks whether content received is less than Content-Length. - * Used in {@link CronetHttpURLConnection}. + * Used in {@link CronvoyHttpURLConnection}. */ abstract void checkReceivedEnoughContent() throws IOException; @@ -37,7 +37,7 @@ public void close() throws IOException { /** * Signals that the request is done. If there is no error, - * {@code exception} is null. Used by {@link CronetHttpURLConnection}. + * {@code exception} is null. Used by {@link CronvoyHttpURLConnection}. */ void setRequestCompleted(IOException exception) { mException = exception; diff --git a/mobile/library/java/org/chromium/net/urlconnection/CronetURLStreamHandlerFactory.java b/mobile/library/java/org/chromium/net/urlconnection/CronvoyURLStreamHandlerFactory.java similarity index 85% rename from mobile/library/java/org/chromium/net/urlconnection/CronetURLStreamHandlerFactory.java rename to mobile/library/java/org/chromium/net/urlconnection/CronvoyURLStreamHandlerFactory.java index 04d5fd2b51f2..a3ac3654079a 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/CronetURLStreamHandlerFactory.java +++ b/mobile/library/java/org/chromium/net/urlconnection/CronvoyURLStreamHandlerFactory.java @@ -34,16 +34,16 @@ * * {@hide} */ -public final class CronetURLStreamHandlerFactory implements URLStreamHandlerFactory { +public final class CronvoyURLStreamHandlerFactory implements URLStreamHandlerFactory { private final ExperimentalCronetEngine mCronetEngine; /** - * Creates a {@link CronetURLStreamHandlerFactory} to handle HTTP and HTTPS + * Creates a {@link CronvoyURLStreamHandlerFactory} to handle HTTP and HTTPS * traffic. * @param cronetEngine the {@link CronetEngine} to be used. * @throws NullPointerException if config is null. */ - public CronetURLStreamHandlerFactory(ExperimentalCronetEngine cronetEngine) { + public CronvoyURLStreamHandlerFactory(ExperimentalCronetEngine cronetEngine) { if (cronetEngine == null) { throw new NullPointerException("CronetEngine is null."); } @@ -51,13 +51,13 @@ public CronetURLStreamHandlerFactory(ExperimentalCronetEngine cronetEngine) { } /** - * Returns a {@link CronetHttpURLStreamHandler} for HTTP and HTTPS, and + * Returns a {@link CronvoyHttpURLStreamHandler} for HTTP and HTTPS, and * {@code null} for other protocols. */ @Override public URLStreamHandler createURLStreamHandler(String protocol) { if ("http".equals(protocol) || "https".equals(protocol)) { - return new CronetHttpURLStreamHandler(mCronetEngine); + return new CronvoyHttpURLStreamHandler(mCronetEngine); } return null; } diff --git a/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java b/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java index 607061acbef5..728bf3546f37 100644 --- a/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java +++ b/mobile/test/java/org/chromium/net/BidirectionalStreamTest.java @@ -20,8 +20,8 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.filters.SmallTest; -import org.chromium.net.impl.BidirectionalStreamNetworkException; -import org.chromium.net.impl.CronetBidirectionalStream; +import org.chromium.net.impl.CronvoyBidirectionalStreamNetworkException; +import org.chromium.net.impl.CronvoyBidirectionalStream; import org.chromium.net.impl.Errors.NetError; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestUtil; @@ -40,7 +40,7 @@ import org.chromium.net.testing.MetricsTestUtil.TestRequestFinishedListener; import org.chromium.net.testing.TestBidirectionalStreamCallback.FailureType; import org.chromium.net.testing.TestBidirectionalStreamCallback.ResponseStep; -import org.chromium.net.impl.UrlResponseInfoImpl; +import org.chromium.net.impl.CronvoyUrlResponseInfoImpl; import java.nio.ByteBuffer; import java.util.AbstractMap; @@ -111,7 +111,7 @@ private static UrlResponseInfo createUrlResponseInfo(String[] urls, String messa headersList.add( new AbstractMap.SimpleImmutableEntry(headers[i], headers[i + 1])); } - UrlResponseInfoImpl urlResponseInfo = new UrlResponseInfoImpl( + CronvoyUrlResponseInfoImpl urlResponseInfo = new CronvoyUrlResponseInfoImpl( Arrays.asList(urls), statusCode, message, headersList, false, "h2", null, receivedBytes); return urlResponseInfo; } @@ -365,7 +365,7 @@ public void onWriteCompleted(BidirectionalStream stream, UrlResponseInfo info, if (mNumWriteCompleted <= 3) { // "6" is in pending queue. List pendingData = - ((CronetBidirectionalStream)stream).getPendingDataForTesting(); + ((CronvoyBidirectionalStream)stream).getPendingDataForTesting(); assertEquals(1, pendingData.size()); ByteBuffer pendingBuffer = pendingData.get(0); byte[] content = new byte[pendingBuffer.remaining()]; @@ -373,11 +373,11 @@ public void onWriteCompleted(BidirectionalStream stream, UrlResponseInfo info, assertTrue(Arrays.equals("6".getBytes(), content)); // "4" and "5" have been flushed. - assertEquals(0, ((CronetBidirectionalStream)stream).getFlushDataForTesting().size()); + assertEquals(0, ((CronvoyBidirectionalStream)stream).getFlushDataForTesting().size()); } else if (mNumWriteCompleted == 5) { // Now flush "6", which is still in pending queue. List pendingData = - ((CronetBidirectionalStream)stream).getPendingDataForTesting(); + ((CronvoyBidirectionalStream)stream).getPendingDataForTesting(); assertEquals(1, pendingData.size()); ByteBuffer pendingBuffer = pendingData.get(0); byte[] content = new byte[pendingBuffer.remaining()]; @@ -386,8 +386,8 @@ public void onWriteCompleted(BidirectionalStream stream, UrlResponseInfo info, stream.flush(); - assertEquals(0, ((CronetBidirectionalStream)stream).getPendingDataForTesting().size()); - assertEquals(0, ((CronetBidirectionalStream)stream).getFlushDataForTesting().size()); + assertEquals(0, ((CronvoyBidirectionalStream)stream).getPendingDataForTesting().size()); + assertEquals(0, ((CronvoyBidirectionalStream)stream).getFlushDataForTesting().size()); } } }; @@ -397,8 +397,8 @@ public void onWriteCompleted(BidirectionalStream stream, UrlResponseInfo info, callback.addWriteData("4".getBytes(), false); callback.addWriteData("5".getBytes(), true); callback.addWriteData("6".getBytes(), false); - CronetBidirectionalStream stream = - (CronetBidirectionalStream)mCronetEngine + CronvoyBidirectionalStream stream = + (CronvoyBidirectionalStream)mCronetEngine .newBidirectionalStreamBuilder(url, callback, callback.getExecutor()) .addHeader("foo", "bar") .addHeader("empty", "") @@ -467,8 +467,8 @@ private ByteBuffer getDummyData() { return dummyData; } }; - CronetBidirectionalStream stream = - (CronetBidirectionalStream)mCronetEngine + CronvoyBidirectionalStream stream = + (CronvoyBidirectionalStream)mCronetEngine .newBidirectionalStreamBuilder(url, callback, callback.getExecutor()) .build(); stream.start(); @@ -1326,8 +1326,8 @@ public void testExecutorShutdownBeforeStreamIsDone() { callback.setAutoAdvance(false); BidirectionalStream.Builder builder = mCronetEngine.newBidirectionalStreamBuilder( Http2TestServer.getEchoMethodUrl(), callback, callback.getExecutor()); - CronetBidirectionalStream stream = - (CronetBidirectionalStream)builder.setHttpMethod("GET").build(); + CronvoyBidirectionalStream stream = + (CronvoyBidirectionalStream)builder.setHttpMethod("GET").build(); stream.start(); callback.waitForNextReadStep(); assertFalse(callback.isDone()); @@ -1394,8 +1394,8 @@ public void testCronetEngineShutdown() throws Exception { callback.setAutoAdvance(false); BidirectionalStream.Builder builder = mCronetEngine.newBidirectionalStreamBuilder( Http2TestServer.getEchoMethodUrl(), callback, callback.getExecutor()); - CronetBidirectionalStream stream = - (CronetBidirectionalStream)builder.setHttpMethod("GET").build(); + CronvoyBidirectionalStream stream = + (CronvoyBidirectionalStream)builder.setHttpMethod("GET").build(); stream.start(); try { mCronetEngine.shutdown(); @@ -1438,8 +1438,8 @@ public void testCronetEngineShutdownAfterStreamFailure() throws Exception { TestBidirectionalStreamCallback callback = new ShutdownTestBidirectionalStreamCallback(); BidirectionalStream.Builder builder = mCronetEngine.newBidirectionalStreamBuilder( Http2TestServer.getEchoMethodUrl(), callback, callback.getExecutor()); - CronetBidirectionalStream stream = - (CronetBidirectionalStream)builder.setHttpMethod("GET").build(); + CronvoyBidirectionalStream stream = + (CronvoyBidirectionalStream)builder.setHttpMethod("GET").build(); stream.start(); callback.setFailure(FailureType.THROW_SYNC, ResponseStep.ON_READ_COMPLETED); callback.blockForDone(); @@ -1455,8 +1455,8 @@ public void testCronetEngineShutdownAfterStreamCancel() throws Exception { TestBidirectionalStreamCallback callback = new ShutdownTestBidirectionalStreamCallback(); BidirectionalStream.Builder builder = mCronetEngine.newBidirectionalStreamBuilder( Http2TestServer.getEchoMethodUrl(), callback, callback.getExecutor()); - CronetBidirectionalStream stream = - (CronetBidirectionalStream)builder.setHttpMethod("GET").build(); + CronvoyBidirectionalStream stream = + (CronvoyBidirectionalStream)builder.setHttpMethod("GET").build(); // Block callback when response starts to verify that shutdown fails // if there are active requests. @@ -1522,7 +1522,7 @@ private static String bufferContentsToString(ByteBuffer byteBuffer, int start, i private static void checkSpecificErrorCode(NetError netError, int errorCode, boolean immediatelyRetryable) throws Exception { NetworkException exception = - new BidirectionalStreamNetworkException("", errorCode, netError.getErrorCode()); + new CronvoyBidirectionalStreamNetworkException("", errorCode, netError.getErrorCode()); assertEquals(immediatelyRetryable, exception.immediatelyRetryable()); assertEquals(netError.getErrorCode(), exception.getCronetInternalErrorCode()); assertEquals(errorCode, exception.getErrorCode()); diff --git a/mobile/test/java/org/chromium/net/CronetEngineBuilderTest.java b/mobile/test/java/org/chromium/net/CronetEngineBuilderTest.java index 441bcafeb4a1..d5003821e7bb 100644 --- a/mobile/test/java/org/chromium/net/CronetEngineBuilderTest.java +++ b/mobile/test/java/org/chromium/net/CronetEngineBuilderTest.java @@ -1,7 +1,7 @@ package org.chromium.net; -import static org.chromium.net.CronetProvider.PROVIDER_NAME_APP_PACKAGED; -import static org.chromium.net.CronetProvider.PROVIDER_NAME_FALLBACK; +import static org.chromium.net.CronvoyProvider.PROVIDER_NAME_APP_PACKAGED; +import static org.chromium.net.CronvoyProvider.PROVIDER_NAME_FALLBACK; import static org.chromium.net.testing.CronetTestRule.getContext; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -52,15 +52,15 @@ public void testVersionComparison() { @SmallTest @Feature({"Cronet"}) public void testProviderOrdering() { - final CronetProvider[] availableProviders = new CronetProvider[] { + final CronvoyProvider[] availableProviders = new CronvoyProvider[] { createMockCronetProvider(PROVIDER_NAME_APP_PACKAGED, "99.77", true), createMockCronetProvider(PROVIDER_NAME_FALLBACK, "99.99", true), createMockCronetProvider("Some other provider", "99.88", true), }; - ArrayList providers = new ArrayList<>(Arrays.asList(availableProviders)); - List orderedProviders = - CronetEngine.Builder.getEnabledCronetProviders(getContext(), providers); + ArrayList providers = new ArrayList<>(Arrays.asList(availableProviders)); + List orderedProviders = + CronetEngine.Builder.getEnabledCronvoyProviders(getContext(), providers); // Check the result assertEquals(availableProviders[2], orderedProviders.get(0)); @@ -76,15 +76,15 @@ public void testProviderOrdering() { @SmallTest @Feature({"Cronet"}) public void testThatDisabledProvidersAreExcluded() { - final CronetProvider[] availableProviders = new CronetProvider[] { + final CronvoyProvider[] availableProviders = new CronvoyProvider[] { createMockCronetProvider(PROVIDER_NAME_FALLBACK, "99.99", true), createMockCronetProvider(PROVIDER_NAME_APP_PACKAGED, "99.77", true), createMockCronetProvider("Some other provider", "99.88", false), }; - ArrayList providers = new ArrayList<>(Arrays.asList(availableProviders)); - List orderedProviders = - CronetEngine.Builder.getEnabledCronetProviders(getContext(), providers); + ArrayList providers = new ArrayList<>(Arrays.asList(availableProviders)); + List orderedProviders = + CronetEngine.Builder.getEnabledCronvoyProviders(getContext(), providers); assertEquals("Unexpected number of providers in the list", 2, orderedProviders.size()); assertEquals(PROVIDER_NAME_APP_PACKAGED, orderedProviders.get(0).getName()); @@ -113,9 +113,9 @@ private void assertIllegalArgumentException(String s1, String s2) { fail("Expected IllegalArgumentException"); } - private static CronetProvider createMockCronetProvider(String mName, String mVersion, - boolean mEnabled) { - CronetProvider mock = mock(CronetProvider.class); + private static CronvoyProvider createMockCronetProvider(String mName, String mVersion, + boolean mEnabled) { + CronvoyProvider mock = mock(CronvoyProvider.class); when(mock.getName()).thenReturn(mName); when(mock.getVersion()).thenReturn(mVersion); when(mock.isEnabled()).thenReturn(mEnabled); diff --git a/mobile/test/java/org/chromium/net/CronetUrlRequestContextTest.java b/mobile/test/java/org/chromium/net/CronetUrlRequestContextTest.java index e99eba9844a0..fc8f5e4de5fc 100644 --- a/mobile/test/java/org/chromium/net/CronetUrlRequestContextTest.java +++ b/mobile/test/java/org/chromium/net/CronetUrlRequestContextTest.java @@ -28,8 +28,8 @@ import java.util.concurrent.Executor; import java.util.concurrent.FutureTask; import java.util.concurrent.atomic.AtomicReference; -import org.chromium.net.impl.CronetUrlRequestContext; -import org.chromium.net.impl.NativeCronetEngineBuilderImpl; +import org.chromium.net.impl.CronvoyUrlRequestContext; +import org.chromium.net.impl.NativeCronvoyEngineBuilderImpl; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; import org.chromium.net.testing.CronetTestRule.RequiresMinApi; @@ -229,8 +229,8 @@ public void run() { // Create new request context, but its initialization on the main thread // will be stuck behind blockingTask. - final CronetUrlRequestContext cronetEngine = - (CronetUrlRequestContext) new CronetEngine.Builder(getContext()).build(); + final CronvoyUrlRequestContext cronetEngine = + (CronvoyUrlRequestContext) new CronetEngine.Builder(getContext()).build(); // Unblock the main thread, so context gets initialized and shutdown on // it. block.open(); @@ -252,8 +252,8 @@ public void testInitAndShutdownOnMainThread() throws Exception { @Override public void run() { // Create new request context, loading the library. - final CronetUrlRequestContext cronetEngine = - (CronetUrlRequestContext) new CronetEngine.Builder(getContext()).build(); + final CronvoyUrlRequestContext cronetEngine = + (CronvoyUrlRequestContext) new CronetEngine.Builder(getContext()).build(); // Shutdown right after init. cronetEngine.shutdown(); // Verify that context is shutdown. @@ -1281,7 +1281,7 @@ public void testSetLibraryLoaderIsEnforcedByDefaultEmbeddedProvider() throws Exc @Ignore("Library Loader not needed") public void testSetLibraryLoaderIsIgnoredInNativeCronetEngineBuilderImpl() throws Exception { CronetEngine.Builder builder = - new CronetEngine.Builder(new NativeCronetEngineBuilderImpl(getContext())); + new CronetEngine.Builder(new NativeCronvoyEngineBuilderImpl(getContext())); TestBadLibraryLoader loader = new TestBadLibraryLoader(); builder.setLibraryLoader(loader); CronetEngine engine = builder.build(); @@ -1294,7 +1294,7 @@ public void testSetLibraryLoaderIsIgnoredInNativeCronetEngineBuilderImpl() throw @Feature({"Cronet"}) public void testNativeCronetEngineBuilderImplSetsCorrectVersionString() throws Exception { CronetEngine.Builder builder = - new CronetEngine.Builder(new NativeCronetEngineBuilderImpl(getContext())); + new CronetEngine.Builder(new NativeCronvoyEngineBuilderImpl(getContext())); CronetEngine engine = builder.build(); assertTrue(engine.getVersionString().startsWith("Cronet/")); } diff --git a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java index 53e3a9602ce6..99d7de35ce27 100644 --- a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java +++ b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java @@ -36,10 +36,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import io.envoyproxy.envoymobile.engine.AndroidNetworkMonitor; -import org.chromium.net.impl.CronetUrlRequest; +import org.chromium.net.impl.CronvoyUrlRequest; import org.chromium.net.impl.Errors.EnvoyMobileError; import org.chromium.net.impl.Errors.NetError; -import org.chromium.net.impl.UrlResponseInfoImpl; +import org.chromium.net.impl.CronvoyUrlResponseInfoImpl; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; import org.chromium.net.testing.CronetTestRule.RequiresMinApi; @@ -186,9 +186,9 @@ private static UrlResponseInfo createUrlResponseInfo(String[] urls, String messa headersList.add( new AbstractMap.SimpleImmutableEntry(headers[i], headers[i + 1])); } - UrlResponseInfoImpl unknown = - new UrlResponseInfoImpl(Arrays.asList(urls), statusCode, message, headersList, false, - "unknown", ":0", receivedBytes); + CronvoyUrlResponseInfoImpl unknown = + new CronvoyUrlResponseInfoImpl(Arrays.asList(urls), statusCode, message, headersList, false, + "unknown", ":0", receivedBytes); return unknown; } @@ -1922,7 +1922,7 @@ public void testExecutorShutdown() { callback.setAutoAdvance(false); UrlRequest.Builder builder = mMockUrlRequestJobFactory.getCronetEngine().newUrlRequestBuilder( NativeTestServer.getEchoBodyURL(), callback, callback.getExecutor()); - CronetUrlRequest urlRequest = (CronetUrlRequest)builder.build(); + CronvoyUrlRequest urlRequest = (CronvoyUrlRequest)builder.build(); urlRequest.start(); callback.waitForNextStep(); assertFalse(callback.isDone()); @@ -2024,7 +2024,7 @@ public void testDestroyUploadDataStreamAdapterOnSucceededCallback() throws Excep TestUploadDataProvider.SuccessCallbackMode.SYNC, callback.getExecutor()); builder.setUploadDataProvider(dataProvider, callback.getExecutor()); builder.addHeader("content-type", "useless/string"); - CronetUrlRequest request = (CronetUrlRequest)builder.build(); + CronvoyUrlRequest request = (CronvoyUrlRequest)builder.build(); final ConditionVariable uploadDataStreamAdapterDestroyed = new ConditionVariable(); // request.setOnDestroyedUploadCallbackForTesting(new Runnable() { // @Override @@ -2209,7 +2209,8 @@ private void checkSpecificErrorCode(@EnvoyMobileError long envoyMobileError, Net ((NetworkException)callback.mError).getCronetInternalErrorCode()); assertEquals(errorCode, ((NetworkException)callback.mError).getErrorCode()); assertEquals(immediatelyRetryable, ((NetworkException)callback.mError).immediatelyRetryable()); - assertContains("Exception in CronetUrlRequest: net::ERR_" + name, callback.mError.getMessage()); + assertContains("Exception in CronvoyUrlRequest: net::ERR_" + name, + callback.mError.getMessage()); assertEquals(0, callback.mRedirectCount); assertTrue(callback.mOnErrorCalled); assertEquals(ResponseStep.ON_FAILED, callback.mResponseStep); diff --git a/mobile/test/java/org/chromium/net/RequestFinishedInfoTest.java b/mobile/test/java/org/chromium/net/RequestFinishedInfoTest.java index 8f04f18de196..3e48ccba3130 100644 --- a/mobile/test/java/org/chromium/net/RequestFinishedInfoTest.java +++ b/mobile/test/java/org/chromium/net/RequestFinishedInfoTest.java @@ -18,7 +18,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; -import org.chromium.net.impl.CronetMetrics; +import org.chromium.net.impl.CronvoyMetrics; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; import org.chromium.net.testing.CronetTestRule.RequiresMinApi; @@ -360,9 +360,9 @@ public void testMetricsGetters() throws Exception { long receivedByteCount = 15; // Make sure nothing gets reordered inside the Metrics class RequestFinishedInfo.Metrics metrics = - new CronetMetrics(requestStart, dnsStart, dnsEnd, connectStart, connectEnd, sslStart, - sslEnd, sendingStart, sendingEnd, pushStart, pushEnd, responseStart, - requestEnd, socketReused, sentByteCount, receivedByteCount); + new CronvoyMetrics(requestStart, dnsStart, dnsEnd, connectStart, connectEnd, sslStart, + sslEnd, sendingStart, sendingEnd, pushStart, pushEnd, responseStart, + requestEnd, socketReused, sentByteCount, receivedByteCount); assertEquals(new Date(requestStart), metrics.getRequestStart()); // -1 timestamp should translate to null assertNull(metrics.getDnsEnd()); diff --git a/mobile/test/java/org/chromium/net/UrlResponseInfoTest.java b/mobile/test/java/org/chromium/net/UrlResponseInfoTest.java index 8e9105e525ff..80890c9811e1 100644 --- a/mobile/test/java/org/chromium/net/UrlResponseInfoTest.java +++ b/mobile/test/java/org/chromium/net/UrlResponseInfoTest.java @@ -9,7 +9,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.chromium.net.impl.UrlResponseInfoImpl; +import org.chromium.net.impl.CronvoyUrlResponseInfoImpl; import org.chromium.net.testing.Feature; import org.junit.Test; import org.junit.runner.RunWith; @@ -38,9 +38,9 @@ public void testPublicAPI() throws Exception { final String proxyServer = "example.com"; final long receivedByteCount = 42; - final UrlResponseInfo info = - new UrlResponseInfoImpl(urlChain, httpStatusCode, httpStatusText, allHeadersList, wasCached, - negotiatedProtocol, proxyServer, receivedByteCount); + final UrlResponseInfo info = new CronvoyUrlResponseInfoImpl( + urlChain, httpStatusCode, httpStatusText, allHeadersList, wasCached, negotiatedProtocol, + proxyServer, receivedByteCount); assertEquals(info.getUrlChain(), urlChain); try { info.getUrlChain().add("example.com"); diff --git a/mobile/test/java/org/chromium/net/impl/CronetBidirectionalStateTest.java b/mobile/test/java/org/chromium/net/impl/CronetBidirectionalStateTest.java index 825b34446cd4..fb16d1e4fcc7 100644 --- a/mobile/test/java/org/chromium/net/impl/CronetBidirectionalStateTest.java +++ b/mobile/test/java/org/chromium/net/impl/CronetBidirectionalStateTest.java @@ -5,8 +5,8 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; -import org.chromium.net.impl.CronetBidirectionalState.NextAction; -import org.chromium.net.impl.CronetBidirectionalState.Event; +import org.chromium.net.impl.CronvoyBidirectionalState.NextAction; +import org.chromium.net.impl.CronvoyBidirectionalState.Event; import org.junit.Test; import org.junit.runner.RunWith; @@ -21,7 +21,8 @@ @RunWith(AndroidJUnit4.class) public class CronetBidirectionalStateTest { - private final CronetBidirectionalState mCronetBidirectionalState = new CronetBidirectionalState(); + private final CronvoyBidirectionalState mCronetBidirectionalState = + new CronvoyBidirectionalState(); // ================= USER_START.* ================= diff --git a/mobile/test/java/org/chromium/net/impl/CronvoyEngineTest.java b/mobile/test/java/org/chromium/net/impl/CronvoyEngineTest.java index 27bf4dda197e..c4a23a981dc8 100644 --- a/mobile/test/java/org/chromium/net/impl/CronvoyEngineTest.java +++ b/mobile/test/java/org/chromium/net/impl/CronvoyEngineTest.java @@ -45,7 +45,7 @@ public class CronvoyEngineTest { private static final String TEST_URL_PATH = "get/flowers"; - private static org.chromium.net.impl.CronetUrlRequestContext cronvoyEngine; + private static org.chromium.net.impl.CronvoyUrlRequestContext cronvoyEngine; private final MockWebServer mockWebServer = new MockWebServer(); @@ -64,10 +64,10 @@ public static void shutdown() { @Before public void setUp() { if (cronvoyEngine == null) { - NativeCronetEngineBuilderImpl nativeCronetEngineBuilder = - new NativeCronetEngineBuilderImpl(ApplicationProvider.getApplicationContext()); + NativeCronvoyEngineBuilderImpl nativeCronetEngineBuilder = + new NativeCronvoyEngineBuilderImpl(ApplicationProvider.getApplicationContext()); nativeCronetEngineBuilder.setUserAgent("Cronvoy"); - cronvoyEngine = new CronetUrlRequestContext(nativeCronetEngineBuilder); + cronvoyEngine = new CronvoyUrlRequestContext(nativeCronetEngineBuilder); } } diff --git a/mobile/test/java/org/chromium/net/testing/CronetTestRule.java b/mobile/test/java/org/chromium/net/testing/CronetTestRule.java index ba3abf38e012..c03fe2ba1907 100644 --- a/mobile/test/java/org/chromium/net/testing/CronetTestRule.java +++ b/mobile/test/java/org/chromium/net/testing/CronetTestRule.java @@ -24,9 +24,9 @@ import org.chromium.net.CronetEngine; import org.chromium.net.ExperimentalCronetEngine; import org.chromium.net.UrlResponseInfo; -import org.chromium.net.impl.CronetEngineBuilderImpl; -import org.chromium.net.impl.NativeCronetProvider; -import org.chromium.net.impl.UserAgent; +import org.chromium.net.impl.CronvoyEngineBuilderImpl; +import org.chromium.net.impl.NativeCronvoyProvider; +import org.chromium.net.impl.CronvoyUserAgent; import org.junit.Assert; import org.junit.rules.TestRule; import org.junit.runner.Description; @@ -272,7 +272,7 @@ public CronetTestFramework startCronetTestFramework() { * @return the {@code CronetEngine.Builder} that builds Chromium-based {@code Cronet engine}. */ public static ExperimentalCronetEngine.Builder createNativeEngineBuilder(Context context) { - return (ExperimentalCronetEngine.Builder) new NativeCronetProvider(context).createBuilder(); + return (ExperimentalCronetEngine.Builder) new NativeCronvoyProvider(context).createBuilder(); } public void assertResponseEquals(UrlResponseInfo expected, UrlResponseInfo actual) { diff --git a/mobile/test/java/org/chromium/net/testing/CronetTestRuleTest.java b/mobile/test/java/org/chromium/net/testing/CronetTestRuleTest.java index 82cbb265fa35..8519dd113b48 100644 --- a/mobile/test/java/org/chromium/net/testing/CronetTestRuleTest.java +++ b/mobile/test/java/org/chromium/net/testing/CronetTestRuleTest.java @@ -7,7 +7,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.filters.SmallTest; -import org.chromium.net.impl.CronetUrlRequestContext; +import org.chromium.net.impl.CronvoyUrlRequestContext; import org.chromium.net.testing.CronetTestRule.CronetTestFramework; import org.chromium.net.testing.CronetTestRule.RequiresMinApi; import org.junit.After; @@ -67,6 +67,6 @@ public void testRequiresMinApiMustRun() { public void testRunOnlyNativeMustRun() { assertFalse(mTestWasRun); mTestWasRun = true; - assertEquals(mTestFramework.mCronetEngine.getClass(), CronetUrlRequestContext.class); + assertEquals(mTestFramework.mCronetEngine.getClass(), CronvoyUrlRequestContext.class); } } diff --git a/mobile/test/java/org/chromium/net/testing/CronetTestUtil.java b/mobile/test/java/org/chromium/net/testing/CronetTestUtil.java index 33d347180fed..6f7468c0ba2a 100644 --- a/mobile/test/java/org/chromium/net/testing/CronetTestUtil.java +++ b/mobile/test/java/org/chromium/net/testing/CronetTestUtil.java @@ -1,7 +1,7 @@ package org.chromium.net.testing; import org.chromium.net.ExperimentalCronetEngine; -import org.chromium.net.impl.NativeCronetEngineBuilderImpl; +import org.chromium.net.impl.NativeCronvoyEngineBuilderImpl; /** * Utilities for Cronet testing @@ -12,9 +12,9 @@ public static void setMockCertVerifierForTesting(ExperimentalCronetEngine.Builde getCronetEngineBuilderImpl(builder).setMockCertVerifierForTesting(); } - public static NativeCronetEngineBuilderImpl + public static NativeCronvoyEngineBuilderImpl getCronetEngineBuilderImpl(ExperimentalCronetEngine.Builder builder) { - return (NativeCronetEngineBuilderImpl)builder.getBuilderDelegate(); + return (NativeCronvoyEngineBuilderImpl)builder.getBuilderDelegate(); } public static boolean nativeCanGetTaggedBytes() { diff --git a/mobile/test/java/org/chromium/net/urlconnection/CronetFixedModeOutputStreamTest.java b/mobile/test/java/org/chromium/net/urlconnection/CronetFixedModeOutputStreamTest.java index 3398de5ed0a2..4682d2c432ae 100644 --- a/mobile/test/java/org/chromium/net/urlconnection/CronetFixedModeOutputStreamTest.java +++ b/mobile/test/java/org/chromium/net/urlconnection/CronetFixedModeOutputStreamTest.java @@ -16,7 +16,7 @@ import java.net.URL; import org.chromium.net.CronetEngine; import org.chromium.net.NetworkException; -import org.chromium.net.impl.CallbackExceptionImpl; +import org.chromium.net.impl.CronvoyCallbackExceptionImpl; import org.chromium.net.testing.CronetTestRule; import org.chromium.net.testing.CronetTestRule.CompareDefaultWithCronet; import org.chromium.net.testing.CronetTestRule.OnlyRunCronetHttpURLConnection; @@ -392,7 +392,7 @@ public void testJavaBufferSizeLargerThanNativeBufferSize() throws Exception { // (largeData.length % 17384 = 9448, largeData.length % 18384 = 16752). int[] bufferLengths = new int[] {17384, 18384}; for (int length : bufferLengths) { - CronetFixedModeOutputStream.setDefaultBufferLengthForTesting(length); + CronvoyFixedModeOutputStream.setDefaultBufferLengthForTesting(length); // Run the following three tests with this custom buffer size. testFixedLengthStreamingModeLargeDataWriteOneByte(); testFixedLengthStreamingModeLargeData(); @@ -450,7 +450,7 @@ public void testRewindWithCronet() throws Exception { connection.setDoOutput(true); connection.setRequestMethod("POST"); connection.setFixedLengthStreamingMode(TestUtil.UPLOAD_DATA.length); - thrown.expect(instanceOf(CallbackExceptionImpl.class)); + thrown.expect(instanceOf(CronvoyCallbackExceptionImpl.class)); thrown.expectMessage("Exception received from UploadDataProvider"); thrown.expectCause( new CauseMatcher(HttpRetryException.class, "Cannot retry streamed Http body")); diff --git a/mobile/test/java/org/chromium/net/urlconnection/CronetHttpURLStreamHandlerTest.java b/mobile/test/java/org/chromium/net/urlconnection/CronetHttpURLStreamHandlerTest.java index 04227ab2e147..9bb4cdf9b36a 100644 --- a/mobile/test/java/org/chromium/net/urlconnection/CronetHttpURLStreamHandlerTest.java +++ b/mobile/test/java/org/chromium/net/urlconnection/CronetHttpURLStreamHandlerTest.java @@ -47,8 +47,8 @@ public void tearDown() throws Exception { @Feature({"Cronet"}) public void testOpenConnectionHttp() throws Exception { URL url = new URL(NativeTestServer.getEchoMethodURL()); - CronetHttpURLStreamHandler streamHandler = - new CronetHttpURLStreamHandler(mTestFramework.mCronetEngine); + CronvoyHttpURLStreamHandler streamHandler = + new CronvoyHttpURLStreamHandler(mTestFramework.mCronetEngine); HttpURLConnection connection = (HttpURLConnection)streamHandler.openConnection(url); assertEquals(200, connection.getResponseCode()); assertEquals("OK", connection.getResponseMessage()); @@ -61,8 +61,8 @@ public void testOpenConnectionHttp() throws Exception { @Feature({"Cronet"}) public void testOpenConnectionHttps() throws Exception { URL url = new URL("https://example.com"); - CronetHttpURLStreamHandler streamHandler = - new CronetHttpURLStreamHandler(mTestFramework.mCronetEngine); + CronvoyHttpURLStreamHandler streamHandler = + new CronvoyHttpURLStreamHandler(mTestFramework.mCronetEngine); HttpURLConnection connection = (HttpURLConnection)streamHandler.openConnection(url); assertNotNull(connection); } @@ -72,8 +72,8 @@ public void testOpenConnectionHttps() throws Exception { @Feature({"Cronet"}) public void testOpenConnectionProtocolNotSupported() throws Exception { URL url = new URL("ftp://example.com"); - CronetHttpURLStreamHandler streamHandler = - new CronetHttpURLStreamHandler(mTestFramework.mCronetEngine); + CronvoyHttpURLStreamHandler streamHandler = + new CronvoyHttpURLStreamHandler(mTestFramework.mCronetEngine); try { streamHandler.openConnection(url); fail(); @@ -87,8 +87,8 @@ public void testOpenConnectionProtocolNotSupported() throws Exception { @Feature({"Cronet"}) public void testOpenConnectionWithProxy() throws Exception { URL url = new URL(NativeTestServer.getEchoMethodURL()); - CronetHttpURLStreamHandler streamHandler = - new CronetHttpURLStreamHandler(mTestFramework.mCronetEngine); + CronvoyHttpURLStreamHandler streamHandler = + new CronvoyHttpURLStreamHandler(mTestFramework.mCronetEngine); Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 8080)); try { streamHandler.openConnection(url, proxy); diff --git a/mobile/test/java/org/chromium/net/urlconnection/CronetInputStreamTest.java b/mobile/test/java/org/chromium/net/urlconnection/CronetInputStreamTest.java index 4d0613de7323..cc064fbb6c5a 100644 --- a/mobile/test/java/org/chromium/net/urlconnection/CronetInputStreamTest.java +++ b/mobile/test/java/org/chromium/net/urlconnection/CronetInputStreamTest.java @@ -26,13 +26,13 @@ public class CronetInputStreamTest { @Rule public final CronetTestRule mTestRule = new CronetTestRule(); - private CronetHttpURLConnection mMockConnection; + private CronvoyHttpURLConnection mMockConnection; @Before public void setUp() throws Exception { // Disable StrictMode constraints for mock initialization. try (StrictModeContext ignored = StrictModeContext.allowAllVmPolicies()) { - mMockConnection = Mockito.mock(CronetHttpURLConnection.class); + mMockConnection = Mockito.mock(CronvoyHttpURLConnection.class); } } @@ -133,7 +133,7 @@ public void testRead_afterDataCompleted() throws Exception { } private void runTestCase(CronetInputStreamTestCase testCase) throws Exception { - try (CronetInputStream underTest = new CronetInputStream(mMockConnection)) { + try (CronvoyInputStream underTest = new CronvoyInputStream(mMockConnection)) { testCase.runTestCase(underTest); } } @@ -163,6 +163,6 @@ private static IOException assertThrowsIoException(Callable callable) throws } private static interface CronetInputStreamTestCase { - void runTestCase(CronetInputStream underTest) throws Exception; + void runTestCase(CronvoyInputStream underTest) throws Exception; } } diff --git a/mobile/test/java/org/chromium/net/urlconnection/CronetURLStreamHandlerFactoryTest.java b/mobile/test/java/org/chromium/net/urlconnection/CronetURLStreamHandlerFactoryTest.java index 1e731172e07f..92aa05bdb935 100644 --- a/mobile/test/java/org/chromium/net/urlconnection/CronetURLStreamHandlerFactoryTest.java +++ b/mobile/test/java/org/chromium/net/urlconnection/CronetURLStreamHandlerFactoryTest.java @@ -25,7 +25,7 @@ public class CronetURLStreamHandlerFactoryTest { public void testRequireConfig() throws Exception { mTestRule.startCronetTestFramework(); try { - new CronetURLStreamHandlerFactory(null); + new CronvoyURLStreamHandlerFactory(null); fail(); } catch (NullPointerException e) { assertEquals("CronetEngine is null.", e.getMessage()); diff --git a/mobile/test/java/org/chromium/net/urlconnection/MessageLoopTest.java b/mobile/test/java/org/chromium/net/urlconnection/MessageLoopTest.java index d5a1afb2d4e4..6b7d6a9a16e3 100644 --- a/mobile/test/java/org/chromium/net/urlconnection/MessageLoopTest.java +++ b/mobile/test/java/org/chromium/net/urlconnection/MessageLoopTest.java @@ -42,7 +42,7 @@ public Thread newThread(Runnable r) { @SmallTest @Feature({"Cronet"}) public void testInterrupt() throws Exception { - final MessageLoop loop = new MessageLoop(); + final CronvoyMessageLoop loop = new CronvoyMessageLoop(); assertFalse(loop.isRunning()); Future future = mExecutorService.submit(new Runnable() { @Override @@ -85,7 +85,7 @@ public void run() { @SmallTest @Feature({"Cronet"}) public void testTaskFailed() throws Exception { - final MessageLoop loop = new MessageLoop(); + final CronvoyMessageLoop loop = new CronvoyMessageLoop(); assertFalse(loop.isRunning()); Future future = mExecutorService.submit(new Runnable() { @Override @@ -136,7 +136,7 @@ public void run() { @SmallTest @Feature({"Cronet"}) public void testLoopWithTimeout() throws Exception { - final MessageLoop loop = new MessageLoop(); + final CronvoyMessageLoop loop = new CronvoyMessageLoop(); assertFalse(loop.isRunning()); // The MessageLoop queue is empty. Use a timeout of 100ms to check that // it doesn't block forever. From c11896c2acf3245c162f6ae74244fdb35c469d92 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 2 May 2023 20:58:17 -0400 Subject: [PATCH 086/740] lb: moving ring hash to extensions (#27110) Signed-off-by: Alyssa Wilk --- changelogs/current.yaml | 2 +- source/common/upstream/BUILD | 16 ------------ .../common/upstream/cluster_manager_impl.cc | 10 ++++---- .../load_balancing_policies/ring_hash/BUILD | 22 +++++++++++++++- .../ring_hash/config.cc | 12 +++++---- .../ring_hash}/ring_hash_lb.cc | 2 +- .../ring_hash}/ring_hash_lb.h | 0 .../load_balancing_policies/subset/BUILD | 2 +- .../subset/subset_lb.cc | 2 +- test/common/upstream/BUILD | 25 ++----------------- .../upstream/load_balancer_benchmark.cc | 2 +- .../load_balancing_policies/ring_hash/BUILD | 23 +++++++++++++++++ .../ring_hash}/ring_hash_lb_test.cc | 2 +- test/integration/BUILD | 2 ++ test/server/BUILD | 1 + tools/code_format/config.yaml | 1 + 16 files changed, 68 insertions(+), 56 deletions(-) rename source/{common/upstream => extensions/load_balancing_policies/ring_hash}/ring_hash_lb.cc (99%) rename source/{common/upstream => extensions/load_balancing_policies/ring_hash}/ring_hash_lb.h (100%) rename test/{common/upstream => extensions/load_balancing_policies/ring_hash}/ring_hash_lb_test.cc (99%) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index a0984d789730..aeea0b877f8a 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -4,7 +4,7 @@ behavior_changes: # *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* - area: build change: | - Moved the subset and maglev LB code into extensions. If you use these load balancers and override + Moved the subset, ring_hash, and maglev LB code into extensions. If you use these load balancers and override extensions_build_config.bzl you will need to include them explicitly. - area: build change: | diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 16eec190993a..337b8f539648 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -85,7 +85,6 @@ envoy_cc_library( ":load_balancer_lib", ":load_stats_reporter_lib", ":od_cds_api_lib", - ":ring_hash_lb_lib", ":host_utility_lib", "//envoy/api:api_interface", "//envoy/config:xds_resources_delegate_interface", @@ -383,21 +382,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "ring_hash_lb_lib", - srcs = ["ring_hash_lb.cc"], - hdrs = ["ring_hash_lb.h"], - external_deps = [ - "abseil_inlined_vector", - ], - deps = [ - ":thread_aware_lb_lib", - "//source/common/common:minimal_logger_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/load_balancing_policies/ring_hash/v3:pkg_cc_proto", - ], -) - envoy_cc_library( name = "upstream_lib", srcs = ["upstream_impl.cc"], diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 150cbbae7299..927c78b45b23 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -45,7 +45,6 @@ #include "source/common/upstream/cluster_factory_impl.h" #include "source/common/upstream/load_balancer_impl.h" #include "source/common/upstream/priority_conn_pool_map_impl.h" -#include "source/common/upstream/ring_hash_lb.h" #ifdef ENVOY_ENABLE_QUIC #include "source/common/http/conn_pool_grid.h" @@ -907,10 +906,11 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust // because the thread_aware_lb_ field takes precedence over the subset lb). if (cluster_reference.info()->lbType() == LoadBalancerType::RingHash) { if (!cluster_reference.info()->lbSubsetInfo().isEnabled()) { - cluster_entry_it->second->thread_aware_lb_ = std::make_unique( - cluster_reference.prioritySet(), cluster_reference.info()->lbStats(), - cluster_reference.info()->statsScope(), runtime_, random_, - cluster_reference.info()->lbRingHashConfig(), cluster_reference.info()->lbConfig()); + auto& factory = Config::Utility::getAndCheckFactoryByName( + "envoy.load_balancing_policies.ring_hash"); + cluster_entry_it->second->thread_aware_lb_ = + factory.create(*cluster_reference.info(), cluster_reference.prioritySet(), runtime_, + random_, time_source_); } } else if (cluster_reference.info()->lbType() == LoadBalancerType::Maglev) { if (!cluster_reference.info()->lbSubsetInfo().isEnabled()) { diff --git a/source/extensions/load_balancing_policies/ring_hash/BUILD b/source/extensions/load_balancing_policies/ring_hash/BUILD index 03bf3e916424..6e9d90339947 100644 --- a/source/extensions/load_balancing_policies/ring_hash/BUILD +++ b/source/extensions/load_balancing_policies/ring_hash/BUILD @@ -1,6 +1,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_extension", + "envoy_cc_library", "envoy_extension_package", ) @@ -8,15 +9,34 @@ licenses(["notice"]) # Apache 2 envoy_extension_package() +envoy_cc_library( + name = "ring_hash_lb_lib", + srcs = ["ring_hash_lb.cc"], + hdrs = ["ring_hash_lb.h"], + external_deps = [ + "abseil_inlined_vector", + ], + deps = [ + "//source/common/common:minimal_logger_lib", + "//source/common/upstream:thread_aware_lb_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/ring_hash/v3:pkg_cc_proto", + ], +) + envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + extra_visibility = [ + # previously considered core code. + "//test:__subpackages__", + ], deps = [ + ":ring_hash_lb_lib", "//source/common/common:minimal_logger_lib", "//source/common/upstream:load_balancer_factory_base_lib", "//source/common/upstream:load_balancer_lib", - "//source/common/upstream:ring_hash_lb_lib", "@envoy_api//envoy/extensions/load_balancing_policies/ring_hash/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/load_balancing_policies/ring_hash/config.cc b/source/extensions/load_balancing_policies/ring_hash/config.cc index ef6f5a4d45ef..41972d1d5252 100644 --- a/source/extensions/load_balancing_policies/ring_hash/config.cc +++ b/source/extensions/load_balancing_policies/ring_hash/config.cc @@ -1,6 +1,6 @@ #include "source/extensions/load_balancing_policies/ring_hash/config.h" -#include "source/common/upstream/ring_hash_lb.h" +#include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" namespace Envoy { namespace Extensions { @@ -16,10 +16,12 @@ Upstream::ThreadAwareLoadBalancerPtr Factory::create(const Upstream::ClusterInfo dynamic_cast( cluster_info.loadBalancingPolicy().get()); - // The load balancing policy configuration will be loaded and validated in the main thread when we - // load the cluster configuration. So we can assume the configuration is valid here. - ASSERT(typed_config != nullptr, - "Invalid load balancing policy configuration for ring hash load balancer"); + // Assume legacy config. + if (!typed_config) { + return std::make_unique( + priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random, + cluster_info.lbRingHashConfig(), cluster_info.lbConfig()); + } return std::make_unique( priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random, diff --git a/source/common/upstream/ring_hash_lb.cc b/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.cc similarity index 99% rename from source/common/upstream/ring_hash_lb.cc rename to source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.cc index 8413d037fedc..586941f7a3a1 100644 --- a/source/common/upstream/ring_hash_lb.cc +++ b/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.cc @@ -1,4 +1,4 @@ -#include "source/common/upstream/ring_hash_lb.h" +#include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" #include #include diff --git a/source/common/upstream/ring_hash_lb.h b/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h similarity index 100% rename from source/common/upstream/ring_hash_lb.h rename to source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h diff --git a/source/extensions/load_balancing_policies/subset/BUILD b/source/extensions/load_balancing_policies/subset/BUILD index 14d5eceaf0f2..ec66a84e18a7 100644 --- a/source/extensions/load_balancing_policies/subset/BUILD +++ b/source/extensions/load_balancing_policies/subset/BUILD @@ -22,9 +22,9 @@ envoy_cc_library( "//source/common/protobuf", "//source/common/protobuf:utility_lib", "//source/common/upstream:load_balancer_lib", - "//source/common/upstream:ring_hash_lb_lib", "//source/common/upstream:upstream_lib", "//source/extensions/load_balancing_policies/maglev:maglev_lb_lib", + "//source/extensions/load_balancing_policies/ring_hash:ring_hash_lb_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.cc b/source/extensions/load_balancing_policies/subset/subset_lb.cc index 16ac15fcf551..532f56693af2 100644 --- a/source/extensions/load_balancing_policies/subset/subset_lb.cc +++ b/source/extensions/load_balancing_policies/subset/subset_lb.cc @@ -12,8 +12,8 @@ #include "source/common/config/well_known_names.h" #include "source/common/protobuf/utility.h" #include "source/common/upstream/load_balancer_impl.h" -#include "source/common/upstream/ring_hash_lb.h" #include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" +#include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" #include "absl/container/node_hash_set.h" diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 511388c4f87a..8bebd8275dc3 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -82,6 +82,7 @@ envoy_cc_test( "//source/extensions/health_checkers/http:health_checker_lib", "//source/extensions/health_checkers/tcp:health_checker_lib", "//source/extensions/load_balancing_policies/maglev:config", + "//source/extensions/load_balancing_policies/ring_hash:config", "//source/extensions/network/dns_resolver/cares:config", "//source/extensions/transport_sockets/tls:config", "//test/config:v2_link_hacks", @@ -426,28 +427,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "ring_hash_lb_test", - srcs = ["ring_hash_lb_test.cc"], - deps = [ - ":utility_lib", - "//envoy/router:router_interface", - "//source/common/network:utility_lib", - "//source/common/upstream:ring_hash_lb_lib", - "//source/common/upstream:upstream_includes", - "//source/common/upstream:upstream_lib", - "//test/mocks:common_lib", - "//test/mocks/runtime:runtime_mocks", - "//test/mocks/upstream:cluster_info_mocks", - "//test/mocks/upstream:host_mocks", - "//test/mocks/upstream:host_set_mocks", - "//test/mocks/upstream:load_balancer_context_mock", - "//test/mocks/upstream:priority_set_mocks", - "//test/test_common:simulated_time_system_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "bounded_load_hlb_test", srcs = ["bounded_load_hlb_test.cc"], @@ -469,9 +448,9 @@ envoy_cc_benchmark_binary( deps = [ "//source/common/common:random_generator_lib", "//source/common/memory:stats_lib", - "//source/common/upstream:ring_hash_lb_lib", "//source/common/upstream:upstream_lib", "//source/extensions/load_balancing_policies/maglev:config", + "//source/extensions/load_balancing_policies/ring_hash:config", "//source/extensions/load_balancing_policies/subset:config", "//test/common/upstream:utility_lib", "//test/mocks/upstream:cluster_info_mocks", diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc index c882d88e7a10..89b001f2b7fc 100644 --- a/test/common/upstream/load_balancer_benchmark.cc +++ b/test/common/upstream/load_balancer_benchmark.cc @@ -6,9 +6,9 @@ #include "source/common/common/random_generator.h" #include "source/common/memory/stats.h" -#include "source/common/upstream/ring_hash_lb.h" #include "source/common/upstream/upstream_impl.h" #include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" +#include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" #include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include "test/benchmark/main.h" diff --git a/test/extensions/load_balancing_policies/ring_hash/BUILD b/test/extensions/load_balancing_policies/ring_hash/BUILD index 55fb1a90c188..e44db5929a93 100644 --- a/test/extensions/load_balancing_policies/ring_hash/BUILD +++ b/test/extensions/load_balancing_policies/ring_hash/BUILD @@ -11,6 +11,29 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_extension_cc_test( + name = "ring_hash_lb_test", + srcs = ["ring_hash_lb_test.cc"], + extension_names = ["envoy.load_balancing_policies.ring_hash"], + deps = [ + "//envoy/router:router_interface", + "//source/common/network:utility_lib", + "//source/common/upstream:upstream_includes", + "//source/common/upstream:upstream_lib", + "//source/extensions/load_balancing_policies/ring_hash:ring_hash_lb_lib", + "//test/common/upstream:utility_lib", + "//test/mocks:common_lib", + "//test/mocks/runtime:runtime_mocks", + "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:host_mocks", + "//test/mocks/upstream:host_set_mocks", + "//test/mocks/upstream:load_balancer_context_mock", + "//test/mocks/upstream:priority_set_mocks", + "//test/test_common:simulated_time_system_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + ], +) + envoy_extension_cc_test( name = "config_test", srcs = ["config_test.cc"], diff --git a/test/common/upstream/ring_hash_lb_test.cc b/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc similarity index 99% rename from test/common/upstream/ring_hash_lb_test.cc rename to test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc index 6a9ae5f81363..8fb244f6af8a 100644 --- a/test/common/upstream/ring_hash_lb_test.cc +++ b/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc @@ -7,8 +7,8 @@ #include "envoy/router/router.h" #include "source/common/network/utility.h" -#include "source/common/upstream/ring_hash_lb.h" #include "source/common/upstream/upstream_impl.h" +#include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" #include "test/common/upstream/utility.h" #include "test/mocks/common.h" diff --git a/test/integration/BUILD b/test/integration/BUILD index 7c05d4a84a8f..0178b436412b 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -313,6 +313,7 @@ envoy_cc_test_binary( "//source/exe:platform_impl_lib", "//source/extensions/clusters/static:static_cluster_lib", "//source/extensions/clusters/strict_dns:strict_dns_cluster_lib", + "//source/extensions/load_balancing_policies/ring_hash:config", ], ) @@ -446,6 +447,7 @@ envoy_cc_test( "//source/common/buffer:buffer_lib", "//source/common/http:header_map_lib", "//source/extensions/filters/http/buffer:config", + "//source/extensions/load_balancing_policies/ring_hash:config", "//test/integration/filters:encode1xx_local_reply_config_lib", "//test/integration/filters:metadata_stop_all_filter_config_lib", "//test/integration/filters:on_local_reply_filter_config_lib", diff --git a/test/server/BUILD b/test/server/BUILD index ed3858549df0..74ae0259db08 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -56,6 +56,7 @@ envoy_cc_test( "//source/common/upstream:cluster_manager_lib", "//source/extensions/access_loggers/file:config", "//source/extensions/clusters/static:static_cluster_lib", + "//source/extensions/load_balancing_policies/ring_hash:config", "//source/extensions/load_balancing_policies/subset:config", "//source/extensions/stat_sinks/statsd:config", "//source/extensions/transport_sockets/raw_buffer:config", diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index d36f14206e1a..c8e71e928107 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -321,3 +321,4 @@ visibility_excludes: - source/extensions/config_subscription/filesystem/BUILD - source/extensions/config_subscription/grpc/BUILD - source/extensions/load_balancing_policies/subset/BUILD +- source/extensions/load_balancing_policies/ring_hash/BUILD From f2a6dc5a2cec23353d6acdba7e406809b93fb345 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Tue, 2 May 2023 21:08:31 -0400 Subject: [PATCH 087/740] Remove unused ConnectionManagerImpl::http_context_ member variable (#27123) Remove unused memeber variable Signed-off-by: Yan Avlasov --- source/common/http/conn_manager_impl.cc | 5 ++--- source/common/http/conn_manager_impl.h | 10 ---------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index f33b7b5d3d2c..ac7b110a05bf 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -96,9 +96,8 @@ ConnectionManagerImpl::ConnectionManagerImpl(ConnectionManagerConfig& config, conn_length_(new Stats::HistogramCompletableTimespanImpl( stats_.named_.downstream_cx_length_ms_, time_source)), drain_close_(drain_close), user_agent_(http_context.userAgentContext()), - random_generator_(random_generator), http_context_(http_context), runtime_(runtime), - local_info_(local_info), cluster_manager_(cluster_manager), - listener_stats_(config_.listenerStats()), + random_generator_(random_generator), runtime_(runtime), local_info_(local_info), + cluster_manager_(cluster_manager), listener_stats_(config_.listenerStats()), overload_state_(overload_manager.getThreadLocalOverloadState()), accept_new_http_stream_(overload_manager.getLoadShedPoint( "envoy.load_shed_points.http_connection_manager_decode_headers")), diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 8250bd6cb1be..94c5228fbcdb 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -170,15 +170,6 @@ class ConnectionManagerImpl : Logger::Loggable, const Network::Connection* connection(); uint64_t streamId() { return stream_id_; } - // This is a helper function for encodeHeaders and responseDataTooLarge which allows for - // shared code for the two headers encoding paths. It does header munging, updates timing - // stats, and sends the headers to the encoder. - void encodeHeadersInternal(ResponseHeaderMap& headers, bool end_stream); - // This is a helper function for encodeData and responseDataTooLarge which allows for shared - // code for the two data encoding paths. It does stats updates and tracks potential end of - // stream. - void encodeDataInternal(Buffer::Instance& data, bool end_stream); - // Http::StreamCallbacks void onResetStream(StreamResetReason reason, absl::string_view transport_failure_reason) override; @@ -568,7 +559,6 @@ class ConnectionManagerImpl : Logger::Loggable, Event::TimerPtr connection_duration_timer_; Event::TimerPtr drain_timer_; Random::RandomGenerator& random_generator_; - Http::Context& http_context_; Runtime::Loader& runtime_; const LocalInfo::LocalInfo& local_info_; Upstream::ClusterManager& cluster_manager_; From 5f75ac64f120cb3a8c242dac0f11550c85ad2e92 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 3 May 2023 03:35:00 -0400 Subject: [PATCH 088/740] Redirect trace log output to /dev/null for coverage tests (#27118) Signed-off-by: Yan Avlasov --- .bazelrc | 1 + test/common/common/logger_test.cc | 1 + test/test_runner.cc | 14 +++++++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index ef72b14b6ee0..1544661efcf0 100644 --- a/.bazelrc +++ b/.bazelrc @@ -195,6 +195,7 @@ build:coverage --collect_code_coverage build:coverage --test_tag_filters=-nocoverage build:coverage --instrumentation_filter="//source(?!/common/quic/platform)[/:],//envoy[/:],//contrib(?!/.*/test)[/:]" build:test-coverage --test_arg="-l trace" +build:test-coverage --test_arg="--log-path /dev/null" build:fuzz-coverage --config=plain-fuzzer build:fuzz-coverage --run_under=@envoy//bazel/coverage:fuzz_coverage_wrapper.sh diff --git a/test/common/common/logger_test.cc b/test/common/common/logger_test.cc index 72127fa0e265..b026eddc6419 100644 --- a/test/common/common/logger_test.cc +++ b/test/common/common/logger_test.cc @@ -120,6 +120,7 @@ class LoggerCustomFlagsTest : public testing::TestWithParam { void expectLogMessage(const std::string& pattern, const std::string& message, const std::string& expected) { + StderrSinkDelegate stacked(Envoy::Logger::Registry::getSink()); auto formatter = std::make_unique(); formatter ->add_flag( diff --git a/test/test_runner.cc b/test/test_runner.cc index ed1ca1b845ef..5f193b5b7e3f 100644 --- a/test/test_runner.cc +++ b/test/test_runner.cc @@ -64,9 +64,19 @@ class RuntimeManagingListener : public ::testing::EmptyTestEventListener { bool disable_; }; +bool isDeathTestChild(int argc, char** argv) { + for (int i = 0; i < argc; ++i) { + if (absl::StartsWith(argv[i], "--gtest_internal_run_death_test")) { + return true; + } + } + return false; +} + } // namespace int TestRunner::RunTests(int argc, char** argv) { + const bool is_death_test_child = isDeathTestChild(argc, argv); ::testing::InitGoogleMock(&argc, argv); // We hold on to process_wide to provide RAII cleanup of process-wide // state. @@ -140,7 +150,9 @@ int TestRunner::RunTests(int argc, char** argv) { std::unique_ptr file_logger; // Redirect all logs to fake file when --log-path arg is specified in command line. - if (!TestEnvironment::getOptions().logPath().empty()) { + // However do not redirect to file from death test children as the parent typically + // looks for specific output in stderr + if (!TestEnvironment::getOptions().logPath().empty() && !is_death_test_child) { file_logger = std::make_unique( TestEnvironment::getOptions().logPath(), access_log_manager, Logger::Registry::getSink()); } From 97a2f2b94e944c58d4214cf5ecd4fce818a2351e Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 3 May 2023 08:48:31 +0100 Subject: [PATCH 089/740] ci: Workaround dep checker (cves) issue again (#27141) Signed-off-by: Ryan Northey --- ci/do_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 24c83c5d95e6..17a7296114e0 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -553,7 +553,7 @@ case $CI_TARGET in bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:check \ --action_env=TODAY_DATE \ -- -v warn \ - -c cves release_dates releases + -c cves release_dates releases || echo "WARNING: Dependency check failed" # Run dependabot tests echo "Check dependabot ..." From 6a69b1b1c9503775c15736db2a14675ef6b86c79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 09:02:50 +0100 Subject: [PATCH 090/740] build(deps): bump debian from `9404b05` to `f4da3f9` in /examples/shared/golang (#27139) build(deps): bump debian in /examples/shared/golang Bumps debian from `9404b05` to `f4da3f9`. --- updated-dependencies: - dependency-name: debian dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/golang/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile index 094b72f8a5fd..fdc947229b08 100644 --- a/examples/shared/golang/Dockerfile +++ b/examples/shared/golang/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim@sha256:9404b05bd09b57c76eccc0c5505b3c88b5feccac808d9b193a4fbac87bb44745 as os-base +FROM debian:bullseye-slim@sha256:f4da3f9b18fc242b739807a0fb3e77747f644f2fb3f67f4403fafce2286b431a as os-base RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache From 177676dd95bd4f2031e63cfdcd07da5bf5d7cdf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 08:07:00 +0000 Subject: [PATCH 091/740] build(deps): bump redis from `f50031a` to `393b84a` in /examples/redis (#27135) Bumps redis from `f50031a` to `393b84a`. --- updated-dependencies: - dependency-name: redis dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/redis/Dockerfile-redis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/redis/Dockerfile-redis b/examples/redis/Dockerfile-redis index 710f087a64c6..5b6cb4021be6 100644 --- a/examples/redis/Dockerfile-redis +++ b/examples/redis/Dockerfile-redis @@ -1 +1 @@ -FROM redis@sha256:f50031a49f41e493087fb95f96fdb3523bb25dcf6a3f0b07c588ad3cdbe1d0aa +FROM redis@sha256:393b84ac0c167421173657a77f36add89bf24e23b123045e5e09501fc370474a From fe676477d3a19ab87e1e2f6cb198cf3224919642 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 09:08:08 +0100 Subject: [PATCH 092/740] build(deps): bump postgres from `6cc9726` to `8a2975e` in /examples/shared/postgres (#27136) build(deps): bump postgres in /examples/shared/postgres Bumps postgres from `6cc9726` to `8a2975e`. --- updated-dependencies: - dependency-name: postgres dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile index c7af1edce02a..a4d972878649 100644 --- a/examples/shared/postgres/Dockerfile +++ b/examples/shared/postgres/Dockerfile @@ -1,3 +1,3 @@ -FROM postgres:latest@sha256:6cc97262444f1c45171081bc5a1d4c28b883ea46a6e0d1a45a8eac4a7f4767ab +FROM postgres:latest@sha256:8a2975ef8f47391d186cfe43b38d4ccef88b27e40397ae6ad3b961ffacb57029 COPY docker-healthcheck.sh /usr/local/bin/ HEALTHCHECK CMD ["docker-healthcheck.sh"] From 2e483104a86c28f8fca0da4e16e4c8be7c317179 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 08:17:25 +0000 Subject: [PATCH 093/740] build(deps): bump elasticsearch from 8.7.0 to 8.7.1 in /examples/skywalking (#27140) build(deps): bump elasticsearch in /examples/skywalking Bumps elasticsearch from 8.7.0 to 8.7.1. --- updated-dependencies: - dependency-name: elasticsearch dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/skywalking/Dockerfile-elasticsearch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/skywalking/Dockerfile-elasticsearch b/examples/skywalking/Dockerfile-elasticsearch index 8bbae1478d83..faa8bcdab143 100644 --- a/examples/skywalking/Dockerfile-elasticsearch +++ b/examples/skywalking/Dockerfile-elasticsearch @@ -1 +1 @@ -FROM elasticsearch:8.7.0@sha256:53e51bde90897d77fdbc53677331504bbbdcc788cb7aadcd8fe707a77733dbb5 +FROM elasticsearch:8.7.1@sha256:160814e5972521291420c29edf4c0348b8591ac9235156f0dbf34befcf362825 From adabedca8347ddcd860b1836d2f2ee4db44d1b56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 09:21:57 +0100 Subject: [PATCH 094/740] build(deps): bump golang from 1.20.3-bullseye to 1.20.4-bullseye in /examples/shared/golang (#27138) build(deps): bump golang in /examples/shared/golang Bumps golang from 1.20.3-bullseye to 1.20.4-bullseye. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/golang/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile index fdc947229b08..b48848365ca2 100644 --- a/examples/shared/golang/Dockerfile +++ b/examples/shared/golang/Dockerfile @@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache -FROM golang:1.20.3-bullseye@sha256:595c9af0430dd84bad33020e7e9e328af4bd1a1aabd46a03b5bf6f252cdbecf3 as golang-base +FROM golang:1.20.4-bullseye@sha256:d282e704848c81e313b17c68e27edd44202937db196398915077830557f82588 as golang-base FROM golang-base as golang-control-plane-builder From 047f3c8f2952a52542d5ee79196259e1c63df4cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 09:40:37 +0000 Subject: [PATCH 095/740] build(deps): bump debian from `9404b05` to `f4da3f9` in /examples/shared/websocket (#27137) build(deps): bump debian in /examples/shared/websocket Bumps debian from `9404b05` to `f4da3f9`. --- updated-dependencies: - dependency-name: debian dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/websocket/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/websocket/Dockerfile b/examples/shared/websocket/Dockerfile index a7a9aaf43ca8..e070c5148ea0 100644 --- a/examples/shared/websocket/Dockerfile +++ b/examples/shared/websocket/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim@sha256:9404b05bd09b57c76eccc0c5505b3c88b5feccac808d9b193a4fbac87bb44745 as websocket-base +FROM debian:bullseye-slim@sha256:f4da3f9b18fc242b739807a0fb3e77747f644f2fb3f67f4403fafce2286b431a as websocket-base ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ From 3ea7ff04dd421646f6154dd5d0f6bd0f241c5ce2 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 3 May 2023 09:06:08 -0400 Subject: [PATCH 096/740] stats: moving a single-use utilility inline into the class (#27113) Risk Level: low Testing: code still covered Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- source/common/common/regex.cc | 11 ----------- source/common/common/regex.h | 10 ---------- source/common/stats/tag_extractor_impl.cc | 12 ++++++++++-- test/common/common/regex_test.cc | 16 ---------------- 4 files changed, 10 insertions(+), 39 deletions(-) diff --git a/source/common/common/regex.cc b/source/common/common/regex.cc index 679efd3029a3..0281d658988d 100644 --- a/source/common/common/regex.cc +++ b/source/common/common/regex.cc @@ -74,16 +74,5 @@ ProtobufTypes::MessagePtr GoogleReEngineFactory::createEmptyConfigProto() { REGISTER_FACTORY(GoogleReEngineFactory, EngineFactory); -std::regex Utility::parseStdRegex(const std::string& regex, std::regex::flag_type flags) { - // TODO(zuercher): In the future, PGV (https://github.com/bufbuild/protoc-gen-validate) - // annotations may allow us to remove this in favor of direct validation of regular - // expressions. - try { - return std::regex(regex, flags); - } catch (const std::regex_error& e) { - throw EnvoyException(fmt::format("Invalid regex '{}': {}", regex, e.what())); - } -} - } // namespace Regex } // namespace Envoy diff --git a/source/common/common/regex.h b/source/common/common/regex.h index 31fa48d332b3..92b2720bc231 100644 --- a/source/common/common/regex.h +++ b/source/common/common/regex.h @@ -70,16 +70,6 @@ enum class Type { Re2, StdRegex }; */ class Utility { public: - /** - * Constructs a std::regex, converting any std::regex_error exception into an EnvoyException. - * @param regex std::string containing the regular expression to parse. - * @param flags std::regex::flag_type containing parser flags. Defaults to std::regex::optimize. - * @return std::regex constructed from regex and flags. - * @throw EnvoyException if the regex string is invalid. - */ - static std::regex parseStdRegex(const std::string& regex, - std::regex::flag_type flags = std::regex::optimize); - /** * Construct a compiled regex matcher from a match config. */ diff --git a/source/common/stats/tag_extractor_impl.cc b/source/common/stats/tag_extractor_impl.cc index 5e321fb81913..2ac20ee1648f 100644 --- a/source/common/stats/tag_extractor_impl.cc +++ b/source/common/stats/tag_extractor_impl.cc @@ -17,6 +17,15 @@ namespace Envoy { namespace Stats { +namespace { +std::regex parseStdRegex(const std::string& regex) { + TRY_ASSERT_MAIN_THREAD { return std::regex(regex, std::regex::optimize); } + END_TRY + catch (const std::regex_error& e) { + throw EnvoyException(fmt::format("Invalid regex '{}': {}", regex, e.what())); + } +} +} // namespace const std::vector& TagExtractionContext::tokens() { if (tokens_.empty()) { @@ -87,8 +96,7 @@ bool TagExtractorImplBase::substrMismatch(absl::string_view stat_name) const { TagExtractorStdRegexImpl::TagExtractorStdRegexImpl(absl::string_view name, absl::string_view regex, absl::string_view substr) - : TagExtractorImplBase(name, regex, substr), - regex_(Regex::Utility::parseStdRegex(std::string(regex))) {} + : TagExtractorImplBase(name, regex, substr), regex_(parseStdRegex(std::string(regex))) {} std::string& TagExtractorImplBase::addTagReturningValueRef(std::vector& tags) const { tags.emplace_back(); diff --git a/test/common/common/regex_test.cc b/test/common/common/regex_test.cc index d480f620ca1d..b44e67008a93 100644 --- a/test/common/common/regex_test.cc +++ b/test/common/common/regex_test.cc @@ -13,22 +13,6 @@ namespace Envoy { namespace Regex { namespace { -TEST(Utility, ParseStdRegex) { - EXPECT_THROW_WITH_REGEX(Utility::parseStdRegex("(+invalid)"), EnvoyException, - "Invalid regex '\\(\\+invalid\\)': .+"); - - { - std::regex regex = Utility::parseStdRegex("x*"); - EXPECT_NE(0, regex.flags() & std::regex::optimize); - } - - { - std::regex regex = Utility::parseStdRegex("x*", std::regex::icase); - EXPECT_NE(0, regex.flags() & std::regex::icase); - EXPECT_EQ(0, regex.flags() & std::regex::optimize); - } -} - TEST(Utility, ParseRegex) { ScopedInjectableLoader engine(std::make_unique()); { From fe02d956348141ca22fc1290608c2dd8bec0a53e Mon Sep 17 00:00:00 2001 From: Michael Finch Date: Wed, 3 May 2023 08:11:14 -0700 Subject: [PATCH 097/740] aws: support fetching credentials from the AWS credentials file (#25532) Signed-off-by: Michael Finch --- changelogs/current.yaml | 5 + .../http_filters/_include/aws_credentials.rst | 16 ++ .../http/http_filters/aws_lambda_filter.rst | 2 + .../aws_request_signing_filter.rst | 2 + source/common/runtime/runtime_features.cc | 1 + .../common/aws/credentials_provider_impl.cc | 101 ++++++++++- .../common/aws/credentials_provider_impl.h | 49 ++++- test/extensions/common/aws/BUILD | 2 + .../aws/credentials_provider_impl_test.cc | 171 ++++++++++++++++++ 9 files changed, 339 insertions(+), 10 deletions(-) create mode 100644 docs/root/configuration/http/http_filters/_include/aws_credentials.rst diff --git a/changelogs/current.yaml b/changelogs/current.yaml index aeea0b877f8a..1f3ff79085a7 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -30,6 +30,11 @@ minor_behavior_changes: Allow malformed URL encoded triplets in the default header validator. This behavior can be reverted by setting runtime flag ``envoy.reloadable_features.uhv_allow_malformed_url_encoding`` to false, in which case requests with malformed URL encoded triplets in path are rejected. This setting is only applicable when the Unversal Header Validator is enabled and has no effect otherwise. +- area: aws + change: | + Added support for fetching credentials from the AWS credentials file, which only happens if credentials cannot be fetched + from environment variables. This behavioral change can be reverted by setting runtime guard + ``envoy.reloadable_features.enable_aws_credentials_file`` to ``false``. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* diff --git a/docs/root/configuration/http/http_filters/_include/aws_credentials.rst b/docs/root/configuration/http/http_filters/_include/aws_credentials.rst new file mode 100644 index 000000000000..be3eab4df5b6 --- /dev/null +++ b/docs/root/configuration/http/http_filters/_include/aws_credentials.rst @@ -0,0 +1,16 @@ +Credentials +----------- + +The filter uses a few different credentials providers to obtain an AWS access key ID, AWS secret access key, and AWS session token. +It moves through the credentials providers in the order described below, stopping when one of them returns an access key ID and a +secret access key (the session token is optional). + +1. Environment variables. The environment variables AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN are used. + +2. The AWS credentials file. The environment variables AWS_SHARED_CREDENTIALS_FILE and AWS_PROFILE are respected if they are set, else + the file '~/.aws/credentials' and profile 'default' are used. The fields 'aws_access_key_id', 'aws_secret_access_key', and + 'aws_session_token' defined for the profile in the credentials file are used. These credentials are cached for 1 hour. + +3. Either EC2 instance metadata or ECS task metadata. For EC2 instance metadata, the fields 'AccessKeyId', 'SecretAccessKey', and + 'Token' are used, and credentials are cached for 1 hour. For ECS task metadata, the fields AccessKeyId', 'SecretAccessKey', and + 'Token' are used, and credentials are cached for 1 hour or until they expire (according to the field 'Expiration'). diff --git a/docs/root/configuration/http/http_filters/aws_lambda_filter.rst b/docs/root/configuration/http/http_filters/aws_lambda_filter.rst index ddd80719553e..6dbad84fc713 100644 --- a/docs/root/configuration/http/http_filters/aws_lambda_filter.rst +++ b/docs/root/configuration/http/http_filters/aws_lambda_filter.rst @@ -186,6 +186,8 @@ An example with the Lambda metadata applied to a weighted-cluster: sni: "*.amazonaws.com" +.. include:: _include/aws_credentials.rst + Statistics ---------- diff --git a/docs/root/configuration/http/http_filters/aws_request_signing_filter.rst b/docs/root/configuration/http/http_filters/aws_request_signing_filter.rst index 0e492f8b7622..2f33a1e4f6ef 100644 --- a/docs/root/configuration/http/http_filters/aws_request_signing_filter.rst +++ b/docs/root/configuration/http/http_filters/aws_request_signing_filter.rst @@ -48,6 +48,8 @@ Example filter configuration: - exact: x-amzn-trace-id +.. include:: _include/aws_credentials.rst + Statistics ---------- diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index b89591a516c9..cd245cfc0b76 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -34,6 +34,7 @@ RUNTIME_GUARD(envoy_reloadable_features_append_query_parameters_path_rewriter); RUNTIME_GUARD(envoy_reloadable_features_conn_pool_delete_when_idle); RUNTIME_GUARD(envoy_reloadable_features_delta_xds_subscription_state_tracking_fix); RUNTIME_GUARD(envoy_reloadable_features_do_not_count_mapped_pages_as_free); +RUNTIME_GUARD(envoy_reloadable_features_enable_aws_credentials_file); RUNTIME_GUARD(envoy_reloadable_features_enable_compression_bomb_protection); RUNTIME_GUARD(envoy_reloadable_features_enable_intermediate_ca); RUNTIME_GUARD(envoy_reloadable_features_enable_update_listener_socket_options); diff --git a/source/extensions/common/aws/credentials_provider_impl.cc b/source/extensions/common/aws/credentials_provider_impl.cc index 5826ac604b07..ee83614e0885 100644 --- a/source/extensions/common/aws/credentials_provider_impl.cc +++ b/source/extensions/common/aws/credentials_provider_impl.cc @@ -1,13 +1,19 @@ #include "source/extensions/common/aws/credentials_provider_impl.h" +#include + #include "envoy/common/exception.h" #include "source/common/common/lock_guard.h" #include "source/common/http/message_impl.h" #include "source/common/http/utility.h" #include "source/common/json/json_loader.h" +#include "source/common/runtime/runtime_features.h" #include "source/extensions/common/aws/utility.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_split.h" + namespace Envoy { namespace Extensions { namespace Common { @@ -26,6 +32,11 @@ constexpr char EXPIRATION[] = "Expiration"; constexpr char EXPIRATION_FORMAT[] = "%E4Y%m%dT%H%M%S%z"; constexpr char TRUE[] = "true"; +constexpr char AWS_SHARED_CREDENTIALS_FILE[] = "AWS_SHARED_CREDENTIALS_FILE"; +constexpr char AWS_PROFILE[] = "AWS_PROFILE"; +constexpr char DEFAULT_AWS_SHARED_CREDENTIALS_FILE[] = "~/.aws/credentials"; +constexpr char DEFAULT_AWS_PROFILE[] = "default"; + constexpr char AWS_CONTAINER_CREDENTIALS_RELATIVE_URI[] = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; constexpr char AWS_CONTAINER_CREDENTIALS_FULL_URI[] = "AWS_CONTAINER_CREDENTIALS_FULL_URI"; constexpr char AWS_CONTAINER_AUTHORIZATION_TOKEN[] = "AWS_CONTAINER_AUTHORIZATION_TOKEN"; @@ -62,13 +73,94 @@ Credentials EnvironmentCredentialsProvider::getCredentials() { return Credentials(access_key_id, secret_access_key, session_token); } -void MetadataCredentialsProviderBase::refreshIfNeeded() { +void CachedCredentialsProviderBase::refreshIfNeeded() { const Thread::LockGuard lock(lock_); if (needsRefresh()) { refresh(); } } +bool CredentialsFileCredentialsProvider::needsRefresh() { + return api_.timeSource().systemTime() - last_updated_ > REFRESH_INTERVAL; +} + +std::string getEnvironmentVariableOrDefault(const std::string& variable_name, + const std::string& default_value) { + const char* value = getenv(variable_name.c_str()); + return (value != nullptr) && (value[0] != '\0') ? value : default_value; +} + +void CredentialsFileCredentialsProvider::refresh() { + ENVOY_LOG(debug, "Getting AWS credentials from the credentials file"); + + const auto credentials_file = getEnvironmentVariableOrDefault( + AWS_SHARED_CREDENTIALS_FILE, DEFAULT_AWS_SHARED_CREDENTIALS_FILE); + const auto profile = getEnvironmentVariableOrDefault(AWS_PROFILE, DEFAULT_AWS_PROFILE); + + extractCredentials(credentials_file, profile); +} + +void CredentialsFileCredentialsProvider::extractCredentials(const std::string& credentials_file, + const std::string& profile) { + // Update last_updated_ now so that even if this function returns before successfully + // extracting credentials, this function won't be called again until after the REFRESH_INTERVAL. + // This prevents envoy from attempting and failing to read the credentials file on every request + // if there are errors extracting credentials from it (e.g. if the credentials file doesn't + // exist). + last_updated_ = api_.timeSource().systemTime(); + + std::ifstream file(credentials_file); + if (!file) { + ENVOY_LOG(debug, "Error opening credentials file {}", credentials_file); + return; + } + + std::string access_key_id, secret_access_key, session_token; + const auto profile_start = absl::StrFormat("[%s]", profile); + + bool found_profile = false; + std::string line; + while (std::getline(file, line)) { + line = std::string(StringUtil::trim(line)); + if (line.empty()) { + continue; + } + + if (line == profile_start) { + found_profile = true; + continue; + } + + if (found_profile) { + // Stop reading once we find the start of the next profile. + if (absl::StartsWith(line, "[")) { + break; + } + + std::vector parts = absl::StrSplit(line, absl::MaxSplits('=', 1)); + if (parts.size() == 2) { + const auto key = StringUtil::toUpper(StringUtil::trim(parts[0])); + const auto val = StringUtil::trim(parts[1]); + if (key == AWS_ACCESS_KEY_ID) { + access_key_id = val; + } else if (key == AWS_SECRET_ACCESS_KEY) { + secret_access_key = val; + } else if (key == AWS_SESSION_TOKEN) { + session_token = val; + } + } + } + } + + ENVOY_LOG(debug, "Found following AWS credentials for profile '{}' in {}: {}={}, {}={}, {}={}", + profile, credentials_file, AWS_ACCESS_KEY_ID, access_key_id, AWS_SECRET_ACCESS_KEY, + secret_access_key.empty() ? "" : "*****", AWS_SESSION_TOKEN, + session_token.empty() ? "" : "*****"); + + cached_credentials_ = Credentials(access_key_id, secret_access_key, session_token); + last_updated_ = api_.timeSource().systemTime(); +} + bool InstanceProfileCredentialsProvider::needsRefresh() { return api_.timeSource().systemTime() - last_updated_ > REFRESH_INTERVAL; } @@ -253,6 +345,13 @@ DefaultCredentialsProviderChain::DefaultCredentialsProviderChain( ENVOY_LOG(debug, "Using environment credentials provider"); add(factories.createEnvironmentCredentialsProvider()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_aws_credentials_file")) { + ENVOY_LOG(debug, "Using credentials file credentials provider"); + add(factories.createCredentialsFileCredentialsProvider(api)); + } else { + ENVOY_LOG(debug, "Not using credential file credentials provider because it is not enabled"); + } + const auto relative_uri = absl::NullSafeStringView(std::getenv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI)); const auto full_uri = absl::NullSafeStringView(std::getenv(AWS_CONTAINER_CREDENTIALS_FULL_URI)); diff --git a/source/extensions/common/aws/credentials_provider_impl.h b/source/extensions/common/aws/credentials_provider_impl.h index 4f8214301173..0b207620ad88 100644 --- a/source/extensions/common/aws/credentials_provider_impl.h +++ b/source/extensions/common/aws/credentials_provider_impl.h @@ -29,22 +29,15 @@ class EnvironmentCredentialsProvider : public CredentialsProvider, Credentials getCredentials() override; }; -class MetadataCredentialsProviderBase : public CredentialsProvider, - public Logger::Loggable { +class CachedCredentialsProviderBase : public CredentialsProvider, + public Logger::Loggable { public: - using MetadataFetcher = std::function(Http::RequestMessage&)>; - - MetadataCredentialsProviderBase(Api::Api& api, const MetadataFetcher& metadata_fetcher) - : api_(api), metadata_fetcher_(metadata_fetcher) {} - Credentials getCredentials() override { refreshIfNeeded(); return cached_credentials_; } protected: - Api::Api& api_; - MetadataFetcher metadata_fetcher_; SystemTime last_updated_; Credentials cached_credentials_; Thread::MutexBasicLockable lock_; @@ -55,6 +48,36 @@ class MetadataCredentialsProviderBase : public CredentialsProvider, virtual void refresh() PURE; }; +/** + * Retrieve AWS credentials from the credentials file. + * + * Adheres to conventions specified in: + * https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html + */ +class CredentialsFileCredentialsProvider : public CachedCredentialsProviderBase { +public: + CredentialsFileCredentialsProvider(Api::Api& api) : api_(api) {} + +private: + Api::Api& api_; + + bool needsRefresh() override; + void refresh() override; + void extractCredentials(const std::string& credentials_file, const std::string& profile); +}; + +class MetadataCredentialsProviderBase : public CachedCredentialsProviderBase { +public: + using MetadataFetcher = std::function(Http::RequestMessage&)>; + + MetadataCredentialsProviderBase(Api::Api& api, const MetadataFetcher& metadata_fetcher) + : api_(api), metadata_fetcher_(metadata_fetcher) {} + +protected: + Api::Api& api_; + MetadataFetcher metadata_fetcher_; +}; + /** * Retrieve AWS credentials from the instance metadata. * @@ -120,6 +143,9 @@ class CredentialsProviderChainFactories { virtual CredentialsProviderSharedPtr createEnvironmentCredentialsProvider() const PURE; + virtual CredentialsProviderSharedPtr + createCredentialsFileCredentialsProvider(Api::Api& api) const PURE; + virtual CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, absl::string_view credential_uri, absl::string_view authorization_token = {}) const PURE; @@ -151,6 +177,11 @@ class DefaultCredentialsProviderChain : public CredentialsProviderChain, return std::make_shared(); } + CredentialsProviderSharedPtr + createCredentialsFileCredentialsProvider(Api::Api& api) const override { + return std::make_shared(api); + } + CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, absl::string_view credential_uri, absl::string_view authorization_token = {}) const override { diff --git a/test/extensions/common/aws/BUILD b/test/extensions/common/aws/BUILD index 7f91f09b1090..55ebdf79f19f 100644 --- a/test/extensions/common/aws/BUILD +++ b/test/extensions/common/aws/BUILD @@ -58,8 +58,10 @@ envoy_cc_test( "//test/extensions/common/aws:aws_mocks", "//test/mocks/api:api_mocks", "//test/mocks/event:event_mocks", + "//test/mocks/runtime:runtime_mocks", "//test/test_common:environment_lib", "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", ], ) diff --git a/test/extensions/common/aws/credentials_provider_impl_test.cc b/test/extensions/common/aws/credentials_provider_impl_test.cc index 35c7faa8517c..3452b62d62ec 100644 --- a/test/extensions/common/aws/credentials_provider_impl_test.cc +++ b/test/extensions/common/aws/credentials_provider_impl_test.cc @@ -3,8 +3,10 @@ #include "test/extensions/common/aws/mocks.h" #include "test/mocks/api/mocks.h" #include "test/mocks/event/mocks.h" +#include "test/mocks/runtime/mocks.h" #include "test/test_common/environment.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" using testing::_; using testing::InSequence; @@ -17,6 +19,35 @@ namespace Extensions { namespace Common { namespace Aws { +const char CREDENTIALS_FILE[] = "test-credentials.json"; +const char CREDENTIALS_FILE_CONTENTS[] = + R"( +[default] +aws_access_key_id=default_access_key +aws_secret_access_key=default_secret +aws_session_token=default_token + +# This profile has leading spaces that should get trimmed. + [profile1] +# The "=" in the value should not interfere with how this line is parsed. +aws_access_key_id=profile1_acc=ess_key +aws_secret_access_key=profile1_secret +foo=bar +aws_session_token=profile1_token + +[profile2] +aws_access_key_id=profile2_access_key + +[profile3] +aws_access_key_id=profile3_access_key +aws_secret_access_key= + +[profile4] +aws_access_key_id = profile4_access_key +aws_secret_access_key = profile4_secret +aws_session_token = profile4_token +)"; + class EvironmentCredentialsProviderTest : public testing::Test { public: ~EvironmentCredentialsProviderTest() override { @@ -62,6 +93,129 @@ TEST_F(EvironmentCredentialsProviderTest, NoSessionToken) { EXPECT_FALSE(credentials.sessionToken().has_value()); } +class CredentialsFileCredentialsProviderTest : public testing::Test { +public: + CredentialsFileCredentialsProviderTest() + : api_(Api::createApiForTest(time_system_)), provider_(*api_) {} + + ~CredentialsFileCredentialsProviderTest() override { + TestEnvironment::unsetEnvVar("AWS_SHARED_CREDENTIALS_FILE"); + TestEnvironment::unsetEnvVar("AWS_PROFILE"); + } + + void setUpTest(std::string file_contents, std::string profile) { + auto file_path = TestEnvironment::writeStringToFileForTest(CREDENTIALS_FILE, file_contents); + TestEnvironment::setEnvVar("AWS_SHARED_CREDENTIALS_FILE", file_path, 1); + TestEnvironment::setEnvVar("AWS_PROFILE", profile, 1); + } + + Event::SimulatedTimeSystem time_system_; + Api::ApiPtr api_; + CredentialsFileCredentialsProvider provider_; +}; + +TEST_F(CredentialsFileCredentialsProviderTest, FileDoesNotExist) { + TestEnvironment::setEnvVar("AWS_SHARED_CREDENTIALS_FILE", "/file/does/not/exist", 1); + + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(CredentialsFileCredentialsProviderTest, ProfileDoesNotExist) { + setUpTest(CREDENTIALS_FILE_CONTENTS, "invalid_profile"); + + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(CredentialsFileCredentialsProviderTest, IncompleteProfile) { + setUpTest(CREDENTIALS_FILE_CONTENTS, "profile2"); + + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("profile2_access_key", credentials.accessKeyId().value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(CredentialsFileCredentialsProviderTest, DefaultProfile) { + setUpTest(CREDENTIALS_FILE_CONTENTS, ""); + + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("default_access_key", credentials.accessKeyId().value()); + EXPECT_EQ("default_secret", credentials.secretAccessKey().value()); + EXPECT_EQ("default_token", credentials.sessionToken().value()); +} + +TEST_F(CredentialsFileCredentialsProviderTest, CompleteProfile) { + setUpTest(CREDENTIALS_FILE_CONTENTS, "profile1"); + + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("profile1_acc=ess_key", credentials.accessKeyId().value()); + EXPECT_EQ("profile1_secret", credentials.secretAccessKey().value()); + EXPECT_EQ("profile1_token", credentials.sessionToken().value()); +} + +TEST_F(CredentialsFileCredentialsProviderTest, EmptySecret) { + setUpTest(CREDENTIALS_FILE_CONTENTS, "profile3"); + + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("profile3_access_key", credentials.accessKeyId().value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(CredentialsFileCredentialsProviderTest, SpacesBetweenParts) { + setUpTest(CREDENTIALS_FILE_CONTENTS, "profile4"); + + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("profile4_access_key", credentials.accessKeyId().value()); + EXPECT_EQ("profile4_secret", credentials.secretAccessKey().value()); + EXPECT_EQ("profile4_token", credentials.sessionToken().value()); +} + +TEST_F(CredentialsFileCredentialsProviderTest, RefreshInterval) { + InSequence sequence; + TestEnvironment::setEnvVar("AWS_SHARED_CREDENTIALS_FILE", "/file/does/not/exist", 1); + + auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); + + // Credentials won't be extracted even after we switch to a legitimate profile + // with valid credentials. + setUpTest(CREDENTIALS_FILE_CONTENTS, "profile1"); + credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); + + // Credentials will be extracted again after the REFRESH_INTERVAL. + time_system_.advanceTimeWait(std::chrono::hours(2)); + credentials = provider_.getCredentials(); + EXPECT_EQ("profile1_acc=ess_key", credentials.accessKeyId().value()); + EXPECT_EQ("profile1_secret", credentials.secretAccessKey().value()); + EXPECT_EQ("profile1_token", credentials.sessionToken().value()); + + // Previously cached credentials will be used. + setUpTest(CREDENTIALS_FILE_CONTENTS, "default"); + credentials = provider_.getCredentials(); + EXPECT_EQ("profile1_acc=ess_key", credentials.accessKeyId().value()); + EXPECT_EQ("profile1_secret", credentials.secretAccessKey().value()); + EXPECT_EQ("profile1_token", credentials.sessionToken().value()); + + // Credentials will be extracted again after the REFRESH_INTERVAL. + time_system_.advanceTimeWait(std::chrono::hours(2)); + credentials = provider_.getCredentials(); + EXPECT_EQ("default_access_key", credentials.accessKeyId().value()); + EXPECT_EQ("default_secret", credentials.secretAccessKey().value()); + EXPECT_EQ("default_token", credentials.sessionToken().value()); +} + class MessageMatcher : public testing::MatcherInterface { public: explicit MessageMatcher(const Http::TestRequestHeaderMapImpl& expected_headers) @@ -517,6 +671,8 @@ class DefaultCredentialsProviderChainTest : public testing::Test { class MockCredentialsProviderChainFactories : public CredentialsProviderChainFactories { public: MOCK_METHOD(CredentialsProviderSharedPtr, createEnvironmentCredentialsProvider, (), (const)); + MOCK_METHOD(CredentialsProviderSharedPtr, createCredentialsFileCredentialsProvider, (Api::Api&), + (const)); MOCK_METHOD(CredentialsProviderSharedPtr, createTaskRoleCredentialsProvider, (Api::Api&, const MetadataCredentialsProviderBase::MetadataFetcher&, absl::string_view, absl::string_view), @@ -532,24 +688,37 @@ class DefaultCredentialsProviderChainTest : public testing::Test { }; TEST_F(DefaultCredentialsProviderChainTest, NoEnvironmentVars) { + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)); + DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); +} + +TEST_F(DefaultCredentialsProviderChainTest, CredentialsFileDisabled) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({{"envoy.reloadable_features.enable_aws_credentials_file", "false"}}); + + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))).Times(0); EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)); DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); } TEST_F(DefaultCredentialsProviderChainTest, MetadataDisabled) { TestEnvironment::setEnvVar("AWS_EC2_METADATA_DISABLED", "true", 1); + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)).Times(0); DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); } TEST_F(DefaultCredentialsProviderChainTest, MetadataNotDisabled) { TestEnvironment::setEnvVar("AWS_EC2_METADATA_DISABLED", "false", 1); + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)); DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); } TEST_F(DefaultCredentialsProviderChainTest, RelativeUri) { TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/path/to/creds", 1); + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); EXPECT_CALL(factories_, createTaskRoleCredentialsProvider(Ref(*api_), _, "169.254.170.2:80/path/to/creds", "")); DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); @@ -557,6 +726,7 @@ TEST_F(DefaultCredentialsProviderChainTest, RelativeUri) { TEST_F(DefaultCredentialsProviderChainTest, FullUriNoAuthorizationToken) { TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://host/path/to/creds", 1); + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); EXPECT_CALL(factories_, createTaskRoleCredentialsProvider(Ref(*api_), _, "http://host/path/to/creds", "")); DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); @@ -565,6 +735,7 @@ TEST_F(DefaultCredentialsProviderChainTest, FullUriNoAuthorizationToken) { TEST_F(DefaultCredentialsProviderChainTest, FullUriWithAuthorizationToken) { TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://host/path/to/creds", 1); TestEnvironment::setEnvVar("AWS_CONTAINER_AUTHORIZATION_TOKEN", "auth_token", 1); + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); EXPECT_CALL(factories_, createTaskRoleCredentialsProvider( Ref(*api_), _, "http://host/path/to/creds", "auth_token")); DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); From e0a67c150f46f48d5169dbb6cff0aa129322f1da Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 3 May 2023 11:26:14 -0400 Subject: [PATCH 098/740] mobile: removing APIs to disable happy eyeballs (#27111) Risk Level: low Testing: n/a Docs Changes: updated Release Notes: mobile release notes inline Signed-off-by: Alyssa Wilk --- mobile/docs/root/api/starting_envoy.rst | 14 ------------ mobile/docs/root/intro/version_history.rst | 1 + mobile/library/cc/engine_builder.cc | 11 +--------- mobile/library/cc/engine_builder.h | 2 -- mobile/library/common/jni/jni_interface.cc | 18 +++++++-------- .../engine/EnvoyConfiguration.java | 22 ++++++++----------- .../envoymobile/engine/JniLibrary.java | 2 +- .../impl/NativeCronvoyEngineBuilderImpl.java | 19 ++++++++-------- .../envoyproxy/envoymobile/EngineBuilder.kt | 15 ------------- .../library/objective-c/EnvoyConfiguration.h | 2 -- .../library/objective-c/EnvoyConfiguration.mm | 3 --- mobile/library/swift/EngineBuilder.swift | 15 ------------- mobile/test/cc/unit/envoy_config_test.cc | 15 ------------- .../engine/EnvoyConfigurationTest.kt | 8 +------ .../envoymobile/EngineBuilderTest.kt | 10 --------- mobile/test/swift/EngineBuilderTests.swift | 14 ------------ 16 files changed, 30 insertions(+), 141 deletions(-) diff --git a/mobile/docs/root/api/starting_envoy.rst b/mobile/docs/root/api/starting_envoy.rst index 5ea1379891e7..f40d8244122e 100644 --- a/mobile/docs/root/api/starting_envoy.rst +++ b/mobile/docs/root/api/starting_envoy.rst @@ -326,20 +326,6 @@ Defaults to ``NWPathMonitor``, but can be configured to use ``SCNetworkReachabil // Swift builder.setNetworkMonitoringMode(.pathMonitor) -~~~~~~~~~~~~~~~~~~~~~~~ -``enableHappyEyeballs`` -~~~~~~~~~~~~~~~~~~~~~~~ - -Specify whether to use Happy Eyeballs when multiple IP stacks may be supported. Defaults to true. - -**Example**:: - - // Kotlin - builder.enableHappyEyeballs(true) - - // Swift - builder.enableHappyEyeballs(true) - ~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``enableGzipDecompression`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/mobile/docs/root/intro/version_history.rst b/mobile/docs/root/intro/version_history.rst index 5d26891dc0de..7f4e13f1a52e 100644 --- a/mobile/docs/root/intro/version_history.rst +++ b/mobile/docs/root/intro/version_history.rst @@ -48,6 +48,7 @@ Features: - api: Add support for String Accessors to the C++ EngineBuilder. (:issue:`#2498 <2498>`) - api: Add support for Native Filters and Platform Filters to the C++ EngineBuilder. (:issue:`#2498 <2498>`) - api: added upstream protocol to final stream intel. (:issue:`#2613 <2613>`) +- api: removed ``enableHappyEyeballs`` turning up happy eyeballs by default. - build: Add a build feature ``exclude_certificates`` to disable inclusion of the Envoy Mobile certificate list, for use when using platform certificate validation. - build: Add a build feature ``envoy_http_datagrams`` to allow disabling HTTP Datagram support. (:issue:`#23564 <23564>`) - build: Add a build feature ``envoy_mobile_stats_reporting`` to allow disabling stats reporting. (:issue:`26086 <26086>`) diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc index 5bc4ba46a585..beaa3b2c493c 100644 --- a/mobile/library/cc/engine_builder.cc +++ b/mobile/library/cc/engine_builder.cc @@ -181,11 +181,6 @@ EngineBuilder& EngineBuilder::enableAdminInterface(bool admin_interface_on) { } #endif -EngineBuilder& EngineBuilder::enableHappyEyeballs(bool happy_eyeballs_on) { - enable_happy_eyeballs_ = happy_eyeballs_on; - return *this; -} - #ifdef ENVOY_ENABLE_QUIC EngineBuilder& EngineBuilder::enableHttp3(bool http3_on) { enable_http3_ = http3_on; @@ -575,11 +570,7 @@ std::unique_ptr EngineBuilder::generate envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig dfp_config; auto* dns_cache_config = dfp_config.mutable_dns_cache_config(); dns_cache_config->set_name("base_dns_cache"); - if (enable_happy_eyeballs_) { - dns_cache_config->set_dns_lookup_family(envoy::config::cluster::v3::Cluster::ALL); - } else { - dns_cache_config->set_dns_lookup_family(envoy::config::cluster::v3::Cluster::V4_PREFERRED); - } + dns_cache_config->set_dns_lookup_family(envoy::config::cluster::v3::Cluster::ALL); dns_cache_config->mutable_host_ttl()->set_seconds(86400); dns_cache_config->mutable_dns_min_refresh_rate()->set_seconds(dns_min_refresh_seconds_); dns_cache_config->mutable_dns_refresh_rate()->set_seconds(dns_refresh_seconds_); diff --git a/mobile/library/cc/engine_builder.h b/mobile/library/cc/engine_builder.h index 51b5f92ac5ef..449169ed5ef9 100644 --- a/mobile/library/cc/engine_builder.h +++ b/mobile/library/cc/engine_builder.h @@ -61,7 +61,6 @@ class EngineBuilder { EngineBuilder& enableGzipDecompression(bool gzip_decompression_on); EngineBuilder& enableBrotliDecompression(bool brotli_decompression_on); EngineBuilder& enableSocketTagging(bool socket_tagging_on); - EngineBuilder& enableHappyEyeballs(bool happy_eyeballs_on); #ifdef ENVOY_ENABLE_QUIC EngineBuilder& enableHttp3(bool http3_on); #endif @@ -180,7 +179,6 @@ class EngineBuilder { absl::flat_hash_map key_value_stores_{}; bool admin_interface_enabled_ = false; - bool enable_happy_eyeballs_ = true; bool enable_interface_binding_ = false; bool enable_drain_post_dns_refresh_ = false; bool enforce_trust_chain_verification_ = true; diff --git a/mobile/library/common/jni/jni_interface.cc b/mobile/library/common/jni/jni_interface.cc index 5ac75a5f790c..10795b3fab28 100644 --- a/mobile/library/common/jni/jni_interface.cc +++ b/mobile/library/common/jni/jni_interface.cc @@ -1202,8 +1202,8 @@ void configureBuilder( jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache, jlong dns_cache_save_interval_seconds, jboolean enable_drain_post_dns_refresh, jboolean enable_http3, jboolean enable_gzip_decompression, jboolean enable_brotli_decompression, - jboolean enable_socket_tagging, jboolean enable_happy_eyeballs, - jboolean enable_interface_binding, jlong h2_connection_keepalive_idle_interval_milliseconds, + jboolean enable_socket_tagging, jboolean enable_interface_binding, + jlong h2_connection_keepalive_idle_interval_milliseconds, jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host, jlong stats_flush_seconds, jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id, @@ -1238,7 +1238,6 @@ void configureBuilder( builder.enableGzipDecompression(enable_gzip_decompression == JNI_TRUE); builder.enableBrotliDecompression(enable_brotli_decompression == JNI_TRUE); builder.enableSocketTagging(enable_socket_tagging == JNI_TRUE); - builder.enableHappyEyeballs(enable_happy_eyeballs == JNI_TRUE); #ifdef ENVOY_ENABLE_QUIC builder.enableHttp3(enable_http3 == JNI_TRUE); #endif @@ -1304,8 +1303,8 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache, jlong dns_cache_save_interval_seconds, jboolean enable_drain_post_dns_refresh, jboolean enable_http3, jboolean enable_gzip_decompression, jboolean enable_brotli_decompression, - jboolean enable_socket_tagging, jboolean enable_happy_eyeballs, - jboolean enable_interface_binding, jlong h2_connection_keepalive_idle_interval_milliseconds, + jboolean enable_socket_tagging, jboolean enable_interface_binding, + jlong h2_connection_keepalive_idle_interval_milliseconds, jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host, jlong stats_flush_seconds, jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id, @@ -1324,11 +1323,10 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr dns_min_refresh_seconds, dns_preresolve_hostnames, enable_dns_cache, dns_cache_save_interval_seconds, enable_drain_post_dns_refresh, enable_http3, enable_gzip_decompression, enable_brotli_decompression, enable_socket_tagging, - enable_happy_eyeballs, enable_interface_binding, - h2_connection_keepalive_idle_interval_milliseconds, h2_connection_keepalive_timeout_seconds, - max_connections_per_host, stats_flush_seconds, stream_idle_timeout_seconds, - per_try_idle_timeout_seconds, app_version, app_id, trust_chain_verification, filter_chain, - stat_sinks, enable_platform_certificates_validation, + enable_interface_binding, h2_connection_keepalive_idle_interval_milliseconds, + h2_connection_keepalive_timeout_seconds, max_connections_per_host, stats_flush_seconds, + stream_idle_timeout_seconds, per_try_idle_timeout_seconds, app_version, app_id, + trust_chain_verification, filter_chain, stat_sinks, enable_platform_certificates_validation, enable_skip_dns_lookup_for_proxied_requests, runtime_guards, rtds_layer_name, rtds_timeout_seconds, ads_address, ads_port, ads_token, ads_token_lifetime, ads_root_certs, node_id, node_region, node_zone, node_sub_zone, cds_resources_locator, cds_timeout_seconds, diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java index d3fae5440e1a..0dbc74961b4d 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java @@ -44,7 +44,6 @@ public enum TrustChainVerification { public final Boolean enableGzipDecompression; public final Boolean enableBrotliDecompression; public final Boolean enableSocketTagging; - public final Boolean enableHappyEyeballs; public final Boolean enableInterfaceBinding; public final Integer h2ConnectionKeepaliveIdleIntervalMilliseconds; public final Integer h2ConnectionKeepaliveTimeoutSeconds; @@ -114,8 +113,6 @@ public enum TrustChainVerification { * decompression. * compression. * @param enableSocketTagging whether to enable socket tagging. - * @param enableHappyEyeballs whether to enable RFC 6555 handling for - * IPv4/IPv6. * @param enableInterfaceBinding whether to allow interface binding. * @param h2ConnectionKeepaliveIdleIntervalMilliseconds rate in milliseconds seconds to send h2 * pings on stream creation. @@ -163,7 +160,7 @@ public EnvoyConfiguration( int dnsQueryTimeoutSeconds, int dnsMinRefreshSeconds, List dnsPreresolveHostnames, boolean enableDNSCache, int dnsCacheSaveIntervalSeconds, boolean enableDrainPostDnsRefresh, boolean enableHttp3, boolean enableGzipDecompression, boolean enableBrotliDecompression, - boolean enableSocketTagging, boolean enableHappyEyeballs, boolean enableInterfaceBinding, + boolean enableSocketTagging, boolean enableInterfaceBinding, int h2ConnectionKeepaliveIdleIntervalMilliseconds, int h2ConnectionKeepaliveTimeoutSeconds, int maxConnectionsPerHost, int statsFlushSeconds, int streamIdleTimeoutSeconds, int perTryIdleTimeoutSeconds, String appVersion, String appId, @@ -195,7 +192,6 @@ public EnvoyConfiguration( this.enableGzipDecompression = enableGzipDecompression; this.enableBrotliDecompression = enableBrotliDecompression; this.enableSocketTagging = enableSocketTagging; - this.enableHappyEyeballs = enableHappyEyeballs; this.enableInterfaceBinding = enableInterfaceBinding; this.h2ConnectionKeepaliveIdleIntervalMilliseconds = h2ConnectionKeepaliveIdleIntervalMilliseconds; @@ -262,14 +258,14 @@ public long createBootstrap() { dnsFailureRefreshSecondsBase, dnsFailureRefreshSecondsMax, dnsQueryTimeoutSeconds, dnsMinRefreshSeconds, dns_preresolve, enableDNSCache, dnsCacheSaveIntervalSeconds, enableDrainPostDnsRefresh, enableHttp3, enableGzipDecompression, enableBrotliDecompression, - enableSocketTagging, enableHappyEyeballs, enableInterfaceBinding, - h2ConnectionKeepaliveIdleIntervalMilliseconds, h2ConnectionKeepaliveTimeoutSeconds, - maxConnectionsPerHost, statsFlushSeconds, streamIdleTimeoutSeconds, - perTryIdleTimeoutSeconds, appVersion, appId, enforceTrustChainVerification, filter_chain, - stats_sinks, enablePlatformCertificatesValidation, enableSkipDNSLookupForProxiedRequests, - runtime_guards, rtdsLayerName, rtdsTimeoutSeconds, adsAddress, adsPort, adsToken, - adsTokenLifetime, adsRootCerts, nodeId, nodeRegion, nodeZone, nodeSubZone, - cdsResourcesLocator, cdsTimeoutSeconds, enableCds); + enableSocketTagging, enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds, + h2ConnectionKeepaliveTimeoutSeconds, maxConnectionsPerHost, statsFlushSeconds, + streamIdleTimeoutSeconds, perTryIdleTimeoutSeconds, appVersion, appId, + enforceTrustChainVerification, filter_chain, stats_sinks, + enablePlatformCertificatesValidation, enableSkipDNSLookupForProxiedRequests, runtime_guards, + rtdsLayerName, rtdsTimeoutSeconds, adsAddress, adsPort, adsToken, adsTokenLifetime, + adsRootCerts, nodeId, nodeRegion, nodeZone, nodeSubZone, cdsResourcesLocator, + cdsTimeoutSeconds, enableCds); } static class ConfigurationException extends RuntimeException { diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java index f016241b8558..c69667fb72f5 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java @@ -302,7 +302,7 @@ public static native long createBootstrap( long dnsQueryTimeoutSeconds, long dnsMinRefreshSeconds, byte[][] dnsPreresolveHostnames, boolean enableDNSCache, long dnsCacheSaveIntervalSeconds, boolean enableDrainPostDnsRefresh, boolean enableHttp3, boolean enableGzipDecompression, boolean enableBrotliDecompression, - boolean enableSocketTagging, boolean enableHappyEyeballs, boolean enableInterfaceBinding, + boolean enableSocketTagging, boolean enableInterfaceBinding, long h2ConnectionKeepaliveIdleIntervalMilliseconds, long h2ConnectionKeepaliveTimeoutSeconds, long maxConnectionsPerHost, long statsFlushSeconds, long streamIdleTimeoutSeconds, long perTryIdleTimeoutSeconds, String appVersion, String appId, diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java index 404e6addbb58..ec0cecbd35de 100644 --- a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java @@ -50,7 +50,6 @@ public class NativeCronvoyEngineBuilderImpl extends CronvoyEngineBuilderImpl { private boolean mEnableDrainPostDnsRefresh = false; private boolean mEnableGzipDecompression = true; private boolean mEnableSocketTag = true; - private boolean mEnableHappyEyeballs = true; private boolean mEnableInterfaceBinding = false; private boolean mEnableProxying = false; private boolean mEnableSkipDNSLookupForProxiedRequests = false; @@ -139,14 +138,14 @@ private EnvoyConfiguration createEnvoyConfiguration() { mDnsFailureRefreshSecondsBase, mDnsFailureRefreshSecondsMax, mDnsQueryTimeoutSeconds, mDnsMinRefreshSeconds, mDnsPreresolveHostnames, mEnableDNSCache, mDnsCacheSaveIntervalSeconds, mEnableDrainPostDnsRefresh, quicEnabled(), - mEnableGzipDecompression, brotliEnabled(), mEnableSocketTag, mEnableHappyEyeballs, - mEnableInterfaceBinding, mH2ConnectionKeepaliveIdleIntervalMilliseconds, - mH2ConnectionKeepaliveTimeoutSeconds, mMaxConnectionsPerHost, mStatsFlushSeconds, - mStreamIdleTimeoutSeconds, mPerTryIdleTimeoutSeconds, mAppVersion, mAppId, - mTrustChainVerification, nativeFilterChain, platformFilterChain, stringAccessors, - keyValueStores, statSinks, runtimeGuards, mEnableSkipDNSLookupForProxiedRequests, - mEnablePlatformCertificatesValidation, mRtdsLayerName, mRtdsTimeoutSeconds, mAdsAddress, - mAdsPort, mAdsToken, mAdsTokenLifetime, mAdsRootCerts, mNodeId, mNodeRegion, mNodeZone, - mNodeSubZone, mCdsResourcesLocator, mCdsTimeoutSeconds, mEnableCds); + mEnableGzipDecompression, brotliEnabled(), mEnableSocketTag, mEnableInterfaceBinding, + mH2ConnectionKeepaliveIdleIntervalMilliseconds, mH2ConnectionKeepaliveTimeoutSeconds, + mMaxConnectionsPerHost, mStatsFlushSeconds, mStreamIdleTimeoutSeconds, + mPerTryIdleTimeoutSeconds, mAppVersion, mAppId, mTrustChainVerification, nativeFilterChain, + platformFilterChain, stringAccessors, keyValueStores, statSinks, runtimeGuards, + mEnableSkipDNSLookupForProxiedRequests, mEnablePlatformCertificatesValidation, + mRtdsLayerName, mRtdsTimeoutSeconds, mAdsAddress, mAdsPort, mAdsToken, mAdsTokenLifetime, + mAdsRootCerts, mNodeId, mNodeRegion, mNodeZone, mNodeSubZone, mCdsResourcesLocator, + mCdsTimeoutSeconds, mEnableCds); } } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt index fa92aecea67c..b6e1189ee0bf 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt @@ -56,7 +56,6 @@ open class EngineBuilder( private var dnsCacheSaveIntervalSeconds = 1 private var enableDrainPostDnsRefresh = false internal var enableHttp3 = true - private var enableHappyEyeballs = true private var enableGzipDecompression = true private var enableBrotliDecompression = false private var enableSocketTagging = false @@ -240,19 +239,6 @@ open class EngineBuilder( return this } - /** - * Specify whether to use Happy Eyeballs when multiple IP stacks may be supported. Defaults to - * true. - * - * @param enableHappyEyeballs whether to enable RFC 6555 handling for IPv4/IPv6. - * - * @return This builder. - */ - fun enableHappyEyeballs(enableHappyEyeballs: Boolean): EngineBuilder { - this.enableHappyEyeballs = enableHappyEyeballs - return this - } - /** * Specify whether to do gzip response decompression or not. Defaults to true. * @@ -684,7 +670,6 @@ open class EngineBuilder( enableGzipDecompression, enableBrotliDecompression, enableSocketTagging, - enableHappyEyeballs, enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds, h2ConnectionKeepaliveTimeoutSeconds, diff --git a/mobile/library/objective-c/EnvoyConfiguration.h b/mobile/library/objective-c/EnvoyConfiguration.h index 6e0a057804e7..313f77ab1ba5 100644 --- a/mobile/library/objective-c/EnvoyConfiguration.h +++ b/mobile/library/objective-c/EnvoyConfiguration.h @@ -22,7 +22,6 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic, assign) UInt32 dnsRefreshSeconds; @property (nonatomic, assign) BOOL enableDNSCache; @property (nonatomic, assign) UInt32 dnsCacheSaveIntervalSeconds; -@property (nonatomic, assign) BOOL enableHappyEyeballs; @property (nonatomic, assign) BOOL enableHttp3; @property (nonatomic, assign) BOOL enableGzipDecompression; @property (nonatomic, assign) BOOL enableBrotliDecompression; @@ -76,7 +75,6 @@ NS_ASSUME_NONNULL_BEGIN dnsPreresolveHostnames:(NSArray *)dnsPreresolveHostnames enableDNSCache:(BOOL)enableDNSCache dnsCacheSaveIntervalSeconds:(UInt32)dnsCacheSaveIntervalSeconds - enableHappyEyeballs:(BOOL)enableHappyEyeballs enableHttp3:(BOOL)enableHttp3 enableGzipDecompression:(BOOL)enableGzipDecompression enableBrotliDecompression:(BOOL)enableBrotliDecompression diff --git a/mobile/library/objective-c/EnvoyConfiguration.mm b/mobile/library/objective-c/EnvoyConfiguration.mm index 2a3b095ebe21..aa76f9135d75 100644 --- a/mobile/library/objective-c/EnvoyConfiguration.mm +++ b/mobile/library/objective-c/EnvoyConfiguration.mm @@ -78,7 +78,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled dnsPreresolveHostnames:(NSArray *)dnsPreresolveHostnames enableDNSCache:(BOOL)enableDNSCache dnsCacheSaveIntervalSeconds:(UInt32)dnsCacheSaveIntervalSeconds - enableHappyEyeballs:(BOOL)enableHappyEyeballs enableHttp3:(BOOL)enableHttp3 enableGzipDecompression:(BOOL)enableGzipDecompression enableBrotliDecompression:(BOOL)enableBrotliDecompression @@ -143,7 +142,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled self.dnsPreresolveHostnames = dnsPreresolveHostnames; self.enableDNSCache = enableDNSCache; self.dnsCacheSaveIntervalSeconds = dnsCacheSaveIntervalSeconds; - self.enableHappyEyeballs = enableHappyEyeballs; self.enableHttp3 = enableHttp3; self.enableGzipDecompression = enableGzipDecompression; self.enableBrotliDecompression = enableBrotliDecompression; @@ -232,7 +230,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled } builder.addDnsPreresolveHostnames(hostnames); } - builder.enableHappyEyeballs(self.enableHappyEyeballs); builder.addDnsRefreshSeconds(self.dnsRefreshSeconds); builder.enableDrainPostDnsRefresh(self.enableDrainPostDnsRefresh); builder.enableInterfaceBinding(self.enableInterfaceBinding); diff --git a/mobile/library/swift/EngineBuilder.swift b/mobile/library/swift/EngineBuilder.swift index 68ad83ca3535..2e187639be2a 100644 --- a/mobile/library/swift/EngineBuilder.swift +++ b/mobile/library/swift/EngineBuilder.swift @@ -30,7 +30,6 @@ open class EngineBuilder: NSObject { private var dnsRefreshSeconds: UInt32 = 60 private var enableDNSCache: Bool = false private var dnsCacheSaveIntervalSeconds: UInt32 = 1 - private var enableHappyEyeballs: Bool = true private var enableGzipDecompression: Bool = true private var enableBrotliDecompression: Bool = false #if ENVOY_ENABLE_QUIC @@ -218,18 +217,6 @@ open class EngineBuilder: NSObject { return self } - /// Specify whether to use Happy Eyeballs when multiple IP stacks may be supported. Defaults to - /// true. - /// - /// - parameter enableHappyEyeballs: whether to enable RFC 6555 handling for IPv4/IPv6. - /// - /// - returns: This builder. - @discardableResult - public func enableHappyEyeballs(_ enableHappyEyeballs: Bool) -> Self { - self.enableHappyEyeballs = enableHappyEyeballs - return self - } - /// Specify whether to do gzip response decompression or not. Defaults to true. /// /// - parameter enableGzipDecompression: whether or not to gunzip responses. @@ -728,7 +715,6 @@ open class EngineBuilder: NSObject { dnsPreresolveHostnames: self.dnsPreresolveHostnames, enableDNSCache: self.enableDNSCache, dnsCacheSaveIntervalSeconds: self.dnsCacheSaveIntervalSeconds, - enableHappyEyeballs: self.enableHappyEyeballs, enableHttp3: self.enableHttp3, enableGzipDecompression: self.enableGzipDecompression, enableBrotliDecompression: self.enableBrotliDecompression, @@ -802,7 +788,6 @@ private extension EngineBuilder { cxxBuilder.addDnsMinRefreshSeconds(Int32(self.dnsMinRefreshSeconds)) cxxBuilder.addDnsPreresolveHostnames(self.dnsPreresolveHostnames.toCXX()) cxxBuilder.enableDnsCache(self.enableDNSCache, Int32(self.dnsCacheSaveIntervalSeconds)) - cxxBuilder.enableHappyEyeballs(self.enableHappyEyeballs) #if ENVOY_ENABLE_QUIC cxxBuilder.enableHttp3(self.enableHttp3) #endif diff --git a/mobile/test/cc/unit/envoy_config_test.cc b/mobile/test/cc/unit/envoy_config_test.cc index f51fa2102e5b..200814196abb 100644 --- a/mobile/test/cc/unit/envoy_config_test.cc +++ b/mobile/test/cc/unit/envoy_config_test.cc @@ -212,21 +212,6 @@ TEST(TestConfig, EnableDrainPostDnsRefresh) { EXPECT_THAT(bootstrap->ShortDebugString(), HasSubstr("enable_drain_post_dns_refresh: true")); } -TEST(TestConfig, EnableHappyEyeballs) { - EngineBuilder engine_builder; - - std::unique_ptr bootstrap = engine_builder.generateBootstrap(); - std::string bootstrap_str = bootstrap->ShortDebugString(); - EXPECT_THAT(bootstrap_str, Not(HasSubstr("dns_lookup_family: V4_PREFERRED"))); - EXPECT_THAT(bootstrap_str, HasSubstr("dns_lookup_family: ALL")); - - engine_builder.enableHappyEyeballs(false); - bootstrap = engine_builder.generateBootstrap(); - bootstrap_str = bootstrap->ShortDebugString(); - EXPECT_THAT(bootstrap_str, HasSubstr("dns_lookup_family: V4_PREFERRED")); - EXPECT_THAT(bootstrap_str, Not(HasSubstr("dns_lookup_family: ALL"))); -} - TEST(TestConfig, EnforceTrustChainVerification) { EngineBuilder engine_builder; diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt index 53ca21f4fd67..acdb2bd171d7 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt @@ -84,7 +84,6 @@ class EnvoyConfigurationTest { enableGzipDecompression: Boolean = true, enableBrotliDecompression: Boolean = false, enableSocketTagging: Boolean = false, - enableHappyEyeballs: Boolean = false, enableInterfaceBinding: Boolean = false, h2ConnectionKeepaliveIdleIntervalMilliseconds: Int = 222, h2ConnectionKeepaliveTimeoutSeconds: Int = 333, @@ -134,7 +133,6 @@ class EnvoyConfigurationTest { enableGzipDecompression, enableBrotliDecompression, enableSocketTagging, - enableHappyEyeballs, enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds, h2ConnectionKeepaliveTimeoutSeconds, @@ -185,7 +183,7 @@ class EnvoyConfigurationTest { assertThat(resolvedTemplate).contains("base_interval: 345s") assertThat(resolvedTemplate).contains("max_interval: 456s") assertThat(resolvedTemplate).contains("dns_query_timeout: 321s") - assertThat(resolvedTemplate).contains("dns_lookup_family: V4_PREFERRED") + assertThat(resolvedTemplate).contains("dns_lookup_family: ALL") assertThat(resolvedTemplate).contains("dns_min_refresh_rate: 12s") assertThat(resolvedTemplate).contains("preresolve_hostnames:") assertThat(resolvedTemplate).contains("hostname1") @@ -255,7 +253,6 @@ class EnvoyConfigurationTest { enableDrainPostDnsRefresh = true, enableDNSCache = true, dnsCacheSaveIntervalSeconds = 101, - enableHappyEyeballs = true, enableHttp3 = false, enableGzipDecompression = false, enableBrotliDecompression = true, @@ -283,9 +280,6 @@ class EnvoyConfigurationTest { // dnsCacheSaveIntervalSeconds = 101 assertThat(resolvedTemplate).contains("save_interval: 101") - // enableHappyEyeballs = true - assertThat(resolvedTemplate).contains("dns_lookup_family: ALL") - // enableHttp3 = false assertThat(resolvedTemplate).doesNotContain("name: alternate_protocols_cache"); diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt index 6da0ac4c1796..8d3d1af832b6 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt @@ -34,16 +34,6 @@ class EngineBuilderTest { assertThat(engine.envoyConfiguration.grpcStatsDomain).isEqualTo("stats.envoyproxy.io") } - @Test - fun `enabling happy eyeballs overrides default`() { - engineBuilder = EngineBuilder(Standard()) - engineBuilder.addEngineType { envoyEngine } - engineBuilder.enableHappyEyeballs(true) - - val engine = engineBuilder.build() as EngineImpl - assertThat(engine.envoyConfiguration.enableHappyEyeballs).isTrue() - } - @Test fun `enabling interface binding overrides default`() { engineBuilder = EngineBuilder(Standard()) diff --git a/mobile/test/swift/EngineBuilderTests.swift b/mobile/test/swift/EngineBuilderTests.swift index 989def5bcc37..bfd83b08438b 100644 --- a/mobile/test/swift/EngineBuilderTests.swift +++ b/mobile/test/swift/EngineBuilderTests.swift @@ -96,20 +96,6 @@ final class EngineBuilderTests: XCTestCase { } #endif - func testEnablingHappyEyeballsAddsToConfigurationWhenRunningEnvoy() { - let expectation = self.expectation(description: "Run called with enabled happy eyeballs") - MockEnvoyEngine.onRunWithConfig = { config, _ in - XCTAssertTrue(config.enableHappyEyeballs) - expectation.fulfill() - } - - _ = EngineBuilder() - .addEngineType(MockEnvoyEngine.self) - .enableHappyEyeballs(true) - .build() - self.waitForExpectations(timeout: 0.01) - } - func testEnablingInterfaceBindingAddsToConfigurationWhenRunningEnvoy() { let expectation = self.expectation(description: "Run called with enabled interface binding") MockEnvoyEngine.onRunWithConfig = { config, _ in From 21e351b151b82229a77f1bee73eaadc0d8c279ce Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 3 May 2023 16:54:22 +0100 Subject: [PATCH 099/740] ci: Cleanups/optimizations (#27149) Signed-off-by: Ryan Northey --- .bazelrc | 1 + ci/do_ci.sh | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.bazelrc b/.bazelrc index 1544661efcf0..7fcb7286984c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -260,6 +260,7 @@ build:remote --strategy=Genrule=remote,sandboxed,local build:remote --remote_timeout=7200 build:remote --google_default_credentials=true build:remote --remote_download_toplevel +build:remote --nobuild_runfile_links # Windows bazel does not allow sandboxed as a spawn strategy build:remote-windows --spawn_strategy=remote,local diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 17a7296114e0..1310838e9f53 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -32,7 +32,7 @@ fi function collect_build_profile() { declare -g build_profile_count=${build_profile_count:-1} - mv -f "$(bazel info output_base)/command.profile.gz" "${ENVOY_BUILD_PROFILE}/${build_profile_count}-$1.profile.gz" || true + mv -f "$(bazel info "${BAZEL_BUILD_OPTIONS[@]}" output_base)/command.profile.gz" "${ENVOY_BUILD_PROFILE}/${build_profile_count}-$1.profile.gz" || true ((build_profile_count++)) } @@ -225,21 +225,30 @@ case $CI_TARGET in ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" mkdir -p "$ENVOY_BINARY_DIR" + # As the binary build package enforces compiler options, adding here to ensure the tests and distribution build + # reuse settings and any already compiled artefacts, the bundle itself will always be compiled + # `--stripopt=--strip-all -c opt` + BAZEL_RELEASE_OPTIONS=( + --stripopt=--strip-all + -c opt) + # Run release tests - echo "Testing ${TEST_TARGETS[*]} with options: ${BAZEL_BUILD_OPTIONS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_minimal -c opt "${TEST_TARGETS[@]}" + echo "Testing with:" + echo " targets: ${TEST_TARGETS[*]}" + echo " build options: ${BAZEL_BUILD_OPTIONS[*]}" + echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}" + + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_minimal "${BAZEL_RELEASE_OPTIONS[@]}" "${TEST_TARGETS[@]}" # Build release binaries - # As the binary build package enforces `-c opt`, adding here to ensure the distribution build reuses - # any already compiled artefacts, the bundle itself will always be compiled `-c opt` - bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c opt //distribution/binary:release + bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" //distribution/binary:release # Copy release binaries to binary export directory cp -a "bazel-bin/distribution/binary/release.tar.zst" "${ENVOY_BINARY_DIR}/release.tar.zst" # Grab the schema_validator_tool # TODO(phlax): bundle this with the release when #26390 is resolved - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel --stripopt=--strip-all -c opt //test/tools/schema_validator:schema_validator_tool.stripped + bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel "${BAZEL_RELEASE_OPTIONS[@]}" //test/tools/schema_validator:schema_validator_tool.stripped # Copy schema_validator_tool to binary export directory cp -a bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped "${ENVOY_BINARY_DIR}/schema_validator_tool" @@ -502,7 +511,7 @@ case $CI_TARGET in ;; fuzz) setup_clang_toolchain - FUZZ_TEST_TARGETS=("$(bazel query "attr('tags','fuzzer',${TEST_TARGETS[*]})")") + FUZZ_TEST_TARGETS=("$(bazel query "${BAZEL_GLOBAL_OPTIONS[@]}" "attr('tags','fuzzer',${TEST_TARGETS[*]})")") echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS[*]}" echo "Building envoy fuzzers and executing 100 fuzz iterations..." bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --config=asan-fuzzer "${FUZZ_TEST_TARGETS[@]}" --test_arg="-runs=10" From 3fe4b8d335fa339ef6f17325c8d31f87ade7bb1a Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 3 May 2023 17:08:57 +0100 Subject: [PATCH 100/740] ci: Add `--test_verbose_timeout_warnings` (#27150) Signed-off-by: Ryan Northey --- .bazelrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.bazelrc b/.bazelrc index 7fcb7286984c..eb023f6c5535 100644 --- a/.bazelrc +++ b/.bazelrc @@ -44,6 +44,8 @@ build --test_summary=terse build --incompatible_config_setting_private_default_visibility build --incompatible_enforce_config_setting_visibility +test --test_verbose_timeout_warnings + # Allow tags to influence execution requirements common --experimental_allow_tags_propagation From 8fd8b68adfe64eecc25c8e609568a09839b53fd1 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 3 May 2023 12:50:41 -0400 Subject: [PATCH 101/740] test: racheting coverage (#27151) Signed-off-by: Alyssa Wilk --- test/per_file_coverage.sh | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 685c13b430d0..c3c94ccd8be3 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -3,30 +3,29 @@ # directory:coverage_percent # for existing directories with low coverage. declare -a KNOWN_LOW_COVERAGE=( -"source/common:96.0" -"source/common/api:82.4" +"source/common:96.2" +"source/common/api:82.6" "source/common/api/posix:81.3" "source/common/common/posix:92.7" "source/common/config:96.1" "source/common/crypto:88.1" "source/common/event:95.1" # Emulated edge events guards don't report LCOV "source/common/filesystem/posix:96.2" # FileReadToEndNotReadable fails in some env; createPath can't test all failure branches. -"source/common/http:96.3" -"source/common/http/http2:95.0" +"source/common/http:96.5" +"source/common/http/http2:95.2" "source/common/json:93.4" "source/common/matcher:94.6" "source/common/network:94.4" # Flaky, `activateFileEvents`, `startSecureTransport` and `ioctl`, listener_socket do not always report LCOV "source/common/network/dns_resolver:91.6" # A few lines of MacOS code not tested in linux scripts. Tested in MacOS scripts "source/common/protobuf:96.3" -"source/common/quic:93.4" -"source/common/router:96.1" +"source/common/quic:93.5" +"source/common/router:96.4" "source/common/secret:95.0" "source/common/signal:87.2" # Death tests don't report LCOV "source/common/singleton:95.7" "source/common/tcp:93.0" "source/common/thread:0.0" # Death tests don't report LCOV "source/common/tracing:97.1" -"source/common/upstream:96.4" "source/common/watchdog:58.6" # Death tests don't report LCOV "source/exe:94.5" "source/extensions/access_loggers/wasm:93.5" @@ -38,20 +37,20 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/filters/common:96.5" "source/extensions/filters/common/fault:94.5" "source/extensions/filters/common/rbac:90.5" -"source/extensions/filters/http/cache:93.3" +"source/extensions/filters/http/cache:93.4" "source/extensions/filters/http/grpc_json_transcoder:95.6" "source/extensions/filters/http/ip_tagging:88.0" "source/extensions/filters/http/kill_request:91.7" # Death tests don't report LCOV "source/extensions/filters/http/wasm:1.9" -"source/extensions/filters/http/rate_limit_quota:94" # TODO(tyxia) WIP. Improve coverage as development continues +"source/extensions/filters/http/rate_limit_quota:94.8" # TODO(tyxia) WIP. Improve coverage as development continues "source/extensions/filters/listener/original_dst:82.9" "source/extensions/filters/listener/original_src:92.1" -"source/extensions/filters/network/common:96.1" +"source/extensions/filters/network/common:96.2" "source/extensions/filters/network/common/redis:96.4" "source/extensions/filters/network/mongo_proxy:96.0" "source/extensions/filters/network/sni_cluster:88.9" "source/extensions/filters/network/thrift_proxy/router:96.5" -"source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata:96.2" +"source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata:96.3" "source/extensions/filters/network/wasm:76.9" "source/extensions/http/cache/simple_http_cache:95.9" "source/extensions/rate_limit_descriptors:95.5" @@ -63,24 +62,24 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/tracers/common/ot:71.7" "source/extensions/tracers/opencensus:93.2" "source/extensions/tracers/zipkin:95.8" -"source/extensions/transport_sockets:95.6" -"source/extensions/transport_sockets/tls:94.8" +"source/extensions/transport_sockets:95.7" +"source/extensions/transport_sockets/tls:94.9" "source/extensions/transport_sockets/tls/cert_validator:95.1" "source/extensions/transport_sockets/tls/private_key:88.9" "source/extensions/wasm_runtime/wamr:0.0" # Not enabled in coverage build "source/extensions/wasm_runtime/wasmtime:0.0" # Not enabled in coverage build "source/extensions/wasm_runtime/wavm:0.0" # Not enabled in coverage build "source/extensions/watchdog:83.3" # Death tests within extensions -"source/extensions/listener_managers/validation_listener_manager:69.6" +"source/extensions/listener_managers/validation_listener_manager:70.0" "source/extensions/watchdog/profile_action:83.3" -"source/server:93.3" # flaky: be careful adjusting. See https://github.com/envoyproxy/envoy/issues/15239 +"source/server:93.8" # flaky: be careful adjusting. See https://github.com/envoyproxy/envoy/issues/15239 "source/server/admin:profiler-lib:83" "source/extensions/load_balancing_policies/common:94" # Death tests don't report LCOV -"source/server/config_validation:83.8" -"source/extensions/health_checkers:95.5" +"source/server/config_validation:88.2" +"source/extensions/health_checkers:95.9" "source/extensions/health_checkers/http:93.8" "source/extensions/health_checkers/grpc:92.0" -"source/extensions/load_balancing_policies:94.7" +"source/extensions/load_balancing_policies:95.5" "source/extensions/load_balancing_policies/subset:94.3" ) From 9b7144374cca05b029fdcac816bf81eb87e6f54e Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Wed, 3 May 2023 17:36:54 -0400 Subject: [PATCH 102/740] docs: Link to CODEOWNERS from OWNERS.md (#27157) Signed-off-by: Ali Beyad --- OWNERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS.md b/OWNERS.md index 273cd62f0445..90f167fd9166 100644 --- a/OWNERS.md +++ b/OWNERS.md @@ -1,5 +1,6 @@ * See [CONTRIBUTING.md](CONTRIBUTING.md) for general contribution guidelines. * See [GOVERNANCE.md](GOVERNANCE.md) for governance guidelines and maintainer responsibilities. +* See [CODEOWNERS](CODEOWNERS) for a detailed list of owners for the various source directories. This page lists all active maintainers and their areas of expertise. This can be used for routing PRs, questions, etc. to the right place. From d7b5c19b72ce0eef4a0391d14b0093718e8e8b14 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Thu, 4 May 2023 00:05:56 -0700 Subject: [PATCH 103/740] Bump MACOSX_DEPLOYMENT_TARGET to 10.8 (#27130) Signed-off-by: Networking team --- bazel/foreign_cc/luajit.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/foreign_cc/luajit.patch b/bazel/foreign_cc/luajit.patch index b6da9912e867..98fc8f6ded92 100644 --- a/bazel/foreign_cc/luajit.patch +++ b/bazel/foreign_cc/luajit.patch @@ -149,7 +149,7 @@ index 00000000..1201542c + shutil.copytree(src_dir, os.path.basename(src_dir)) + os.chdir(os.path.basename(src_dir)) + -+ os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.6" ++ os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.8" + os.environ["DEFAULT_CC"] = os.environ.get("CC", "") + os.environ["TARGET_CFLAGS"] = os.environ.get("CFLAGS", "") + " -fno-function-sections -fno-data-sections" + os.environ["TARGET_LDFLAGS"] = os.environ.get("CFLAGS", "") + " -fno-function-sections -fno-data-sections" From a723174541001a0b6b7339320aab2660b9bfde9a Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 4 May 2023 10:01:18 +0100 Subject: [PATCH 104/740] Revert "ci: Workaround dep checker (cves) issue again (#27141)" (#27156) This reverts commit 97a2f2b94e944c58d4214cf5ecd4fce818a2351e. Signed-off-by: Ryan Northey --- ci/do_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 1310838e9f53..9f9554901cd6 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -562,7 +562,7 @@ case $CI_TARGET in bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:check \ --action_env=TODAY_DATE \ -- -v warn \ - -c cves release_dates releases || echo "WARNING: Dependency check failed" + -c cves release_dates releases # Run dependabot tests echo "Check dependabot ..." From 58f712e733b8131a57087472e50a24f56b52c15c Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 4 May 2023 10:21:33 +0100 Subject: [PATCH 105/740] ci: Improve/clarify var name (#27175) Signed-off-by: Ryan Northey --- .azure-pipelines/stages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index c6d5348f6aa0..c08c96310b77 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -143,7 +143,7 @@ stages: authGCP: $(GcpServiceAccountKey) authGithub: $(GitHubPublicRepoOnlyAccessToken) authGPGPassphrase: $(MaintainerGPGKeyPassphrase) - authGPGKey: $(MaintainerGPGKey) + authGPGKey: $(MaintainerGPGKeySecureFileDownloadPath) authGPGPath: $(MaintainerGPGKey.secureFilePath) authNetlifyURL: $(NetlifyTriggerURL) authSSHDocsKeyPublic: $(DocsPublicKey) From fdbdc066d6565abc652e047497f0d70fa90ace12 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 4 May 2023 12:22:45 +0100 Subject: [PATCH 106/740] ci: Improve/add tests for GPG signing (#27080) Signed-off-by: Ryan Northey --- .azure-pipelines/bazel.yml | 8 +-- .azure-pipelines/env.yml | 16 +++++ .azure-pipelines/gpg.yml | 80 +++++++++++++++++++++++ .azure-pipelines/gpg/activate-key.sh | 64 ++++++++++++++++++ .azure-pipelines/gpg/generate-test-key.sh | 21 ++++++ .azure-pipelines/stage/prechecks.yml | 78 ++++++++++++++++++---- .azure-pipelines/stage/publish.yml | 67 +++++++++---------- .azure-pipelines/stages.yml | 3 + ci/do_ci.sh | 28 ++++---- distribution/packages.bzl | 14 +--- 10 files changed, 305 insertions(+), 74 deletions(-) create mode 100644 .azure-pipelines/gpg.yml create mode 100755 .azure-pipelines/gpg/activate-key.sh create mode 100755 .azure-pipelines/gpg/generate-test-key.sh diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index 6a5d8e6961f4..f7a768f862ad 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -64,10 +64,6 @@ steps: fetchDepth: ${{ parameters.repoFetchDepth }} fetchTags: ${{ parameters.repoFetchTags }} -- ${{ each step in parameters.stepsPre }}: - - ${{ each pair in step }}: - ${{ pair.key }}: ${{ pair.value }} - # Set up tmpfs directories for self-hosted agents which have a surplus of mem. # # NB: Do not add any directory that grow larger than spare memory capacity! @@ -132,6 +128,10 @@ steps: version: "$(cacheKeyBuildImage)" arch: "${{ parameters.artifactSuffix }}" +- ${{ each step in parameters.stepsPre }}: + - ${{ each pair in step }}: + ${{ pair.key }}: ${{ pair.value }} + - bash: | echo "disk space at beginning of build:" df -h diff --git a/.azure-pipelines/env.yml b/.azure-pipelines/env.yml index bad63dc4db7f..aa9d16de691a 100644 --- a/.azure-pipelines/env.yml +++ b/.azure-pipelines/env.yml @@ -213,3 +213,19 @@ jobs: echo "env.outputs['publish.netlify']: $(publish.netlify)" displayName: "Print build environment" + +- job: test_artifacts + dependsOn: [] + displayName: Test artifacts + condition: ne(variables['Build.DefinitionName'], 'envoy-postsubmit') + pool: + vmImage: "ubuntu-20.04" + steps: + - script: $(Build.SourcesDirectory)/.azure-pipelines/gpg/generate-test-key.sh + displayName: "Generate snakeoil GPG key for testing" + workingDirectory: $(Build.StagingDirectory) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/envoy" + artifactName: test.env + timeoutInMinutes: 10 diff --git a/.azure-pipelines/gpg.yml b/.azure-pipelines/gpg.yml new file mode 100644 index 000000000000..b9085e9ff547 --- /dev/null +++ b/.azure-pipelines/gpg.yml @@ -0,0 +1,80 @@ + +parameters: +- name: nameDownloadTask + type: string + default: MaintainerGPGKey +- name: downloadKey + type: boolean + default: true + +# Auth +- name: authGPGPassphrase + type: string + default: "" +- name: authGPGPath + type: string + default: "" +- name: authGPGKey + type: string + default: "" + +## Paths +# The configured GNUPGHOME. +# This is used to set the path to the password file +# and should point to the GPG homedir at the point of use, +# ie inside the Docker container (/build/.gnupg) +- name: pathGPGConfiguredHome + type: string + default: "" +# The created GNUPGHOME +# This is where the GPG homedir will actually be created, and should point to +# the path on the host (ie /tmp/envoy-docker-build/.gnupg) +- name: pathGPGHome + type: string + default: "" + +# All steps should include this condition +- name: condition + type: string + default: eq('true', 'true') + + +steps: +# Secure key - postsubmit +- task: DownloadSecureFile@1 + name: ${{ parameters.nameDownloadTask }} + displayName: 'Download maintainer GPG key' + condition: | + and(succeeded(), + ${{ parameters.condition }}, + eq(${{ parameters.downloadKey }}, true), + eq(variables['Build.DefinitionName'], 'envoy-postsubmit')) + inputs: + # NB: This is the path to the key and **must** be set on a per-pipeline basis + secureFile: '${{ parameters.authGPGKey }}' + +# Snakeoil key - non-postsubmit +- task: DownloadBuildArtifacts@0 + displayName: 'Download snakeoil GPG key' + inputs: + buildType: current + artifactName: "test.env" + itemPattern: "test.env/ci.snakeoil.gpg.key" + targetPath: $(Build.StagingDirectory) + condition: | + and(succeeded(), + ${{ parameters.condition }}, + eq(${{ parameters.downloadKey }}, true), + ne(variables['Build.DefinitionName'], 'envoy-postsubmit')) + +- script: $(Build.SourcesDirectory)/.azure-pipelines/gpg/activate-key.sh + displayName: "Import and activate GPG key" + condition: | + and(succeeded(), + ${{ parameters.condition }}) + env: + GPG_PASSPHRASE: ${{ parameters.authGPGPassphrase }} + GPG_KEYFILE: ${{ parameters.authGPGPath }} + GNUPGHOME_CONFIGURED: ${{ parameters.pathGPGConfiguredHome }} + GNUPGHOME: ${{ parameters.pathGPGHome }} + workingDirectory: $(Build.StagingDirectory) diff --git a/.azure-pipelines/gpg/activate-key.sh b/.azure-pipelines/gpg/activate-key.sh new file mode 100755 index 000000000000..4f48ce6e54b4 --- /dev/null +++ b/.azure-pipelines/gpg/activate-key.sh @@ -0,0 +1,64 @@ +#!/bin/bash -eu + +set -o pipefail + +# The key configured here can either be the snakeoil key created +# in CI, or in PostSubmit its the actual maintainer key downloaded +# from an azp SecureFile + +# If the intended destination of the `.gnupg` file is not the same +# as where it is configured (ie configuring on the host for use in a +# container), GNUPGHOME_CONFIGURED can be set pointing to where `.gnupg` +# will be. + +GNUPGHOME="${GNUPGHOME:-${HOME}/.gnupg}" +GPG_PASSPHRASE="${GPG_PASSPHRASE:-HACKME}" +GPG_KEYFILE="${GPG_KEYFILE:-test.env/ci.snakeoil.gpg.key}" + +# In the case of non-PostSubmit AZP leaves the `secureFilePath` variable +# uniterpolated, so matching here is matching that its not set. +# +# shellcheck disable=SC2016 +if [[ "$GPG_KEYFILE" == '$(MaintainerGPGKey.secureFilePath)' ]]; then + GPG_PASSPHRASE=HACKME + GPG_KEYFILE="test.env/ci.snakeoil.gpg.key" +fi + +# The configured home, may be different if the configuration +# is to be used inside a container or different env to the one +# in which the configuration is created. +if [[ -z "$GNUPGHOME_CONFIGURED" ]]; then + GNUPGHOME_CONFIGURED="$GNUPGHOME" +fi + +if [[ ! -e "$GNUPGHOME" ]]; then + mkdir -p "$GNUPGHOME" + chmod 700 "${GNUPGHOME}" +fi + +# Reload the gpg-agent +eval "$(gpg-agent --daemon --allow-loopback-pinentry)" + +# Load the key +echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --import "$GPG_KEYFILE" + +# Set the passphrase in a file +echo "$GPG_PASSPHRASE" > "${GNUPGHOME}/gpg-passphrase" +chmod 600 "${GNUPGHOME}/gpg-passphrase" + +# Configure gpg to use the file - not the configured path may be different +{ + echo "use-agent" + echo "pinentry-mode loopback" + echo "passphrase-file ${GNUPGHOME_CONFIGURED}/gpg-passphrase" +} >> "$GNUPGHOME/gpg.conf" + +# Configure gpg-agent +echo "allow-loopback-pinentry" >> "$GNUPGHOME/gpg-agent.conf" +echo RELOADAGENT | gpg-connect-agent + +if [[ "$GNUPGHOME" != "$GNUPGHOME_CONFIGURED" ]]; then + echo "GPG configured in ${GNUPGHOME} for ${GNUPGHOME_CONFIGURED}" +else + echo "GPG configured in ${GNUPGHOME}" +fi diff --git a/.azure-pipelines/gpg/generate-test-key.sh b/.azure-pipelines/gpg/generate-test-key.sh new file mode 100755 index 000000000000..f473e5fc133e --- /dev/null +++ b/.azure-pipelines/gpg/generate-test-key.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e + +gpg --batch --gen-key < envoy/ci.snakeoil.gpg.key diff --git a/.azure-pipelines/stage/prechecks.yml b/.azure-pipelines/stage/prechecks.yml index 8e4bd99b02f0..22c1db3ab9c4 100644 --- a/.azure-pipelines/stage/prechecks.yml +++ b/.azure-pipelines/stage/prechecks.yml @@ -23,6 +23,15 @@ parameters: - name: authGCP type: string default: "" +- name: authGPGPassphrase + type: string + default: "" +- name: authGPGPath + type: string + default: "" +- name: authGPGKey + type: string + default: "" jobs: @@ -50,6 +59,62 @@ jobs: cacheVersion: $(cacheKeyBazel) publishEnvoy: false publishTestResults: false + stepsPre: + ## Ensure we can sign things + # Signing on the host + - template: ../gpg.yml + parameters: + condition: and(not(canceled()), eq(variables['CI_TARGET'], 'docs')) + authGPGPassphrase: ${{ parameters.authGPGPassphrase }} + authGPGPath: ${{ parameters.authGPGPath }} + authGPGKey: ${{ parameters.authGPGKey }} + - bash: | + set -e + echo AUTHORITY > /tmp/authority + gpg --clearsign /tmp/authority + cat /tmp/authority.asc + gpg --verify /tmp/authority.asc + rm -rf ~/.gnupg + displayName: "Ensure host CI can sign with GPG" + condition: and(not(canceled()), eq(variables['CI_TARGET'], 'docs')) + + # Signing in the Docker container + - template: ../gpg.yml + parameters: + condition: and(not(canceled()), eq(variables['CI_TARGET'], 'docs')) + # Reuse key downloaded above + downloadKey: false + nameDownloadTask: MaintainerGPGKey2 + authGPGPassphrase: ${{ parameters.authGPGPassphrase }} + authGPGPath: ${{ parameters.authGPGPath }} + authGPGKey: ${{ parameters.authGPGKey }} + # GNUPGHOME inside the container + pathGPGConfiguredHome: /build/.gnupg + pathGPGHome: /tmp/envoy-docker-build/.gnupg + - bash: | + set -e + ci/run_envoy_docker.sh " + echo AUTHORITY > /tmp/authority \ + && gpg --clearsign /tmp/authority \ + && cat /tmp/authority.asc \ + && gpg --verify /tmp/authority.asc" + rm -rf /tmp/envoy-docker-build/.gnupg + displayName: "Ensure container CI can sign with GPG" + condition: and(not(canceled()), eq(variables['CI_TARGET'], 'docs')) + + # Docker regression tests + - script: | + DOCKER_CI_FIX_DIFF=$(Build.StagingDirectory)/fix_docker.diff DOCKER_CI_FIX=1 ci/test_docker_ci.sh + workingDirectory: $(Build.SourcesDirectory) + displayName: Docker build regression test + condition: eq(variables['CI_TARGET'], 'docs') + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/fix_docker.diff" + artifactName: "docker_ci" + timeoutInMinutes: 10 + condition: and(failed(), eq(variables['CI_TARGET'], 'docs')) + stepsPost: # Format fixes @@ -78,19 +143,6 @@ jobs: GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} condition: eq(variables['CI_TARGET'], 'docs') - # Docker regression tests - - script: | - DOCKER_CI_FIX_DIFF=$(Build.StagingDirectory)/fix_docker.diff DOCKER_CI_FIX=1 ci/test_docker_ci.sh - workingDirectory: $(Build.SourcesDirectory) - displayName: Docker build regression test - condition: eq(variables['CI_TARGET'], 'docs') - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/fix_docker.diff" - artifactName: "docker_ci" - timeoutInMinutes: 10 - condition: and(failed(), eq(variables['CI_TARGET'], 'docs')) - - job: dependencies displayName: Precheck dependencies timeoutInMinutes: 20 diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index c74654654e55..3e15b7253ed7 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -155,6 +155,19 @@ jobs: - template: ../bazel.yml parameters: ciTarget: bazel.distribution + stepsPre: + - template: ../gpg.yml + parameters: + authGPGPassphrase: ${{ parameters.authGPGPassphrase }} + authGPGPath: ${{ parameters.authGPGPath }} + authGPGKey: ${{ parameters.authGPGKey }} + pathGPGConfiguredHome: /build/.gnupg + pathGPGHome: $(Build.StagingDirectory)/.gnupg + stepsPost: + - bash: | + set -e + rm -rf $(Build.StagingDirectory)/.gnupg + - job: package_arm64 displayName: Linux debs (arm64) dependsOn: [] @@ -170,6 +183,7 @@ jobs: artifactName: "bazel.release" itemPattern: "bazel.release/arm64/bin/*" targetPath: $(Build.StagingDirectory) + - template: ../bazel.yml parameters: managedAgent: false @@ -177,6 +191,18 @@ jobs: rbe: false artifactSuffix: ".arm64" bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" + stepsPre: + - template: ../gpg.yml + parameters: + authGPGPassphrase: ${{ parameters.authGPGPassphrase }} + authGPGPath: ${{ parameters.authGPGPath }} + authGPGKey: ${{ parameters.authGPGKey }} + pathGPGConfiguredHome: /build/.gnupg + pathGPGHome: $(Build.StagingDirectory)/.gnupg + stepsPost: + - bash: | + set -e + rm -rf $(Build.StagingDirectory)/.gnupg - job: success dependsOn: ["docker", "package_x64", "package_arm64"] @@ -277,44 +303,19 @@ jobs: artifactName: "bazel.release.arm64" itemPattern: "bazel.release.arm64/bin/*" targetPath: $(Build.StagingDirectory) - - - task: DownloadSecureFile@1 - name: MaintainerGPGKey - displayName: 'Download maintainer GPG key' - inputs: - # NB: This is the path to the key and **must** be set on a per-pipeline basis - secureFile: '${{ parameters.authGPGKey }}' - + - template: ../gpg.yml + parameters: + authGPGPassphrase: ${{ parameters.authGPGPassphrase }} + authGPGPath: ${{ parameters.authGPGPath }} + authGPGKey: ${{ parameters.authGPGKey }} + pathGPGConfiguredHome: /build/.gnupg + pathGPGHome: $(Build.StagingDirectory)/.gnupg - script: ./ci/run_envoy_docker.sh './ci/do_ci.sh publish' displayName: "Publish release to Github" workingDirectory: $(Build.SourcesDirectory) env: AZP_BRANCH: $(Build.SourceBranch) GITHUB_TOKEN: ${{ parameters.authGithub }} - - # TODO(phlax): combine this with publish step - bash: | set -e - - VERSION="$(cat VERSION.txt)" - - mkdir -p linux/amd64 linux/arm64 publish - - # linux/amd64 - tar xf $(Build.StagingDirectory)/bazel.release/release.tar.zst -C ./linux/amd64 - cp -a linux/amd64/envoy "publish/envoy-${VERSION}-linux-x86_64" - cp -a linux/amd64/envoy-contrib "publish/envoy-contrib-${VERSION}-linux-x86_64" - - # linux/arm64 - tar xf $(Build.StagingDirectory)/bazel.release.arm64/release.tar.zst -C ./linux/arm64 - cp -a linux/arm64/envoy "publish/envoy-${VERSION}-linux-aarch_64" - cp -a linux/arm64/envoy-contrib "publish/envoy-contrib-${VERSION}-linux-aarch_64" - - echo "$MAINTAINER_GPG_KEY_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --import "$MAINTAINER_GPG_KEY_PATH" - - ci/publish_github_assets.sh "v${VERSION}" "${PWD}/publish" - workingDirectory: $(Build.SourcesDirectory) - env: - GITHUB_TOKEN: ${{ parameters.authGithub }} - MAINTAINER_GPG_KEY_PASSPHRASE: ${{ parameters.authGPGPassphrase }} - MAINTAINER_GPG_KEY_PATH: ${{ parameters.authGPGPath }} + rm -rf $(Build.StagingDirectory)/.gnupg diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index c08c96310b77..1023ce7eefd2 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -69,6 +69,9 @@ stages: cacheTestResults: ${{ parameters.cacheTestResults }} authGithub: $(GitHubPublicRepoOnlyAccessToken) authGCP: $(GcpServiceAccountKey) + authGPGPassphrase: $(MaintainerGPGKeyPassphrase) + authGPGKey: $(MaintainerGPGKeySecureFileDownloadPath) + authGPGPath: $(MaintainerGPGKey.secureFilePath) bucketGCP: $(GcsArtifactBucket) checkDeps: variables['CHECK_DEPS'] diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 9f9554901cd6..ba2c6b93d1da 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -268,18 +268,6 @@ case $CI_TARGET in fi bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/zstd -- --stdout -d "$ENVOY_RELEASE_TARBALL" | tar xfO - envoy > distribution/custom/envoy - # By default the packages will be signed by the first available key. - # If there is no key available, a throwaway key is created - # and the packages signed with it, for the purpose of testing only. - if ! gpg --list-secret-keys "*"; then - export PACKAGES_MAINTAINER_NAME="Envoy CI" - export PACKAGES_MAINTAINER_EMAIL="envoy-ci@for.testing.only" - BAZEL_BUILD_OPTIONS+=( - "--action_env=PACKAGES_GEN_KEY=1" - "--action_env=PACKAGES_MAINTAINER_NAME" - "--action_env=PACKAGES_MAINTAINER_EMAIL") - fi - # Build the packages bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c opt --//distribution:envoy-binary=//distribution:custom/envoy //distribution:packages.tar.gz if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then @@ -591,6 +579,22 @@ case $CI_TARGET in # from the current commit (as this only happens on non-PRs we are safe from merges) BUILD_SHA="$(git rev-parse HEAD)" bazel run "${BAZEL_BUILD_OPTIONS[@]}" @envoy_repo//:publish -- --publish-commitish="$BUILD_SHA" + + # TODO(phlax): move this to pytooling + mkdir -p linux/amd64 linux/arm64 publish + + # linux/amd64 + tar xf /build/bazel.release/release.tar.zst -C ./linux/amd64 + cp -a linux/amd64/envoy "publish/envoy-${version}-linux-x86_64" + cp -a linux/amd64/envoy-contrib "publish/envoy-contrib-${version}-linux-x86_64" + + # linux/arm64 + tar xf /build/bazel.release.arm64/release.tar.zst -C ./linux/arm64 + cp -a linux/arm64/envoy "publish/envoy-${version}-linux-aarch_64" + cp -a linux/arm64/envoy-contrib "publish/envoy-contrib-${version}-linux-aarch_64" + + "${ENVOY_SRCDIR}ci/publish_github_assets.sh" "v${version}" "${PWD}/publish" + exit 0 fi fi diff --git a/distribution/packages.bzl b/distribution/packages.bzl index d788330cbad1..dacd54829fb8 100644 --- a/distribution/packages.bzl +++ b/distribution/packages.bzl @@ -52,22 +52,12 @@ def envoy_pkg_distros( native.genrule( name = name, cmd = """ - SIGNING_ARGS=() \ - && if [[ -n $${PACKAGES_GEN_KEY+x} ]]; then \ - SIGNING_ARGS+=("--gen-key"); \ - fi \ - && if [[ -n $${PACKAGES_MAINTAINER_NAME+x} ]]; then \ - SIGNING_ARGS+=("--maintainer-name" "$${PACKAGES_MAINTAINER_NAME}"); \ - fi \ - && if [[ -n $${PACKAGES_MAINTAINER_EMAIL+x} ]]; then \ - SIGNING_ARGS+=("--maintainer-email" "$${PACKAGES_MAINTAINER_EMAIL}"); \ - fi \ - && $(location //tools/distribution:sign) \ + $(location //tools/distribution:sign) \ --out $@ \ - "$${SIGNING_ARGS[@]}" \ $(location :distro_packages) """, outs = ["%s.tar.gz" % name], srcs = [":distro_packages"], tools = ["//tools/distribution:sign"], + tags = ["no-remote"], ) From 7639f15a84fffa44b89aae5b5e09d70b42770d05 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 4 May 2023 14:47:33 +0100 Subject: [PATCH 107/740] mobile/ci: Trigger run with .bazelrc (and fix) (#27172) Signed-off-by: Ryan Northey --- mobile/.bazelrc | 1 + mobile/tools/should_run_ci.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mobile/.bazelrc b/mobile/.bazelrc index 5168401fb5eb..df4f01c554d9 100644 --- a/mobile/.bazelrc +++ b/mobile/.bazelrc @@ -237,6 +237,7 @@ build:remote-ci-linux-coverage --strategy=CoverageReport=local,remote # TODO(lfpino): Reference upstream Bazel issue here on the incompatibility of remote caching and LLVM coverage. build:remote-ci-linux-coverage --noremote_accept_cached build:remote-ci-linux-coverage --config=remote-ci-common +build:remote-ci-linux-coverage --build_runfile_links # IPv6 tests fail on CI build:remote-ci-linux-coverage --test_env=ENVOY_IP_TEST_VERSIONS=v4only ############################################################################# diff --git a/mobile/tools/should_run_ci.sh b/mobile/tools/should_run_ci.sh index 2476a9d7e928..1ee2622db98f 100755 --- a/mobile/tools/should_run_ci.sh +++ b/mobile/tools/should_run_ci.sh @@ -52,6 +52,8 @@ if grep -q "^mobile/" <<< "$changed_files"; then success "mobile" elif grep -q "^bazel/repository_locations\.bzl" <<< "$changed_files"; then success "bazel/repository_locations.bzl" +elif grep -q "^\.bazelrc" <<< "$changed_files"; then + success ".bazelrc" elif grep -q "^\.github/workflows/mobile-*" <<< "$changed_files"; then success "GitHub Workflows" else From a484ae183d3690de6d3cf9037d43d451dfe686be Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 4 May 2023 15:32:36 +0100 Subject: [PATCH 108/740] ci: Dont run scheduled CodeQL in non-Envoy repos (#27180) Signed-off-by: Ryan Northey --- .github/workflows/codeql-daily.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index cdf39487c846..7132d6b49656 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -16,6 +16,7 @@ jobs: # CodeQL runs on ubuntu-20.04 runs-on: ubuntu-20.04 + if: github.repository == 'envoyproxy/envoy' steps: - name: Checkout repository From a0b07e302ba8aac610764c7d13820cf70ad0c45c Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 4 May 2023 15:38:04 +0100 Subject: [PATCH 109/740] proto_format: Use char (not text) len when encoding diff (#27182) Signed-off-by: Ryan Northey --- tools/proto_format/proto_sync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/proto_format/proto_sync.py b/tools/proto_format/proto_sync.py index 8d8f2364e2fa..d3aa72667745 100755 --- a/tools/proto_format/proto_sync.py +++ b/tools/proto_format/proto_sync.py @@ -35,11 +35,14 @@ def blob_hash(self, stream, size): nread += len(data) hasher.update(data) if nread != size: + # TODO(phlax): move this to pytooling asap. + # This would not pass type checking `BytesIO` has no `stream.name` raise ValueError('%s: expected %u bytes, found %u bytes' % (stream.name, size, nread)) return hasher.hexdigest()[:10] def string_hash(self, text): - return self.blob_hash(io.BytesIO(text.encode("utf-8")), len(text)) + chars = text.encode("utf-8") + return self.blob_hash(io.BytesIO(chars), len(chars)) def diff(self, t1, t2, path, mode, remove=False): fhash = self.string_hash(t1) From ca6ff33b91f9ef16f24ec8f1ae9d0dfa3a4e7eb2 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Thu, 4 May 2023 10:55:59 -0400 Subject: [PATCH 110/740] Fix flaky tests in ShadowPolicyIntegrationTest (#27160) Signed-off-by: Yan Avlasov --- test/integration/shadow_policy_integration_test.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/integration/shadow_policy_integration_test.cc b/test/integration/shadow_policy_integration_test.cc index d34796163a67..190bff3858dc 100644 --- a/test/integration/shadow_policy_integration_test.cc +++ b/test/integration/shadow_policy_integration_test.cc @@ -762,14 +762,13 @@ TEST_P(ShadowPolicyIntegrationTest, RequestMirrorPolicyWithShadowBackpressure) { cleanupUpstreamAndDownstream(); - EXPECT_EQ(test_server_->counter("http.config_test.downstream_flow_control_paused_reading_total") - ->value(), - 1); - EXPECT_EQ(test_server_->counter("cluster.cluster_0.upstream_cx_total")->value(), 1); - EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1); + test_server_->waitForCounterEq("http.config_test.downstream_flow_control_paused_reading_total", + 1); + test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_total", 1); + test_server_->waitForCounterEq("cluster.cluster_1.upstream_cx_total", 1); // Main cluster saw no reset; shadow cluster saw remote reset. - EXPECT_EQ(test_server_->counter("cluster.cluster_0.upstream_rq_completed")->value(), 1); - EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_rq_completed")->value(), 1); + test_server_->waitForCounterEq("cluster.cluster_0.upstream_rq_completed", 1); + test_server_->waitForCounterEq("cluster.cluster_1.upstream_rq_completed", 1); } // Test request mirroring / shadowing with the cluster name in policy. From 0478eba2a495027bf6ac8e787c42e2f5b9eb553b Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Thu, 4 May 2023 10:59:01 -0400 Subject: [PATCH 111/740] Load Shed Point: HTTP1 Server Codec Dispatch (#27120) * Plumb loadshed point for HTTP1 codec server dispatch and add runtime flag and test for codec error after 1xx header encoding for HTTP1. Signed-off-by: Kevin Baichoo --- changelogs/current.yaml | 11 + .../overload_manager/overload_manager.rst | 5 + source/common/http/conn_manager_config.h | 5 +- source/common/http/conn_manager_impl.cc | 31 ++- source/common/http/conn_manager_impl.h | 4 + source/common/http/conn_manager_utility.cc | 5 +- source/common/http/conn_manager_utility.h | 3 +- source/common/http/http1/BUILD | 1 + source/common/http/http1/codec_impl.cc | 29 ++- source/common/http/http1/codec_impl.h | 6 +- source/common/http/status.cc | 12 + source/common/http/status.h | 7 + source/common/runtime/runtime_features.cc | 1 + .../network/http_connection_manager/config.cc | 12 +- .../network/http_connection_manager/config.h | 3 +- source/server/admin/admin.cc | 6 +- source/server/admin/admin.h | 3 +- test/common/http/BUILD | 1 + test/common/http/codec_impl_fuzz_test.cc | 4 +- .../http/conn_manager_impl_fuzz_test.cc | 2 +- test/common/http/conn_manager_impl_test_2.cc | 34 +++ .../common/http/conn_manager_impl_test_base.h | 2 +- test/common/http/http1/BUILD | 2 + test/common/http/http1/codec_impl_test.cc | 68 +++++- .../http/http1/http1_connection_fuzz_test.cc | 5 +- test/common/http/status_test.cc | 20 ++ test/integration/BUILD | 22 +- test/integration/fake_upstream.cc | 3 +- test/integration/fake_upstream.h | 2 + test/integration/overload_integration_test.cc | 225 ++++++++++++++++++ test/mocks/http/mocks.h | 8 +- 31 files changed, 492 insertions(+), 50 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 1f3ff79085a7..32208dc7ab0e 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -20,6 +20,14 @@ minor_behavior_changes: - area: custom response change: | The filter now traverses matchers from most specific to least specific per filter config till a match is found for the response. +- area: http1 + change: | + The HTTP1 server-side codec no longer considers encoding 1xx headers as + starting the response. This allows the codec to raise protocol errors, + sending detailed local replies instead of just closing the connection. This + behavior can be reverted by setting runtime flag + ``envoy.reloadable_features.http1_allow_codec_error_response_after_1xx_headers`` + to false. - area: uhv change: | Preserve case of %-encoded triplets in the default header validator. This behavior can be reverted by setting runtime flag @@ -80,6 +88,9 @@ new_features: change: | added load shed point ``envoy.load_shed_points.http_connection_manager_decode_headers`` that rejects new http streams by sending a local reply. +- area: load shed point + change: | + added load shed point ``envoy.load_shed_points.http1_server_abort_dispatch`` that rejects HTTP1 server processing of requests. - area: matchers change: | Added :ref:`RuntimeFraction ` input diff --git a/docs/root/configuration/operations/overload_manager/overload_manager.rst b/docs/root/configuration/operations/overload_manager/overload_manager.rst index b2629e7381db..c4c088414826 100644 --- a/docs/root/configuration/operations/overload_manager/overload_manager.rst +++ b/docs/root/configuration/operations/overload_manager/overload_manager.rst @@ -156,6 +156,11 @@ The following core load shed points are supported: right after the http codec has finished parsing headers but before the :ref:`HTTP Filter Chain is instantiated `. + * - envoy.load_shed_points.http1_server_abort_dispatch + - Envoy will reject processing HTTP1 at the codec level. If a response has + not yet started, Envoy will send a local reply. Envoy will then close the + connection. + .. _config_overload_manager_reducing_timeouts: Reducing timeouts diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index d6d23cf2a419..427131f4b14a 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -228,11 +228,14 @@ class ConnectionManagerConfig { * @param connection supplies the owning connection. * @param data supplies the currently available read data. * @param callbacks supplies the callbacks to install into the codec. + * @param overload_manager supplies overload manager that the codec can + * integrate with. * @return a codec or nullptr if no codec can be created. */ virtual ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& data, - ServerConnectionCallbacks& callbacks) PURE; + ServerConnectionCallbacks& callbacks, + Server::OverloadManager& overload_manager) PURE; /** * @return DateProvider& the date provider to use for diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index ac7b110a05bf..041d002760a5 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -98,6 +98,7 @@ ConnectionManagerImpl::ConnectionManagerImpl(ConnectionManagerConfig& config, drain_close_(drain_close), user_agent_(http_context.userAgentContext()), random_generator_(random_generator), runtime_(runtime), local_info_(local_info), cluster_manager_(cluster_manager), listener_stats_(config_.listenerStats()), + overload_manager_(overload_manager), overload_state_(overload_manager.getThreadLocalOverloadState()), accept_new_http_stream_(overload_manager.getLoadShedPoint( "envoy.load_shed_points.http_connection_manager_decode_headers")), @@ -393,21 +394,30 @@ RequestDecoder& ConnectionManagerImpl::newStream(ResponseEncoder& response_encod return **streams_.begin(); } -void ConnectionManagerImpl::handleCodecError(absl::string_view error) { +void ConnectionManagerImpl::handleCodecErrorImpl(absl::string_view error, absl::string_view details, + StreamInfo::ResponseFlag response_flag) { ENVOY_CONN_LOG(debug, "dispatch error: {}", read_callbacks_->connection(), error); - read_callbacks_->connection().streamInfo().setResponseFlag( - StreamInfo::ResponseFlag::DownstreamProtocolError); + read_callbacks_->connection().streamInfo().setResponseFlag(response_flag); // HTTP/1.1 codec has already sent a 400 response if possible. HTTP/2 codec has already sent // GOAWAY. - doConnectionClose(Network::ConnectionCloseType::FlushWriteAndDelay, - StreamInfo::ResponseFlag::DownstreamProtocolError, - absl::StrCat("codec_error:", StringUtil::replaceAllEmptySpace(error))); + doConnectionClose(Network::ConnectionCloseType::FlushWriteAndDelay, response_flag, details); +} + +void ConnectionManagerImpl::handleCodecError(absl::string_view error) { + handleCodecErrorImpl(error, absl::StrCat("codec_error:", StringUtil::replaceAllEmptySpace(error)), + StreamInfo::ResponseFlag::DownstreamProtocolError); +} + +void ConnectionManagerImpl::handleCodecOverloadError(absl::string_view error) { + handleCodecErrorImpl(error, + absl::StrCat("overload_error:", StringUtil::replaceAllEmptySpace(error)), + StreamInfo::ResponseFlag::OverloadManager); } void ConnectionManagerImpl::createCodec(Buffer::Instance& data) { ASSERT(!codec_); - codec_ = config_.createCodec(read_callbacks_->connection(), data, *this); + codec_ = config_.createCodec(read_callbacks_->connection(), data, *this, overload_manager_); switch (codec_->protocol()) { case Protocol::Http3: @@ -445,6 +455,13 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool stats_.named_.downstream_cx_protocol_error_.inc(); handleCodecError(status.message()); return Network::FilterStatus::StopIteration; + } else if (isEnvoyOverloadError(status)) { + // The other codecs aren't wired to send this status. + ASSERT(codec_->protocol() < Protocol::Http2, + "Expected only HTTP1.1 and below to send overload error."); + stats_.named_.downstream_rq_overload_close_.inc(); + handleCodecOverloadError(status.message()); + return Network::FilterStatus::StopIteration; } ASSERT(status.ok()); diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 94c5228fbcdb..fa693a067e65 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -535,7 +535,10 @@ class ConnectionManagerImpl : Logger::Loggable, void onDrainTimeout(); void startDrainSequence(); Tracing::Tracer& tracer() { return *config_.tracer(); } + void handleCodecErrorImpl(absl::string_view error, absl::string_view details, + StreamInfo::ResponseFlag response_flag); void handleCodecError(absl::string_view error); + void handleCodecOverloadError(absl::string_view error); void doConnectionClose(absl::optional close_type, absl::optional response_flag, absl::string_view details); @@ -564,6 +567,7 @@ class ConnectionManagerImpl : Logger::Loggable, Upstream::ClusterManager& cluster_manager_; Network::ReadFilterCallbacks* read_callbacks_{}; ConnectionManagerListenerStats& listener_stats_; + Server::OverloadManager& overload_manager_; Server::ThreadLocalOverloadState& overload_state_; Server::LoadShedPoint* accept_new_http_stream_{nullptr}; // References into the overload manager thread local state map. Using these lets us avoid a diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index 8498f16ed270..03237b518b40 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -61,7 +61,8 @@ ServerConnectionPtr ConnectionManagerUtility::autoCreateCodec( const envoy::config::core::v3::Http2ProtocolOptions& http2_options, uint32_t max_request_headers_kb, uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction - headers_with_underscores_action) { + headers_with_underscores_action, + Server::OverloadManager& overload_manager) { if (determineNextProtocol(connection, data) == Utility::AlpnNames::get().Http2) { Http2::CodecStats& stats = Http2::CodecStats::atomicGet(http2_codec_stats, scope); return std::make_unique( @@ -71,7 +72,7 @@ ServerConnectionPtr ConnectionManagerUtility::autoCreateCodec( Http1::CodecStats& stats = Http1::CodecStats::atomicGet(http1_codec_stats, scope); return std::make_unique( connection, stats, callbacks, http1_settings, max_request_headers_kb, - max_request_headers_count, headers_with_underscores_action); + max_request_headers_count, headers_with_underscores_action, overload_manager); } } diff --git a/source/common/http/conn_manager_utility.h b/source/common/http/conn_manager_utility.h index 4eb91de37037..a93d53706914 100644 --- a/source/common/http/conn_manager_utility.h +++ b/source/common/http/conn_manager_utility.h @@ -45,7 +45,8 @@ class ConnectionManagerUtility { const envoy::config::core::v3::Http2ProtocolOptions& http2_options, uint32_t max_request_headers_kb, uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction - headers_with_underscores_action); + headers_with_underscores_action, + Server::OverloadManager& overload_manager); /* The result after calling mutateRequestHeaders(), containing the final remote address. Note that * an extension used for detecting the original IP of the request might decide it should be diff --git a/source/common/http/http1/BUILD b/source/common/http/http1/BUILD index 3a2ce1e39af6..bdd794dc0723 100644 --- a/source/common/http/http1/BUILD +++ b/source/common/http/http1/BUILD @@ -43,6 +43,7 @@ envoy_cc_library( "//envoy/http:codec_interface", "//envoy/http:header_map_interface", "//envoy/network:connection_interface", + "//envoy/server/overload:overload_manager_interface", "//source/common/buffer:buffer_lib", "//source/common/buffer:watermark_buffer_lib", "//source/common/common:assert_lib", diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index 4b36728f02c6..c9f06c1acf55 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -130,6 +130,11 @@ void StreamEncoderImpl::encodeFormattedHeader(absl::string_view key, absl::strin void ResponseEncoderImpl::encode1xxHeaders(const ResponseHeaderMap& headers) { ASSERT(HeaderUtility::isSpecial1xx(headers)); encodeHeaders(headers, false); + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.http1_allow_codec_error_response_after_1xx_headers")) { + // Don't consider 100-continue responses as the actual response. + started_response_ = false; + } } void StreamEncoderImpl::encodeHeadersBase(const RequestOrResponseHeaderMap& headers, @@ -1048,7 +1053,8 @@ ServerConnectionImpl::ServerConnectionImpl( const Http1Settings& settings, uint32_t max_request_headers_kb, const uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction - headers_with_underscores_action) + headers_with_underscores_action, + Server::OverloadManager& overload_manager) : ConnectionImpl(connection, stats, settings, MessageType::Request, max_request_headers_kb, max_request_headers_count), callbacks_(callbacks), @@ -1059,7 +1065,9 @@ ServerConnectionImpl::ServerConnectionImpl( [&]() -> void { this->onBelowLowWatermark(); }, [&]() -> void { this->onAboveHighWatermark(); }, []() -> void { /* TODO(adisuissa): handle overflow watermark */ })), - headers_with_underscores_action_(headers_with_underscores_action) { + headers_with_underscores_action_(headers_with_underscores_action), + abort_dispatch_( + overload_manager.getLoadShedPoint("envoy.load_shed_points.http1_server_abort_dispatch")) { owned_output_buffer_->setWatermarks(connection.bufferLimit()); // Inform parent output_buffer_ = owned_output_buffer_.get(); @@ -1240,6 +1248,11 @@ void ServerConnectionImpl::onBody(Buffer::Instance& data) { } Http::Status ServerConnectionImpl::dispatch(Buffer::Instance& data) { + if (abort_dispatch_ != nullptr && abort_dispatch_->shouldShedLoad()) { + RETURN_IF_ERROR(sendOverloadError()); + return envoyOverloadError("Aborting Server Dispatch"); + } + if (active_request_ != nullptr && active_request_->remote_complete_) { // Eagerly read disable the connection if the downstream is sending pipelined requests as we // serially process them. Reading from the connection will be re-enabled after the active @@ -1298,6 +1311,18 @@ void ServerConnectionImpl::onResetStream(StreamResetReason reason) { } } +Status ServerConnectionImpl::sendOverloadError() { + const bool latched_dispatching = dispatching_; + + // The codec might be in the early stages of server dispatching where this isn't yet + // flipped to true. + dispatching_ = true; + error_code_ = Http::Code::InternalServerError; + auto status = sendProtocolError(Envoy::StreamInfo::ResponseCodeDetails::get().Overload); + dispatching_ = latched_dispatching; + return status; +} + Status ServerConnectionImpl::sendProtocolError(absl::string_view details) { // We do this here because we may get a protocol error before we have a logical stream. if (active_request_ == nullptr) { diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index 437c64d5ad37..4343f4f96176 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -11,6 +11,7 @@ #include "envoy/config/core/v3/protocol.pb.h" #include "envoy/http/codec.h" #include "envoy/network/connection.h" +#include "envoy/server/overload/overload_manager.h" #include "source/common/buffer/watermark_buffer.h" #include "source/common/common/assert.h" @@ -463,7 +464,8 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { ServerConnectionCallbacks& callbacks, const Http1Settings& settings, uint32_t max_request_headers_kb, const uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction - headers_with_underscores_action); + headers_with_underscores_action, + Server::OverloadManager& overload_manager); bool supportsHttp10() override { return codec_settings_.accept_http_10_; } protected: @@ -522,6 +524,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { void onBody(Buffer::Instance& data) override; void onResetStream(StreamResetReason reason) override; Status sendProtocolError(absl::string_view details) override; + Status sendOverloadError(); void onAboveHighWatermark() override; void onBelowLowWatermark() override; HeaderMap& headersOrTrailers() override { @@ -571,6 +574,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { // The action to take when a request header name contains underscore characters. const envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headers_with_underscores_action_; + Server::LoadShedPoint* abort_dispatch_{nullptr}; }; /** diff --git a/source/common/http/status.cc b/source/common/http/status.cc index c3500fcae9f9..967d8a6ae640 100644 --- a/source/common/http/status.cc +++ b/source/common/http/status.cc @@ -25,6 +25,8 @@ absl::string_view statusCodeToString(StatusCode code) { return "CodecClientError"; case StatusCode::InboundFramesWithEmptyPayload: return "InboundFramesWithEmptyPayloadError"; + case StatusCode::EnvoyOverloadError: + return "EnvoyOverloadError"; } return ""; } @@ -116,6 +118,12 @@ Status inboundFramesWithEmptyPayloadError() { return status; } +Status envoyOverloadError(absl::string_view message) { + absl::Status status(absl::StatusCode::kInternal, message); + storePayload(status, EnvoyStatusPayload(StatusCode::EnvoyOverloadError)); + return status; +} + // Methods for checking and extracting error information StatusCode getStatusCode(const Status& status) { return status.ok() ? StatusCode::Ok : getPayload(status).status_code_; @@ -148,5 +156,9 @@ bool isInboundFramesWithEmptyPayloadError(const Status& status) { return getStatusCode(status) == StatusCode::InboundFramesWithEmptyPayload; } +bool isEnvoyOverloadError(const Status& status) { + return getStatusCode(status) == StatusCode::EnvoyOverloadError; +} + } // namespace Http } // namespace Envoy diff --git a/source/common/http/status.h b/source/common/http/status.h index 97c7dac96ca1..858311b1d9bf 100644 --- a/source/common/http/status.h +++ b/source/common/http/status.h @@ -74,6 +74,11 @@ enum class StatusCode : int { * Indicates that peer sent too many consecutive DATA frames with empty payload. */ InboundFramesWithEmptyPayload = 5, + + /** + * Indicates that Envoy is overloaded and may shed load. + */ + EnvoyOverloadError = 6, }; using Status = absl::Status; @@ -94,6 +99,7 @@ Status bufferFloodError(absl::string_view message); Status prematureResponseError(absl::string_view message, Http::Code http_code); Status codecClientError(absl::string_view message); Status inboundFramesWithEmptyPayloadError(); +Status envoyOverloadError(absl::string_view message); /** * Returns Envoy::StatusCode of the given status object. @@ -109,6 +115,7 @@ ABSL_MUST_USE_RESULT bool isBufferFloodError(const Status& status); ABSL_MUST_USE_RESULT bool isPrematureResponseError(const Status& status); ABSL_MUST_USE_RESULT bool isCodecClientError(const Status& status); ABSL_MUST_USE_RESULT bool isInboundFramesWithEmptyPayloadError(const Status& status); +ABSL_MUST_USE_RESULT bool isEnvoyOverloadError(const Status& status); /** * Returns Http::Code value of the PrematureResponseError status. diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index cd245cfc0b76..3960237d9609 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -41,6 +41,7 @@ RUNTIME_GUARD(envoy_reloadable_features_enable_update_listener_socket_options); RUNTIME_GUARD(envoy_reloadable_features_expand_agnostic_stream_lifetime); RUNTIME_GUARD(envoy_reloadable_features_finish_reading_on_decode_trailers); RUNTIME_GUARD(envoy_reloadable_features_format_ports_as_numbers); +RUNTIME_GUARD(envoy_reloadable_features_http1_allow_codec_error_response_after_1xx_headers); RUNTIME_GUARD(envoy_reloadable_features_http2_decode_metadata_with_quiche); RUNTIME_GUARD(envoy_reloadable_features_http2_validate_authority_with_quiche); RUNTIME_GUARD(envoy_reloadable_features_http_allow_partial_urls_in_referer); diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index da548397fe13..3499b90b6beb 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -653,16 +653,15 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( } } -Http::ServerConnectionPtr -HttpConnectionManagerConfig::createCodec(Network::Connection& connection, - const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks) { +Http::ServerConnectionPtr HttpConnectionManagerConfig::createCodec( + Network::Connection& connection, const Buffer::Instance& data, + Http::ServerConnectionCallbacks& callbacks, Server::OverloadManager& overload_manager) { switch (codec_type_) { case CodecType::HTTP1: return std::make_unique( connection, Http::Http1::CodecStats::atomicGet(http1_codec_stats_, context_.scope()), callbacks, http1_settings_, maxRequestHeadersKb(), maxRequestHeadersCount(), - headersWithUnderscoresAction()); + headersWithUnderscoresAction(), overload_manager); case CodecType::HTTP2: return std::make_unique( connection, callbacks, @@ -681,7 +680,8 @@ HttpConnectionManagerConfig::createCodec(Network::Connection& connection, return Http::ConnectionManagerUtility::autoCreateCodec( connection, data, callbacks, context_.scope(), context_.api().randomGenerator(), http1_codec_stats_, http2_codec_stats_, http1_settings_, http2_options_, - maxRequestHeadersKb(), maxRequestHeadersCount(), headersWithUnderscoresAction()); + maxRequestHeadersKb(), maxRequestHeadersCount(), headersWithUnderscoresAction(), + overload_manager); } PANIC_DUE_TO_CORRUPT_ENUM; } diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index 0f3e014dd8d9..d1237c8e710b 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -157,7 +157,8 @@ class HttpConnectionManagerConfig : Logger::Loggable, } Http::ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks) override; + Http::ServerConnectionCallbacks& callbacks, + Server::OverloadManager& overload_manager) override; Http::DateProvider& dateProvider() override { return date_provider_; } std::chrono::milliseconds drainTimeout() const override { return drain_timeout_; } FilterChainFactory& filterFactory() override { return *this; } diff --git a/source/server/admin/admin.cc b/source/server/admin/admin.cc index 3e5bea114e49..1025bd5e44e1 100644 --- a/source/server/admin/admin.cc +++ b/source/server/admin/admin.cc @@ -232,13 +232,15 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server, Http::ServerConnectionPtr AdminImpl::createCodec(Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks) { + Http::ServerConnectionCallbacks& callbacks, + Server::OverloadManager& overload_manager) { return Http::ConnectionManagerUtility::autoCreateCodec( connection, data, callbacks, *server_.stats().rootScope(), server_.api().randomGenerator(), http1_codec_stats_, http2_codec_stats_, Http::Http1Settings(), ::Envoy::Http2::Utility::initializeAndValidateOptions( envoy::config::core::v3::Http2ProtocolOptions()), - maxRequestHeadersKb(), maxRequestHeadersCount(), headersWithUnderscoresAction()); + maxRequestHeadersKb(), maxRequestHeadersCount(), headersWithUnderscoresAction(), + overload_manager); } bool AdminImpl::createNetworkFilterChain(Network::Connection& connection, diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 00589f2526c5..d29f93e1a1ad 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -129,7 +129,8 @@ class AdminImpl : public Admin, } Http::ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks) override; + Http::ServerConnectionCallbacks& callbacks, + Server::OverloadManager& overload_manager) override; Http::DateProvider& dateProvider() override { return date_provider_; } std::chrono::milliseconds drainTimeout() const override { return std::chrono::milliseconds(100); } Http::FilterChainFactory& filterFactory() override { return *this; } diff --git a/test/common/http/BUILD b/test/common/http/BUILD index f19c28c0f951..5e8e183e9c68 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -112,6 +112,7 @@ envoy_proto_library( "//test/fuzz:utility_lib", "//test/mocks/http:http_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/server:overload_manager_mocks", "//test/test_common:test_runtime_lib", ], ) diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index 1428e96886f4..7c024c98a827 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -23,6 +23,7 @@ #include "test/fuzz/utility.h" #include "test/mocks/http/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/server/overload_manager.h" #include "test/test_common/test_runtime.h" #include "gmock/gmock.h" @@ -554,6 +555,7 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi NiceMock server_connection; NiceMock server_callbacks; NiceMock random; + NiceMock overload_manager_; NiceMock conn_manager_config; uint32_t max_request_headers_kb = Http::DEFAULT_MAX_REQUEST_HEADERS_KB; uint32_t max_request_headers_count = Http::DEFAULT_MAX_HEADERS_COUNT; @@ -607,7 +609,7 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi server = std::make_unique( server_connection, Http1::CodecStats::atomicGet(http1_stats, scope), server_callbacks, server_http1settings, max_request_headers_kb, max_request_headers_count, - headers_with_underscores_action); + headers_with_underscores_action, overload_manager_); } // We track whether the connection should be closed for HTTP/1, since stream resets imply diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index f88dd9c62881..0943f9b8d214 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -133,7 +133,7 @@ class FuzzConfig : public ConnectionManagerConfig { return access_log_flush_interval_; } ServerConnectionPtr createCodec(Network::Connection&, const Buffer::Instance&, - ServerConnectionCallbacks&) override { + ServerConnectionCallbacks&, Server::OverloadManager&) override { return ServerConnectionPtr{codec_}; } DateProvider& dateProvider() override { return date_provider_; } diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index ca9bfa2d1699..d4cb41cae9ec 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -263,6 +263,40 @@ TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { StreamInfo::ResponseFlag::DownstreamProtocolError)); } +TEST_F(HttpConnectionManagerImplTest, EnvoyOverloadError) { + std::shared_ptr log_handler = + std::make_shared>(); + access_logs_ = {log_handler}; + setup(false, ""); + ASSERT_EQ(0U, stats_.named_.downstream_rq_overload_close_.value()); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> Http::Status { + conn_manager_->newStream(response_encoder_); + return envoyOverloadError("Envoy Overloaded"); + })); + + EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)).Times(2); + EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); + + // Overload should result in local reply followed by abortive close. + EXPECT_CALL(filter_callbacks_.connection_, + close(Network::ConnectionCloseType::FlushWriteAndDelay, _)); + + EXPECT_CALL(*log_handler, log(_, _, _, _, _)) + .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, + const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + ASSERT_TRUE(stream_info.responseCodeDetails().has_value()); + EXPECT_EQ("overload_error:Envoy_Overloaded", stream_info.responseCodeDetails().value()); + })); + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + + EXPECT_TRUE(filter_callbacks_.connection_.streamInfo().hasResponseFlag( + StreamInfo::ResponseFlag::OverloadManager)); + EXPECT_EQ(1U, stats_.named_.downstream_rq_overload_close_.value()); +} + TEST_F(HttpConnectionManagerImplTest, IdleTimeoutNoCodec) { // Not used in the test. delete codec_; diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index 92f0da9eeb64..5c13b13e584d 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -66,7 +66,7 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { return access_log_flush_interval_; } ServerConnectionPtr createCodec(Network::Connection&, const Buffer::Instance&, - ServerConnectionCallbacks&) override { + ServerConnectionCallbacks&, Server::OverloadManager&) override { return ServerConnectionPtr{codec_}; } DateProvider& dateProvider() override { return date_provider_; } diff --git a/test/common/http/http1/BUILD b/test/common/http/http1/BUILD index 40482870a042..c7d409367281 100644 --- a/test/common/http/http1/BUILD +++ b/test/common/http/http1/BUILD @@ -36,6 +36,7 @@ envoy_cc_test( "//test/mocks/local_info:local_info_mocks", "//test/mocks/network:network_mocks", "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/server:overload_manager_mocks", "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", @@ -82,6 +83,7 @@ envoy_cc_fuzz_test( "//test/fuzz:utility_lib", "//test/mocks/http:http_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/server:overload_manager_mocks", "//test/test_common:test_runtime_lib", ], ) diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 06d24d86b8b6..821e446959d9 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -20,6 +20,7 @@ #include "test/mocks/buffer/mocks.h" #include "test/mocks/http/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/server/overload_manager.h" #include "test/test_common/logging.h" #include "test/test_common/printers.h" #include "test/test_common/test_runtime.h" @@ -151,6 +152,7 @@ class Http1CodecTestBase : public testing::TestWithParam { NiceMock codec_settings_; Stats::TestUtil::TestStore store_; Http::Http1::CodecStats::AtomicPtr http1_codec_stats_; + NiceMock overload_manager_; }; class Http1ServerConnectionImplTest : public Http1CodecTestBase { @@ -159,7 +161,7 @@ class Http1ServerConnectionImplTest : public Http1CodecTestBase { createHeaderValidator(); codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, - max_request_headers_count_, headers_with_underscores_action_); + max_request_headers_count_, headers_with_underscores_action_, overload_manager_); } ~Http1ServerConnectionImplTest() override { @@ -243,7 +245,8 @@ void Http1ServerConnectionImplTest::expect400(Buffer::OwnedImpl& buffer, codec_settings_.allow_absolute_url_ = true; codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, - max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); + max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW, + overload_manager_); MockRequestDecoder decoder; Http::ResponseEncoder* response_encoder = nullptr; @@ -271,7 +274,8 @@ void Http1ServerConnectionImplTest::expectHeadersTest(Protocol p, bool allow_abs codec_settings_.allow_absolute_url_ = allow_absolute_url; codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, - max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); + max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW, + overload_manager_); } MockRequestDecoder decoder; @@ -292,7 +296,8 @@ void Http1ServerConnectionImplTest::expectTrailersTest(bool enable_trailers) { codec_settings_.enable_trailers_ = enable_trailers; codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, - max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); + max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW, + overload_manager_); } InSequence sequence; @@ -328,7 +333,8 @@ void Http1ServerConnectionImplTest::testTrailersExceedLimit(std::string trailer_ codec_settings_.enable_trailers_ = enable_trailers; codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, - max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); + max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW, + overload_manager_); std::string exception_reason; NiceMock decoder; EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); @@ -406,7 +412,8 @@ void Http1ServerConnectionImplTest::testServerAllowChunkedContentLength(uint32_t createHeaderValidator(); codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, - max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); + max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW, + overload_manager_); MockRequestDecoderShimWithUhv decoder(header_validator_.get(), connection_); Http::ResponseEncoder* response_encoder = nullptr; @@ -805,7 +812,8 @@ TEST_P(Http1ServerConnectionImplTest, CodecHasCorrectStreamErrorIfTrue) { codec_settings_.stream_error_on_invalid_http_message_ = true; codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, - max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); + max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW, + overload_manager_); Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\n"); NiceMock decoder; @@ -824,7 +832,8 @@ TEST_P(Http1ServerConnectionImplTest, CodecHasCorrectStreamErrorIfFalse) { codec_settings_.stream_error_on_invalid_http_message_ = false; codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, - max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); + max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW, + overload_manager_); Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\n"); NiceMock decoder; @@ -2327,6 +2336,49 @@ TEST_P(Http1ServerConnectionImplTest, TestSmugglingAllowChunkedContentLength100) testServerAllowChunkedContentLength(100, true); } +TEST_P(Http1ServerConnectionImplTest, LoadShedPointCanCloseConnectionOnDispatchOfNewStream) { + Server::MockLoadShedPoint mock_abort_dispatch; + EXPECT_CALL(overload_manager_, getLoadShedPoint(_)).WillOnce(Return(&mock_abort_dispatch)); + + initialize(); + + EXPECT_CALL(mock_abort_dispatch, shouldShedLoad()).WillOnce(Return(true)); + MockRequestDecoder decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + EXPECT_CALL(decoder, sendLocalReply(_, _, _, _, _)); + + Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\n\r\n"); + const auto status = codec_->dispatch(buffer); + + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(isEnvoyOverloadError(status)); +} + +TEST_P(Http1ServerConnectionImplTest, LoadShedPointCanCloseConnectionOnDispatchOfContinuingStream) { + Server::MockLoadShedPoint mock_abort_dispatch; + EXPECT_CALL(overload_manager_, getLoadShedPoint(_)).WillOnce(Return(&mock_abort_dispatch)); + + initialize(); + + EXPECT_CALL(mock_abort_dispatch, shouldShedLoad()).WillOnce(Return(false)); + MockRequestDecoder decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + + Buffer::OwnedImpl request_line_buffer("GET / HTTP/1.1\r\n"); + auto status = codec_->dispatch(request_line_buffer); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(0, request_line_buffer.length()); + + EXPECT_CALL(mock_abort_dispatch, shouldShedLoad()).WillOnce(Return(true)); + EXPECT_CALL(decoder, sendLocalReply(_, _, _, _, _)); + EXPECT_CALL(decoder, decodeHeaders_(_, true)).Times(0); + Buffer::OwnedImpl headers_buffer("final-header: value\r\n\r\n"); + status = codec_->dispatch(headers_buffer); + + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(isEnvoyOverloadError(status)); +} + TEST_P(Http1ServerConnectionImplTest, ShouldDumpParsedAndPartialHeadersWithoutAllocatingMemoryIfProcessingHeaders) { if (parser_impl_ == Http1ParserImpl::BalsaParser) { diff --git a/test/common/http/http1/http1_connection_fuzz_test.cc b/test/common/http/http1/http1_connection_fuzz_test.cc index b7f86c6f827e..af3888748f2e 100644 --- a/test/common/http/http1/http1_connection_fuzz_test.cc +++ b/test/common/http/http1/http1_connection_fuzz_test.cc @@ -7,6 +7,7 @@ #include "test/fuzz/utility.h" #include "test/mocks/http/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/server/overload_manager.h" #include "test/test_common/test_runtime.h" #include "absl/strings/substitute.h" @@ -54,7 +55,8 @@ class Http1Harness { mock_server_connection_, Http1::CodecStats::atomicGet(http1_stats_, *stats_store_.rootScope()), mock_server_callbacks_, server_settings_, Http::DEFAULT_MAX_REQUEST_HEADERS_KB, - Http::DEFAULT_MAX_HEADERS_COUNT, envoy::config::core::v3::HttpProtocolOptions::ALLOW); + Http::DEFAULT_MAX_HEADERS_COUNT, envoy::config::core::v3::HttpProtocolOptions::ALLOW, + overload_manager_); Status status = server_->dispatch(payload); } @@ -71,6 +73,7 @@ class Http1Harness { NiceMock orphan_request_decoder_; NiceMock mock_server_connection_; NiceMock mock_server_callbacks_; + testing::NiceMock overload_manager_; ServerConnectionPtr server_; }; diff --git a/test/common/http/status_test.cc b/test/common/http/status_test.cc index 11bdd104f190..c8d6d77a869c 100644 --- a/test/common/http/status_test.cc +++ b/test/common/http/status_test.cc @@ -17,6 +17,7 @@ TEST(Status, Ok) { EXPECT_FALSE(isPrematureResponseError(status)); EXPECT_FALSE(isCodecClientError(status)); EXPECT_FALSE(isInboundFramesWithEmptyPayloadError(status)); + EXPECT_FALSE(isEnvoyOverloadError(status)); } TEST(Status, CodecProtocolError) { @@ -30,6 +31,7 @@ TEST(Status, CodecProtocolError) { EXPECT_FALSE(isPrematureResponseError(status)); EXPECT_FALSE(isCodecClientError(status)); EXPECT_FALSE(isInboundFramesWithEmptyPayloadError(status)); + EXPECT_FALSE(isEnvoyOverloadError(status)); } TEST(Status, BufferFloodError) { @@ -43,6 +45,7 @@ TEST(Status, BufferFloodError) { EXPECT_FALSE(isPrematureResponseError(status)); EXPECT_FALSE(isCodecClientError(status)); EXPECT_FALSE(isInboundFramesWithEmptyPayloadError(status)); + EXPECT_FALSE(isEnvoyOverloadError(status)); } TEST(Status, PrematureResponseError) { @@ -57,6 +60,7 @@ TEST(Status, PrematureResponseError) { EXPECT_EQ(Http::Code::ProxyAuthenticationRequired, getPrematureResponseHttpCode(status)); EXPECT_FALSE(isCodecClientError(status)); EXPECT_FALSE(isInboundFramesWithEmptyPayloadError(status)); + EXPECT_FALSE(isEnvoyOverloadError(status)); } TEST(Status, CodecClientError) { @@ -70,6 +74,7 @@ TEST(Status, CodecClientError) { EXPECT_FALSE(isPrematureResponseError(status)); EXPECT_TRUE(isCodecClientError(status)); EXPECT_FALSE(isInboundFramesWithEmptyPayloadError(status)); + EXPECT_FALSE(isEnvoyOverloadError(status)); } TEST(Status, InboundFramesWithEmptyPayload) { @@ -84,6 +89,21 @@ TEST(Status, InboundFramesWithEmptyPayload) { EXPECT_FALSE(isPrematureResponseError(status)); EXPECT_FALSE(isCodecClientError(status)); EXPECT_TRUE(isInboundFramesWithEmptyPayloadError(status)); + EXPECT_FALSE(isEnvoyOverloadError(status)); +} + +TEST(Status, EnvoyOverloadError) { + auto status = envoyOverloadError("foobar"); + EXPECT_FALSE(status.ok()); + EXPECT_EQ("foobar", status.message()); + EXPECT_EQ("EnvoyOverloadError: foobar", toString(status)); + EXPECT_EQ(StatusCode::EnvoyOverloadError, getStatusCode(status)); + EXPECT_FALSE(isCodecProtocolError(status)); + EXPECT_FALSE(isBufferFloodError(status)); + EXPECT_FALSE(isPrematureResponseError(status)); + EXPECT_FALSE(isCodecClientError(status)); + EXPECT_FALSE(isInboundFramesWithEmptyPayloadError(status)); + EXPECT_TRUE(isEnvoyOverloadError(status)); } TEST(Status, ReturnIfError) { diff --git a/test/integration/BUILD b/test/integration/BUILD index 0178b436412b..d49e57963533 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -897,8 +897,6 @@ envoy_cc_test_library( ], deps = [ ":utility_lib", - "//test/mocks/runtime:runtime_mocks", - "//test/mocks/protobuf:protobuf_mocks", "//envoy/api:api_interface", "//envoy/grpc:status", "//envoy/http:codec_interface", @@ -930,6 +928,9 @@ envoy_cc_test_library( "//source/extensions/listener_managers/listener_manager:active_raw_udp_listener_config", "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//test/mocks/http:header_validator_mocks", + "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:overload_manager_mocks", "//test/test_common:network_utility_lib", "//test/test_common:test_time_system_interface", "//test/test_common:utility_lib", @@ -954,25 +955,25 @@ envoy_cc_test_library( "base_integration_test.h", ], deps = [ - "//source/extensions/config_subscription/grpc:grpc_subscription_lib", - "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", - "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib", - "//source/extensions/load_balancing_policies/maglev:config", - "//source/server:proto_descriptors_lib", - "//source/extensions/request_id/uuid:config", ":autonomous_upstream_lib", ":fake_upstream_lib", ":integration_tcp_client_lib", ":utility_lib", "//source/common/common:thread_lib", "//source/common/config:api_version_lib", - "//source/extensions/network/dns_resolver/cares:config", - "//source/extensions/clusters/static:static_cluster_lib", "//source/extensions/clusters/eds:eds_lib", + "//source/extensions/clusters/static:static_cluster_lib", + "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib", + "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "//source/extensions/load_balancing_policies/maglev:config", + "//source/extensions/network/dns_resolver/cares:config", + "//source/extensions/request_id/uuid:config", "//source/extensions/transport_sockets/tls:context_config_lib", "//source/extensions/transport_sockets/tls:context_lib", "//source/extensions/transport_sockets/tls:ssl_socket_lib", "//source/server:process_context_lib", + "//source/server:proto_descriptors_lib", "//test/common/grpc:grpc_client_integration_lib", "//test/config:utility_lib", "//test/mocks/buffer:buffer_mocks", @@ -1365,6 +1366,7 @@ envoy_cc_test( ":base_overload_integration_test_lib", ":http_protocol_integration_lib", "//test/common/config:dummy_config_proto_cc_proto", + "//test/test_common:test_runtime_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", ], diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 35c73f6d24a7..96ca4e3afd54 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -387,7 +387,7 @@ FakeHttpConnection::FakeHttpConnection( Http::Http1::CodecStats& stats = fake_upstream.http1CodecStats(); codec_ = std::make_unique( shared_connection_.connection(), stats, *this, http1_settings, max_request_headers_kb, - max_request_headers_count, headers_with_underscores_action); + max_request_headers_count, headers_with_underscores_action, overload_manager_); } else if (type == Http::CodecType::HTTP2) { envoy::config::core::v3::Http2ProtocolOptions http2_options = fake_upstream.http2Options(); Http::Http2::CodecStats& stats = fake_upstream.http2CodecStats(); @@ -636,6 +636,7 @@ FakeUpstream::FakeUpstream(Network::DownstreamTransportSocketFactoryPtr&& transp ->mutable_max_rx_datagram_size() ->set_value(config.udp_fake_upstream_->max_rx_datagram_size_.value()); } + dispatcher_->post([this]() -> void { socket_factories_[0]->doFinalPreWorkerInit(); handler_->addListener(absl::nullopt, listener_, runtime_); diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 0b2c2cc15715..b4e28712d38b 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -49,6 +49,7 @@ #include "test/mocks/common.h" #include "test/mocks/runtime/mocks.h" +#include "test/mocks/server/overload_manager.h" #include "test/test_common/test_time_system.h" #include "test/test_common/utility.h" @@ -528,6 +529,7 @@ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeCo const Http::CodecType type_; Http::ServerConnectionPtr codec_; std::list new_streams_ ABSL_GUARDED_BY(lock_); + testing::NiceMock overload_manager_; testing::NiceMock random_; testing::NiceMock header_validator_stats_; Http::HeaderValidatorFactoryPtr header_validator_factory_; diff --git a/test/integration/overload_integration_test.cc b/test/integration/overload_integration_test.cc index 1e4b56e8951d..3200dced73ee 100644 --- a/test/integration/overload_integration_test.cc +++ b/test/integration/overload_integration_test.cc @@ -6,6 +6,7 @@ #include "test/integration/base_overload_integration_test.h" #include "test/integration/http_protocol_integration.h" #include "test/integration/ssl_utility.h" +#include "test/test_common/test_runtime.h" #include "absl/strings/str_cat.h" @@ -485,4 +486,228 @@ TEST_P(LoadShedPointIntegrationTest, AcceptNewHttpStreamShedsLoad) { test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 1); } +TEST_P(LoadShedPointIntegrationTest, Http1ServerDispatchAbortShedsLoadWhenNewRequest) { + // Test only applies to HTTP1. + if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) { + return; + } + autonomous_upstream_ = true; + initializeOverloadManager( + TestUtility::parseYaml(R"EOF( + name: "envoy.load_shed_points.http1_server_abort_dispatch" + triggers: + - name: "envoy.resource_monitors.testonly.fake_resource_monitor" + threshold: + value: 0.90 + )EOF")); + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 0); + + // Put envoy in overloaded state and check that the dispatch fails. + updateResource(0.95); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http1_server_abort_dispatch.scale_percent", 100); + + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + auto [encoder, decoder] = codec_client_->startRequest(default_request_headers_); + + // We should get rejected local reply and connection close. + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 1); + ASSERT_TRUE(decoder->waitForEndStream()); + EXPECT_EQ(decoder->headers().getStatusValue(), "500"); + ASSERT_TRUE(codec_client_->waitForDisconnect()); + + // Disable overload, we should allow connections. + updateResource(0.80); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http1_server_abort_dispatch.scale_percent", 0); + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ(response->headers().getStatusValue(), "200"); +} + +// Test using 100-continue to start the response before doing triggering Overload. +// Even though Envoy has encoded the 1xx headers, 1xx headers are not the actual response +// so Envoy should still send the local reply when shedding load. +TEST_P( + LoadShedPointIntegrationTest, + Http1ServerDispatchAbortShedsLoadWithLocalReplyWhen1xxResponseStartedWithFlagAllowCodecErrorResponseAfter1xxHeadersEnabled) { + // Test only applies to HTTP1. + if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) { + return; + } + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.http1_allow_codec_error_response_after_1xx_headers", "true"}}); + + initializeOverloadManager( + TestUtility::parseYaml(R"EOF( + name: "envoy.load_shed_points.http1_server_abort_dispatch" + triggers: + - name: "envoy.resource_monitors.testonly.fake_resource_monitor" + threshold: + value: 0.90 + )EOF")); + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 0); + + // Start the 100-continue request + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + auto encoder_decoder = + codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "POST"}, + {":path", "/dynamo/url"}, + {":scheme", "http"}, + {":authority", "sni.lyft.com"}, + {"expect", "100-contINUE"}}); + + // Wait for 100-continue + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + // The continue headers should arrive immediately. + response->waitFor1xxHeaders(); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + + // Put envoy in overloaded state and check that it rejects the continuing + // dispatch. + updateResource(0.95); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http1_server_abort_dispatch.scale_percent", 100); + codec_client_->sendData(*request_encoder_, 10, true); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ(response->headers().getStatusValue(), "500"); + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 1); +} + +// TODO(kbaichoo): remove this test when the runtime flag is removed. +TEST_P( + LoadShedPointIntegrationTest, + Http1ServerDispatchAbortShedsLoadWithLocalReplyWhen1xxResponseStartedWithFlagAllowCodecErrorResponseAfter1xxHeadersDisabled) { + // Test only applies to HTTP1. + if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) { + return; + } + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.http1_allow_codec_error_response_after_1xx_headers", "false"}}); + + initializeOverloadManager( + TestUtility::parseYaml(R"EOF( + name: "envoy.load_shed_points.http1_server_abort_dispatch" + triggers: + - name: "envoy.resource_monitors.testonly.fake_resource_monitor" + threshold: + value: 0.90 + )EOF")); + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 0); + + // Start the 100-continue request + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + auto encoder_decoder = + codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "POST"}, + {":path", "/dynamo/url"}, + {":scheme", "http"}, + {":authority", "sni.lyft.com"}, + {"expect", "100-contINUE"}}); + + // Wait for 100-continue + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + // The continue headers should arrive immediately. + response->waitFor1xxHeaders(); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + + // Put envoy in overloaded state and check that it rejects the continuing + // dispatch. + updateResource(0.95); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http1_server_abort_dispatch.scale_percent", 100); + codec_client_->sendData(*request_encoder_, 10, true); + + // Since the runtime flag `http1_allow_codec_error_response_after_1xx_headers` + // is disabled the downstream client ends up just getting a closed connection. + // Envoy's downstream codec does not provide a local reply as we've started + // the response. + ASSERT_TRUE(codec_client_->waitForDisconnect()); + EXPECT_FALSE(response->complete()); + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 1); +} + +TEST_P(LoadShedPointIntegrationTest, Http1ServerDispatchShouldNotAbortEncodingUpstreamResponse) { + // Test only applies to HTTP1. + if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) { + return; + } + initializeOverloadManager( + TestUtility::parseYaml(R"EOF( + name: "envoy.load_shed_points.http1_server_abort_dispatch" + triggers: + - name: "envoy.resource_monitors.testonly.fake_resource_monitor" + threshold: + value: 0.90 + )EOF")); + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 0); + + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(default_response_headers_, false); + + // Put envoy in overloaded state, the response should succeed. + updateResource(0.95); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http1_server_abort_dispatch.scale_percent", 100); + upstream_request_->encodeData(100, true); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ(response->headers().getStatusValue(), "200"); + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 0); +} + +TEST_P(LoadShedPointIntegrationTest, Http1ServerDispatchAbortClosesConnectionWhenResponseStarted) { + // Test only applies to HTTP1. + if (downstreamProtocol() != Http::CodecClient::Type::HTTP1) { + return; + } + initializeOverloadManager( + TestUtility::parseYaml(R"EOF( + name: "envoy.load_shed_points.http1_server_abort_dispatch" + triggers: + - name: "envoy.resource_monitors.testonly.fake_resource_monitor" + threshold: + value: 0.90 + )EOF")); + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 0); + + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + auto encoder_decoder = codec_client_->startRequest(Http::TestRequestHeaderMapImpl{ + {":method", "POST"}, + {":path", "/dynamo/url"}, + {":scheme", "http"}, + {":authority", "sni.lyft.com"}, + }); + + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + upstream_request_->encodeHeaders(default_response_headers_, false); + + // Put envoy in overloaded state, the next dispatch should fail. + updateResource(0.95); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http1_server_abort_dispatch.scale_percent", 100); + + Buffer::OwnedImpl data("hello world"); + request_encoder_->encodeData(data, true); + + ASSERT_TRUE(codec_client_->waitForDisconnect()); + EXPECT_FALSE(response->complete()); + test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 1); +} + } // namespace Envoy diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index aa99fae6f823..1cefdb07635b 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -589,8 +589,9 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { // Http::ConnectionManagerConfig ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& instance, - ServerConnectionCallbacks& callbacks) override { - return ServerConnectionPtr{createCodec_(connection, instance, callbacks)}; + ServerConnectionCallbacks& callbacks, + Server::OverloadManager& overload_manager) override { + return ServerConnectionPtr{createCodec_(connection, instance, callbacks, overload_manager)}; } MOCK_METHOD(const RequestIDExtensionSharedPtr&, requestIDExtension, ()); @@ -598,7 +599,8 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_METHOD(bool, flushAccessLogOnNewRequest, ()); MOCK_METHOD(const absl::optional&, accessLogFlushInterval, ()); MOCK_METHOD(ServerConnection*, createCodec_, - (Network::Connection&, const Buffer::Instance&, ServerConnectionCallbacks&)); + (Network::Connection&, const Buffer::Instance&, ServerConnectionCallbacks&, + Server::OverloadManager&)); MOCK_METHOD(DateProvider&, dateProvider, ()); MOCK_METHOD(std::chrono::milliseconds, drainTimeout, (), (const)); MOCK_METHOD(FilterChainFactory&, filterFactory, ()); From e2a40eaff8baf35b269a936526512417033b916a Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Thu, 4 May 2023 11:20:12 -0400 Subject: [PATCH 112/740] build: Adds a envoy_select_signal_trace Bazel macro (#27155) Signed-off-by: Ali Beyad --- bazel/envoy_build_system.bzl | 2 ++ bazel/envoy_internal.bzl | 6 ++--- bazel/envoy_select.bzl | 7 ++++++ mobile/library/common/BUILD | 12 ++++------ mobile/test/common/integration/BUILD | 11 ++++----- source/common/event/BUILD | 8 ++----- source/exe/BUILD | 34 +++++++++++----------------- test/fuzz/BUILD | 6 ++--- test/test_common/BUILD | 6 ++--- 9 files changed, 40 insertions(+), 52 deletions(-) diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index 43c74a0810f0..a64c9babbfea 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -31,6 +31,7 @@ load( _envoy_select_envoy_mobile_stats_reporting = "envoy_select_envoy_mobile_stats_reporting", _envoy_select_google_grpc = "envoy_select_google_grpc", _envoy_select_hot_restart = "envoy_select_hot_restart", + _envoy_select_signal_trace = "envoy_select_signal_trace", _envoy_select_static_extension_registration = "envoy_select_static_extension_registration", _envoy_select_wasm_cpp_tests = "envoy_select_wasm_cpp_tests", _envoy_select_wasm_rust_tests = "envoy_select_wasm_rust_tests", @@ -240,6 +241,7 @@ envoy_select_enable_http3 = _envoy_select_enable_http3 envoy_select_enable_yaml = _envoy_select_enable_yaml envoy_select_hot_restart = _envoy_select_hot_restart envoy_select_enable_http_datagrams = _envoy_select_enable_http_datagrams +envoy_select_signal_trace = _envoy_select_signal_trace envoy_select_wasm_cpp_tests = _envoy_select_wasm_cpp_tests envoy_select_wasm_rust_tests = _envoy_select_wasm_rust_tests envoy_select_wasm_v8 = _envoy_select_wasm_v8 diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index d8b3d33f3b38..472dfd2fa483 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -1,6 +1,6 @@ # DO NOT LOAD THIS FILE. Targets from this file should be considered private # and not used outside of the @envoy//bazel package. -load(":envoy_select.bzl", "envoy_select_admin_html", "envoy_select_disable_logging", "envoy_select_google_grpc", "envoy_select_hot_restart", "envoy_select_static_extension_registration") +load(":envoy_select.bzl", "envoy_select_admin_html", "envoy_select_disable_logging", "envoy_select_google_grpc", "envoy_select_hot_restart", "envoy_select_signal_trace", "envoy_select_static_extension_registration") # Compute the final copts based on various options. def envoy_copts(repository, test = False): @@ -96,9 +96,6 @@ def envoy_copts(repository, test = False): repository + "//bazel:linux_x86_64": ["-DTCMALLOC"], repository + "//bazel:linux_aarch64": ["-DTCMALLOC"], "//conditions:default": ["-DGPERFTOOLS_TCMALLOC"], - }) + select({ - repository + "//bazel:disable_signal_trace": [], - "//conditions:default": ["-DENVOY_HANDLE_SIGNALS"], }) + select({ repository + "//bazel:disable_object_dump_on_signal_trace": [], "//conditions:default": ["-DENVOY_OBJECT_TRACE_ON_DUMP"], @@ -128,6 +125,7 @@ def envoy_copts(repository, test = False): _envoy_select_perf_annotation(["-DENVOY_PERF_ANNOTATION"]) + \ _envoy_select_perfetto(["-DENVOY_PERFETTO"]) + \ envoy_select_google_grpc(["-DENVOY_GOOGLE_GRPC"], repository) + \ + envoy_select_signal_trace(["-DENVOY_HANDLE_SIGNALS"], repository) + \ _envoy_select_path_normalization_by_default(["-DENVOY_NORMALIZE_PATH_BY_DEFAULT"], repository) # References to Envoy external dependencies should be wrapped with this function. diff --git a/bazel/envoy_select.bzl b/bazel/envoy_select.bzl index 19cc8a8f8b01..7fee86b3eeab 100644 --- a/bazel/envoy_select.bzl +++ b/bazel/envoy_select.bzl @@ -108,6 +108,13 @@ def envoy_select_hot_restart(xs, repository = ""): "//conditions:default": xs, }) +# Selects the given values if signal trace is enabled in the current build. +def envoy_select_signal_trace(xs, repository = ""): + return select({ + repository + "//bazel:disable_signal_trace": [], + "//conditions:default": xs, + }) + # Selects the given values depending on the Wasm runtimes enabled in the current build, # and the ability to build tests using Proxy-Wasm C++ SDK on the current platform. def envoy_select_wasm_cpp_tests(xs): diff --git a/mobile/library/common/BUILD b/mobile/library/common/BUILD index 7af18c311cc7..4fb65e2063f9 100644 --- a/mobile/library/common/BUILD +++ b/mobile/library/common/BUILD @@ -1,4 +1,4 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package", "envoy_select_signal_trace") licenses(["notice"]) # Apache 2 @@ -53,12 +53,10 @@ envoy_cc_library( "@envoy//source/common/common:random_generator_lib", "@envoy//source/common/runtime:runtime_lib", "@envoy//source/exe:stripped_main_base_lib", - ] + select({ - "@envoy//bazel:disable_signal_trace": [], - "//conditions:default": [ - "@envoy//source/common/signal:sigaction_lib", - ], - }), + ] + envoy_select_signal_trace( + ["@envoy//source/common/signal:sigaction_lib"], + "@envoy", + ), ) envoy_cc_library( diff --git a/mobile/test/common/integration/BUILD b/mobile/test/common/integration/BUILD index feabf1c65faf..501f65c8c588 100644 --- a/mobile/test/common/integration/BUILD +++ b/mobile/test/common/integration/BUILD @@ -4,6 +4,7 @@ load( "envoy_cc_test_library", "envoy_package", "envoy_select_google_grpc", + "envoy_select_signal_trace", ) licenses(["notice"]) # Apache 2 @@ -159,12 +160,10 @@ envoy_cc_test_library( "@envoy//test/mocks/server:transport_socket_factory_context_mocks", "@envoy//test/test_common:environment_lib", "@envoy_build_config//:extension_registry", - ] + select({ - "@envoy//bazel:disable_signal_trace": [], - "//conditions:default": [ - "@envoy//source/common/signal:sigaction_lib", - ], - }), + ] + envoy_select_signal_trace( + ["@envoy//source/common/signal:sigaction_lib"], + "@envoy", + ), ) envoy_cc_test_library( diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 302319d88c80..608cd4dcce57 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -5,6 +5,7 @@ load( "envoy_cc_posix_library", "envoy_cc_win32_library", "envoy_package", + "envoy_select_signal_trace", ) licenses(["notice"]) # Apache 2 @@ -130,12 +131,7 @@ envoy_cc_library( "//source/common/common:minimal_logger_lib", "//source/common/common:thread_lib", "//source/common/signal:fatal_error_handler_lib", - ] + select({ - "//bazel:disable_signal_trace": [], - "//conditions:default": [ - "//source/common/signal:sigaction_lib", - ], - }), + ] + envoy_select_signal_trace(["//source/common/signal:sigaction_lib"]), ) envoy_cc_library( diff --git a/source/exe/BUILD b/source/exe/BUILD index 2e16a10266a7..f2b01f7b655d 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -8,6 +8,7 @@ load( "envoy_cc_win32_library", "envoy_package", "envoy_select_enable_http3", + "envoy_select_signal_trace", ) load("//source/extensions:all_extensions.bzl", "envoy_all_core_extensions", "envoy_all_extensions") load("//bazel:repositories.bzl", "PPC_SKIP_TARGETS", "WINDOWS_SKIP_TARGETS") @@ -77,13 +78,10 @@ envoy_cc_library( "//source/common/grpc:google_grpc_context_lib", "//source/server:hot_restart_lib", "//source/server:hot_restart_nop_lib", - ] + select({ - "//bazel:disable_signal_trace": [], - "//conditions:default": [ - "//source/common/signal:sigaction_lib", - ":terminate_handler_lib", - ], - }), + ] + envoy_select_signal_trace([ + "//source/common/signal:sigaction_lib", + ":terminate_handler_lib", + ]), ) envoy_cc_library( @@ -107,13 +105,10 @@ envoy_cc_library( "//source/server:hot_restart_lib", "//source/server:hot_restart_nop_lib", "//source/server/config_validation:server_lib", - ] + select({ - "//bazel:disable_signal_trace": [], - "//conditions:default": [ - "//source/common/signal:sigaction_lib", - ":terminate_handler_lib", - ], - }), + ] + envoy_select_signal_trace([ + "//source/common/signal:sigaction_lib", + ":terminate_handler_lib", + ]), ) # provides a library target for Envoy server builds with the versioning information set up correctly. @@ -180,13 +175,10 @@ envoy_cc_library( "//source/server:hot_restart_lib", "//source/server:hot_restart_nop_lib", "//source/server/config_validation:server_lib", - ] + select({ - "//bazel:disable_signal_trace": [], - "//conditions:default": [ - "//source/common/signal:sigaction_lib", - ":terminate_handler_lib", - ], - }), + ] + envoy_select_signal_trace([ + "//source/common/signal:sigaction_lib", + ":terminate_handler_lib", + ]), ) envoy_cc_library( diff --git a/test/fuzz/BUILD b/test/fuzz/BUILD index dd2753923fe4..51b17c68f410 100644 --- a/test/fuzz/BUILD +++ b/test/fuzz/BUILD @@ -4,6 +4,7 @@ load( "envoy_cc_test_library", "envoy_package", "envoy_proto_library", + "envoy_select_signal_trace", ) load( "@rules_fuzzing//fuzzing:cc_defs.bzl", @@ -36,10 +37,7 @@ envoy_cc_test_library( "//test:test_listener_lib", "//test/test_common:environment_lib", "//test/test_common:utility_lib", - ] + select({ - "//bazel:disable_signal_trace": [], - "//conditions:default": ["//source/common/signal:sigaction_lib"], - }), + ] + envoy_select_signal_trace(["//source/common/signal:sigaction_lib"]), ) envoy_cc_test_library( diff --git a/test/test_common/BUILD b/test/test_common/BUILD index 4f753c1be9cb..1fa38a94a182 100644 --- a/test/test_common/BUILD +++ b/test/test_common/BUILD @@ -5,6 +5,7 @@ load( "envoy_cc_test", "envoy_cc_test_library", "envoy_package", + "envoy_select_signal_trace", ) licenses(["notice"]) # Apache 2 @@ -31,10 +32,7 @@ envoy_cc_test_library( "//source/common/json:json_loader_lib", "//source/common/network:utility_lib", "//source/server:options_lib", - ] + select({ - "//bazel:disable_signal_trace": [], - "//conditions:default": ["//source/common/signal:sigaction_lib"], - }), + ] + envoy_select_signal_trace(["//source/common/signal:sigaction_lib"]), ) envoy_cc_test_library( From 5880c2d25399566a5376c274817ba9d86d2091a8 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Thu, 4 May 2023 14:40:46 -0400 Subject: [PATCH 113/740] Use fastbuild mode for compile_time_options build (#27185) Signed-off-by: Yan Avlasov --- ci/do_ci.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index ba2c6b93d1da..10467dac322a 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -417,13 +417,13 @@ case $CI_TARGET in # Building all the dependencies from scratch to link them against libc++. echo "Building and testing with wasm=wamr: ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wamr "${COMPILE_TIME_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wamr "${COMPILE_TIME_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only echo "Building and testing with wasm=wasmtime: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wasmtime --define admin_html=disabled --define admin_functionality=disabled "${COMPILE_TIME_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wasmtime --define admin_html=disabled --define admin_functionality=disabled "${COMPILE_TIME_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only echo "Building and testing with wasm=wavm: ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only # these tests under "-c opt" to save time in CI. @@ -433,7 +433,7 @@ case $CI_TARGET in bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c opt @envoy//test/common/common:assert_test --define log_fast_debug_assert_in_release=enabled --define log_debug_assert_in_release=disabled echo "Building binary with wasm=wavm... and logging disabled" - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm --define enable_logging=disabled "${COMPILE_TIME_OPTIONS[@]}" -c dbg @envoy//source/exe:envoy-static --build_tag_filters=-nofips + bazel build "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm --define enable_logging=disabled "${COMPILE_TIME_OPTIONS[@]}" -c fastbuild @envoy//source/exe:envoy-static --build_tag_filters=-nofips collect_build_profile build ;; api) From 334a50e7ab10ff3247c300fdf32a2425e9deaa55 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Thu, 4 May 2023 22:09:14 -0400 Subject: [PATCH 114/740] Remove dead code (#27189) Signed-off-by: Yan Avlasov --- source/common/http/filter_manager.h | 7 ------- source/common/router/config_impl.h | 1 - source/common/router/upstream_request.cc | 1 - .../health_checkers/common/health_checker_base_impl.h | 3 --- test/integration/utility.h | 2 +- 5 files changed, 1 insertion(+), 13 deletions(-) diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index 09c131d673d5..5c93ef2f9574 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -757,11 +757,6 @@ class FilterManager : public ScopeTrackedObject, */ virtual void executeLocalReplyIfPrepared() PURE; - /** - * Whether the Filter Manager has a prepared local reply that it has not sent. - */ - virtual bool hasPreparedLocalReply() const PURE; - // Possibly increases buffer_limit_ to the value of limit. void setBufferLimit(uint32_t limit); @@ -1117,8 +1112,6 @@ class DownstreamFilterManager : public FilterManager { */ void executeLocalReplyIfPrepared() override; - bool hasPreparedLocalReply() const override { return prepared_local_reply_ != nullptr; } - /** * Sends a local reply by constructing a response and skipping the encoder filters. The * resulting response will be passed out via the FilterManagerCallbacks. diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 3694640f491e..e5f570e3cfe8 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -1610,7 +1610,6 @@ class ConfigImpl : public Config { const std::vector& shadowPolicies() const { return shared_config_->shadowPolicies(); } - ClusterSpecifierPluginSharedPtr clusterSpecifierPlugin(absl::string_view provider) const; bool ignorePathParametersInPathMatching() const { return shared_config_->ignorePathParametersInPathMatching(); } diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index 6766a5cb5987..c8e764e6b3cf 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -75,7 +75,6 @@ class UpstreamFilterManager : public Http::FilterManager { details); } void executeLocalReplyIfPrepared() override {} - bool hasPreparedLocalReply() const override { return false; } UpstreamRequest& upstream_request_; }; diff --git a/source/extensions/health_checkers/common/health_checker_base_impl.h b/source/extensions/health_checkers/common/health_checker_base_impl.h index 32d5fc55925c..1e7e308348cd 100644 --- a/source/extensions/health_checkers/common/health_checker_base_impl.h +++ b/source/extensions/health_checkers/common/health_checker_base_impl.h @@ -68,7 +68,6 @@ class HealthCheckerImplBase : public HealthChecker, ActiveHealthCheckSession(HealthCheckerImplBase& parent, HostSharedPtr host); void handleSuccess(bool degraded = false); - void handleDegraded(); void handleFailure(envoy::data::core::v3::HealthCheckFailureType type, bool retriable = false); HostSharedPtr host_; @@ -147,8 +146,6 @@ class HealthCheckerImplBase : public HealthChecker, static MetadataConstSharedPtr initTransportSocketMatchMetadata(const envoy::config::core::v3::HealthCheck& config); - static const std::chrono::milliseconds NO_TRAFFIC_INTERVAL; - std::list callbacks_; const std::chrono::milliseconds interval_; const std::chrono::milliseconds no_traffic_interval_; diff --git a/test/integration/utility.h b/test/integration/utility.h index f91e2407a6b0..497877b059df 100644 --- a/test/integration/utility.h +++ b/test/integration/utility.h @@ -85,7 +85,7 @@ class RawConnectionDriver { Event::Dispatcher& dispatcher, Network::TransportSocketPtr transport_socket = nullptr); ~RawConnectionDriver(); - const Network::Connection& connection() { return *client_; } + testing::AssertionResult run(Event::Dispatcher::RunType run_type = Event::Dispatcher::RunType::Block, std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); From 15093123d9427ff485f2b1e5eafe21906d67b376 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 May 2023 08:12:06 +0100 Subject: [PATCH 115/740] build(deps): bump redis from `393b84a` to `ea30bef` in /examples/redis (#27163) Bumps redis from `393b84a` to `ea30bef`. --- updated-dependencies: - dependency-name: redis dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/redis/Dockerfile-redis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/redis/Dockerfile-redis b/examples/redis/Dockerfile-redis index 5b6cb4021be6..f5847517ea98 100644 --- a/examples/redis/Dockerfile-redis +++ b/examples/redis/Dockerfile-redis @@ -1 +1 @@ -FROM redis@sha256:393b84ac0c167421173657a77f36add89bf24e23b123045e5e09501fc370474a +FROM redis@sha256:ea30bef6a1424d032295b90db20a869fc8db76331091543b7a80175cede7d887 From 54234e1d4f7c532faeac558c67a9d6f6eb9b7db3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 May 2023 08:12:25 +0100 Subject: [PATCH 116/740] build(deps): bump nginx from `63b44e8` to `480868e` in /examples/local_ratelimit (#27164) build(deps): bump nginx in /examples/local_ratelimit Bumps nginx from `63b44e8` to `480868e`. --- updated-dependencies: - dependency-name: nginx dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/local_ratelimit/Dockerfile-nginx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/local_ratelimit/Dockerfile-nginx b/examples/local_ratelimit/Dockerfile-nginx index 15b1370bddfc..46b5fa67f3a1 100644 --- a/examples/local_ratelimit/Dockerfile-nginx +++ b/examples/local_ratelimit/Dockerfile-nginx @@ -1 +1 @@ -FROM nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94 +FROM nginx@sha256:480868e8c8c797794257e2abd88d0f9a8809b2fe956cbfbc05dcc0bca1f7cd43 From 78e3f1e58c43d726d991b25cac39ecfee405daa7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 May 2023 08:15:56 +0100 Subject: [PATCH 117/740] build(deps): bump python from `286f2f1` to `551c952` in /examples/shared/python (#27168) build(deps): bump python in /examples/shared/python Bumps python from `286f2f1` to `551c952`. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/python/Dockerfile b/examples/shared/python/Dockerfile index bf1aad685da4..cc94f641fbff 100644 --- a/examples/shared/python/Dockerfile +++ b/examples/shared/python/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.3-slim-bullseye@sha256:286f2f1d6f2f730a44108656afb04b131504b610a6cb2f3413918e98dabba67e as python-base +FROM python:3.11.3-slim-bullseye@sha256:551c9529e77896518ac5693d7e98ee5e12051d625de450ac2a68da1eae15ec87 as python-base RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache ARG PYTHON_REQUIREMENTS_FILE=aiohttp/requirements.txt From 2eecc7122493d96d96c1543a00f9316c66455be0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 May 2023 08:16:28 +0100 Subject: [PATCH 118/740] build(deps): bump confluentinc/cp-zookeeper from `cb54862` to `8fb8e15` in /examples/kafka (#27169) build(deps): bump confluentinc/cp-zookeeper in /examples/kafka Bumps confluentinc/cp-zookeeper from `cb54862` to `8fb8e15`. --- updated-dependencies: - dependency-name: confluentinc/cp-zookeeper dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/kafka/Dockerfile-zookeeper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka/Dockerfile-zookeeper b/examples/kafka/Dockerfile-zookeeper index 78c2e5d7f15f..1cd685cc6876 100644 --- a/examples/kafka/Dockerfile-zookeeper +++ b/examples/kafka/Dockerfile-zookeeper @@ -1 +1 @@ -FROM confluentinc/cp-zookeeper:latest@sha256:cb54862841849dff061a3439bb96e7e80cf8697cfdbfc5b56231c7e2f3e9b098 +FROM confluentinc/cp-zookeeper:latest@sha256:8fb8e15d702d6935d21622ac03e789ba79bcedd3fa809134556d9fed0f88f0e9 From 97b07d0781d34b00db8774269b2b28933b7af7db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 May 2023 08:16:37 +0100 Subject: [PATCH 119/740] build(deps): bump confluentinc/cp-kafka from `1333e64` to `328bb7b` in /examples/kafka (#27167) build(deps): bump confluentinc/cp-kafka in /examples/kafka Bumps confluentinc/cp-kafka from `1333e64` to `328bb7b`. --- updated-dependencies: - dependency-name: confluentinc/cp-kafka dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/kafka/Dockerfile-kafka | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka/Dockerfile-kafka b/examples/kafka/Dockerfile-kafka index 33d357575770..46f63575c21e 100644 --- a/examples/kafka/Dockerfile-kafka +++ b/examples/kafka/Dockerfile-kafka @@ -1 +1 @@ -FROM confluentinc/cp-kafka:latest@sha256:1333e64f9b3e0b18f1c1fbaa5f4593eedb8a515ca03b88f2ef7cb4d65010e21c +FROM confluentinc/cp-kafka:latest@sha256:328bb7b00e15f173bee5f68a9da0c97651ca961e2f7736563dc55bbebac80e5e From c9b5bf22407bd96974e53dc5120924d2f179aa1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 May 2023 08:17:24 +0100 Subject: [PATCH 120/740] build(deps): bump postgres from `8a2975e` to `78a275d` in /examples/shared/postgres (#27165) build(deps): bump postgres in /examples/shared/postgres Bumps postgres from `8a2975e` to `78a275d`. --- updated-dependencies: - dependency-name: postgres dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile index a4d972878649..bc236287dd27 100644 --- a/examples/shared/postgres/Dockerfile +++ b/examples/shared/postgres/Dockerfile @@ -1,3 +1,3 @@ -FROM postgres:latest@sha256:8a2975ef8f47391d186cfe43b38d4ccef88b27e40397ae6ad3b961ffacb57029 +FROM postgres:latest@sha256:78a275d4c891f7b3a33d3f1a78eda9f1d744954d9e20122bfdc97cdda25cddaf COPY docker-healthcheck.sh /usr/local/bin/ HEALTHCHECK CMD ["docker-healthcheck.sh"] From 75c31d7af31c751d0e82530b9bd96f23d7b1b239 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 May 2023 08:17:42 +0100 Subject: [PATCH 121/740] build(deps): bump golang from `d282e70` to `995b84e` in /examples/shared/golang (#27170) build(deps): bump golang in /examples/shared/golang Bumps golang from `d282e70` to `995b84e`. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/golang/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile index b48848365ca2..a5eba1863a66 100644 --- a/examples/shared/golang/Dockerfile +++ b/examples/shared/golang/Dockerfile @@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache -FROM golang:1.20.4-bullseye@sha256:d282e704848c81e313b17c68e27edd44202937db196398915077830557f82588 as golang-base +FROM golang:1.20.4-bullseye@sha256:995b84ed1eb95c3aefa2787b00bad9129a6b0edc5363328bcde8b49d909fd4df as golang-base FROM golang-base as golang-control-plane-builder From 536caa8999ba2a89f5962fda9293da9c390b9386 Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 5 May 2023 11:00:20 +0100 Subject: [PATCH 122/740] examples: Workaround kitware apt issue in wasm example (#27206) Signed-off-by: Ryan Northey --- examples/shared/build/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/shared/build/Dockerfile b/examples/shared/build/Dockerfile index d4faf35fa785..277bdd0687a4 100644 --- a/examples/shared/build/Dockerfile +++ b/examples/shared/build/Dockerfile @@ -1,6 +1,8 @@ FROM envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get -qq update \ - && apt-get -qq install --no-install-recommends -y gosu \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ + # apt-get -qq update \ + apt-get -qq install --no-install-recommends -y gosu \ && groupadd -f envoygroup \ && useradd -g envoygroup -m -d /home/envoybuild envoybuild From 7b40f0a2533cb229308599380a7bf52a027f7769 Mon Sep 17 00:00:00 2001 From: doujiang24 Date: Fri, 5 May 2023 07:07:45 -0700 Subject: [PATCH 123/740] golang filter: fix segfault: should check filter expired before touch mutex_ lock. (#27177) * golang filter: fix segfault: should check filter expired before touch mutex_ lock. otherwise, might lead to segfault. Signed-off-by: doujiang24 --- .../filters/http/source/golang_filter.cc | 36 +++++++++---------- .../filters/http/source/golang_filter.h | 4 +++ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc index 05eebfceb2e9..ba5bb9b6fcf3 100644 --- a/contrib/golang/filters/http/source/golang_filter.cc +++ b/contrib/golang/filters/http/source/golang_filter.cc @@ -498,6 +498,7 @@ CAPIStatus Filter::sendLocalReply(Http::Code response_code, std::string body_text, std::function modify_headers, Grpc::Status::GrpcStatus grpc_status, std::string details) { + // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -513,11 +514,8 @@ Filter::sendLocalReply(Http::Code response_code, std::string body_text, auto weak_ptr = weak_from_this(); state.getDispatcher().post( [this, &state, weak_ptr, response_code, body_text, modify_headers, grpc_status, details] { - ASSERT(state.isThreadSafe()); - // TODO: do not need lock here, since it's the work thread now. - Thread::ReleasableLockGuard lock(mutex_); - if (!weak_ptr.expired() && !has_destroyed_) { - lock.release(); + if (!weak_ptr.expired() && !hasDestroyed()) { + ASSERT(state.isThreadSafe()); sendLocalReplyInternal(response_code, body_text, modify_headers, grpc_status, details); } else { ENVOY_LOG(debug, "golang filter has gone or destroyed in sendLocalReply"); @@ -539,6 +537,7 @@ CAPIStatus Filter::sendPanicReply(absl::string_view details) { } CAPIStatus Filter::continueStatus(GolangStatus status) { + // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -556,11 +555,8 @@ CAPIStatus Filter::continueStatus(GolangStatus status) { // TODO: skip post event to dispatcher, and return continue in the caller, // when it's invoked in the current envoy thread, for better performance & latency. state.getDispatcher().post([this, &state, weak_ptr, status] { - ASSERT(state.isThreadSafe()); - // TODO: do not need lock here, since it's the work thread now. - Thread::ReleasableLockGuard lock(mutex_); - if (!weak_ptr.expired() && !has_destroyed_) { - lock.release(); + if (!weak_ptr.expired() && !hasDestroyed()) { + ASSERT(state.isThreadSafe()); continueStatusInternal(status); } else { ENVOY_LOG(debug, "golang filter has gone or destroyed in continueStatus event"); @@ -685,8 +681,8 @@ CAPIStatus Filter::setHeader(absl::string_view key, absl::string_view value, hea // safety. otherwise, there might be race between reading in the envoy worker thread and writing // in the Go thread. state.getDispatcher().post([this, weak_ptr, key_str, value_str, act] { - Thread::LockGuard lock(mutex_); - if (!weak_ptr.expired() && !has_destroyed_) { + if (!weak_ptr.expired() && !hasDestroyed()) { + Thread::LockGuard lock(mutex_); switch (act) { case HeaderAdd: headers_->addCopy(Http::LowerCaseString(key_str), value_str); @@ -740,8 +736,8 @@ CAPIStatus Filter::removeHeader(absl::string_view key) { // safety. otherwise, there might be race between reading in the envoy worker thread and writing // in the Go thread. state.getDispatcher().post([this, weak_ptr, key_str] { - Thread::LockGuard lock(mutex_); - if (!weak_ptr.expired() && !has_destroyed_) { + if (!weak_ptr.expired() && !hasDestroyed()) { + Thread::LockGuard lock(mutex_); headers_->remove(Http::LowerCaseString(key_str)); onHeadersModified(); } else { @@ -753,6 +749,7 @@ CAPIStatus Filter::removeHeader(absl::string_view key) { } CAPIStatus Filter::copyBuffer(Buffer::Instance* buffer, char* data) { + // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -778,6 +775,7 @@ CAPIStatus Filter::copyBuffer(Buffer::Instance* buffer, char* data) { CAPIStatus Filter::setBufferHelper(Buffer::Instance* buffer, absl::string_view& value, bufferAction action) { + // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -842,6 +840,7 @@ CAPIStatus Filter::setTrailer(absl::string_view key, absl::string_view value) { } CAPIStatus Filter::getIntegerValue(int id, uint64_t* value) { + // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -879,6 +878,7 @@ CAPIStatus Filter::getIntegerValue(int id, uint64_t* value) { } CAPIStatus Filter::getStringValue(int id, GoString* value_str) { + // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -937,6 +937,7 @@ CAPIStatus Filter::getStringValue(int id, GoString* value_str) { CAPIStatus Filter::setDynamicMetadata(std::string filter_name, std::string key, absl::string_view buf) { + // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -955,11 +956,8 @@ CAPIStatus Filter::setDynamicMetadata(std::string filter_name, std::string key, // of the buffer slice and pass that to the dispatcher. auto buff_copy = std::string(buf); state.getDispatcher().post([this, &state, weak_ptr, filter_name, key, buff_copy] { - ASSERT(state.isThreadSafe()); - // TODO: do not need lock here, since it's the work thread now. - Thread::ReleasableLockGuard lock(mutex_); - if (!weak_ptr.expired() && !has_destroyed_) { - lock.release(); + if (!weak_ptr.expired() && !hasDestroyed()) { + ASSERT(state.isThreadSafe()); setDynamicMetadataInternal(state, filter_name, key, buff_copy); } else { ENVOY_LOG(info, "golang filter has gone or destroyed in setDynamicMetadata"); diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h index 610b229f1337..c9d9c7f14d47 100644 --- a/contrib/golang/filters/http/source/golang_filter.h +++ b/contrib/golang/filters/http/source/golang_filter.h @@ -180,6 +180,10 @@ class Filter : public Http::StreamFilter, CAPIStatus setDynamicMetadata(std::string filter_name, std::string key, absl::string_view buf); private: + bool hasDestroyed() { + Thread::LockGuard lock(mutex_); + return has_destroyed_; + }; ProcessorState& getProcessorState(); bool doHeaders(ProcessorState& state, Http::RequestOrResponseHeaderMap& headers, bool end_stream); From 506340a1ce61a49a4438c8235e9c60d3396d76be Mon Sep 17 00:00:00 2001 From: "Dr. Andre Vehreschild" <101638173+vehre-x41@users.noreply.github.com> Date: Fri, 5 May 2023 16:14:51 +0200 Subject: [PATCH 124/740] [fuzz] Address various issues in the health_check_fuzz_test (#27181) * [fuzz] Fix expect to allow multiple call. Oss-Fuzz issue #27356 Signed-off-by: Andre Vehreschild * [fuzz] Prevent trigger_interval timer on close connection. Oss-fuzz issue #51726 Signed-off-by: Andre Vehreschild * [fuzz] Prevent multiple CONNECTED actions. OSS-fuzz issue #50311 Signed-off-by: Andre Vehreschild * [fuzz] Prevent access to unset bytes_meter. Accessing downstream's bytes meter in header substitution is now guarded against unset bytes meter. OSS-fuzz issue #46835 Signed-off-by: Andre Vehreschild --- .../formatter/substitution_formatter.cc | 14 ++++++----- test/common/upstream/health_check_fuzz.cc | 23 ++++++++++++++++--- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index a337e70de86e..3937d2ffbee6 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -991,8 +991,8 @@ const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnow [](const std::string&, const absl::optional&) { return std::make_unique( [](const StreamInfo::StreamInfo& stream_info) { - return stream_info.getDownstreamBytesMeter() - ->wireBytesReceived(); + auto bytes_meter = stream_info.getDownstreamBytesMeter(); + return bytes_meter ? bytes_meter->wireBytesReceived() : 0; }); }}}, {"DOWNSTREAM_HEADER_BYTES_RECEIVED", @@ -1000,8 +1000,8 @@ const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnow [](const std::string&, const absl::optional&) { return std::make_unique( [](const StreamInfo::StreamInfo& stream_info) { - return stream_info.getDownstreamBytesMeter() - ->headerBytesReceived(); + auto bytes_meter = stream_info.getDownstreamBytesMeter(); + return bytes_meter ? bytes_meter->headerBytesReceived() : 0; }); }}}, {"PROTOCOL", @@ -1077,7 +1077,8 @@ const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnow [](const std::string&, const absl::optional&) { return std::make_unique( [](const StreamInfo::StreamInfo& stream_info) { - return stream_info.getDownstreamBytesMeter()->wireBytesSent(); + auto bytes_meter = stream_info.getDownstreamBytesMeter(); + return bytes_meter ? bytes_meter->wireBytesSent() : 0; }); }}}, {"DOWNSTREAM_HEADER_BYTES_SENT", @@ -1085,7 +1086,8 @@ const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnow [](const std::string&, const absl::optional&) { return std::make_unique( [](const StreamInfo::StreamInfo& stream_info) { - return stream_info.getDownstreamBytesMeter()->headerBytesSent(); + auto bytes_meter = stream_info.getDownstreamBytesMeter(); + return bytes_meter ? bytes_meter->headerBytesSent() : 0; }); }}}, {"DURATION", diff --git a/test/common/upstream/health_check_fuzz.cc b/test/common/upstream/health_check_fuzz.cc index dcc5ab5579f9..b69d7037c2fd 100644 --- a/test/common/upstream/health_check_fuzz.cc +++ b/test/common/upstream/health_check_fuzz.cc @@ -172,6 +172,10 @@ void HttpHealthCheckFuzz::triggerIntervalTimer(bool expect_client_create) { } if (expect_client_create) { expectClientCreate(0); + } else if (test_sessions_[0]->client_connection_->state() != Network::Connection::State::Open) { + // No client connection to reuse. + ENVOY_LOG_MISC(trace, "Interval timer on closed connection; ignored."); + return; } expectStreamCreate(0); ENVOY_LOG_MISC(trace, "Triggered interval timer"); @@ -269,6 +273,10 @@ void TcpHealthCheckFuzz::triggerIntervalTimer(bool expect_client_create) { if (expect_client_create) { ENVOY_LOG_MISC(trace, "Creating client"); expectClientCreate(); + } else if (connection_->state() != Network::Connection::State::Open) { + // Without a client no interval timer possible. + ENVOY_LOG_MISC(trace, "Interval timer on closed connection; ignored."); + return; } ENVOY_LOG_MISC(trace, "Triggered interval timer"); interval_timer_->invokeCallback(); @@ -293,7 +301,12 @@ void TcpHealthCheckFuzz::raiseEvent(const Network::ConnectionEvent& event_type, // set by multiple code paths. handleFailure() turns on interval and turns off timeout. However, // other action of the fuzzer account for this by explicitly invoking a client after // expect_close_ gets set to true, turning expect_close_ back to false. - connection_->raiseEvent(event_type); + if (event_type == Network::ConnectionEvent::Connected && + connection_->state() != Network::Connection::State::Open) { + ENVOY_LOG_MISC(trace, "Event CONNECTED on closed connection; ignoring"); + } else { + connection_->raiseEvent(event_type); + } if (!last_action && event_type != Network::ConnectionEvent::Connected) { if (!interval_timer_->enabled_) { return; @@ -447,6 +460,10 @@ void GrpcHealthCheckFuzz::triggerIntervalTimer(bool expect_client_create) { if (expect_client_create) { expectClientCreate(); ENVOY_LOG_MISC(trace, "Created client"); + } else if (test_session_->client_connection_->state() != Network::Connection::State::Open) { + // No client connection to reuse. + ENVOY_LOG_MISC(trace, "Interval timer on closed connection; ignored."); + return; } expectStreamCreate(); ENVOY_LOG_MISC(trace, "Created stream"); @@ -511,8 +528,8 @@ void GrpcHealthCheckFuzz::expectClientCreate() { void GrpcHealthCheckFuzz::expectStreamCreate() { test_session_->request_encoder_.stream_.callbacks_.clear(); EXPECT_CALL(*test_session_->codec_, newStream(_)) - .WillOnce(DoAll(SaveArgAddress(&test_session_->stream_response_callbacks_), - ReturnRef(test_session_->request_encoder_))); + .WillRepeatedly(DoAll(SaveArgAddress(&test_session_->stream_response_callbacks_), + ReturnRef(test_session_->request_encoder_))); } Network::ConnectionEvent From 60e68963cf6509638b13b67f627481f107dae947 Mon Sep 17 00:00:00 2001 From: code Date: Fri, 5 May 2023 22:45:01 +0800 Subject: [PATCH 125/740] minor fix: eliminate the gmock warning of ssl sock test (#27208) * minor fix: eliminate the gmock warning of ssl sock test Signed-off-by: wbpcode --- .../transport_sockets/tls/ssl_socket_test.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 4ab57d8c5cf9..3a8d1bd817fc 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -5924,10 +5924,12 @@ class SslReadBufferLimitTest : public SslSocketTest { EXPECT_CALL(*client_write_buffer, move(_)) .WillRepeatedly(DoAll(AddBufferToStringWithoutDraining(&data_written), Invoke(client_write_buffer, &MockWatermarkBuffer::baseMove))); - EXPECT_CALL(*client_write_buffer, drain(_)).Times(2).WillOnce(Invoke([&](uint64_t n) -> void { - client_write_buffer->baseDrain(n); - dispatcher_->exit(); - })); + EXPECT_CALL(*client_write_buffer, drain(_)) + .Times(2) + .WillRepeatedly(Invoke([&](uint64_t n) -> void { + client_write_buffer->baseDrain(n); + dispatcher_->exit(); + })); client_connection_->write(buffer_to_write, false); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_EQ(data_to_write, data_written); From aa91f9b852eee1da06e248b74c35de6632468c48 Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Fri, 5 May 2023 12:19:03 -0400 Subject: [PATCH 126/740] xds: Improvements to the xDS delegate and its KV store test (#27153) This commit contains a few improvements to xDS: 1. Fixes a bug where the legacy GrpcMuxImpl did not keep track of xDS delegate initialization per type url, so it could happen that some type urls would not get initialized from the xDS delegate if the xDS management server was down. 2. Adds the ability for the FakeUpstream to optionally differ listener initialization and leave it in manual control of the programmer, instead of initializing the listener for incoming connections by default in the constructor. This is useful if you want to start a FakeUpstream and claim a listener port, but not have the FakeUpstream listen for incoming connections yet until a deferred time. If using deferred initialization, the programmer must explicitely call the FakeUpstream::initializeServer() function. 3. Fixes flakiness in the KeyValueStoreXdsDelegateIntegrationTest by creating, but not initializing, the FakeUpstream (using the functionality described in 2). This ensures that the KV store is always used when restarting the Envoy server, then initializing the FakeUpstream, then ensuring the management server's xDS is used when connectivity is re-established. 4. Using the FakeUpstream with deferred initialization also allows us to get around the problem of using the same port and getting ocassional "address already in use" errors, while still enabling the FakeUpstream to start in a state where it isn't listening for incoming connections, so that the xDS delegate / KV store kicks in. Fixes #26974 Signed-off-by: Ali Beyad --- .../kv_store_xds_delegate_integration_test.cc | 49 ++++++++++--- source/common/config/grpc_mux_impl.cc | 6 +- source/common/config/grpc_mux_impl.h | 3 +- test/integration/fake_upstream.cc | 72 ++++++++++++++----- test/integration/fake_upstream.h | 21 ++++-- 5 files changed, 117 insertions(+), 34 deletions(-) diff --git a/contrib/config/test/kv_store_xds_delegate_integration_test.cc b/contrib/config/test/kv_store_xds_delegate_integration_test.cc index 27c54162935f..049d9e727454 100644 --- a/contrib/config/test/kv_store_xds_delegate_integration_test.cc +++ b/contrib/config/test/kv_store_xds_delegate_integration_test.cc @@ -250,6 +250,25 @@ class KeyValueStoreXdsDelegateIntegrationTest // Set up a new Envoy, using the previous Envoy's configuration, and create the test server. ConfigHelper helper(version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + + // Close the connection between Envoy and the xDS FakeUpstreams. + closeConnection(sds_connection_); + closeConnection(rtds_connection_); + // Bring down the xDS FakeUpstreams. + getSdsUpstream().reset(); + getRtdsUpstream().reset(); + + // Reset the xDS upstreams to new FakeUpstreams, since we'll re-establish a connection to xDS + // after the Envoy server gets re-created and the KV store loaded. + fake_upstreams_.pop_back(); // Removes the RTDS upstream. + fake_upstreams_.pop_back(); // Removes the SDS upstream. + // Adds the new SDS upstream. + fake_upstreams_.emplace_back(std::make_unique( + 0, version_, configWithType(Http::CodecType::HTTP2), /*defer_initialization=*/true)); + // Adds the new RTDS upstream. + fake_upstreams_.emplace_back(std::make_unique( + 0, version_, configWithType(Http::CodecType::HTTP2), /*defer_initialization=*/true)); + std::vector ports; std::vector zero; for (auto& upstream : fake_upstreams_) { @@ -268,13 +287,6 @@ class KeyValueStoreXdsDelegateIntegrationTest named_ports.push_back(static_resources.listeners(i).name()); } - // Simulate the upstream xDS servers going down. - closeConnection(sds_connection_); - closeConnection(rtds_connection_); - rtds_upstream_port_ = getRtdsUpstream()->localAddress()->ip()->port(); - getSdsUpstream().reset(); - getRtdsUpstream().reset(); - // Create and start the new Envoy. createGeneratedApiTestServer(bootstrap_path, named_ports, {false, true, false}, false, test_server_); @@ -366,7 +378,7 @@ TEST_P(KeyValueStoreXdsDelegateIntegrationTest, BasicSuccess) { // Two runtime loads are expected, one for the admin layer and one for the RTDS layer. test_server_->waitForCounterGe("runtime.load_success", 2); - // Verify that the latest resource values are used by Envoy. + // Verify that the latest resource values in the KV store are used by Envoy. EXPECT_EQ(2, test_server_->counter("xds.kv_store.load_success")->value()); EXPECT_EQ(0, test_server_->counter("xds.kv_store.resources_not_found")->value()); EXPECT_EQ(0, test_server_->counter("xds.kv_store.resource_missing")->value()); @@ -375,6 +387,27 @@ TEST_P(KeyValueStoreXdsDelegateIntegrationTest, BasicSuccess) { EXPECT_EQ("whatevs", getRuntimeKey("foo")); EXPECT_EQ("yar", getRuntimeKey("bar")); EXPECT_EQ("saz", getRuntimeKey("baz")); + + // Start the FakeUpstream's listener and re-establish the connection with the RTDS upstream. + getRtdsUpstream()->initializeServer(); + initXdsStream(*getRtdsUpstream(), rtds_connection_, rtds_stream_); + + // Send v2 of the RTDS layer. + auto rtds_resource_v2 = TestUtility::parseYaml(R"EOF( + name: some_rtds_layer + layer: + foo: zoo + baz: jazz + )EOF"); + sendSotwDiscoveryResponse( + Config::TypeUrl::get().Runtime, {rtds_resource_v2}, /*version=*/"3", rtds_stream_.get()); + + test_server_->waitForCounterGe("runtime.load_success", 3); + + // Verify that the values from the xDS response are used instead of from the persisted xDS once + // connectivity is re-established. + EXPECT_EQ("zoo", getRuntimeKey("foo")); + EXPECT_EQ("jazz", getRuntimeKey("baz")); } // A KeyValueStore implementation that returns an invalid proto field value for a Cluster resource. diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 1c26cdc439c9..e35114c721b8 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -332,7 +332,7 @@ void GrpcMuxImpl::onDiscoveryResponse( xds_config_tracker_->onConfigRejected(*message, error_detail->message()); } } - previously_fetched_data_ = true; + api_state.previously_fetched_data_ = true; api_state.request_.set_response_nonce(message->nonce()); ASSERT(api_state.paused()); queueDiscoveryRequest(type_url); @@ -445,7 +445,7 @@ void GrpcMuxImpl::onEstablishmentFailure() { watch->callbacks_.onConfigUpdateFailed( Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); } - if (!previously_fetched_data_) { + if (!api_state.second->previously_fetched_data_) { // On the initialization of the gRPC mux, if connection to the xDS server fails, load the // persisted config, if available. The locally persisted config will be used until // connectivity is established with the xDS server. @@ -453,7 +453,7 @@ void GrpcMuxImpl::onEstablishmentFailure() { /*type_url=*/api_state.first, absl::flat_hash_set{api_state.second->request_.resource_names().begin(), api_state.second->request_.resource_names().end()}); - previously_fetched_data_ = true; + api_state.second->previously_fetched_data_ = true; } } } diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 7bc66280e290..28ebf809a925 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -184,6 +184,8 @@ class GrpcMuxImpl : public GrpcMux, // The identifier for the server that sent the most recent response, or // empty if there is none. std::string control_plane_identifier_{}; + // If true, xDS resources were previously fetched from an xDS source or an xDS delegate. + bool previously_fetched_data_{false}; }; bool isHeartbeatResource(const std::string& type_url, const DecodedResource& resource) { @@ -213,7 +215,6 @@ class GrpcMuxImpl : public GrpcMux, XdsResourcesDelegateOptRef xds_resources_delegate_; const std::string target_xds_authority_; bool first_stream_request_; - bool previously_fetched_data_{false}; // Helper function for looking up and potentially allocating a new ApiState. ApiState& apiStateFor(absl::string_view type_url); diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 96ca4e3afd54..940838d39f64 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -280,10 +280,9 @@ AssertionResult FakeStream::waitForData(Event::Dispatcher& client_dispatcher, return succeeded; } -testing::AssertionResult -FakeStream::waitForData(Event::Dispatcher& client_dispatcher, - const FakeStream::ValidatorFunction& data_validator, - std::chrono::milliseconds timeout) { +AssertionResult FakeStream::waitForData(Event::Dispatcher& client_dispatcher, + const FakeStream::ValidatorFunction& data_validator, + std::chrono::milliseconds timeout) { absl::MutexLock lock(&lock_); if (!waitForWithDispatcherRun( time_system_, lock_, @@ -599,9 +598,10 @@ makeListenSocket(const FakeUpstreamConfig& config, } FakeUpstream::FakeUpstream(uint32_t port, Network::Address::IpVersion version, - const FakeUpstreamConfig& config) + const FakeUpstreamConfig& config, const bool defer_initialization) : FakeUpstream(Network::Test::createRawBufferDownstreamSocketFactory(), - makeListenSocket(config, makeAddress(port, version)), config) {} + makeListenSocket(config, makeAddress(port, version)), config, + defer_initialization) {} FakeUpstream::FakeUpstream(Network::DownstreamTransportSocketFactoryPtr&& transport_socket_factory, const Network::Address::InstanceConstSharedPtr& address, @@ -616,7 +616,8 @@ FakeUpstream::FakeUpstream(Network::DownstreamTransportSocketFactoryPtr&& transp makeListenSocket(config, makeAddress(port, version)), config) {} FakeUpstream::FakeUpstream(Network::DownstreamTransportSocketFactoryPtr&& transport_socket_factory, - Network::SocketPtr&& listen_socket, const FakeUpstreamConfig& config) + Network::SocketPtr&& listen_socket, const FakeUpstreamConfig& config, + const bool defer_initialization) : http_type_(config.upstream_protocol_), http2_options_(config.http2_options_), http3_options_(config.http3_options_), quic_options_(config.quic_options_), socket_(Network::SocketSharedPtr(listen_socket.release())), @@ -637,6 +638,19 @@ FakeUpstream::FakeUpstream(Network::DownstreamTransportSocketFactoryPtr&& transp ->set_value(config.udp_fake_upstream_->max_rx_datagram_size_.value()); } + if (!defer_initialization) { + initializeServer(); + } +} + +FakeUpstream::~FakeUpstream() { cleanUp(); }; + +void FakeUpstream::initializeServer() { + if (initialized_) { + // Already initialized. + return; + } + dispatcher_->post([this]() -> void { socket_factories_[0]->doFinalPreWorkerInit(); handler_->addListener(absl::nullopt, listener_, runtime_); @@ -644,10 +658,9 @@ FakeUpstream::FakeUpstream(Network::DownstreamTransportSocketFactoryPtr&& transp }); thread_ = api_->threadFactory().createThread([this]() -> void { threadRoutine(); }); server_initialized_.waitReady(); + initialized_ = true; } -FakeUpstream::~FakeUpstream() { cleanUp(); }; - void FakeUpstream::cleanUp() { if (thread_.get()) { dispatcher_->exit(); @@ -705,6 +718,11 @@ void FakeUpstream::threadRoutine() { AssertionResult FakeUpstream::waitForHttpConnection(Event::Dispatcher& client_dispatcher, FakeHttpConnectionPtr& connection, milliseconds timeout) { + if (!initialized_) { + return AssertionFailure() + << "Must initialize the FakeUpstream first by calling initializeServer()."; + } + { absl::MutexLock lock(&lock_); @@ -759,6 +777,10 @@ FakeUpstream::waitForHttpConnection(Event::Dispatcher& client_dispatcher, FakeUpstream& upstream = *it; { absl::MutexLock lock(&upstream.lock_); + if (!upstream.isInitialized()) { + return AssertionFailure() + << "Must initialize the FakeUpstream first by calling initializeServer()."; + } if (!waitForWithDispatcherRun( upstream.time_system_, upstream.lock_, [&upstream]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(upstream.lock_) { @@ -784,7 +806,7 @@ FakeUpstream::waitForHttpConnection(Event::Dispatcher& client_dispatcher, } ABSL_MUST_USE_RESULT -testing::AssertionResult FakeUpstream::assertPendingConnectionsEmpty() { +AssertionResult FakeUpstream::assertPendingConnectionsEmpty() { return runOnDispatcherThreadAndWait([&]() { absl::MutexLock lock(&lock_); return new_connections_.empty() ? AssertionSuccess() : AssertionFailure(); @@ -793,6 +815,11 @@ testing::AssertionResult FakeUpstream::assertPendingConnectionsEmpty() { AssertionResult FakeUpstream::waitForRawConnection(FakeRawConnectionPtr& connection, milliseconds timeout) { + if (!initialized_) { + return AssertionFailure() + << "Must initialize the FakeUpstream first by calling initializeServer()."; + } + { absl::MutexLock lock(&lock_); const auto reached = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { @@ -847,8 +874,13 @@ SharedConnectionWrapper& FakeUpstream::consumeConnection() { return *connection_wrapper; } -testing::AssertionResult FakeUpstream::waitForUdpDatagram(Network::UdpRecvData& data_to_fill, - std::chrono::milliseconds timeout) { +AssertionResult FakeUpstream::waitForUdpDatagram(Network::UdpRecvData& data_to_fill, + std::chrono::milliseconds timeout) { + if (!initialized_) { + return AssertionFailure() + << "Must initialize the FakeUpstream first by calling initializeServer()."; + } + absl::MutexLock lock(&lock_); const auto reached = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { return !received_datagrams_.empty(); @@ -893,9 +925,14 @@ void FakeUpstream::sendUdpDatagram(const std::string& buffer, }); } -testing::AssertionResult FakeUpstream::rawWriteConnection(uint32_t index, const std::string& data, - bool end_stream, - std::chrono::milliseconds timeout) { +AssertionResult FakeUpstream::rawWriteConnection(uint32_t index, const std::string& data, + bool end_stream, + std::chrono::milliseconds timeout) { + if (!initialized_) { + return AssertionFailure() + << "Must initialize the FakeUpstream first by calling initializeServer()."; + } + absl::MutexLock lock(&lock_); auto iter = consumed_connections_.begin(); std::advance(iter, index); @@ -994,9 +1031,8 @@ Network::FilterStatus FakeRawConnection::ReadFilter::onData(Buffer::Instance& da } ABSL_MUST_USE_RESULT -testing::AssertionResult -FakeHttpConnection::waitForInexactRawData(const char* data, std::string* out, - std::chrono::milliseconds timeout) { +AssertionResult FakeHttpConnection::waitForInexactRawData(const char* data, std::string* out, + std::chrono::milliseconds timeout) { absl::MutexLock lock(&lock_); const auto reached = [this, data, &out]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { char peek_buf[200]; diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index b4e28712d38b..c0b99df16891 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -653,15 +653,26 @@ class FakeUpstream : Logger::Loggable, const Network::Address::InstanceConstSharedPtr& address, const FakeUpstreamConfig& config); - // Creates a fake upstream bound to INADDR_ANY and the specified |port|. - FakeUpstream(uint32_t port, Network::Address::IpVersion version, - const FakeUpstreamConfig& config); + // Creates a fake upstream bound to INADDR_ANY and the specified `port`. + // Set `defer_initialization` to true if you want the FakeUpstream to not immediately listen for + // incoming connections, and instead want to control when the FakeUpstream is available for + // listening. If `defer_initialization` is set to true, call initializeServer() before invoking + // any other functions in this class. + FakeUpstream(uint32_t port, Network::Address::IpVersion version, const FakeUpstreamConfig& config, + bool defer_initialization = false); FakeUpstream(Network::DownstreamTransportSocketFactoryPtr&& transport_socket_factory, uint32_t port, Network::Address::IpVersion version, const FakeUpstreamConfig& config); ~FakeUpstream() override; + // Initializes the FakeUpstream's server. + void initializeServer(); + + // Returns true if the server has been initialized, i.e. that initializeServer() executed + // successfully. Returns false otherwise. + bool isInitialized() { return initialized_; } + Http::CodecType httpType() { return http_type_; } // Returns the new connection via the connection argument. @@ -760,7 +771,8 @@ class FakeUpstream : Logger::Loggable, private: FakeUpstream(Network::DownstreamTransportSocketFactoryPtr&& transport_socket_factory, - Network::SocketPtr&& connection, const FakeUpstreamConfig& config); + Network::SocketPtr&& connection, const FakeUpstreamConfig& config, + bool defer_initialization = false); class FakeListenSocketFactory : public Network::ListenSocketFactory { public: @@ -930,6 +942,7 @@ class FakeUpstream : Logger::Loggable, #ifdef ENVOY_ENABLE_QUIC Quic::QuicStatNames quic_stat_names_ = Quic::QuicStatNames(stats_store_.symbolTable()); #endif + bool initialized_ = false; }; using FakeUpstreamPtr = std::unique_ptr; From f76cc4da2816759ed141ba71db8090b6aff52239 Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 5 May 2023 18:30:25 +0100 Subject: [PATCH 127/740] bazel: Use vendored tools from `foreign_cc` (#27211) Signed-off-by: Ryan Northey Co-authored-by: Keith Smiley --- bazel/README.md | 17 +---------------- bazel/dependency_imports.bzl | 3 +-- bazel/repository_locations.bzl | 6 +++--- bazel/setup_clang.sh | 8 ++++---- ci/run_envoy_docker.sh | 2 ++ 5 files changed, 11 insertions(+), 25 deletions(-) diff --git a/bazel/README.md b/bazel/README.md index 37e25cd85dd5..116399e4904c 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -75,12 +75,8 @@ for how to update or override dependencies. ```console sudo apt-get install \ autoconf \ - automake \ - cmake \ curl \ libtool \ - make \ - ninja-build \ patch \ python3-pip \ unzip \ @@ -92,13 +88,11 @@ for how to update or override dependencies. ```console dnf install \ aspell-en \ - cmake \ libatomic \ libstdc++ \ libstdc++-static \ libtool \ lld \ - ninja-build \ patch \ python3-pip ``` @@ -128,7 +122,7 @@ for how to update or override dependencies. ### macOS On macOS, you'll need to install several dependencies. This can be accomplished via [Homebrew](https://brew.sh/): ```console - brew install coreutils wget cmake libtool go bazel automake ninja clang-format autoconf aspell python@3.10 + brew install coreutils wget libtool go bazel clang-format autoconf aspell ``` _notes_: `coreutils` is used for `realpath`, `gmd5sum` and `gsha256sum` @@ -227,15 +221,6 @@ for how to update or override dependencies. The Windows SDK contains header files and libraries you need when building Windows applications. Bazel always uses the latest, but you can specify a different version by setting the environment variable `BAZEL_WINSDK_FULL_VERSION`. See [bazel/windows](https://docs.bazel.build/versions/master/windows.html) - Ensure `CMake` and `ninja` binaries are on the PATH. The versions packaged with VC++ Build - Tools are sufficient in most cases, but are 32 bit binaries. These flavors will not run in - the project's GCP CI remote build environment, so 64 bit builds from the CMake and ninja - projects are used instead. - ```cmd - set PATH=%USERPROFILE%\VSBT2019\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin;%PATH% - set PATH=%USERPROFILE%\VSBT2019\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja;%PATH% - ``` - [MSYS2 shell](https://msys2.github.io/): Install to a path with no spaces, e.g. C:\msys64. Set the `BAZEL_SH` environment variable to the path of the installed MSYS2 `bash.exe` diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index 681617f1b87f..72c90bf19d74 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -23,8 +23,7 @@ JQ_VERSION = "1.6" YQ_VERSION = "4.24.4" def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, yq_version = YQ_VERSION): - # TODO: allow building of tools for easier onboarding - rules_foreign_cc_dependencies(register_default_tools = False, register_built_tools = False) + rules_foreign_cc_dependencies() go_rules_dependencies() go_register_toolchains(go_version) envoy_download_go_sdks(go_version) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 2bf42e8df83a..8e00f134b23d 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -870,11 +870,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Rules for using foreign build systems in Bazel", project_desc = "Rules for using foreign build systems in Bazel", project_url = "https://github.com/bazelbuild/rules_foreign_cc", - version = "0.8.0", - sha256 = "6041f1374ff32ba711564374ad8e007aef77f71561a7ce784123b9b4b88614fc", + version = "0.9.0", + sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51", strip_prefix = "rules_foreign_cc-{version}", urls = ["https://github.com/bazelbuild/rules_foreign_cc/archive/{version}.tar.gz"], - release_date = "2022-04-18", + release_date = "2022-08-02", use_category = ["build", "dataplane_core", "controlplane"], license = "Apache-2.0", license_url = "https://github.com/bazelbuild/rules_foreign_cc/blob/{version}/LICENSE", diff --git a/bazel/setup_clang.sh b/bazel/setup_clang.sh index 4a2c1c82a611..a20b21cb1c34 100755 --- a/bazel/setup_clang.sh +++ b/bazel/setup_clang.sh @@ -19,10 +19,10 @@ LLVM_TARGET="$(llvm-config --host-target)" RT_LIBRARY_PATH="${LLVM_LIBDIR}/clang/${LLVM_VERSION}/lib/${LLVM_TARGET}" echo "# Generated file, do not edit. If you want to disable clang, just delete this file. -build:clang --action_env='PATH=${PATH}' -build:clang --action_env=CC=clang -build:clang --action_env=CXX=clang++ -build:clang --action_env='LLVM_CONFIG=${LLVM_PREFIX}/bin/llvm-config' +build:clang --action_env='PATH=${PATH}' --host_action_env='PATH=${PATH}' +build:clang --action_env=CC=clang --host_action_env=CC=clang +build:clang --action_env=CXX=clang++ --host_action_env=CXX=clang++ +build:clang --action_env='LLVM_CONFIG=${LLVM_PREFIX}/bin/llvm-config' --host_action_env='LLVM_CONFIG=${LLVM_PREFIX}/bin/llvm-config' build:clang --repo_env='LLVM_CONFIG=${LLVM_PREFIX}/bin/llvm-config' build:clang --linkopt='-L$(llvm-config --libdir)' build:clang --linkopt='-Wl,-rpath,$(llvm-config --libdir)' diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 355a1f74d41f..e9c52afec150 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -60,6 +60,8 @@ else && usermod -a -G pcap envoybuild \ && chown envoybuild:envoygroup /build \ && chown envoybuild /proc/self/fd/2 \ + && rm -rf /usr/bin/cmake \ + && cmake &> /dev/null || echo 'No cmake here!' \ && sudo -EHs -u envoybuild bash -c 'cd /source && $*'") fi From a3f01a5190b56e576842ac3b08b337c181a02bda Mon Sep 17 00:00:00 2001 From: Joseph Straceski Date: Fri, 5 May 2023 13:45:30 -0400 Subject: [PATCH 128/740] Updating ext_proc filter clear route cache logic. (#26388) Adding in the ability to disable route cache clearing ont he ext_proc filter. Changing the behavior of the clearing to check if there is a header mutation. Risk Level: Low Testing: test/extensions/filters/http/ext_proc/filter_test.cc Docs Changes: docs/root/configuration/http/http_filters/ext_proc_filter.rst Release Notes: changelogs/current.yaml Signed-off-by: Joseph Straceski --- .../filters/http/ext_proc/v3/ext_proc.proto | 7 +- changelogs/current.yaml | 11 ++ .../http/http_filters/ext_proc_filter.rst | 2 + .../filters/http/ext_proc/ext_proc.h | 12 +- .../filters/http/ext_proc/processor_state.cc | 24 +++- .../filters/http/ext_proc/processor_state.h | 3 + .../filters/http/ext_proc/filter_test.cc | 107 ++++++++++++++++-- 7 files changed, 152 insertions(+), 14 deletions(-) diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto index e4b5b695948f..b916f874f30b 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto @@ -95,7 +95,7 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // messages, and the server must reply with // :ref:`ProcessingResponse `. -// [#next-free-field: 11] +// [#next-free-field: 12] message ExternalProcessor { // Configuration for the gRPC service that the filter will communicate with. // The filter supports both the "Envoy" and "Google" gRPC clients. @@ -168,6 +168,11 @@ message ExternalProcessor { // :ref:`override_message_timeout ` // If not specified, by default it is 0, which will effectively disable the ``override_message_timeout`` API. google.protobuf.Duration max_message_timeout = 10; + + // Prevents clearing the route-cache when the + // :ref:`clear_route_cache ` + // field is set in an external processor response. + bool disable_clear_route_cache = 11; } // Extra settings that may be added to per-route configuration for a diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 32208dc7ab0e..6f20d0b1c295 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -38,6 +38,11 @@ minor_behavior_changes: Allow malformed URL encoded triplets in the default header validator. This behavior can be reverted by setting runtime flag ``envoy.reloadable_features.uhv_allow_malformed_url_encoding`` to false, in which case requests with malformed URL encoded triplets in path are rejected. This setting is only applicable when the Unversal Header Validator is enabled and has no effect otherwise. +- area: ext_proc + change: | + When :ref:`clear_route_cache ` is set, ext_proc will check + for header mutations beforce clearing the route cache. Failures due to this check will be counted under the + clear_route_cache_ignored stat. - area: aws change: | Added support for fetching credentials from the AWS credentials file, which only happens if credentials cannot be fetched @@ -109,6 +114,12 @@ new_features: added new configuration field :ref:`domain ` to allow for setting rate limit domains on a per-route basis. +- area: ext_proc + change: | + added new configuration field + :ref:`disable_clear_route_cache ` + to force the ext_proc filter from clearing the route cache. Failures to clear from setting this field will be counted under the + clear_route_cache_disabled stat. - area: redis_proxy change: | added new field :ref:`connection_rate_limit diff --git a/docs/root/configuration/http/http_filters/ext_proc_filter.rst b/docs/root/configuration/http/http_filters/ext_proc_filter.rst index bb3ea275db62..9826bbf63115 100644 --- a/docs/root/configuration/http/http_filters/ext_proc_filter.rst +++ b/docs/root/configuration/http/http_filters/ext_proc_filter.rst @@ -50,3 +50,5 @@ The following statistics are supported: failure_mode_allowed, Counter, The number of times an error was ignored due to configuration message_timeouts, Counter, The number of times a message failed to receive a response within the configured timeout rejected_header_mutations, Counter, The number of rejected header mutations + clear_route_cache_ignored, Counter, The number of clear cache request that were ignored + clear_route_cache_disabled, Counter, The number of clear cache requests that were rejected from being disabled diff --git a/source/extensions/filters/http/ext_proc/ext_proc.h b/source/extensions/filters/http/ext_proc/ext_proc.h index d2cb2cc5e602..0bc052997e88 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.h +++ b/source/extensions/filters/http/ext_proc/ext_proc.h @@ -35,7 +35,9 @@ namespace ExternalProcessing { COUNTER(message_timeouts) \ COUNTER(rejected_header_mutations) \ COUNTER(override_message_timeout_received) \ - COUNTER(override_message_timeout_ignored) + COUNTER(override_message_timeout_ignored) \ + COUNTER(clear_route_cache_ignored) \ + COUNTER(clear_route_cache_disabled) struct ExtProcFilterStats { ALL_EXT_PROC_FILTER_STATS(GENERATE_COUNTER_STRUCT) @@ -73,8 +75,9 @@ class FilterConfig { const std::chrono::milliseconds message_timeout, const uint32_t max_message_timeout_ms, Stats::Scope& scope, const std::string& stats_prefix) - : failure_mode_allow_(config.failure_mode_allow()), message_timeout_(message_timeout), - max_message_timeout_ms_(max_message_timeout_ms), + : failure_mode_allow_(config.failure_mode_allow()), + disable_clear_route_cache_(config.disable_clear_route_cache()), + message_timeout_(message_timeout), max_message_timeout_ms_(max_message_timeout_ms), stats_(generateStats(stats_prefix, config.stat_prefix(), scope)), processing_mode_(config.processing_mode()), mutation_checker_(config.mutation_rules()) {} @@ -94,6 +97,8 @@ class FilterConfig { return mutation_checker_; } + bool disableClearRouteCache() const { return disable_clear_route_cache_; } + private: ExtProcFilterStats generateStats(const std::string& prefix, const std::string& filter_stats_prefix, Stats::Scope& scope) { @@ -102,6 +107,7 @@ class FilterConfig { } const bool failure_mode_allow_; + const bool disable_clear_route_cache_; const std::chrono::milliseconds message_timeout_; const uint32_t max_message_timeout_ms_; diff --git a/source/extensions/filters/http/ext_proc/processor_state.cc b/source/extensions/filters/http/ext_proc/processor_state.cc index 81c225bb236c..e4cd7e0b9175 100644 --- a/source/extensions/filters/http/ext_proc/processor_state.cc +++ b/source/extensions/filters/http/ext_proc/processor_state.cc @@ -1,3 +1,4 @@ +#include "processor_state.h" #include "source/extensions/filters/http/ext_proc/processor_state.h" #include "source/common/buffer/buffer_impl.h" @@ -91,9 +92,11 @@ absl::Status ProcessorState::handleHeadersResponse(const HeadersResponse& respon return mut_status; } } + if (common_response.clear_route_cache()) { - filter_callbacks_->downstreamCallbacks()->clearRouteCache(); + clearRouteCache(common_response); } + onFinishProcessorCall(Grpc::Status::Ok); if (common_response.status() == CommonResponse::CONTINUE_AND_REPLACE) { @@ -321,9 +324,10 @@ absl::Status ProcessorState::handleBodyResponse(const BodyResponse& response) { onFinishProcessorCall(Grpc::Status::FailedPrecondition); } - if (response.response().clear_route_cache()) { - filter_callbacks_->downstreamCallbacks()->clearRouteCache(); + if (common_response.clear_route_cache()) { + clearRouteCache(common_response); } + headers_ = nullptr; if (send_trailers_ && trailers_available_) { @@ -361,6 +365,20 @@ absl::Status ProcessorState::handleTrailersResponse(const TrailersResponse& resp return absl::FailedPreconditionError("spurious message"); } +void ProcessorState::clearRouteCache(const CommonResponse& common_response) { + // Only clear the route cache if there is a mutation to the header and clearing is allowed. + if (filter_.config().disableClearRouteCache()) { + filter_.stats().clear_route_cache_disabled_.inc(); + ENVOY_LOG(debug, "NOT clearing route cache, it is disabled in the config"); + } else if (common_response.has_header_mutation()) { + ENVOY_LOG(debug, "clearing route cache"); + filter_callbacks_->downstreamCallbacks()->clearRouteCache(); + } else { + filter_.stats().clear_route_cache_ignored_.inc(); + ENVOY_LOG(debug, "NOT clearing route cache, no header mutations detected"); + } +} + void ProcessorState::enqueueStreamingChunk(Buffer::Instance& data, bool end_stream, bool delivered) { chunk_queue_.push(data, end_stream, delivered); diff --git a/source/extensions/filters/http/ext_proc/processor_state.h b/source/extensions/filters/http/ext_proc/processor_state.h index 636269353a8c..32bff6adabd0 100644 --- a/source/extensions/filters/http/ext_proc/processor_state.h +++ b/source/extensions/filters/http/ext_proc/processor_state.h @@ -208,6 +208,9 @@ class ProcessorState : public Logger::Loggable { ChunkQueue chunk_queue_; absl::optional call_start_time_ = absl::nullopt; const envoy::config::core::v3::TrafficDirection traffic_direction_; + +private: + void clearRouteCache(const envoy::service::ext_proc::v3::CommonResponse& common_response); }; class DecodingProcessorState : public ProcessorState { diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc index 3f7fe50b915b..aec63f731000 100644 --- a/test/extensions/filters/http/ext_proc/filter_test.cc +++ b/test/extensions/filters/http/ext_proc/filter_test.cc @@ -1978,9 +1978,9 @@ TEST_F(HttpFilterTest, ProcessingModeResponseHeadersOnlyWithoutCallingDecodeHead EXPECT_EQ(1, config_->stats().streams_closed_.value()); } -// Using the default configuration, verify that the "clear_route_cache_ flag makes the appropriate -// callback on the filter. -TEST_F(HttpFilterTest, ClearRouteCache) { +// Using the default configuration, verify that the "clear_route_cache" flag makes the appropriate +// callback on the filter when header modifications are also present. +TEST_F(HttpFilterTest, ClearRouteCacheHeaderMutation) { initialize(R"EOF( grpc_service: envoy_grpc: @@ -1993,6 +1993,10 @@ TEST_F(HttpFilterTest, ClearRouteCache) { EXPECT_CALL(decoder_callbacks_.downstream_callbacks_, clearRouteCache()); processRequestHeaders(false, [](const HttpHeaders&, ProcessingResponse&, HeadersResponse& resp) { + auto* resp_headers_mut = resp.mutable_response()->mutable_header_mutation(); + auto* resp_add = resp_headers_mut->add_set_headers(); + resp_add->mutable_header()->set_key("x-new-header"); + resp_add->mutable_header()->set_value("new"); resp.mutable_response()->set_clear_route_cache(true); }); @@ -2007,15 +2011,104 @@ TEST_F(HttpFilterTest, ClearRouteCache) { EXPECT_CALL(encoder_callbacks_.downstream_callbacks_, clearRouteCache()); processResponseBody([](const HttpBody&, ProcessingResponse&, BodyResponse& resp) { + auto* resp_headers_mut = resp.mutable_response()->mutable_header_mutation(); + auto* resp_add = resp_headers_mut->add_set_headers(); + resp_add->mutable_header()->set_key("x-new-header"); + resp_add->mutable_header()->set_value("new"); resp.mutable_response()->set_clear_route_cache(true); }); filter_->onDestroy(); - EXPECT_EQ(1, config_->stats().streams_started_.value()); - EXPECT_EQ(3, config_->stats().stream_msgs_sent_.value()); - EXPECT_EQ(3, config_->stats().stream_msgs_received_.value()); - EXPECT_EQ(1, config_->stats().streams_closed_.value()); + EXPECT_EQ(config_->stats().streams_started_.value(), 1); + EXPECT_EQ(config_->stats().stream_msgs_sent_.value(), 3); + EXPECT_EQ(config_->stats().stream_msgs_received_.value(), 3); + EXPECT_EQ(config_->stats().streams_closed_.value(), 1); +} + +// Verify that the "disable_route_cache_clearing" setting prevents the "clear_route_cache" flag +// from performing route clearing callbacks when enabled. +TEST_F(HttpFilterTest, ClearRouteCacheDisabledHeaderMutation) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" + processing_mode: + response_body_mode: "BUFFERED" + disable_clear_route_cache: true + )EOF"); + + EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, true)); + processRequestHeaders(false, [](const HttpHeaders&, ProcessingResponse&, HeadersResponse& resp) { + auto* resp_headers_mut = resp.mutable_response()->mutable_header_mutation(); + auto* resp_add = resp_headers_mut->add_set_headers(); + resp_add->mutable_header()->set_key("x-new-header"); + resp_add->mutable_header()->set_value("new"); + resp.mutable_response()->set_clear_route_cache(true); + }); + + EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->encodeHeaders(response_headers_, false)); + processResponseHeaders(true, absl::nullopt); + + Buffer::OwnedImpl resp_data("foo"); + Buffer::OwnedImpl buffered_response_data; + setUpEncodingBuffering(buffered_response_data); + + EXPECT_EQ(FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(resp_data, true)); + processResponseBody([](const HttpBody&, ProcessingResponse&, BodyResponse& resp) { + auto* resp_headers_mut = resp.mutable_response()->mutable_header_mutation(); + auto* resp_add = resp_headers_mut->add_set_headers(); + resp_add->mutable_header()->set_key("x-new-header"); + resp_add->mutable_header()->set_value("new"); + resp.mutable_response()->set_clear_route_cache(true); + }); + + filter_->onDestroy(); + + EXPECT_EQ(config_->stats().clear_route_cache_ignored_.value(), 0); + EXPECT_EQ(config_->stats().clear_route_cache_disabled_.value(), 2); + EXPECT_EQ(config_->stats().streams_started_.value(), 1); + EXPECT_EQ(config_->stats().stream_msgs_sent_.value(), 3); + EXPECT_EQ(config_->stats().stream_msgs_received_.value(), 3); + EXPECT_EQ(config_->stats().streams_closed_.value(), 1); +} + +// Using the default configuration, verify that the "clear_route_cache" flag does not preform +// route clearing callbacks on the filter when no header changes are present. +TEST_F(HttpFilterTest, ClearRouteCacheUnchanged) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" + processing_mode: + response_body_mode: "BUFFERED" + )EOF"); + + EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, true)); + processRequestHeaders(false, [](const HttpHeaders&, ProcessingResponse&, HeadersResponse& resp) { + resp.mutable_response()->set_clear_route_cache(true); + }); + + EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->encodeHeaders(response_headers_, false)); + processResponseHeaders(true, absl::nullopt); + + Buffer::OwnedImpl resp_data("foo"); + Buffer::OwnedImpl buffered_response_data; + setUpEncodingBuffering(buffered_response_data); + + EXPECT_EQ(FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(resp_data, true)); + processResponseBody([](const HttpBody&, ProcessingResponse&, BodyResponse& resp) { + resp.mutable_response()->set_clear_route_cache(true); + }); + + filter_->onDestroy(); + + EXPECT_EQ(config_->stats().clear_route_cache_ignored_.value(), 2); + EXPECT_EQ(config_->stats().clear_route_cache_disabled_.value(), 0); + EXPECT_EQ(config_->stats().streams_started_.value(), 1); + EXPECT_EQ(config_->stats().stream_msgs_sent_.value(), 3); + EXPECT_EQ(config_->stats().stream_msgs_received_.value(), 3); + EXPECT_EQ(config_->stats().streams_closed_.value(), 1); } // Using the default configuration, turn a GET into a POST. From e5228c9363443ba58f295d64adccff6c724b33bb Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Fri, 5 May 2023 16:21:38 -0400 Subject: [PATCH 129/740] Fix flake: //test/integration:overload_integration_test (#27221) Signed-off-by: Kevin Baichoo --- test/integration/overload_integration_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/overload_integration_test.cc b/test/integration/overload_integration_test.cc index 3200dced73ee..cf3aa27fbf1e 100644 --- a/test/integration/overload_integration_test.cc +++ b/test/integration/overload_integration_test.cc @@ -696,6 +696,7 @@ TEST_P(LoadShedPointIntegrationTest, Http1ServerDispatchAbortClosesConnectionWhe ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); upstream_request_->encodeHeaders(default_response_headers_, false); + response->waitForHeaders(); // Put envoy in overloaded state, the next dispatch should fail. updateResource(0.95); From 70be007eadddc2d574098487390d0e4fa0488680 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Fri, 5 May 2023 19:12:49 -0400 Subject: [PATCH 130/740] Deflake TcpProxyManyConnections test (#27223) Signed-off-by: Yan Avlasov --- test/integration/BUILD | 30 +++++++ test/integration/integration_tcp_client.cc | 7 +- test/integration/integration_tcp_client.h | 1 + .../integration/tcp_proxy_integration_test.cc | 40 ---------- .../tcp_proxy_many_connections_test.cc | 79 +++++++++++++++++++ 5 files changed, 116 insertions(+), 41 deletions(-) create mode 100644 test/integration/tcp_proxy_many_connections_test.cc diff --git a/test/integration/BUILD b/test/integration/BUILD index d49e57963533..95186ba1941d 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1563,6 +1563,36 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "tcp_proxy_many_connections_test", + size = "large", + srcs = [ + "tcp_proxy_many_connections_test.cc", + ], + shard_count = 1, + tags = [ + "cpu:4", + ], + deps = [ + ":integration_lib", + "//source/common/config:api_version_lib", + "//source/common/event:dispatcher_includes", + "//source/common/event:dispatcher_lib", + "//source/common/network:utility_lib", + "//source/extensions/access_loggers/file:config", + "//source/extensions/filters/network/common:factory_base_lib", + "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/load_balancing_policies/subset:config", + "//source/extensions/transport_sockets/tls:config", + "//source/extensions/transport_sockets/tls:context_config_lib", + "//source/extensions/transport_sockets/tls:context_lib", + "//test/mocks/runtime:runtime_mocks", + "//test/mocks/secret:secret_mocks", + "//test/test_common:registry_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "tcp_tunneling_integration_test", size = "large", diff --git a/test/integration/integration_tcp_client.cc b/test/integration/integration_tcp_client.cc index 34b52775dbdd..fcf5251850de 100644 --- a/test/integration/integration_tcp_client.cc +++ b/test/integration/integration_tcp_client.cc @@ -109,12 +109,17 @@ void IntegrationTcpClient::waitForDisconnect(bool ignore_spurious_events) { } void IntegrationTcpClient::waitForHalfClose(bool ignore_spurious_events) { + waitForHalfClose(TestUtility::DefaultTimeout, ignore_spurious_events); +} + +void IntegrationTcpClient::waitForHalfClose(std::chrono::milliseconds timeout, + bool ignore_spurious_events) { if (payload_reader_->readLastByte()) { return; } Event::TimerPtr timeout_timer = connection_->dispatcher().createTimer([this]() -> void { connection_->dispatcher().exit(); }); - timeout_timer->enableTimer(TestUtility::DefaultTimeout); + timeout_timer->enableTimer(timeout); if (ignore_spurious_events) { while (!payload_reader_->readLastByte() && timeout_timer->enabled()) { diff --git a/test/integration/integration_tcp_client.h b/test/integration/integration_tcp_client.h index d4936ad51b9b..f29561d300b9 100644 --- a/test/integration/integration_tcp_client.h +++ b/test/integration/integration_tcp_client.h @@ -39,6 +39,7 @@ class IntegrationTcpClient { waitForData(size_t length, std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); void waitForDisconnect(bool ignore_spurious_events = false); void waitForHalfClose(bool ignore_spurious_events = false); + void waitForHalfClose(std::chrono::milliseconds timeout, bool ignore_spurious_events = false); void readDisable(bool disabled); ABSL_MUST_USE_RESULT AssertionResult write(const std::string& data, bool end_stream = false, bool verify = true, diff --git a/test/integration/tcp_proxy_integration_test.cc b/test/integration/tcp_proxy_integration_test.cc index 20837ec3f68d..ebb9b4362e64 100644 --- a/test/integration/tcp_proxy_integration_test.cc +++ b/test/integration/tcp_proxy_integration_test.cc @@ -163,46 +163,6 @@ TEST_P(TcpProxyIntegrationTest, TcpProxyDownstreamDisconnectBytesMeter) { "\r?.*"))); } -TEST_P(TcpProxyIntegrationTest, TcpProxyManyConnections) { - autonomous_upstream_ = true; - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* static_resources = bootstrap.mutable_static_resources(); - for (int i = 0; i < static_resources->clusters_size(); ++i) { - auto* cluster = static_resources->mutable_clusters(i); - auto* thresholds = cluster->mutable_circuit_breakers()->add_thresholds(); - thresholds->mutable_max_connections()->set_value(1027); - thresholds->mutable_max_pending_requests()->set_value(1027); - } - }); - initialize(); -// The large number of connection is meant to regression test -// https://github.com/envoyproxy/envoy/issues/19033 but fails on apple CI -// TODO(alyssawilk) debug. -#if defined(__APPLE__) - const int num_connections = 50; -#else - const int num_connections = 1026; -#endif - std::vector clients(num_connections); - - for (int i = 0; i < num_connections; ++i) { - clients[i] = makeTcpConnection(lookupPort("tcp_proxy")); - } - test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_total", num_connections); - for (int i = 0; i < num_connections; ++i) { - IntegrationTcpClientPtr& tcp_client = clients[i]; - // The autonomous upstream is an HTTP upstream, so send raw HTTP. - // This particular request will result in the upstream sending a response, - // and flush-closing due to the 'close_after_response' header. - ASSERT_TRUE(tcp_client->write( - "GET / HTTP/1.1\r\nHost: foo\r\nclose_after_response: yes\r\ncontent-length: 0\r\n\r\n", - false)); - tcp_client->waitForHalfClose(); - tcp_client->close(); - EXPECT_THAT(tcp_client->data(), HasSubstr("aaaaaaaaaa")); - } -} - TEST_P(TcpProxyIntegrationTest, TcpProxyRandomBehavior) { autonomous_upstream_ = true; initialize(); diff --git a/test/integration/tcp_proxy_many_connections_test.cc b/test/integration/tcp_proxy_many_connections_test.cc new file mode 100644 index 000000000000..18d8ed2c01f1 --- /dev/null +++ b/test/integration/tcp_proxy_many_connections_test.cc @@ -0,0 +1,79 @@ +#include +#include + +#include "source/common/network/utility.h" + +#include "test/integration/integration.h" +#include "test/integration/utility.h" +#include "test/test_common/registry.h" + +#include "gtest/gtest.h" + +using testing::HasSubstr; + +namespace Envoy { + +class TcpProxyManyConnectionsTest : public testing::TestWithParam, + public BaseIntegrationTest { +public: + TcpProxyManyConnectionsTest() : BaseIntegrationTest(GetParam(), ConfigHelper::tcpProxyConfig()) { + enableHalfClose(true); + } + + void initialize() override { + config_helper_.renameListener("tcp_proxy"); + BaseIntegrationTest::initialize(); + } + + // Multiplier for increasing timeouts in this test. It was empirically determined by running a + // debug build with CPU oversubscribed by a factor of 5 on 3GHz CPU. + static constexpr int timeout_scaling_factor_ = 20; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, TcpProxyManyConnectionsTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(TcpProxyManyConnectionsTest, TcpProxyManyConnections) { + autonomous_upstream_ = true; + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* static_resources = bootstrap.mutable_static_resources(); + for (int i = 0; i < static_resources->clusters_size(); ++i) { + auto* cluster = static_resources->mutable_clusters(i); + cluster->mutable_connect_timeout()->set_seconds(timeout_scaling_factor_); + auto* thresholds = cluster->mutable_circuit_breakers()->add_thresholds(); + thresholds->mutable_max_connections()->set_value(1027); + thresholds->mutable_max_pending_requests()->set_value(1027); + } + }); + initialize(); +// The large number of connection is meant to regression test +// https://github.com/envoyproxy/envoy/issues/19033 but fails on apple CI +// TODO(alyssawilk) debug. +#if defined(__APPLE__) + const int num_connections = 50; +#else + const int num_connections = 1026; +#endif + std::vector clients(num_connections); + + for (int i = 0; i < num_connections; ++i) { + clients[i] = makeTcpConnection(lookupPort("tcp_proxy")); + test_server_->waitForGaugeGe("cluster.cluster_0.upstream_cx_active", i, + TestUtility::DefaultTimeout * timeout_scaling_factor_); + } + for (int i = 0; i < num_connections; ++i) { + IntegrationTcpClientPtr& tcp_client = clients[i]; + // The autonomous upstream is an HTTP upstream, so send raw HTTP. + // This particular request will result in the upstream sending a response, + // and flush-closing due to the 'close_after_response' header. + ASSERT_TRUE(tcp_client->write( + "GET / HTTP/1.1\r\nHost: foo\r\nclose_after_response: yes\r\ncontent-length: 0\r\n\r\n", + false)); + tcp_client->waitForHalfClose(TestUtility::DefaultTimeout * timeout_scaling_factor_); + tcp_client->close(); + EXPECT_THAT(tcp_client->data(), HasSubstr("aaaaaaaaaa")); + } +} + +} // namespace Envoy From 294dacde669772efd447e0b4f2af76a1083146f2 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Fri, 5 May 2023 20:49:33 -0400 Subject: [PATCH 131/740] Deflake tap_filter_integration_test under tsan (#27186) Signed-off-by: Yan Avlasov --- .../extensions/filters/http/tap/tap_filter_integration_test.cc | 3 +++ test/test_common/logging.h | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/http/tap/tap_filter_integration_test.cc b/test/extensions/filters/http/tap/tap_filter_integration_test.cc index 968cebca58e1..af716d2917da 100644 --- a/test/extensions/filters/http/tap/tap_filter_integration_test.cc +++ b/test/extensions/filters/http/tap/tap_filter_integration_test.cc @@ -25,6 +25,9 @@ class TapIntegrationTest : public testing::TestWithParamsetShouldEscape(false); } void initializeFilter(const std::string& filter_config) { diff --git a/test/test_common/logging.h b/test/test_common/logging.h index b5855755c3f5..09314642a79c 100644 --- a/test/test_common/logging.h +++ b/test/test_common/logging.h @@ -207,7 +207,6 @@ using ExpectedLogMessages = std::vector; do { \ Envoy::LogLevelSetter save_levels(spdlog::level::trace); \ Envoy::Logger::DelegatingLogSinkSharedPtr sink_ptr = Envoy::Logger::Registry::getSink(); \ - sink_ptr->setShouldEscape(false); \ std::string loglevel = loglevel_raw; \ std::string substr = substr_raw; \ Envoy::LogRecordingSink log_recorder(sink_ptr); \ From 5523257c7059f0748611b0d83cfa7c28268d0df9 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Sat, 6 May 2023 00:31:59 -0400 Subject: [PATCH 132/740] stats: revert "Prom stats perf improvements" (#27226) Commit Message: Reverts #24998 See #27173 Additional Description: Risk Level: n/a Testing: n/a Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a --- source/server/admin/BUILD | 27 +- source/server/admin/admin.cc | 10 +- source/server/admin/grouped_stats_request.cc | 117 ------- source/server/admin/grouped_stats_request.h | 61 ---- source/server/admin/prometheus_stats.cc | 285 ++++++++++++++++++ source/server/admin/prometheus_stats.h | 51 ++++ source/server/admin/stats_handler.cc | 108 ++++--- source/server/admin/stats_handler.h | 56 +++- source/server/admin/stats_render.cc | 165 ---------- source/server/admin/stats_render.h | 122 +------- source/server/admin/stats_request.cc | 190 ++++++++---- source/server/admin/stats_request.h | 124 +++----- .../server/admin/ungrouped_stats_request.cc | 129 -------- source/server/admin/ungrouped_stats_request.h | 49 --- test/server/admin/BUILD | 38 +-- test/server/admin/admin_test.cc | 7 +- .../admin/grouped_stats_request_test.cc | 90 ------ test/server/admin/prometheus_stats_test.cc | 279 ++++++++--------- test/server/admin/stats_handler_speed_test.cc | 7 +- test/server/admin/stats_handler_test.cc | 7 +- test/server/admin/stats_render_test.cc | 2 + test/server/admin/stats_request_test.cc | 167 ++++++++++ test/server/admin/stats_request_test_base.cc | 65 ---- test/server/admin/stats_request_test_base.h | 82 ----- .../admin/ungrouped_stats_request_test.cc | 88 ------ tools/code_format/config.yaml | 7 +- 26 files changed, 973 insertions(+), 1360 deletions(-) delete mode 100644 source/server/admin/grouped_stats_request.cc delete mode 100644 source/server/admin/grouped_stats_request.h create mode 100644 source/server/admin/prometheus_stats.cc create mode 100644 source/server/admin/prometheus_stats.h delete mode 100644 source/server/admin/ungrouped_stats_request.cc delete mode 100644 source/server/admin/ungrouped_stats_request.h delete mode 100644 test/server/admin/grouped_stats_request_test.cc create mode 100644 test/server/admin/stats_request_test.cc delete mode 100644 test/server/admin/stats_request_test_base.cc delete mode 100644 test/server/admin/stats_request_test_base.h delete mode 100644 test/server/admin/ungrouped_stats_request_test.cc diff --git a/source/server/admin/BUILD b/source/server/admin/BUILD index 7d4123c7d5aa..7fa1d22fac92 100644 --- a/source/server/admin/BUILD +++ b/source/server/admin/BUILD @@ -139,18 +139,11 @@ envoy_cc_library( envoy_cc_library( name = "stats_request_lib", - srcs = [ - "grouped_stats_request.cc", - "stats_request.cc", - "ungrouped_stats_request.cc", - ], - hdrs = [ - "grouped_stats_request.h", - "stats_request.h", - "ungrouped_stats_request.h", - ], + srcs = ["stats_request.cc"], + hdrs = ["stats_request.h"], deps = [ ":handler_ctx_lib", + ":prometheus_stats_lib", ":stats_params_lib", ":stats_render_lib", ":utils_lib", @@ -190,6 +183,7 @@ envoy_cc_library( hdrs = ["stats_handler.h"], deps = [ ":handler_ctx_lib", + ":prometheus_stats_lib", ":stats_render_lib", ":stats_request_lib", ":utils_lib", @@ -203,6 +197,19 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "prometheus_stats_lib", + srcs = ["prometheus_stats.cc"], + hdrs = ["prometheus_stats.h"], + deps = [ + ":stats_params_lib", + ":utils_lib", + "//envoy/stats:custom_stat_namespaces_interface", + "//source/common/buffer:buffer_lib", + "//source/common/stats:histogram_lib", + ], +) + envoy_cc_library( name = "listeners_handler_lib", srcs = ["listeners_handler.cc"], diff --git a/source/server/admin/admin.cc b/source/server/admin/admin.cc index 1025bd5e44e1..f71f71c5abae 100644 --- a/source/server/admin/admin.cc +++ b/source/server/admin/admin.cc @@ -187,7 +187,15 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server, makeHandler("/ready", "print server state, return 200 if LIVE, otherwise return 503", MAKE_ADMIN_HANDLER(server_info_handler_.handlerReady), false, false), stats_handler_.statsHandler(false /* not active mode */), - stats_handler_.prometheusStatsHandler(), + makeHandler("/stats/prometheus", "print server stats in prometheus format", + MAKE_ADMIN_HANDLER(stats_handler_.handlerPrometheusStats), false, false, + {{ParamDescriptor::Type::Boolean, "usedonly", + "Only include stats that have been written by system since restart"}, + {ParamDescriptor::Type::Boolean, "text_readouts", + "Render text_readouts as new gaugues with value 0 (increases Prometheus " + "data size)"}, + {ParamDescriptor::Type::String, "filter", + "Regular expression (Google re2) for filtering stats"}}), makeHandler("/stats/recentlookups", "Show recent stat-name lookups", MAKE_ADMIN_HANDLER(stats_handler_.handlerStatsRecentLookups), false, false), makeHandler("/stats/recentlookups/clear", "clear list of stat-name lookups and counter", diff --git a/source/server/admin/grouped_stats_request.cc b/source/server/admin/grouped_stats_request.cc deleted file mode 100644 index d4b86a81f6fb..000000000000 --- a/source/server/admin/grouped_stats_request.cc +++ /dev/null @@ -1,117 +0,0 @@ -#include "source/server/admin/grouped_stats_request.h" - -#include -#include - -#include "source/server/admin/stats_render.h" - -namespace Envoy { -namespace Server { - -GroupedStatsRequest::GroupedStatsRequest(Stats::Store& stats, const StatsParams& params, - Stats::CustomStatNamespaces& custom_namespaces, - UrlHandlerFn url_handler_fn) - : StatsRequest(stats, params, url_handler_fn), custom_namespaces_(custom_namespaces), - global_symbol_table_(stats.constSymbolTable()) { - - // The "type" query param is ignored for prometheus stats, so always start from - // counters; also, skip the TextReadouts phase unless that stat type is explicitly - // requested via query param. - if (params_.prometheus_text_readouts_) { - phases_ = {Phase::Counters, Phase::Gauges, Phase::TextReadouts, Phase::Histograms}; - } else { - phases_ = {Phase::Counters, Phase::Gauges, Phase::Histograms}; - } - phase_index_ = 0; -} - -template Stats::IterateFn GroupedStatsRequest::saveMatchingStat() { - return [this](const Stats::RefcountPtr& stat) -> bool { - // Check if unused. - if (params_.used_only_ && !stat->used()) { - return true; - } - - // Check if filtered. - if (params_.re2_filter_ != nullptr && - !re2::RE2::PartialMatch(stat->name(), *params_.re2_filter_)) { - return true; - } - - // Capture stat. - std::string tag_extracted_name = global_symbol_table_.toString(stat->tagExtractedStatName()); - stat_map_.insert({tag_extracted_name, std::vector>({})}); - absl::get>>(stat_map_[tag_extracted_name]) - .emplace_back(stat); - return true; - }; -} - -Stats::IterateFn GroupedStatsRequest::saveMatchingStatForTextReadout() { - return saveMatchingStat(); -} - -Stats::IterateFn GroupedStatsRequest::saveMatchingStatForGauge() { - return saveMatchingStat(); -} - -Stats::IterateFn GroupedStatsRequest::saveMatchingStatForCounter() { - return saveMatchingStat(); -} - -Stats::IterateFn GroupedStatsRequest::saveMatchingStatForHistogram() { - return saveMatchingStat(); -} - -template -void GroupedStatsRequest::renderStat(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - auto prefixed_tag_extracted_name = prefixedTagExtractedName(name); - if (prefixed_tag_extracted_name.has_value()) { - ++phase_stat_count_; - - // Sort group. - std::vector group = absl::get>(variant); - global_symbol_table_.sortByStatNames( - group.begin(), group.end(), - [](const SharedStatType& stat_ptr) -> Stats::StatName { return stat_ptr->statName(); }); - - // Render group. - render_->generate(response, prefixed_tag_extracted_name.value(), group); - } -} - -void GroupedStatsRequest::processTextReadout(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - renderStat(name, response, variant); -} - -void GroupedStatsRequest::processCounter(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - - renderStat(name, response, variant); -} - -void GroupedStatsRequest::processGauge(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - renderStat(name, response, variant); -} - -void GroupedStatsRequest::processHistogram(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - renderStat(name, response, variant); -} - -template -absl::optional -GroupedStatsRequest::prefixedTagExtractedName(const std::string& tag_extracted_name) { - return Envoy::Server::PrometheusStatsRender::metricName(tag_extracted_name, custom_namespaces_); -} - -void GroupedStatsRequest::setRenderPtr(Http::ResponseHeaderMap&) { - ASSERT(params_.format_ == StatsFormat::Prometheus); - render_ = std::make_unique(); -} - -} // namespace Server -} // namespace Envoy diff --git a/source/server/admin/grouped_stats_request.h b/source/server/admin/grouped_stats_request.h deleted file mode 100644 index 8fce7a9cf901..000000000000 --- a/source/server/admin/grouped_stats_request.h +++ /dev/null @@ -1,61 +0,0 @@ -#pragma once - -#include "source/server/admin/stats_request.h" - -namespace Envoy { -namespace Server { - -// In the context of this class, a stat is represented by a group of -// gauges, counters etc. with the same tag-extracted name. This class -// is currently used for Prometheus stats only (and has some Prometheus-specific -// logic in it), but if needed could be generalized for other -// formats. -// TODO(rulex123): cleanup any Prometheus-specific logic if we decide to have a grouped view -// for HTML or JSON stats. -class GroupedStatsRequest - : public StatsRequest, - std::vector, std::vector, - std::vector> { - -public: - GroupedStatsRequest(Stats::Store& stats, const StatsParams& params, - Stats::CustomStatNamespaces& custom_namespaces, - UrlHandlerFn url_handler_fn = nullptr); - -protected: - Stats::IterateFn saveMatchingStatForTextReadout() override; - Stats::IterateFn saveMatchingStatForGauge() override; - Stats::IterateFn saveMatchingStatForCounter() override; - Stats::IterateFn saveMatchingStatForHistogram() override; - template Stats::IterateFn saveMatchingStat(); - - void processTextReadout(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) override; - void processGauge(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) override; - void processCounter(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) override; - void processHistogram(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) override; - - // GroupedStatsRequest - template - absl::optional prefixedTagExtractedName(const std::string& tag_extracted_name); - - template - void renderStat(const std::string& name, Buffer::Instance& response, const StatOrScopes& variant); - - void setRenderPtr(Http::ResponseHeaderMap& response_headers) override; - StatsRenderBase& render() override { - ASSERT(render_ != nullptr); - return *render_; - } - -private: - Stats::CustomStatNamespaces& custom_namespaces_; - const Stats::SymbolTable& global_symbol_table_; - std::unique_ptr render_; -}; - -} // namespace Server -} // namespace Envoy diff --git a/source/server/admin/prometheus_stats.cc b/source/server/admin/prometheus_stats.cc new file mode 100644 index 000000000000..d65891f1c29a --- /dev/null +++ b/source/server/admin/prometheus_stats.cc @@ -0,0 +1,285 @@ +#include "source/server/admin/prometheus_stats.h" + +#include "source/common/common/empty_string.h" +#include "source/common/common/macros.h" +#include "source/common/common/regex.h" +#include "source/common/stats/histogram_impl.h" + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" + +namespace Envoy { +namespace Server { + +namespace { + +const Regex::CompiledGoogleReMatcher& promRegex() { + CONSTRUCT_ON_FIRST_USE(Regex::CompiledGoogleReMatcher, "[^a-zA-Z0-9_]", false); +} + +/** + * Take a string and sanitize it according to Prometheus conventions. + */ +std::string sanitizeName(const absl::string_view name) { + // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by + // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/. + // The initial [a-zA-Z_] constraint is always satisfied by the namespace prefix. + return promRegex().replaceAll(name, "_"); +} + +/** + * Take tag values and sanitize it for text serialization, according to + * Prometheus conventions. + */ +std::string sanitizeValue(const absl::string_view value) { + // Removes problematic characters from Prometheus tag values to prevent + // text serialization issues. This matches the prometheus text formatting code: + // https://github.com/prometheus/common/blob/88f1636b699ae4fb949d292ffb904c205bf542c9/expfmt/text_create.go#L419-L420. + // The goal is to replace '\' with "\\", newline with "\n", and '"' with "\"". + return absl::StrReplaceAll(value, { + {R"(\)", R"(\\)"}, + {"\n", R"(\n)"}, + {R"(")", R"(\")"}, + }); +} + +/* + * Determine whether a metric has never been emitted and choose to + * not show it if we only wanted used metrics. + */ +template +static bool shouldShowMetric(const StatType& metric, const StatsParams& params) { + // This duplicates logic in StatsRequest::populateStatsFromScopes, but differs + // in one subtle way: in Prometheus we only use metric.name() for filtering, + // not rendering, so we only construct the name if there's a filter. + if (params.used_only_ && !metric.used()) { + return false; + } + if (params.re2_filter_ != nullptr && + !re2::RE2::PartialMatch(metric.name(), *params.re2_filter_)) { + return false; + } + return true; +} + +/* + * Comparator for Stats::Metric that does not require a string representation + * to make the comparison, for memory efficiency. + */ +struct MetricLessThan { + bool operator()(const Stats::Metric* a, const Stats::Metric* b) const { + ASSERT(&a->constSymbolTable() == &b->constSymbolTable()); + return a->constSymbolTable().lessThan(a->statName(), b->statName()); + } +}; + +/** + * Processes a stat type (counter, gauge, histogram) by generating all output lines, sorting + * them by tag-extracted metric name, and then outputting them in the correct sorted order into + * response. + * + * @param response The buffer to put the output into. + * @param used_only Whether to only output stats that are used. + * @param regex A filter on which stats to output. + * @param metrics The metrics to output stats for. This must contain all stats of the given type + * to be included in the same output. + * @param generate_output A function which returns the output text for this metric. + * @param type The name of the prometheus metric type for used in TYPE annotations. + */ +template +uint64_t outputStatType( + Buffer::Instance& response, const StatsParams& params, + const std::vector>& metrics, + const std::function& generate_output, + absl::string_view type, const Stats::CustomStatNamespaces& custom_namespaces) { + + /* + * From + * https:*github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting: + * + * All lines for a given metric must be provided as one single group, with the optional HELP and + * TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated + * expositions is preferred but not required, i.e. do not sort if the computational cost is + * prohibitive. + */ + + // This is an unsorted collection of dumb-pointers (no need to increment then decrement every + // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will + // be sorted before producing the final output to satisfy the "preferred" ordering from the + // prometheus spec: metrics will be sorted by their tags' textual representation, which will be + // consistent across calls. + using StatTypeUnsortedCollection = std::vector; + + // Return early to avoid crashing when getting the symbol table from the first metric. + if (metrics.empty()) { + return 0; + } + + // There should only be one symbol table for all of the stats in the admin + // interface. If this assumption changes, the name comparisons in this function + // will have to change to compare to convert all StatNames to strings before + // comparison. + const Stats::SymbolTable& global_symbol_table = metrics.front()->constSymbolTable(); + + // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements + // of the exposition format. + std::map groups( + global_symbol_table); + + for (const auto& metric : metrics) { + ASSERT(&global_symbol_table == &metric->constSymbolTable()); + if (!shouldShowMetric(*metric, params)) { + continue; + } + groups[metric->tagExtractedStatName()].push_back(metric.get()); + } + + auto result = groups.size(); + for (auto& group : groups) { + const absl::optional prefixed_tag_extracted_name = + PrometheusStatsFormatter::metricName(global_symbol_table.toString(group.first), + custom_namespaces); + if (!prefixed_tag_extracted_name.has_value()) { + --result; + continue; + } + response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name.value(), type)); + + // Sort before producing the final output to satisfy the "preferred" ordering from the + // prometheus spec: metrics will be sorted by their tags' textual representation, which will + // be consistent across calls. + std::sort(group.second.begin(), group.second.end(), MetricLessThan()); + + for (const auto& metric : group.second) { + response.add(generate_output(*metric, prefixed_tag_extracted_name.value())); + } + } + return result; +} + +/* + * Return the prometheus output for a numeric Stat (Counter or Gauge). + */ +template +std::string generateNumericOutput(const StatType& metric, + const std::string& prefixed_tag_extracted_name) { + const std::string tags = PrometheusStatsFormatter::formattedTags(metric.tags()); + return fmt::format("{0}{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, metric.value()); +} + +/* + * Returns the prometheus output for a TextReadout in gauge format. + * It is a workaround of a limitation of prometheus which stores only numeric metrics. + * The output is a gauge named the same as a given text-readout. The value of returned gauge is + * always equal to 0. Returned gauge contains all tags of a given text-readout and one additional + * tag {"text_value":"textReadout.value"}. + */ +std::string generateTextReadoutOutput(const Stats::TextReadout& text_readout, + const std::string& prefixed_tag_extracted_name) { + auto tags = text_readout.tags(); + tags.push_back(Stats::Tag{"text_value", text_readout.value()}); + const std::string formattedTags = PrometheusStatsFormatter::formattedTags(tags); + return fmt::format("{0}{{{1}}} 0\n", prefixed_tag_extracted_name, formattedTags); +} + +/* + * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded + * newlines) that contains all the individual bucket counts and sum/count for a single histogram + * (metric_name plus all tags). + */ +std::string generateHistogramOutput(const Stats::ParentHistogram& histogram, + const std::string& prefixed_tag_extracted_name) { + const std::string tags = PrometheusStatsFormatter::formattedTags(histogram.tags()); + const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ","); + + const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics(); + Stats::ConstSupportedBuckets& supported_buckets = stats.supportedBuckets(); + const std::vector& computed_buckets = stats.computedBuckets(); + std::string output; + for (size_t i = 0; i < supported_buckets.size(); ++i) { + double bucket = supported_buckets[i]; + uint64_t value = computed_buckets[i]; + // We want to print the bucket in a fixed point (non-scientific) format. The fmt library + // doesn't have a specific modifier to format as a fixed-point value only so we use the + // 'g' operator which prints the number in general fixed point format or scientific format + // with precision 50 to round the number up to 32 significant digits in fixed point format + // which should cover pretty much all cases + output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", prefixed_tag_extracted_name, + hist_tags, bucket, value)); + } + + output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", prefixed_tag_extracted_name, + hist_tags, stats.sampleCount())); + output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", prefixed_tag_extracted_name, tags, + stats.sampleSum())); + output.append(fmt::format("{0}_count{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, + stats.sampleCount())); + + return output; +}; + +} // namespace + +std::string PrometheusStatsFormatter::formattedTags(const std::vector& tags) { + std::vector buf; + buf.reserve(tags.size()); + for (const Stats::Tag& tag : tags) { + buf.push_back(fmt::format("{}=\"{}\"", sanitizeName(tag.name_), sanitizeValue(tag.value_))); + } + return absl::StrJoin(buf, ","); +} + +absl::optional +PrometheusStatsFormatter::metricName(const std::string& extracted_name, + const Stats::CustomStatNamespaces& custom_namespaces) { + const absl::optional custom_namespace_stripped = + custom_namespaces.stripRegisteredPrefix(extracted_name); + if (custom_namespace_stripped.has_value()) { + // This case the name has a custom namespace, and it is a custom metric. + const std::string sanitized_name = sanitizeName(custom_namespace_stripped.value()); + // We expose these metrics without modifying (e.g. without "envoy_"), + // so we have to check the "user-defined" stat name complies with the Prometheus naming + // convention. Specifically the name must start with the "[a-zA-Z_]" pattern. + // All the characters in sanitized_name are already in "[a-zA-Z0-9_]" pattern + // thanks to sanitizeName above, so the only thing we have to do is check + // if it does not start with digits. + if (sanitized_name.empty() || absl::ascii_isdigit(sanitized_name.front())) { + return absl::nullopt; + } + return sanitized_name; + } + + // If it does not have a custom namespace, add namespacing prefix to avoid conflicts, as per best + // practice: https://prometheus.io/docs/practices/naming/#metric-names Also, naming conventions on + // https://prometheus.io/docs/concepts/data_model/ + return absl::StrCat("envoy_", sanitizeName(extracted_name)); +} + +uint64_t PrometheusStatsFormatter::statsAsPrometheus( + const std::vector& counters, + const std::vector& gauges, + const std::vector& histograms, + const std::vector& text_readouts, Buffer::Instance& response, + const StatsParams& params, const Stats::CustomStatNamespaces& custom_namespaces) { + + uint64_t metric_name_count = 0; + metric_name_count += outputStatType(response, params, counters, + generateNumericOutput, + "counter", custom_namespaces); + + metric_name_count += outputStatType( + response, params, gauges, generateNumericOutput, "gauge", custom_namespaces); + + // TextReadout stats are returned in gauge format, so "gauge" type is set intentionally. + metric_name_count += outputStatType( + response, params, text_readouts, generateTextReadoutOutput, "gauge", custom_namespaces); + + metric_name_count += outputStatType( + response, params, histograms, generateHistogramOutput, "histogram", custom_namespaces); + + return metric_name_count; +} + +} // namespace Server +} // namespace Envoy diff --git a/source/server/admin/prometheus_stats.h b/source/server/admin/prometheus_stats.h new file mode 100644 index 000000000000..9677e86685c1 --- /dev/null +++ b/source/server/admin/prometheus_stats.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include + +#include "envoy/buffer/buffer.h" +#include "envoy/stats/custom_stat_namespaces.h" +#include "envoy/stats/histogram.h" +#include "envoy/stats/stats.h" + +#include "source/server/admin/stats_params.h" + +namespace Envoy { +namespace Server { +/** + * Formatter for metric/labels exported to Prometheus. + * + * See: https://prometheus.io/docs/concepts/data_model + */ +class PrometheusStatsFormatter { +public: + /** + * Extracts counters and gauges and relevant tags, appending them to + * the response buffer after sanitizing the metric / label names. + * @return uint64_t total number of metric types inserted in response. + */ + static uint64_t statsAsPrometheus(const std::vector& counters, + const std::vector& gauges, + const std::vector& histograms, + const std::vector& text_readouts, + Buffer::Instance& response, const StatsParams& params, + const Stats::CustomStatNamespaces& custom_namespaces); + /** + * Format the given tags, returning a string as a comma-separated list + * of ="" pairs. + */ + static std::string formattedTags(const std::vector& tags); + + /** + * Format the given metric name, and prefixed with "envoy_" if it does not have a custom + * stat namespace. If it has a custom stat namespace AND the name without the custom namespace + * has a valid prometheus namespace, the trimmed name is returned. + * Otherwise, return nullopt. + */ + static absl::optional + metricName(const std::string& extracted_name, + const Stats::CustomStatNamespaces& custom_namespace_factory); +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/admin/stats_handler.cc b/source/server/admin/stats_handler.cc index 9cdc923bad17..be645ebddd4f 100644 --- a/source/server/admin/stats_handler.cc +++ b/source/server/admin/stats_handler.cc @@ -10,9 +10,8 @@ #include "source/common/common/empty_string.h" #include "source/common/http/headers.h" #include "source/common/http/utility.h" -#include "source/server/admin/grouped_stats_request.h" -#include "source/server/admin/stats_params.h" -#include "source/server/admin/ungrouped_stats_request.h" +#include "source/server/admin/prometheus_stats.h" +#include "source/server/admin/stats_request.h" #include "absl/strings/numbers.h" @@ -72,22 +71,31 @@ Http::Code StatsHandler::handlerStatsRecentLookupsEnable(Http::ResponseHeaderMap return Http::Code::OK; } -Admin::RequestPtr StatsHandler::makeRequest(AdminStream& admin_stream, StatsParams& params) { +Admin::RequestPtr StatsHandler::makeRequest(AdminStream& admin_stream) { + StatsParams params; Buffer::OwnedImpl response; Http::Code code = params.parse(admin_stream.getRequestHeaders().getPathValue(), response); if (code != Http::Code::OK) { return Admin::makeStaticTextRequest(response, code); } + if (params.format_ == StatsFormat::Prometheus) { + // TODO(#16139): modify streaming algorithm to cover Prometheus. + // + // This may be easiest to accomplish by populating the set + // with tagExtractedName(), and allowing for vectors of + // stats as multiples will have the same tag-extracted names. + // Ideally we'd find a way to do this without slowing down + // the non-Prometheus implementations. + Buffer::OwnedImpl response; + prometheusFlushAndRender(params, response); + return Admin::makeStaticTextRequest(response, code); + } + if (server_.statsConfig().flushOnAdmin()) { server_.flushStats(); } - if (params.format_ == StatsFormat::Prometheus) { - return makePrometheusRequest( - server_.stats(), params, server_.api().customStatNamespaces(), - [this]() -> Admin::UrlHandler { return prometheusStatsHandler(); }); - } #ifdef ENVOY_ADMIN_HTML const bool active_mode = params.format_ == StatsFormat::ActiveHtml; return makeRequest(server_.stats(), params, [this, active_mode]() -> Admin::UrlHandler { @@ -100,15 +108,48 @@ Admin::RequestPtr StatsHandler::makeRequest(AdminStream& admin_stream, StatsPara } Admin::RequestPtr StatsHandler::makeRequest(Stats::Store& stats, const StatsParams& params, - UngroupedStatsRequest::UrlHandlerFn url_handler_fn) { - return std::make_unique(stats, params, url_handler_fn); + StatsRequest::UrlHandlerFn url_handler_fn) { + return std::make_unique(stats, params, url_handler_fn); } -Admin::RequestPtr -StatsHandler::makePrometheusRequest(Stats::Store& stats, const StatsParams& params, - Stats::CustomStatNamespaces& custom_namespaces, - GroupedStatsRequest::UrlHandlerFn url_handler_fn) { - return std::make_unique(stats, params, custom_namespaces, url_handler_fn); +Http::Code StatsHandler::handlerPrometheusStats(Http::ResponseHeaderMap&, + Buffer::Instance& response, + AdminStream& admin_stream) { + return prometheusStats(admin_stream.getRequestHeaders().getPathValue(), response); +} + +Http::Code StatsHandler::prometheusStats(absl::string_view path_and_query, + Buffer::Instance& response) { + StatsParams params; + Http::Code code = params.parse(path_and_query, response); + if (code != Http::Code::OK) { + return code; + } + + if (server_.statsConfig().flushOnAdmin()) { + server_.flushStats(); + } + + prometheusFlushAndRender(params, response); + return Http::Code::OK; +} + +void StatsHandler::prometheusFlushAndRender(const StatsParams& params, Buffer::Instance& response) { + if (server_.statsConfig().flushOnAdmin()) { + server_.flushStats(); + } + prometheusRender(server_.stats(), server_.api().customStatNamespaces(), params, response); +} + +void StatsHandler::prometheusRender(Stats::Store& stats, + const Stats::CustomStatNamespaces& custom_namespaces, + const StatsParams& params, Buffer::Instance& response) { + const std::vector& text_readouts_vec = + params.prometheus_text_readouts_ ? stats.textReadouts() + : std::vector(); + PrometheusStatsFormatter::statsAsPrometheus(stats.counters(), stats.gauges(), stats.histograms(), + text_readouts_vec, response, params, + custom_namespaces); } Http::Code StatsHandler::handlerContention(Http::ResponseHeaderMap& response_headers, @@ -154,34 +195,13 @@ Admin::UrlHandler StatsHandler::statsHandler(bool active_mode) { } params.insert(params.end(), common_params.begin(), common_params.end()); - return {"/stats", - "print server stats", - [this](AdminStream& admin_stream) -> Admin::RequestPtr { - StatsParams params; - return makeRequest(admin_stream, params); - }, - false, - false, - params}; -} - -Admin::UrlHandler StatsHandler::prometheusStatsHandler() { - return {"/stats/prometheus", - "print server stats in prometheus format", - [this](AdminStream& admin_stream) -> Admin::RequestPtr { - StatsParams params; - params.format_ = StatsFormat::Prometheus; - return makeRequest(admin_stream, params); - }, - false, - false, - {{Admin::ParamDescriptor::Type::Boolean, "usedonly", - "Only include stats that have been written by system since restart"}, - {Admin::ParamDescriptor::Type::Boolean, "text_readouts", - "Render text_readouts as new gaugues with value 0 (increases Prometheus " - "data size)"}, - {Admin::ParamDescriptor::Type::String, "filter", - "Regular expression (Google re2) for filtering stats"}}}; + return { + "/stats", + "print server stats", + [this](AdminStream& admin_stream) -> Admin::RequestPtr { return makeRequest(admin_stream); }, + false, + false, + params}; } } // namespace Server diff --git a/source/server/admin/stats_handler.h b/source/server/admin/stats_handler.h index 9570bd96ce94..fb29712600b8 100644 --- a/source/server/admin/stats_handler.h +++ b/source/server/admin/stats_handler.h @@ -9,9 +9,8 @@ #include "envoy/server/admin.h" #include "envoy/server/instance.h" -#include "source/server/admin/grouped_stats_request.h" #include "source/server/admin/handler_ctx.h" -#include "source/server/admin/ungrouped_stats_request.h" +#include "source/server/admin/stats_request.h" #include "source/server/admin/utils.h" #include "absl/strings/string_view.h" @@ -34,6 +33,42 @@ class StatsHandler : public HandlerContextBase { Buffer::Instance& response, AdminStream&); Http::Code handlerStatsRecentLookupsEnable(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&); + Http::Code handlerPrometheusStats(Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, AdminStream&); + + /** + * Parses and executes a prometheus stats request. + * + * @param path_and_query the URL path and query + * @param response buffer into which to write response + * @return http response code + */ + Http::Code prometheusStats(absl::string_view path_and_query, Buffer::Instance& response); + + /** + * Checks the server_ to see if a flush is needed, and then renders the + * prometheus stats request. + * + * @params params the already-parsed parameters. + * @param response buffer into which to write response + */ + void prometheusFlushAndRender(const StatsParams& params, Buffer::Instance& response); + + /** + * Renders the stats as prometheus. This is broken out as a separately + * callable API to facilitate the benchmark + * (test/server/admin/stats_handler_speed_test.cc) which does not have a + * server object. + * + * @params stats the stats store to read + * @param custom_namespaces namespace mappings used for prometheus + * @params params the already-parsed parameters. + * @param response buffer into which to write response + */ + static void prometheusRender(Stats::Store& stats, + const Stats::CustomStatNamespaces& custom_namespaces, + const StatsParams& params, Buffer::Instance& response); + Http::Code handlerContention(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&); @@ -51,17 +86,14 @@ class StatsHandler : public HandlerContextBase { */ Admin::UrlHandler statsHandler(bool active_mode); - Admin::UrlHandler prometheusStatsHandler(); - - static Admin::RequestPtr - makeRequest(Stats::Store& stats, const StatsParams& params, - UngroupedStatsRequest::UrlHandlerFn url_handler_fn = nullptr); + static Admin::RequestPtr makeRequest(Stats::Store& stats, const StatsParams& params, + StatsRequest::UrlHandlerFn url_handler_fn = nullptr); + Admin::RequestPtr makeRequest(AdminStream&); - static Admin::RequestPtr - makePrometheusRequest(Stats::Store& stats, const StatsParams& params, - Stats::CustomStatNamespaces& custom_namespaces, - GroupedStatsRequest::UrlHandlerFn url_handler_fn = nullptr); - Admin::RequestPtr makeRequest(AdminStream& admin_stream, StatsParams& params); +private: + static Http::Code prometheusStats(absl::string_view path_and_query, Buffer::Instance& response, + Stats::Store& stats, + Stats::CustomStatNamespaces& custom_namespaces); }; } // namespace Server diff --git a/source/server/admin/stats_render.cc b/source/server/admin/stats_render.cc index bc3fb3eb1fe0..e75dab1b0a28 100644 --- a/source/server/admin/stats_render.cc +++ b/source/server/admin/stats_render.cc @@ -1,24 +1,14 @@ #include "source/server/admin/stats_render.h" -#include - -#include "source/common/common/empty_string.h" -#include "source/common/common/regex.h" #include "source/common/json/json_sanitizer.h" #include "source/common/stats/histogram_impl.h" -#include "absl/strings/str_replace.h" - namespace { constexpr absl::string_view JsonNameTag = "{\"name\":\""; constexpr absl::string_view JsonValueTag = "\",\"value\":"; constexpr absl::string_view JsonValueTagQuote = "\",\"value\":\""; constexpr absl::string_view JsonCloseBrace = "}"; constexpr absl::string_view JsonQuoteCloseBrace = "\"}"; - -const Envoy::Regex::CompiledGoogleReMatcher& prometheusRegex() { - CONSTRUCT_ON_FIRST_USE(Envoy::Regex::CompiledGoogleReMatcher, "[^a-zA-Z0-9_]", false); -} } // namespace namespace Envoy { @@ -272,160 +262,5 @@ void StatsJsonRender::collectBuckets(const std::string& name, *histogram_array_->add_values() = ValueUtil::structValue(histogram_obj); } -// Writes output for a Prometheus stat of type Gauge. -void PrometheusStatsRender::generate(Buffer::Instance& response, - const std::string& prefixed_tag_extracted_name, - const std::vector& gauge) { - outputStatType(response, gauge, prefixed_tag_extracted_name, - generateNumericOutput, "gauge"); -} - -// Writes output for a Prometheus stat of type Counter. -void PrometheusStatsRender::generate(Buffer::Instance& response, - const std::string& prefixed_tag_extracted_name, - const std::vector& counter) { - outputStatType(response, counter, prefixed_tag_extracted_name, - generateNumericOutput, - "counter"); -} - -// Writes output for a Prometheus stat of type Text Readout. -void PrometheusStatsRender::generate(Buffer::Instance& response, - const std::string& prefixed_tag_extracted_name, - const std::vector& text_readout) { - // text readout stats are returned in gauge format, so "gauge" type is set intentionally. - outputStatType(response, text_readout, prefixed_tag_extracted_name, - generateTextReadoutOutput, "gauge"); -} - -// Writes output for a Prometheus stat of type Histogram. -void PrometheusStatsRender::generate(Buffer::Instance& response, - const std::string& prefixed_tag_extracted_name, - const std::vector& histogram) { - outputStatType(response, histogram, prefixed_tag_extracted_name, - generateHistogramOutput, "histogram"); -} - -void PrometheusStatsRender::finalize(Buffer::Instance&) {} - -std::string PrometheusStatsRender::formattedTags(const std::vector& tags) { - std::vector buf; - buf.reserve(tags.size()); - for (const Stats::Tag& tag : tags) { - buf.push_back(absl::StrCat(sanitizeName(tag.name_), "=\"", sanitizeValue(tag.value_), "\"")); - } - return absl::StrJoin(buf, ","); -} - -absl::optional -PrometheusStatsRender::metricName(const std::string& extracted_name, - const Stats::CustomStatNamespaces& custom_namespaces) { - const absl::optional custom_namespace_stripped = - custom_namespaces.stripRegisteredPrefix(extracted_name); - if (custom_namespace_stripped.has_value()) { - // This case the name has a custom namespace, and it is a custom metric. - const std::string sanitized_name = sanitizeName(custom_namespace_stripped.value()); - // We expose these metrics without modifying (e.g. without "envoy_"), - // so we have to check the "user-defined" stat name complies with the Prometheus naming - // convention. Specifically the name must start with the "[a-zA-Z_]" pattern. - // All the characters in sanitized_name are already in "[a-zA-Z0-9_]" pattern - // thanks to sanitizeName above, so the only thing we have to do is check - // if it does not start with digits. - if (sanitized_name.empty() || absl::ascii_isdigit(sanitized_name.front())) { - return absl::nullopt; - } - return sanitized_name; - } - - // If it does not have a custom namespace, add namespacing prefix to avoid conflicts, as per best - // practice: https://prometheus.io/docs/practices/naming/#metric-names Also, naming conventions on - // https://prometheus.io/docs/concepts/data_model/ - return absl::StrCat("envoy_", sanitizeName(extracted_name)); -} - -std::string PrometheusStatsRender::sanitizeName(const absl::string_view name) { - // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by - // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/. - // The initial [a-zA-Z_] constraint is always satisfied by the namespace prefix. - return prometheusRegex().replaceAll(name, "_"); -} - -std::string PrometheusStatsRender::sanitizeValue(const absl::string_view value) { - // Removes problematic characters from Prometheus tag values to prevent - // text serialization issues. This matches the prometheus text formatting code: - // https://github.com/prometheus/common/blob/88f1636b699ae4fb949d292ffb904c205bf542c9/expfmt/text_create.go#L419-L420. - // The goal is to replace '\' with "\\", newline with "\n", and '"' with "\"". - return absl::StrReplaceAll(value, { - {R"(\)", R"(\\)"}, - {"\n", R"(\n)"}, - {R"(")", R"(\")"}, - }); -} - -template -void PrometheusStatsRender::outputStatType( - Buffer::Instance& response, const std::vector& metrics, - const std::string& prefixed_tag_extracted_name, - const std::function& generate_output, - absl::string_view type) { - response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name, type)); - for (const auto& metric : metrics) { - response.add(generate_output(metric, prefixed_tag_extracted_name)); - } -} - -template -std::string -PrometheusStatsRender::generateNumericOutput(const StatType& metric, - const std::string& prefixed_tag_extracted_name) { - const std::string tags = formattedTags(metric->tags()); - return absl::StrCat(prefixed_tag_extracted_name, "{", tags, "} ", metric->value(), "\n"); -} - -std::string -PrometheusStatsRender::generateTextReadoutOutput(const Stats::TextReadoutSharedPtr& metric, - const std::string& prefixed_tag_extracted_name) { - auto tags = metric->tags(); - tags.push_back(Stats::Tag{"text_value", metric->value()}); - const std::string formTags = formattedTags(tags); - return absl::StrCat(prefixed_tag_extracted_name, "{", formTags, "} 0\n"); -} - -std::string -PrometheusStatsRender::generateHistogramOutput(const Stats::HistogramSharedPtr& metric, - const std::string& prefixed_tag_extracted_name) { - auto parent_histogram = dynamic_cast(metric.get()); - if (parent_histogram != nullptr) { - const std::string tags = formattedTags(parent_histogram->tags()); - const std::string hist_tags = parent_histogram->tags().empty() ? EMPTY_STRING : (tags + ","); - - const Stats::HistogramStatistics& stats = parent_histogram->cumulativeStatistics(); - Stats::ConstSupportedBuckets& supported_buckets = stats.supportedBuckets(); - const std::vector& computed_buckets = stats.computedBuckets(); - std::string output; - for (size_t i = 0; i < supported_buckets.size(); ++i) { - double bucket = supported_buckets[i]; - uint64_t value = computed_buckets[i]; - // We want to print the bucket in a fixed point (non-scientific) format. The fmt library - // doesn't have a specific modifier to format as a fixed-point value only so we use the - // 'g' operator which prints the number in general fixed point format or scientific format - // with precision 50 to round the number up to 32 significant digits in fixed point format - // which should cover pretty much all cases - output.append(absl::StrCat(prefixed_tag_extracted_name, "_bucket{", hist_tags, "le=\"", - fmt::format("{0:.32g}", bucket), "\"} ", value, "\n")); - } - - output.append(absl::StrCat(prefixed_tag_extracted_name, "_bucket{", hist_tags, "le=\"+Inf\"} ", - stats.sampleCount(), "\n")); - output.append(absl::StrCat(prefixed_tag_extracted_name, "_sum{", tags, "} ", - fmt::format("{0:.32g}", stats.sampleSum()), "\n")); - output.append(absl::StrCat(prefixed_tag_extracted_name, "_count{", tags, "} ", - stats.sampleCount(), "\n")); - return output; - } - return EMPTY_STRING; -} - } // namespace Server } // namespace Envoy diff --git a/source/server/admin/stats_render.h b/source/server/admin/stats_render.h index efc55c804c91..477c3eb6695e 100644 --- a/source/server/admin/stats_render.h +++ b/source/server/admin/stats_render.h @@ -1,7 +1,5 @@ #pragma once -#include - #include "envoy/server/admin.h" #include "envoy/stats/stats.h" @@ -12,29 +10,14 @@ namespace Envoy { namespace Server { -// Abstract base class for rendering stats, which captures logic -// that is shared across all formats (e.g. finalizing rendering of stats). -// The APIs for generating stats output vary between formats (e.g. -// JSON vs. Prometheus) and are defined in derived classes: having a base -// render class avoids code duplication while affording the flexibility to -// capture any differences in how we generate stats output. -class StatsRenderBase { -public: - virtual ~StatsRenderBase() = default; - - // Completes rendering any buffered data. - virtual void finalize(Buffer::Instance& response) PURE; - - // Indicates that no stats for a particular type have been found. - virtual void noStats(Buffer::Instance&, absl::string_view) {} -}; - -// Abstract class for rendering ungrouped stats. -// Every method is called "generate" differing only by the data type, to -// facilitate templatized call-sites. -class StatsRender : public StatsRenderBase { +// Abstract class for rendering stats. Every method is called "generate" +// differing only by the data type, to facilitate templatized call-sites. +// +// There are currently Json and Text implementations of this interface, and in +// #19546 an HTML version will be added to provide a hierarchical view. +class StatsRender { public: - ~StatsRender() override = default; + virtual ~StatsRender() = default; // Writes a fragment for a numeric value, for counters and gauges. virtual void generate(Buffer::Instance& response, const std::string& name, uint64_t value) PURE; @@ -46,6 +29,12 @@ class StatsRender : public StatsRenderBase { // Writes a histogram value. virtual void generate(Buffer::Instance& response, const std::string& name, const Stats::ParentHistogram& histogram) PURE; + + // Completes rendering any buffered data. + virtual void finalize(Buffer::Instance& response) PURE; + + // Indicates that no stats for a particular type have been found. + virtual void noStats(Buffer::Instance&, absl::string_view) {} }; // Implements the Render interface for simple textual representation of stats. @@ -106,90 +95,5 @@ class StatsJsonRender : public StatsRender { std::string value_buffer_; // Used for Json::sanitize for text-readout values. }; -// Implements the Render base interface for textual representation of Prometheus stats -// (see: https://prometheus.io/docs/concepts/data_model). -// The APIs for rendering Prometheus stats take as input a string (the stat's name) -// and a vector of counters, gauges etc. (each entry in the vector holds a set of labels -// and the stat's value for that set of labels). The Prometheus stat is rendered -// as per the text-based format described at -// https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format. -class PrometheusStatsRender : public StatsRenderBase { -public: - void generate(Buffer::Instance& response, const std::string& prefixed_tag_extracted_name, - const std::vector& histogram); - - void generate(Buffer::Instance& response, const std::string& prefixed_tag_extracted_name, - const std::vector& gauge); - - void generate(Buffer::Instance& response, const std::string& prefixed_tag_extracted_name, - const std::vector& counter); - - void generate(Buffer::Instance& response, const std::string& prefixed_tag_extracted_name, - const std::vector& text_readout); - - void finalize(Buffer::Instance&) override; - - /** - * Format the given tags, returning a string as a comma-separated list - * of ="" pairs. - */ - static std::string formattedTags(const std::vector& tags); - - /** - * Format the given metric name, and prefixed with "envoy_" if it does not have a custom - * stat namespace. If it has a custom stat namespace AND the name without the custom namespace - * has a valid prometheus namespace, the trimmed name is returned. - * Otherwise, return nullopt. - */ - static absl::optional - metricName(const std::string& extracted_name, - const Stats::CustomStatNamespaces& custom_namespace_factory); - -private: - /** - * Take a string and sanitize it according to Prometheus conventions. - */ - static std::string sanitizeName(const absl::string_view name); - - /** - * Take tag values and sanitize it for text serialization, according to - * Prometheus conventions. - */ - static std::string sanitizeValue(const absl::string_view value); - - template - void outputStatType( - Envoy::Buffer::Instance& response, const std::vector& metrics, - const std::string& prefixed_tag_extracted_name, - const std::function& generate_output, - absl::string_view type); - - /* - * Return the prometheus output for a numeric Stat (Counter or Gauge). - */ - template - static std::string generateNumericOutput(const StatType& metric, - const std::string& prefixed_tag_extracted_name); - - /* - * Returns the prometheus output for a TextReadout in gauge format. - * It is a workaround of a limitation of prometheus which stores only numeric metrics. - * The output is a gauge named the same as a given text-readout. The value of returned gauge is - * always equal to 0. Returned gauge contains all tags of a given text-readout and one additional - * tag {"text_value":"textReadout.value"}. - */ - static std::string generateTextReadoutOutput(const Stats::TextReadoutSharedPtr& metric, - const std::string& prefixed_tag_extracted_name); - - /* - * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded - * newlines) that contains all the individual bucket counts and sum/count for a single histogram - * (metric_name plus all tags). - */ - static std::string generateHistogramOutput(const Stats::HistogramSharedPtr& histogram, - const std::string& prefixed_tag_extracted_name); -}; - } // namespace Server } // namespace Envoy diff --git a/source/server/admin/stats_request.cc b/source/server/admin/stats_request.cc index 410c10909ae8..954eb1127dbf 100644 --- a/source/server/admin/stats_request.cc +++ b/source/server/admin/stats_request.cc @@ -1,22 +1,55 @@ #include "source/server/admin/stats_request.h" +#ifdef ENVOY_ADMIN_HTML +#include "source/server/admin/stats_html_render.h" +#endif + namespace Envoy { namespace Server { -template -StatsRequest::StatsRequest( - Stats::Store& stats, const StatsParams& params, UrlHandlerFn url_handler_fn) - : params_(params), url_handler_fn_(url_handler_fn), stats_(stats) {} +StatsRequest::StatsRequest(Stats::Store& stats, const StatsParams& params, + UrlHandlerFn url_handler_fn) + : params_(params), stats_(stats), url_handler_fn_(url_handler_fn) { + switch (params_.type_) { + case StatsType::TextReadouts: + case StatsType::All: + phase_ = Phase::TextReadouts; + break; + case StatsType::Counters: + case StatsType::Gauges: + phase_ = Phase::CountersAndGauges; + break; + case StatsType::Histograms: + phase_ = Phase::Histograms; + break; + } +} -template -Http::Code StatsRequest::start( - Http::ResponseHeaderMap& response_headers) { - setRenderPtr(response_headers); +Http::Code StatsRequest::start(Http::ResponseHeaderMap& response_headers) { + switch (params_.format_) { + case StatsFormat::Json: + render_ = std::make_unique(response_headers, response_, params_); + break; + case StatsFormat::Text: + render_ = std::make_unique(params_); + break; #ifdef ENVOY_ADMIN_HTML - if (params_.format_ == StatsFormat::ActiveHtml) { - return Http::Code::OK; + case StatsFormat::ActiveHtml: + case StatsFormat::Html: { + auto html_render = std::make_unique(response_headers, response_, params_); + html_render->setupStatsPage(url_handler_fn_(), params_, response_); + render_ = std::move(html_render); + if (params_.format_ == StatsFormat::ActiveHtml) { + return Http::Code::OK; + } + break; } #endif + case StatsFormat::Prometheus: + // TODO(#16139): once Prometheus shares this algorithm here, this becomes a legitimate choice. + IS_ENVOY_BUG("reached Prometheus case in switch unexpectedly"); + return Http::Code::BadRequest; + } // Populate the top-level scopes and the stats underneath any scopes with an empty name. // We will have to de-dup, but we can do that after sorting. @@ -31,9 +64,7 @@ Http::Code StatsRequest: return Http::Code::OK; } -template -bool StatsRequest::nextChunk( - Buffer::Instance& response) { +bool StatsRequest::nextChunk(Buffer::Instance& response) { if (response_.length() > 0) { ASSERT(response.length() == 0); response.move(response_); @@ -42,35 +73,38 @@ bool StatsRequest::nextC // nextChunk's contract is to add up to chunk_size_ additional bytes. The // caller is not required to drain the bytes after each call to nextChunk. - StatsRenderBase& stats_render = render(); const uint64_t starting_response_length = response.length(); while (response.length() - starting_response_length < chunk_size_) { while (stat_map_.empty()) { if (phase_stat_count_ == 0) { - stats_render.noStats(response, phase_labels_[phases_.at(phase_index_)]); + render_->noStats(response, phase_string_); } else { phase_stat_count_ = 0; } if (params_.type_ != StatsType::All) { - stats_render.finalize(response); + render_->finalize(response); return false; } - - // Check if we are at the last phase: in that case, we are done; - // if not, increment phase index and start next phase. - if (phase_index_ == phases_.size() - 1) { - stats_render.finalize(response); - return false; - } else { - phase_index_++; + switch (phase_) { + case Phase::TextReadouts: + phase_ = Phase::CountersAndGauges; + phase_string_ = "Counters and Gauges"; + startPhase(); + break; + case Phase::CountersAndGauges: + phase_ = Phase::Histograms; + phase_string_ = "Histograms"; startPhase(); + break; + case Phase::Histograms: + render_->finalize(response); + return false; } } auto iter = stat_map_.begin(); StatOrScopes variant = std::move(iter->second); StatOrScopesIndex index = static_cast(variant.index()); - switch (index) { case StatOrScopesIndex::Scopes: // Erase the current element before adding new ones, as absl::btree_map @@ -81,78 +115,100 @@ bool StatsRequest::nextC populateStatsForCurrentPhase(absl::get(variant)); break; case StatOrScopesIndex::TextReadout: - processTextReadout(iter->first, response, variant); + renderStat(iter->first, response, variant); stat_map_.erase(iter); + ++phase_stat_count_; break; case StatOrScopesIndex::Counter: - processCounter(iter->first, response, variant); + renderStat(iter->first, response, variant); stat_map_.erase(iter); + ++phase_stat_count_; break; case StatOrScopesIndex::Gauge: - processGauge(iter->first, response, variant); + renderStat(iter->first, response, variant); stat_map_.erase(iter); + ++phase_stat_count_; break; - case StatOrScopesIndex::Histogram: - processHistogram(iter->first, response, variant); + case StatOrScopesIndex::Histogram: { + auto histogram = absl::get(variant); + auto parent_histogram = dynamic_cast(histogram.get()); + if (parent_histogram != nullptr) { + render_->generate(response, iter->first, *parent_histogram); + ++phase_stat_count_; + } stat_map_.erase(iter); - break; + } } } return true; } -template -void StatsRequest::startPhase() { +void StatsRequest::startPhase() { ASSERT(stat_map_.empty()); // Insert all the scopes in the alphabetically ordered map. As we iterate // through the map we'll erase the scopes and replace them with the stats held // in the scopes. for (const Stats::ConstScopeSharedPtr& scope : scopes_) { - // The operator[] of btree_map runs a try_emplace behind the scenes, - // inserting the variant into the map when the lookup key does not exist. StatOrScopes& variant = stat_map_[stats_.symbolTable().toString(scope->prefix())]; - ASSERT(static_cast(variant.index()) == StatOrScopesIndex::Scopes); + if (variant.index() == absl::variant_npos) { + variant = ScopeVec(); + } absl::get(variant).emplace_back(scope); } } -template -void StatsRequest::populateStatsForCurrentPhase(const ScopeVec& scope_vec) { - Phase current_phase = phases_.at(phase_index_); - for (const Stats::ConstScopeSharedPtr& scope : scope_vec) { - switch (current_phase) { - case Phase::TextReadouts: - scope->iterate(saveMatchingStatForTextReadout()); - break; - case Phase::CountersAndGauges: - if (params_.type_ != StatsType::Gauges) { - scope->iterate(saveMatchingStatForCounter()); - } - if (params_.type_ != StatsType::Counters) { - scope->iterate(saveMatchingStatForGauge()); - } - break; - case Phase::Counters: - scope->iterate(saveMatchingStatForCounter()); - break; - case Phase::Gauges: - scope->iterate(saveMatchingStatForGauge()); - break; - case Phase::Histograms: - scope->iterate(saveMatchingStatForHistogram()); - break; +void StatsRequest::populateStatsForCurrentPhase(const ScopeVec& scope_vec) { + switch (phase_) { + case Phase::TextReadouts: + populateStatsFromScopes(scope_vec); + break; + case Phase::CountersAndGauges: + if (params_.type_ != StatsType::Gauges) { + populateStatsFromScopes(scope_vec); } + if (params_.type_ != StatsType::Counters) { + populateStatsFromScopes(scope_vec); + } + break; + case Phase::Histograms: + populateStatsFromScopes(scope_vec); + break; } } -template class StatsRequest; +template void StatsRequest::populateStatsFromScopes(const ScopeVec& scope_vec) { + Stats::IterateFn check_stat = [this](const Stats::RefcountPtr& stat) -> bool { + if (params_.used_only_ && !stat->used()) { + return true; + } + + // Capture the name if we did not early-exit due to used_only -- we'll use + // the name for both filtering and for capturing the stat in the map. + // stat->name() takes a symbol table lock and builds a string, so we only + // want to call it once. + // + // This duplicates logic in shouldShowMetric in prometheus_stats.cc, but + // differs in that Prometheus only uses stat->name() for filtering, not + // rendering, so it only grab the name if there's a filter. + std::string name = stat->name(); + if (params_.re2_filter_ != nullptr && !re2::RE2::PartialMatch(name, *params_.re2_filter_)) { + return true; + } + stat_map_[name] = stat; + return true; + }; + for (const Stats::ConstScopeSharedPtr& scope : scope_vec) { + scope->iterate(check_stat); + } +} -template class StatsRequest< - std::vector, std::vector, - std::vector, std::vector>; +template +void StatsRequest::renderStat(const std::string& name, Buffer::Instance& response, + StatOrScopes& variant) { + auto stat = absl::get(variant); + render_->generate(response, name, stat->value()); +} } // namespace Server } // namespace Envoy diff --git a/source/server/admin/stats_request.h b/source/server/admin/stats_request.h index 711b246c0218..41d43632a83f 100644 --- a/source/server/admin/stats_request.h +++ b/source/server/admin/stats_request.h @@ -1,9 +1,5 @@ #pragma once -#include -#include -#include - #include "envoy/server/admin.h" #include "source/server/admin/stats_params.h" @@ -16,16 +12,35 @@ namespace Envoy { namespace Server { -// Captures context for a streaming request, implementing the AdminHandler interface. The class -// templating allows derived classes to specify how each stat type (Text Readout, Counter, etc.) -// is represented, catering for different exposition formats (e.g. grouped, ungrouped). -template +// Captures context for a streaming request, implementing the AdminHandler interface. class StatsRequest : public Admin::Request { + using ScopeVec = std::vector; + using StatOrScopes = absl::variant; + + // Ordered to match the StatsOrScopes variant. + enum class StatOrScopesIndex { Scopes, TextReadout, Counter, Gauge, Histogram }; + + // In order to keep the output consistent with the fully buffered behavior + // prior to the chunked implementation that buffered each type, we iterate + // over all scopes for each type. This enables the complex chunking + // implementation to pass the tests that capture the buffered behavior. There + // is not a significant cost to this, but in a future PR we may choose to + // co-mingle the types. Note that histograms are groups together in the data + // JSON data model, so we won't be able to fully co-mingle. + enum class Phase { + TextReadouts, + CountersAndGauges, + Histograms, + }; public: + using UrlHandlerFn = std::function; + static constexpr uint64_t DefaultChunkSize = 2 * 1000 * 1000; - using UrlHandlerFn = std::function; + StatsRequest(Stats::Store& stats, const StatsParams& params, + UrlHandlerFn url_handler_fn = nullptr); // Admin::Request Http::Code start(Http::ResponseHeaderMap& response_headers) override; @@ -52,23 +67,15 @@ class StatsRequest : public Admin::Request { // introduce flow-control so that we don't buffer the all the serialized stats // while waiting for a slow client. // - // For text/JSON/HTML stats, we do 3 passes through all the scopes_, and we emit + // Note that we do 3 passes through all the scopes_, so that we can emit // text-readouts first, then the intermingled counters and gauges, and finally - // the histograms. For prometheus stats, we do 3 or 4 passes (this depends on whether Text - // Readouts are explicitly requested) through all the scopes_, and we emit counters first, - // then gauges, possibly text readouts and finally histograms. + // the histograms. bool nextChunk(Buffer::Instance& response) override; - // To duplicate prior behavior for this class, we do several passes over all the stats. For - // text/JSON/HTML stats, we do 3 passes: + // To duplicate prior behavior for this class, we do three passes over all the stats: // 1. text readouts across all scopes // 2. counters and gauges, co-mingled, across all scopes // 3. histograms across all scopes. - // For prometheus stats, we do 3 or 4 passes: - // 1. counters across all scopes - // 2. gauges across all scopes - // 3. text readouts (only if explicitly requested via query param) - // 4. histograms across all scopes. // It would be little more efficient to co-mingle all the stats, but three // passes over the scopes is OK. In the future we may decide to organize the // result data differently, but in the process of changing from buffering @@ -76,78 +83,35 @@ class StatsRequest : public Admin::Request { // to reason about if the tests don't change their expectations. void startPhase(); - // Sets the chunk size. - void setChunkSize(uint64_t chunk_size) { chunk_size_ = chunk_size; } - -protected: - // Ordered to match the StatsOrScopes variant. - enum class StatOrScopesIndex { Scopes, TextReadout, Counter, Gauge, Histogram }; - - // In order to keep the output consistent with the fully buffered behavior - // prior to the chunked implementation that buffered each type, we iterate - // over all scopes for each type. This enables the complex chunking - // implementation to pass the tests that capture the buffered behavior. There - // is not a significant cost to this, but in a future PR we may choose to - // co-mingle the types. Note that histograms are grouped together in the data - // JSON data model, so we won't be able to fully co-mingle. - enum class Phase { - TextReadouts, - CountersAndGauges, - Counters, - Gauges, - Histograms, - }; - - using ScopeVec = std::vector; - - using StatOrScopes = - absl::variant; - - StatsRequest(Stats::Store& stats, const StatsParams& params, - UrlHandlerFn url_handler_fn = nullptr); - - ~StatsRequest() override = default; - // Iterates over scope_vec and populates the metric types associated with the // current phase. void populateStatsForCurrentPhase(const ScopeVec& scope_vec); - virtual Stats::IterateFn saveMatchingStatForTextReadout() PURE; - virtual Stats::IterateFn saveMatchingStatForGauge() PURE; - virtual Stats::IterateFn saveMatchingStatForCounter() PURE; - virtual Stats::IterateFn saveMatchingStatForHistogram() PURE; + // Populates all the metrics of the templatized type from scope_vec. Here we + // exploit that Scope::iterate is a generic templatized function to avoid code + // duplication. + template void populateStatsFromScopes(const ScopeVec& scope_vec); - virtual void processTextReadout(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) PURE; - virtual void processGauge(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) PURE; - virtual void processCounter(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) PURE; - virtual void processHistogram(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) PURE; + // Renders the templatized type, exploiting the fact that Render::generate is + // generic to avoid code duplication. + template + void renderStat(const std::string& name, Buffer::Instance& response, StatOrScopes& variant); - virtual void setRenderPtr(Http::ResponseHeaderMap& response_headers) PURE; - virtual StatsRenderBase& render() PURE; + // Sets the chunk size. + void setChunkSize(uint64_t chunk_size) { chunk_size_ = chunk_size; } +private: StatsParams params_; + std::unique_ptr render_; + Stats::Store& stats_; + ScopeVec scopes_; absl::btree_map stat_map_; + Phase phase_{Phase::TextReadouts}; + uint64_t phase_stat_count_{0}; + absl::string_view phase_string_{"text readouts"}; Buffer::OwnedImpl response_; UrlHandlerFn url_handler_fn_; - - // Phase-related state. - uint64_t phase_stat_count_{0}; - uint32_t phase_index_{0}; - std::vector phases_; - -private: - Stats::Store& stats_; - ScopeVec scopes_; uint64_t chunk_size_{DefaultChunkSize}; - std::map phase_labels_{{Phase::TextReadouts, "Text Readouts"}, - {Phase::CountersAndGauges, "Counters and Gauges"}, - {Phase::Histograms, "Histograms"}, - {Phase::Counters, "Counters"}, - {Phase::Gauges, "Gauges"}}; }; } // namespace Server diff --git a/source/server/admin/ungrouped_stats_request.cc b/source/server/admin/ungrouped_stats_request.cc deleted file mode 100644 index 0b3d02c37cce..000000000000 --- a/source/server/admin/ungrouped_stats_request.cc +++ /dev/null @@ -1,129 +0,0 @@ -#include "source/server/admin/ungrouped_stats_request.h" - -#ifdef ENVOY_ADMIN_HTML -#include "source/server/admin/stats_html_render.h" -#endif - -namespace Envoy { -namespace Server { - -UngroupedStatsRequest::UngroupedStatsRequest(Stats::Store& stats, const StatsParams& params, - UrlHandlerFn url_handler_fn) - : StatsRequest(stats, params, url_handler_fn) { - phases_ = {Phase::TextReadouts, Phase::CountersAndGauges, Phase::Histograms}; - switch (params_.type_) { - case StatsType::TextReadouts: - case StatsType::All: - phase_index_ = 0; - break; - case StatsType::Counters: - case StatsType::Gauges: - phase_index_ = 1; - break; - case StatsType::Histograms: - phase_index_ = 2; - break; - } -} - -template Stats::IterateFn UngroupedStatsRequest::saveMatchingStat() { - return [this](const Stats::RefcountPtr& stat) -> bool { - // Check if used. - if (params_.used_only_ && !stat->used()) { - return true; - } - - // Capture the name if we did not early-exit due to used_only -- we'll use - // the name for both filtering and for capturing the stat in the map. - // stat->name() takes a symbol table lock and builds a string, so we only - // want to call it once. - // - // This duplicates logic in saveMatchingStat in grouped_stats_request.cc, but - // differs in that Prometheus only uses stat->name() for filtering, not - // rendering, so it only grabs the name if there's a filter. - std::string name = stat->name(); - - // Check if filtered. - if (params_.re2_filter_ != nullptr && !re2::RE2::PartialMatch(name, *params_.re2_filter_)) { - return true; - } - - stat_map_[name] = stat; - return true; - }; -} - -Stats::IterateFn UngroupedStatsRequest::saveMatchingStatForTextReadout() { - return saveMatchingStat(); -} - -Stats::IterateFn UngroupedStatsRequest::saveMatchingStatForGauge() { - return saveMatchingStat(); -} - -Stats::IterateFn UngroupedStatsRequest::saveMatchingStatForCounter() { - return saveMatchingStat(); -} - -Stats::IterateFn UngroupedStatsRequest::saveMatchingStatForHistogram() { - return saveMatchingStat(); -} - -template -void UngroupedStatsRequest::renderStat(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - auto stat = absl::get(variant); - render_.get()->generate(response, name, stat->value()); - phase_stat_count_++; -} - -void UngroupedStatsRequest::processTextReadout(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - renderStat(name, response, variant); -} - -void UngroupedStatsRequest::processCounter(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - renderStat(name, response, variant); -} - -void UngroupedStatsRequest::processGauge(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - renderStat(name, response, variant); -} - -void UngroupedStatsRequest::processHistogram(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) { - auto histogram = absl::get(variant); - auto parent_histogram = dynamic_cast(histogram.get()); - if (parent_histogram != nullptr) { - dynamic_cast(render_.get())->generate(response, name, *parent_histogram); - ++phase_stat_count_; - } -} - -void UngroupedStatsRequest::setRenderPtr(Http::ResponseHeaderMap& response_headers) { - switch (params_.format_) { - case StatsFormat::Json: - render_ = std::make_unique(response_headers, response_, params_); - break; - case StatsFormat::Text: - render_ = std::make_unique(params_); - break; -#ifdef ENVOY_ADMIN_HTML - case StatsFormat::ActiveHtml: - case StatsFormat::Html: { - auto html_render = std::make_unique(response_headers, response_, params_); - html_render->setupStatsPage(url_handler_fn_(), params_, response_); - render_ = std::move(html_render); - break; - } -#endif - case StatsFormat::Prometheus: - IS_ENVOY_BUG("reached Prometheus case in switch unexpectedly"); - break; - } -} - -} // namespace Server -} // namespace Envoy diff --git a/source/server/admin/ungrouped_stats_request.h b/source/server/admin/ungrouped_stats_request.h deleted file mode 100644 index 47f402aee9a7..000000000000 --- a/source/server/admin/ungrouped_stats_request.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include "source/server/admin/stats_request.h" - -namespace Envoy { -namespace Server { - -// In the context of this class, a stat is represented by an individual -// gauge, counter etc.; this is in contrast to the GroupedStatsRequest class, -// where a stat is a group of gauges, counters etc. with a common tag-extracted name. -// This class is currently used for all non-Prometheus formats (e.g. text, JSON). -class UngroupedStatsRequest - : public StatsRequest { - -public: - UngroupedStatsRequest(Stats::Store& stats, const StatsParams& params, - UrlHandlerFn url_handler_fn = nullptr); - -protected: - Stats::IterateFn saveMatchingStatForTextReadout() override; - Stats::IterateFn saveMatchingStatForGauge() override; - Stats::IterateFn saveMatchingStatForCounter() override; - Stats::IterateFn saveMatchingStatForHistogram() override; - template Stats::IterateFn saveMatchingStat(); - - void processTextReadout(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) override; - void processGauge(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) override; - void processCounter(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) override; - void processHistogram(const std::string& name, Buffer::Instance& response, - const StatOrScopes& variant) override; - - template - void renderStat(const std::string& name, Buffer::Instance& response, const StatOrScopes& variant); - - void setRenderPtr(Http::ResponseHeaderMap& response_headers) override; - StatsRenderBase& render() override { - ASSERT(render_ != nullptr); - return *render_; - } - -private: - std::unique_ptr render_; -}; -} // namespace Server -} // namespace Envoy diff --git a/test/server/admin/BUILD b/test/server/admin/BUILD index 36e37168dec7..8c1a28f7d2ee 100644 --- a/test/server/admin/BUILD +++ b/test/server/admin/BUILD @@ -127,25 +127,8 @@ envoy_cc_test_library( ) envoy_cc_test( - name = "ungrouped_stats_request_test", - srcs = envoy_select_admin_functionality(["ungrouped_stats_request_test.cc"]), - deps = [ - ":stats_request_test_base", - ], -) - -envoy_cc_test( - name = "grouped_stats_request_test", - srcs = envoy_select_admin_functionality(["grouped_stats_request_test.cc"]), - deps = [ - ":stats_request_test_base", - ], -) - -envoy_cc_test_library( - name = "stats_request_test_base", - srcs = envoy_select_admin_functionality(["stats_request_test_base.cc"]), - hdrs = ["stats_request_test_base.h"], + name = "stats_request_test", + srcs = envoy_select_admin_functionality(["stats_request_test.cc"]), deps = [ "//source/common/stats:thread_local_store_lib", "//source/server/admin:stats_request_lib", @@ -156,14 +139,6 @@ envoy_cc_test_library( ], ) -envoy_cc_test( - name = "prometheus_stats_test", - srcs = envoy_select_admin_functionality(["prometheus_stats_test.cc"]), - deps = [ - ":stats_request_test_base", - ], -) - envoy_cc_benchmark_binary( name = "stats_handler_speed_test", srcs = envoy_select_admin_functionality(["stats_handler_speed_test.cc"]), @@ -189,6 +164,15 @@ envoy_cc_test( deps = [":admin_instance_lib"], ) +envoy_cc_test( + name = "prometheus_stats_test", + srcs = envoy_select_admin_functionality(["prometheus_stats_test.cc"]), + deps = [ + "//source/server/admin:prometheus_stats_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "logs_handler_test", srcs = envoy_select_admin_functionality(["logs_handler_test.cc"]), diff --git a/test/server/admin/admin_test.cc b/test/server/admin/admin_test.cc index b030005493cd..ec9d0f39d1d9 100644 --- a/test/server/admin/admin_test.cc +++ b/test/server/admin/admin_test.cc @@ -15,7 +15,7 @@ #include "source/common/protobuf/utility.h" #include "source/common/upstream/upstream_impl.h" #include "source/extensions/access_loggers/common/file_access_log_impl.h" -#include "source/server/admin/ungrouped_stats_request.h" +#include "source/server/admin/stats_request.h" #include "test/server/admin/admin_instance.h" #include "test/test_common/logging.h" @@ -254,8 +254,7 @@ TEST_P(AdminInstanceTest, StatsWithMultipleChunks) { uint32_t expected_size = 0; // Declare enough counters so that we are sure to exceed the chunk size. - const uint32_t n = - (UngroupedStatsRequest::DefaultChunkSize + prefix.size() / 2) / prefix.size() + 1; + const uint32_t n = (StatsRequest::DefaultChunkSize + prefix.size() / 2) / prefix.size() + 1; for (uint32_t i = 0; i <= n; ++i) { const std::string name = absl::StrCat(prefix, i); store.counterFromString(name); @@ -263,7 +262,7 @@ TEST_P(AdminInstanceTest, StatsWithMultipleChunks) { } EXPECT_EQ(Http::Code::OK, getCallback("/stats", header_map, response)); EXPECT_LT(expected_size, response.length()); - EXPECT_LT(UngroupedStatsRequest::DefaultChunkSize, response.length()); + EXPECT_LT(StatsRequest::DefaultChunkSize, response.length()); EXPECT_THAT(response.toString(), StartsWith(absl::StrCat(prefix, "0: 0\n", prefix))); } diff --git a/test/server/admin/grouped_stats_request_test.cc b/test/server/admin/grouped_stats_request_test.cc deleted file mode 100644 index c731c8aa18c8..000000000000 --- a/test/server/admin/grouped_stats_request_test.cc +++ /dev/null @@ -1,90 +0,0 @@ -#include "source/common/stats/custom_stat_namespaces_impl.h" - -#include "test/mocks/stats/mocks.h" -#include "test/server/admin/stats_request_test_base.h" - -namespace Envoy { -namespace Server { - -class GroupedStatsRequestTest : public StatsRequestTestBase { -protected: - std::unique_ptr makeRequest(bool used_only, bool text_readouts = false) { - StatsParams params; - params.used_only_ = used_only; - params.prometheus_text_readouts_ = text_readouts; - params.format_ = StatsFormat::Prometheus; - return std::make_unique(store_, params, custom_namespaces_); - } - - Stats::CustomStatNamespacesImpl custom_namespaces_; -}; - -TEST_F(GroupedStatsRequestTest, Empty) { EXPECT_EQ(0, iterateChunks(*makeRequest(false))); } - -TEST_F(GroupedStatsRequestTest, OneCounter) { - Stats::StatNameTagVector c1Tags{{makeStatName("cluster"), makeStatName("c1")}}; - Stats::Counter& c1 = store_.rootScope()->counterFromStatNameWithTags( - makeStatName("cluster.upstream.cx.total"), c1Tags); - c1.add(10); - - EXPECT_EQ(1, iterateChunks(*makeRequest(false))); -} - -TEST_F(GroupedStatsRequestTest, OneGauge) { - Stats::StatNameTagVector c1Tags{{makeStatName("cluster"), makeStatName("c1")}}; - store_.rootScope()->gaugeFromStatNameWithTags(makeStatName("foo"), c1Tags, - Stats::Gauge::ImportMode::Accumulate); - EXPECT_EQ(1, iterateChunks(*makeRequest(false))); -} - -TEST_F(GroupedStatsRequestTest, OneHistogram) { - Stats::StatNameTagVector c1Tags{{makeStatName("cluster"), makeStatName("c1")}}; - store_.rootScope()->histogramFromStatNameWithTags(makeStatName("foo"), c1Tags, - Stats::Histogram::Unit::Milliseconds); - EXPECT_EQ(1, iterateChunks(*makeRequest(false))); -} - -TEST_F(GroupedStatsRequestTest, OneTextReadout) { - Stats::StatNameTagVector c1Tags{{makeStatName("cluster"), makeStatName("c1")}}; - store_.rootScope()->textReadoutFromStatNameWithTags(makeStatName("foo"), c1Tags); - // text readouts are not included in the returned prometheus stats, unless specifically asked for - // via query param - EXPECT_EQ(0, iterateChunks(*makeRequest(false))); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, true))); -} - -TEST_F(GroupedStatsRequestTest, OneScope) { - Stats::ScopeSharedPtr scope = store_.createScope("foo"); - EXPECT_EQ(0, iterateChunks(*makeRequest(false))); -} - -// example output: -// # TYPE envoy_foo6 counter -// envoy_foo6{cluster="c1"} 0 -TEST_F(GroupedStatsRequestTest, ManyStatsSmallChunkSize) { - for (uint32_t i = 0; i < 10; ++i) { - Stats::StatNameTagVector tags{{makeStatName("cluster"), makeStatName("c1")}}; - store_.rootScope()->counterFromStatNameWithTags(makeStatName(absl::StrCat("foo", i)), tags); - } - std::unique_ptr request = makeRequest(false); - request->setChunkSize(50); - EXPECT_EQ(10, iterateChunks(*request)); -} - -TEST_F(GroupedStatsRequestTest, ManyStatsSmallChunkSizeNoDrain) { - for (uint32_t i = 0; i < 10; ++i) { - Stats::StatNameTagVector tags{{makeStatName("cluster"), makeStatName("c1")}}; - store_.rootScope()->counterFromStatNameWithTags(makeStatName(absl::StrCat("foo", i)), tags); - } - std::unique_ptr request = makeRequest(false); - request->setChunkSize(50); - EXPECT_EQ(10, iterateChunks(*request, false)); -} - -TEST_F(GroupedStatsRequestTest, OneStatUsedOnly) { - Stats::ScopeSharedPtr scope = store_.createScope("foo"); - EXPECT_EQ(0, iterateChunks(*makeRequest(true))); -} - -} // namespace Server -} // namespace Envoy diff --git a/test/server/admin/prometheus_stats_test.cc b/test/server/admin/prometheus_stats_test.cc index 5b57be204ec0..c27af6121130 100644 --- a/test/server/admin/prometheus_stats_test.cc +++ b/test/server/admin/prometheus_stats_test.cc @@ -1,10 +1,14 @@ -#include "envoy/stats/histogram.h" +#include +#include +#include #include "source/common/stats/custom_stat_namespaces_impl.h" +#include "source/server/admin/prometheus_stats.h" #include "test/mocks/stats/mocks.h" -#include "test/server/admin/stats_request_test_base.h" +#include "test/test_common/utility.h" +using testing::NiceMock; using testing::ReturnRef; namespace Envoy { @@ -34,9 +38,6 @@ class HistogramWrapper { histogram_t* histogram_; }; -// TODO(rulex123): these tests should be moved to grouped_stats_request_test.cc, -// but are left here to ease code review while we move Prometheus stats to the -// streaming API. class PrometheusStatsFormatterTest : public testing::Test { protected: PrometheusStatsFormatterTest() : alloc_(*symbol_table_), pool_(*symbol_table_) {} @@ -46,16 +47,16 @@ class PrometheusStatsFormatterTest : public testing::Test { void addCounter(const std::string& name, Stats::StatNameTagVector cluster_tags) { Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_); Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_); - test_scope_ptr_->counters_.push_back(alloc_.makeCounter( - name_storage.statName(), tag_extracted_name_storage.statName(), cluster_tags)); + counters_.push_back(alloc_.makeCounter(name_storage.statName(), + tag_extracted_name_storage.statName(), cluster_tags)); } void addGauge(const std::string& name, Stats::StatNameTagVector cluster_tags) { Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_); Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_); - test_scope_ptr_->gauges_.push_back( - alloc_.makeGauge(name_storage.statName(), tag_extracted_name_storage.statName(), - cluster_tags, Stats::Gauge::ImportMode::Accumulate)); + gauges_.push_back(alloc_.makeGauge(name_storage.statName(), + tag_extracted_name_storage.statName(), cluster_tags, + Stats::Gauge::ImportMode::Accumulate)); } void addTextReadout(const std::string& name, const std::string& value, @@ -65,17 +66,15 @@ class PrometheusStatsFormatterTest : public testing::Test { Stats::TextReadoutSharedPtr textReadout = alloc_.makeTextReadout( name_storage.statName(), tag_extracted_name_storage.statName(), cluster_tags); textReadout->set(value); - test_scope_ptr_->text_readouts_.push_back(textReadout); + textReadouts_.push_back(textReadout); } - using MockHistogramSharedPtr = Stats::RefcountPtr>; - void addHistogram(MockHistogramSharedPtr histogram) { - test_scope_ptr_->histograms_.push_back(histogram); - } + using MockHistogramSharedPtr = Stats::RefcountPtr>; + void addHistogram(MockHistogramSharedPtr histogram) { histograms_.push_back(histogram); } MockHistogramSharedPtr makeHistogram(const std::string& name, Stats::StatNameTagVector cluster_tags) { - auto histogram = MockHistogramSharedPtr(new testing::NiceMock()); + auto histogram = MockHistogramSharedPtr(new NiceMock()); histogram->name_ = baseName(name, cluster_tags); histogram->setTagExtractedName(name); histogram->setTags(cluster_tags); @@ -99,93 +98,67 @@ class PrometheusStatsFormatterTest : public testing::Test { void clearStorage() { pool_.clear(); - test_scope_ptr_->counters_.clear(); - test_scope_ptr_->histograms_.clear(); - test_scope_ptr_->gauges_.clear(); - test_scope_ptr_->text_readouts_.clear(); + counters_.clear(); + gauges_.clear(); + histograms_.clear(); + textReadouts_.clear(); EXPECT_EQ(0, symbol_table_->numSymbols()); } - std::unique_ptr makeRequest(bool used_only, bool text_readouts = false) { - StatsParams params; - params.used_only_ = used_only; - params.prometheus_text_readouts_ = text_readouts; - params.format_ = StatsFormat::Prometheus; - return std::make_unique(mock_store_, params, custom_namespaces_); - } - - std::unique_ptr makeRequest(StatsParams& params) { - return std::make_unique(mock_store_, params, custom_namespaces_); - } - - std::string response(GroupedStatsRequest& request) { - Http::TestResponseHeaderMapImpl response_headers; - Http::Code code = request.start(response_headers); - EXPECT_EQ(Http::Code::OK, code); - Buffer::OwnedImpl data; - while (request.nextChunk(data)) { - } - return data.toString(); - } - Stats::TestUtil::TestSymbolTable symbol_table_; Stats::AllocatorImpl alloc_; Stats::StatNamePool pool_; - Stats::CustomStatNamespacesImpl custom_namespaces_; - testing::NiceMock mock_store_; - std::shared_ptr test_scope_ptr_ = std::make_shared("", mock_store_); + std::vector counters_; + std::vector gauges_; + std::vector histograms_; + std::vector textReadouts_; }; -// TODO(rulex123): these tests should be moved to stats_render_test.cc, -// but are left here to ease code review while we move Prometheus stats to the -// streaming API. -class PrometheusStatsRenderTest : public testing::Test {}; - -TEST_F(PrometheusStatsRenderTest, MetricName) { +TEST_F(PrometheusStatsFormatterTest, MetricName) { Stats::CustomStatNamespacesImpl custom_namespaces; std::string raw = "vulture.eats-liver"; std::string expected = "envoy_vulture_eats_liver"; - auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); EXPECT_TRUE(actual.has_value()); EXPECT_EQ(expected, actual.value()); } -TEST_F(PrometheusStatsRenderTest, SanitizeMetricName) { +TEST_F(PrometheusStatsFormatterTest, SanitizeMetricName) { Stats::CustomStatNamespacesImpl custom_namespaces; std::string raw = "An.artist.plays-violin@019street"; std::string expected = "envoy_An_artist_plays_violin_019street"; - auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); EXPECT_EQ(expected, actual.value()); } -TEST_F(PrometheusStatsRenderTest, SanitizeMetricNameDigitFirst) { +TEST_F(PrometheusStatsFormatterTest, SanitizeMetricNameDigitFirst) { Stats::CustomStatNamespacesImpl custom_namespaces; std::string raw = "3.artists.play-violin@019street"; std::string expected = "envoy_3_artists_play_violin_019street"; - auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); EXPECT_TRUE(actual.has_value()); EXPECT_EQ(expected, actual.value()); } -TEST_F(PrometheusStatsRenderTest, CustomNamespace) { +TEST_F(PrometheusStatsFormatterTest, CustomNamespace) { Stats::CustomStatNamespacesImpl custom_namespaces; custom_namespaces.registerStatNamespace("promstattest"); std::string raw = "promstattest.vulture.eats-liver"; std::string expected = "vulture_eats_liver"; - auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); EXPECT_TRUE(actual.has_value()); EXPECT_EQ(expected, actual.value()); } -TEST_F(PrometheusStatsRenderTest, CustomNamespaceWithInvalidPromnamespace) { +TEST_F(PrometheusStatsFormatterTest, CustomNamespaceWithInvalidPromnamespace) { Stats::CustomStatNamespacesImpl custom_namespaces; custom_namespaces.registerStatNamespace("promstattest"); std::string raw = "promstattest.1234abcd.eats-liver"; - auto actual = PrometheusStatsRender::metricName(raw, custom_namespaces); + auto actual = PrometheusStatsFormatter::metricName(raw, custom_namespaces); EXPECT_FALSE(actual.has_value()); } -TEST_F(PrometheusStatsRenderTest, FormattedTags) { +TEST_F(PrometheusStatsFormatterTest, FormattedTags) { std::vector tags; Stats::Tag tag1 = {"a.tag-name", "a.tag-value"}; Stats::Tag tag2 = {"another_tag_name", "another_tag-value"}; @@ -196,13 +169,16 @@ TEST_F(PrometheusStatsRenderTest, FormattedTags) { tags.push_back(tag3); std::string expected = "a_tag_name=\"a.tag-value\",another_tag_name=\"another_tag-value\"," "replace_problematic=\"val\\\"ue with\\\\ some\\n issues\""; - auto actual = PrometheusStatsRender::formattedTags(tags); + auto actual = PrometheusStatsFormatter::formattedTags(tags); EXPECT_EQ(expected, actual); } -TEST_F(PrometheusStatsFormatterTest, MetricNameCollision) { +TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) { + Stats::CustomStatNamespacesImpl custom_namespaces; + // Create two counters and two gauges with each pair having the same name, - // but having different tag names and values. 2 groups should be rendered. + // but having different tag names and values. + //`statsAsPrometheus()` should return two implying it found two unique stat names addCounter("cluster.test_cluster_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); @@ -213,23 +189,18 @@ TEST_F(PrometheusStatsFormatterTest, MetricNameCollision) { addGauge("cluster.test_cluster_2.upstream_cx_total", {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); - - const std::string expected_output = - R"EOF(# TYPE envoy_cluster_test_cluster_1_upstream_cx_total counter -envoy_cluster_test_cluster_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 -envoy_cluster_test_cluster_1_upstream_cx_total{another_tag_name="another_tag-value"} 0 -# TYPE envoy_cluster_test_cluster_2_upstream_cx_total gauge -envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 -envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0 -)EOF"; - - EXPECT_EQ(expected_output, response(*makeRequest(false))); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(2UL, size); } TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) { + Stats::CustomStatNamespacesImpl custom_namespaces; + // Create two counters and two gauges, all with unique names. - // 4 groups should be rendered. + // statsAsPrometheus() should return four implying it found + // four unique stat names. addCounter("cluster.test_cluster_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); @@ -240,23 +211,14 @@ TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) { addGauge("cluster.test_cluster_4.upstream_cx_total", {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); - - const std::string expected_output = - R"EOF(# TYPE envoy_cluster_test_cluster_1_upstream_cx_total counter -envoy_cluster_test_cluster_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 -# TYPE envoy_cluster_test_cluster_2_upstream_cx_total counter -envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name="another_tag-value"} 0 -# TYPE envoy_cluster_test_cluster_3_upstream_cx_total gauge -envoy_cluster_test_cluster_3_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 -# TYPE envoy_cluster_test_cluster_4_upstream_cx_total gauge -envoy_cluster_test_cluster_4_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0 -)EOF"; - - EXPECT_EQ(expected_output, response(*makeRequest(false))); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(4UL, size); } TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { + Stats::CustomStatNamespacesImpl custom_namespaces; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(std::vector(0)); Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); @@ -265,7 +227,11 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { ON_CALL(*histogram, cumulativeStatistics()).WillByDefault(ReturnRef(h1_cumulative_statistics)); addHistogram(histogram); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram envoy_histogram1_bucket{le="0.5"} 0 @@ -292,10 +258,11 @@ envoy_histogram1_sum{} 0 envoy_histogram1_count{} 0 )EOF"; - EXPECT_EQ(expected_output, response(*makeRequest(false))); + EXPECT_EQ(expected_output, response.toString()); } TEST_F(PrometheusStatsFormatterTest, HistogramWithNonDefaultBuckets) { + Stats::CustomStatNamespacesImpl custom_namespaces; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(std::vector(0)); Stats::ConstSupportedBuckets buckets{10, 20}; @@ -307,7 +274,10 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNonDefaultBuckets) { addHistogram(histogram); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram envoy_histogram1_bucket{le="10"} 0 @@ -317,12 +287,13 @@ envoy_histogram1_sum{} 0 envoy_histogram1_count{} 0 )EOF"; - EXPECT_EQ(expected_output, response(*makeRequest(false))); + EXPECT_EQ(expected_output, response.toString()); } // Test that scaled percents are emitted in the expected 0.0-1.0 range, and that the buckets // apply to the final output range, not the internal scaled range. TEST_F(PrometheusStatsFormatterTest, HistogramWithScaledPercent) { + Stats::CustomStatNamespacesImpl custom_namespaces; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(std::vector(0)); Stats::ConstSupportedBuckets buckets{0.5, 1.0}; @@ -342,7 +313,10 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithScaledPercent) { addHistogram(histogram); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram envoy_histogram1_bucket{le="0.5"} 1 @@ -352,10 +326,11 @@ envoy_histogram1_sum{} 2.2599999999999997868371792719699 envoy_histogram1_count{} 3 )EOF"; - EXPECT_EQ(expected_output, response(*makeRequest(false))); + EXPECT_EQ(expected_output, response.toString()); } TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) { + Stats::CustomStatNamespacesImpl custom_namespaces; HistogramWrapper h1_cumulative; // Force large counts to prove that the +Inf bucket doesn't overflow to scientific notation. @@ -372,7 +347,10 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) { addHistogram(histogram); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram envoy_histogram1_bucket{le="0.5"} 0 @@ -399,11 +377,12 @@ envoy_histogram1_sum{} 105105105000 envoy_histogram1_count{} 101100000 )EOF"; - EXPECT_EQ(expected_output, response(*makeRequest(false))); + EXPECT_EQ(expected_output, response.toString()); } TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { - custom_namespaces_.registerStatNamespace("promtest"); + Stats::CustomStatNamespacesImpl custom_namespaces; + custom_namespaces.registerStatNamespace("promtest"); addCounter("cluster.test_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); @@ -430,7 +409,10 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { addHistogram(histogram1); EXPECT_CALL(*histogram1, cumulativeStatistics()).WillOnce(ReturnRef(h1_cumulative_statistics)); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(7UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 @@ -469,10 +451,12 @@ envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 )EOF"; - EXPECT_EQ(expected_output, response(*makeRequest(false))); + EXPECT_EQ(expected_output, response.toString()); } TEST_F(PrometheusStatsFormatterTest, OutputWithTextReadoutsInGaugeFormat) { + Stats::CustomStatNamespacesImpl custom_namespaces; + addCounter("cluster.upstream_cx_total_count", {{makeStat("cluster"), makeStat("c1")}}); addGauge("cluster.upstream_cx_total", {{makeStat("cluster"), makeStat("c1")}}); // Text readouts that should be returned in gauge format. @@ -482,7 +466,10 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithTextReadoutsInGaugeFormat) { {makeStat("tag2"), makeStat("\n")}, {makeStat("tag3"), makeStat(R"(")")}}); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(4UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_upstream_cx_total_count counter envoy_cluster_upstream_cx_total_count{cluster="c1"} 0 @@ -494,7 +481,7 @@ envoy_control_plane_identifier{cluster="c1",text_value="CP-1"} 0 envoy_invalid_tag_values{tag1="\\",tag2="\n",tag3="\"",text_value="test"} 0 )EOF"; - EXPECT_EQ(expected_output, response(*makeRequest(false, true))); + EXPECT_EQ(expected_output, response.toString()); } // Test that output groups all metrics of the same name (with different tags) together, @@ -502,6 +489,7 @@ envoy_invalid_tag_values{tag1="\\",tag2="\n",tag3="\"",text_value="test"} 0 // should be sorted by their tags; the format specifies that it is preferred that metrics // are always grouped in the same order, and sorting is an easy way to ensure this. TEST_F(PrometheusStatsFormatterTest, OutputSortedByMetricName) { + Stats::CustomStatNamespacesImpl custom_namespaces; const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(h1_values); @@ -526,7 +514,10 @@ TEST_F(PrometheusStatsFormatterTest, OutputSortedByMetricName) { } } - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(6UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_upstream_cx_connect_fail counter envoy_cluster_upstream_cx_connect_fail{cluster="aaa"} 0 @@ -680,10 +671,11 @@ envoy_cluster_upstream_rq_time_sum{cluster="ccc"} 5532 envoy_cluster_upstream_rq_time_count{cluster="ccc"} 7 )EOF"; - EXPECT_EQ(expected_output, response(*makeRequest(false))); + EXPECT_EQ(expected_output, response.toString()); } TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { + Stats::CustomStatNamespacesImpl custom_namespaces; addCounter("cluster.test_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); addCounter("cluster.test_2.upstream_cx_total", @@ -705,7 +697,12 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { addHistogram(histogram1); EXPECT_CALL(*histogram1, cumulativeStatistics()).WillOnce(ReturnRef(h1_cumulative_statistics)); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + Buffer::OwnedImpl response; + StatsParams params; + params.used_only_ = true; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0 @@ -732,10 +729,11 @@ envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 )EOF"; - EXPECT_EQ(expected_output, response(*makeRequest(true))); + EXPECT_EQ(expected_output, response.toString()); } TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { + Stats::CustomStatNamespacesImpl custom_namespaces; const std::vector h1_values = {}; HistogramWrapper h1_cumulative; h1_cumulative.setHistogramValues(h1_values); @@ -747,49 +745,31 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; histogram1->used_ = false; addHistogram(histogram1); - - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); + StatsParams params; { + params.used_only_ = true; EXPECT_CALL(*histogram1, cumulativeStatistics()).Times(0); - std::string result = response(*makeRequest(true)); - EXPECT_EQ(EMPTY_STRING, result); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + EXPECT_EQ(0UL, size); } { + params.used_only_ = false; EXPECT_CALL(*histogram1, cumulativeStatistics()).WillOnce(ReturnRef(h1_cumulative_statistics)); - const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 0 -envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 0 -envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 0 -)EOF"; - - EXPECT_EQ(expected_output, response(*makeRequest(false))); + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + EXPECT_EQ(1UL, size); } } TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) { + Stats::CustomStatNamespacesImpl custom_namespaces; addCounter("cluster.test_1.upstream_cx_total", {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); addCounter("cluster.test_2.upstream_cx_total", @@ -810,27 +790,30 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) { histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; addHistogram(histogram1); - ON_CALL(mock_store_, rootScope()).WillByDefault(testing::Return(test_scope_ptr_)); - const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 )EOF"; { - Buffer::OwnedImpl res; + Buffer::OwnedImpl response; StatsParams params; - ASSERT_EQ(Http::Code::OK, params.parse("/stats?filter=cluster.test_1.upstream_cx_total", res)); - params.format_ = StatsFormat::Prometheus; - EXPECT_EQ(expected_output, response(*makeRequest(params))); + ASSERT_EQ(Http::Code::OK, + params.parse("/stats?filter=cluster.test_1.upstream_cx_total", response)); + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + EXPECT_EQ(1UL, size); + EXPECT_EQ(expected_output, response.toString()); } { - Buffer::OwnedImpl res; + Buffer::OwnedImpl response; StatsParams params; ASSERT_EQ(Http::Code::OK, - params.parse("/stats?filter=cluster.test_1.upstream_cx_total&safe", res)); - params.format_ = StatsFormat::Prometheus; - EXPECT_EQ(expected_output, response(*makeRequest(params))); + params.parse("/stats?filter=cluster.test_1.upstream_cx_total&safe", response)); + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + EXPECT_EQ(1UL, size); + EXPECT_EQ(expected_output, response.toString()); } } diff --git a/test/server/admin/stats_handler_speed_test.cc b/test/server/admin/stats_handler_speed_test.cc index 8dc9ed7b0230..4c045936d094 100644 --- a/test/server/admin/stats_handler_speed_test.cc +++ b/test/server/admin/stats_handler_speed_test.cc @@ -36,12 +36,11 @@ class StatsHandlerTest { */ uint64_t handlerStats(const StatsParams& params) { Buffer::OwnedImpl data; - Admin::RequestPtr request; if (params.format_ == Envoy::Server::StatsFormat::Prometheus) { - request = StatsHandler::makePrometheusRequest(store_, params, custom_namespaces_); - } else { - request = StatsHandler::makeRequest(store_, params); + Envoy::Server::StatsHandler::prometheusRender(store_, custom_namespaces_, params, data); + return data.length(); } + Admin::RequestPtr request = StatsHandler::makeRequest(store_, params); auto response_headers = Http::ResponseHeaderMapImpl::create(); request->start(*response_headers); uint64_t count = 0; diff --git a/test/server/admin/stats_handler_test.cc b/test/server/admin/stats_handler_test.cc index aa3e3c245903..529f0f0993e3 100644 --- a/test/server/admin/stats_handler_test.cc +++ b/test/server/admin/stats_handler_test.cc @@ -5,7 +5,7 @@ #include "source/common/stats/custom_stat_namespaces_impl.h" #include "source/common/stats/thread_local_store.h" #include "source/server/admin/stats_handler.h" -#include "source/server/admin/ungrouped_stats_request.h" +#include "source/server/admin/stats_request.h" #include "test/mocks/server/admin_stream.h" #include "test/mocks/server/instance.h" @@ -87,8 +87,7 @@ class StatsHandlerTest { EXPECT_CALL(api_, customStatNamespaces()).WillRepeatedly(ReturnRef(custom_namespaces_)); StatsHandler handler(instance); request_headers_.setPath(url); - StatsParams params; - Admin::RequestPtr request = handler.makeRequest(admin_stream_, params); + Admin::RequestPtr request = handler.makeRequest(admin_stream_); Http::TestResponseHeaderMapImpl response_headers; Http::Code code = request->start(response_headers); Buffer::OwnedImpl data; @@ -1172,7 +1171,7 @@ class ThreadedTest : public testing::Test { } void statsEndpoint() { - UngroupedStatsRequest request(*store_, StatsParams()); + StatsRequest request(*store_, StatsParams()); Http::TestResponseHeaderMapImpl response_headers; request.start(response_headers); Buffer::OwnedImpl data; diff --git a/test/server/admin/stats_render_test.cc b/test/server/admin/stats_render_test.cc index ff1bd37caff3..b2f9ce223153 100644 --- a/test/server/admin/stats_render_test.cc +++ b/test/server/admin/stats_render_test.cc @@ -1,3 +1,5 @@ +#include + #include "test/server/admin/stats_render_test_base.h" namespace Envoy { diff --git a/test/server/admin/stats_request_test.cc b/test/server/admin/stats_request_test.cc new file mode 100644 index 000000000000..23bd46a4e2c1 --- /dev/null +++ b/test/server/admin/stats_request_test.cc @@ -0,0 +1,167 @@ +#include + +#include "source/common/buffer/buffer_impl.h" +#include "source/common/stats/thread_local_store.h" +#include "source/server/admin/stats_request.h" + +#include "test/mocks/event/mocks.h" +#include "test/mocks/stats/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::StartsWith; + +namespace Envoy { +namespace Server { + +class StatsRequestTest : public testing::Test { +protected: + StatsRequestTest() : pool_(symbol_table_), alloc_(symbol_table_), store_(alloc_) { + store_.addSink(sink_); + store_.initializeThreading(main_thread_dispatcher_, tls_); + } + + ~StatsRequestTest() override { + tls_.shutdownGlobalThreading(); + store_.shutdownThreading(); + tls_.shutdownThread(); + } + + std::unique_ptr makeRequest(bool used_only, StatsFormat format, StatsType type) { + StatsParams params; + params.used_only_ = used_only; + params.type_ = type; + params.format_ = format; + return std::make_unique(store_, params); + } + + // Executes a request, counting the chunks that were generated. + uint32_t iterateChunks(StatsRequest& request, bool drain = true, + Http::Code expect_code = Http::Code::OK) { + Http::TestResponseHeaderMapImpl response_headers; + Http::Code code = request.start(response_headers); + EXPECT_EQ(expect_code, code); + if (code != Http::Code::OK) { + return 0; + } + Buffer::OwnedImpl data; + uint32_t num_chunks = 0; + bool more = true; + do { + more = request.nextChunk(data); + uint64_t size = data.length(); + if (size > 0) { + ++num_chunks; + if (drain) { + data.drain(size); + } + } + } while (more); + return num_chunks; + } + + // Executes a request, returning the rendered buffer as a string. + std::string response(StatsRequest& request) { + Http::TestResponseHeaderMapImpl response_headers; + Http::Code code = request.start(response_headers); + EXPECT_EQ(Http::Code::OK, code); + Buffer::OwnedImpl data; + while (request.nextChunk(data)) { + } + return data.toString(); + } + + Stats::StatName makeStatName(absl::string_view name) { return pool_.add(name); } + + Stats::SymbolTableImpl symbol_table_; + Stats::StatNamePool pool_; + Stats::AllocatorImpl alloc_; + NiceMock sink_; + NiceMock main_thread_dispatcher_; + NiceMock tls_; + Stats::ThreadLocalStoreImpl store_; + Buffer::OwnedImpl response_; +}; + +TEST_F(StatsRequestTest, Empty) { + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); +} + +TEST_F(StatsRequestTest, OneCounter) { + store_.rootScope()->counterFromStatName(makeStatName("foo")); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); +} + +TEST_F(StatsRequestTest, OneGauge) { + store_.rootScope()->gaugeFromStatName(makeStatName("foo"), Stats::Gauge::ImportMode::Accumulate); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); +} + +TEST_F(StatsRequestTest, OneHistogram) { + store_.rootScope()->histogramFromStatName(makeStatName("foo"), + Stats::Histogram::Unit::Milliseconds); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Histograms))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); +} + +TEST_F(StatsRequestTest, OneTextReadout) { + store_.rootScope()->textReadoutFromStatName(makeStatName("foo")); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::TextReadouts))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); +} + +TEST_F(StatsRequestTest, OneScope) { + Stats::ScopeSharedPtr scope = store_.createScope("foo"); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); +} + +TEST_F(StatsRequestTest, ManyStatsSmallChunkSize) { + for (uint32_t i = 0; i < 100; ++i) { + store_.rootScope()->counterFromStatName(makeStatName(absl::StrCat("foo", i))); + } + std::unique_ptr request = makeRequest(false, StatsFormat::Text, StatsType::All); + request->setChunkSize(100); + EXPECT_EQ(9, iterateChunks(*request)); +} + +TEST_F(StatsRequestTest, ManyStatsSmallChunkSizeNoDrain) { + for (uint32_t i = 0; i < 100; ++i) { + store_.rootScope()->counterFromStatName(makeStatName(absl::StrCat("foo", i))); + } + std::unique_ptr request = makeRequest(false, StatsFormat::Text, StatsType::All); + request->setChunkSize(100); + EXPECT_EQ(9, iterateChunks(*request, false)); +} + +TEST_F(StatsRequestTest, OneStatUsedOnly) { + store_.rootScope()->counterFromStatName(makeStatName("foo")); + EXPECT_EQ(0, iterateChunks(*makeRequest(true, StatsFormat::Text, StatsType::All))); +} + +TEST_F(StatsRequestTest, OneStatJson) { + store_.rootScope()->counterFromStatName(makeStatName("foo")); + EXPECT_THAT(response(*makeRequest(false, StatsFormat::Json, StatsType::All)), StartsWith("{")); +} + +TEST_F(StatsRequestTest, OneStatPrometheus) { + // Currently the rendering infrastructure does not support Prometheus -- that + // gets rendered using a different code-path. This will be fixed at some + // point, to make Prometheus consume less resource, and when that occurs this + // test can exercise that. + store_.rootScope()->counterFromStatName(makeStatName("foo")); + EXPECT_ENVOY_BUG(iterateChunks(*makeRequest(false, StatsFormat::Prometheus, StatsType::All), true, + Http::Code::BadRequest), + "reached Prometheus case in switch unexpectedly"); +} + +} // namespace Server +} // namespace Envoy diff --git a/test/server/admin/stats_request_test_base.cc b/test/server/admin/stats_request_test_base.cc deleted file mode 100644 index 52c655027ab2..000000000000 --- a/test/server/admin/stats_request_test_base.cc +++ /dev/null @@ -1,65 +0,0 @@ -#include "test/server/admin/stats_request_test_base.h" - -#include - -namespace Envoy { -namespace Server { - -template -StatsRequestTestBase::StatsRequestTestBase() - : pool_(symbol_table_), alloc_(symbol_table_), store_(alloc_) { - store_.addSink(sink_); - store_.initializeThreading(main_thread_dispatcher_, tls_); -} - -template StatsRequestTestBase::~StatsRequestTestBase() { - tls_.shutdownGlobalThreading(); - store_.shutdownThreading(); - tls_.shutdownThread(); -} - -// Executes a request, counting the chunks that were generated. -template -uint32_t StatsRequestTestBase::iterateChunks(T& request, bool drain, Http::Code expect_code) { - Http::TestResponseHeaderMapImpl response_headers; - Http::Code code = request.start(response_headers); - EXPECT_EQ(expect_code, code); - if (code != Http::Code::OK) { - return 0; - } - Buffer::OwnedImpl data; - uint32_t num_chunks = 0; - bool more = true; - do { - uint64_t prev_size = data.length(); - more = request.nextChunk(data); - uint64_t size = data.length(); - if ((drain && size > 0) || (!drain && size != prev_size)) { - ++num_chunks; - if (drain) { - data.drain(size); - } - } - } while (more); - return num_chunks; -} - -// Executes a request, returning the rendered buffer as a string. -template std::string StatsRequestTestBase::response(T& request) { - Http::TestResponseHeaderMapImpl response_headers; - Http::Code code = request.start(response_headers); - EXPECT_EQ(Http::Code::OK, code); - Buffer::OwnedImpl data; - while (request.nextChunk(data)) { - } - return data.toString(); -} - -TestScope::TestScope(const std::string& prefix, Stats::MockStore& store) - : Stats::IsolatedScopeImpl(prefix, store) {} - -template class StatsRequestTestBase; -template class StatsRequestTestBase; - -} // namespace Server -} // namespace Envoy diff --git a/test/server/admin/stats_request_test_base.h b/test/server/admin/stats_request_test_base.h deleted file mode 100644 index 2a6bbcf7faa4..000000000000 --- a/test/server/admin/stats_request_test_base.h +++ /dev/null @@ -1,82 +0,0 @@ -#pragma once - -#include -#include - -#include "source/common/buffer/buffer_impl.h" -#include "source/common/stats/thread_local_store.h" -#include "source/server/admin/grouped_stats_request.h" -#include "source/server/admin/ungrouped_stats_request.h" - -#include "test/mocks/event/mocks.h" -#include "test/mocks/stats/mocks.h" -#include "test/mocks/thread_local/mocks.h" -#include "test/test_common/utility.h" - -#include "gtest/gtest.h" - -namespace Envoy { -namespace Server { - -template class StatsRequestTestBase : public testing::Test { -protected: - StatsRequestTestBase(); - ~StatsRequestTestBase() override; - - uint32_t iterateChunks(T& request, bool drain = true, Http::Code expect_code = Http::Code::OK); - - Stats::StatName makeStatName(absl::string_view name) { return pool_.add(name); } - - std::string response(T& request); - - Stats::SymbolTableImpl symbol_table_; - Stats::StatNamePool pool_; - Stats::AllocatorImpl alloc_; - testing::NiceMock sink_; - testing::NiceMock main_thread_dispatcher_; - testing::NiceMock tls_; - Stats::ThreadLocalStoreImpl store_; - Buffer::OwnedImpl response_; -}; - -// TODO(jmarantz): investigate whether we can reuse one of the pre-existing -// scope classes instead of introducing a new subclass of IsolatedScopeImpl -// (used only by a set of tests in prometheus_stats_test.cc). -class TestScope : public Stats::IsolatedScopeImpl { -public: - TestScope(const std::string& prefix, Stats::MockStore& store); - - bool iterate(const Stats::IterateFn& fn) const override { - std::for_each(histograms_.begin(), histograms_.end(), fn); - return true; - } - - bool iterate(const Stats::IterateFn& fn) const override { - std::for_each(counters_.begin(), counters_.end(), fn); - return true; - } - - bool iterate(const Stats::IterateFn& fn) const override { - std::for_each(gauges_.begin(), gauges_.end(), fn); - return true; - } - - bool iterate(const Stats::IterateFn& fn) const override { - std::for_each(text_readouts_.begin(), text_readouts_.end(), fn); - return true; - } - -public: - std::vector histograms_; - std::vector counters_; - std::vector gauges_; - std::vector text_readouts_; -}; - -class MockStore : public Stats::MockStore { -public: - MOCK_METHOD(Stats::ScopeSharedPtr, rootScope, ()); -}; - -} // namespace Server -} // namespace Envoy diff --git a/test/server/admin/ungrouped_stats_request_test.cc b/test/server/admin/ungrouped_stats_request_test.cc deleted file mode 100644 index 65f7d9e0551b..000000000000 --- a/test/server/admin/ungrouped_stats_request_test.cc +++ /dev/null @@ -1,88 +0,0 @@ -#include "test/server/admin/stats_request_test_base.h" - -namespace Envoy { -namespace Server { - -class UngroupedStatsRequestTest : public StatsRequestTestBase { -protected: - std::unique_ptr makeRequest(bool used_only, StatsFormat format, - StatsType type) { - StatsParams params; - params.used_only_ = used_only; - params.type_ = type; - params.format_ = format; - return std::make_unique(store_, params); - } -}; - -TEST_F(UngroupedStatsRequestTest, Empty) { - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); -} - -TEST_F(UngroupedStatsRequestTest, OneCounter) { - store_.rootScope()->counterFromStatName(makeStatName("foo")); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); -} - -TEST_F(UngroupedStatsRequestTest, OneGauge) { - store_.rootScope()->gaugeFromStatName(makeStatName("foo"), Stats::Gauge::ImportMode::Accumulate); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); -} - -TEST_F(UngroupedStatsRequestTest, OneHistogram) { - store_.rootScope()->histogramFromStatName(makeStatName("foo"), - Stats::Histogram::Unit::Milliseconds); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Histograms))); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); -} - -TEST_F(UngroupedStatsRequestTest, OneTextReadout) { - store_.rootScope()->textReadoutFromStatName(makeStatName("foo")); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); - EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::TextReadouts))); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); -} - -TEST_F(UngroupedStatsRequestTest, OneScope) { - Stats::ScopeSharedPtr scope = store_.createScope("foo"); - EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); -} - -TEST_F(UngroupedStatsRequestTest, ManyStatsSmallChunkSize) { - for (uint32_t i = 0; i < 100; ++i) { - store_.rootScope()->counterFromStatName(makeStatName(absl::StrCat("foo", i))); - } - std::unique_ptr request = - makeRequest(false, StatsFormat::Text, StatsType::All); - request->setChunkSize(100); - EXPECT_EQ(9, iterateChunks(*request)); -} - -TEST_F(UngroupedStatsRequestTest, ManyStatsSmallChunkSizeNoDrain) { - for (uint32_t i = 0; i < 100; ++i) { - store_.rootScope()->counterFromStatName(makeStatName(absl::StrCat("foo", i))); - } - std::unique_ptr request = - makeRequest(false, StatsFormat::Text, StatsType::All); - request->setChunkSize(100); - EXPECT_EQ(9, iterateChunks(*request, false)); -} - -TEST_F(UngroupedStatsRequestTest, OneStatUsedOnly) { - store_.rootScope()->counterFromStatName(makeStatName("foo")); - EXPECT_EQ(0, iterateChunks(*makeRequest(true, StatsFormat::Text, StatsType::All))); -} - -TEST_F(UngroupedStatsRequestTest, OneStatJson) { - store_.rootScope()->counterFromStatName(makeStatName("foo")); - EXPECT_THAT(response(*makeRequest(false, StatsFormat::Json, StatsType::All)), - testing::StartsWith("{")); -} - -} // namespace Server -} // namespace Envoy diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index c8e71e928107..f79ef30ac258 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -193,11 +193,10 @@ paths: - source/common/stats/tag_extractor_impl.h - source/common/stats/tag_extractor_impl.cc - source/common/version/version.cc - - source/server/admin/grouped_stats_request.cc - - source/server/admin/ungrouped_stats_request.cc - - source/server/admin/prometheus_stats_render.h - - source/server/admin/prometheus_stats_render.cc + - source/server/admin/prometheus_stats.h + - source/server/admin/prometheus_stats.cc - source/server/admin/stats_params.h + - source/server/admin/stats_request.cc - source/server/admin/utils.h - source/server/admin/utils.cc - tools/clang_tools/api_booster/main.cc From f88e29647931c7c1f1d2ff181e4d95602b0928fa Mon Sep 17 00:00:00 2001 From: Gustavo Gabriel Moyano Date: Sat, 6 May 2023 11:14:41 -0300 Subject: [PATCH 133/740] oauth2: fix Max-Age attribute of Set-Cookie response header (#26764) Signed-off-by: Gustavo Gabriel Moyano --- changelogs/current.yaml | 7 ++++ source/common/runtime/runtime_features.cc | 1 + .../extensions/filters/http/oauth2/filter.cc | 14 ++++++-- .../extensions/filters/http/oauth2/filter.h | 1 + .../filters/http/oauth2/filter_test.cc | 33 +++++++++++++++++++ 5 files changed, 53 insertions(+), 3 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 6f20d0b1c295..4a9dc3fd95f8 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -51,6 +51,13 @@ minor_behavior_changes: bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* +- area: oauth2 + change: | + The Max-Age attribute of Set-Cookie HTTP response header was being assigned a value representing Seconds Since + the Epoch, causing cookies to expire in ~53 years. This was fixed an now it is being assinged a value representing + the number of seconds until the cookie expires. + This behavioral change can be temporarily reverted by setting runtime guard + ``envoy.reloadable_features.oauth_use_standard_max_age_value`` to false. - area: tls change: | Fix build FIPS compliance when using both FIPS mode and Wasm extensions (``--define boringssl=fips`` and ``--define wasm=v8``). diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 3960237d9609..ff9d03110511 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -53,6 +53,7 @@ RUNTIME_GUARD(envoy_reloadable_features_initialize_upstream_filters); RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name); RUNTIME_GUARD(envoy_reloadable_features_no_full_scan_certs_on_sni_mismatch); RUNTIME_GUARD(envoy_reloadable_features_oauth_header_passthrough_fix); +RUNTIME_GUARD(envoy_reloadable_features_oauth_use_standard_max_age_value); RUNTIME_GUARD(envoy_reloadable_features_oauth_use_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_original_dst_rely_on_idle_timeout); RUNTIME_GUARD(envoy_reloadable_features_prohibit_route_refresh_after_response_headers_sent); diff --git a/source/extensions/filters/http/oauth2/filter.cc b/source/extensions/filters/http/oauth2/filter.cc index 64f3fc07a4d7..af111b3c26b1 100644 --- a/source/extensions/filters/http/oauth2/filter.cc +++ b/source/extensions/filters/http/oauth2/filter.cc @@ -460,6 +460,7 @@ void OAuth2Filter::updateTokens(const std::string& access_token, const std::stri access_token_ = access_token; id_token_ = id_token; refresh_token_ = refresh_token; + expires_in_ = std::to_string(expires_in.count()); const auto new_epoch = time_source_.systemTime() + expires_in; new_expires_ = std::to_string( @@ -510,10 +511,17 @@ void OAuth2Filter::finishGetAccessTokenFlow() { void OAuth2Filter::addResponseCookies(Http::ResponseHeaderMap& headers, const std::string& encoded_token) const { + std::string max_age; + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.oauth_use_standard_max_age_value")) { + max_age = expires_in_; + } else { + max_age = new_expires_; + } + // We use HTTP Only cookies for the HMAC and Expiry. - const std::string cookie_tail = fmt::format(CookieTailFormatString, new_expires_); - const std::string cookie_tail_http_only = - fmt::format(CookieTailHttpOnlyFormatString, new_expires_); + const std::string cookie_tail = fmt::format(CookieTailFormatString, max_age); + const std::string cookie_tail_http_only = fmt::format(CookieTailHttpOnlyFormatString, max_age); const CookieNames& cookie_names = config_->cookieNames(); diff --git a/source/extensions/filters/http/oauth2/filter.h b/source/extensions/filters/http/oauth2/filter.h index fc3eaab4f1d7..93e555035ebd 100644 --- a/source/extensions/filters/http/oauth2/filter.h +++ b/source/extensions/filters/http/oauth2/filter.h @@ -261,6 +261,7 @@ class OAuth2Filter : public Http::PassThroughDecoderFilter, public FilterCallbac std::string access_token_; // TODO - see if we can avoid this being a member variable std::string id_token_; std::string refresh_token_; + std::string expires_in_; std::string new_expires_; absl::string_view host_; std::string state_; diff --git a/test/extensions/filters/http/oauth2/filter_test.cc b/test/extensions/filters/http/oauth2/filter_test.cc index 8cf6b8125821..ddf8f16f4fb5 100644 --- a/test/extensions/filters/http/oauth2/filter_test.cc +++ b/test/extensions/filters/http/oauth2/filter_test.cc @@ -1214,6 +1214,39 @@ TEST_F(OAuth2Test, OAuthTestFullFlowPostWithParameters) { */ TEST_F(OAuth2Test, OAuthAccessTokenSucessWithTokens) { + // Set SystemTime to a fixed point so we get consistent HMAC encodings between test runs. + test_time_.setSystemTime(SystemTime(std::chrono::seconds(1000))); + + // Expected response after the callback is complete. + Http::TestRequestHeaderMapImpl expected_headers{ + {Http::Headers::get().Status.get(), "302"}, + {Http::Headers::get().SetCookie.get(), + "OauthHMAC=" + "MjI2YmI5YTRiZjJlNTFlNDUzZWVjOWUzYmU1MThlNGQyNDgyNzA0ZTBkMGQyY2M3M2QyMzg3NTRkZTY0YmU5YQ==;" + "version=1;path=/;Max-Age=600;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "OauthExpires=1600;version=1;path=/;Max-Age=600;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "BearerToken=access_code;version=1;path=/;Max-Age=600;secure"}, + {Http::Headers::get().SetCookie.get(), + "IdToken=some-id-token;version=1;path=/;Max-Age=600;secure"}, + {Http::Headers::get().SetCookie.get(), + "RefreshToken=some-refresh-token;version=1;path=/;Max-Age=600;secure"}, + {Http::Headers::get().Location.get(), ""}, + }; + + EXPECT_CALL(decoder_callbacks_, encodeHeaders_(HeaderMapEqualRef(&expected_headers), true)); + + filter_->onGetAccessTokenSuccess("access_code", "some-id-token", "some-refresh-token", + std::chrono::seconds(600)); +} + +TEST_F(OAuth2Test, OAuthAccessTokenSucessWithTokens_oauth_use_standard_max_age_value) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({ + {"envoy.reloadable_features.oauth_use_standard_max_age_value", "false"}, + }); + // Set SystemTime to a fixed point so we get consistent HMAC encodings between test runs. test_time_.setSystemTime(SystemTime(std::chrono::seconds(0))); From 244a13c1c3819886143b3c6a95d59a6fd3cd1ba5 Mon Sep 17 00:00:00 2001 From: code Date: Mon, 8 May 2023 10:11:52 +0800 Subject: [PATCH 134/740] minor opt: fix the content of exceptions to make it more accurate (#27241) Commit Message: minor opt: fix the content of exceptions to make it more accurate Additional Description: Risk Level: n/a. Testing: n/a. Docs Changes: n/a. Release Notes: n/a. Platform Specific Features: n/a. Signed-off-by: wbpcode --- source/common/router/config_impl.cc | 13 +++++++------ test/common/router/config_impl_test.cc | 6 ++++-- .../http_typed_per_filter_config_test.cc | 5 ++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index c7bda0dc134e..014920a35bcd 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -2130,13 +2130,14 @@ RouteSpecificFilterConfigConstSharedPtr PerFilterConfigs::createRouteSpecificFil auto object = factory->createRouteSpecificFilterConfig(*proto_config, factory_context, validator); if (object == nullptr) { if (is_optional) { - ENVOY_LOG(debug, - "The filter {} doesn't support virtual host-specific configurations, and it is " - "optional, so ignore it.", - name); + ENVOY_LOG( + debug, + "The filter {} doesn't support virtual host or route specific configurations, and it is " + "optional, so ignore it.", + name); } else { - throw EnvoyException( - fmt::format("The filter {} doesn't support virtual host-specific configurations", name)); + throw EnvoyException(fmt::format( + "The filter {} doesn't support virtual host or route specific configurations", name)); } } return object; diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 3b47694f1a50..2b848d3d9319 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -10292,7 +10292,8 @@ TEST_F(PerFilterConfigsTest, DefaultFilterImplementationAnyWithCheckPerVirtualHo EXPECT_THROW_WITH_MESSAGE( TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true), EnvoyException, - "The filter test.default.filter doesn't support virtual host-specific configurations"); + "The filter test.default.filter doesn't support virtual host or route specific " + "configurations"); } TEST_F(PerFilterConfigsTest, OptionalDefaultFilterImplementationAnyWithCheckPerVirtualHost) { @@ -10333,7 +10334,8 @@ TEST_F(PerFilterConfigsTest, DefaultFilterImplementationAnyWithCheckPerRoute) { EXPECT_THROW_WITH_MESSAGE( TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true), EnvoyException, - "The filter test.default.filter doesn't support virtual host-specific configurations"); + "The filter test.default.filter doesn't support virtual host or route specific " + "configurations"); } TEST_F(PerFilterConfigsTest, OptionalDefaultFilterImplementationAnyWithCheckPerRoute) { diff --git a/test/integration/http_typed_per_filter_config_test.cc b/test/integration/http_typed_per_filter_config_test.cc index 2ce029871d65..43b82354dbd1 100644 --- a/test/integration/http_typed_per_filter_config_test.cc +++ b/test/integration/http_typed_per_filter_config_test.cc @@ -30,9 +30,8 @@ TEST_F(HTTPTypedPerFilterConfigTest, RejectUnsupportedTypedPerFilterConfig) { auto size = hcm.http_filters_size(); hcm.mutable_http_filters()->SwapElements(size - 2, size - 1); }); - EXPECT_DEATH( - initialize(), - "The filter set-response-code-filter doesn't support virtual host-specific configurations"); + EXPECT_DEATH(initialize(), "The filter set-response-code-filter doesn't support virtual host or " + "route specific configurations"); } TEST_F(HTTPTypedPerFilterConfigTest, RejectUnknownHttpFilterInTypedPerFilterConfig) { From 4a05f11e6f45d6dda0e34836a25fe7071ab3eb14 Mon Sep 17 00:00:00 2001 From: code Date: Mon, 8 May 2023 11:10:56 +0800 Subject: [PATCH 135/740] fix typo of comments of http filter interface (#27227) Commit Message: fix typo of comments of http filter interface Additional Description: Risk Level: n/a. Testing: n/a. Docs Changes: n/a. Release Notes: n/a. Platform Specific Features: n/a. Signed-off-by: wbpcode --- envoy/http/filter.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/envoy/http/filter.h b/envoy/http/filter.h index 9ce3ebc697a9..98176a29d8fd 100644 --- a/envoy/http/filter.h +++ b/envoy/http/filter.h @@ -432,9 +432,9 @@ class StreamFilterCallbacks { virtual OptRef upstreamCallbacks() PURE; /** - * Return a handle to the downstream callbacks. This is valid for downstream filters, and nullopt - * for upstream filters. - */ + * Return a handle to the downstream callbacks. This is valid for downstream filters, and nullopt + * for upstream filters. + */ virtual OptRef downstreamCallbacks() PURE; }; From 7aaaa197ab8dab7b47a64c2b6662486cb2033706 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 09:59:59 +0100 Subject: [PATCH 136/740] build(deps): bump orjson from 3.8.11 to 3.8.12 in /tools/base (#27247) Bumps [orjson](https://github.com/ijl/orjson) from 3.8.11 to 3.8.12. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.8.11...3.8.12) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 99 ++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 60a6c8567166..bbf0b485b6d2 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -785,58 +785,53 @@ multidict==6.0.4 \ # -r requirements.in # aiohttp # yarl -orjson==3.8.11 \ - --hash=sha256:04b60dfc1251742e79bb075d7a7c4e37078b932a02e6f005c45761bd90c69189 \ - --hash=sha256:08729e339ff3146e6de56c1166f014c3d2ec3e79ffb76d6c55d52cc892e5e477 \ - --hash=sha256:0a53b3c02a38aadc5302661c2ca18645093971488992df77ce14fef16f598b2e \ - --hash=sha256:0bc3d1b93a73b46a698c054697eb2d27bdedbc5ea0d11ec5f1a6bfbec36346b5 \ - --hash=sha256:0f9415b86ef154bf247fa78a6918aac50089c296e26fb6cf15bc9d7e6402a1f8 \ - --hash=sha256:1103e597c16f82c241e1b02beadc9c91cecd93e60433ca73cb6464dcc235f37c \ - --hash=sha256:12f647d4da0aab1997e25bed4fa2b76782b5b9d2d1bf3066b5f0a57d34d833c4 \ - --hash=sha256:173b8f8c750590f432757292cfb197582e5c14347b913b4017561d47af0e759b \ - --hash=sha256:176d742f53434541e50a5e659694073aa51dcbd8f29a1708a4fa1a320193c615 \ - --hash=sha256:1e97fdbb779a3b8f5d9fc7dfddef5325f81ee45897eb7cb4638d5d9734d42514 \ - --hash=sha256:1fedcc428416e23a6c9de62a000c22ae33bbe0108302ad5d5935e29ea739bf37 \ - --hash=sha256:235926b38ed9b76ab2bca99ff26ece79c1c46bc10079b06e660b087aecffbe69 \ - --hash=sha256:2fc050f8e7f2e4061c8c9968ad0be745b11b03913b77ffa8ceca65914696886c \ - --hash=sha256:3485c458670c0edb79ca149fe201f199dd9ccfe7ca3acbdef617e3c683e7b97f \ - --hash=sha256:358e515b8b19a275b259f5ee1e0efa2859b1d976b5ed5d016ac59f9e6c8788a3 \ - --hash=sha256:37f38c8194ce086e6a9816b4b8dde5e7f383feeed92feec0385d99baf64f9b6e \ - --hash=sha256:382f15861a4bf447ab9d07106010e61b217ef6d4245c6cf64af0c12c4c5e2346 \ - --hash=sha256:3afccf7f8684dca7f017837a315de0a1ab5c095de22a4eed206d079f9325ed72 \ - --hash=sha256:3b65424ceee82b94e3613233b67ef110dc58f9d83b0076ec47a506289552a861 \ - --hash=sha256:3c55065bc2075a5ea6ffb30462d84fd3aa5bbb7ae600855c325ee5753feec715 \ - --hash=sha256:4118dcd2b5a27a22af5ad92414073f25d93bca1868f1f580056003c84841062f \ - --hash=sha256:553fdaf9f4b5060a0dcc517ae0c511c289c184a83d6719d03c5602ed0eef0390 \ - --hash=sha256:58c068f93d701f9466f667bf3b5cb4e4946aee940df2b07ca5101f1cf1b60ce4 \ - --hash=sha256:5c3b5405edc3a5f9e34516ee1a729f6c46aecf6de960ae07a7b3e95ebdd0e1d9 \ - --hash=sha256:5ff10789cbc08a9fd94507c907ba55b9315e99f20345ff8ef34fac432dacd948 \ - --hash=sha256:62eb8bdcf6f4cdbe12743e88ad98696277a75f91a35e8fb93a7ea2b9f4a7000c \ - --hash=sha256:66f0c9e4e8f6641497a7dc50591af3704b11468e9fc90cfb5874f28b0a61edb5 \ - --hash=sha256:6d7b050135669d2335e40120215ad4120e29958c139f8bab68ce06a1cb1a1b2c \ - --hash=sha256:714c3e2be6ed7e4ff6e887926d6e171bfd94fdee76d7d3bfa74ee19237a2d49d \ - --hash=sha256:71a656f1c62e84c69060093e20cedff6a92e472d53ff5b8b9026b1b298542a68 \ - --hash=sha256:7c7b4fae3b8fc69c8e76f1c0694f3decfe8a57f87e7ac7779ebb59cd71135438 \ - --hash=sha256:7e4ded77ac7432a155d1d27a83bcadf722750aea3b9e6c4d47f2a92054ab71cb \ - --hash=sha256:882c77126c42dd93bb35288632d69b1e393863a2b752de3e5fe0112833609496 \ - --hash=sha256:9486963d2e65482c565dacb366adb36d22aa22acf7274b61490244c3d87fa631 \ - --hash=sha256:982ab319b7a5ece4199caf2a2b3a28e62a8e289cb6418548ef98bced7e2a6cfe \ - --hash=sha256:98befa717efaab7ddb847ebe47d473f6bd6f0cb53e98e6c3d487c7c58ba2e174 \ - --hash=sha256:9fa900bdd84b4576c8dd6f3e2a00b35797f29283af328c6e3d70addfa4c2d599 \ - --hash=sha256:b369019e597b59c4b97e9f925a3b725321fa1481c129d76c74c6ea3823f5d1e8 \ - --hash=sha256:b68a07794834b7bd53ae2a8b4fe4bf010734cae3f0917d434c83b97acf8e5bce \ - --hash=sha256:bf48ed8d4b6ab9f23b7ee642462369d7133412d72824bad89f9bf4311c06c6a1 \ - --hash=sha256:c2d3e6b65458ed71b6797f321d6e8bfeeadee9d3d31cac47806a608ea745edd7 \ - --hash=sha256:c67ac094a4dde914297543af19f22532d7124f3a35245580d8b756c4ff2f5884 \ - --hash=sha256:cdf201e77d3fac9d8d6f68d872ef45dccfe46f30b268bb88b6c5af5065b433aa \ - --hash=sha256:d47f97b99beb9bcac6e288a76b559543a61e0187443d8089204b757726b1d000 \ - --hash=sha256:d70b6db9d4e1e6057829cd7fe119c217cebaf989f88d14b2445fa69fc568d03e \ - --hash=sha256:d7d5aecccfaf2052cd07ed5bec8efba9ddfea055682fcd346047b1a3e9da3034 \ - --hash=sha256:e14903bfeb591a9117b7d40d81e3ebca9700b4e77bd829d6f22ea57941bb0ebf \ - --hash=sha256:ef52f1d5a2f89ef9049781c90ea35d5edf74374ed6ed515c286a706d1b290267 \ - --hash=sha256:f2ef933da50b31c112b252be03d1ef59e0d0552c1a08e48295bd529ce42aaab8 \ - --hash=sha256:f4e4a1001933166fd1c257b920b241b35322bef99ed7329338bf266ac053abe7 \ - --hash=sha256:f7aeefac55848aeb29f20b91fa55f9e488f446201bb1bb31dc17480d113d8955 +orjson==3.8.12 \ + --hash=sha256:062e67108c218fdb9475edd5272b1629c05b56c66416fa915de5656adde30e73 \ + --hash=sha256:06e528f9a84fbb4000fd0eee573b5db543ee70ae586fdbc53e740b0ac981701c \ + --hash=sha256:0ba645c92801417933fa74448622ba614a275ea82df05e888095c7742d913bb4 \ + --hash=sha256:135f29cf936283a0cd1b8bce86540ca181108f2a4d4483eedad6b8026865d2a9 \ + --hash=sha256:29706dd8189835bcf1781faed286e99ae54fd6165437d364dfdbf0276bf39b19 \ + --hash=sha256:2ad149ed76dce2bbdfbadd61c35959305e77141badf364a158beb4ef3d88ec37 \ + --hash=sha256:355055e0977c43b0e5325b9312b7208c696fe20cd54eed1d6fc80b0a4d6721f5 \ + --hash=sha256:397670665f94cf5cff779054781d80395084ba97191d82f7b3a86f0a20e6102b \ + --hash=sha256:3fa58ca064c640fa9d823f98fbbc8e71940ecb78cea3ac2507da1cbf49d60b51 \ + --hash=sha256:44f7bb4c995652106276442de1147c9993716d1e2d79b7fd435afa154ff236b9 \ + --hash=sha256:4fd240e736ce52cd757d74142d9933fd35a3184396be887c435f0574e0388654 \ + --hash=sha256:62f999798f2fa55e567d483864ebfc30120fb055c2696a255979439323a5b15c \ + --hash=sha256:6cae2ff288a80e81ce30313e735c5436495ab58cf8d4fbe84900e616d0ee7a78 \ + --hash=sha256:6d1acf52d3a4b9384af09a5c2658c3a7a472a4d62a0ad1fe2c8fab8ef460c9b4 \ + --hash=sha256:6f1b01f641f5e87168b819ac1cbd81aa6278e7572c326f3d27e92dea442a2c0d \ + --hash=sha256:6f568205519bb0197ca91915c5da6058cfbb59993e557b02dfc3b2718b34770a \ + --hash=sha256:710c40c214b753392e46f9275fd795e9630dd737a5ab4ac6e4ee1a02fe83cc0d \ + --hash=sha256:77710774faed337ac4ad919dadc5f3b655b0cd40518e5386e6f1f116de9c6c25 \ + --hash=sha256:7d50d9b1ae409ea15534365fec0ce8a5a5f7dc94aa790aacfb8cfec87ab51aa4 \ + --hash=sha256:7d63f524048825e05950db3b6998c756d5377a5e8c469b2e3bdb9f3217523d74 \ + --hash=sha256:7e405d54c84c30d9b1c918c290bcf4ef484a45c69d5583a95db81ffffba40b44 \ + --hash=sha256:7e549468867991f6f9cfbd9c5bbc977330173bd8f6ceb79973bbd4634e13e1b9 \ + --hash=sha256:82d65e478a21f98107b4eb8390104746bb3024c27084b57edab7d427385f1f70 \ + --hash=sha256:834b50df79f1fe89bbaced3a1c1d8c8c92cc99e84cdcd374d8da4974b3560d2a \ + --hash=sha256:83e8c740a718fa6d511a82e463adc7ab17631c6eea81a716b723e127a9c51d57 \ + --hash=sha256:8682f752c19f6a7d9fc727fd98588b4c8b0dce791b5794bb814c7379ccd64a79 \ + --hash=sha256:8d153b228b6e24f8bccf732a51e01e8e938eef59efed9030c5c257778fbe0804 \ + --hash=sha256:8f00038bf5d07439d13c0c2c5cd6ad48eb86df7dbd7a484013ce6a113c421b14 \ + --hash=sha256:96fb1eb82b578eb6c0e53e3cf950839fe98ea210626f87c8204bd4fc2cc6ba02 \ + --hash=sha256:9a6c1594d5a9ff56e5babc4a87ac372af38d37adef9e06744e9f158431e33f43 \ + --hash=sha256:9f0f042cf002a474a6aea006dd9f8d7a5497e35e5fb190ec78eb4d232ec19955 \ + --hash=sha256:a72b50719bdd6bb0acfca3d4d1c841aa4b191f3ff37268e7aba04e5d6be44ccd \ + --hash=sha256:aff761de5ed5543a0a51e9f703668624749aa2239de5d7d37d9c9693daeaf5dc \ + --hash=sha256:becbd5af6d035a7ec2ee3239d4700929d52d8517806b97dd04efcc37289403f7 \ + --hash=sha256:c6390ce0bce24c107fc275736aa8a4f768ef7eb5df935d7dca0cc99815eb5d99 \ + --hash=sha256:c84046e890e13a119404a83f2e09e622509ed4692846ff94c4ca03654fbc7fb5 \ + --hash=sha256:cd6fbd1413559572e81b5ac64c45388147c3ba85cc3df2eaa11002945e0dbd1f \ + --hash=sha256:d937503e4dfba5edc8d5e0426d3cc97ed55716e93212b2e12a198664487b9965 \ + --hash=sha256:dc27a8ec13f28e92dc1ea89bf1232d77e7d3ebfd5c1ccf4f3729a70561cb63bd \ + --hash=sha256:de3d096dde3e46d01841abc1982b906694ab3c92f338d37a2e6184739dc8a958 \ + --hash=sha256:eb16e0195febd24b44f4db1ab3be85ecf6038f92fd511370cebc004b3d422294 \ + --hash=sha256:ebb03e4c7648f7bb299872002a6120082da018f41ba7a9ebf4ceae8d765443d2 \ + --hash=sha256:ec4f0130d9a27cb400423e09e0f9e46480e9e977f05fdcf663a7a2c68735513e \ + --hash=sha256:efb3a10030462a22c731682434df5c137a67632a8339f821cd501920b169007e \ + --hash=sha256:f480ae7b84369b1860d8867f0baf8d885fede400fda390ce088bfa8edf97ffdc \ + --hash=sha256:f4e22b0aa70c963ac01fcd620de15be21a5027711b0e5d4b96debcdeea43e3ae # via # -r requirements.in # envoy-base-utils From 4cd5d3e0d104d20df7ce9350d2038f50084f4dfd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 10:00:19 +0100 Subject: [PATCH 137/740] build(deps): bump golang from `995b84e` to `2dc5c56` in /examples/shared/golang (#27250) build(deps): bump golang in /examples/shared/golang Bumps golang from `995b84e` to `2dc5c56`. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/golang/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile index a5eba1863a66..feaafc04b9b4 100644 --- a/examples/shared/golang/Dockerfile +++ b/examples/shared/golang/Dockerfile @@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache -FROM golang:1.20.4-bullseye@sha256:995b84ed1eb95c3aefa2787b00bad9129a6b0edc5363328bcde8b49d909fd4df as golang-base +FROM golang:1.20.4-bullseye@sha256:2dc5c568c8a314583090e887e8d96d313e081e2c5333d0a7b935906baf77cee9 as golang-base FROM golang-base as golang-control-plane-builder From c6d52610053eab16ae52e8c0d5268edf3c2dfaaa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 09:56:36 +0000 Subject: [PATCH 138/740] build(deps): bump node from 20.0-bullseye-slim to 20.1-bullseye-slim in /examples/shared/node (#27199) build(deps): bump node in /examples/shared/node Bumps node from 20.0-bullseye-slim to 20.1-bullseye-slim. --- updated-dependencies: - dependency-name: node dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/node/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile index fe3a4113dff8..344e4ac0dd05 100644 --- a/examples/shared/node/Dockerfile +++ b/examples/shared/node/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.0-bullseye-slim@sha256:2cd3fdd49eb8bc1b942be824030469beca57fb6723edaaa9041e593b79a56cd1 as node-base +FROM node:20.1-bullseye-slim@sha256:bc5812b018fa74ea7dbe759cb6c0b456ff96a5c2bc8765e132438f6a75cd6946 as node-base FROM node-base as node-http-auth From 1542e0f58f398650b1cab336c190ce85ebb18f08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 11:10:36 +0100 Subject: [PATCH 139/740] build(deps): bump google.golang.org/grpc from 1.54.0 to 1.55.0 in /examples/ext_authz/auth/grpc-service (#27196) build(deps): bump google.golang.org/grpc Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.54.0 to 1.55.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.54.0...v1.55.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/ext_authz/auth/grpc-service/go.mod | 3 +-- examples/ext_authz/auth/grpc-service/go.sum | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/examples/ext_authz/auth/grpc-service/go.mod b/examples/ext_authz/auth/grpc-service/go.mod index f03d2c306801..a1a9385404d8 100644 --- a/examples/ext_authz/auth/grpc-service/go.mod +++ b/examples/ext_authz/auth/grpc-service/go.mod @@ -3,9 +3,8 @@ module github.com/envoyproxy/envoy/examples/ext_authz/auth/grpc-service go 1.14 require ( - github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43 // indirect github.com/envoyproxy/go-control-plane v0.11.0 github.com/golang/protobuf v1.5.3 google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 - google.golang.org/grpc v1.54.0 + google.golang.org/grpc v1.55.0 ) diff --git a/examples/ext_authz/auth/grpc-service/go.sum b/examples/ext_authz/auth/grpc-service/go.sum index 0aec789ce4c2..ed297be4e25b 100644 --- a/examples/ext_authz/auth/grpc-service/go.sum +++ b/examples/ext_authz/auth/grpc-service/go.sum @@ -547,8 +547,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43 h1:XP+uhjN0yBCN/tPkr8Z0BNDc5rZam9RG6UWyf2FrSQ0= -github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 h1:58f1tJ1ra+zFINPlwLWvQsR9CzAKt2e+EWV2yX9oXQ4= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -568,8 +568,9 @@ github.com/envoyproxy/go-control-plane v0.11.0 h1:jtLewhRR2vMRNnq2ZZUoCjUlgut+Y0 github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= -github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.0 h1:oIfnZFdC0YhpNNEX+SuIqko4cqqVZeN9IGTrhZje83Y= +github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -589,6 +590,7 @@ github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -707,6 +709,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -832,6 +835,7 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -916,6 +920,7 @@ golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1097,6 +1102,7 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1340,8 +1346,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1357,8 +1363,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 908af27449f370c03f4b021570f747302a419306 Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 8 May 2023 11:18:46 +0100 Subject: [PATCH 140/740] ci: Remove gcloud from build image (#27207) Signed-off-by: Ryan Northey --- .azure-pipelines/stage/checks.yml | 8 +- .azure-pipelines/stage/prechecks.yml | 6 +- .azure-pipelines/stage/publish.yml | 6 +- ci/do_ci.sh | 17 +- ci/upload_gcs_artifact.sh | 30 ++- tools/base/requirements.in | 2 + tools/base/requirements.txt | 209 +++++++++++++++++- tools/gsutil/BUILD | 24 ++ ..._crcfunext.cpython-310-x86_64-linux-gnu.so | Bin 0 -> 53824 bytes tools/gsutil/gsutil.py | 4 + tools/gsutil/vendor_util.sh | 33 +++ 11 files changed, 331 insertions(+), 8 deletions(-) create mode 100644 tools/gsutil/BUILD create mode 100755 tools/gsutil/crcmod/_crcfunext.cpython-310-x86_64-linux-gnu.so create mode 100644 tools/gsutil/gsutil.py create mode 100755 tools/gsutil/vendor_util.sh diff --git a/.azure-pipelines/stage/checks.yml b/.azure-pipelines/stage/checks.yml index 20cf344f6fd4..c64ab6bd8422 100644 --- a/.azure-pipelines/stage/checks.yml +++ b/.azure-pipelines/stage/checks.yml @@ -114,12 +114,18 @@ jobs: bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" cacheTestResults: ${{ parameters.cacheTestResults }} - - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)' + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh $(CI_TARGET)-upload' displayName: "Upload $(CI_TARGET) Report to GCS" env: ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci" + BAZEL_REMOTE_CACHE: $(LocalBuildCache) + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)" + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + BAZEL_REMOTE_INSTANCE_BRANCH: "$(Build.SourceBranchName)" condition: always() - job: complete diff --git a/.azure-pipelines/stage/prechecks.yml b/.azure-pipelines/stage/prechecks.yml index 22c1db3ab9c4..4a445157d0cb 100644 --- a/.azure-pipelines/stage/prechecks.yml +++ b/.azure-pipelines/stage/prechecks.yml @@ -135,10 +135,14 @@ jobs: # Publish docs - script: | - ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs' + ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload' displayName: "Upload Docs to GCS" env: ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_RBE: "1" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} condition: eq(variables['CI_TARGET'], 'docs') diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index 3e15b7253ed7..4034eeef16cc 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -259,10 +259,14 @@ jobs: condition: and(eq(variables['isMain'], 'true'), eq(variables['PostSubmit'], true)) - script: | - ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs' + ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload' displayName: "Upload Docs to GCS" env: ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_RBE: "1" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} condition: eq(variables['isMain'], 'true') diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 10467dac322a..27410f17d34f 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -476,6 +476,17 @@ case $CI_TARGET in BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]} --define tcmalloc=gperftools" "${ENVOY_SRCDIR}"/test/run_envoy_bazel_coverage.sh "${COVERAGE_TEST_TARGETS[@]}" collect_build_profile coverage ;; + coverage-upload|fuzz_coverage-upload) + setup_clang_toolchain + + if [[ "$CI_TARGET" == "fuzz_coverage-upload" ]]; then + TARGET=fuzz_coverage + else + TARGET=coverage + fi + + "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" "/source/generated/${TARGET}" "$TARGET" + ;; clang_tidy) # clang-tidy will warn on standard library issues with libc++ ENVOY_STDLIB="libstdc++" @@ -529,6 +540,10 @@ case $CI_TARGET in # Build docs. "${ENVOY_SRCDIR}"/docs/build.sh ;; + docs-upload) + setup_clang_toolchain + "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" /source/generated/docs docs + ;; deps) setup_clang_toolchain @@ -593,7 +608,7 @@ case $CI_TARGET in cp -a linux/arm64/envoy "publish/envoy-${version}-linux-aarch_64" cp -a linux/arm64/envoy-contrib "publish/envoy-contrib-${version}-linux-aarch_64" - "${ENVOY_SRCDIR}ci/publish_github_assets.sh" "v${version}" "${PWD}/publish" + "${ENVOY_SRCDIR}/ci/publish_github_assets.sh" "v${version}" "${PWD}/publish" exit 0 fi diff --git a/ci/upload_gcs_artifact.sh b/ci/upload_gcs_artifact.sh index aea2f6cea5a0..43c0032cc936 100755 --- a/ci/upload_gcs_artifact.sh +++ b/ci/upload_gcs_artifact.sh @@ -12,8 +12,23 @@ if [[ -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then exit 1 fi +read -ra BAZEL_STARTUP_OPTIONS <<< "${BAZEL_STARTUP_OPTION_LIST:-}" +read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTION_LIST:-}" + +remove_key () { + rm -rf "$KEYFILE" +} + +trap remove_key EXIT + # Fail when service account key is not specified -bash -c 'echo ${GCP_SERVICE_ACCOUNT_KEY}' | base64 --decode | gcloud auth activate-service-account --key-file=- +KEYFILE="$(mktemp)" +bash -c 'echo ${GCP_SERVICE_ACCOUNT_KEY}' | base64 --decode > "$KEYFILE" + +cat < ~/.boto +[Credentials] +gs_service_key_file=${KEYFILE} +EOF SOURCE_DIRECTORY="$1" TARGET_SUFFIX="$2" @@ -37,7 +52,11 @@ fi GCS_LOCATION="${GCS_ARTIFACT_BUCKET}/${UPLOAD_PATH}/${TARGET_SUFFIX}" echo "Uploading to gs://${GCS_LOCATION} ..." -gsutil -mq rsync -dr "${SOURCE_DIRECTORY}" "gs://${GCS_LOCATION}" +bazel "${BAZEL_STARTUP_OPTIONS[@]}" run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/gsutil \ + -- -mq rsync \ + -dr "${SOURCE_DIRECTORY}" \ + "gs://${GCS_LOCATION}" # For PR uploads, add a redirect `PR_NUMBER` -> `COMMIT_SHA` if [[ "$BUILD_REASON" == "PullRequest" ]]; then @@ -48,7 +67,12 @@ if [[ "$BUILD_REASON" == "PullRequest" ]]; then > "${TMP_REDIRECT}/index.html" GCS_REDIRECT="${GCS_ARTIFACT_BUCKET}/${REDIRECT_PATH}/${TARGET_SUFFIX}" echo "Uploading redirect to gs://${GCS_REDIRECT} ..." - gsutil -h "Cache-Control:no-cache,max-age=0" -mq rsync -dr "${TMP_REDIRECT}" "gs://${GCS_REDIRECT}" + bazel "${BAZEL_STARTUP_OPTIONS[@]}" run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/gsutil \ + -- -h "Cache-Control:no-cache,max-age=0" \ + -mq rsync \ + -dr "${TMP_REDIRECT}" \ + "gs://${GCS_REDIRECT}" fi if [[ "${COVERAGE_FAILED}" -eq 1 ]]; then diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 4e3cc6c11406..c082aa98a5c0 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -17,6 +17,8 @@ envoy.gpg.sign>=0.2.0 flake8>=6 frozendict>=2.3.7 gitpython +google-cloud-storage +gsutil jinja2 multidict>=6.0.2 orjson diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index bbf0b485b6d2..ee17efec730f 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -182,6 +182,7 @@ aiohttp==3.8.4 \ # envoy-dependency-check # envoy-github-abstract # envoy-github-release + # google-auth # slackclient aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ @@ -191,6 +192,10 @@ alabaster==0.7.13 \ --hash=sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3 \ --hash=sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2 # via sphinx +argcomplete==3.0.8 \ + --hash=sha256:b9ca96448e14fa459d7450a4ab5a22bbf9cee4ba7adddf03e65c398b5daeea28 \ + --hash=sha256:e36fd646839933cbec7941c662ecb65338248667358dd3d968405a4506a60d9b + # via gsutil async-timeout==4.0.2 \ --hash=sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15 \ --hash=sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c @@ -203,6 +208,14 @@ babel==2.11.0 \ --hash=sha256:1ad3eca1c885218f6dce2ab67291178944f810a10a9b5f3cb8382a5a232b64fe \ --hash=sha256:5ef4b3226b0180dedded4229651c8b0e1a3a6a2837d45a073272f313e4cf97f6 # via sphinx +boto==2.49.0 \ + --hash=sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8 \ + --hash=sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a + # via gcs-oauth2-boto-plugin +cachetools==5.3.0 \ + --hash=sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14 \ + --hash=sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4 + # via google-auth certifi==2022.12.7 \ --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 @@ -380,6 +393,9 @@ coloredlogs==15.0.1 \ # via # -r requirements.in # aio-run-runner +crcmod==1.7 \ + --hash=sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e + # via gsutil cryptography==39.0.1 \ --hash=sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4 \ --hash=sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f \ @@ -404,7 +420,9 @@ cryptography==39.0.1 \ --hash=sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106 \ --hash=sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a \ --hash=sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8 - # via pyjwt + # via + # pyjwt + # pyopenssl dependatool==0.2.2 \ --hash=sha256:8e66850c79e37325735efa67ce06e2d5a939c0dab758f37b9bd3d09d0fb1f9a4 \ --hash=sha256:dff28853a7252d6a5d670c2519165506902c0d4746cbbdac99d2ad63ed96d82d @@ -484,6 +502,12 @@ envoy-gpg-sign==0.2.0 \ --hash=sha256:53ef217a05555d725d467ceb70fbf7bc623eeb973a41996e8bbe1f295d8c9aab \ --hash=sha256:8bca326766a2b82864ec6274c51d99c9924f2ec773316c2f13034925ddf50772 # via -r requirements.in +fasteners==0.18 \ + --hash=sha256:1d4caf5f8db57b0e4107d94fd5a1d02510a450dced6ca77d1839064c1bacf20c \ + --hash=sha256:cb7c13ef91e0c7e4fe4af38ecaf6b904ec3f5ce0dda06d34924b6b74b869d953 + # via + # google-apitools + # gsutil flake8==6.0.0 \ --hash=sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7 \ --hash=sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181 @@ -611,6 +635,9 @@ frozenlist==1.3.3 \ # via # aiohttp # aiosignal +gcs-oauth2-boto-plugin==3.0 \ + --hash=sha256:f4120b08b7f8d32904674c98f07d4caf4083a58343c0c0fa0016e0f0254dfe31 + # via gsutil gidgethub==5.2.1 \ --hash=sha256:8ac9cf5314aac0f3a33f46127d4a538d0ed1a7030027eb2fc4aa3b55c66abcef \ --hash=sha256:a533f85a57955261433c250701476304cc374037c0ab66b0e5372b4846749d5c @@ -628,6 +655,127 @@ gitpython==3.1.31 \ --hash=sha256:8ce3bcf69adfdf7c7d503e78fd3b1c492af782d58893b650adb2ac8912ddd573 \ --hash=sha256:f04893614f6aa713a60cbbe1e6a97403ef633103cdd0ef5eb6efe0deb98dbe8d # via -r requirements.in +google-api-core==2.11.0 \ + --hash=sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22 \ + --hash=sha256:ce222e27b0de0d7bc63eb043b956996d6dccab14cc3b690aaea91c9cc99dc16e + # via + # google-cloud-core + # google-cloud-storage +google-apitools==0.5.32 \ + --hash=sha256:b78f74116558e0476e19501b5b4b2ac7c93261a69c5449c861ea95cbc853c688 \ + --hash=sha256:c3763e52289f61e21c41d5531e20fbda9cc8484a088b8686fd460770db8bad13 + # via gsutil +google-auth[aiohttp]==2.17.3 \ + --hash=sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc \ + --hash=sha256:f586b274d3eb7bd932ea424b1c702a30e0393a2e2bc4ca3eae8263ffd8be229f + # via + # google-api-core + # google-cloud-core + # google-cloud-storage + # gsutil +google-cloud-core==2.3.2 \ + --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \ + --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a + # via google-cloud-storage +google-cloud-storage==2.9.0 \ + --hash=sha256:83a90447f23d5edd045e0037982c270302e3aeb45fc1288d2c2ca713d27bad94 \ + --hash=sha256:9b6ae7b509fc294bdacb84d0f3ea8e20e2c54a8b4bbe39c5707635fec214eff3 + # via -r requirements.in +google-crc32c==1.5.0 \ + --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ + --hash=sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876 \ + --hash=sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c \ + --hash=sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289 \ + --hash=sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298 \ + --hash=sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02 \ + --hash=sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f \ + --hash=sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2 \ + --hash=sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a \ + --hash=sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb \ + --hash=sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210 \ + --hash=sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5 \ + --hash=sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee \ + --hash=sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c \ + --hash=sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a \ + --hash=sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314 \ + --hash=sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd \ + --hash=sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65 \ + --hash=sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37 \ + --hash=sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4 \ + --hash=sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13 \ + --hash=sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894 \ + --hash=sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31 \ + --hash=sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e \ + --hash=sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709 \ + --hash=sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740 \ + --hash=sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc \ + --hash=sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d \ + --hash=sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c \ + --hash=sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c \ + --hash=sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d \ + --hash=sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906 \ + --hash=sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61 \ + --hash=sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57 \ + --hash=sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c \ + --hash=sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a \ + --hash=sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438 \ + --hash=sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946 \ + --hash=sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7 \ + --hash=sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96 \ + --hash=sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091 \ + --hash=sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae \ + --hash=sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d \ + --hash=sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88 \ + --hash=sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2 \ + --hash=sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd \ + --hash=sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541 \ + --hash=sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728 \ + --hash=sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178 \ + --hash=sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968 \ + --hash=sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346 \ + --hash=sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8 \ + --hash=sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93 \ + --hash=sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7 \ + --hash=sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273 \ + --hash=sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462 \ + --hash=sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94 \ + --hash=sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd \ + --hash=sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e \ + --hash=sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57 \ + --hash=sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b \ + --hash=sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9 \ + --hash=sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a \ + --hash=sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100 \ + --hash=sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325 \ + --hash=sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183 \ + --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ + --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 + # via google-resumable-media +google-reauth==0.1.1 \ + --hash=sha256:cb39074488d74c8853074dde47368bbf8f739d4a4338b89aab696c895b6d8368 \ + --hash=sha256:f9f6852a55c2c5453d581cd01f3d1278e86147c03d008409800390a834235892 + # via + # gcs-oauth2-boto-plugin + # gsutil +google-resumable-media==2.5.0 \ + --hash=sha256:218931e8e2b2a73a58eb354a288e03a0fd5fb1c4583261ac6e4c078666468c93 \ + --hash=sha256:da1bd943e2e114a56d85d6848497ebf9be6a14d3db23e9fc57581e7c3e8170ec + # via google-cloud-storage +googleapis-common-protos==1.59.0 \ + --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \ + --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f + # via google-api-core +gsutil==5.23 \ + --hash=sha256:76cc1d45dcf995f737a16eba69ee57b36fa3d973092f689bdef021d4584c842a + # via -r requirements.in +httplib2==0.20.4 \ + --hash=sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585 \ + --hash=sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543 + # via + # gcs-oauth2-boto-plugin + # google-apitools + # gsutil + # oauth2client humanfriendly==10.0 \ --hash=sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477 \ --hash=sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc @@ -706,6 +854,10 @@ mccabe==0.7.0 \ --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \ --hash=sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e # via flake8 +monotonic==1.6 \ + --hash=sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7 \ + --hash=sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c + # via gsutil multidict==6.0.4 \ --hash=sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9 \ --hash=sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8 \ @@ -785,6 +937,12 @@ multidict==6.0.4 \ # -r requirements.in # aiohttp # yarl +oauth2client==4.1.3 \ + --hash=sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac \ + --hash=sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6 + # via + # gcs-oauth2-boto-plugin + # google-apitools orjson==3.8.12 \ --hash=sha256:062e67108c218fdb9475edd5272b1629c05b56c66416fa915de5656adde30e73 \ --hash=sha256:06e528f9a84fbb4000fd0eee573b5db543ee70ae586fdbc53e740b0ac981701c \ @@ -878,6 +1036,21 @@ protobuf==4.21.12 \ # via # envoy-base-utils # envoy-docs-sphinx-runner + # google-api-core + # googleapis-common-protos +pyasn1==0.5.0 \ + --hash=sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57 \ + --hash=sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde + # via + # oauth2client + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 \ + --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ + --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d + # via + # google-auth + # oauth2client pycodestyle==2.10.0 \ --hash=sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053 \ --hash=sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610 @@ -918,6 +1091,16 @@ pynacl==1.5.0 \ --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \ --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543 # via pygithub +pyopenssl==23.1.1 \ + --hash=sha256:841498b9bec61623b1b6c47ebbc02367c07d60e0e195f19790817f10cc8db0b7 \ + --hash=sha256:9e0c526404a210df9d2b18cd33364beadb0dc858a739b885677bc65e105d4a4c + # via + # gcs-oauth2-boto-plugin + # gsutil +pyparsing==3.0.9 \ + --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ + --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc + # via httplib2 pyreadline==2.1 \ --hash=sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1 # via -r requirements.in @@ -932,6 +1115,9 @@ pytz==2022.7.1 \ # aio-core # babel # envoy-base-utils +pyu2f==0.1.5 \ + --hash=sha256:a3caa3a11842fc7d5746376f37195e6af5f17c0a15737538bb1cebf656fb306b + # via google-reauth pyyaml==6.0 \ --hash=sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf \ --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ @@ -982,12 +1168,33 @@ requests==2.28.2 \ --hash=sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa \ --hash=sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf # via + # google-api-core + # google-auth + # google-cloud-storage # pygithub # sphinx +retry-decorator==1.1.1 \ + --hash=sha256:e1e8ad02e518fe11073f2ea7d80b6b8be19daa27a60a1838aff7c731ddcf2ebe + # via + # gcs-oauth2-boto-plugin + # gsutil +rsa==4.7.2 \ + --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ + --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 + # via + # gcs-oauth2-boto-plugin + # google-auth + # oauth2client six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via + # gcs-oauth2-boto-plugin + # google-apitools + # google-auth + # gsutil + # oauth2client + # pyu2f # sphinxcontrib-httpdomain # thrift slackclient==2.9.4 \ diff --git a/tools/gsutil/BUILD b/tools/gsutil/BUILD new file mode 100644 index 000000000000..6f265485df9c --- /dev/null +++ b/tools/gsutil/BUILD @@ -0,0 +1,24 @@ +load("//bazel:envoy_build_system.bzl", "envoy_package") +load("@base_pip3//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_binary", "py_library") + +licenses(["notice"]) # Apache 2 + +envoy_package() + +# Please see ./tools/gsutil/vendor_util.sh for a Docker-based +# method of updating this vendored lib. +# TODO(phlax): figure a way to keep this updated (eg requirements.txt) +py_library( + name = "crcmod", + data = ["crcmod/_crcfunext.cpython-310-x86_64-linux-gnu.so"], +) + +py_binary( + name = "gsutil", + srcs = ["gsutil.py"], + deps = [ + ":crcmod", + requirement("gsutil"), + ], +) diff --git a/tools/gsutil/crcmod/_crcfunext.cpython-310-x86_64-linux-gnu.so b/tools/gsutil/crcmod/_crcfunext.cpython-310-x86_64-linux-gnu.so new file mode 100755 index 0000000000000000000000000000000000000000..9ed6a477524d2cc3f3734a038f892fbbfbac3340 GIT binary patch literal 53824 zcmeHw3wTu3wf{LYnPd(LlgSIhJ4(=iCM1D`paet0BPfp$5=EMavdDOw-Vp)0l5iZc~n6>Qq79^9n#xXVe&(_+M;H zW!ki4(Gci?QG(K`;gvkK;UZAwQ&X2}U#Gg9>T!GJR2f%F+UpdwPVM#5QFOFc%8xD| zoNiT}xIuSD$>jhU)pO4mdiRRK=^mvzow}u%Wh7P)5QAaTBE}kqPE+eWr|PAq#j2i8 zwYaO_h`0Ye?KP=--U?xynqH>b)2VK673z^*{@P2MxmmS0k{#Tt-#S(5WqlG4@|h(o zS`CoxAz75UxjbLxsr=}juP*4TdaxiedgYwPyQHqnPR$xwwh1$PX?>f&Cp% z8NQ`0%YE&^Xs|OBiwC1E%Nx2Q;b4outve`X!e`Td#%FdheD%?!+PZD((zlz z&2l|xbbJ;vec<)%DH<~5YdJ8CecM5)!8qh^4__d0@Bn5vUPx#iQ;G>GZF$G_j zEA-E;l$7QM%LLRK^s2PMt|6b_uBABZTK7;zOYF$ z)TVu071{85ZY0VQ8=lu)Qr#%G;blIQf*Knh12J*c+3;#<$7PK+d{%FMIMI8?SoX-MHQL^D_*iam${#W8j6x ztDbg$j~g1OdL8H^1JizuBzuZM^j%bPf$S;C_C83-zmt-GL+2ZJ)P7tCz3*@xp=9pS zcmYgNhGH^>HEB@g9W8@GJ~fgZ+^6ppj0y>WZmv=ZjJuy{kop2i)^irtOdUSEc^ zaM_N9#omK2p@}VfvakLPQC)diaiIZ|W^Wmw?Dl!ZjoX*FKiu>cg`Pzd{6j6=U2`n*{W_QHEgdr_z2YkpKFm;?D^eO3A&Kb z)hjwxepjM=fs}Vn6h0So{X>_s&cWF=UCQ^V@-wCUGble;2Dqc6EPKk;5HB%am3?xk z!e3{fYVaZZyiL7K+9w017-FAXvdkeq{K}O1mC|CyfTr8>Gye|2QxJ zG13lZfirQkkDVPPftck6*kkEj5~!+8>2V{f5{qV}7&rS8`mPYVQHoBL-&TCc8EP_ z936ZFZBmWLi6q4y&!UX&@dXr$J;H;>OsUNtKZZSIxL~yGeXCFg%dE6p#3b)ZJad32 zo1`B=s!47nA=o5(p4sktcn%3x^#f7^^!fTxvCHE~VV5Tpdc95PqKZzH-u%-engdTl=Ab)E*AnGF`liV3V-c*X$L3L_FL4^qE?uyk|7@B$Erpjc2u3H^miK7#Q4cY3OhZXu*17mO!tE^)a4poetV+)c`5%xfs`Nl zkm{?s9yMGKOZl0q{(dPxhwQW)u<)_Pv&V?hUu&PI!QpSSPgKNf|PZTNalbml>3Ed1ur^=J_&2%YW znJ@Nf=K8kzCP&I2ROOXYzJ}~G2JjI3Od`S&?3124-xPzxVEbT^pS|T6<7fZ-Qe!y# zyos_A?4#CfRXl*QA6;0SPHVR0l18uFI%MMMn3cA&14Ar5v2Gg=-r0{{R-AsY4p6Fz zhFP~gU&;1)9Vdn~F8=tQ*k?ad*eAJedsyhMRdlL6xo+Dg<=@B^K96vH+q&%%DIcZk z-zepuA^WTYJj6bqB}RX(eXa(Fzs)|Cs>X2kS&Xs~?4#CfHFo>_IBB26Jk+uGEw>CL znCKj3qIdg#MXta8h&IukfKp9#T_Vh3UPIiOUcvUMS9(2cj0lfC5_>w_Crjp5b@_oIyMGiZIl(+-cH=i^u(#EZSzyB8K`G@36a?NjNXZBO>@ zJ=wd{(zjCS+#mG4Mf!aGFpRSMvf_+`Zg6kJBt~nttZB598+O05b0+KaK~e+8%|6s8 zKDQ$sG;aEZu36Ek^5nQ#CFSdJ+)I2ma(&ylnJeY{Rrz^R&f{i2;Gye-L+4;iXxpDY z^#KkbuyK9}j1HZJD(Df-Q&{l0F~HPG2aw0F;~F@wf#VuDu7TqkIIe-?8aS?j;~F@w zf#VuDu7Tqk`2SP`E6<8eHA3Nac(Jv;sG+%`D9*3Ku3QTA%0~KEkAJooKnbj2L`{^F zifWZewsJOSD`!h~RwZX=RZ6y6N~=*SS8F&Fh_nYwBmClQtf;3i7B6ZG76rqEZU@*A zi3W=T{_a3uw?B^8U5orAr#6SwD~k2?_C}&y9PbJiwehRBMGKmi7Da=X^o63qc2z;% zqfImu1HhPUwg9{1|AierI4nzbi?&9|5-sI2`C7zm99*xCV}E;J5~kYv8yBj%(n!299gs z|5O9|^K$z0aagMIMW4Z=EAbf~wTaN5-8ovs#RguY<4b=I4=*|LMXzSlg_rc?QlH18 zy$fBs{DV&hB9uR@_B}4j;5vZU=IGL&MJl9s$8n94#JEz)r$0M|m(KW7Wca|5%a=dfL zd963q{Qr5PKhK^jNV&3mol2WjdXY+dRk}%~H>h-%N*_|`9+jr@*ZECN8X7*;`W(JA ziPfLm)EL-(5i_&rhb7f11{pG&G!6 zRMOhk7moK8%`KZ7rb#Qc*(Gq!Ks`4mi_aCw)S4edyx`N z+~%LM5z=Ph&Bfn>V7Q8oGDJA%i(}`a-09Akh;iyn%Qyi^S{+a>x6^e07lK3@#WSZP z<6$s#n(03Q%IR>;o{Wf<<_d-5hT}AZmb4jTsFu_1dK-b!@c^P}TImGh?f74GA??<2 zn+ZPT1$^5ChEGBJX?q#YaDSI7o9Qo6g9N+ZMErW$RpD+%ZR;>dU9&Pco>=rcw5!_s zHc?qmgUmIDGhXXGaCg;krqH?-Z#B8*a;C`gAey>P<4lS57W56Et?YVdxY>oa3-M9Hnz5$ezR zfb}3Ex$8n}8R|S_?Z**~%QvbNna8a4)a{E{wmnuEP_8!jePHmk;dl$|(lf^W5Xeyr z;f1bKJeh5HFD=8}0fG!S6=adYNUqUjZ^B-POlB3t%5d)>@DJdU;l33h)BdNX%>(vX z)JP+YqZC%oSf2MaAUpx8vCZ=m@EPm_o|EB!8P_t#(+dM+Y!~>O=%agl7Wqcjw?Uo> zJ!NAUAn}|CqLzB+5w4Knp)17uT+|9cYB_5ZPS!Izh zF1R#n-!wd{u(HU$+e5E_c~+sB>|N6QJy37ffz z_HV%d4@j#N|Gax#mmq=e&_(aw95QF2btQb+yU$DV z7g-^oy!$3oro=iQ|Gm$8mtl=jZe_qvyx(WC8jGa!K9@sZg-~ZbNUWdF`4TdX)} z-X9eF5G^!WPeV=KA9?npWR10r7`!m%MU-4*T}%?ZSokv}UDiYld++|SuOQQFEr!c` zU-G<)IvcFJU@z~_awwW@vVKgNmzmpEs{$7D{yg`3klkR7gKoUP$bA->TdjU#{Yw6? zk?gV_z9C=4kG%#Nbckb|Pb#xrfn$_s`jPp;fn;dx!}Cl3P#8Fmuz$Dxc)8K(5ft?IigI z#71Tw>C{(-?3W?gi_E+?sb6l%cA0q_NS&5sd(FIeFe<&H zOm>$IX5OP@$I)hP9lE;7%=V zmV0YhH$i?!cODhEF{a5lj{ih0M~{;nJHOBov&gI2!yR$Y0>FlK5WRKuF$=HZm{ugCua(m&l}=pT^8A&p>1J5=cS_Ly%0&TxMR|CyG&k!huZ z!%3u%kNG82y{4(imy%S9bRxXT@ty2+C?mt2h5rQ+G&uo)A~=J({=}G*nV^&iXeiE0 z61+kpJz3Dm1S^PuByH0KV~E^vTn+U)zUv7yRh+1(ogIUyRExhLP3`sE#-w*^(#H~{ zNzFLE51({=FZ*dG{edQZbr7lHc#SOdboN_Jby!o;I1_a&gCvII2$|uTY%kCm?lJh! z5|Wb&6>}BR3JeU#bKcXLbOHX4rHC{}3eSbtIiB~f;lc~4kb;ux+hg!h#}5knxbRZ^ zXJt{$kVYsb8;)s2^}_=agIMA2s?gU zKu+cO2C@4|!CTCNhIqk`K}pd>RoqKdFBD{=sSI}x{V||1KbxbCeSNQ<39p3}?sx2z&?0@lrvQo7te7p&&7a22-*v!%>T2@Az54 zolJV4CVezP%1-5RkN|%gPb1J%k1&E+kl||)V;z5C*y|zq0>>B{L*B1ohIOo!Ora}{ zy5zXj;Ta2o$Grjj8{?ltAiQGS6Ywk3nD9j!99KA;AD~_LPDtP6t?=#z@kBf}HN2IX zZ1IVBvTAr|Nz|O!9Sj@Z*)qvSqPEr*&Zx0X_PKAkf)){~IUJC8GU)*On2wJbp+tVRlP;H@%{&}hV>!MgJI6Ntwl7`L^$KMeoC`lFJ}s^ zN2p7eaHhz5lctI&PrfDA&(W?omi<|9E4O|K!+PVKsj*(7p7e31&iXRk+`CR@7ps`& z&h=cA4Z zr;k~8!nWRPxMYvDkb3$#&OB#bpN-5`&b(lKmFjHc%*)o7$UUy*4Aug~;PafpT7Y!D zgENP%Eb8`koH?3XLG$JHoWWXP6`5ft_W^4GdOGj@0+(Se&_TVpiF1YcKB{eqINqC` z=R;HZP2O9qpOLknqjqi`O;d?s`Dh^A#uePw1j^i=OLK_Vx|5vZOPndR7Lr2lU?mn= z)zsvj1vHbCSpP#Nxr0x~z?)ru&s#M@WW$S8}CM=7Bt#lrTltkc0k*^Vg{CIFIUlrSMeL&u~}be*uMz zvj8N6))>m7n1W?~k@stC9P0$C`E}N!VQr_(H$W&rwA+x5&FqZ zA91ciF5m0@ruAI_*7Y>{zLo#qXxOlRMRwa=m4MjOq-Lx#U7ND7&;#^vE8`d8wGcD&-gbla#doav7DJMdBnjTt0p(FL$}-m(m8P8xm!5H~#5aC`<1 zk+mVb`Pl-0h_>>*0#86Z$j=dYGc25+D=@axMt+{aV<2aKzQ9i+*5wxn{2q*x zKStml#La>+GKsGMzF;iFy!rnM%CZd)ZKTKK=k7wzGY@(hJ3+`Z5EsWz6!-v|DvSuc z4fw)dVbcoz(J~L){NEz+ZpQE!SD1DJ;oO&kS<(1b_o?V3FWtwVtv09s3oVU5C-Wi{ z6dIn((C-OZ!jPPx5oor>fZ_RX4DbmnC(yx)<5yIxH1|Qso7DnDucm|G)4*fmC=u{p z=x$;LkyaQx^(63p}T;6!?7Dduo%wwTLZKR|z~F@=skYFzqd-wg?;`KCKD{eX(k{ zmte2p-++FLa|B*Rc%_qd&{eV0$(!)qX-W^DMSrJd3jJ=#cG99k3e}$V(C10Z1pfkL zIBB`S{|+0Sv_jxp&{mFzVMjaYbMjvV@L< zYMqaMmMqKv4(b@zIB+Ogo=sCiiM0(qDp|pqa_dP9>yni$O^x**lxgxdp-!FkH5j*K z)kI>}Xk7_Qlr(#3bJb*-B=qV`+IX(9ZbK|6Y007u)kP5g`>5;r8EiGZN_3o#rl+?H zd>{O4dQjlsq23JTn(gHk52NBUDtKkuwdwHLUl*s;Z7NZfgeydDVh(r=Y}Z)k40ih52WIL7`QK zidBo$uH!CMfXYbJ?vE@v*Xj>4&{&f`p%h402Ts(fR{ zp`Bjqd5rn0z*ySX++e*&;slxOR_jS>r;}ydW!+Di5SKh)-AA3fm=Dt)vVMwbv#OiP z9<%DermBa@_E?{zcCN{y{rYoO2r;edb37DYu*MSEHm>us^)&T#JJ&g2If?b>Idjk& zK#Z%pjx&d?Tw-v&ht>^8r-s^%sv8-`lKHpP?VGZmK~*=F#jnF$v&ZDOAzSPV#SGd2 z8damSV?boM`|+RmWVZki7bJ^~iGYZ+hWmHm=dGIT{Vpi19i)=!EUIDAK|s}v0xF<0 zC}QFGDF#s0DNcgj5ooEXs??i@{#lhI)@+s@8M4Og@f1Pa)}Kf*vrnRo*UH0~sH&bw zRcM{}BJpe_o;6N_Gu-5n8SIG}Zt7!(n=FT;df+_}$f%ww-n8m8-;>gg)h}+39_-&+QZu z|BS&lM+U2BDF#u^DT03;L!jnVfiHo6YDxuOhR8E_%mjMK;MtAtou+1?)!=!$gb>eu zsBGR5iC}xcr?y^Zr+x&9+6I9`P)DtrwYDHg)GiTxANpI{DDYVDtX(SbTTo8znF6;! z#kI=>{t@Aq3*1TcD+KO?fVC?HJ`o;Y+a&N2czW$BfggpOwao%Afo^MC1YSyVo-J^U z@aG8pI`mMxM&Pegzs?o-V(6syJb}v*rE5PU@SAYl+Vcgz7k*oNp}=RtcC{A?9DvSi z{Q{?v%xwbyhTwp}FJi3KwhMeG^(83qV%W5{L*N}GXQ#j`NrznmAErKr1pY75`NaZ% zKyt1XI7IsI7I-i5?-BT6k}WK7Gxg;XflG-#D)26XV*-yQI4< zU~<^nEecO^ZWovyBiDXj;4x&Q9RlBhaZ!7nz-Lk2>jnNF!rvh9e1w47odW-e+PzWW zi;2${1ilGlr1mC(&!f6G3;aFunJ)@F0AHG~*H(~cz8Xg!*kZn{5WV{WpJAFjs7|1; z>Uj`S)#c{B4FyDW(C!p2TWo8iSAvAnp0KH zx#(Vf@%S$jmGwK=xqcc)d#{nUmg~1t7d|4YQ;HvD3^DP{2c{uILe0Oy+zo0?VL?R= z5~i|fsJ;tgwNXiXjVfLa{Venr5k8A}=N2Ui9DqGP#GqR^=>)=ij=*^qPG;C~3o4q^ z#;!me$K`M&^Q6M_2;PcdVxByvj^N3V#4MS(8h)HbtKmqIVm`{4wus7V9r-@sET?|) zWZK8XYMEmKFJGOzn!pCvNju>t<~a_NBK(0O#u;G9mwCtAMOb!U1Ev2NK*mQB1U~iX0=27|i$xx-!Q~8~UXO{tyF*%{!J)#WHdM+5|}+ ziGjhu*D3E~xBxkokPSf9;y#tmgUxXco^YKz4_|&Nofezp9oclw>fCw56{q5pLHPWP zBWraLA&y>NdXzLW(UCSr-SMYObT;w&um2IJ^=UNE^kHVhX9_( zZB%SjoFc%UkBZQ$^F6Z#`m~(-Cq<^4s~qc5(oB7sZmuD@rkm%IG)5)u+v&O#3@V;g znXSlN!`xaN>riu*S#DHJ#vDS|%_zJ}nnIybS)%aw5{C;_=IhA(KsZ>=!{t)bsOYfq zcs2u6RKnR_av&_Z!0d+#jZs0v$o{n+g2JBZIDnowtDs!{>)%R2(`PGNg z6#LbGDLngCx(X@eS0}>x?M@~C3saT*u55|4^9TeWj_wWMFIuoZ96#Yw*k+lp_| z#Yw-~sEXOIW)x63*e$-KUqa&#kU^d19Y;>qzO>uq81)v^{3dSkrQIgSsOf1qSivp6 z^qBUgqbf#yJJpvSH?x)$QJ{HW#i;LyFI@^AeYnM!o)9nj5fHR_6<>PN98EGMW7JbA zzVx!{osj^vFa5}LefK2gOVeC92E)z1bS;3bxY?H;0q_uRqv8q0iG0aKn|5Cst41ke zj6^2{srQEjsoyIO>`bG^h#=%lr!YI4Gkr!Ck|Q)oTkK5j3jaOeE54{Qsxg?K@r)<^k*VU5F^s24+8C{%orcZ3eztP1>XQGo1x>S%Nv(ES!6TG=aTuQ(9 zn4I<&&Co8jMZ(jo(DpBIi%V^h@H8nM2d}urr9PMHQd?79YFnyHT`Mj{Th9JjdC=Tlv3M~X{zsXiJBfV+%Ne#vy@PFF59E(51_xY?yH2XGl~cB%UT&_he3 zVwvJZF4cmHf4fUvt~jtu-L5j^QkOG3n@jy(6_PuAQQBgc`bgo~rOL1=p^IERweW@hKaMGh4)&^h>tM zx$eeOwKKIyeA*B4ALABhYLWPqgRM&@ZgHlwQ=RFYRA*X~>P+W~GX=oo0^H(E=ZTwq z6NseQ2)~xL*qQE7_@4k@K~Ko(B4^5mVNxc$ zS8Y5_=3<%bKCx9>tg9uRX)-oZnrl!OC!J}&t@vtPoOGr$Rk1kJW{ONx#hLUQ-AQNq zRhf3CsS=s!eVBc?#hIo`WO9KAz2zd#G%eMcPD*vAlT)3kM4agy@L7dhoN2na3H?mp zy|~4hW~4gPDJjnMfa;x*07O@YIoEW3wNyEinQ0glli8U%kZH%w&U77s&*L^KKCd{D zGyNVF|8{43N^xLk`i07nGZjLua0U2_T9KFG#Dgw!rgAQ1N2sh-cy=aw;6fKU)A_=| z9+@t;@rWmRRNQ5&_61!n=}b@AiXYL%NoRW1R{WwaPCC2YXzH}2W*kD*$;Z2B1a_i$J7*6stfvpafu74PgmOf0~wcw_fb z*6BsG@cHoaYTnl!<>mKTS@_BT+||6T`0YqzheWEB-(iCtc`rTX8-V&p|NhLf5KdwO%5V&T7EeH21M1>5r)-9qHm( z+L8A0X720I^@F&@k@oRk?j7WRi(4G&=@dtLCdHASO>v~}vm?=~9dmJuBR$7Xas%?$ z;uc4GKE;uKkZ`0LIvhBtdX@xO`RLlqt~r&;j}C#`Yq;5u=pBG%xY>`^0f^%^Dt@FG zkstM=;@|E^pC}ION8@oIM;H0geaz10NBydheBe9M7W>gv3eSGDTV=?Pek&a8e)N=$ z$3T)t#jCbzlVL^ngrpyRVk@rI#YsP!jPouJTBniatr&!JPs|l{*c`=jX7WeIB(XK}Ra@^v+T}+pZgQ3ChyHzRFNC2dLT;_V$hqIOY zz5;IlgPYyA9;bVCxY>Q_TZ_AJ8x`G(5xH+o)nNBMp`2O56_VS=kA{T7yA%g@-^W#k z+_#O{+1%GtB>c#IH%eRVzJ&_U?z=)|$dm3D4tDoF+s5OmB#(-9wra2FYDxF~qOJJP zx;W{+yKTi2&~Qo^+^34!eQW&&j&}Du%m`8^9kR~91^Yo@PM|k<=YIn4My6^miqCkL zGCy_j(^;c#I*uafKI8Smy6=p^7{+}@NzQ`LgNp7&8D`FcUjm?ekq4RAuvDje(PA^V zZW5+6x+nMx>oy~3&|Nz^xBh7~Pj}r-;79j_yu!NcKuGt5+(N_5F{gR*GM%0Yo*Xkb zgLb)&F^1D~TIT7Qb2I6&m9yB$%$yGjoa>%s5XO}a3_ic43LrxUVW`NRXF5HZnNr&X z!WwO3R)bnNJtkfH5xa<5bZ|66Zz7-*{E5vZLR^&CC zkJV?AKx;D3b)Ca1p6N0w@)V=;*-5xU#H(OmsYJ%e0Rt>OlyLCbA;Qg8!m&Rqo>fXX z!9h5DpO|*brMlrcgK!|KNtHHJq@6psb@GwZ?2Y0%w!J;fO?J*0se@Hp#44C?*C~R5 zs-j%1uFN3k4|}xKcmy{H1v)$lOQJAzCSz)H8csNhY5oZ_t%(b0+-7foyekr(H8X}M zFEjgM(V3xepu4XoRbHAB(rQ2Ri^G z$TDw$*!RTZ(LVgn^5Uh-76Ad(Bq7uYMS#TlTC6|j3r3^e_^D^9hD=zM^@V|L7dpJs z8|jVv0{Esmp(7sQcxfO0g$2{wPv0*G7&7(v<6TlQj_-{VC?PW@jGiE@(w@MeMtd+8 zh}sdo{ayYTF+pK(e|xa4uM-*6ZR?K*{n4nu-%dRgsVHN1D)flHGw)a^RO`lXN~2rR zU~g2KCxpFDTO`tL&v$f3{Bb)X5b42BUT?6YqW*B_poV*bJ(1|3KE&yFvh8)qD)xMP zC@?4=8H9l3!Ej%Xy#n67wr5kcjtV{eOVxH7cv--n4|MxugLK#t8Z6|XJXrSj2mHM; z{QkA*j`S^d>+SD|`q>#^(RkD!2)5xDw}mR!9SRKUdLY<6NW~%i61iQh-hTXEcU-lR z%%J_wVAxKGY%m(amm`M!gSxRk=wBPfHzy7nNHk(%!AnG@FoccuLQFeNZz!;~8@`yV zh$g9zftb|l?Z*g;L{%;j3CHo9;8Fx-t1n^Hq!gN>eKda9$&*d2tf--fa74^S(}8Ti2&%b}lTJ8wxd(@obXr?uH-I@ia~bh*;s%i}BGH3}te zcLz{gU2edx^ta432k5?Rf-Zk-e!7`vI`?0WnKI3h?#|_84gRO4Z^r+0_gC@X9d}Kq z1l1|YbB(4XJr9R)NY4>$`niWX{B|_qj%& ziK25|nb|DWg;%?>0GM|(vy60S%>5hi$Z(S^86=AHF8b$QHa~F`Av$$%wva{2ZuQ;)nRrf9fn-> zA5zQL{vNeFYFEpif3aHFU1qYo%yUhETHr39rj{fBxLQ7S91Y_x2km;f{%_PvHu+F` zfdrT(I>4y7qry=7d9Hj##^kMdRD)V@QpUnlDAke3O5gQ&I91e9{0|K_55mw_{~oWp z&1oBKuU_`A9c+KKSG|!KZ2wn5=y|(dzWy)O3uevav^AQiEgow-Q$x@{ch34dBJ&^Z zO8E=E6k~+B{GSk+p_IYig1P+Drz|(M{11iZzg+%z=;fDbwn22qzgjQsEHi11ldo1c zLnf<#rlVv|B_(X*Mh|(Kp!u|X~}Q+Y&${2*B9=K z1>5n(Nz)&Z@4dp5>oAJJ>$k z-&XE#tMu0d{MFU|n%U)*6_wNv+J;Gk5DdE?5w5Af-X9CDXf#&_QL1>pp=Z!=<{Uh*Pkz196-vf&(mG>q?uC+Tcb z!IrVmM(qKMRxTFRgu{1boi*R>{k9ZO}m^=83s9!Z)Gn|Bp}@4ZhxoY>y8Ad ziN&;m*Gdxa_BBLdydXKmGHm~wY5Tqu@+rp?bl={NoK;4K_u#H53LKe2Pv;@tE??uX^4cwK~_$4u!9Dq?ilqmS-7AL z=GTS%;{?m_T@=D%R%r0X$wXd20K8pl=t$!rHVg0dM^&~*eMi;CUg)C7UI+?h~bS9p?N{W ztRo+M2`47LK-XGdhd@9I;tidGETB^m75l}_MZUBB zQ3-6Mrse*yzcbig&p!|$90ER!TXoPws`Ilg*hy#vL2NxhaY*Qo!rOz?6&O21-F(`E7=ymT*r3O-(gvXr zEk=@jqJ*Ddgcp2iX*kx4@x~E{G%>4E#mhK+H)ix=&f^FX=t7?RmY63SaIz$= z@fhprW$F9yLmZu9{PYKEP#+Se6w>yK+cwOa}h7jRa^wGeOF&ZQ3Rg~t4-VqApsE+5320D4vg^4vl506q*Tg-zJ3j`4kfma;1 zIoKT}I((M7!(`$XIHckGAAci8>9tN0oE|O5%T%fIC~8CaBtfE zalZsxUY1A+oXw93T!#eeqIN=VFX%4wjL}; zNFW%F7HRBgXr2ycr9p|LiX#?a2x{Fy{3uLf>S=@R)c7KWO883{j;iCAZ+d+69UyHT zv>0q-iy7;9GNJ&$jiLZSLj-^V8Y0oiAiGc`=hIJL9LMmq+KEw@p#ddxgE&o6Y798U z6pBZfHNYNL!+a8DZ-5W86hTZXcq+n!l%GXF4~m~^-L{Tk0x)6%k_XG)uGPUn6vm3% z=N4Md@-&5+cU`dEh=j3J=ng^*ObW1`kF)|Vv{a+v74xqP>exeuYe8U-^u=Xxk?Yfp zD>jumB^mYAlqgKiEOC4oQ9Z3Hw9uM|Za&)v z0Xzsc5$5m;oX%57j8{|eH z6fG^GJD-o9C{@qq7^5OD6u5zR7{s_a7>o2p1BvMlh2dZ)=F4@YQ`oQx%PpIzYU-6} zA>$C^3jLEAUq+`B@u3%uTge_W6z}6cv=P6Agc5g_8odyh!BrhH7XeOlys@N?F#uc-kZY(hU zoiQ5P;oy3sBe5m31ywDVTE!$|)avfYASZ^ggLKq3OYxRR6}qw_LMv$=MTylBmq|3mQzRZ6D!8gh=kpPX&o?MUpeI=I^+oB& zwR~Mz-9{sk9)tIEMTwanKCMHCz#1U?4!G#ut$oiv!u39{)F0GzjRNATkR#q03(%k`_1=g0rPH~RXrLZ=JE(+FT z|GExo)6xd~=apL*iZE@J_J!9YQkC*nBz7zw!kiv4cIhUd1k?z0*zyMEaqRFAkm2m- zE?Km~x2U;!WivfPM9h$Q3O9#FXrD!!L5?5fE3{dVc-oU#0Gt=<)!O2%esWw<4Ooh| z8ZaXQJk5j=gZ0S9g%b~+`GiwI4pJ}=lgp6fqq#_Lzv1h_0*)Lp$U|R681U}lMGqno znw0$wuQ;g$4N45f6pD4cL{Y)e>H$G8h&?Pe=`^BNujf;QmA#ZZN1kc=ICQW-Q=p?? zl1q#c-E%c$T4;->aTH#8Ck#FEZd^A_E0m;`5OFcs@tbovtT6f#Q$FqdF~z>8OXAR?8}Nnnb~s&qfl_m1r0vVo~p#;G5X1sLm+oXkOIP z+Ps3igLMwq4PX}I2tlJUx!NF4fm2~v)9>H$VGH6A*2@OsC#_PbOZ;qi9iwb9i4IXV zX!T$oOln7_Ijk55$(=MBB*0<|qZDgkbeyIgOlepyQU?AO3Ji`MTK9{Hv_fKmAS>Q!fm^dr>fuNGFq|HD!UHk2wsi;jY%)XwjIivqPmZ+i z!$4mU?ZjBe+DXo4S3{32gy&V{!cb>OLIrsn<_dmg1?Pg$bIi_6jz%eNB$w-4MaOxp ztFZkNU#AlnrDs}^&Z?A!ee$pfuGQZi>g+;j5RXPsv#*pO3v5I2MajvL5LoKFI-4c$ zvyX3(t`6ElEo7yIG{1!+V&VKrX_M5Vj*=JPR4k$nd(Z&yPN;FPqct|puGj%o+jf-< zI);P?r7p%)$YU9m065M>17J-XGrTc z`BkqXl5g-zE4dzP$6VWm{X$<9GeLJ>PZ)8O&lhZiPC8D3z~1ZRbWDqPikZo28NJm9 zzR45JI3I2L@WoQtc3=sP#Q}l^R$b@}dn{J7?Gc<^^E#O=3O;!G2XgU-l2nC>r!C@- zwlAeUP_z$=S62xNDubYMYODUbaFf&BU_2g%rv=~*J#KG$r<&<_A8svRJ5s3oS;6qR@pyRf(7PeIv3Kj zJgVwtCMI8wX*jM*PJHrQDAA1VK~D=UZ+Mg5heJuN{=}A15HMp0?_5Rp^YGT+LOdj) zy^-*Qa@e==z8e(44umr=4g*@<$;B5`h4}ze5f35~RyRvNf@Ndk@e1O!=1(h66~$5x z{t|;qKWrgA>J6k?Zi(FVjDw>yS((Qx8G|;?c%~^2gw*ad=>{Q?(!9X4j<9J8Mtef= zUEO)abQ*b5sDm5-473POVMD?@#yw=y$7J~FXH&jgZtQnG)zj0UpUVx`cg5-o^gzn089g;PDvt zPik4eV$qrwb=b*Eo#gbVZOZ0>SWuC;8eWD>T=*;pU#8KjLH!Y^G<@|+>)i>;VFA2g z&X;MlXmD-<)v7@|$ClwZ;3$iXW0R-G)xnjJn3+Ard)<(50i1^r;Uh_DO~W?NV4>3X^M)zAKlm zW0_o6|At}%eQNOM2~?{F?Horkc^%&FrHj4{oi5X8*5EpYavISq?$#OlvK~bzK=Qws zTr<=uMARVt{1shna-F$UAwSjRAE*wFQWeOS1xjaAR0@-8kUrN%7m;bWRN+Q6xo$(J zBbi*6>-1PAuTzaSCS+{Y;1BRw2)d4Ca$TK%NP{luQ-hZ#P^}uYa~#R!zfsi&64h5} z(4z$6QLZ!N71A_Ti3rL6Vsgz;rw~zt^zCMJvB`BNPci&dli#Xpe@&%i%X?MjZzQB@ z(%`ohipVtlq{5{e_!_Wbhq-P1m+SOcHh)32`f@_ZRt@T(lsuNrb@g8*s<&t` z6Q7o$3xPv}c8(+2Jg%x;m#E&X!Fv>nhq=ytElDTsCI5@fHA9`k<{I3RV2H@BGxsax zr`o(k5!R}dj9H6M1JSiar5NTKT%}M%rr}nF8`%?RfBepBiWq3PKd4_C#pAVkUq9d z7mstDc_&FHrOE#?&NV}wVw`L6nFK?ObDeopAwSjTU5fCtDkWoXQ97gFo}vrmT!Zv$ z0CW+VhVN9kk!`LU(dkGw*X24rmdzhhtv;3zvQ>j;;?tjW9n0pr`gar6TQo@DuSyrZ zNP~8cBiXzKUjRbadX-|FYmmMzpDrHfI&*W9PD+#i#par!PGNHm($D45#q+t&T&Iwq zYICZe*M5Mszxo&2_m>k7e^F)#{prkgXbg z3!mwy>sU6|)qRQTEgIaOK(%Vn&T%B0zptv%cX`rFJ~u4hX&fFFpJo)Q4b5OaF5`e& z@DIkP8+wU97@uKmRSTxUc(?J`uy_l<;cBN8*l8dd<6CJ`FDHHpqi9%smhr-{_|e9J zVeuYgQ%)*(nnJRTqea8v!K8dRe2(F*7!IFn+%POY&p5h#IC^}=?uy~?1%`WA{21f8 zmBZ2F`!1NufFXrrO0Wh?~5CPSKm@L1h2lgY6xC^Rn!ptloX6) zokr0RJRQBH8Sf6H7Eo++7(?w#gO~6RrP%!hg+H8vp98#UB=w_?dW^-4A8M~2#iu+M zfbHrBY#KxDek<_9$^T{GNlyKdbbUzrHHANbFBPFnf1%np0ds!~7;WgUI(|aoQ@^kF zdkUZW)uTUD_|$Ka+^_Jd-w5~%g-`v0x7QTDT$xhK|67HBp$GsjJ?wuE*u*cX=3AYxfn+W6f-qEj7=lZe_;gtSAZW* zo@anRaTqy`Kr|kU<5>~j0l`b1!A?Bx#q+y3y-|nPRq>b_?-Tht@pT=3yf%gR{PAJ8 zz72S^Oz(%|<&(0S^6HA=iSQz0IOO9mbHPh@alC`Yudevo`Gt49ElDqUN)8Xq@w&8~ zbkNHw`b9TM`2}Bovmfuj;q7n2(#Jg{<%B;I1T;RDLf9Jlk7v#LRekjxHS)HT&$qaF z!SY4EMJpEK?M)V&dK8P`z2$H_UTX0zJa@%{gtv(TUy`Xt1O#6%!5gKIBI*-PT0}YQ~iiEAqxRuAkRKi-e>OY zp|^SQaUi_q4jMdS#|Lo|AFIRTzG2_$8Twv)g8h=j`^_W0Im8+qR)2WWeuJOv#-A=p zcE7T$3cvLh>+ivHOr-Isq+J>c8G|G+aO( z;aBU*f?aBWbhU$<&Iu10DN;kjXv*Ic0yD5C&XAH`(JO0<#f&lxS9+%lIQnz|s?Y}> zftM`5aifdc+UU`*%F#zSH4LR`kUk9gPr7J@MhA}wPkib9fliAQUMK3TOJ|KTK=%52 zAEDFY1f>9}+k*u_e>gA@(e3N~hE8>TT`#r$vrwP*<+{G!m+164Q`)7ocmj2Od@UPa ztMQ+B<=5@#l-4hw*y`*3kWR~0{Z#oC z8c%*{eUp@5@0)bWol`VaC$;}qsW($pP* z)`MbeT$55?@7HwtkS0`V3VUN`N`1Yr)2XJ{Y;<0yx2M!UpbB)lOIK8>uBX%cQtIn{ zqE7qt3k@mtHQWD4sjv5wI#uTnnw;&B$N>0;s$Z#*um}CHDG> z{dk>J=}OcTK-bgh$3WZb>wU6JQk){uSF z(BlqW|C9u!0J^?TM+4BXo%AcJeeKS=ER}ZR2=#xd>i=)Gt~t1UjV&3W{-d-J$Mv>W zjAVz%T0KJj`#vibPD~m9sq)vW`l /dev/null 2>&1 +DOCKER_AVAILABLE="$?" + +if [[ ! "$DOCKER_AVAILABLE" ]]; then + echo "No docker daemon, exiting" + exit 1 +fi + +ARCH=x86_64 +HOST_UID="$(id -u)" +HOST_GID="$(id -g)" + +docker run --rm \ + -v "$PWD/tools/gsutil/crcmod:/output" \ + ubuntu:20.04 bash -c "\ + export DEBIAN_FRONTEND=noninteractive \ + && apt-get -qq update -y \ + && apt-get -qq install -y ca-certificates software-properties-common \ + && add-apt-repository ppa:deadsnakes/ppa \ + && apt-get -qq update -y \ + && apt-get -qq install -y build-essential curl python3.10 python3.10-dev \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 \ + && update-alternatives --install /usr/bin/${ARCH}-linux-gnu-gcc ${ARCH}-linux-gnu-gcc /usr/bin/${ARCH}-linux-gnu-gcc-9 1 \ + && pip install crcmod \ + && chown -R ${HOST_UID}:${HOST_GID} /usr/local/lib/python3.10/dist-packages/crcmod/ \ + && cp -a /usr/local/lib/python3.10/dist-packages/crcmod/*.so /output/" From 60ce9c7a8edef37bf06175f6fc5fcec9e0f39bbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 11:46:52 +0100 Subject: [PATCH 141/740] build(deps): bump github/codeql-action from 2.3.2 to 2.3.3 (#27194) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.3.2 to 2.3.3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/f3feb00acb00f31a6f60280e6ace9ca31d91c76a...29b1f65c5e92e24fe6b6647da1eaabe529cec70f) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 7132d6b49656..78225d327520 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -33,7 +33,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@f3feb00acb00f31a6f60280e6ace9ca31d91c76a + uses: github/codeql-action/init@29b1f65c5e92e24fe6b6647da1eaabe529cec70f # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -64,4 +64,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@f3feb00acb00f31a6f60280e6ace9ca31d91c76a + uses: github/codeql-action/analyze@29b1f65c5e92e24fe6b6647da1eaabe529cec70f diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index bfabcf7f692d..183c47767366 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -45,7 +45,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@f3feb00acb00f31a6f60280e6ace9ca31d91c76a + uses: github/codeql-action/init@29b1f65c5e92e24fe6b6647da1eaabe529cec70f # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -78,4 +78,4 @@ jobs: - name: Perform CodeQL Analysis if: env.BUILD_TARGETS != '' - uses: github/codeql-action/analyze@f3feb00acb00f31a6f60280e6ace9ca31d91c76a + uses: github/codeql-action/analyze@29b1f65c5e92e24fe6b6647da1eaabe529cec70f From 6bd192b335b3382c0e24eaae4a541df328accefb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 11:18:54 +0000 Subject: [PATCH 142/740] build(deps): bump requests from 2.29.0 to 2.30.0 in /examples/grpc-bridge/client (#27195) build(deps): bump requests in /examples/grpc-bridge/client Bumps [requests](https://github.com/psf/requests) from 2.29.0 to 2.30.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.29.0...v2.30.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/client/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index fc107979ec9d..fd668e2d6171 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -129,9 +129,9 @@ protobuf==4.22.3 \ # via # -r requirements.in # grpcio-tools -requests==2.29.0 \ - --hash=sha256:e8f3c9be120d3333921d213eef078af392fba3933ab7ed2d1cba3b56f2568c3b \ - --hash=sha256:f2e34a75f4749019bb0e3effb66683630e4ffeaf75819fb51bebef1bf5aef059 +requests==2.30.0 \ + --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ + --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 # via -r requirements.in urllib3==1.26.7 \ --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \ From ec7229e00e177df950bc4bb889a6e3f146d3d025 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 11:42:31 +0000 Subject: [PATCH 143/740] build(deps): bump requests from 2.28.2 to 2.30.0 in /mobile/docs (#27198) Bumps [requests](https://github.com/psf/requests) from 2.28.2 to 2.30.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.28.2...v2.30.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mobile/docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/docs/requirements.txt b/mobile/docs/requirements.txt index 9d0f4837f503..c471e6d216e3 100644 --- a/mobile/docs/requirements.txt +++ b/mobile/docs/requirements.txt @@ -158,9 +158,9 @@ pyparsing==3.0.9 \ pytz==2023.3 \ --hash=sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588 \ --hash=sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb -requests==2.28.2 \ - --hash=sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa \ - --hash=sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf +requests==2.30.0 \ + --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ + --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 six==1.16.0 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 From 4b83586a2196dc2d4d515815a1b33ffb954763a2 Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Mon, 8 May 2023 08:09:23 -0400 Subject: [PATCH 144/740] HTTP2 ServerConnection load shed point. (#27187) * Get HTTP2 ServerConnection load shed point. Signed-off-by: Kevin Baichoo --- changelogs/current.yaml | 5 ++ .../http/http_conn_man/stats.rst | 5 +- .../overload_manager/overload_manager.rst | 4 ++ source/common/http/conn_manager_utility.cc | 2 +- source/common/http/http2/codec_impl.cc | 14 +++- source/common/http/http2/codec_impl.h | 5 +- source/common/http/http2/codec_stats.h | 3 +- .../network/http_connection_manager/config.cc | 2 +- test/common/http/codec_impl_fuzz_test.cc | 2 +- test/common/http/http2/BUILD | 2 + test/common/http/http2/codec_impl_test.cc | 64 +++++++++++++++++++ test/common/http/http2/codec_impl_test_util.h | 15 ++++- .../http/http2/http2_connection_fuzz_test.cc | 4 +- test/integration/fake_upstream.cc | 8 ++- test/integration/overload_integration_test.cc | 46 +++++++++++++ 15 files changed, 166 insertions(+), 15 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 4a9dc3fd95f8..0414e2f19940 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -103,6 +103,11 @@ new_features: - area: load shed point change: | added load shed point ``envoy.load_shed_points.http1_server_abort_dispatch`` that rejects HTTP1 server processing of requests. +- area: load shed point + change: | + added load shed point ``envoy.load_shed_points.http2_server_go_away_on_dispatch`` that sends + ``GOAWAY`` for HTTP2 server processing of requests. When a ``GOAWAY`` frame is submitted by + this the counter ``http2.goaway_sent`` will be incremented. - area: matchers change: | Added :ref:`RuntimeFraction ` input diff --git a/docs/root/configuration/http/http_conn_man/stats.rst b/docs/root/configuration/http/http_conn_man/stats.rst index cdf7d71df7ff..4f7ee8dd5b79 100644 --- a/docs/root/configuration/http/http_conn_man/stats.rst +++ b/docs/root/configuration/http/http_conn_man/stats.rst @@ -161,14 +161,16 @@ On the upstream side all http2 statistics are rooted at ``cluster..http2.` :widths: 1, 1, 2 ``dropped_headers_with_underscores``, Counter, Total number of dropped headers with names containing underscores. This action is configured by setting the :ref:`headers_with_underscores_action config setting `. + ``goaway_sent``, Counter, Total number ``GOAWAY`` frames that have been submitted to the codec to send. ``header_overflow``, Counter, Total number of connections reset due to the headers being larger than the :ref:`configured value `. ``headers_cb_no_stream``, Counter, Total number of errors where a header callback is called without an associated stream. This tracks an unexpected occurrence due to an as yet undiagnosed bug ``inbound_empty_frames_flood``, Counter, Total number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. ``inbound_priority_frames_flood``, Counter, Total number of connections terminated for exceeding the limit on inbound frames of type PRIORITY. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. ``inbound_window_update_frames_flood``, Counter, Total number of connections terminated for exceeding the limit on inbound frames of type WINDOW_UPDATE. The limit is configured by setting the :ref:`max_inbound_window_updateframes_per_data_frame_sent config setting `. + ``keepalive_timeout``, Counter, Total number of connections closed due to :ref:`keepalive timeout ` ``metadata_empty_frames``, Counter, Total number of metadata frames that were received and contained empty maps. - ``outbound_flood``, Counter, Total number of connections terminated for exceeding the limit on outbound frames of all types. The limit is configured by setting the :ref:`max_outbound_frames config setting `. ``outbound_control_flood``, Counter, "Total number of connections terminated for exceeding the limit on outbound frames of types PING, SETTINGS and RST_STREAM. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `." + ``outbound_flood``, Counter, Total number of connections terminated for exceeding the limit on outbound frames of all types. The limit is configured by setting the :ref:`max_outbound_frames config setting `. ``requests_rejected_with_underscores_in_headers``, Counter, Total numbers of rejected requests due to header names containing underscores. This action is configured by setting the :ref:`headers_with_underscores_action config setting `. ``rx_messaging_error``, Counter, Total number of invalid received frames that violated `section 8 `_ of the HTTP/2 spec. This will result in a ``tx_reset`` ``rx_reset``, Counter, Total number of reset stream frames received by Envoy @@ -176,7 +178,6 @@ On the upstream side all http2 statistics are rooted at ``cluster..http2.` ``trailers``, Counter, Total number of trailers seen on requests coming from downstream ``tx_flush_timeout``, Counter, Total number of :ref:`stream idle timeouts ` waiting for open stream window to flush the remainder of a stream ``tx_reset``, Counter, Total number of reset stream frames transmitted by Envoy - ``keepalive_timeout``, Counter, Total number of connections closed due to :ref:`keepalive timeout ` ``streams_active``, Gauge, Active streams as observed by the codec ``pending_send_bytes``, Gauge, Currently buffered body data in bytes waiting to be written when stream/connection window is opened. ``deferred_stream_close``, Gauge, Number of HTTP/2 streams where the stream has been closed but processing of the stream close has been deferred due to network backup. This is expected to be incremented when a downstream stream is backed up and the corresponding upstream stream has received end stream but we defer processing of the upstream stream close due to downstream backup. This is decremented as we finally delete the stream when either the deferred close stream has its buffered data drained or receives a reset. diff --git a/docs/root/configuration/operations/overload_manager/overload_manager.rst b/docs/root/configuration/operations/overload_manager/overload_manager.rst index c4c088414826..ac66e556514f 100644 --- a/docs/root/configuration/operations/overload_manager/overload_manager.rst +++ b/docs/root/configuration/operations/overload_manager/overload_manager.rst @@ -161,6 +161,10 @@ The following core load shed points are supported: not yet started, Envoy will send a local reply. Envoy will then close the connection. + * - envoy.load_shed_points.http2_server_go_away_on_dispatch + - Envoy will send a ``GOAWAY`` while processing HTTP2 requests at the codec + level which will eventually drain the HTTP/2 connection. + .. _config_overload_manager_reducing_timeouts: Reducing timeouts diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index 03237b518b40..39805b8de253 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -67,7 +67,7 @@ ServerConnectionPtr ConnectionManagerUtility::autoCreateCodec( Http2::CodecStats& stats = Http2::CodecStats::atomicGet(http2_codec_stats, scope); return std::make_unique( connection, callbacks, stats, random, http2_options, max_request_headers_kb, - max_request_headers_count, headers_with_underscores_action); + max_request_headers_count, headers_with_underscores_action, overload_manager); } else { Http1::CodecStats& stats = Http1::CodecStats::atomicGet(http1_codec_stats, scope); return std::make_unique( diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 3a3a8e464231..3e4a76d1f3aa 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -1017,7 +1017,7 @@ int ConnectionImpl::onData(int32_t stream_id, const uint8_t* data, size_t len) { void ConnectionImpl::goAway() { adapter_->SubmitGoAway(adapter_->GetHighestReceivedStreamId(), http2::adapter::Http2ErrorCode::HTTP2_NO_ERROR, ""); - + stats_.goaway_sent_.inc(); if (sendPendingFramesAndHandleError()) { // Intended to check through coverage that this error case is tested return; @@ -2046,10 +2046,13 @@ ServerConnectionImpl::ServerConnectionImpl( const envoy::config::core::v3::Http2ProtocolOptions& http2_options, const uint32_t max_request_headers_kb, const uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction - headers_with_underscores_action) + headers_with_underscores_action, + Server::OverloadManager& overload_manager) : ConnectionImpl(connection, stats, random_generator, http2_options, max_request_headers_kb, max_request_headers_count), - callbacks_(callbacks), headers_with_underscores_action_(headers_with_underscores_action) { + callbacks_(callbacks), headers_with_underscores_action_(headers_with_underscores_action), + should_send_go_away_on_dispatch_(overload_manager.getLoadShedPoint( + "envoy.load_shed_points.http2_server_go_away_on_dispatch")) { Http2Options h2_options(http2_options, max_request_headers_kb); auto visitor = std::make_unique( @@ -2132,6 +2135,11 @@ Status ServerConnectionImpl::trackInboundFrames(int32_t stream_id, size_t length Http::Status ServerConnectionImpl::dispatch(Buffer::Instance& data) { // Make sure downstream outbound queue was not flooded by the upstream frames. RETURN_IF_ERROR(protocol_constraints_.checkOutboundFrameLimits()); + if (should_send_go_away_on_dispatch_ != nullptr && !sent_go_away_on_dispatch_ && + should_send_go_away_on_dispatch_->shouldShedLoad()) { + ConnectionImpl::goAway(); + sent_go_away_on_dispatch_ = true; + } return ConnectionImpl::dispatch(data); } diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 8d33db4bab1b..5d4a6033d34f 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -768,7 +768,8 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { const uint32_t max_request_headers_kb, const uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction - headers_with_underscores_action); + headers_with_underscores_action, + Server::OverloadManager& overload_manager); private: // ConnectionImpl @@ -796,6 +797,8 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { // The action to take when a request header name contains underscore characters. envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headers_with_underscores_action_; + Server::LoadShedPoint* should_send_go_away_on_dispatch_{nullptr}; + bool sent_go_away_on_dispatch_{false}; }; } // namespace Http2 diff --git a/source/common/http/http2/codec_stats.h b/source/common/http/http2/codec_stats.h index c3346916c780..fb403bea301d 100644 --- a/source/common/http/http2/codec_stats.h +++ b/source/common/http/http2/codec_stats.h @@ -15,11 +15,13 @@ namespace Http2 { */ #define ALL_HTTP2_CODEC_STATS(COUNTER, GAUGE) \ COUNTER(dropped_headers_with_underscores) \ + COUNTER(goaway_sent) \ COUNTER(header_overflow) \ COUNTER(headers_cb_no_stream) \ COUNTER(inbound_empty_frames_flood) \ COUNTER(inbound_priority_frames_flood) \ COUNTER(inbound_window_update_frames_flood) \ + COUNTER(keepalive_timeout) \ COUNTER(metadata_empty_frames) \ COUNTER(outbound_control_flood) \ COUNTER(outbound_flood) \ @@ -30,7 +32,6 @@ namespace Http2 { COUNTER(trailers) \ COUNTER(tx_flush_timeout) \ COUNTER(tx_reset) \ - COUNTER(keepalive_timeout) \ GAUGE(streams_active, Accumulate) \ GAUGE(pending_send_bytes, Accumulate) \ GAUGE(deferred_stream_close, Accumulate) diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 3499b90b6beb..a0df43f3a9f9 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -667,7 +667,7 @@ Http::ServerConnectionPtr HttpConnectionManagerConfig::createCodec( connection, callbacks, Http::Http2::CodecStats::atomicGet(http2_codec_stats_, context_.scope()), context_.api().randomGenerator(), http2_options_, maxRequestHeadersKb(), - maxRequestHeadersCount(), headersWithUnderscoresAction()); + maxRequestHeadersCount(), headersWithUnderscoresAction(), overload_manager); case CodecType::HTTP3: return Config::Utility::getAndCheckFactoryByName( "quic.http_server_connection.default") diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index 7c024c98a827..b7fc7a420f7e 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -603,7 +603,7 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi server = std::make_unique( server_connection, server_callbacks, Http2::CodecStats::atomicGet(http2_stats, scope), random, server_http2_options, max_request_headers_kb, max_request_headers_count, - headers_with_underscores_action); + headers_with_underscores_action, overload_manager_); } else { const Http1Settings server_http1settings{fromHttp1Settings(input.h1_settings().server())}; server = std::make_unique( diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index 3772270b11d6..2f07020e559a 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -57,6 +57,7 @@ envoy_cc_test_library( deps = [ "//source/common/http/http2:codec_lib", "//test/mocks:common_lib", + "//test/mocks/server:overload_manager_mocks", ], ) @@ -229,6 +230,7 @@ envoy_cc_fuzz_test( "//test/common/http/http2:http2_frame", "//test/mocks/http:http_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/server:overload_manager_mocks", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", ], diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 778dbddae095..c08764d1e036 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -605,9 +605,11 @@ TEST_P(Http2CodecImplTest, ShutdownNotice) { EXPECT_TRUE(request_encoder_->encodeHeaders(request_headers, true).ok()); driveToCompletion(); + ASSERT_EQ(0, server_stats_store_.counter("http2.goaway_sent").value()); EXPECT_CALL(client_callbacks_, onGoAway(_)); server_->shutdownNotice(); server_->goAway(); + EXPECT_EQ(1, server_stats_store_.counter("http2.goaway_sent").value()); TestResponseHeaderMapImpl response_headers{{":status", "200"}}; EXPECT_CALL(response_decoder_, decodeHeaders_(_, true)); @@ -2924,8 +2926,11 @@ TEST_P(Http2CodecImplTest, LargeRequestHeadersExceedPerHeaderLimit) { EXPECT_CALL(request_decoder_, decodeHeaders_(_, _)).Times(0); EXPECT_CALL(client_callbacks_, onGoAway(_)); + ASSERT_EQ(0, server_stats_store_.counter("http2.goaway_sent").value()); server_->shutdownNotice(); server_->goAway(); + EXPECT_EQ(1, server_stats_store_.counter("http2.goaway_sent").value()); + EXPECT_TRUE(request_encoder_->encodeHeaders(request_headers, true).ok()); driveToCompletion(); } @@ -4239,6 +4244,65 @@ TEST_P(Http2CodecImplTest, ChunkProcessingShouldNotScheduleIfReadDisabled) { } } +TEST_P(Http2CodecImplTest, ServerDispatchLoadShedPointCanCauseServerToSendGoAway) { + initialize(); + ASSERT_EQ(0, server_stats_store_.counter("http2.goaway_sent").value()); + + TestRequestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(server_->server_go_away_on_dispatch, shouldShedLoad()).WillOnce(Return(true)); + EXPECT_CALL(client_callbacks_, onGoAway(_)); + + if (http2_implementation_ == Http2Impl::Oghttp2) { + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); + EXPECT_TRUE(request_encoder_->encodeHeaders(request_headers, true).ok()); + } else { + // nghttp2 does not raise the headers to the decoder. + EXPECT_TRUE(request_encoder_->encodeHeaders(request_headers, true).ok()); + } + driveToCompletion(); + + EXPECT_EQ(1, server_stats_store_.counter("http2.goaway_sent").value()); +} + +TEST_P(Http2CodecImplTest, ServerDispatchLoadShedPointIsOnlyConsultedOncePerDispatch) { + initialize(); + + int times_shed_load_invoked = 0; + EXPECT_CALL(server_->server_go_away_on_dispatch, shouldShedLoad()) + .WillRepeatedly(Invoke([×_shed_load_invoked]() { + ++times_shed_load_invoked; + return false; + })); + + // Drive new streams to be created within a single server dispatch. + const int num_streams_to_create = 20; + TestRequestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + + std::vector request_encoders; + std::vector response_encoders; + request_encoders.reserve(num_streams_to_create); + response_encoders.reserve(num_streams_to_create); + for (int i = 0; i < num_streams_to_create; ++i) { + request_encoders.push_back(&client_->newStream(response_decoder_)); + EXPECT_CALL(server_callbacks_, newStream(_, _)) + .WillRepeatedly(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoder& { + response_encoders.push_back(&encoder); + encoder.getStream().addCallbacks(server_stream_callbacks_); + return request_decoder_; + })); + + EXPECT_TRUE(request_encoders[i]->encodeHeaders(request_headers, true).ok()); + } + + // All the newly created streams are queued in the connection buffer. + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)).Times(num_streams_to_create); + driveToCompletion(); + EXPECT_EQ(1, times_shed_load_invoked); + EXPECT_EQ(0, server_stats_store_.counter("http2.goaway_sent").value()); +} + TEST_P(Http2CodecImplTest, CheckHeaderPaddedWhitespaceValidation) { // Per https://datatracker.ietf.org/doc/html/rfc9113#section-8.2.1, // leading & trailing whitespace characters in headers are not valid, but this is a new diff --git a/test/common/http/http2/codec_impl_test_util.h b/test/common/http/http2/codec_impl_test_util.h index 0568b0b5552d..cd93fe8f9c60 100644 --- a/test/common/http/http2/codec_impl_test_util.h +++ b/test/common/http/http2/codec_impl_test_util.h @@ -6,6 +6,7 @@ #include "source/common/http/utility.h" #include "test/mocks/common.h" +#include "test/mocks/server/overload_manager.h" #include "quiche/http2/adapter/http2_adapter.h" @@ -61,8 +62,20 @@ class TestCodecSettingsProvider { absl::node_hash_map settings_; }; +class TestCodecOverloadManagerProvider { +public: + TestCodecOverloadManagerProvider() { + ON_CALL(overload_manager_, getLoadShedPoint(testing::_)) + .WillByDefault(testing::Return(&server_go_away_on_dispatch)); + } + + testing::NiceMock overload_manager_; + testing::NiceMock server_go_away_on_dispatch; +}; + class TestServerConnectionImpl : public TestCodecStatsProvider, public TestCodecSettingsProvider, + public TestCodecOverloadManagerProvider, public ServerConnectionImpl { public: TestServerConnectionImpl( @@ -75,7 +88,7 @@ class TestServerConnectionImpl : public TestCodecStatsProvider, : TestCodecStatsProvider(scope), ServerConnectionImpl(connection, callbacks, http2CodecStats(), random, http2_options, max_request_headers_kb, max_request_headers_count, - headers_with_underscores_action) {} + headers_with_underscores_action, overload_manager_) {} http2::adapter::Http2Adapter* adapter() { return adapter_.get(); } using ServerConnectionImpl::getStream; diff --git a/test/common/http/http2/http2_connection_fuzz_test.cc b/test/common/http/http2/http2_connection_fuzz_test.cc index 603585e01b2b..02c413ace42d 100644 --- a/test/common/http/http2/http2_connection_fuzz_test.cc +++ b/test/common/http/http2/http2_connection_fuzz_test.cc @@ -10,6 +10,7 @@ #include "test/fuzz/fuzz_runner.h" #include "test/mocks/http/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/server/overload_manager.h" #include "test/test_common/test_runtime.h" #include "absl/strings/string_view.h" @@ -191,7 +192,7 @@ class Http2Harness { mock_server_connection_, mock_server_callbacks_, Http2::CodecStats::atomicGet(http2_stats_, scope), random_, server_settings_, Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT, - envoy::config::core::v3::HttpProtocolOptions::ALLOW); + envoy::config::core::v3::HttpProtocolOptions::ALLOW, overload_manager_); Buffer::OwnedImpl payload; payload.add(Http2Frame::Preamble, 24); @@ -212,6 +213,7 @@ class Http2Harness { NiceMock mock_server_connection_; NiceMock mock_server_callbacks_; NiceMock random_; + NiceMock overload_manager_; ServerConnectionPtr server_; }; diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 940838d39f64..665e08680716 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -340,10 +340,11 @@ class TestHttp2ServerConnectionImpl : public Http::Http2::ServerConnectionImpl { const envoy::config::core::v3::Http2ProtocolOptions& http2_options, const uint32_t max_request_headers_kb, const uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction - headers_with_underscores_action) + headers_with_underscores_action, + Server::OverloadManager& overload_manager) : ServerConnectionImpl(connection, callbacks, stats, random_generator, http2_options, max_request_headers_kb, max_request_headers_count, - headers_with_underscores_action) {} + headers_with_underscores_action, overload_manager) {} void updateConcurrentStreams(uint32_t max_streams) { absl::InlinedVector settings; @@ -392,7 +393,8 @@ FakeHttpConnection::FakeHttpConnection( Http::Http2::CodecStats& stats = fake_upstream.http2CodecStats(); codec_ = std::make_unique( shared_connection_.connection(), *this, stats, random_, http2_options, - max_request_headers_kb, max_request_headers_count, headers_with_underscores_action); + max_request_headers_kb, max_request_headers_count, headers_with_underscores_action, + overload_manager_); } else { ASSERT(type == Http::CodecType::HTTP3); #ifdef ENVOY_ENABLE_QUIC diff --git a/test/integration/overload_integration_test.cc b/test/integration/overload_integration_test.cc index cf3aa27fbf1e..f621b89f658a 100644 --- a/test/integration/overload_integration_test.cc +++ b/test/integration/overload_integration_test.cc @@ -711,4 +711,50 @@ TEST_P(LoadShedPointIntegrationTest, Http1ServerDispatchAbortClosesConnectionWhe test_server_->waitForCounterEq("http.config_test.downstream_rq_overload_close", 1); } +TEST_P(LoadShedPointIntegrationTest, Http2ServerDispatchSendsGoAwayCompletingPendingRequests) { + // Test only applies to HTTP2. + if (downstreamProtocol() != Http::CodecClient::Type::HTTP2) { + return; + } + autonomous_upstream_ = true; + initializeOverloadManager( + TestUtility::parseYaml(R"EOF( + name: "envoy.load_shed_points.http2_server_go_away_on_dispatch" + triggers: + - name: "envoy.resource_monitors.testonly.fake_resource_monitor" + threshold: + value: 0.90 + )EOF")); + + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + auto [first_request_encoder, first_request_decoder] = + codec_client_->startRequest(default_request_headers_); + test_server_->waitForCounterEq("http.config_test.downstream_rq_http2_total", 1); + + // Put envoy in overloaded state to send GOAWAY frames. + updateResource(0.95); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http2_server_go_away_on_dispatch.scale_percent", 100); + + auto second_request_decoder = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + // Wait for reply of the first request which should be allowed to complete. + // The downstream should also receive the GOAWAY. + Buffer::OwnedImpl first_request_body{"foo"}; + first_request_encoder.encodeData(first_request_body, true); + ASSERT_TRUE(first_request_decoder->waitForEndStream()); + + EXPECT_TRUE(codec_client_->sawGoAway()); + test_server_->waitForCounterEq("http2.goaway_sent", 1); + + // The GOAWAY gets submitted with the first created stream as the last stream + // that will be processed on this connection, so the second stream's frames + // are ignored. + EXPECT_FALSE(second_request_decoder->complete()); + + updateResource(0.80); + test_server_->waitForGaugeEq( + "overload.envoy.load_shed_points.http2_server_go_away_on_dispatch.scale_percent", 0); +} + } // namespace Envoy From 8db288b2d55231a42d3fdab18d55224d4af2bd55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 15:32:19 +0100 Subject: [PATCH 145/740] build(deps): bump protobuf from 4.22.3 to 4.22.4 in /examples/grpc-bridge/client (#27193) build(deps): bump protobuf in /examples/grpc-bridge/client Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 4.22.3 to 4.22.4. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v4.22.3...v4.22.4) --- updated-dependencies: - dependency-name: protobuf dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/client/requirements.txt | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index fd668e2d6171..4d7a5033281a 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -112,20 +112,20 @@ idna==3.2 \ --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \ --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3 # via requests -protobuf==4.22.3 \ - --hash=sha256:13233ee2b9d3bd9a5f216c1fa2c321cd564b93d8f2e4f521a85b585447747997 \ - --hash=sha256:23452f2fdea754a8251d0fc88c0317735ae47217e0d27bf330a30eec2848811a \ - --hash=sha256:52f0a78141078077cfe15fe333ac3e3a077420b9a3f5d1bf9b5fe9d286b4d881 \ - --hash=sha256:70659847ee57a5262a65954538088a1d72dfc3e9882695cab9f0c54ffe71663b \ - --hash=sha256:7760730063329d42a9d4c4573b804289b738d4931e363ffbe684716b796bde51 \ - --hash=sha256:7cf56e31907c532e460bb62010a513408e6cdf5b03fb2611e4b67ed398ad046d \ - --hash=sha256:8b54f56d13ae4a3ec140076c9d937221f887c8f64954673d46f63751209e839a \ - --hash=sha256:d14fc1a41d1a1909998e8aff7e80d2a7ae14772c4a70e4bf7db8a36690b54425 \ - --hash=sha256:d4b66266965598ff4c291416be429cef7989d8fae88b55b62095a2331511b3fa \ - --hash=sha256:e0e630d8e6a79f48c557cd1835865b593d0547dce221c66ed1b827de59c66c97 \ - --hash=sha256:ecae944c6c2ce50dda6bf76ef5496196aeb1b85acb95df5843cd812615ec4b61 \ - --hash=sha256:f08aa300b67f1c012100d8eb62d47129e53d1150f4469fd78a29fa3cb68c66f2 \ - --hash=sha256:f2f4710543abec186aee332d6852ef5ae7ce2e9e807a3da570f36de5a732d88e +protobuf==4.22.4 \ + --hash=sha256:11b28b4e779d7f275e3ea0efa3938f4d4e8ed3ca818f9fec3b193f8e9ada99fd \ + --hash=sha256:144d5b46df5e44f914f715accaadf88d617242ba5a40cacef4e8de7effa79954 \ + --hash=sha256:21fbaef7f012232eb8d6cb8ba334e931fc6ff8570f5aaedc77d5b22a439aa909 \ + --hash=sha256:3b21074b7fb748d8e123acaef9fa63a84fdc1436dc71199d2317b139f77dd6f4 \ + --hash=sha256:4bfb28d48628deacdb66a95aaa7b6640f3dc82b4edd34db444c7a3cdd90b01fb \ + --hash=sha256:5128b4d5efcaef92189e076077ae389700606ff81d2126b8361dc01f3e026197 \ + --hash=sha256:7b42086d6027be2730151b49f27b2f5be40f3b036adf7b8da5917f4567f268c3 \ + --hash=sha256:8fd329e5dd7b6c4b878cab4b85bb6cec880e2adaf4e8aa2c75944dcbb05e1ff1 \ + --hash=sha256:9537ae27d43318acf8ce27d0359fe28e6ebe4179c3350bc055bb60ff4dc4fcd3 \ + --hash=sha256:a4e661247896c2ffea4b894bca2d8657e752bedb8f3c66d7befa2557291be1e8 \ + --hash=sha256:b7728b5da9eee15c0aa3baaee79e94fa877ddcf7e3d2f34b1eab586cd26eea89 \ + --hash=sha256:e98e26328d7c668541d1052b02de4205b1094ef6b2ce57167440d3e39876db48 \ + --hash=sha256:f4a711588c3a79b6f9c44af4d7f4a2ae868e27063654683932ab6462f90e9656 # via # -r requirements.in # grpcio-tools From 59b8aa0490a4ac46ef37a41b554ae136ece8b61d Mon Sep 17 00:00:00 2001 From: danzh Date: Mon, 8 May 2023 10:45:45 -0400 Subject: [PATCH 146/740] Quiche roll 20230504221454 (#27191) * Update QUICHE from 32d62793d to 3a06b4d60 https://github.com/google/quiche/compare/32d62793d..3a06b4d60 ``` $ git log 32d62793d..3a06b4d60 --date=short --no-merges --format="%ad %al %s" 2023-05-03 diannahu Avoid interim response handling for 101 Switching Protocols responses in BalsaFrame. 2023-05-03 diannahu Add a class to represent a sequence of BalsaHeaders. 2023-05-03 quiche-dev Include debug_mode in the Public Metadata fingerprint. 2023-05-03 quiche-dev Add support for CreateContextForMultiPortPath running async. 2023-05-03 bnc Do not evict unacknowledged QPACK entries. 2023-05-01 quiche-dev Use simple regex in quic_connection_test.cc 2023-05-01 haoyuewang Clean up count_reverse_path_validation_stats. ``` Signed-off-by: Dan Zhang --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 8e00f134b23d..1a79c7a0ad15 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1054,12 +1054,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "32d62793deb3e7d3c77df57615ecd0ab4ab42859", - sha256 = "2aabe1824f7821cbf3ac6e05e23a571836ca141a243d6a2ab41e6f0037d6f370", + version = "3a06b4d609a514aff792d6c3cf033a3dedb42edb", + sha256 = "23dd8b079cf9d21f3fb5a82a44a9241c52973e457a0c89eb46bf521cd41199bd", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2023-04-27", + release_date = "2023-05-03", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", @@ -1150,7 +1150,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "compiler-rt", project_desc = "LLVM compiler runtime library", project_url = "https://compiler-rt.llvm.org", - # Note: the llvm/clang version should match the version specifed in: + # Note: the llvm/clang version should match the version specified in: # - .github/workflows/codeql-daily.yml # - .github/workflows/codeql-push.yml # - https://github.com/envoyproxy/envoy-build-tools/blob/main/build_container/build_container_ubuntu.sh#L84 From 658d9bbb7b0be7b0317cc9efea1a6f31460e9cee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 15:58:17 +0100 Subject: [PATCH 147/740] build(deps): bump google.golang.org/grpc from 1.54.0 to 1.55.0 in /examples/load-reporting-service (#27197) build(deps): bump google.golang.org/grpc Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.54.0 to 1.55.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.54.0...v1.55.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/load-reporting-service/go.mod | 4 +--- examples/load-reporting-service/go.sum | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/examples/load-reporting-service/go.mod b/examples/load-reporting-service/go.mod index 4998f6aa7d55..99b9d340b241 100644 --- a/examples/load-reporting-service/go.mod +++ b/examples/load-reporting-service/go.mod @@ -3,9 +3,7 @@ module github.com/envoyproxy/envoy/examples/load-reporting-service go 1.13 require ( - github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43 // indirect github.com/envoyproxy/go-control-plane v0.11.0 github.com/golang/protobuf v1.5.3 - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect - google.golang.org/grpc v1.54.0 + google.golang.org/grpc v1.55.0 ) diff --git a/examples/load-reporting-service/go.sum b/examples/load-reporting-service/go.sum index 0aec789ce4c2..ed297be4e25b 100644 --- a/examples/load-reporting-service/go.sum +++ b/examples/load-reporting-service/go.sum @@ -547,8 +547,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43 h1:XP+uhjN0yBCN/tPkr8Z0BNDc5rZam9RG6UWyf2FrSQ0= -github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 h1:58f1tJ1ra+zFINPlwLWvQsR9CzAKt2e+EWV2yX9oXQ4= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -568,8 +568,9 @@ github.com/envoyproxy/go-control-plane v0.11.0 h1:jtLewhRR2vMRNnq2ZZUoCjUlgut+Y0 github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= -github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.0 h1:oIfnZFdC0YhpNNEX+SuIqko4cqqVZeN9IGTrhZje83Y= +github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -589,6 +590,7 @@ github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -707,6 +709,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -832,6 +835,7 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -916,6 +920,7 @@ golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1097,6 +1102,7 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1340,8 +1346,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1357,8 +1363,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From e1ea4357856f592ea4e2d1a388d505b70b598d41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 15:59:25 +0100 Subject: [PATCH 148/740] build(deps): bump jaegertracing/all-in-one from `ac83e8d` to `a34185d` in /examples/shared/jaeger (#27251) build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger Bumps jaegertracing/all-in-one from `ac83e8d` to `a34185d`. --- updated-dependencies: - dependency-name: jaegertracing/all-in-one dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/jaeger/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile index d05f454acee7..1adb0c095895 100644 --- a/examples/shared/jaeger/Dockerfile +++ b/examples/shared/jaeger/Dockerfile @@ -1,4 +1,4 @@ -FROM jaegertracing/all-in-one@sha256:ac83e8d52ce8365390beda54de65235d392077da6e1e35f8b848f95db7e9d6b0 +FROM jaegertracing/all-in-one@sha256:a34185d2d040d44788580359ec9bffac794c4ec3e1c83b1da9b1448e9c8185a7 HEALTHCHECK \ --interval=1s \ --timeout=1s \ From ac76c49706359bbcdf0f89555570f91fc4e11b1a Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 8 May 2023 16:26:00 +0100 Subject: [PATCH 149/740] deps: Bump build image -> `818d288` (#27222) Signed-off-by: Ryan Northey --- .azure-pipelines/stage/prechecks.yml | 2 +- .bazelrc | 2 +- .devcontainer/Dockerfile | 2 +- .github/workflows/mobile-android_build.yml | 2 +- .github/workflows/mobile-android_tests.yml | 2 +- .github/workflows/mobile-asan.yml | 2 +- .github/workflows/mobile-cc_tests.yml | 2 +- .github/workflows/mobile-core.yml | 2 +- .github/workflows/mobile-coverage.yml | 2 +- .github/workflows/mobile-docs.yml | 2 +- .github/workflows/mobile-format.yml | 2 +- .github/workflows/mobile-perf.yml | 6 +++--- .github/workflows/mobile-tsan.yml | 2 +- .github/workflows/mobile_release.yml | 2 +- bazel/repository_locations.bzl | 6 +++--- examples/shared/build/Dockerfile | 2 +- mobile/third_party/rbe_configs/config/BUILD | 8 ++++---- 17 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.azure-pipelines/stage/prechecks.yml b/.azure-pipelines/stage/prechecks.yml index 4a445157d0cb..8aeb26c2656e 100644 --- a/.azure-pipelines/stage/prechecks.yml +++ b/.azure-pipelines/stage/prechecks.yml @@ -37,7 +37,7 @@ parameters: jobs: - job: prechecks displayName: Precheck - timeoutInMinutes: 20 + timeoutInMinutes: 30 pool: vmImage: "ubuntu-20.04" variables: diff --git a/.bazelrc b/.bazelrc index eb023f6c5535..4b8b6c96f71c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -304,7 +304,7 @@ build:remote-clang-cl --config=rbe-toolchain-clang-cl # Docker sandbox # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/main/toolchains/rbe_toolchains_config.bzl#L8 -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b438647c983a..e323afcb8855 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM gcr.io/envoy-ci/envoy-build:321658b6b50abda6869f89fac275f59bf3b1e757 +FROM gcr.io/envoy-ci/envoy-build:818d28832abf2a7c0cb2bff00435be231729a0bf ARG USERNAME=vscode ARG USER_UID=501 diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml index cf889b38ac2d..200a58d5b4b5 100644 --- a/.github/workflows/mobile-android_build.yml +++ b/.github/workflows/mobile-android_build.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:mobile-321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml index c198cef9b363..4291842163fa 100644 --- a/.github/workflows/mobile-android_tests.yml +++ b/.github/workflows/mobile-android_tests.yml @@ -91,7 +91,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:mobile-321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-asan.yml b/.github/workflows/mobile-asan.yml index ca8952a56f03..72c2ec95ade1 100644 --- a/.github/workflows/mobile-asan.yml +++ b/.github/workflows/mobile-asan.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 180 container: - image: envoyproxy/envoy-build-ubuntu:mobile-321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-cc_tests.yml b/.github/workflows/mobile-cc_tests.yml index 8c5793afeff0..f1da9a974d9c 100644 --- a/.github/workflows/mobile-cc_tests.yml +++ b/.github/workflows/mobile-cc_tests.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf steps: - uses: actions/checkout@v3 - name: Add safe directory diff --git a/.github/workflows/mobile-core.yml b/.github/workflows/mobile-core.yml index ccd32f84d9c9..edf0a078d8ff 100644 --- a/.github/workflows/mobile-core.yml +++ b/.github/workflows/mobile-core.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf steps: - uses: actions/checkout@v3 - name: Ensure no listener leaks diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml index b69a4e05209d..2a6dd22db74e 100644 --- a/.github/workflows/mobile-coverage.yml +++ b/.github/workflows/mobile-coverage.yml @@ -20,7 +20,7 @@ jobs: run: shell: bash container: - image: envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/mobile-docs.yml b/.github/workflows/mobile-docs.yml index c06d7d85c414..292c96c1dd6f 100644 --- a/.github/workflows/mobile-docs.yml +++ b/.github/workflows/mobile-docs.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 20 container: - image: envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf steps: - uses: actions/checkout@v3 - name: Add safe directory diff --git a/.github/workflows/mobile-format.yml b/.github/workflows/mobile-format.yml index 1ff61808a92d..a42b71b5a9ae 100644 --- a/.github/workflows/mobile-format.yml +++ b/.github/workflows/mobile-format.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 45 container: - image: envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf env: CLANG_FORMAT: /opt/llvm/bin/clang-format BUILDIFIER_BIN: /usr/local/bin/buildifier diff --git a/.github/workflows/mobile-perf.yml b/.github/workflows/mobile-perf.yml index 346580233e06..18bb4cacdb85 100644 --- a/.github/workflows/mobile-perf.yml +++ b/.github/workflows/mobile-perf.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 30 container: - image: envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf steps: - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 diff --git a/.github/workflows/mobile-tsan.yml b/.github/workflows/mobile-tsan.yml index 2af0fc12d6b1..02ea14dfeb7d 100644 --- a/.github/workflows/mobile-tsan.yml +++ b/.github/workflows/mobile-tsan.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:mobile-321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile_release.yml b/.github/workflows/mobile_release.yml index 4e1b3c7b8b14..4fe5f89f5fa3 100644 --- a/.github/workflows/mobile_release.yml +++ b/.github/workflows/mobile_release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:mobile-321658b6b50abda6869f89fac275f59bf3b1e757 + image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 1a79c7a0ad15..0145b5db237b 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -90,11 +90,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "envoy-build-tools", project_desc = "Common build tools shared by the Envoy/UDPA ecosystem", project_url = "https://github.com/envoyproxy/envoy-build-tools", - version = "b0452fc4dfb1b02357bd4ce2d55a1056b53e2ffd", - sha256 = "fbc095421b95427e2dc4ba02aa4459796e020fbc1a3b577168e781dd3d420119", + version = "2c557f5c899527b3331c316b2d5ca137fcb046b9", + sha256 = "f7e256ee9ee4d02536fb2124c6b79694633e187fe9b33017273af92e88bbdd6c", strip_prefix = "envoy-build-tools-{version}", urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"], - release_date = "2023-02-22", + release_date = "2023-05-06", use_category = ["build"], license = "Apache-2.0", license_url = "https://github.com/envoyproxy/envoy-build-tools/blob/{version}/LICENSE", diff --git a/examples/shared/build/Dockerfile b/examples/shared/build/Dockerfile index 277bdd0687a4..9e8e5992fbca 100644 --- a/examples/shared/build/Dockerfile +++ b/examples/shared/build/Dockerfile @@ -1,4 +1,4 @@ -FROM envoyproxy/envoy-build-ubuntu:321658b6b50abda6869f89fac275f59bf3b1e757 +FROM envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ diff --git a/mobile/third_party/rbe_configs/config/BUILD b/mobile/third_party/rbe_configs/config/BUILD index ea43bf063118..b9281332ad58 100644 --- a/mobile/third_party/rbe_configs/config/BUILD +++ b/mobile/third_party/rbe_configs/config/BUILD @@ -43,8 +43,8 @@ platform( ], exec_properties = { # Please update both the commented tag and the sha256 - # mobile-321658b6b50abda6869f89fac275f59bf3b1e757 - "container-image": "docker://envoyproxy/envoy-build-ubuntu@sha256:edcd937aefcc3c01a89df65389bfbb46fda438edaeefd6c859a4f59afb2191ee", + # mobile-818d28832abf2a7c0cb2bff00435be231729a0bf + "container-image": "docker://envoyproxy/envoy-build-ubuntu@sha256:20034a0e6afa28e33125760b2dbb069ba9bf0ba717727fbb2d200a4b633297fe", "OSFamily": "Linux", "Pool": "linux", }, @@ -60,8 +60,8 @@ platform( ], exec_properties = { # Please update both the commented tag and the sha256 - # mobile-321658b6b50abda6869f89fac275f59bf3b1e757 - "container-image": "docker://envoyproxy/envoy-build-ubuntu@sha256:edcd937aefcc3c01a89df65389bfbb46fda438edaeefd6c859a4f59afb2191ee", + # mobile-818d28832abf2a7c0cb2bff00435be231729a0bf + "container-image": "docker://envoyproxy/envoy-build-ubuntu@sha256:20034a0e6afa28e33125760b2dbb069ba9bf0ba717727fbb2d200a4b633297fe", "OSFamily": "Linux", "Pool": "linux", # Necessary to workaround https://github.com/google/sanitizers/issues/916, otherwise, dangling threads in the From 5b5d13955d576369b5593f6ec29e3cfe42ef9fff Mon Sep 17 00:00:00 2001 From: code Date: Mon, 8 May 2023 23:29:47 +0800 Subject: [PATCH 150/740] factory context refactor: refactor the transport factory context (#27081) Signed-off-by: wbpcode --- .../source/cryptomb_private_key_provider.cc | 10 +- .../private_key_providers/test/config_test.cc | 4 +- .../test/fake_factory.cc | 4 +- .../network/test/postgres_integration_test.cc | 2 +- .../source/qat_private_key_provider.cc | 7 +- .../private_key_providers/test/config_test.cc | 4 +- .../private_key_providers/test/ops_test.cc | 5 +- .../filters/network/source/utility.h | 1 + .../filters/network/test/conn_manager_test.cc | 3 +- .../filters/network/test/router_test.cc | 3 +- contrib/sxg/filters/http/test/filter_test.cc | 9 +- envoy/server/transport_socket_config.h | 72 ++------- mobile/test/common/integration/test_server.cc | 5 +- .../quic/quic_transport_socket_factory.cc | 8 +- source/common/secret/sds_api.h | 41 +++-- .../upstream/health_discovery_service.cc | 3 +- source/common/upstream/upstream_impl.cc | 16 +- source/extensions/clusters/eds/leds.cc | 2 +- .../listener_manager/listener_impl.cc | 3 +- .../transport_sockets/alts/config.cc | 7 +- .../internal_upstream/config.cc | 2 +- .../proxy_protocol/config.cc | 2 +- .../transport_sockets/tap/config.cc | 17 ++- .../transport_sockets/tcp_stats/config.cc | 2 +- .../transport_sockets/tls/config.cc | 6 +- .../tls/context_config_impl.cc | 5 +- .../tls/context_config_impl.h | 2 +- source/extensions/upstreams/http/config.h | 2 +- source/server/server.h | 6 +- source/server/transport_socket_config_impl.h | 34 ++--- .../grpc_client_integration_test_harness.h | 2 +- .../quic_transport_socket_factory_test.cc | 2 +- .../common/secret/secret_manager_impl_test.cc | 69 ++++----- test/common/upstream/BUILD | 1 + test/common/upstream/hds_test.cc | 4 +- test/extensions/clusters/eds/leds_test.cc | 2 +- .../filters/http/oauth2/filter_test.cc | 9 +- .../alts/alts_integration_test.cc | 2 +- .../transport_sockets/alts/config_test.cc | 6 +- .../upstream_starttls_integration_test.cc | 2 +- test/extensions/transport_sockets/tls/BUILD | 1 + .../tls/context_impl_test.cc | 35 ++--- .../tls/handshaker_factory_test.cc | 12 +- .../transport_sockets/tls/ssl_certs_test.h | 2 +- .../transport_sockets/tls/ssl_socket_test.cc | 52 ++++--- .../tls/test_private_key_method_provider.cc | 3 +- test/integration/base_integration_test.cc | 4 +- .../integration/quic_http_integration_test.cc | 4 +- test/integration/ssl_utility.cc | 4 +- test/integration/utility.cc | 4 +- test/integration/xfcc_integration_test.h | 2 +- test/mocks/server/BUILD | 36 +++-- test/mocks/server/instance.cc | 36 +---- test/mocks/server/instance.h | 134 +---------------- test/mocks/server/server_factory_context.cc | 43 ++++++ test/mocks/server/server_factory_context.h | 140 ++++++++++++++++++ .../transport_socket_factory_context.cc | 12 +- .../server/transport_socket_factory_context.h | 26 +--- 58 files changed, 443 insertions(+), 493 deletions(-) create mode 100644 test/mocks/server/server_factory_context.cc create mode 100644 test/mocks/server/server_factory_context.h diff --git a/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc b/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc index e1a7fc96d4ae..f94f37db6323 100644 --- a/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc +++ b/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc @@ -516,9 +516,10 @@ CryptoMbPrivateKeyMethodProvider::CryptoMbPrivateKeyMethodProvider( const envoy::extensions::private_key_providers::cryptomb::v3alpha:: CryptoMbPrivateKeyMethodConfig& conf, Server::Configuration::TransportSocketFactoryContext& factory_context, IppCryptoSharedPtr ipp) - : api_(factory_context.api()), - tls_(ThreadLocal::TypedSlot::makeUnique(factory_context.threadLocal())), - stats_(generateCryptoMbStats("cryptomb", factory_context.scope())) { + : api_(factory_context.serverFactoryContext().api()), + tls_(ThreadLocal::TypedSlot::makeUnique( + factory_context.serverFactoryContext().threadLocal())), + stats_(generateCryptoMbStats("cryptomb", factory_context.statsScope())) { if (!ipp->mbxIsCryptoMbApplicable(0)) { throw EnvoyException("Multi-buffer CPU instructions not available."); @@ -527,8 +528,7 @@ CryptoMbPrivateKeyMethodProvider::CryptoMbPrivateKeyMethodProvider( std::chrono::milliseconds poll_delay = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(conf, poll_delay, 200)); - std::string private_key = - Config::DataSource::read(conf.private_key(), false, factory_context.api()); + std::string private_key = Config::DataSource::read(conf.private_key(), false, api_); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); diff --git a/contrib/cryptomb/private_key_providers/test/config_test.cc b/contrib/cryptomb/private_key_providers/test/config_test.cc index 370ce3953692..158936f49037 100644 --- a/contrib/cryptomb/private_key_providers/test/config_test.cc +++ b/contrib/cryptomb/private_key_providers/test/config_test.cc @@ -35,8 +35,8 @@ parsePrivateKeyProviderFromV3Yaml(const std::string& yaml_string) { class CryptoMbConfigTest : public Event::TestUsingSimulatedTime, public testing::Test { public: CryptoMbConfigTest() : api_(Api::createApiForTest(store_, time_system_)) { - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); - ON_CALL(factory_context_, threadLocal()).WillByDefault(ReturnRef(tls_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, threadLocal()).WillByDefault(ReturnRef(tls_)); ON_CALL(factory_context_, sslContextManager()).WillByDefault(ReturnRef(context_manager_)); ON_CALL(context_manager_, privateKeyMethodManager()) .WillByDefault(ReturnRef(private_key_method_manager_)); diff --git a/contrib/cryptomb/private_key_providers/test/fake_factory.cc b/contrib/cryptomb/private_key_providers/test/fake_factory.cc index 7ae205f52f9e..ddd2a5d363e8 100644 --- a/contrib/cryptomb/private_key_providers/test/fake_factory.cc +++ b/contrib/cryptomb/private_key_providers/test/fake_factory.cc @@ -135,8 +135,8 @@ FakeCryptoMbPrivateKeyMethodFactory::createPrivateKeyMethodProviderInstance( std::make_shared(supported_instruction_set_); // We need to get more RSA key params in order to be able to use BoringSSL signing functions. - std::string private_key = - Config::DataSource::read(conf.private_key(), false, private_key_provider_context.api()); + std::string private_key = Config::DataSource::read( + conf.private_key(), false, private_key_provider_context.serverFactoryContext().api()); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); diff --git a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc index 6fb93661e68c..4f1e3c829fb8 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc @@ -313,7 +313,7 @@ class UpstreamSSLBaseIntegrationTest : public PostgresBaseIntegrationTest { TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), downstream_tls_context); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); auto cfg = std::make_unique( downstream_tls_context, mock_factory_ctx); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); diff --git a/contrib/qat/private_key_providers/source/qat_private_key_provider.cc b/contrib/qat/private_key_providers/source/qat_private_key_provider.cc index 6b2d808d254b..863637604157 100644 --- a/contrib/qat/private_key_providers/source/qat_private_key_provider.cc +++ b/contrib/qat/private_key_providers/source/qat_private_key_provider.cc @@ -347,9 +347,9 @@ QatPrivateKeyMethodProvider::QatPrivateKeyMethodProvider( const envoy::extensions::private_key_providers::qat::v3alpha::QatPrivateKeyMethodConfig& conf, Server::Configuration::TransportSocketFactoryContext& factory_context, LibQatCryptoSharedPtr libqat) - : api_(factory_context.api()), libqat_(libqat) { + : api_(factory_context.serverFactoryContext().api()), libqat_(libqat) { - manager_ = factory_context.singletonManager().getTyped( + manager_ = factory_context.serverFactoryContext().singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(qat_manager), [libqat] { return std::make_shared(libqat); }); @@ -358,8 +358,7 @@ QatPrivateKeyMethodProvider::QatPrivateKeyMethodProvider( std::chrono::milliseconds poll_delay = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(conf, poll_delay, 5)); - std::string private_key = - Config::DataSource::read(conf.private_key(), false, factory_context.api()); + std::string private_key = Config::DataSource::read(conf.private_key(), false, api_); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); diff --git a/contrib/qat/private_key_providers/test/config_test.cc b/contrib/qat/private_key_providers/test/config_test.cc index b5527bc97bc8..518c9ba22d6a 100644 --- a/contrib/qat/private_key_providers/test/config_test.cc +++ b/contrib/qat/private_key_providers/test/config_test.cc @@ -48,11 +48,11 @@ class QatConfigTest : public Event::TestUsingSimulatedTime, public testing::Test QatConfigTest() : api_(Api::createApiForTest(store_, time_system_)), libqat_(std::make_shared()), fsm_(libqat_) { - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); ON_CALL(factory_context_, sslContextManager()).WillByDefault(ReturnRef(context_manager_)); ON_CALL(context_manager_, privateKeyMethodManager()) .WillByDefault(ReturnRef(private_key_method_manager_)); - ON_CALL(factory_context_, singletonManager()).WillByDefault(ReturnRef(fsm_)); + ON_CALL(factory_context_.server_context_, singletonManager()).WillByDefault(ReturnRef(fsm_)); } Ssl::PrivateKeyMethodProviderSharedPtr createWithConfig(std::string yaml) { diff --git a/contrib/qat/private_key_providers/test/ops_test.cc b/contrib/qat/private_key_providers/test/ops_test.cc index 0e2f77322107..68922f08bfef 100644 --- a/contrib/qat/private_key_providers/test/ops_test.cc +++ b/contrib/qat/private_key_providers/test/ops_test.cc @@ -59,8 +59,9 @@ class QatProviderTest : public testing::Test { dispatcher_(api_->allocateDispatcher("test_thread")), libqat_(std::make_shared()), fsm_(libqat_) { handle_.setLibqat(libqat_); - ON_CALL(factory_context_, api()).WillByDefault(testing::ReturnRef(*api_)); - ON_CALL(factory_context_, singletonManager()).WillByDefault(testing::ReturnRef(fsm_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, singletonManager()) + .WillByDefault(testing::ReturnRef(fsm_)); } Stats::TestUtil::TestStore store_; diff --git a/contrib/sip_proxy/filters/network/source/utility.h b/contrib/sip_proxy/filters/network/source/utility.h index 1cc4a69872d0..29465c09cab2 100644 --- a/contrib/sip_proxy/filters/network/source/utility.h +++ b/contrib/sip_proxy/filters/network/source/utility.h @@ -156,6 +156,7 @@ class Utility { public: static const std::string& localAddress(Server::Configuration::FactoryContext& context) { return context.getTransportSocketFactoryContext() + .serverFactoryContext() .localInfo() .address() ->ip() diff --git a/contrib/sip_proxy/filters/network/test/conn_manager_test.cc b/contrib/sip_proxy/filters/network/test/conn_manager_test.cc index 6b687031d9e3..8cf88bed959f 100644 --- a/contrib/sip_proxy/filters/network/test/conn_manager_test.cc +++ b/contrib/sip_proxy/filters/network/test/conn_manager_test.cc @@ -80,7 +80,8 @@ class SipConnectionManagerTest : public testing::Test { EXPECT_CALL(context_, getTransportSocketFactoryContext()) .WillRepeatedly(testing::ReturnRef(factory_context_)); - EXPECT_CALL(factory_context_, localInfo()).WillRepeatedly(testing::ReturnRef(local_info_)); + EXPECT_CALL(factory_context_.server_context_, localInfo()) + .WillRepeatedly(testing::ReturnRef(local_info_)); ON_CALL(random_, random()).WillByDefault(Return(42)); filter_ = std::make_unique( *config_, random_, filter_callbacks_.connection_.dispatcher_.timeSource(), context_, diff --git a/contrib/sip_proxy/filters/network/test/router_test.cc b/contrib/sip_proxy/filters/network/test/router_test.cc index f50895c58ead..9964207138da 100644 --- a/contrib/sip_proxy/filters/network/test/router_test.cc +++ b/contrib/sip_proxy/filters/network/test/router_test.cc @@ -105,7 +105,8 @@ class SipRouterTest : public testing::Test { EXPECT_CALL(context_, getTransportSocketFactoryContext()) .WillRepeatedly(testing::ReturnRef(factory_context_)); - EXPECT_CALL(factory_context_, localInfo()).WillRepeatedly(testing::ReturnRef(local_info_)); + EXPECT_CALL(factory_context_.server_context_, localInfo()) + .WillRepeatedly(testing::ReturnRef(local_info_)); transaction_infos_ = std::make_shared(); context_.cluster_manager_.initializeThreadLocalClusters({cluster_name_}); diff --git a/contrib/sxg/filters/http/test/filter_test.cc b/contrib/sxg/filters/http/test/filter_test.cc index 5a874067c10a..92bf1fa92f9e 100644 --- a/contrib/sxg/filters/http/test/filter_test.cc +++ b/contrib/sxg/filters/http/test/filter_test.cc @@ -360,14 +360,13 @@ TEST_F(FilterTest, SdsDynamicGenericSecret) { NiceMock secret_context; NiceMock local_info; Api::ApiPtr api = Api::createApiForTest(); - Stats::IsolatedStoreImpl stats; NiceMock init_manager; Init::TargetHandlePtr init_handle; NiceMock dispatcher; - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api)); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); EXPECT_CALL(secret_context, initManager()).Times(0); EXPECT_CALL(init_manager, add(_)) .WillRepeatedly(Invoke([&init_handle](const Init::Target& target) { diff --git a/envoy/server/transport_socket_config.h b/envoy/server/transport_socket_config.h index 756c27f64365..c39723b573ca 100644 --- a/envoy/server/transport_socket_config.h +++ b/envoy/server/transport_socket_config.h @@ -30,14 +30,23 @@ class TransportSocketFactoryContext { virtual ~TransportSocketFactoryContext() = default; /** - * @return ServerFactoryContext which lifetime is no shorter than the server. + * @return ServerFactoryContext& the server factory context. */ - virtual ServerFactoryContext& getServerFactoryContext() PURE; + virtual ServerFactoryContext& serverFactoryContext() PURE; /** - * @return OptRef the global HTTP admin endpoint for the server. + * @return Upstream::ClusterManager& singleton for use by the entire server. + * TODO(wbpcode): clusterManager() of ServerFactoryContext still be invalid when loading + * static cluster. So we need to provide an cluster manager reference here. + * This could be removed after https://github.com/envoyproxy/envoy/issues/26653 is resolved. */ - virtual OptRef admin() PURE; + virtual Upstream::ClusterManager& clusterManager() PURE; + + /** + * @return ProtobufMessage::ValidationVisitor& validation visitor for cluster configuration + * messages. + */ + virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; /** * @return Ssl::ContextManager& the SSL context manager. @@ -47,7 +56,7 @@ class TransportSocketFactoryContext { /** * @return Stats::Scope& the transport socket's stats scope. */ - virtual Stats::Scope& scope() PURE; + virtual Stats::Scope& statsScope() PURE; /** * Return the instance of secret manager. @@ -55,60 +64,9 @@ class TransportSocketFactoryContext { virtual Secret::SecretManager& secretManager() PURE; /** - * @return the instance of ClusterManager. - */ - virtual Upstream::ClusterManager& clusterManager() PURE; - - /** - * @return information about the local environment the server is running in. - */ - virtual const LocalInfo::LocalInfo& localInfo() const PURE; - - /** - * @return Event::Dispatcher& the main thread's dispatcher. - */ - virtual Event::Dispatcher& mainThreadDispatcher() PURE; - - /** - * @return Server::Options& the command-line options that Envoy was started with. - */ - virtual const Options& options() PURE; - - /** - * @return the server-wide stats store. - */ - virtual Stats::Store& stats() PURE; - - /** - * @return a reference to the instance of an init manager. + * @return the init manager of the particular context. */ virtual Init::Manager& initManager() PURE; - - /** - * @return the server's singleton manager. - */ - virtual Singleton::Manager& singletonManager() PURE; - - /** - * @return the server's TLS slot allocator. - */ - virtual ThreadLocal::SlotAllocator& threadLocal() PURE; - - /** - * @return ProtobufMessage::ValidationVisitor& validation visitor for filter configuration - * messages. - */ - virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; - - /** - * @return reference to the Api object - */ - virtual Api::Api& api() PURE; - - /** - * @return reference to the access log manager object - */ - virtual AccessLog::AccessLogManager& accessLogManager() PURE; }; using TransportSocketFactoryContextPtr = std::unique_ptr; diff --git a/mobile/test/common/integration/test_server.cc b/mobile/test/common/integration/test_server.cc index 771360fc3da0..f47ef32aeb00 100644 --- a/mobile/test/common/integration/test_server.cc +++ b/mobile/test/common/integration/test_server.cc @@ -53,8 +53,9 @@ Network::DownstreamTransportSocketFactoryPtr TestServer::createUpstreamTlsContex TestServer::TestServer() : api_(Api::createApiForTest(stats_store_, time_system_)), version_(Network::Address::IpVersion::v4), upstream_config_(time_system_), port_(0) { - ON_CALL(factory_context_, api()).WillByDefault(testing::ReturnRef(*api_)); - ON_CALL(factory_context_, scope()).WillByDefault(testing::ReturnRef(*stats_store_.rootScope())); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(factory_context_, statsScope()) + .WillByDefault(testing::ReturnRef(*stats_store_.rootScope())); } void TestServer::startTestServer(bool use_quic) { diff --git a/source/common/quic/quic_transport_socket_factory.cc b/source/common/quic/quic_transport_socket_factory.cc index 05d5ddc0c0ca..f7f21ef5db8f 100644 --- a/source/common/quic/quic_transport_socket_factory.cc +++ b/source/common/quic/quic_transport_socket_factory.cc @@ -30,8 +30,8 @@ QuicServerTransportSocketConfigFactory::createTransportSocketFactory( } auto factory = std::make_unique( - PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_transport, enable_early_data, true), context.scope(), - std::move(server_config)); + PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_transport, enable_early_data, true), + context.statsScope(), std::move(server_config)); factory->initialize(); return factory; } @@ -59,9 +59,9 @@ QuicClientTransportSocketConfigFactory::createTransportSocketFactory( QuicClientTransportSocketFactory::QuicClientTransportSocketFactory( Ssl::ClientContextConfigPtr config, Server::Configuration::TransportSocketFactoryContext& factory_context) - : QuicTransportSocketFactoryBase(factory_context.scope(), "client"), + : QuicTransportSocketFactoryBase(factory_context.statsScope(), "client"), fallback_factory_(std::make_unique( - std::move(config), factory_context.sslContextManager(), factory_context.scope())) {} + std::move(config), factory_context.sslContextManager(), factory_context.statsScope())) {} ProtobufTypes::MessagePtr QuicClientTransportSocketConfigFactory::createEmptyConfigProto() { return std::make_unique(); diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index a09d87b6ac52..edc5db1e0ad8 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -137,13 +137,13 @@ class TlsCertificateSdsApi : public SdsApi, public TlsCertificateConfigProvider const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - Config::Utility::checkLocalInfo("TlsCertificateSdsApi", secret_provider_context.localInfo()); + auto& server_context = secret_provider_context.serverFactoryContext(); + Config::Utility::checkLocalInfo("TlsCertificateSdsApi", server_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - secret_provider_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), - destructor_cb, secret_provider_context.mainThreadDispatcher(), - secret_provider_context.api()); + server_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), + destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); } TlsCertificateSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, @@ -223,14 +223,14 @@ class CertificateValidationContextSdsApi : public SdsApi, const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. + auto& server_context = secret_provider_context.serverFactoryContext(); Config::Utility::checkLocalInfo("CertificateValidationContextSdsApi", - secret_provider_context.localInfo()); + server_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - secret_provider_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), - destructor_cb, secret_provider_context.mainThreadDispatcher(), - secret_provider_context.api()); + server_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), + destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); } CertificateValidationContextSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, const std::string& sds_config_name, @@ -318,14 +318,13 @@ class TlsSessionTicketKeysSdsApi : public SdsApi, public TlsSessionTicketKeysCon const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - Config::Utility::checkLocalInfo("TlsSessionTicketKeysSdsApi", - secret_provider_context.localInfo()); + auto& server_context = secret_provider_context.serverFactoryContext(); + Config::Utility::checkLocalInfo("TlsSessionTicketKeysSdsApi", server_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - secret_provider_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), - destructor_cb, secret_provider_context.mainThreadDispatcher(), - secret_provider_context.api()); + server_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), + destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); } TlsSessionTicketKeysSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, @@ -392,13 +391,13 @@ class GenericSecretSdsApi : public SdsApi, public GenericSecretConfigProvider { const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - Config::Utility::checkLocalInfo("GenericSecretSdsApi", secret_provider_context.localInfo()); + auto& server_context = secret_provider_context.serverFactoryContext(); + Config::Utility::checkLocalInfo("GenericSecretSdsApi", server_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - secret_provider_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), - destructor_cb, secret_provider_context.mainThreadDispatcher(), - secret_provider_context.api()); + server_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), + destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); } GenericSecretSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index 7a8386069184..30f16ac78084 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -516,8 +516,7 @@ ProdClusterInfoFactory::createClusterInfo(const CreateClusterInfoParams& params) Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( params.server_context_, params.ssl_context_manager_, *scope, - params.server_context_.clusterManager(), params.stats_, - params.server_context_.messageValidationVisitor()); + params.server_context_.clusterManager(), params.server_context_.messageValidationVisitor()); // TODO(JimmyCYJ): Support SDS for HDS cluster. Network::UpstreamTransportSocketFactoryPtr socket_factory = diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 130404364e3a..228807771a0c 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -882,11 +882,14 @@ class FactoryContextImpl : public Server::Configuration::CommonFactoryContext { // other contexts taken from TransportSocketFactoryContext. FactoryContextImpl(Stats::Scope& stats_scope, Envoy::Runtime::Loader& runtime, Server::Configuration::TransportSocketFactoryContext& c) - : admin_(c.admin()), server_scope_(*c.stats().rootScope()), stats_scope_(stats_scope), - cluster_manager_(c.clusterManager()), local_info_(c.localInfo()), - dispatcher_(c.mainThreadDispatcher()), runtime_(runtime), - singleton_manager_(c.singletonManager()), tls_(c.threadLocal()), api_(c.api()), - options_(c.options()), message_validation_visitor_(c.messageValidationVisitor()) {} + : admin_(c.serverFactoryContext().admin()), + server_scope_(c.serverFactoryContext().serverScope()), stats_scope_(stats_scope), + cluster_manager_(c.clusterManager()), local_info_(c.serverFactoryContext().localInfo()), + dispatcher_(c.serverFactoryContext().mainThreadDispatcher()), runtime_(runtime), + singleton_manager_(c.serverFactoryContext().singletonManager()), + tls_(c.serverFactoryContext().threadLocal()), api_(c.serverFactoryContext().api()), + options_(c.serverFactoryContext().options()), + message_validation_visitor_(c.messageValidationVisitor()) {} Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } Event::Dispatcher& mainThreadDispatcher() override { return dispatcher_; } @@ -1374,8 +1377,7 @@ ClusterImplBase::ClusterImplBase(const envoy::config::cluster::v3::Cluster& clus transport_factory_context_ = std::make_unique( server_context, cluster_context.sslContextManager(), *stats_scope, - cluster_context.clusterManager(), server_context.serverScope().store(), - cluster_context.messageValidationVisitor()); + cluster_context.clusterManager(), cluster_context.messageValidationVisitor()); transport_factory_context_->setInitManager(init_manager_); auto socket_factory = createTransportSocketFactory(cluster, *transport_factory_context_); diff --git a/source/extensions/clusters/eds/leds.cc b/source/extensions/clusters/eds/leds.cc index 2e2ed0db6941..91c5ad6f32a7 100644 --- a/source/extensions/clusters/eds/leds.cc +++ b/source/extensions/clusters/eds/leds.cc @@ -17,7 +17,7 @@ LedsSubscription::LedsSubscription( Stats::Scope& cluster_stats_scope, const UpdateCb& callback) : Envoy::Config::SubscriptionBase( factory_context.messageValidationVisitor(), leds_config.leds_collection_name()), - local_info_(factory_context.localInfo()), cluster_name_(cluster_name), + local_info_(factory_context.serverFactoryContext().localInfo()), cluster_name_(cluster_name), stats_scope_(cluster_stats_scope.createScope("leds.")), stats_({ALL_LEDS_STATS(POOL_COUNTER(*stats_scope_))}), callback_(callback) { const xds::core::v3::ResourceLocator leds_resource_locator = diff --git a/source/extensions/listener_managers/listener_manager/listener_impl.cc b/source/extensions/listener_managers/listener_manager/listener_impl.cc index 94b29a189e71..47909c0de470 100644 --- a/source/extensions/listener_managers/listener_manager/listener_impl.cc +++ b/source/extensions/listener_managers/listener_manager/listener_impl.cc @@ -354,8 +354,7 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, transport_factory_context_( std::make_shared( parent_.server_.serverFactoryContext(), parent_.server_.sslContextManager(), - listenerScope(), parent_.server_.clusterManager(), parent_.server_.stats(), - validation_visitor_)), + listenerScope(), parent_.server_.clusterManager(), validation_visitor_)), quic_stat_names_(parent_.quicStatNames()), missing_listener_config_stats_({ALL_MISSING_LISTENER_CONFIG_STATS( POOL_COUNTER(listener_factory_context_->listenerScope()))}) { diff --git a/source/extensions/transport_sockets/alts/config.cc b/source/extensions/transport_sockets/alts/config.cc index 7c3d18d27d7a..0af3cb780775 100644 --- a/source/extensions/transport_sockets/alts/config.cc +++ b/source/extensions/transport_sockets/alts/config.cc @@ -97,9 +97,10 @@ TransportSocketFactoryPtr createTransportSocketFactoryHelper( Server::Configuration::TransportSocketFactoryContext& factory_ctxt) { // A reference to this is held in the factory closure to keep the singleton // instance alive. - auto alts_shared_state = factory_ctxt.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(alts_shared_state), - [] { return std::make_shared(); }); + auto alts_shared_state = + factory_ctxt.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(alts_shared_state), + [] { return std::make_shared(); }); auto config = MessageUtil::downcastAndValidate( message, factory_ctxt.messageValidationVisitor()); diff --git a/source/extensions/transport_sockets/internal_upstream/config.cc b/source/extensions/transport_sockets/internal_upstream/config.cc index 2993e34d2815..5517c5c34203 100644 --- a/source/extensions/transport_sockets/internal_upstream/config.cc +++ b/source/extensions/transport_sockets/internal_upstream/config.cc @@ -104,7 +104,7 @@ InternalSocketFactory::InternalSocketFactory( const envoy::extensions::transport_sockets::internal_upstream::v3::InternalUpstreamTransport& config_proto, Network::UpstreamTransportSocketFactoryPtr&& inner_factory) - : PassthroughFactory(std::move(inner_factory)), config_(config_proto, context.scope()) {} + : PassthroughFactory(std::move(inner_factory)), config_(config_proto, context.statsScope()) {} Network::TransportSocketPtr InternalSocketFactory::createTransportSocket(Network::TransportSocketOptionsConstSharedPtr options, diff --git a/source/extensions/transport_sockets/proxy_protocol/config.cc b/source/extensions/transport_sockets/proxy_protocol/config.cc index cf8f1e08b387..bdd57dfdc84f 100644 --- a/source/extensions/transport_sockets/proxy_protocol/config.cc +++ b/source/extensions/transport_sockets/proxy_protocol/config.cc @@ -27,7 +27,7 @@ UpstreamProxyProtocolSocketConfigFactory::createTransportSocketFactory( auto inner_transport_factory = inner_config_factory.createTransportSocketFactory(*inner_factory_config, context); return std::make_unique( - std::move(inner_transport_factory), outer_config.config(), context.scope()); + std::move(inner_transport_factory), outer_config.config(), context.statsScope()); } ProtobufTypes::MessagePtr UpstreamProxyProtocolSocketConfigFactory::createEmptyConfigProto() { diff --git a/source/extensions/transport_sockets/tap/config.cc b/source/extensions/transport_sockets/tap/config.cc index b84b173d2d36..3b565bee92e9 100644 --- a/source/extensions/transport_sockets/tap/config.cc +++ b/source/extensions/transport_sockets/tap/config.cc @@ -44,11 +44,14 @@ UpstreamTapSocketConfigFactory::createTransportSocketFactory( outer_config.transport_socket(), context.messageValidationVisitor(), inner_config_factory); auto inner_transport_factory = inner_config_factory.createTransportSocketFactory(*inner_factory_config, context); + + auto& server_context = context.serverFactoryContext(); return std::make_unique( outer_config, - std::make_unique(context.mainThreadDispatcher().timeSource()), - context.admin(), context.singletonManager(), context.threadLocal(), - context.mainThreadDispatcher(), std::move(inner_transport_factory)); + std::make_unique( + server_context.mainThreadDispatcher().timeSource()), + server_context.admin(), server_context.singletonManager(), server_context.threadLocal(), + server_context.mainThreadDispatcher(), std::move(inner_transport_factory)); } Network::DownstreamTransportSocketFactoryPtr @@ -65,11 +68,13 @@ DownstreamTapSocketConfigFactory::createTransportSocketFactory( outer_config.transport_socket(), context.messageValidationVisitor(), inner_config_factory); auto inner_transport_factory = inner_config_factory.createTransportSocketFactory( *inner_factory_config, context, server_names); + auto& server_context = context.serverFactoryContext(); return std::make_unique( outer_config, - std::make_unique(context.mainThreadDispatcher().timeSource()), - context.admin(), context.singletonManager(), context.threadLocal(), - context.mainThreadDispatcher(), std::move(inner_transport_factory)); + std::make_unique( + server_context.mainThreadDispatcher().timeSource()), + server_context.admin(), server_context.singletonManager(), server_context.threadLocal(), + server_context.mainThreadDispatcher(), std::move(inner_transport_factory)); } ProtobufTypes::MessagePtr TapSocketConfigFactory::createEmptyConfigProto() { diff --git a/source/extensions/transport_sockets/tcp_stats/config.cc b/source/extensions/transport_sockets/tcp_stats/config.cc index aafc940cb01f..0e362f959b1e 100644 --- a/source/extensions/transport_sockets/tcp_stats/config.cc +++ b/source/extensions/transport_sockets/tcp_stats/config.cc @@ -16,7 +16,7 @@ TcpStatsSocketFactory::TcpStatsSocketFactory( Server::Configuration::TransportSocketFactoryContext& context, const envoy::extensions::transport_sockets::tcp_stats::v3::Config& config) { #if defined(__linux__) - config_ = std::make_shared(config, context.scope()); + config_ = std::make_shared(config, context.statsScope()); #else UNREFERENCED_PARAMETER(config); UNREFERENCED_PARAMETER(context); diff --git a/source/extensions/transport_sockets/tls/config.cc b/source/extensions/transport_sockets/tls/config.cc index 75cc7b34a4ba..f2612251584d 100644 --- a/source/extensions/transport_sockets/tls/config.cc +++ b/source/extensions/transport_sockets/tls/config.cc @@ -20,8 +20,8 @@ Network::UpstreamTransportSocketFactoryPtr UpstreamSslSocketFactory::createTrans const envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext&>( message, context.messageValidationVisitor()), context); - return std::make_unique(std::move(client_config), - context.sslContextManager(), context.scope()); + return std::make_unique( + std::move(client_config), context.sslContextManager(), context.statsScope()); } ProtobufTypes::MessagePtr UpstreamSslSocketFactory::createEmptyConfigProto() { @@ -41,7 +41,7 @@ DownstreamSslSocketFactory::createTransportSocketFactory( message, context.messageValidationVisitor()), context); return std::make_unique( - std::move(server_config), context.sslContextManager(), context.scope(), server_names); + std::move(server_config), context.sslContextManager(), context.statsScope(), server_names); } ProtobufTypes::MessagePtr DownstreamSslSocketFactory::createEmptyConfigProto() { diff --git a/source/extensions/transport_sockets/tls/context_config_impl.cc b/source/extensions/transport_sockets/tls/context_config_impl.cc index f6a7e30a4f35..021c1d881fea 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.cc +++ b/source/extensions/transport_sockets/tls/context_config_impl.cc @@ -173,8 +173,9 @@ ContextConfigImpl::ContextConfigImpl( const unsigned default_min_protocol_version, const unsigned default_max_protocol_version, const std::string& default_cipher_suites, const std::string& default_curves, Server::Configuration::TransportSocketFactoryContext& factory_context) - : api_(factory_context.api()), options_(factory_context.options()), - singleton_manager_(factory_context.singletonManager()), + : api_(factory_context.serverFactoryContext().api()), + options_(factory_context.serverFactoryContext().options()), + singleton_manager_(factory_context.serverFactoryContext().singletonManager()), alpn_protocols_(RepeatedPtrUtil::join(config.alpn_protocols(), ",")), cipher_suites_(StringUtil::nonEmptyStringOrDefault( RepeatedPtrUtil::join(config.tls_params().cipher_suites(), ":"), default_cipher_suites)), diff --git a/source/extensions/transport_sockets/tls/context_config_impl.h b/source/extensions/transport_sockets/tls/context_config_impl.h index da3a526d163c..23ca329abe8e 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.h +++ b/source/extensions/transport_sockets/tls/context_config_impl.h @@ -46,7 +46,7 @@ class ContextConfigImpl : public virtual Ssl::ContextConfig { const Network::Address::IpList& tlsKeyLogRemote() const override { return tls_keylog_remote_; }; const std::string& tlsKeyLogPath() const override { return tls_keylog_path_; }; AccessLog::AccessLogManager& accessLogManager() const override { - return factory_context_.accessLogManager(); + return factory_context_.serverFactoryContext().accessLogManager(); } bool isReady() const override { diff --git a/source/extensions/upstreams/http/config.h b/source/extensions/upstreams/http/config.h index 93879e9deb6b..9e1d4c78283b 100644 --- a/source/extensions/upstreams/http/config.h +++ b/source/extensions/upstreams/http/config.h @@ -71,7 +71,7 @@ class ProtocolOptionsConfigFactory : public Server::Configuration::ProtocolOptio const envoy::extensions::upstreams::http::v3::HttpProtocolOptions&>( config, context.messageValidationVisitor()); return std::make_shared(typed_config, - context.getServerFactoryContext()); + context.serverFactoryContext()); } std::string category() const override { return "envoy.upstream_options"; } std::string name() const override { diff --git a/source/server/server.h b/source/server/server.h index d9c9f175a6dd..d5140d0e895a 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -191,6 +191,7 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, ThreadLocal::Instance& threadLocal() override { return server_.threadLocal(); } OptRef admin() override { return server_.admin(); } TimeSource& timeSource() override { return api().timeSource(); } + AccessLog::AccessLogManager& accessLogManager() override { return server_.accessLogManager(); } Api::Api& api() override { return server_.api(); } Grpc::Context& grpcContext() override { return server_.grpcContext(); } Router::Context& routerContext() override { return server_.routerContext(); } @@ -200,12 +201,11 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return server_.bootstrap(); } // Configuration::TransportSocketFactoryContext - ServerFactoryContext& getServerFactoryContext() override { return *this; }; + ServerFactoryContext& serverFactoryContext() override { return *this; } Ssl::ContextManager& sslContextManager() override { return server_.sslContextManager(); } Secret::SecretManager& secretManager() override { return server_.secretManager(); } - Stats::Store& stats() override { return server_.stats(); } + Stats::Scope& statsScope() override { return *server_scope_; } Init::Manager& initManager() override { return server_.initManager(); } - AccessLog::AccessLogManager& accessLogManager() override { return server_.accessLogManager(); } ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { // Server has two message validation visitors, one for static and // other for dynamic configuration. Choose the dynamic validation diff --git a/source/server/transport_socket_config_impl.h b/source/server/transport_socket_config_impl.h index edbb65282e19..87f9aaeac244 100644 --- a/source/server/transport_socket_config_impl.h +++ b/source/server/transport_socket_config_impl.h @@ -14,11 +14,10 @@ class TransportSocketFactoryContextImpl : public TransportSocketFactoryContext { public: TransportSocketFactoryContextImpl(Server::Configuration::ServerFactoryContext& server_context, Ssl::ContextManager& context_manager, Stats::Scope& stats_scope, - Upstream::ClusterManager& cm, Stats::Store& stats, + Upstream::ClusterManager& cm, ProtobufMessage::ValidationVisitor& validation_visitor) : server_context_(server_context), context_manager_(context_manager), - stats_scope_(stats_scope), cluster_manager_(cm), stats_(stats), - validation_visitor_(validation_visitor) {} + stats_scope_(stats_scope), cluster_manager_(cm), validation_visitor_(validation_visitor) {} /** * Pass an init manager to register dynamic secret provider. @@ -27,42 +26,29 @@ class TransportSocketFactoryContextImpl : public TransportSocketFactoryContext { void setInitManager(Init::Manager& init_manager) { init_manager_ = &init_manager; } // TransportSocketFactoryContext - ServerFactoryContext& getServerFactoryContext() override { return server_context_; }; - OptRef admin() override { return server_context_.admin(); } + ServerFactoryContext& serverFactoryContext() override { return server_context_; } + Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } + ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { + return validation_visitor_; + } Ssl::ContextManager& sslContextManager() override { return context_manager_; } - Stats::Scope& scope() override { return stats_scope_; } + Stats::Scope& statsScope() override { return stats_scope_; } Secret::SecretManager& secretManager() override { return clusterManager().clusterManagerFactory().secretManager(); } - Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } - const LocalInfo::LocalInfo& localInfo() const override { return server_context_.localInfo(); } - Event::Dispatcher& mainThreadDispatcher() override { - return server_context_.mainThreadDispatcher(); - } - Stats::Store& stats() override { return stats_; } Init::Manager& initManager() override { ASSERT(init_manager_ != nullptr); return *init_manager_; } - Singleton::Manager& singletonManager() override { return server_context_.singletonManager(); } - ThreadLocal::SlotAllocator& threadLocal() override { return server_context_.threadLocal(); } - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { - return validation_visitor_; - } - Api::Api& api() override { return server_context_.api(); } - const Server::Options& options() override { return server_context_.options(); } - AccessLog::AccessLogManager& accessLogManager() override { - return server_context_.accessLogManager(); - } private: Server::Configuration::ServerFactoryContext& server_context_; Ssl::ContextManager& context_manager_; Stats::Scope& stats_scope_; Upstream::ClusterManager& cluster_manager_; - Stats::Store& stats_; - Init::Manager* init_manager_{}; ProtobufMessage::ValidationVisitor& validation_visitor_; + + Init::Manager* init_manager_{}; }; using TransportSocketFactoryContextImplPtr = std::unique_ptr; diff --git a/test/common/grpc/grpc_client_integration_test_harness.h b/test/common/grpc/grpc_client_integration_test_harness.h index dd3bb0831afe..7c1baf03a87e 100644 --- a/test/common/grpc/grpc_client_integration_test_harness.h +++ b/test/common/grpc/grpc_client_integration_test_harness.h @@ -493,7 +493,7 @@ class GrpcClientIntegrationTest : public GrpcClientIntegrationParamTest { class GrpcSslClientIntegrationTest : public GrpcClientIntegrationTest { public: GrpcSslClientIntegrationTest() { - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); } void TearDown() override { // Reset some state in the superclass before we destruct context_manager_ in our destructor, it diff --git a/test/common/quic/quic_transport_socket_factory_test.cc b/test/common/quic/quic_transport_socket_factory_test.cc index 0b28fc060978..89aa18174ab1 100644 --- a/test/common/quic/quic_transport_socket_factory_test.cc +++ b/test/common/quic/quic_transport_socket_factory_test.cc @@ -17,7 +17,7 @@ class QuicServerTransportSocketFactoryConfigTest : public Event::TestUsingSimula public: QuicServerTransportSocketFactoryConfigTest() : server_api_(Api::createApiForTest(server_stats_store_, simTime())) { - ON_CALL(context_, api()).WillByDefault(ReturnRef(*server_api_)); + ON_CALL(context_.server_context_, api()).WillByDefault(ReturnRef(*server_api_)); } void verifyQuicServerTransportSocketFactory(std::string yaml, bool expect_early_data) { diff --git a/test/common/secret/secret_manager_impl_test.cc b/test/common/secret/secret_manager_impl_test.cc index daca620733e7..ead2895c984b 100644 --- a/test/common/secret/secret_manager_impl_test.cc +++ b/test/common/secret/secret_manager_impl_test.cc @@ -270,7 +270,6 @@ TEST_F(SecretManagerImplTest, DeduplicateDynamicTlsCertificateSecretProvider) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -278,10 +277,10 @@ TEST_F(SecretManagerImplTest, DeduplicateDynamicTlsCertificateSecretProvider) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); envoy::config::core::v3::ConfigSource config_source; TestUtility::loadFromYaml(R"( @@ -353,7 +352,6 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretUpdateSuccess) { envoy::config::core::v3::ConfigSource config_source; NiceMock local_info; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -361,11 +359,11 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretUpdateSuccess) { .WillOnce(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); - EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api_)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); @@ -403,18 +401,17 @@ TEST_F(SecretManagerImplTest, SdsDynamicGenericSecret) { NiceMock secret_context; NiceMock validation_visitor; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock local_info; Init::TargetHandlePtr init_target_handle; NiceMock init_watcher; - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(*dispatcher_)); EXPECT_CALL(secret_context, messageValidationVisitor()).WillOnce(ReturnRef(validation_visitor)); - EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api_)); EXPECT_CALL(init_manager, add(_)) .WillOnce(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); @@ -452,7 +449,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandler) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -460,10 +456,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandler) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); @@ -722,7 +718,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerWarmingSecrets) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -730,10 +725,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerWarmingSecrets) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); @@ -870,7 +865,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSecrets) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -878,10 +872,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSecrets) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); const std::string tls_certificate = R"EOF( @@ -947,7 +941,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticValidationContext) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -955,10 +948,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticValidationContext) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); const std::string validation_context = R"EOF( @@ -995,7 +988,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSessionTicketsContext) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -1003,10 +995,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSessionTicketsContext) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); const std::string stek_context = R"EOF( @@ -1076,7 +1068,6 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretPrivateKeyProviderUpdateSuccess) { envoy::config::core::v3::ConfigSource config_source; NiceMock local_info; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -1084,11 +1075,11 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretPrivateKeyProviderUpdateSuccess) { .WillOnce(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); - EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api_)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 8bebd8275dc3..d52e677e5fec 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -60,6 +60,7 @@ envoy_cc_test( envoy_cc_test( name = "cluster_manager_impl_test", + size = "large", srcs = ["cluster_manager_impl_test.cc"], args = [ # Force creation of c-ares DnsResolverImpl when running test on macOS. diff --git a/test/common/upstream/hds_test.cc b/test/common/upstream/hds_test.cc index 8fdc4eead1d1..e3ee0677e7c1 100644 --- a/test/common/upstream/hds_test.cc +++ b/test/common/upstream/hds_test.cc @@ -583,7 +583,7 @@ TEST_F(HdsTest, TestSocketContext) { params.stats_.createScope(fmt::format("cluster.{}.", params.cluster_.name())); Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( params.server_context_, params.ssl_context_manager_, *scope, - params.server_context_.clusterManager(), params.stats_, + params.server_context_.clusterManager(), params.server_context_.messageValidationVisitor()); // Create a mock socket_factory for the scope of this unit test. @@ -1072,7 +1072,7 @@ TEST_F(HdsTest, TestUpdateSocketContext) { params.stats_.createScope(fmt::format("cluster.{}.", params.cluster_.name())); Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( params.server_context_, params.ssl_context_manager_, *scope, - params.server_context_.clusterManager(), params.stats_, + params.server_context_.clusterManager(), params.server_context_.messageValidationVisitor()); // Create a mock socket_factory for the scope of this unit test. diff --git a/test/extensions/clusters/eds/leds_test.cc b/test/extensions/clusters/eds/leds_test.cc index 8e30b3fbeb11..14a0fc5c0958 100644 --- a/test/extensions/clusters/eds/leds_test.cc +++ b/test/extensions/clusters/eds/leds_test.cc @@ -90,7 +90,7 @@ class LedsTest : public testing::Test { cluster_scope_ = stats_.createScope("cluster.xds_cluster."); Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( server_context_, ssl_context_manager_, *cluster_scope_, server_context_.cluster_manager_, - stats_, validation_visitor_); + validation_visitor_); // Setup LEDS subscription. EXPECT_CALL(server_context_.cluster_manager_.subscription_factory_, diff --git a/test/extensions/filters/http/oauth2/filter_test.cc b/test/extensions/filters/http/oauth2/filter_test.cc index ddf8f16f4fb5..970bbf360e81 100644 --- a/test/extensions/filters/http/oauth2/filter_test.cc +++ b/test/extensions/filters/http/oauth2/filter_test.cc @@ -207,14 +207,13 @@ TEST_F(OAuth2Test, SdsDynamicGenericSecret) { NiceMock secret_context; NiceMock local_info; Api::ApiPtr api = Api::createApiForTest(); - Stats::IsolatedStoreImpl stats; NiceMock init_manager; Init::TargetHandlePtr init_handle; NiceMock dispatcher; - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api)); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); EXPECT_CALL(secret_context, initManager()).Times(0); EXPECT_CALL(init_manager, add(_)) .WillRepeatedly(Invoke([&init_handle](const Init::Target& target) { diff --git a/test/extensions/transport_sockets/alts/alts_integration_test.cc b/test/extensions/transport_sockets/alts/alts_integration_test.cc index 570646820764..4cc0b8f435f7 100644 --- a/test/extensions/transport_sockets/alts/alts_integration_test.cc +++ b/test/extensions/transport_sockets/alts/alts_integration_test.cc @@ -160,7 +160,7 @@ class AltsIntegrationTestBase : public Event::TestUsingSimulatedTime, } }; FakeSingletonManager fsm; - ON_CALL(mock_factory_ctx, singletonManager()).WillByDefault(ReturnRef(fsm)); + ON_CALL(mock_factory_ctx.server_context_, singletonManager()).WillByDefault(ReturnRef(fsm)); UpstreamAltsTransportSocketConfigFactory factory; envoy::extensions::transport_sockets::alts::v3::Alts alts_config; diff --git a/test/extensions/transport_sockets/alts/config_test.cc b/test/extensions/transport_sockets/alts/config_test.cc index 625994ba5404..a848115facf8 100644 --- a/test/extensions/transport_sockets/alts/config_test.cc +++ b/test/extensions/transport_sockets/alts/config_test.cc @@ -19,7 +19,8 @@ namespace { TEST(UpstreamAltsConfigTest, CreateSocketFactory) { NiceMock factory_context; Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; - EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); + EXPECT_CALL(factory_context.server_context_, singletonManager()) + .WillRepeatedly(ReturnRef(singleton_manager)); UpstreamAltsTransportSocketConfigFactory factory; ProtobufTypes::MessagePtr config = factory.createEmptyConfigProto(); @@ -39,7 +40,8 @@ TEST(UpstreamAltsConfigTest, CreateSocketFactory) { TEST(DownstreamAltsConfigTest, CreateSocketFactory) { NiceMock factory_context; Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; - EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); + EXPECT_CALL(factory_context.server_context_, singletonManager()) + .WillRepeatedly(ReturnRef(singleton_manager)); DownstreamAltsTransportSocketConfigFactory factory; ProtobufTypes::MessagePtr config = factory.createEmptyConfigProto(); diff --git a/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc b/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc index 232b95c1acde..fe298d275640 100644 --- a/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc +++ b/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc @@ -269,7 +269,7 @@ void StartTlsIntegrationTest::initialize() { TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), downstream_tls_context); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); auto cfg = std::make_unique( downstream_tls_context, mock_factory_ctx); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); diff --git a/test/extensions/transport_sockets/tls/BUILD b/test/extensions/transport_sockets/tls/BUILD index 375bbb3072e3..098ba510db74 100644 --- a/test/extensions/transport_sockets/tls/BUILD +++ b/test/extensions/transport_sockets/tls/BUILD @@ -13,6 +13,7 @@ envoy_package() envoy_cc_test( name = "ssl_socket_test", + size = "large", srcs = [ "ssl_certs_test.h", "ssl_socket_test.cc", diff --git a/test/extensions/transport_sockets/tls/context_impl_test.cc b/test/extensions/transport_sockets/tls/context_impl_test.cc index 4f082a192725..8e6bed283c68 100644 --- a/test/extensions/transport_sockets/tls/context_impl_test.cc +++ b/test/extensions/transport_sockets/tls/context_impl_test.cc @@ -955,13 +955,12 @@ TEST_F(SslServerContextImplTicketTest, TicketKeySdsNotReady) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock cluster_manager; NiceMock init_manager; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); // EXPECT_CALL(factory_context_, random()).WillOnce(ReturnRef(random)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, clusterManager()).WillOnce(ReturnRef(cluster_manager)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); auto* sds_secret_configs = tls_context.mutable_session_ticket_keys_sds_secret_config(); @@ -1408,13 +1407,12 @@ TEST_F(ClientContextConfigImplTest, TlsCertificatesAndSdsConfig) { TEST_F(ClientContextConfigImplTest, SecretNotReady) { envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; NiceMock local_info; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); @@ -1440,13 +1438,12 @@ TEST_F(ClientContextConfigImplTest, ValidationContextNotReady) { client_cert->mutable_private_key()->set_filename(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_key.pem")); NiceMock local_info; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_validation_context_sds_secret_config(); sds_secret_configs->set_name("abc.com"); @@ -1847,13 +1844,12 @@ TEST_F(ServerContextConfigImplTest, MultiSdsConfig) { TEST_F(ServerContextConfigImplTest, SecretNotReady) { envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; NiceMock local_info; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); @@ -1879,13 +1875,12 @@ TEST_F(ServerContextConfigImplTest, ValidationContextNotReady) { server_cert->mutable_private_key()->set_filename(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_key.pem")); NiceMock local_info; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_validation_context_sds_secret_config(); sds_secret_configs->set_name("abc.com"); diff --git a/test/extensions/transport_sockets/tls/handshaker_factory_test.cc b/test/extensions/transport_sockets/tls/handshaker_factory_test.cc index 64b5e9d0f05d..0221a468d700 100644 --- a/test/extensions/transport_sockets/tls/handshaker_factory_test.cc +++ b/test/extensions/transport_sockets/tls/handshaker_factory_test.cc @@ -129,7 +129,7 @@ TEST_F(HandshakerFactoryTest, SetMockFunctionCb) { static_cast(custom_process_object_for_test)); NiceMock mock_factory_ctx; - EXPECT_CALL(mock_factory_ctx.api_, processContext()) + EXPECT_CALL(mock_factory_ctx.server_context_.api_, processContext()) .WillRepeatedly(Return(std::reference_wrapper(*process_context_impl))); Extensions::TransportSockets::Tls::ClientSslSocketFactory socket_factory( @@ -155,7 +155,7 @@ TEST_F(HandshakerFactoryTest, SetSpecificSslCtxOption) { static_cast(custom_process_object_for_test)); NiceMock mock_factory_ctx; - EXPECT_CALL(mock_factory_ctx.api_, processContext()) + EXPECT_CALL(mock_factory_ctx.server_context_.api_, processContext()) .WillRepeatedly(Return(std::reference_wrapper(*process_context_impl))); Extensions::TransportSockets::Tls::ClientSslSocketFactory socket_factory( @@ -181,7 +181,7 @@ TEST_F(HandshakerFactoryTest, HandshakerContextProvidesObjectsFromParentContext) static_cast(custom_process_object_for_test)); NiceMock mock_factory_ctx; - EXPECT_CALL(mock_factory_ctx.api_, processContext()) + EXPECT_CALL(mock_factory_ctx.server_context_.api_, processContext()) .WillRepeatedly(Return(std::reference_wrapper(*process_context_impl))); MockFunction mock_factory_cb; @@ -191,8 +191,8 @@ TEST_F(HandshakerFactoryTest, HandshakerContextProvidesObjectsFromParentContext) .WillOnce(WithArg<1>([&](Ssl::HandshakerFactoryContext& context) { // Check that the objects available via the context are the same ones // provided to the parent context. - EXPECT_THAT(context.api(), Ref(mock_factory_ctx.api_)); - EXPECT_THAT(context.options(), Ref(mock_factory_ctx.options_)); + EXPECT_THAT(context.api(), Ref(mock_factory_ctx.server_context_.api_)); + EXPECT_THAT(context.options(), Ref(mock_factory_ctx.server_context_.options_)); })); Extensions::TransportSockets::Tls::ClientSslSocketFactory socket_factory( @@ -289,7 +289,7 @@ TEST_F(HandshakerFactoryDownstreamTest, ServerHandshakerProvidesCertificates) { static_cast(custom_process_object_for_test)); NiceMock mock_factory_ctx; - EXPECT_CALL(mock_factory_ctx.api_, processContext()) + EXPECT_CALL(mock_factory_ctx.server_context_.api_, processContext()) .WillRepeatedly(Return(std::reference_wrapper(*process_context_impl))); Extensions::TransportSockets::Tls::ServerContextConfigImpl server_context_config( diff --git a/test/extensions/transport_sockets/tls/ssl_certs_test.h b/test/extensions/transport_sockets/tls/ssl_certs_test.h index 0fe7d2183b27..fca49c721c7d 100644 --- a/test/extensions/transport_sockets/tls/ssl_certs_test.h +++ b/test/extensions/transport_sockets/tls/ssl_certs_test.h @@ -12,7 +12,7 @@ namespace Envoy { class SslCertsTest : public testing::Test { protected: SslCertsTest() : api_(Api::createApiForTest(store_, time_system_)) { - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); } Event::SimulatedTimeSystem time_system_; diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 3a8d1bd817fc..c571da735acc 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -329,7 +329,7 @@ void testUtil(const TestUtilOptions& options) { NiceMock runtime; testing::NiceMock server_factory_context; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); // For private key method testing. NiceMock context_manager; @@ -370,7 +370,7 @@ void testUtil(const TestUtilOptions& options) { Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_tls_context, client_factory_context); @@ -684,7 +684,7 @@ void testUtilV2(const TestUtilOptionsV2& options) { testing::NiceMock server_factory_context; NiceMock runtime; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; const envoy::config::core::v3::TransportSocket& transport_socket = @@ -708,7 +708,7 @@ void testUtilV2(const TestUtilOptionsV2& options) { Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(options.clientCtxProto(), client_factory_context); @@ -3420,7 +3420,7 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, NiceMock runtime; testing::NiceMock server_factory_context; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext server_tls_context1; TestUtility::loadFromYaml(TestEnvironment::substitute(server_ctx_yaml1), server_tls_context1); @@ -3454,7 +3454,7 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_tls_context, client_factory_context); @@ -3572,7 +3572,7 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml NiceMock runtime; testing::NiceMock server_factory_context; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext server_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(server_ctx_yaml), server_tls_context); @@ -3595,7 +3595,7 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_tls_context, client_factory_context); @@ -4294,7 +4294,7 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya Api::ApiPtr server_api = Api::createApiForTest(server_stats_store, time_system_); testing::NiceMock server_factory_context; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext server_ctx_proto; TestUtility::loadFromYaml(TestEnvironment::substitute(server_ctx_yaml), server_ctx_proto); @@ -4321,7 +4321,7 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system_); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_ctx_proto, client_factory_context); @@ -5683,14 +5683,14 @@ TEST_P(SslSocketTest, OverrideApplicationProtocols) { // Validate that if downstream secrets are not yet downloaded from SDS server, Envoy creates // NotReadySslSocket object to handle downstream connection. TEST_P(SslSocketTest, DownstreamNotReadySslSocket) { - Stats::TestUtil::TestStore stats_store; - NiceMock local_info; testing::NiceMock factory_context; + NiceMock local_info; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(factory_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context, stats()).WillOnce(ReturnRef(stats_store)); + + EXPECT_CALL(factory_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); EXPECT_CALL(factory_context, initManager()).WillRepeatedly(ReturnRef(init_manager)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; @@ -5704,7 +5704,8 @@ TEST_P(SslSocketTest, DownstreamNotReadySslSocket) { ContextManagerImpl manager(time_system_); ServerSslSocketFactory server_ssl_socket_factory( - std::move(server_cfg), manager, *stats_store.rootScope(), std::vector{}); + std::move(server_cfg), manager, *factory_context.server_context_.store_.rootScope(), + std::vector{}); auto transport_socket = server_ssl_socket_factory.createDownstreamTransportSocket(); EXPECT_FALSE(transport_socket->startSecureTransport()); // Noop transport_socket->configureInitialCongestionWindow(200, std::chrono::microseconds(223)); // Noop @@ -5722,15 +5723,14 @@ TEST_P(SslSocketTest, DownstreamNotReadySslSocket) { // Validate that if upstream secrets are not yet downloaded from SDS server, Envoy creates // NotReadySslSocket object to handle upstream connection. TEST_P(SslSocketTest, UpstreamNotReadySslSocket) { - Stats::TestUtil::TestStore stats_store; NiceMock local_info; testing::NiceMock factory_context; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context, stats()).WillOnce(ReturnRef(stats_store)); + EXPECT_CALL(factory_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; auto sds_secret_configs = @@ -5742,8 +5742,8 @@ TEST_P(SslSocketTest, UpstreamNotReadySslSocket) { EXPECT_FALSE(client_cfg->isReady()); ContextManagerImpl manager(time_system_); - ClientSslSocketFactory client_ssl_socket_factory(std::move(client_cfg), manager, - *stats_store.rootScope()); + ClientSslSocketFactory client_ssl_socket_factory( + std::move(client_cfg), manager, *factory_context.server_context_.store_.rootScope()); auto transport_socket = client_ssl_socket_factory.createTransportSocket(nullptr, nullptr); EXPECT_EQ(EMPTY_STRING, transport_socket->protocol()); EXPECT_EQ(nullptr, transport_socket->ssl()); @@ -5764,17 +5764,15 @@ TEST_P(SslSocketTest, TestTransportSocketCallback) { // Make SslSocket. testing::NiceMock factory_context; - Stats::TestUtil::TestStore stats_store; - ON_CALL(factory_context, stats()).WillByDefault(ReturnRef(stats_store)); NiceMock local_info; - ON_CALL(factory_context, localInfo()).WillByDefault(ReturnRef(local_info)); + ON_CALL(factory_context.server_context_, localInfo()).WillByDefault(ReturnRef(local_info)); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; auto client_cfg = std::make_unique(tls_context, factory_context); ContextManagerImpl manager(time_system_); - ClientSslSocketFactory client_ssl_socket_factory(std::move(client_cfg), manager, - *stats_store.rootScope()); + ClientSslSocketFactory client_ssl_socket_factory( + std::move(client_cfg), manager, *factory_context.server_context_.store_.rootScope()); Network::TransportSocketPtr transport_socket = client_ssl_socket_factory.createTransportSocket(nullptr, nullptr); diff --git a/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc index 6cae75d1db06..1f01175716ea 100644 --- a/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc +++ b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc @@ -350,7 +350,8 @@ TestPrivateKeyMethodProvider::TestPrivateKeyMethodProvider( } } - std::string private_key = factory_context.api().fileSystem().fileReadToEnd(private_key_path); + std::string private_key = + factory_context.serverFactoryContext().api().fileSystem().fileReadToEnd(private_key_path); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); bssl::UniquePtr pkey(PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, nullptr)); diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 97f9710ddbda..4b45cb7e0f39 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -61,8 +61,8 @@ BaseIntegrationTest::BaseIntegrationTest(const InstanceConstSharedPtrFn& upstrea std::function above_overflow) -> Buffer::Instance* { return new Buffer::WatermarkBuffer(below_low, above_high, above_overflow); })); - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); - ON_CALL(factory_context_, scope()).WillByDefault(ReturnRef(*stats_store_.rootScope())); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_, statsScope()).WillByDefault(ReturnRef(*stats_store_.rootScope())); // Allow extension lookup by name in the integration tests. config_helper_.addRuntimeOverride("envoy.reloadable_features.no_extension_lookup_by_name", "false"); diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 5a934afd78f4..3fadbfd2af8b 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -262,8 +262,8 @@ class QuicHttpIntegrationTestBase : public HttpIntegrationTest { // Initialize the transport socket factory using a customized ssl option. ssl_client_option_.setAlpn(true).setSan(san_to_match_).setSni("lyft.com"); NiceMock context; - ON_CALL(context, api()).WillByDefault(testing::ReturnRef(*api_)); - ON_CALL(context, scope()).WillByDefault(testing::ReturnRef(stats_scope_)); + ON_CALL(context.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(context, statsScope()).WillByDefault(testing::ReturnRef(stats_scope_)); ON_CALL(context, sslContextManager()).WillByDefault(testing::ReturnRef(context_manager_)); envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport quic_transport_socket_config; diff --git a/test/integration/ssl_utility.cc b/test/integration/ssl_utility.cc index bd8e0057b067..af93ec2c3a48 100644 --- a/test/integration/ssl_utility.cc +++ b/test/integration/ssl_utility.cc @@ -117,7 +117,7 @@ createClientSslTransportSocketFactory(const ClientSslTransportOptions& options, initializeUpstreamTlsContextConfig(options, tls_context); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx, api()).WillByDefault(ReturnRef(api)); + ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(ReturnRef(api)); auto cfg = std::make_unique( tls_context, mock_factory_ctx); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); @@ -132,7 +132,7 @@ createUpstreamSslContext(ContextManager& context_manager, Api::Api& api, bool us ConfigHelper::initializeTls({}, *tls_context.mutable_common_tls_context()); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx, api()).WillByDefault(ReturnRef(api)); + ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(ReturnRef(api)); auto cfg = std::make_unique( tls_context, mock_factory_ctx); diff --git a/test/integration/utility.cc b/test/integration/utility.cc index db03dded6453..b19dd19bd264 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -135,8 +135,8 @@ IntegrationUtil::createQuicUpstreamTransportSocketFactory(Api::Api& api, Stats:: Ssl::ContextManager& context_manager, const std::string& san_to_match) { NiceMock context; - ON_CALL(context, api()).WillByDefault(testing::ReturnRef(api)); - ON_CALL(context, scope()).WillByDefault(testing::ReturnRef(*store.rootScope())); + ON_CALL(context.server_context_, api()).WillByDefault(testing::ReturnRef(api)); + ON_CALL(context, statsScope()).WillByDefault(testing::ReturnRef(*store.rootScope())); ON_CALL(context, sslContextManager()).WillByDefault(testing::ReturnRef(context_manager)); envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport quic_transport_socket_config; diff --git a/test/integration/xfcc_integration_test.h b/test/integration/xfcc_integration_test.h index def7807d94b2..c787eaa69497 100644 --- a/test/integration/xfcc_integration_test.h +++ b/test/integration/xfcc_integration_test.h @@ -40,7 +40,7 @@ class XfccIntegrationTest : public testing::TestWithParam{admin_})); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); - ON_CALL(*this, timeSource()).WillByDefault(ReturnRef(time_system_)); - ON_CALL(*this, messageValidationContext()).WillByDefault(ReturnRef(validation_context_)); - ON_CALL(*this, messageValidationVisitor()) - .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); - ON_CALL(*this, drainManager()).WillByDefault(ReturnRef(drain_manager_)); - ON_CALL(*this, statsConfig()).WillByDefault(ReturnRef(stats_config_)); - ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); - ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); - ON_CALL(*this, lifecycleNotifier()).WillByDefault(ReturnRef(lifecycle_notifier_)); - ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); -} -MockServerFactoryContext::~MockServerFactoryContext() = default; - -MockStatsConfig::MockStatsConfig() = default; -MockStatsConfig::~MockStatsConfig() = default; - -} // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/instance.h b/test/mocks/server/instance.h index e8d077a1624e..93b5d4fe43d4 100644 --- a/test/mocks/server/instance.h +++ b/test/mocks/server/instance.h @@ -2,44 +2,13 @@ #include "envoy/server/instance.h" -#include "source/common/grpc/context_impl.h" -#include "source/common/http/context_impl.h" -#include "source/common/quic/quic_stat_names.h" -#include "source/common/router/context_impl.h" -#include "source/common/stats/symbol_table.h" -#include "source/extensions/transport_sockets/tls/context_manager_impl.h" +#include "test/mocks/server/server_factory_context.h" +#include "test/mocks/server/transport_socket_factory_context.h" -#include "test/mocks/access_log/mocks.h" -#include "test/mocks/api/mocks.h" -#include "test/mocks/event/mocks.h" -#include "test/mocks/http/mocks.h" -#include "test/mocks/init/mocks.h" -#include "test/mocks/local_info/mocks.h" -#include "test/mocks/network/mocks.h" -#include "test/mocks/protobuf/mocks.h" -#include "test/mocks/runtime/mocks.h" -#include "test/mocks/secret/mocks.h" -#include "test/mocks/stats/mocks.h" -#include "test/mocks/thread_local/mocks.h" -#include "test/mocks/tracing/mocks.h" -#include "test/mocks/upstream/cluster_manager.h" - -#include "admin.h" -#include "drain_manager.h" #include "gmock/gmock.h" -#include "hot_restart.h" -#include "listener_manager.h" -#include "options.h" -#include "overload_manager.h" -#include "server_lifecycle_notifier.h" -#include "transport_socket_factory_context.h" namespace Envoy { namespace Server { -namespace Configuration { -class MockServerFactoryContext; -class MockStatsConfig; -} // namespace Configuration class MockInstance : public Instance { public: @@ -133,104 +102,5 @@ class MockInstance : public Instance { transport_socket_factory_context_; }; -namespace Configuration { -class MockStatsConfig : public virtual StatsConfig { -public: - MockStatsConfig(); - ~MockStatsConfig() override; - - MOCK_METHOD(const std::list&, sinks, (), (const)); - MOCK_METHOD(std::chrono::milliseconds, flushInterval, (), (const)); - MOCK_METHOD(bool, flushOnAdmin, (), (const)); - MOCK_METHOD(const Stats::SinkPredicates*, sinkPredicates, (), (const)); -}; - -class MockServerFactoryContext : public virtual ServerFactoryContext { -public: - MockServerFactoryContext(); - ~MockServerFactoryContext() override; - - MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); - MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); - MOCK_METHOD(Stats::Scope&, scope, ()); - MOCK_METHOD(Stats::Scope&, serverScope, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); - MOCK_METHOD(OptRef, admin, ()); - MOCK_METHOD(TimeSource&, timeSource, ()); - Event::TestTimeSystem& timeSystem() { return time_system_; } - MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); - Grpc::Context& grpcContext() override { return grpc_context_; } - Router::Context& routerContext() override { return router_context_; } - envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return bootstrap_; } - MOCK_METHOD(Server::DrainManager&, drainManager, ()); - MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); - MOCK_METHOD(StatsConfig&, statsConfig, (), ()); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); - - testing::NiceMock cluster_manager_; - testing::NiceMock dispatcher_; - testing::NiceMock drain_manager_; - testing::NiceMock local_info_; - testing::NiceMock runtime_loader_; - testing::NiceMock store_; - testing::NiceMock thread_local_; - testing::NiceMock validation_context_; - testing::NiceMock stats_config_; - testing::NiceMock access_log_manager_; - testing::NiceMock init_manager_; - testing::NiceMock lifecycle_notifier_; - - Singleton::ManagerPtr singleton_manager_; - testing::NiceMock admin_; - Event::GlobalTimeSystem time_system_; - testing::NiceMock api_; - Grpc::ContextImpl grpc_context_; - Router::ContextImpl router_context_; - envoy::config::bootstrap::v3::Bootstrap bootstrap_; - testing::NiceMock options_; -}; - -// Stateless mock ServerFactoryContext for cases where it needs to be used concurrently in different -// threads. Global state in the MockServerFactoryContext causes thread safety issues in this case. -class StatelessMockServerFactoryContext : public virtual ServerFactoryContext { -public: - StatelessMockServerFactoryContext() = default; - ~StatelessMockServerFactoryContext() override = default; - - MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); - MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); - MOCK_METHOD(Stats::Scope&, scope, ()); - MOCK_METHOD(Stats::Scope&, serverScope, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); - MOCK_METHOD(OptRef, admin, ()); - MOCK_METHOD(TimeSource&, timeSource, ()); - MOCK_METHOD(Event::TestTimeSystem&, timeSystem, ()); - MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); - MOCK_METHOD(Grpc::Context&, grpcContext, ()); - MOCK_METHOD(Router::Context&, routerContext, ()); - MOCK_METHOD(envoy::config::bootstrap::v3::Bootstrap&, bootstrap, ()); - MOCK_METHOD(Server::DrainManager&, drainManager, ()); - MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); - MOCK_METHOD(StatsConfig&, statsConfig, (), ()); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); -}; - -} // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/server_factory_context.cc b/test/mocks/server/server_factory_context.cc new file mode 100644 index 000000000000..0acd1b671908 --- /dev/null +++ b/test/mocks/server/server_factory_context.cc @@ -0,0 +1,43 @@ +#include "test/mocks/server/server_factory_context.h" + +namespace Envoy { +namespace Server { +namespace Configuration { + +using ::testing::Return; +using ::testing::ReturnRef; + +MockServerFactoryContext::MockServerFactoryContext() + : singleton_manager_(new Singleton::ManagerImpl(Thread::threadFactoryForTest())), + grpc_context_(store_.symbolTable()), router_context_(store_.symbolTable()) { + ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); + ON_CALL(*this, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); + ON_CALL(*this, drainDecision()).WillByDefault(ReturnRef(drain_manager_)); + ON_CALL(*this, localInfo()).WillByDefault(ReturnRef(local_info_)); + ON_CALL(*this, runtime()).WillByDefault(ReturnRef(runtime_loader_)); + ON_CALL(*this, scope()).WillByDefault(ReturnRef(*store_.rootScope())); + ON_CALL(*this, serverScope()).WillByDefault(ReturnRef(*store_.rootScope())); + ON_CALL(*this, singletonManager()).WillByDefault(ReturnRef(*singleton_manager_)); + ON_CALL(*this, threadLocal()).WillByDefault(ReturnRef(thread_local_)); + ON_CALL(*this, admin()).WillByDefault(Return(OptRef{admin_})); + ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); + ON_CALL(*this, timeSource()).WillByDefault(ReturnRef(time_system_)); + ON_CALL(*this, messageValidationContext()).WillByDefault(ReturnRef(validation_context_)); + ON_CALL(*this, messageValidationVisitor()) + .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); + ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); + ON_CALL(*this, drainManager()).WillByDefault(ReturnRef(drain_manager_)); + ON_CALL(*this, statsConfig()).WillByDefault(ReturnRef(stats_config_)); + ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); + ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); + ON_CALL(*this, lifecycleNotifier()).WillByDefault(ReturnRef(lifecycle_notifier_)); + ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); +} +MockServerFactoryContext::~MockServerFactoryContext() = default; + +MockStatsConfig::MockStatsConfig() = default; +MockStatsConfig::~MockStatsConfig() = default; + +} // namespace Configuration +} // namespace Server +} // namespace Envoy diff --git a/test/mocks/server/server_factory_context.h b/test/mocks/server/server_factory_context.h new file mode 100644 index 000000000000..77a2610680c0 --- /dev/null +++ b/test/mocks/server/server_factory_context.h @@ -0,0 +1,140 @@ +#pragma once + +#include "envoy/server/factory_context.h" + +#include "source/common/grpc/context_impl.h" +#include "source/common/http/context_impl.h" +#include "source/common/quic/quic_stat_names.h" +#include "source/common/router/context_impl.h" +#include "source/common/stats/symbol_table.h" +#include "source/extensions/transport_sockets/tls/context_manager_impl.h" + +#include "test/mocks/access_log/mocks.h" +#include "test/mocks/api/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/secret/mocks.h" +#include "test/mocks/server/admin.h" +#include "test/mocks/server/drain_manager.h" +#include "test/mocks/server/hot_restart.h" +#include "test/mocks/server/listener_manager.h" +#include "test/mocks/server/options.h" +#include "test/mocks/server/overload_manager.h" +#include "test/mocks/server/server_lifecycle_notifier.h" +#include "test/mocks/stats/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Server { +namespace Configuration { + +class MockStatsConfig : public virtual StatsConfig { +public: + MockStatsConfig(); + ~MockStatsConfig() override; + + MOCK_METHOD(const std::list&, sinks, (), (const)); + MOCK_METHOD(std::chrono::milliseconds, flushInterval, (), (const)); + MOCK_METHOD(bool, flushOnAdmin, (), (const)); + MOCK_METHOD(const Stats::SinkPredicates*, sinkPredicates, (), (const)); +}; + +class MockServerFactoryContext : public virtual ServerFactoryContext { +public: + MockServerFactoryContext(); + ~MockServerFactoryContext() override; + + MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); + MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); + MOCK_METHOD(const Server::Options&, options, ()); + MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); + MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); + MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); + MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(Stats::Scope&, serverScope, ()); + MOCK_METHOD(Singleton::Manager&, singletonManager, ()); + MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); + MOCK_METHOD(OptRef, admin, ()); + MOCK_METHOD(TimeSource&, timeSource, ()); + Event::TestTimeSystem& timeSystem() { return time_system_; } + MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); + MOCK_METHOD(Api::Api&, api, ()); + Grpc::Context& grpcContext() override { return grpc_context_; } + Router::Context& routerContext() override { return router_context_; } + envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return bootstrap_; } + MOCK_METHOD(Server::DrainManager&, drainManager, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); + MOCK_METHOD(StatsConfig&, statsConfig, (), ()); + MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); + + testing::NiceMock cluster_manager_; + testing::NiceMock dispatcher_; + testing::NiceMock drain_manager_; + testing::NiceMock local_info_; + testing::NiceMock runtime_loader_; + testing::NiceMock store_; + testing::NiceMock thread_local_; + testing::NiceMock validation_context_; + testing::NiceMock stats_config_; + testing::NiceMock access_log_manager_; + testing::NiceMock init_manager_; + testing::NiceMock lifecycle_notifier_; + + Singleton::ManagerPtr singleton_manager_; + testing::NiceMock admin_; + Event::GlobalTimeSystem time_system_; + testing::NiceMock api_; + Grpc::ContextImpl grpc_context_; + Router::ContextImpl router_context_; + envoy::config::bootstrap::v3::Bootstrap bootstrap_; + testing::NiceMock options_; +}; + +// Stateless mock ServerFactoryContext for cases where it needs to be used concurrently in different +// threads. Global state in the MockServerFactoryContext causes thread safety issues in this case. +class StatelessMockServerFactoryContext : public virtual ServerFactoryContext { +public: + StatelessMockServerFactoryContext() = default; + ~StatelessMockServerFactoryContext() override = default; + + MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); + MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); + MOCK_METHOD(const Server::Options&, options, ()); + MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); + MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); + MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); + MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(Stats::Scope&, serverScope, ()); + MOCK_METHOD(Singleton::Manager&, singletonManager, ()); + MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); + MOCK_METHOD(OptRef, admin, ()); + MOCK_METHOD(TimeSource&, timeSource, ()); + MOCK_METHOD(Event::TestTimeSystem&, timeSystem, ()); + MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); + MOCK_METHOD(Api::Api&, api, ()); + MOCK_METHOD(Grpc::Context&, grpcContext, ()); + MOCK_METHOD(Router::Context&, routerContext, ()); + MOCK_METHOD(envoy::config::bootstrap::v3::Bootstrap&, bootstrap, ()); + MOCK_METHOD(Server::DrainManager&, drainManager, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); + MOCK_METHOD(StatsConfig&, statsConfig, (), ()); + MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); +}; + +} // namespace Configuration +} // namespace Server +} // namespace Envoy diff --git a/test/mocks/server/transport_socket_factory_context.cc b/test/mocks/server/transport_socket_factory_context.cc index 2f45814530f8..4de5eff68b6a 100644 --- a/test/mocks/server/transport_socket_factory_context.cc +++ b/test/mocks/server/transport_socket_factory_context.cc @@ -1,4 +1,4 @@ -#include "transport_socket_factory_context.h" +#include "test/mocks/server/transport_socket_factory_context.h" #include @@ -12,17 +12,13 @@ namespace Configuration { using ::testing::ReturnRef; MockTransportSocketFactoryContext::MockTransportSocketFactoryContext() - : secret_manager_(std::make_unique(config_tracker_)), - singleton_manager_(Thread::threadFactoryForTest()) { + : secret_manager_(std::make_unique(config_tracker_)) { + ON_CALL(*this, serverFactoryContext()).WillByDefault(ReturnRef(server_context_)); ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); ON_CALL(*this, messageValidationVisitor()) .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); ON_CALL(*this, sslContextManager()).WillByDefault(ReturnRef(context_manager_)); - ON_CALL(*this, scope()).WillByDefault(ReturnRef(*store_.rootScope())); - ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); - ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); - ON_CALL(*this, singletonManager()).WillByDefault(ReturnRef(singleton_manager_)); + ON_CALL(*this, statsScope()).WillByDefault(ReturnRef(*store_.rootScope())); } MockTransportSocketFactoryContext::~MockTransportSocketFactoryContext() = default; diff --git a/test/mocks/server/transport_socket_factory_context.h b/test/mocks/server/transport_socket_factory_context.h index ee23330262d7..fc43416a6560 100644 --- a/test/mocks/server/transport_socket_factory_context.h +++ b/test/mocks/server/transport_socket_factory_context.h @@ -7,16 +7,17 @@ #include "test/mocks/access_log/mocks.h" #include "test/mocks/api/mocks.h" #include "test/mocks/server/options.h" +#include "test/mocks/server/server_factory_context.h" #include "test/mocks/ssl/mocks.h" #include "test/mocks/stats/mocks.h" #include "test/mocks/upstream/cluster_manager.h" -#include "config_tracker.h" #include "gmock/gmock.h" namespace Envoy { namespace Server { namespace Configuration { + class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { public: MockTransportSocketFactoryContext(); @@ -24,32 +25,19 @@ class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { Secret::SecretManager& secretManager() override { return *(secret_manager_); } - MOCK_METHOD(ServerFactoryContext&, getServerFactoryContext, ()); - MOCK_METHOD(OptRef, admin, ()); - MOCK_METHOD(Ssl::ContextManager&, sslContextManager, ()); - MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(ServerFactoryContext&, serverFactoryContext, ()); MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); - MOCK_METHOD(Envoy::Random::RandomGenerator&, random, ()); - MOCK_METHOD(Stats::Store&, stats, ()); - MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(ThreadLocal::SlotAllocator&, threadLocal, ()); MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, ()); + MOCK_METHOD(Ssl::ContextManager&, sslContextManager, ()); + MOCK_METHOD(Stats::Scope&, statsScope, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + testing::NiceMock server_context_; testing::NiceMock cluster_manager_; - testing::NiceMock api_; testing::NiceMock config_tracker_; testing::NiceMock context_manager_; testing::NiceMock store_; - testing::NiceMock options_; std::unique_ptr secret_manager_; - testing::NiceMock access_log_manager_; - Singleton::ManagerImpl singleton_manager_; }; } // namespace Configuration } // namespace Server From aa50e71f82cafe364c9e26c69bce54a67a12645e Mon Sep 17 00:00:00 2001 From: Tony Han Date: Mon, 8 May 2023 23:34:42 +0800 Subject: [PATCH 151/740] fix(router): update scope_key_builder when srds config is updated (#26860) * fix(router): update scope_key_builder when srds config is updated Signed-off-by: Bing Han --- changelogs/current.yaml | 5 + envoy/router/scopes.h | 32 ++- source/common/http/BUILD | 1 + source/common/http/conn_manager_config.h | 7 + source/common/http/conn_manager_impl.cc | 30 ++- source/common/http/conn_manager_impl.h | 4 +- source/common/router/BUILD | 1 + source/common/router/scoped_config_impl.cc | 13 +- source/common/router/scoped_config_impl.h | 20 +- source/common/router/scoped_rds.cc | 51 +++-- source/common/router/scoped_rds.h | 24 +- .../network/http_connection_manager/BUILD | 1 + .../network/http_connection_manager/config.cc | 1 + .../network/http_connection_manager/config.h | 7 + source/server/admin/admin.h | 12 + .../http/conn_manager_impl_fuzz_test.cc | 7 + test/common/http/conn_manager_impl_test_2.cc | 47 +++- .../common/http/conn_manager_impl_test_base.h | 7 + test/common/router/scoped_config_impl_test.cc | 61 ++++-- test/common/router/scoped_rds_test.cc | 207 +++++++++++++----- test/integration/scoped_rds.h | 38 ++++ .../scoped_rds_integration_test.cc | 106 +++++++++ test/mocks/http/mocks.h | 1 + test/mocks/router/mocks.cc | 6 + test/mocks/router/mocks.h | 10 +- 25 files changed, 519 insertions(+), 180 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 0414e2f19940..557090b2e8fd 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -61,6 +61,11 @@ bug_fixes: - area: tls change: | Fix build FIPS compliance when using both FIPS mode and Wasm extensions (``--define boringssl=fips`` and ``--define wasm=v8``). +- area: router + change: | + Fixed the bug that updating :ref:`scope_key_builder + ` + of SRDS config doesn't work and multiple HCM share the same ``scope_key_builder``. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/envoy/router/scopes.h b/envoy/router/scopes.h index a7949facb92e..47ba039756eb 100644 --- a/envoy/router/scopes.h +++ b/envoy/router/scopes.h @@ -80,29 +80,39 @@ class StringKeyFragment : public ScopeKeyFragmentBase { }; /** - * The scoped routing configuration. + * The scoped key builder. */ -class ScopedConfig : public Envoy::Config::ConfigProvider::Config { +class ScopeKeyBuilder { public: - ~ScopedConfig() override = default; + virtual ~ScopeKeyBuilder() = default; /** - * Based on the incoming HTTP request headers, returns the configuration to use for selecting a - * target route. + * Based on the incoming HTTP request headers, returns the hash value of its scope key. * @param headers the request headers to match the scoped routing configuration against. - * @return ConfigConstSharedPtr the router's Config matching the request headers. + * @return unique_ptr of the scope key computed from header. */ - virtual ConfigConstSharedPtr getRouteConfig(const Http::HeaderMap& headers) const PURE; + virtual ScopeKeyPtr computeScopeKey(const Http::HeaderMap&) const PURE; +}; + +/** + * The scoped routing configuration. + */ +class ScopedConfig : public Envoy::Config::ConfigProvider::Config { +public: + ~ScopedConfig() override = default; /** - * Based on the incoming HTTP request headers, returns the hash value of its scope key. - * @param headers the request headers to match the scoped routing configuration against. - * @return unique_ptr of the scope key computed from header. + * Based on the scope key, returns the configuration to use for selecting a target route. + * The scope key can be got via ScopeKeyBuilder. + * + * @param scope_key the scope key. null config will be returned when null. + * @return ConfigConstSharedPtr the router's Config matching the request headers. */ - virtual ScopeKeyPtr computeScopeKey(const Http::HeaderMap&) const { return {}; } + virtual ConfigConstSharedPtr getRouteConfig(const ScopeKeyPtr& scope_key) const PURE; }; using ScopedConfigConstSharedPtr = std::shared_ptr; +using ScopeKeyBuilderPtr = std::unique_ptr; } // namespace Router } // namespace Envoy diff --git a/source/common/http/BUILD b/source/common/http/BUILD index 01e1661768d7..2c04364a7f2b 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -252,6 +252,7 @@ envoy_cc_library( "//envoy/http:original_ip_detection_interface", "//envoy/http:request_id_extension_interface", "//envoy/router:rds_interface", + "//envoy/router:scopes_interface", "//source/common/local_reply:local_reply_lib", "//source/common/network:utility_lib", "//source/common/stats:symbol_table_lib", diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 427131f4b14a..f084b0d95836 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -8,6 +8,7 @@ #include "envoy/http/original_ip_detection.h" #include "envoy/http/request_id_extension.h" #include "envoy/router/rds.h" +#include "envoy/router/scopes.h" #include "envoy/stats/scope.h" #include "envoy/tracing/tracer.h" #include "envoy/type/v3/percent.pb.h" @@ -339,6 +340,12 @@ class ConnectionManagerConfig { */ virtual Config::ConfigProvider* scopedRouteConfigProvider() PURE; + /** + * @return OptRef the scope key builder to calculate the scope key. + * This will return nullptr when scoped routing is not enabled. + */ + virtual OptRef scopeKeyBuilder() PURE; + /** * @return const std::string& the server name to write into responses. */ diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 041d002760a5..276b255eaebc 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -670,9 +670,8 @@ void ConnectionManagerImpl::RdsRouteConfigUpdateRequester::requestRouteConfigUpd const auto& host_header = absl::AsciiStrToLower(parent_.request_headers_->getHostValue()); requestVhdsUpdate(host_header, thread_local_dispatcher, std::move(route_config_updated_cb)); return; - } else if (parent_.snapped_scoped_routes_config_ != nullptr) { - Router::ScopeKeyPtr scope_key = - parent_.snapped_scoped_routes_config_->computeScopeKey(*parent_.request_headers_); + } else if (scope_key_builder_.has_value()) { + Router::ScopeKeyPtr scope_key = scope_key_builder_->computeScopeKey(*parent_.request_headers_); // If scope_key is not null, the scope exists but RouteConfiguration is not initialized. if (scope_key != nullptr) { requestSrdsUpdate(std::move(scope_key), thread_local_dispatcher, @@ -757,10 +756,13 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect connection_manager.config_.makeHeaderValidator(connection_manager.codec_->protocol())) { ASSERT(!connection_manager.config_.isRoutable() || ((connection_manager.config_.routeConfigProvider() == nullptr && - connection_manager.config_.scopedRouteConfigProvider() != nullptr) || + connection_manager.config_.scopedRouteConfigProvider() != nullptr && + connection_manager.config_.scopeKeyBuilder().has_value()) || (connection_manager.config_.routeConfigProvider() != nullptr && - connection_manager.config_.scopedRouteConfigProvider() == nullptr)), - "Either routeConfigProvider or scopedRouteConfigProvider should be set in " + connection_manager.config_.scopedRouteConfigProvider() == nullptr && + !connection_manager.config_.scopeKeyBuilder().has_value())), + "Either routeConfigProvider or (scopedRouteConfigProvider and scopeKeyBuilder) should be " + "set in " "ConnectionManagerImpl."); for (const AccessLog::InstanceSharedPtr& access_log : connection_manager_.config_.accessLogs()) { filter_manager_.addAccessLogHandler(access_log); @@ -775,10 +777,12 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect std::make_unique( connection_manager.config_.routeConfigProvider(), *this); } else if (connection_manager_.config_.isRoutable() && - connection_manager.config_.scopedRouteConfigProvider() != nullptr) { + connection_manager.config_.scopedRouteConfigProvider() != nullptr && + connection_manager.config_.scopeKeyBuilder().has_value()) { route_config_update_requester_ = std::make_unique( - connection_manager.config_.scopedRouteConfigProvider(), *this); + connection_manager.config_.scopedRouteConfigProvider(), + connection_manager.config_.scopeKeyBuilder(), *this); } ScopeTrackerScopeState scope(this, connection_manager_.read_callbacks_->connection().dispatcher()); @@ -1094,7 +1098,8 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt if (connection_manager_.config_.isRoutable()) { if (connection_manager_.config_.routeConfigProvider() != nullptr) { snapped_route_config_ = connection_manager_.config_.routeConfigProvider()->configCast(); - } else if (connection_manager_.config_.scopedRouteConfigProvider() != nullptr) { + } else if (connection_manager_.config_.scopedRouteConfigProvider() != nullptr && + connection_manager_.config_.scopeKeyBuilder().has_value()) { snapped_scoped_routes_config_ = connection_manager_.config_.scopedRouteConfigProvider()->config(); snapScopedRouteConfig(); @@ -1406,7 +1411,9 @@ void ConnectionManagerImpl::startDrainSequence() { void ConnectionManagerImpl::ActiveStream::snapScopedRouteConfig() { // NOTE: if a RDS subscription hasn't got a RouteConfiguration back, a Router::NullConfigImpl is // returned, in that case we let it pass. - snapped_route_config_ = snapped_scoped_routes_config_->getRouteConfig(*request_headers_); + auto scope_key = + connection_manager_.config_.scopeKeyBuilder()->computeScopeKey(*request_headers_); + snapped_route_config_ = snapped_scoped_routes_config_->getRouteConfig(scope_key); if (snapped_route_config_ == nullptr) { ENVOY_STREAM_LOG(trace, "can't find SRDS scope.", *this); // TODO(stevenzzzz): Consider to pass an error message to router filter, so that it can @@ -1515,7 +1522,8 @@ void ConnectionManagerImpl::ActiveStream::refreshCachedRoute(const Router::Route Router::RouteConstSharedPtr route; if (request_headers_ != nullptr) { if (connection_manager_.config_.isRoutable() && - connection_manager_.config_.scopedRouteConfigProvider() != nullptr) { + connection_manager_.config_.scopedRouteConfigProvider() != nullptr && + connection_manager_.config_.scopeKeyBuilder().has_value()) { // NOTE: re-select scope as well in case the scope key header has been changed by a filter. snapScopedRouteConfig(); } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index fa693a067e65..29b8c01b2673 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -126,6 +126,7 @@ class ConnectionManagerImpl : Logger::Loggable, : route_config_provider_(route_config_provider), parent_(parent) {} RdsRouteConfigUpdateRequester(Config::ConfigProvider* scoped_route_config_provider, + OptRef scope_key_builder, ActiveStream& parent) // Expect the dynamic cast to succeed because only ScopedRdsConfigProvider is fully // implemented. Inline provider will be cast to nullptr here but it is not full implemented @@ -133,7 +134,7 @@ class ConnectionManagerImpl : Logger::Loggable, // functional inline scope route provider in the future. : scoped_route_config_provider_( dynamic_cast(scoped_route_config_provider)), - parent_(parent) {} + scope_key_builder_(scope_key_builder), parent_(parent) {} void requestRouteConfigUpdate(Http::RouteConfigUpdatedCallbackSharedPtr route_config_updated_cb); @@ -147,6 +148,7 @@ class ConnectionManagerImpl : Logger::Loggable, private: Router::RouteConfigProvider* route_config_provider_; Router::ScopedRdsConfigProvider* scoped_route_config_provider_; + OptRef scope_key_builder_; ActiveStream& parent_; }; diff --git a/source/common/router/BUILD b/source/common/router/BUILD index d3448de674d4..bbff165c3c65 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -226,6 +226,7 @@ envoy_cc_library( "//envoy/config:config_provider_interface", "//envoy/config:subscription_interface", "//envoy/router:route_config_provider_manager_interface", + "//envoy/router:scopes_interface", "//envoy/stats:stats_interface", "//source/common/common:assert_lib", "//source/common/common:cleanup_lib", diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 51697ac125ac..45a16d9c4c46 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -145,9 +145,7 @@ void ScopedConfigImpl::removeRoutingScopes(const std::vector& scope } } -Router::ConfigConstSharedPtr -ScopedConfigImpl::getRouteConfig(const Http::HeaderMap& headers) const { - ScopeKeyPtr scope_key = scope_key_builder_.computeScopeKey(headers); +Router::ConfigConstSharedPtr ScopedConfigImpl::getRouteConfig(const ScopeKeyPtr& scope_key) const { if (scope_key == nullptr) { return nullptr; } @@ -158,14 +156,5 @@ ScopedConfigImpl::getRouteConfig(const Http::HeaderMap& headers) const { return nullptr; } -ScopeKeyPtr ScopedConfigImpl::computeScopeKey(const Http::HeaderMap& headers) const { - ScopeKeyPtr scope_key = scope_key_builder_.computeScopeKey(headers); - if (scope_key && - scoped_route_info_by_key_.find(scope_key->hash()) != scoped_route_info_by_key_.end()) { - return scope_key; - } - return nullptr; -} - } // namespace Router } // namespace Envoy diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index a036fb393cc0..d7f8bd158ffa 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -55,14 +55,10 @@ class HeaderValueExtractorImpl : public FragmentBuilderBase { /** * Base class for ScopeKeyBuilder implementations. */ -class ScopeKeyBuilderBase { +class ScopeKeyBuilderBase : public ScopeKeyBuilder { public: explicit ScopeKeyBuilderBase(ScopedRoutes::ScopeKeyBuilder&& config) : config_(std::move(config)) {} - virtual ~ScopeKeyBuilderBase() = default; - - // Computes scope key for given headers, returns nullptr if a key can't be computed. - virtual ScopeKeyPtr computeScopeKey(const Http::HeaderMap& headers) const PURE; protected: const ScopedRoutes::ScopeKeyBuilder config_; @@ -111,12 +107,9 @@ using ScopedRouteMap = std::map; */ class ScopedConfigImpl : public ScopedConfig { public: - explicit ScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) - : scope_key_builder_(std::move(scope_key_builder)) {} + ScopedConfigImpl() = default; - ScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder, - const std::vector& scoped_route_infos) - : scope_key_builder_(std::move(scope_key_builder)) { + ScopedConfigImpl(const std::vector& scoped_route_infos) { addOrUpdateRoutingScopes(scoped_route_infos); } @@ -126,12 +119,9 @@ class ScopedConfigImpl : public ScopedConfig { void removeRoutingScopes(const std::vector& scope_names); // Envoy::Router::ScopedConfig - Router::ConfigConstSharedPtr getRouteConfig(const Http::HeaderMap& headers) const override; - // The return value is not null only if the scope corresponding to the header exists. - ScopeKeyPtr computeScopeKey(const Http::HeaderMap& headers) const override; + Router::ConfigConstSharedPtr getRouteConfig(const ScopeKeyPtr& scope_key) const override; private: - ScopeKeyBuilderImpl scope_key_builder_; // From scope name to cached ScopedRouteInfo. absl::flat_hash_map scoped_route_info_by_name_; // Hash by ScopeKey hash to lookup in constant time. @@ -143,7 +133,7 @@ class ScopedConfigImpl : public ScopedConfig { */ class NullScopedConfigImpl : public ScopedConfig { public: - Router::ConfigConstSharedPtr getRouteConfig(const Http::HeaderMap&) const override { + Router::ConfigConstSharedPtr getRouteConfig(const ScopeKeyPtr&) const override { return std::make_shared(); } }; diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 9cefdbd61d67..4ad5fc00f3a7 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -60,17 +60,17 @@ ConfigProviderPtr create( envoy::config::route::v3::ScopedRouteConfiguration, ProtobufTypes::ConstMessagePtrVector>(scoped_route_list.scoped_route_configurations()), factory_context, - ScopedRoutesConfigProviderManagerOptArg( - config.scoped_routes().name(), config.scoped_routes().rds_config_source(), - config.scoped_routes().scope_key_builder(), optional_http_filters)); + ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), + config.scoped_routes().rds_config_source(), + optional_http_filters)); } case envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: ConfigSpecifierCase::kScopedRds: return scoped_routes_config_provider_manager.createXdsConfigProvider( config.scoped_routes().scoped_rds(), factory_context, init_manager, stat_prefix, - ScopedRoutesConfigProviderManagerOptArg( - config.scoped_routes().name(), config.scoped_routes().rds_config_source(), - config.scoped_routes().scope_key_builder(), optional_http_filters)); + ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), + config.scoped_routes().rds_config_source(), + optional_http_filters)); case envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: ConfigSpecifierCase::CONFIG_SPECIFIER_NOT_SET: PANIC("not implemented"); @@ -78,6 +78,16 @@ ConfigProviderPtr create( PANIC_DUE_TO_CORRUPT_ENUM; } +ScopeKeyBuilderPtr createScopeKeyBuilder( + const envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + config) { + ASSERT(config.route_specifier_case() == + envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: + RouteSpecifierCase::kScopedRoutes); + auto scope_key_builder = config.scoped_routes().scope_key_builder(); + return std::make_unique(std::move(scope_key_builder)); +} + } // namespace ScopedRoutesConfigProviderUtil namespace { @@ -117,8 +127,6 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( Server::Configuration::ServerFactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, envoy::config::core::v3::ConfigSource rds_config_source, - envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes::ScopeKeyBuilder - scope_key_builder, const OptionalHttpFilters& optional_http_filters) : Envoy::Config::ImmutableConfigProviderBase(factory_context, config_provider_manager, ConfigProviderInstanceType::Inline, @@ -126,17 +134,14 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( name_(std::move(name)), scopes_(makeScopedRouteInfos(std::move(config_protos), factory_context, config_provider_manager, optional_http_filters)), - config_(std::make_shared(std::move(scope_key_builder), scopes_)), + config_(std::make_shared(scopes_)), rds_config_source_(std::move(rds_config_source)) {} ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRds& scoped_rds, const OptionalHttpFilters& optional_http_filters, const uint64_t manager_identifier, - const std::string& name, - const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: - ScopeKeyBuilder& scope_key_builder, - Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, - envoy::config::core::v3::ConfigSource rds_config_source, + const std::string& name, Server::Configuration::ServerFactoryContext& factory_context, + const std::string& stat_prefix, envoy::config::core::v3::ConfigSource rds_config_source, RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager) : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, @@ -146,8 +151,8 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( factory_context_(factory_context), name_(name), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_), POOL_GAUGE(*scope_))}), - scope_key_builder_(scope_key_builder), rds_config_source_(std::move(rds_config_source)), - stat_prefix_(stat_prefix), route_config_provider_manager_(route_config_provider_manager), + rds_config_source_(std::move(rds_config_source)), stat_prefix_(stat_prefix), + route_config_provider_manager_(route_config_provider_manager), optional_http_filters_(optional_http_filters) { const auto resource_name = getResourceName(); if (scoped_rds.srds_resources_locator().empty()) { @@ -164,10 +169,9 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( *this, resource_decoder_); } - initialize([scope_key_builder]() -> Envoy::Config::ConfigProvider::ConfigConstSharedPtr { - return std::make_shared( - envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: - ScopeKeyBuilder(scope_key_builder)); + // TODO(tony612): consider not using the callback here. + initialize([]() -> Envoy::Config::ConfigProvider::ConfigConstSharedPtr { + return std::make_shared(); }); } @@ -617,8 +621,8 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( config_source_proto); return std::make_shared( scoped_rds_config_source, typed_optarg.optional_http_filters_, manager_identifier, - typed_optarg.scoped_routes_name_, typed_optarg.scope_key_builder_, factory_context, - stat_prefix, typed_optarg.rds_config_source_, + typed_optarg.scoped_routes_name_, factory_context, stat_prefix, + typed_optarg.rds_config_source_, static_cast(config_provider_manager) .routeConfigProviderManager(), static_cast(config_provider_manager)); @@ -634,8 +638,7 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( const auto& typed_optarg = static_cast(optarg); return std::make_unique( std::move(config_protos), typed_optarg.scoped_routes_name_, factory_context, *this, - typed_optarg.rds_config_source_, typed_optarg.scope_key_builder_, - typed_optarg.optional_http_filters_); + typed_optarg.rds_config_source_, typed_optarg.optional_http_filters_); } } // namespace Router diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 3e0947c4fef4..6b76b0594e64 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -10,6 +10,7 @@ #include "envoy/config/subscription.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" #include "envoy/router/route_config_provider_manager.h" +#include "envoy/router/scopes.h" #include "envoy/service/discovery/v3/discovery.pb.h" #include "envoy/stats/scope.h" @@ -34,6 +35,12 @@ Envoy::Config::ConfigProviderPtr create( const std::string& stat_prefix, Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager); +// If enabled in the HttpConnectionManager config, returns a ConfigProvider for scoped routing +// configuration. +ScopeKeyBuilderPtr createScopeKeyBuilder( + const envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + config); + } // namespace ScopedRoutesConfigProviderUtil class ScopedRoutesConfigProviderManager; @@ -51,8 +58,6 @@ class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigPr Server::Configuration::ServerFactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, envoy::config::core::v3::ConfigSource rds_config_source, - envoy::extensions::filters::network::http_connection_manager:: - v3::ScopedRoutes::ScopeKeyBuilder scope_key_builder, const OptionalHttpFilters& optional_http_filters); ~InlineScopedRoutesConfigProvider() override = default; @@ -113,11 +118,8 @@ class ScopedRdsConfigSubscription ScopedRdsConfigSubscription( const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRds& scoped_rds, const OptionalHttpFilters& optional_http_filters, const uint64_t manager_identifier, - const std::string& name, - const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: - ScopeKeyBuilder& scope_key_builder, - Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, - envoy::config::core::v3::ConfigSource rds_config_source, + const std::string& name, Server::Configuration::ServerFactoryContext& factory_context, + const std::string& stat_prefix, envoy::config::core::v3::ConfigSource rds_config_source, RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager); @@ -230,8 +232,6 @@ class ScopedRdsConfigSubscription Stats::ScopeSharedPtr scope_; ScopedRdsStats stats_; Envoy::Config::SubscriptionPtr subscription_; - const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: - ScopeKeyBuilder scope_key_builder_; const envoy::config::core::v3::ConfigSource rds_config_source_; const std::string stat_prefix_; RouteConfigProviderManager& route_config_provider_manager_; @@ -314,16 +314,12 @@ class ScopedRoutesConfigProviderManagerOptArg ScopedRoutesConfigProviderManagerOptArg( std::string scoped_routes_name, const envoy::config::core::v3::ConfigSource& rds_config_source, - const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: - ScopeKeyBuilder& scope_key_builder, const OptionalHttpFilters& optional_http_filters) : scoped_routes_name_(std::move(scoped_routes_name)), rds_config_source_(rds_config_source), - scope_key_builder_(scope_key_builder), optional_http_filters_(optional_http_filters) {} + optional_http_filters_(optional_http_filters) {} const std::string scoped_routes_name_; const envoy::config::core::v3::ConfigSource& rds_config_source_; - const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: - ScopeKeyBuilder& scope_key_builder_; const OptionalHttpFilters& optional_http_filters_; }; diff --git a/source/extensions/filters/network/http_connection_manager/BUILD b/source/extensions/filters/network/http_connection_manager/BUILD index 730d87b77cc1..0dd2d064b2f3 100644 --- a/source/extensions/filters/network/http_connection_manager/BUILD +++ b/source/extensions/filters/network/http_connection_manager/BUILD @@ -29,6 +29,7 @@ envoy_cc_extension( "//envoy/http:request_id_extension_interface", "//envoy/registry", "//envoy/router:route_config_provider_manager_interface", + "//envoy/router:scopes_interface", "//envoy/server:admin_interface", "//envoy/server:options_interface", "//envoy/stats:stats_interface", diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index a0df43f3a9f9..6b5626c8de22 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -487,6 +487,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( scoped_routes_config_provider_ = Router::ScopedRoutesConfigProviderUtil::create( config, context_.getServerFactoryContext(), context_.initManager(), stats_prefix_, scoped_routes_config_provider_manager_); + scope_key_builder_ = Router::ScopedRoutesConfigProviderUtil::createScopeKeyBuilder(config); break; case envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: RouteSpecifierCase::ROUTE_SPECIFIER_NOT_SET: diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index d1237c8e710b..cb65ca146721 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -18,6 +18,7 @@ #include "envoy/http/original_ip_detection.h" #include "envoy/http/request_id_extension.h" #include "envoy/router/route_config_provider_manager.h" +#include "envoy/router/scopes.h" #include "envoy/tracing/tracer_manager.h" #include "source/common/common/logger.h" @@ -186,6 +187,9 @@ class HttpConnectionManagerConfig : Logger::Loggable, Config::ConfigProvider* scopedRouteConfigProvider() override { return scoped_routes_config_provider_.get(); } + OptRef scopeKeyBuilder() override { + return scope_key_builder_ ? *scope_key_builder_ : OptRef{}; + } const std::string& serverName() const override { return server_name_; } HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() const override { @@ -315,6 +319,9 @@ class HttpConnectionManagerConfig : Logger::Loggable, std::chrono::milliseconds request_timeout_; std::chrono::milliseconds request_headers_timeout_; Router::RouteConfigProviderSharedPtr route_config_provider_; + // used to get scope key, then scoped_routes_config_provider_ should be used to get the scoped + // routes + Router::ScopeKeyBuilderPtr scope_key_builder_; Config::ConfigProviderPtr scoped_routes_config_provider_; std::chrono::milliseconds drain_timeout_; bool generate_request_id_; diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index d29f93e1a1ad..8706b6d47fca 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -155,6 +155,7 @@ class AdminImpl : public Admin, Config::ConfigProvider* scopedRouteConfigProvider() override { return &scoped_route_config_provider_; } + OptRef scopeKeyBuilder() override { return scope_key_builder_; } const std::string& serverName() const override { return Http::DefaultServerString::get(); } const absl::optional& schemeToSet() const override { return scheme_; } HttpConnectionManagerProto::ServerHeaderTransformation @@ -304,6 +305,16 @@ class AdminImpl : public Admin, TimeSource& time_source_; }; + /** + * Implementation of ScopeKeyBuilder that returns a null scope key. + */ + struct NullScopeKeyBuilder : public Router::ScopeKeyBuilder { + NullScopeKeyBuilder() = default; + ~NullScopeKeyBuilder() override = default; + + Router::ScopeKeyPtr computeScopeKey(const Http::HeaderMap&) const override { return nullptr; }; + }; + /** * Implementation of OverloadManager that is never overloaded. Using this instead of the real * OverloadManager keeps the admin interface accessible even when the proxy is overloaded. @@ -472,6 +483,7 @@ class AdminImpl : public Admin, Http::ConnectionManagerTracingStats tracing_stats_; NullRouteConfigProvider route_config_provider_; NullScopedRouteConfigProvider scoped_route_config_provider_; + NullScopeKeyBuilder scope_key_builder_; Server::ClustersHandler clusters_handler_; Server::ConfigDumpHandler config_dump_handler_; Server::InitDumpHandler init_dump_handler_; diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 0943f9b8d214..4bc08de034aa 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -170,6 +170,12 @@ class FuzzConfig : public ConnectionManagerConfig { } return nullptr; } + OptRef scopeKeyBuilder() override { + if (use_srds_) { + return scope_key_builder_; + } + return {}; + } const std::string& serverName() const override { return server_name_; } HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() const override { @@ -249,6 +255,7 @@ class FuzzConfig : public ConnectionManagerConfig { bool use_srds_{}; Router::MockRouteConfigProvider route_config_provider_; Router::MockScopedRouteConfigProvider scoped_route_config_provider_; + Router::MockScopeKeyBuilder scope_key_builder_; std::string server_name_; HttpConnectionManagerProto::ServerHeaderTransformation server_transformation_{ HttpConnectionManagerProto::OVERWRITE}; diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index d4cb41cae9ec..5daf2b1a45e6 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -2750,6 +2750,9 @@ TEST_F(HttpConnectionManagerImplTest, TestSrdsRouteNotFound) { setup(false, "", true, true); setupFilterChain(1, 0); // Recreate the chain for second stream. + EXPECT_CALL(*static_cast(scopeKeyBuilder().ptr()), + computeScopeKey(_)) + .Times(2); EXPECT_CALL(*static_cast( scopedRouteConfigProvider()->config().get()), getRouteConfig(_)) @@ -2782,6 +2785,9 @@ TEST_F(HttpConnectionManagerImplTest, TestSrdsRouteNotFound) { TEST_F(HttpConnectionManagerImplTest, TestSrdsUpdate) { setup(false, "", true, true); + EXPECT_CALL(*static_cast(scopeKeyBuilder().ptr()), + computeScopeKey(_)) + .Times(3); EXPECT_CALL(*static_cast( scopedRouteConfigProvider()->config().get()), getRouteConfig(_)) @@ -2842,6 +2848,17 @@ TEST_F(HttpConnectionManagerImplTest, TestSrdsCrossScopeReroute) { std::shared_ptr route2 = std::make_shared>(); EXPECT_CALL(*route_config1, route(_, _, _, _)).WillRepeatedly(Return(route1)); EXPECT_CALL(*route_config2, route(_, _, _, _)).WillRepeatedly(Return(route2)); + EXPECT_CALL(*static_cast(scopeKeyBuilder().ptr()), + computeScopeKey(_)) + .Times(3) + .WillRepeatedly(Invoke([&](const HeaderMap& headers) -> Router::ScopeKeyPtr { + auto& test_headers = dynamic_cast(headers); + if (test_headers.get_("scope_key") == "foo") { + Router::ScopeKey key; + return std::make_unique(std::move(key)); + } + return nullptr; + })); EXPECT_CALL(*static_cast( scopedRouteConfigProvider()->config().get()), getRouteConfig(_)) @@ -2849,13 +2866,13 @@ TEST_F(HttpConnectionManagerImplTest, TestSrdsCrossScopeReroute) { // 2. refreshCachedRoute (both in decodeHeaders(headers,end_stream); // 3. then refreshCachedRoute triggered by decoder_filters_[1]->callbacks_->route(). .Times(3) - .WillRepeatedly(Invoke([&](const HeaderMap& headers) -> Router::ConfigConstSharedPtr { - auto& test_headers = dynamic_cast(headers); - if (test_headers.get_("scope_key") == "foo") { - return route_config1; - } - return route_config2; - })); + .WillRepeatedly( + Invoke([&](const Router::ScopeKeyPtr& scope_key) -> Router::ConfigConstSharedPtr { + if (scope_key != nullptr) { + return route_config1; + } + return route_config2; + })); EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { decoder_ = &conn_manager_->newStream(response_encoder_); RequestHeaderMapPtr headers{new TestRequestHeaderMapImpl{ @@ -2903,6 +2920,9 @@ TEST_F(HttpConnectionManagerImplTest, TestSrdsRouteFound) { std::shared_ptr fake_cluster1 = std::make_shared>(); EXPECT_CALL(cluster_manager_, getThreadLocalCluster(_)).WillOnce(Return(fake_cluster1.get())); + EXPECT_CALL(*static_cast(scopeKeyBuilder().ptr()), + computeScopeKey(_)) + .Times(2); EXPECT_CALL(*scopedRouteConfigProvider()->config(), getRouteConfig(_)) // 1. decodeHeaders() snapping route config. // 2. refreshCachedRoute() later in the same decodeHeaders(). @@ -3116,9 +3136,13 @@ class HttpConnectionManagerImplDeathTest : public HttpConnectionManagerImplTest Config::ConfigProvider* scopedRouteConfigProvider() override { return scoped_route_config_provider2_.get(); } + OptRef scopeKeyBuilder() override { + return scope_key_builder2_ ? *scope_key_builder2_ : OptRef{}; + } std::shared_ptr route_config_provider2_; std::shared_ptr scoped_route_config_provider2_; + std::unique_ptr scope_key_builder2_; }; // HCM config can only have either RouteConfigProvider or ScopedRoutesConfigProvider. @@ -3132,8 +3156,8 @@ TEST_F(HttpConnectionManagerImplDeathTest, InvalidConnectionManagerConfig) { })); // Either RDS or SRDS should be set. EXPECT_DEBUG_DEATH(conn_manager_->onData(fake_input, false), - "Either routeConfigProvider or scopedRouteConfigProvider should be set in " - "ConnectionManagerImpl."); + "Either routeConfigProvider or \\(scopedRouteConfigProvider and " + "scopeKeyBuilder\\) should be set in ConnectionManagerImpl."); route_config_provider2_ = std::make_shared>(); @@ -3142,10 +3166,11 @@ TEST_F(HttpConnectionManagerImplDeathTest, InvalidConnectionManagerConfig) { scoped_route_config_provider2_ = std::make_shared>(); + scope_key_builder2_ = std::make_unique>(); // Can't have RDS and SRDS provider in the same time. EXPECT_DEBUG_DEATH(conn_manager_->onData(fake_input, false), - "Either routeConfigProvider or scopedRouteConfigProvider should be set in " - "ConnectionManagerImpl."); + "Either routeConfigProvider or \\(scopedRouteConfigProvider and " + "scopeKeyBuilder\\) should be set in ConnectionManagerImpl."); route_config_provider2_.reset(); // Only scoped route config provider valid. diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index 5c13b13e584d..1a00ab4548f9 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -104,6 +104,12 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { } return nullptr; } + OptRef scopeKeyBuilder() override { + if (use_srds_) { + return scope_key_builder_; + } + return {}; + } const std::string& serverName() const override { return server_name_; } HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() const override { @@ -197,6 +203,7 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { NiceMock route_config_provider_; std::shared_ptr route_config_{new NiceMock()}; NiceMock scoped_route_config_provider_; + Router::MockScopeKeyBuilder scope_key_builder_; Stats::IsolatedStoreImpl fake_stats_; Http::ContextImpl http_context_; NiceMock runtime_; diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 2a686e28f39c..37d8a7fa6561 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -457,75 +457,88 @@ class ScopedConfigImplTest : public testing::Test { std::shared_ptr scope_info_a_v2_; std::shared_ptr scope_info_b_; ScopedRoutes::ScopeKeyBuilder key_builder_config_; + std::unique_ptr scoped_key_builder_impl_; std::unique_ptr scoped_config_impl_; }; // Test a ScopedConfigImpl returns the correct route Config. TEST_F(ScopedConfigImplTest, PickRoute) { - scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); + scoped_key_builder_impl_ = std::make_unique(std::move(key_builder_config_)); + scoped_config_impl_ = std::make_unique(); scoped_config_impl_->addOrUpdateRoutingScopes({scope_info_a_}); scoped_config_impl_->addOrUpdateRoutingScopes({scope_info_b_}); // Key (foo, bar) maps to scope_info_a_. - ConfigConstSharedPtr route_config = scoped_config_impl_->getRouteConfig(TestRequestHeaderMapImpl{ - {"foo_header", ",,key=value,bar=foo,"}, - {"bar_header", ";val1;bar;val3"}, - }); + ConfigConstSharedPtr route_config = scoped_config_impl_->getRouteConfig( + scoped_key_builder_impl_->computeScopeKey(TestRequestHeaderMapImpl{ + {"foo_header", ",,key=value,bar=foo,"}, + {"bar_header", ";val1;bar;val3"}, + })); EXPECT_EQ(route_config, scope_info_a_->routeConfig()); // Key (bar, baz) maps to scope_info_b_. - route_config = scoped_config_impl_->getRouteConfig(TestRequestHeaderMapImpl{ - {"foo_header", ",,key=value,bar=bar,"}, - {"bar_header", ";val1;baz;val3"}, - }); + route_config = scoped_config_impl_->getRouteConfig( + scoped_key_builder_impl_->computeScopeKey(TestRequestHeaderMapImpl{ + {"foo_header", ",,key=value,bar=bar,"}, + {"bar_header", ";val1;baz;val3"}, + })); EXPECT_EQ(route_config, scope_info_b_->routeConfig()); // No such key (bar, NOT_BAZ). - route_config = scoped_config_impl_->getRouteConfig(TestRequestHeaderMapImpl{ - {"foo_header", ",key=value,bar=bar,"}, - {"bar_header", ";val1;NOT_BAZ;val3"}, - }); + route_config = scoped_config_impl_->getRouteConfig( + scoped_key_builder_impl_->computeScopeKey(TestRequestHeaderMapImpl{ + {"foo_header", ",key=value,bar=bar,"}, + {"bar_header", ";val1;NOT_BAZ;val3"}, + })); EXPECT_EQ(route_config, nullptr); } // Test a ScopedConfigImpl returns the correct route Config before and after scope config update. TEST_F(ScopedConfigImplTest, Update) { - scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); + scoped_key_builder_impl_ = std::make_unique(std::move(key_builder_config_)); + scoped_config_impl_ = std::make_unique(); TestRequestHeaderMapImpl headers{ {"foo_header", ",,key=value,bar=foo,"}, {"bar_header", ";val1;bar;val3"}, }; // Empty ScopeConfig. - EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(scoped_key_builder_impl_->computeScopeKey(headers)), + nullptr); // Add scope_key (bar, baz). scoped_config_impl_->addOrUpdateRoutingScopes({scope_info_b_}); // scope_info_a_ not found - EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(scoped_key_builder_impl_->computeScopeKey(headers)), + nullptr); // scope_info_b_ found - EXPECT_EQ(scoped_config_impl_->getRouteConfig(TestRequestHeaderMapImpl{ - {"foo_header", ",,key=v,bar=bar,"}, {"bar_header", ";val1;baz"}}), + EXPECT_EQ(scoped_config_impl_->getRouteConfig( + scoped_key_builder_impl_->computeScopeKey(TestRequestHeaderMapImpl{ + {"foo_header", ",,key=v,bar=bar,"}, {"bar_header", ";val1;baz"}})), scope_info_b_->routeConfig()); // Add scope_key (foo, bar). scoped_config_impl_->addOrUpdateRoutingScopes({scope_info_a_}); // Found scope_info_a_. - EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), scope_info_a_->routeConfig()); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(scoped_key_builder_impl_->computeScopeKey(headers)), + scope_info_a_->routeConfig()); // Update scope foo_scope. scoped_config_impl_->addOrUpdateRoutingScopes({scope_info_a_v2_}); - EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(scoped_key_builder_impl_->computeScopeKey(headers)), + nullptr); // foo_scope now is keyed by (xyz, xyz). - EXPECT_EQ(scoped_config_impl_->getRouteConfig(TestRequestHeaderMapImpl{ - {"foo_header", ",bar=xyz,foo=bar"}, {"bar_header", ";;xyz"}}), - scope_info_a_v2_->routeConfig()); + EXPECT_EQ( + scoped_config_impl_->getRouteConfig(scoped_key_builder_impl_->computeScopeKey( + TestRequestHeaderMapImpl{{"foo_header", ",bar=xyz,foo=bar"}, {"bar_header", ";;xyz"}})), + scope_info_a_v2_->routeConfig()); // Remove scope "foo_scope". scoped_config_impl_->removeRoutingScopes({"foo_scope"}); // scope_info_a_ is gone. - EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(scoped_key_builder_impl_->computeScopeKey(headers)), + nullptr); // Now delete some non-existent scopes. EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScopes( diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 3880d478315a..eaf598d89631 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -220,20 +220,26 @@ TEST_F(InlineScopedRoutesTest, InlineRouteConfigurations) { key: fragments: { string_key: foo-key-2 } )EOF"); + const auto config = + parseHttpConnectionManagerFromYaml(absl::Substitute(hcm_config, "foo-scoped-routes")); Envoy::Config::ConfigProviderPtr provider = ScopedRoutesConfigProviderUtil::create( - parseHttpConnectionManagerFromYaml(absl::Substitute(hcm_config, "foo-scoped-routes")), - server_factory_context_, context_init_manager_, "foo.", *config_provider_manager_); + config, server_factory_context_, context_init_manager_, "foo.", *config_provider_manager_); + Envoy::Router::ScopeKeyBuilderPtr scope_key_builder = + ScopedRoutesConfigProviderUtil::createScopeKeyBuilder(config); ASSERT_THAT(provider->config(), Not(IsNull())); EXPECT_EQ(provider->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"addr", "foo-key"}}) + ->getRouteConfig(scope_key_builder->computeScopeKey( + TestRequestHeaderMapImpl{{"addr", "foo-key"}})) ->name(), "foo"); EXPECT_EQ(provider->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"addr", "foo-key-2"}}) + ->getRouteConfig(scope_key_builder->computeScopeKey( + TestRequestHeaderMapImpl{{"addr", "foo-key-2"}})) ->name(), "foo2"); EXPECT_EQ(provider->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"addr", "foo-key,foo-key-2"}}) + ->getRouteConfig(scope_key_builder->computeScopeKey( + TestRequestHeaderMapImpl{{"addr", "foo-key,foo-key-2"}})) ->name(), "foo"); } @@ -388,11 +394,13 @@ name: foo_scoped_routes envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes scoped_routes_config; TestUtility::loadFromYaml(config_yaml, scoped_routes_config); + auto scope_key_builder_config = scoped_routes_config.scope_key_builder(); + scope_key_builder_ = std::make_unique(std::move(scope_key_builder_config)); provider_ = config_provider_manager_->createXdsConfigProvider( scoped_routes_config.scoped_rds(), server_factory_context_, context_init_manager_, "foo.", - ScopedRoutesConfigProviderManagerOptArg( - scoped_routes_config.name(), scoped_routes_config.rds_config_source(), - scoped_routes_config.scope_key_builder(), optional_http_filters)); + ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), + scoped_routes_config.rds_config_source(), + optional_http_filters)); srds_subscription_ = server_factory_context_.cluster_manager_.subscription_factory_.callbacks_; } @@ -445,6 +453,7 @@ name: foo_scoped_routes } Envoy::Config::SubscriptionCallbacks* srds_subscription_{}; + Envoy::Router::ScopeKeyBuilderPtr scope_key_builder_; Envoy::Config::ConfigProviderPtr provider_; std::list target_handles_; Init::ExpectableWatcherImpl init_watcher_; @@ -631,24 +640,28 @@ route_configuration_name: foo_routes ASSERT_THAT(getScopedRdsProvider()->config(), Not(IsNull())); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), ""); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})) ->name(), ""); // RDS updates foo_routes. pushRdsConfig({"foo_routes"}, "111"); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})) ->name(), "foo_routes"); @@ -665,11 +678,13 @@ route_configuration_name: foo_routes // now scope key "x-bar-key" points to nowhere. EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})), IsNull()); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); } @@ -710,24 +725,28 @@ route_configuration_name: foo_routes ASSERT_THAT(getScopedRdsProvider()->config(), Not(IsNull())); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), ""); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})) ->name(), ""); // RDS updates foo_routes. pushRdsConfig({"foo_routes"}, "111"); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})) ->name(), "foo_routes"); @@ -743,11 +762,13 @@ route_configuration_name: foo_routes .value()); // now scope key "x-bar-key" points to nowhere. EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})), IsNull()); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); } @@ -786,7 +807,8 @@ route_configuration_name: foo_routes ASSERT_THAT(getScopedRdsProvider(), Not(IsNull())); ASSERT_THAT(getScopedRdsProvider()->config(), Not(IsNull())); EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})), IsNull()); EXPECT_EQ(server_factory_context_.store_.counter("foo.rds.foo_routes.config_reload").value(), 0UL); @@ -827,7 +849,8 @@ route_configuration_name: foo_routes ASSERT_THAT(getScopedRdsProvider(), Not(IsNull())); ASSERT_THAT(getScopedRdsProvider()->config(), Not(IsNull())); EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})), IsNull()); EXPECT_EQ(server_factory_context_.store_.counter("foo.rds.foo_routes.config_reload").value(), 0UL); @@ -866,7 +889,8 @@ route_configuration_name: bar_routes // No RDS "foo_routes" config push happened yet, Router::NullConfig is returned. EXPECT_THAT(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), ""); pushRdsConfig({"foo_routes", "bar_routes"}, "111"); @@ -876,7 +900,8 @@ route_configuration_name: bar_routes 1UL); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); @@ -903,7 +928,8 @@ route_configuration_name: foo_routes // The same scope-key now points to the same route table. EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); @@ -928,7 +954,8 @@ route_configuration_name: foo_routes EXPECT_EQ(getScopedRouteMap().count("foo_scope3"), 1); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})) ->name(), "bar_routes"); @@ -943,12 +970,14 @@ route_configuration_name: foo_routes EXPECT_EQ(getScopedRouteMap().count("foo_scope4"), 1); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})) ->name(), "foo_routes"); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); } @@ -1002,7 +1031,8 @@ route_configuration_name: foo_routes ASSERT_THAT(getScopedRdsProvider(), Not(IsNull())); ASSERT_THAT(getScopedRdsProvider()->config(), Not(IsNull())); EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})), IsNull()); EXPECT_EQ(server_factory_context_.store_.counter("foo.rds.foo_routes.config_reload").value(), 0UL); @@ -1378,21 +1408,25 @@ on_demand: true // Route config for foo key is NullConfigImpl and route config for bar key is nullptr EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), ""); EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})), IsNull()); pushRdsConfig({"foo_routes"}, "111"); // Scope foo now have route config but route config for scope bar is still nullptr. EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})), IsNull()); EXPECT_EQ(2UL, all_scopes_.value()); EXPECT_EQ(1UL, active_scopes_.value()); @@ -1435,15 +1469,17 @@ on_demand: true ASSERT_THAT(getScopedRdsProvider()->config(), Not(IsNull())); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), ""); EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})), IsNull()); EXPECT_EQ(1UL, active_scopes_.value()); - ScopeKeyPtr scope_key = getScopedRdsProvider()->config()->computeScopeKey( + ScopeKeyPtr scope_key = scope_key_builder_->computeScopeKey( TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}); EXPECT_CALL(event_dispatcher_, post(_)); std::function route_config_updated_cb = [](bool) {}; @@ -1453,12 +1489,14 @@ on_demand: true pushRdsConfig({"foo_routes"}, "111"); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})) ->name(), "foo_routes"); // Now we have 1 active on demand scope and 1 eager loading scope. @@ -1502,20 +1540,23 @@ on_demand: true ASSERT_THAT(getScopedRdsProvider()->config(), Not(IsNull())); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), ""); EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})), IsNull()); // Push rds update before on demand srds request. pushRdsConfig({"foo_routes"}, "111"); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); - ScopeKeyPtr scope_key = getScopedRdsProvider()->config()->computeScopeKey( + ScopeKeyPtr scope_key = scope_key_builder_->computeScopeKey( TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}); EXPECT_CALL(server_factory_context_.dispatcher_, post(_)); EXPECT_CALL(event_dispatcher_, post(_)); @@ -1524,7 +1565,8 @@ on_demand: true std::move(route_config_updated_cb)); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}})) ->name(), "foo_routes"); } @@ -1549,7 +1591,8 @@ on_demand: true ASSERT_THAT(getScopedRdsProvider()->config(), Not(IsNull())); EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})), IsNull()); EXPECT_EQ(0UL, active_scopes_.value()); EXPECT_EQ(1UL, on_demand_scopes_.value()); @@ -1565,13 +1608,15 @@ route_configuration_name: foo_routes EXPECT_EQ(1UL, all_scopes_.value()); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), ""); pushRdsConfig({"foo_routes"}, "111"); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); // Now we have 1 eager scope. @@ -1603,7 +1648,8 @@ route_configuration_name: foo_routes ASSERT_THAT(getScopedRdsProvider()->config(), Not(IsNull())); EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); // Update the scope to on demand, rds provider and the route config will be deleted. @@ -1617,7 +1663,8 @@ route_configuration_name: foo_routes )EOF"; srdsUpdateWithYaml({lazy_resource}, "2"); EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})), IsNull()); // The new scope will be on demand and inactive after srds update. EXPECT_EQ(0UL, active_scopes_.value()); @@ -1645,7 +1692,7 @@ on_demand: true EXPECT_EQ(1UL, on_demand_scopes_.value()); // All the on demand updated callbacks will be executed when the route table comes. for (int i = 0; i < 5; i++) { - ScopeKeyPtr scope_key = getScopedRdsProvider()->config()->computeScopeKey( + ScopeKeyPtr scope_key = scope_key_builder_->computeScopeKey( TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}); std::function route_config_updated_cb = [](bool) {}; getScopedRdsProvider()->onDemandRdsUpdate(std::move(scope_key), event_dispatcher_, @@ -1657,7 +1704,7 @@ on_demand: true // Route table have been fetched, callbacks will be executed immediately. for (int i = 0; i < 5; i++) { EXPECT_CALL(event_dispatcher_, post(_)); - ScopeKeyPtr scope_key = getScopedRdsProvider()->config()->computeScopeKey( + ScopeKeyPtr scope_key = scope_key_builder_->computeScopeKey( TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}); std::function route_config_updated_cb = [](bool) {}; getScopedRdsProvider()->onDemandRdsUpdate(std::move(scope_key), event_dispatcher_, @@ -1666,7 +1713,8 @@ on_demand: true // Activating the same on_demand scope multiple times, active_scopes is still 1. EXPECT_EQ(getScopedRdsProvider() ->config() - ->getRouteConfig(TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->getRouteConfig(scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}})) ->name(), "foo_routes"); EXPECT_EQ(1UL, active_scopes_.value()); @@ -1679,9 +1727,8 @@ TEST_F(ScopedRdsTest, DanglingSubscriptionOnDemandUpdate) { Event::PostCb temp_post_cb; EXPECT_CALL(server_factory_context_.dispatcher_, post(_)) .WillOnce([&temp_post_cb](Event::PostCb cb) { temp_post_cb = std::move(cb); }); - std::shared_ptr scope_key = - getScopedRdsProvider()->config()->computeScopeKey( - TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}); + std::shared_ptr scope_key = scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}); getScopedRdsProvider()->onDemandRdsUpdate(scope_key, event_dispatcher_, std::move(route_config_updated_cb)); // Destroy the scoped_rds subscription by destroying its only config provider. @@ -1710,7 +1757,7 @@ on_demand: true EXPECT_EQ(1UL, on_demand_scopes_.value()); // All the on demand updated callbacks will be executed when the route table comes. { - ScopeKeyPtr scope_key = getScopedRdsProvider()->config()->computeScopeKey( + ScopeKeyPtr scope_key = scope_key_builder_->computeScopeKey( TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}); std::function route_config_updated_cb = [](bool scope_exist) { EXPECT_TRUE(scope_exist); @@ -1722,7 +1769,7 @@ on_demand: true EXPECT_CALL(event_dispatcher_, post(_)); pushRdsConfig({"foo_routes"}, "111"); - ScopeKeyPtr scope_key = getScopedRdsProvider()->config()->computeScopeKey( + ScopeKeyPtr scope_key = scope_key_builder_->computeScopeKey( TestRequestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}); // Delete the scope route. EXPECT_NO_THROW(srds_subscription_->onConfigUpdate({}, "2")); @@ -1736,6 +1783,54 @@ on_demand: true std::move(route_config_updated_cb)); } +TEST_F(ScopedRdsTest, OnScopeKeyBuilderUpdate) { + setup(); + + // update scope_key_builder separator to , + const std::string hcm_config = R"EOF( +name: foo_scoped_routes +scope_key_builder: + fragments: + - header_value_extractor: + name: Addr + element: + key: x-foo-key + separator: "," +)EOF"; + envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes + scoped_routes_config; + TestUtility::loadFromYaml(hcm_config, scoped_routes_config); + auto scope_key_builder_config = scoped_routes_config.scope_key_builder(); + scope_key_builder_ = std::make_unique(std::move(scope_key_builder_config)); + provider_ = config_provider_manager_->createXdsConfigProvider( + scoped_routes_config.scoped_rds(), server_factory_context_, context_init_manager_, "foo.", + ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), + scoped_routes_config.rds_config_source(), + OptionalHttpFilters())); + srds_subscription_ = server_factory_context_.cluster_manager_.subscription_factory_.callbacks_; + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-bar-key +)EOF"; + const auto resource = parseScopedRouteConfigurationFromYaml(config_yaml); + init_watcher_.expectReady(); // Only the SRDS parent_init_target_. + context_init_manager_.initialize(init_watcher_); + const auto decoded_resources = TestUtility::decodeResources({resource}); + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(decoded_resources.refvec_, "1")); + + pushRdsConfig({"foo_routes"}, "111"); + + auto config = getScopedRdsProvider()->config()->getRouteConfig( + scope_key_builder_->computeScopeKey( + TestRequestHeaderMapImpl{{"Addr", "x-foo-key,x-bar-key"}})); + ASSERT_THAT(config, Not(IsNull())); + EXPECT_EQ(config->name(), "foo_routes"); +} + } // namespace } // namespace Router } // namespace Envoy diff --git a/test/integration/scoped_rds.h b/test/integration/scoped_rds.h index fb354c726460..194f1d1a3eda 100644 --- a/test/integration/scoped_rds.h +++ b/test/integration/scoped_rds.h @@ -131,6 +131,43 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, grpc_service = srds_api_config_source->add_grpc_services(); setGrpcService(*grpc_service, "srds_cluster", getScopedRdsFakeUpstream().localAddress()); }); + + if (add_listener_) { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + // Copy from old listener and modify the scope key builder. + auto* old_listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* new_listener = bootstrap.mutable_static_resources()->add_listeners(); + new_listener->CopyFrom(*old_listener); + new_listener->set_name("listener_1"); + auto* filter_chain = new_listener->mutable_filter_chains(0); + envoy::config::listener::v3::Filter* filter_config = nullptr; + for (ssize_t i = 0; i < filter_chain->filters_size(); i++) { + if (filter_chain->mutable_filters(i)->name() == "http") { + filter_config = filter_chain->mutable_filters(i); + } + } + ASSERT(filter_config != nullptr); + auto* config = filter_config->mutable_typed_config(); + auto filter = + MessageUtil::anyConvert(*config); + // changing separator to , + const std::string& scope_key_builder_config_yaml = R"EOF( +fragments: + - header_value_extractor: + name: Addr + element_separator: ; + element: + key: x-foo-key + separator: "," +)EOF"; + envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: + ScopeKeyBuilder scope_key_builder; + TestUtility::loadFromYaml(scope_key_builder_config_yaml, scope_key_builder); + *filter.mutable_scoped_routes()->mutable_scope_key_builder() = scope_key_builder; + config->PackFrom(filter); + }); + } modifications_set_up_ = true; } @@ -288,6 +325,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, FakeUpstreamInfo rds_upstream_info_; std::string srds_resources_locator_; bool modifications_set_up_ = false; + bool add_listener_ = false; }; } // namespace Envoy diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index a3ab24568167..4b5aedffa47e 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -564,5 +564,111 @@ route_configuration_name: foo_route1 /*cluster_0*/ 0); } +// Two listeners with the same rds config but different scope_key_builder should use their own +// scope_key_builder to calculate the scope key and routes. +TEST_P(ScopedRdsIntegrationTest, ListenersWithDifferentScopeKeyBuilder) { + constexpr absl::string_view scope_tmpl = R"EOF( +name: {} +route_configuration_name: {} +key: + fragments: + - string_key: {} +)EOF"; + const std::string scope_route1 = fmt::format(scope_tmpl, "foo_scope1", "foo_route1", "foo-route"); + const std::string scope_route2 = fmt::format(scope_tmpl, "foo_scope2", "foo_route1", "bar-route"); + + constexpr absl::string_view route_config_tmpl = R"EOF( + name: {} + virtual_hosts: + - name: integration + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: {} }} +)EOF"; + + on_server_init_function_ = [&]() { + createScopedRdsStream(); + sendSrdsResponse({scope_route1, scope_route2}, {scope_route1, scope_route2}, {}, "1"); + createRdsStream("foo_route1"); + // CreateRdsStream waits for connection which is fired by RDS subscription. + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_0"), "1"); + }; + // add listener_1 with different scope_key_builder + add_listener_ = true; + initialize(); + registerTestServerPorts({"http", "listener_1"}); + + // listener_0 can't match using "," as separator + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key,foo-route"}}); + ASSERT_TRUE(response->waitForEndStream()); + verifyResponse(std::move(response), "404", Http::TestResponseHeaderMapImpl{}, ""); + cleanupUpstreamAndDownstream(); + + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_success", 1); + + // listener_0 can match using "=" as separator + codec_client_ = makeHttpConnection(lookupPort("http")); + response = sendRequestAndWaitForResponse( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key=foo-route"}}, + 456, Http::TestResponseHeaderMapImpl{{":status", "200"}, {"service", "foo-route"}}, 123, 0); + ASSERT_TRUE(response->waitForEndStream()); + verifyResponse(std::move(response), "200", + Http::TestResponseHeaderMapImpl{{":status", "200"}, {"service", "foo-route"}}, + std::string(123, 'a')); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(456, upstream_request_->bodyLength()); + cleanupUpstreamAndDownstream(); + + // listener_1 can't match using "=" as separator + codec_client_ = makeHttpConnection(lookupPort("listener_1")); + response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key=foo-route"}}); + ASSERT_TRUE(response->waitForEndStream()); + verifyResponse(std::move(response), "404", Http::TestResponseHeaderMapImpl{}, ""); + cleanupUpstreamAndDownstream(); + + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_success", 1); + + // listener_1 can match using "," as separator + codec_client_ = makeHttpConnection(lookupPort("listener_1")); + response = sendRequestAndWaitForResponse( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key,foo-route"}}, + 456, Http::TestResponseHeaderMapImpl{{":status", "200"}, {"service", "foo-route"}}, 123, 0); + ASSERT_TRUE(response->waitForEndStream()); + verifyResponse(std::move(response), "200", + Http::TestResponseHeaderMapImpl{{":status", "200"}, {"service", "foo-route"}}, + std::string(123, 'a')); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(456, upstream_request_->bodyLength()); + cleanupUpstreamAndDownstream(); + + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", + // update_attempt only increase after a response + isDelta() ? 1 : 2); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 1); + // The version gauge should be set to xxHash64("1"). + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", + 13237225503670494420UL); +} + } // namespace } // namespace Envoy diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 1cefdb07635b..44ce6bc4b7b1 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -619,6 +619,7 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_METHOD(std::chrono::milliseconds, delayedCloseTimeout, (), (const)); MOCK_METHOD(Router::RouteConfigProvider*, routeConfigProvider, ()); MOCK_METHOD(Config::ConfigProvider*, scopedRouteConfigProvider, ()); + MOCK_METHOD(OptRef, scopeKeyBuilder, ()); MOCK_METHOD(const std::string&, serverName, (), (const)); MOCK_METHOD(HttpConnectionManagerProto::ServerHeaderTransformation, serverHeaderTransformation, (), (const)); diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index 3b995307d79b..3c77af507b18 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -167,6 +167,12 @@ MockScopedRouteConfigProvider::MockScopedRouteConfigProvider() } MockScopedRouteConfigProvider::~MockScopedRouteConfigProvider() = default; +MockScopeKeyBuilder::MockScopeKeyBuilder() { + ON_CALL(*this, computeScopeKey(_)) + .WillByDefault(Invoke([](const Http::HeaderMap&) -> ScopeKeyPtr { return nullptr; })); +} +MockScopeKeyBuilder::~MockScopeKeyBuilder() = default; + MockGenericConnectionPoolCallbacks::MockGenericConnectionPoolCallbacks() { ON_CALL(*this, upstreamToDownstream()).WillByDefault(ReturnRef(upstream_to_downstream_)); } diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index c5b6174c7d2e..9f5976bd814f 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -580,7 +580,7 @@ class MockScopedConfig : public ScopedConfig { MockScopedConfig(); ~MockScopedConfig() override; - MOCK_METHOD(ConfigConstSharedPtr, getRouteConfig, (const Http::HeaderMap& headers), (const)); + MOCK_METHOD(ConfigConstSharedPtr, getRouteConfig, (const ScopeKeyPtr& scope_key), (const)); std::shared_ptr route_config_{new NiceMock()}; }; @@ -600,6 +600,14 @@ class MockScopedRouteConfigProvider : public Envoy::Config::ConfigProvider { std::shared_ptr config_; }; +class MockScopeKeyBuilder : public ScopeKeyBuilder { +public: + MockScopeKeyBuilder(); + ~MockScopeKeyBuilder() override; + + MOCK_METHOD(ScopeKeyPtr, computeScopeKey, (const Http::HeaderMap&), (const)); +}; + class MockGenericConnPool : public GenericConnPool { public: MOCK_METHOD(void, newStream, (GenericConnectionPoolCallbacks * request)); From ca88f87df177794b8599be6378afd5f4cfacdcd1 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Mon, 8 May 2023 12:59:00 -0400 Subject: [PATCH 152/740] stats: reproduce redundant TYPE tags problem in Prometheus. (#27239) Commit Message: Reproduces a scenario where it's difficult to use a streaming optimization for Prometheus stats based on scopes without introducing a bug. Context: Issue that /stats/prometheus endpoint takes too much much memory: Prometheus stats handler used too much memory. #16139 Solution for non-Prometheus endpoints to use less memory for /stats and run faster: admin: Streaming /stats implementation #19693 which I believe is working well. This solution mapped to Prometheus: Prom stats perf improvements #24998 A case where this solution has a duplicate# TYPE line due to two stats with the same tag-extracted-stat name from two different scopes: stats: prometheus scrape failed #27173 Note that the existing unit tests did not exercise that scenario so this PR adds a testcase. Signed-off-by: Joshua Marantz --- test/server/admin/BUILD | 2 + test/server/admin/prometheus_stats_test.cc | 47 ++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/test/server/admin/BUILD b/test/server/admin/BUILD index 8c1a28f7d2ee..950a66bc4c3f 100644 --- a/test/server/admin/BUILD +++ b/test/server/admin/BUILD @@ -168,6 +168,8 @@ envoy_cc_test( name = "prometheus_stats_test", srcs = envoy_select_admin_functionality(["prometheus_stats_test.cc"]), deps = [ + "//source/common/stats:tag_producer_lib", + "//source/common/stats:thread_local_store_lib", "//source/server/admin:prometheus_stats_lib", "//test/test_common:utility_lib", ], diff --git a/test/server/admin/prometheus_stats_test.cc b/test/server/admin/prometheus_stats_test.cc index c27af6121130..dcf9ab510bd2 100644 --- a/test/server/admin/prometheus_stats_test.cc +++ b/test/server/admin/prometheus_stats_test.cc @@ -3,6 +3,8 @@ #include #include "source/common/stats/custom_stat_namespaces_impl.h" +#include "source/common/stats/tag_producer_impl.h" +#include "source/common/stats/thread_local_store.h" #include "source/server/admin/prometheus_stats.h" #include "test/mocks/stats/mocks.h" @@ -261,6 +263,51 @@ envoy_histogram1_count{} 0 EXPECT_EQ(expected_output, response.toString()); } +// Replicate bug https://github.com/envoyproxy/envoy/issues/27173 which fails to +// coalesce stats in different scopes with the same tag-extracted-name. +TEST_F(PrometheusStatsFormatterTest, DifferentNamedScopeSameStat) { + Stats::CustomStatNamespacesImpl custom_namespaces; + Stats::ThreadLocalStoreImpl store(alloc_); + envoy::config::metrics::v3::StatsConfig stats_config; + store.setTagProducer(std::make_unique(stats_config)); + Stats::StatName name = pool_.add("default.total_match_count"); + + Stats::ScopeSharedPtr scope1 = store.rootScope()->createScope("cluster.a"); + counters_.push_back(Stats::CounterSharedPtr(&scope1->counterFromStatName(name))); + + // To reproduce the problem from we will render + // cluster.a.default.total_match_count before we discover the existence of + // cluster.x.default.total_match_count. That will happen because "d" in + // "default" comes before "x" with + // https://github.com/envoyproxy/envoy/pull/24998 + Stats::ScopeSharedPtr scope2 = store.rootScope()->createScope("cluster.x"); + counters_.push_back(Stats::CounterSharedPtr(&scope2->counterFromStatName(name))); + + constexpr absl::string_view expected_output = + R"EOF(# TYPE envoy_cluster_default_total_match_count counter +envoy_cluster_default_total_match_count{envoy_cluster_name="a"} 0 +envoy_cluster_default_total_match_count{envoy_cluster_name="x"} 0 +)EOF"; + + // Note: in the version of prometheus_stats_test.cc that works with the + // streaming GroupedStatsRequest, the test code is slightly different; + // it's based on the local 'store' object rather than being based on + // the counters_ member variable. + // StatsParams params; + // params.type_ = StatsType::Counters; + // params.format_ = StatsFormat::Prometheus; + // auto request = std::make_unique(store, params, custom_namespaces_); + // EXPECT_EQ(expected_output, response(*request)); + // This code is left here so that we can verify the bug is fixed if we decide to + // re-try the streaming Prometheus implementation. + + Buffer::OwnedImpl response; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + EXPECT_EQ(1, size); + EXPECT_EQ(expected_output, response.toString()); +} + TEST_F(PrometheusStatsFormatterTest, HistogramWithNonDefaultBuckets) { Stats::CustomStatNamespacesImpl custom_namespaces; HistogramWrapper h1_cumulative; From c2ccab265b13a7675af6a78587fe6f397c56ea6b Mon Sep 17 00:00:00 2001 From: Ryan Hamilton Date: Mon, 8 May 2023 10:42:57 -0700 Subject: [PATCH 153/740] Add a SystemHelper abstraction for Envoy Mobile (#27237) Add a SystemHelper abstraction for Envoy Mobile which provides a platform-agnostic API for platform-specific system services in a manner that can be swapped out in tests. Risk Level: Low Testing: New unit tests Docs Changes: N/A Release Notes: N/A Signed-off-by: Ryan Hamilton --- mobile/library/common/common/BUILD | 19 +++++++ .../common/common/default_system_helper.cc | 7 +++ .../common/common/default_system_helper.h | 19 +++++++ .../common/default_system_helper_android.cc | 10 ++++ mobile/library/common/common/system_helper.cc | 11 +++++ mobile/library/common/common/system_helper.h | 40 +++++++++++++++ mobile/library/common/http/BUILD | 1 + mobile/library/common/http/client.cc | 3 +- mobile/test/common/integration/BUILD | 1 + .../integration/client_integration_test.cc | 38 ++++++++++++++ mobile/test/common/mocks/common/BUILD | 19 +++++++ mobile/test/common/mocks/common/mocks.cc | 13 +++++ mobile/test/common/mocks/common/mocks.h | 49 +++++++++++++++++++ 13 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 mobile/library/common/common/default_system_helper.cc create mode 100644 mobile/library/common/common/default_system_helper.h create mode 100644 mobile/library/common/common/default_system_helper_android.cc create mode 100644 mobile/library/common/common/system_helper.cc create mode 100644 mobile/library/common/common/system_helper.h create mode 100644 mobile/test/common/mocks/common/BUILD create mode 100644 mobile/test/common/mocks/common/mocks.cc create mode 100644 mobile/test/common/mocks/common/mocks.h diff --git a/mobile/library/common/common/BUILD b/mobile/library/common/common/BUILD index a0cc3a1b5a25..ff561431c347 100644 --- a/mobile/library/common/common/BUILD +++ b/mobile/library/common/common/BUILD @@ -4,6 +4,25 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_library( + name = "system_helper_lib", + srcs = [ + "system_helper.cc", + ] + select({ + "@envoy//bazel:android": ["default_system_helper_android.cc"], + "//conditions:default": ["default_system_helper.cc"], + }), + hdrs = [ + "default_system_helper.h", + "system_helper.h", + ], + repository = "@envoy", + deps = select({ + "@envoy//bazel:android": ["//library/common/jni:android_jni_utility_lib"], + "//conditions:default": [], + }), +) + envoy_cc_library( name = "lambda_logger_delegate_lib", srcs = ["lambda_logger_delegate.cc"], diff --git a/mobile/library/common/common/default_system_helper.cc b/mobile/library/common/common/default_system_helper.cc new file mode 100644 index 000000000000..cd70da12ffd8 --- /dev/null +++ b/mobile/library/common/common/default_system_helper.cc @@ -0,0 +1,7 @@ +#include "library/common/common/default_system_helper.h" + +namespace Envoy { + +bool DefaultSystemHelper::isCleartextPermitted(absl::string_view /*hostname*/) { return true; } + +} // namespace Envoy diff --git a/mobile/library/common/common/default_system_helper.h b/mobile/library/common/common/default_system_helper.h new file mode 100644 index 000000000000..5ca7f4edbdc4 --- /dev/null +++ b/mobile/library/common/common/default_system_helper.h @@ -0,0 +1,19 @@ +#pragma once + +#include "library/common/common/system_helper.h" + +namespace Envoy { + +/** + * Default implementation of SystemHelper which invokes the appropriate + * platform-specific system APIs. + */ +class DefaultSystemHelper : public SystemHelper { +public: + ~DefaultSystemHelper() override = default; + + // SystemHelper: + bool isCleartextPermitted(absl::string_view hostname) override; +}; + +} // namespace Envoy diff --git a/mobile/library/common/common/default_system_helper_android.cc b/mobile/library/common/common/default_system_helper_android.cc new file mode 100644 index 000000000000..cfb9c9daf50f --- /dev/null +++ b/mobile/library/common/common/default_system_helper_android.cc @@ -0,0 +1,10 @@ +#include "library/common/common/default_system_helper.h" +#include "library/common/jni/android_jni_utility.h" + +namespace Envoy { + +bool DefaultSystemHelper::isCleartextPermitted(absl::string_view hostname) { + return is_cleartext_permitted(hostname); +} + +} // namespace Envoy diff --git a/mobile/library/common/common/system_helper.cc b/mobile/library/common/common/system_helper.cc new file mode 100644 index 000000000000..a363a690cd68 --- /dev/null +++ b/mobile/library/common/common/system_helper.cc @@ -0,0 +1,11 @@ +#include "library/common/common/system_helper.h" + +#include "library/common/common/default_system_helper.h" + +namespace Envoy { + +std::unique_ptr SystemHelper::instance_ = std::make_unique(); + +SystemHelper& SystemHelper::getInstance() { return *instance_; } + +} // namespace Envoy diff --git a/mobile/library/common/common/system_helper.h b/mobile/library/common/common/system_helper.h new file mode 100644 index 000000000000..bde21205d17e --- /dev/null +++ b/mobile/library/common/common/system_helper.h @@ -0,0 +1,40 @@ +#pragma once + +#include + +#include "envoy/common/pure.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { + +namespace test { +class SystemHelperPeer; +} // namespace test + +/** + * SystemHelper provided a platform-agnostic API for interacting with platform-specific + * system APIs. The singleton helper is accessible via the `getInstance()` static + * method. Tests may override this singleton via `test::SystemHelperPeer` class. + */ +class SystemHelper { +public: + virtual ~SystemHelper() = default; + + /** + * @return true if the system permits cleartext requests. + */ + virtual bool isCleartextPermitted(absl::string_view hostname) PURE; + + /** + * @return a reference to the current SystemHelper instance. + */ + static SystemHelper& getInstance(); + +private: + friend class test::SystemHelperPeer; + + static std::unique_ptr instance_; +}; + +} // namespace Envoy diff --git a/mobile/library/common/http/BUILD b/mobile/library/common/http/BUILD index 495422b43781..22c4886712a4 100644 --- a/mobile/library/common/http/BUILD +++ b/mobile/library/common/http/BUILD @@ -13,6 +13,7 @@ envoy_cc_library( deps = [ "//library/common/bridge:utility_lib", "//library/common/buffer:bridge_fragment_lib", + "//library/common/common:system_helper_lib", "//library/common/data:utility_lib", "//library/common/event:provisional_dispatcher_lib", "//library/common/extensions/filters/http/local_error:local_error_filter_lib", diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index 129910cbc0e2..f41c432da55b 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -10,6 +10,7 @@ #include "library/common/bridge/utility.h" #include "library/common/buffer/bridge_fragment.h" +#include "library/common/common/system_helper.h" #include "library/common/data/utility.h" #include "library/common/http/header_utility.h" #include "library/common/http/headers.h" @@ -486,7 +487,7 @@ void Client::sendHeaders(envoy_stream_t stream, envoy_headers headers, bool end_ // This is largely a check for the android platform: is_cleartext_permitted // is a no-op for other platforms. if (internal_headers->getSchemeValue() != "https" && - !is_cleartext_permitted(internal_headers->getHostValue())) { + !SystemHelper::getInstance().isCleartextPermitted(internal_headers->getHostValue())) { direct_stream->request_decoder_->sendLocalReply( Http::Code::BadRequest, "Cleartext is not permitted", nullptr, absl::nullopt, ""); return; diff --git a/mobile/test/common/integration/BUILD b/mobile/test/common/integration/BUILD index 501f65c8c588..049a6b9548b1 100644 --- a/mobile/test/common/integration/BUILD +++ b/mobile/test/common/integration/BUILD @@ -21,6 +21,7 @@ envoy_cc_test( repository = "@envoy", deps = [ ":base_client_integration_test_lib", + "//test/common/mocks/common:common_mocks", ], ) diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index 3ffcb1f3e791..a85f1b0eef7d 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -1,6 +1,7 @@ #include "source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.h" #include "test/common/integration/base_client_integration_test.h" +#include "test/common/mocks/common/mocks.h" #include "test/integration/autonomous_upstream.h" #include "library/common/data/utility.h" @@ -8,6 +9,8 @@ #include "library/common/network/proxy_settings.h" #include "library/common/types/c_types.h" +using testing::_; +using testing::Return; using testing::ReturnRef; namespace Envoy { @@ -22,10 +25,17 @@ class ClientIntegrationTest : public BaseClientIntegrationTest, setUpstreamCount(config_helper_.bootstrap().static_resources().clusters_size()); // TODO(abeyad): Add paramaterized tests for HTTP1, HTTP2, and HTTP3. setUpstreamProtocol(Http::CodecType::HTTP1); + helper_handle_ = test::SystemHelperPeer::replaceSystemHelper(); + EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)) + .WillRepeatedly(Return(true)); } + void TearDown() override { BaseClientIntegrationTest::TearDown(); } void basicTest(); + +protected: + std::unique_ptr helper_handle_; }; INSTANTIATE_TEST_SUITE_P(IpVersions, ClientIntegrationTest, @@ -71,6 +81,34 @@ void ClientIntegrationTest::basicTest() { TEST_P(ClientIntegrationTest, Basic) { basicTest(); } +TEST_P(ClientIntegrationTest, ClearTextNotPermitted) { + EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).WillRepeatedly(Return(false)); + + expect_data_streams_ = false; + initialize(); + + Buffer::OwnedImpl request_data = Buffer::OwnedImpl("request body"); + default_request_headers_.addCopy(AutonomousStream::EXPECT_REQUEST_SIZE_BYTES, + std::to_string(request_data.length())); + + stream_prototype_->setOnData([this](envoy_data c_data, bool end_stream) { + if (end_stream) { + EXPECT_EQ(Data::Utility::copyToString(c_data), "Cleartext is not permitted"); + } + cc_.on_data_calls++; + release_envoy_data(c_data); + }); + + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + + terminal_callback_.waitReady(); + + ASSERT_EQ(cc_.on_headers_calls, 1); + ASSERT_EQ(cc_.status, "400"); + ASSERT_EQ(cc_.on_data_calls, 1); + ASSERT_EQ(cc_.on_complete_calls, 1); +} + TEST_P(ClientIntegrationTest, BasicNon2xx) { initialize(); diff --git a/mobile/test/common/mocks/common/BUILD b/mobile/test/common/mocks/common/BUILD new file mode 100644 index 000000000000..41d56371e474 --- /dev/null +++ b/mobile/test/common/mocks/common/BUILD @@ -0,0 +1,19 @@ +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_mock", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_mock( + name = "common_mocks", + srcs = ["mocks.cc"], + hdrs = ["mocks.h"], + repository = "@envoy", + deps = [ + "//library/common/common:system_helper_lib", + ], +) diff --git a/mobile/test/common/mocks/common/mocks.cc b/mobile/test/common/mocks/common/mocks.cc new file mode 100644 index 000000000000..1a394cf0424b --- /dev/null +++ b/mobile/test/common/mocks/common/mocks.cc @@ -0,0 +1,13 @@ +#include "test/common/mocks/common/mocks.h" + +namespace Envoy { +namespace test { + +using testing::_; + +MockSystemHelper::MockSystemHelper() { + ON_CALL(*this, isCleartextPermitted(_)).WillByDefault(testing::Return(true)); +} + +} // namespace test +} // namespace Envoy diff --git a/mobile/test/common/mocks/common/mocks.h b/mobile/test/common/mocks/common/mocks.h new file mode 100644 index 000000000000..8d876bb10101 --- /dev/null +++ b/mobile/test/common/mocks/common/mocks.h @@ -0,0 +1,49 @@ +#include "gmock/gmock.h" +#include "library/common/common/system_helper.h" + +namespace Envoy { +namespace test { + +// Mock implementation of SystemHelper. +class MockSystemHelper : public SystemHelper { +public: + MockSystemHelper(); + + // SystemHelper: + MOCK_METHOD(bool, isCleartextPermitted, (absl::string_view hostname)); +}; + +// SystemHelperPeer allows the replacement of the SystemHelper singleton +// with a MockSystemHelper. +class SystemHelperPeer { +public: + class Handle; + + // Replaces the SystemHelper singleton with a new MockSystemHelper which is + // wrapped in a Handle. The MockSystemHelper can be accessed via the + // Handle's `mock_helper()` accessor. + static std::unique_ptr replaceSystemHelper() { return std::make_unique(); } + + // RAII type for replacing the SystemHelper singleton with a the MockSystemHelper. + // When this object is destroyed, it resets the SystemHelper singleton back + // to the previous state. + class Handle { + public: + Handle() { + previous_ = std::make_unique(); + SystemHelper::instance_.swap(previous_); + } + + ~Handle() { SystemHelper::instance_ = std::move(previous_); } + + test::MockSystemHelper& mock_helper() { + return *static_cast(SystemHelper::instance_.get()); + } + + private: + std::unique_ptr previous_; + }; +}; + +} // namespace test +} // namespace Envoy From a8107ace78d3768fa725a315debaa2599678a95c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 13:38:22 -0600 Subject: [PATCH 154/740] build(deps): bump sphinx from 6.1.3 to 7.0.0 in /mobile/docs (#27073) Dependabot couldn't find the original pull request head commit, 10120058e7b0d2d2b7de12d5bb6b975cc305bcd5. Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mobile/docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/docs/requirements.txt b/mobile/docs/requirements.txt index c471e6d216e3..7734c1c67156 100644 --- a/mobile/docs/requirements.txt +++ b/mobile/docs/requirements.txt @@ -167,9 +167,9 @@ six==1.16.0 \ snowballstemmer==2.2.0 \ --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a \ --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 -Sphinx==6.1.3 \ - --hash=sha256:0dac3b698538ffef41716cf97ba26c1c7788dba73ce6f150c1ff5b4720786dd2 \ - --hash=sha256:807d1cb3d6be87eb78a381c3e70ebd8d346b9a25f3753e9947e866b2786865fc +Sphinx==7.0.0 \ + --hash=sha256:283c44aa28922bb4223777b44ac0d59af50a279ac7690dfe945bb2b9575dc41b \ + --hash=sha256:3cfc1c6756ef1b132687b813ec6ea2214cb7a7e5d1dcb2772006cb895a0fa469 sphinx-rtd-theme==1.2.0 \ --hash=sha256:a0d8bd1a2ed52e0b338cbe19c4b2eef3c5e7a048769753dac6a9f059c7b641b8 \ --hash=sha256:f823f7e71890abe0ac6aaa6013361ea2696fc8d3e1fa798f463e82bdb77eeff2 From e09dc8c7b647557826bc01b3138da6c28a90b024 Mon Sep 17 00:00:00 2001 From: Ryan Hamilton Date: Mon, 8 May 2023 13:42:56 -0700 Subject: [PATCH 155/740] Add cert verification support to the mobile SystemHelper (#27238) Add cert verification support to the mobile SystemHelper Change the PlatformBridgeCertValidator to get the underlying platform verifier from the SystemHelper instead of the Envoy Mobile register API mechanism. This allows the Envoy Mobile certificate verification API to be C++-ified by using string_view instead of envoy_data. This change fixed a memory leak in the Apple platform cert verified. Risk Level: Low Testing: Existing Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A Signed-off-by: Ryan Hamilton --- .github/workflows/mobile-android_tests.yml | 1 + bazel/BUILD | 5 + bazel/envoy_library.bzl | 4 +- mobile/library/common/common/BUILD | 53 +++++++-- .../common/common/default_system_helper.cc | 12 ++ .../common/common/default_system_helper.h | 3 + .../common/default_system_helper_android.cc | 9 ++ .../common/default_system_helper_apple.cc | 16 +++ mobile/library/common/common/system_helper.cc | 8 ++ mobile/library/common/common/system_helper.h | 13 +++ .../cert_validator/platform_bridge/BUILD | 1 + .../cert_validator/platform_bridge/c_types.h | 27 ----- .../cert_validator/platform_bridge/config.cc | 6 +- .../cert_validator/platform_bridge/config.h | 3 - .../platform_bridge_cert_validator.cc | 53 +++++---- .../platform_bridge_cert_validator.h | 9 +- .../common/jni/android_jni_interface.cc | 3 +- .../common/jni/android_network_utility.cc | 27 ++--- .../common/jni/android_network_utility.h | 10 +- mobile/library/common/network/BUILD | 5 +- .../network/apple_platform_cert_verifier.cc | 30 ++--- .../network/apple_platform_cert_verifier.h | 18 +-- mobile/library/objective-c/EnvoyEngineImpl.mm | 5 - .../EnvoyCxxSwiftInterop/cxx_swift_interop.h | 1 - .../cert_validator/platform_bridge/BUILD | 1 + .../platform_bridge_cert_validator_test.cc | 104 ++++++++++-------- .../integration/client_integration_test.cc | 46 ++++++++ mobile/test/common/mocks/common/mocks.cc | 8 +- mobile/test/common/mocks/common/mocks.h | 3 + 29 files changed, 298 insertions(+), 186 deletions(-) create mode 100644 mobile/library/common/common/default_system_helper_apple.cc diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml index 4291842163fa..d6d53894ba14 100644 --- a/.github/workflows/mobile-android_tests.yml +++ b/.github/workflows/mobile-android_tests.yml @@ -82,6 +82,7 @@ jobs: --define envoy_mobile_listener=enabled \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --define=signal_trace=disabled \ + --define=system-helper=android \ //test/java/... kotlintestslinux: if: github.repository == 'envoyproxy/envoy' diff --git a/bazel/BUILD b/bazel/BUILD index e8f22adba5a8..c1557d082179 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -644,6 +644,11 @@ config_setting( values = {"define": "logger=android"}, ) +config_setting( + name = "android_system_helper", + values = {"define": "system-helper=android"}, +) + config_setting( name = "libfuzzer_coverage", define_values = { diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index 1c3929bd58c4..f26690c1292f 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -106,7 +106,8 @@ def envoy_cc_library( include_prefix = None, textual_hdrs = None, alwayslink = None, - defines = []): + defines = [], + linkopts = []): if tcmalloc_dep: deps += tcmalloc_external_deps(repository) @@ -123,6 +124,7 @@ def envoy_cc_library( srcs = srcs, hdrs = hdrs, copts = envoy_copts(repository) + envoy_pch_copts(repository, "//source/common/common:common_pch") + copts, + linkopts = linkopts, visibility = visibility, tags = tags, textual_hdrs = textual_hdrs, diff --git a/mobile/library/common/common/BUILD b/mobile/library/common/common/BUILD index ff561431c347..b8316a763fc6 100644 --- a/mobile/library/common/common/BUILD +++ b/mobile/library/common/common/BUILD @@ -1,28 +1,67 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@bazel_skylib//lib:selects.bzl", "selects") licenses(["notice"]) # Apache 2 envoy_package() +selects.config_setting_group( + name = "use_android_system_helper", + match_any = [ + "@envoy//bazel:android_system_helper", + "@envoy//bazel:android", + ], +) + envoy_cc_library( name = "system_helper_lib", srcs = [ "system_helper.cc", - ] + select({ - "@envoy//bazel:android": ["default_system_helper_android.cc"], - "//conditions:default": ["default_system_helper.cc"], - }), + ], hdrs = [ - "default_system_helper.h", "system_helper.h", ], + copts = select({ + ":use_android_system_helper": ["-DUSE_ANDROID_SYSTEM_HELPER"], + "//conditions:default": [], + }), repository = "@envoy", - deps = select({ - "@envoy//bazel:android": ["//library/common/jni:android_jni_utility_lib"], + deps = [ + ":default_system_helper_lib", + "//library/common/extensions/cert_validator/platform_bridge:c_types_lib", + ] + select({ + ":use_android_system_helper": [ + "//library/common/jni:android_jni_utility_lib", + "//library/common/jni:android_network_utility_lib", + ], + "//conditions:default": [], + }) + select({ + "@envoy//bazel:apple": [ + "//library/common/network:apple_platform_cert_verifier", + ], "//conditions:default": [], }), ) +# Ideally, Bazel select() would choose the appropriate implementation to use. +# It would use the Android implementation on Android, or when the config +# android_system_helper is set. Otherwise, it would use the Apple implementation +# on Apple platforms, or the default otherwise. However, Bazel select() doesn't +# have a good way to express this. Instead, system_helper.cc uses "#if defined" +# and "#include" to select the correct implementation. +envoy_cc_library( + name = "default_system_helper_lib", + srcs = [], + hdrs = [ + "default_system_helper.cc", + "default_system_helper.h", + "default_system_helper_android.cc", + "default_system_helper_apple.cc", + ], + repository = "@envoy", + deps = [], +) + envoy_cc_library( name = "lambda_logger_delegate_lib", srcs = ["lambda_logger_delegate.cc"], diff --git a/mobile/library/common/common/default_system_helper.cc b/mobile/library/common/common/default_system_helper.cc index cd70da12ffd8..877d105203ea 100644 --- a/mobile/library/common/common/default_system_helper.cc +++ b/mobile/library/common/common/default_system_helper.cc @@ -4,4 +4,16 @@ namespace Envoy { bool DefaultSystemHelper::isCleartextPermitted(absl::string_view /*hostname*/) { return true; } +envoy_cert_validation_result +DefaultSystemHelper::validateCertificateChain(const std::vector& /*certs*/, + absl::string_view /*hostname*/) { + envoy_cert_validation_result result; + result.result = ENVOY_FAILURE; + result.tls_alert = 80; // internal error + result.error_details = "Certificate verification not implemented."; + return result; +} + +void DefaultSystemHelper::cleanupAfterCertificateValidation() {} + } // namespace Envoy diff --git a/mobile/library/common/common/default_system_helper.h b/mobile/library/common/common/default_system_helper.h index 5ca7f4edbdc4..9bce1f061159 100644 --- a/mobile/library/common/common/default_system_helper.h +++ b/mobile/library/common/common/default_system_helper.h @@ -14,6 +14,9 @@ class DefaultSystemHelper : public SystemHelper { // SystemHelper: bool isCleartextPermitted(absl::string_view hostname) override; + envoy_cert_validation_result validateCertificateChain(const std::vector& certs, + absl::string_view hostname) override; + void cleanupAfterCertificateValidation() override; }; } // namespace Envoy diff --git a/mobile/library/common/common/default_system_helper_android.cc b/mobile/library/common/common/default_system_helper_android.cc index cfb9c9daf50f..0bc2062217ea 100644 --- a/mobile/library/common/common/default_system_helper_android.cc +++ b/mobile/library/common/common/default_system_helper_android.cc @@ -1,5 +1,6 @@ #include "library/common/common/default_system_helper.h" #include "library/common/jni/android_jni_utility.h" +#include "library/common/jni/android_network_utility.h" namespace Envoy { @@ -7,4 +8,12 @@ bool DefaultSystemHelper::isCleartextPermitted(absl::string_view hostname) { return is_cleartext_permitted(hostname); } +envoy_cert_validation_result +DefaultSystemHelper::validateCertificateChain(const std::vector& certs, + absl::string_view hostname) { + return verify_x509_cert_chain(certs, hostname); +} + +void DefaultSystemHelper::cleanupAfterCertificateValidation() { jvm_detach_thread(); } + } // namespace Envoy diff --git a/mobile/library/common/common/default_system_helper_apple.cc b/mobile/library/common/common/default_system_helper_apple.cc new file mode 100644 index 000000000000..592c62ec4b9a --- /dev/null +++ b/mobile/library/common/common/default_system_helper_apple.cc @@ -0,0 +1,16 @@ +#include "library/common/common/default_system_helper.h" +#include "library/common/network/apple_platform_cert_verifier.h" + +namespace Envoy { + +bool DefaultSystemHelper::isCleartextPermitted(absl::string_view /*hostname*/) { return true; } + +envoy_cert_validation_result +DefaultSystemHelper::validateCertificateChain(const std::vector& certs, + absl::string_view hostname) { + return verify_cert(certs, hostname); +} + +void DefaultSystemHelper::cleanupAfterCertificateValidation() {} + +} // namespace Envoy diff --git a/mobile/library/common/common/system_helper.cc b/mobile/library/common/common/system_helper.cc index a363a690cd68..c2f11fefcd9e 100644 --- a/mobile/library/common/common/system_helper.cc +++ b/mobile/library/common/common/system_helper.cc @@ -2,6 +2,14 @@ #include "library/common/common/default_system_helper.h" +#if defined(USE_ANDROID_SYSTEM_HELPER) +#include "library/common/common/default_system_helper_android.cc" +#elif defined(__APPLE__) +#include "library/common/common/default_system_helper_apple.cc" +#else +#include "library/common/common/default_system_helper.cc" +#endif + namespace Envoy { std::unique_ptr SystemHelper::instance_ = std::make_unique(); diff --git a/mobile/library/common/common/system_helper.h b/mobile/library/common/common/system_helper.h index bde21205d17e..b558c0e13c03 100644 --- a/mobile/library/common/common/system_helper.h +++ b/mobile/library/common/common/system_helper.h @@ -1,10 +1,12 @@ #pragma once #include +#include #include "envoy/common/pure.h" #include "absl/strings/string_view.h" +#include "library/common/extensions/cert_validator/platform_bridge/c_types.h" namespace Envoy { @@ -26,6 +28,17 @@ class SystemHelper { */ virtual bool isCleartextPermitted(absl::string_view hostname) PURE; + /** + * Invokes platform APIs to validate certificates. + */ + virtual envoy_cert_validation_result + validateCertificateChain(const std::vector& certs, absl::string_view hostname) PURE; + + /** + * Invokes platform APIs to clean up after validation is complete. + */ + virtual void cleanupAfterCertificateValidation() PURE; + /** * @return a reference to the current SystemHelper instance. */ diff --git a/mobile/library/common/extensions/cert_validator/platform_bridge/BUILD b/mobile/library/common/extensions/cert_validator/platform_bridge/BUILD index 879d2a4331fb..652cafcb6be3 100644 --- a/mobile/library/common/extensions/cert_validator/platform_bridge/BUILD +++ b/mobile/library/common/extensions/cert_validator/platform_bridge/BUILD @@ -34,6 +34,7 @@ envoy_cc_library( deps = [ ":c_types_lib", ":platform_bridge_cc_proto", + "//library/common/common:system_helper_lib", "@envoy//source/extensions/transport_sockets/tls/cert_validator:cert_validator_lib", ], ) diff --git a/mobile/library/common/extensions/cert_validator/platform_bridge/c_types.h b/mobile/library/common/extensions/cert_validator/platform_bridge/c_types.h index cfaf4c036c9b..ea178052a12a 100644 --- a/mobile/library/common/extensions/cert_validator/platform_bridge/c_types.h +++ b/mobile/library/common/extensions/cert_validator/platform_bridge/c_types.h @@ -13,30 +13,3 @@ typedef struct { uint8_t tls_alert; const char* error_details; } envoy_cert_validation_result; - -#ifdef __cplusplus -extern "C" { // function pointers -#endif - -/** - * Function signature for calling into platform APIs to validate certificates. - */ -typedef envoy_cert_validation_result (*envoy_validate_cert_f)(const envoy_data* certs, uint8_t size, - const char* host_name); - -/** - * Function signature for calling into platform APIs to clean up after validation is complete. - */ -typedef void (*envoy_validation_cleanup_f)(); - -#ifdef __cplusplus -} // function pointers -#endif - -/** - * A bag of function pointers to be registered in the platform registry. - */ -typedef struct { - envoy_validate_cert_f validate_cert; - envoy_validation_cleanup_f validation_cleanup; -} envoy_cert_validator; diff --git a/mobile/library/common/extensions/cert_validator/platform_bridge/config.cc b/mobile/library/common/extensions/cert_validator/platform_bridge/config.cc index 187457a5fc05..223e6ff8e925 100644 --- a/mobile/library/common/extensions/cert_validator/platform_bridge/config.cc +++ b/mobile/library/common/extensions/cert_validator/platform_bridge/config.cc @@ -10,11 +10,7 @@ namespace Tls { CertValidatorPtr PlatformBridgeCertValidatorFactory::createCertValidator( const Envoy::Ssl::CertificateValidationContextConfig* config, SslStats& stats, TimeSource& /*time_source*/) { - if (platform_validator_ == nullptr) { - platform_validator_ = - static_cast(Api::External::retrieveApi("platform_cert_validator")); - } - return std::make_unique(config, stats, platform_validator_); + return std::make_unique(config, stats); } REGISTER_FACTORY(PlatformBridgeCertValidatorFactory, CertValidatorFactory); diff --git a/mobile/library/common/extensions/cert_validator/platform_bridge/config.h b/mobile/library/common/extensions/cert_validator/platform_bridge/config.h index 00f1a364d3f6..6155381a882f 100644 --- a/mobile/library/common/extensions/cert_validator/platform_bridge/config.h +++ b/mobile/library/common/extensions/cert_validator/platform_bridge/config.h @@ -24,9 +24,6 @@ class PlatformBridgeCertValidatorFactory : public CertValidatorFactory, envoy_mobile::extensions::cert_validator::platform_bridge::PlatformBridgeCertValidator>(); } std::string category() const override { return "envoy.tls.cert_validator"; } - -private: - const envoy_cert_validator* platform_validator_ = nullptr; }; DECLARE_FACTORY(PlatformBridgeCertValidatorFactory); diff --git a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc index 0aa61cc41f5e..2918c7824e54 100644 --- a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc +++ b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc @@ -4,6 +4,7 @@ #include #include +#include "library/common/common/system_helper.h" #include "library/common/data/utility.h" namespace Envoy { @@ -12,13 +13,12 @@ namespace TransportSockets { namespace Tls { PlatformBridgeCertValidator::PlatformBridgeCertValidator( - const Envoy::Ssl::CertificateValidationContextConfig* config, SslStats& stats, - const envoy_cert_validator* platform_validator) + const Envoy::Ssl::CertificateValidationContextConfig* config, SslStats& stats) : allow_untrusted_certificate_(config != nullptr && config->trustChainVerification() == envoy::extensions::transport_sockets::tls::v3:: CertificateValidationContext::ACCEPT_UNTRUSTED), - platform_validator_(platform_validator), stats_(stats) { + stats_(stats) { ENVOY_BUG(config != nullptr && config->caCert().empty() && config->certificateRevocationList().empty(), "Invalid certificate validation context config."); @@ -53,7 +53,7 @@ ValidationResults PlatformBridgeCertValidator::doVerifyCertChain( Envoy::Ssl::ClientValidationStatus::NotValidated, absl::nullopt, error}; } - std::vector certs; + std::vector certs; for (uint64_t i = 0; i < sk_X509_num(&cert_chain); i++) { X509* cert = sk_X509_value(&cert_chain, i); unsigned char* cert_in_der = nullptr; @@ -62,7 +62,7 @@ ValidationResults PlatformBridgeCertValidator::doVerifyCertChain( absl::string_view cert_str(reinterpret_cast(cert_in_der), static_cast(der_length)); - certs.push_back(Data::Utility::copyToBridgeData(cert_str)); + certs.push_back(std::string(cert_str)); OPENSSL_free(cert_in_der); } @@ -84,9 +84,9 @@ ValidationResults PlatformBridgeCertValidator::doVerifyCertChain( ValidationJob job; job.result_callback_ = std::move(callback); - job.validation_thread_ = std::thread(&verifyCertChainByPlatform, platform_validator_, - &(job.result_callback_->dispatcher()), std::move(certs), - std::string(host), std::move(subject_alt_names), this); + job.validation_thread_ = + std::thread(&verifyCertChainByPlatform, &(job.result_callback_->dispatcher()), + std::move(certs), std::string(host), std::move(subject_alt_names), this); std::thread::id thread_id = job.validation_thread_.get_id(); validation_jobs_[thread_id] = std::move(job); return {ValidationResults::ValidationStatus::Pending, @@ -94,23 +94,22 @@ ValidationResults PlatformBridgeCertValidator::doVerifyCertChain( } void PlatformBridgeCertValidator::verifyCertChainByPlatform( - const envoy_cert_validator* platform_validator, Event::Dispatcher* dispatcher, - std::vector cert_chain, std::string hostname, + Event::Dispatcher* dispatcher, std::vector cert_chain, std::string hostname, std::vector subject_alt_names, PlatformBridgeCertValidator* parent) { ASSERT(!cert_chain.empty()); ENVOY_LOG(trace, "Start verifyCertChainByPlatform for host {}", hostname); // This is running in a stand alone thread other than the engine thread. - envoy_data leaf_cert_der = cert_chain[0]; - bssl::UniquePtr leaf_cert(d2i_X509( - nullptr, const_cast(&leaf_cert_der.bytes), leaf_cert_der.length)); + const std::string& leaf_cert_der = cert_chain[0]; + const unsigned char* leaf_cert_data = + reinterpret_cast(leaf_cert_der.data()); + bssl::UniquePtr leaf_cert(d2i_X509(nullptr, &leaf_cert_data, leaf_cert_der.length())); envoy_cert_validation_result result = - platform_validator->validate_cert(cert_chain.data(), cert_chain.size(), hostname.c_str()); + SystemHelper::getInstance().validateCertificateChain(cert_chain, hostname); bool success = result.result == ENVOY_SUCCESS; if (!success) { ENVOY_LOG(debug, result.error_details); postVerifyResultAndCleanUp(success, std::move(hostname), result.error_details, result.tls_alert, - ValidationFailureType::FAIL_VERIFY_ERROR, platform_validator, - dispatcher, parent); + ValidationFailureType::FAIL_VERIFY_ERROR, dispatcher, parent); return; } @@ -123,26 +122,26 @@ void PlatformBridgeCertValidator::verifyCertChainByPlatform( error_details = "PlatformBridgeCertValidator_verifySubjectAltName failed: SNI mismatch."; ENVOY_LOG(debug, error_details); postVerifyResultAndCleanUp(success, std::move(hostname), error_details, SSL_AD_BAD_CERTIFICATE, - ValidationFailureType::FAIL_VERIFY_SAN, platform_validator, - dispatcher, parent); + ValidationFailureType::FAIL_VERIFY_SAN, dispatcher, parent); return; } postVerifyResultAndCleanUp(success, std::move(hostname), error_details, - SSL_AD_CERTIFICATE_UNKNOWN, ValidationFailureType::SUCCESS, - platform_validator, dispatcher, parent); + SSL_AD_CERTIFICATE_UNKNOWN, ValidationFailureType::SUCCESS, dispatcher, + parent); } -void PlatformBridgeCertValidator::postVerifyResultAndCleanUp( - bool success, std::string hostname, absl::string_view error_details, uint8_t tls_alert, - ValidationFailureType failure_type, const envoy_cert_validator* platform_validator, - Event::Dispatcher* dispatcher, PlatformBridgeCertValidator* parent) { +void PlatformBridgeCertValidator::postVerifyResultAndCleanUp(bool success, std::string hostname, + absl::string_view error_details, + uint8_t tls_alert, + ValidationFailureType failure_type, + Event::Dispatcher* dispatcher, + PlatformBridgeCertValidator* parent) { ENVOY_LOG(trace, "Finished platform cert validation for {}, post result callback to network thread", hostname); - if (platform_validator->validation_cleanup) { - platform_validator->validation_cleanup(); - } + SystemHelper::getInstance().cleanupAfterCertificateValidation(); + std::weak_ptr weak_alive_indicator(parent->alive_indicator_); dispatcher->post([weak_alive_indicator, success, hostname = std::move(hostname), diff --git a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.h b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.h index 8ece9b644dab..c2912c77799b 100644 --- a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.h +++ b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.h @@ -19,7 +19,7 @@ namespace Tls { class PlatformBridgeCertValidator : public CertValidator, Logger::Loggable { public: PlatformBridgeCertValidator(const Envoy::Ssl::CertificateValidationContextConfig* config, - SslStats& stats, const envoy_cert_validator* platform_validator); + SslStats& stats); ~PlatformBridgeCertValidator() override; @@ -73,9 +73,8 @@ class PlatformBridgeCertValidator : public CertValidator, Logger::Loggable cert_chain, std::string hostname, + static void verifyCertChainByPlatform(Event::Dispatcher* dispatcher, + std::vector cert_chain, std::string hostname, std::vector subject_alt_names, PlatformBridgeCertValidator* parent); @@ -83,7 +82,6 @@ class PlatformBridgeCertValidator : public CertValidator, Logger::Loggable validation_jobs_; std::shared_ptr alive_indicator_{new size_t(1)}; diff --git a/mobile/library/common/jni/android_jni_interface.cc b/mobile/library/common/jni/android_jni_interface.cc index a1c1336fad5e..834e41f83a2a 100644 --- a/mobile/library/common/jni/android_jni_interface.cc +++ b/mobile/library/common/jni/android_jni_interface.cc @@ -13,6 +13,5 @@ Java_io_envoyproxy_envoymobile_engine_AndroidJniLibrary_initialize(JNIEnv* env, jclass, // class jobject class_loader) { set_class_loader(env->NewGlobalRef(class_loader)); - // At this point, we know Android APIs are available. Register cert chain validation JNI calls. - return register_platform_api(cert_validator_name, get_android_cert_validator_api()); + return ENVOY_SUCCESS; } diff --git a/mobile/library/common/jni/android_network_utility.cc b/mobile/library/common/jni/android_network_utility.cc index 0c8690f11f79..976a8e9c6967 100644 --- a/mobile/library/common/jni/android_network_utility.cc +++ b/mobile/library/common/jni/android_network_utility.cc @@ -67,7 +67,7 @@ static void ExtractCertVerifyResult(JNIEnv* env, jobject result, envoy_cert_veri // `auth_type` and `host` are expected to be UTF-8 encoded. jobject call_jvm_verify_x509_cert_chain(JNIEnv* env, const std::vector& cert_chain, - std::string auth_type, std::string host) { + std::string auth_type, absl::string_view hostname) { jni_log("[Envoy]", "jvm_verify_x509_cert_chain"); jclass jcls_AndroidNetworkLibrary = find_class("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); @@ -77,7 +77,8 @@ jobject call_jvm_verify_x509_cert_chain(JNIEnv* env, const std::vector(hostname.data()), hostname.length()); jobject result = env->CallStaticObjectMethod(jcls_AndroidNetworkLibrary, jmid_verifyServerCertificates, chain_byte_array, auth_string, host_string); @@ -91,12 +92,12 @@ jobject call_jvm_verify_x509_cert_chain(JNIEnv* env, const std::vector& cert_chain, - std::string auth_type, std::string host, + std::string auth_type, absl::string_view hostname, envoy_cert_verify_status_t* status, bool* is_issued_by_known_root, std::vector* verified_chain) { JNIEnv* env = get_env(); - jobject result = call_jvm_verify_x509_cert_chain(env, cert_chain, auth_type, host); + jobject result = call_jvm_verify_x509_cert_chain(env, cert_chain, auth_type, hostname); if (Envoy::JNI::Exception::checkAndClear()) { *status = CERT_VERIFY_STATUS_NOT_YET_VALID; } else { @@ -108,22 +109,21 @@ static void jvm_verify_x509_cert_chain(const std::vector& cert_chai env->DeleteLocalRef(result); } -static envoy_cert_validation_result verify_x509_cert_chain(const envoy_data* certs, uint8_t size, - const char* host_name) { +envoy_cert_validation_result verify_x509_cert_chain(const std::vector& certs, + absl::string_view hostname) { jni_log("[Envoy]", "verify_x509_cert_chain"); envoy_cert_verify_status_t result; bool is_issued_by_known_root; std::vector verified_chain; std::vector cert_chain; - for (uint8_t i = 0; i < size; ++i) { - cert_chain.push_back(Envoy::Data::Utility::copyToString(certs[i])); - release_envoy_data(certs[i]); + for (absl::string_view cert : certs) { + cert_chain.push_back(std::string(cert)); } // Android ignores the authType parameter to X509TrustManager.checkServerTrusted, so pass in "RSA" // as dummy value. See https://crbug.com/627154. - jvm_verify_x509_cert_chain(cert_chain, "RSA", host_name, &result, &is_issued_by_known_root, + jvm_verify_x509_cert_chain(cert_chain, "RSA", hostname, &result, &is_issued_by_known_root, &verified_chain); switch (result) { case CERT_VERIFY_STATUS_OK: @@ -152,10 +152,3 @@ static envoy_cert_validation_result verify_x509_cert_chain(const envoy_data* cer } void jvm_detach_thread() { Envoy::JNI::JavaVirtualMachine::detachCurrentThread(); } - -envoy_cert_validator* get_android_cert_validator_api() { - envoy_cert_validator* api = (envoy_cert_validator*)safe_malloc(sizeof(envoy_cert_validator)); - api->validate_cert = verify_x509_cert_chain; - api->validation_cleanup = jvm_detach_thread; - return api; -} diff --git a/mobile/library/common/jni/android_network_utility.h b/mobile/library/common/jni/android_network_utility.h index 52366113c17f..81534b185fd1 100644 --- a/mobile/library/common/jni/android_network_utility.h +++ b/mobile/library/common/jni/android_network_utility.h @@ -3,6 +3,7 @@ #include #include +#include "absl/strings/string_view.h" #include "library/common/api/c_types.h" #include "library/common/extensions/cert_validator/platform_bridge/c_types.h" #include "library/common/jni/import/jni_import.h" @@ -12,10 +13,9 @@ /* Calls up through JNI to validate given certificates. */ jobject call_jvm_verify_x509_cert_chain(JNIEnv* env, const std::vector& cert_chain, - std::string auth_type, std::string host); + std::string auth_type, absl::string_view hostname); -/* Returns a group of C functions to do certificates validation using AndroidNetworkLibrary. - */ -envoy_cert_validator* get_android_cert_validator_api(); +envoy_cert_validation_result verify_x509_cert_chain(const std::vector& certs, + absl::string_view hostname); -static constexpr const char* cert_validator_name = "platform_cert_validator"; +void jvm_detach_thread(); diff --git a/mobile/library/common/network/BUILD b/mobile/library/common/network/BUILD index f0275196f93b..82728955e042 100644 --- a/mobile/library/common/network/BUILD +++ b/mobile/library/common/network/BUILD @@ -74,10 +74,13 @@ envoy_cc_library( "@envoy//bazel:apple": ["apple_platform_cert_verifier.h"], "//conditions:default": [], }), + linkopts = select({ + "@envoy//bazel:apple": ["-framework Security"], + "//conditions:default": [], + }), repository = "@envoy", deps = select({ "@envoy//bazel:apple": [ - "//library/common:envoy_main_interface_lib", "//library/common/extensions/cert_validator/platform_bridge:c_types_lib", "@envoy//bazel:boringssl", ], diff --git a/mobile/library/common/network/apple_platform_cert_verifier.cc b/mobile/library/common/network/apple_platform_cert_verifier.cc index 8d71524e192f..0bbf417f160d 100644 --- a/mobile/library/common/network/apple_platform_cert_verifier.cc +++ b/mobile/library/common/network/apple_platform_cert_verifier.cc @@ -7,7 +7,6 @@ #include #include "library/common/extensions/cert_validator/platform_bridge/c_types.h" -#include "library/common/main_interface.h" #include "openssl/ssl.h" // NOLINT(namespace-envoy) @@ -35,7 +34,7 @@ CFMutableArrayRef CreateTrustPolicies() { // Returns a new CFMutableArrayRef containing the specified certificates // in the form expected by Security.framework and Keychain Services, or // NULL on failure. -CFMutableArrayRef CreateSecCertificateArray(const envoy_data* certs, uint8_t num_certs) { +CFMutableArrayRef CreateSecCertificateArray(const std::vector& certs) { CFMutableArrayRef cert_array = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks); @@ -43,8 +42,9 @@ CFMutableArrayRef CreateSecCertificateArray(const envoy_data* certs, uint8_t num return NULL; } - for (uint8_t i = 0; i < num_certs; ++i) { - CFDataRef cert_data = CFDataCreate(kCFAllocatorDefault, certs[i].bytes, certs[i].length); + for (absl::string_view cert : certs) { + CFDataRef cert_data = CFDataCreate( + kCFAllocatorDefault, reinterpret_cast(cert.data()), cert.length()); if (!cert_data) { CFRelease(cert_array); return NULL; @@ -70,15 +70,15 @@ envoy_cert_validation_result make_result(envoy_status_t status, uint8_t tls_aler return result; } -static envoy_cert_validation_result verify_cert(const envoy_data* certs, uint8_t num_certs, - const char* /*hostname*/) { +envoy_cert_validation_result verify_cert(const std::vector& certs, + absl::string_view /*hostname*/) { CFArrayRef trust_policies = CreateTrustPolicies(); if (!trust_policies) { return make_result(ENVOY_FAILURE, SSL_AD_CERTIFICATE_UNKNOWN, "validation couldn't be conducted."); } - CFMutableArrayRef cert_array = CreateSecCertificateArray(certs, num_certs); + CFMutableArrayRef cert_array = CreateSecCertificateArray(certs); if (!cert_array) { return make_result(ENVOY_FAILURE, SSL_AD_CERTIFICATE_UNKNOWN, "validation couldn't be conducted."); @@ -103,19 +103,3 @@ static envoy_cert_validation_result verify_cert(const envoy_data* certs, uint8_t } return make_result(ENVOY_SUCCESS, 0, ""); } - -#ifdef __cplusplus -extern "C" { -#endif - -void register_apple_platform_cert_verifier() { - envoy_cert_validator* api = - static_cast(safe_malloc(sizeof(envoy_cert_validator))); - api->validate_cert = verify_cert; - api->validation_cleanup = NULL; - register_platform_api("platform_cert_validator", api); -} - -#ifdef __cplusplus -} -#endif diff --git a/mobile/library/common/network/apple_platform_cert_verifier.h b/mobile/library/common/network/apple_platform_cert_verifier.h index 63ba9e39ca53..b559d41a4978 100644 --- a/mobile/library/common/network/apple_platform_cert_verifier.h +++ b/mobile/library/common/network/apple_platform_cert_verifier.h @@ -1,16 +1,10 @@ #pragma once -// NOLINT(namespace-envoy) - -#ifdef __cplusplus -extern "C" { -#endif +#include -/** - * Registers the Apple platform cert verifier API. - */ -void register_apple_platform_cert_verifier(); +#include "absl/strings/string_view.h" +#include "library/common/extensions/cert_validator/platform_bridge/c_types.h" -#ifdef __cplusplus -} -#endif +// NOLINT(namespace-envoy) +envoy_cert_validation_result verify_cert(const std::vector& certs, + absl::string_view hostname); diff --git a/mobile/library/objective-c/EnvoyEngineImpl.mm b/mobile/library/objective-c/EnvoyEngineImpl.mm index 29d0279b696f..5bea922541be 100644 --- a/mobile/library/objective-c/EnvoyEngineImpl.mm +++ b/mobile/library/objective-c/EnvoyEngineImpl.mm @@ -6,7 +6,6 @@ #include "library/common/api/c_types.h" #import "library/common/main_interface.h" -#import "library/common/network/apple_platform_cert_verifier.h" #import "library/common/types/c_types.h" #import "library/common/extensions/key_value/platform/c_types.h" #import "library/cc/engine_builder.h" @@ -510,10 +509,6 @@ - (void)performRegistrationsForConfig:(EnvoyConfiguration *)config { for (NSString *name in config.keyValueStores) { [self registerKeyValueStore:name keyValueStore:config.keyValueStores[name]]; } - - if (config.enablePlatformCertificateValidation) { - register_apple_platform_cert_verifier(); - } } - (int)runWithConfig:(EnvoyConfiguration *)config logLevel:(NSString *)logLevel { diff --git a/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.h b/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.h index ea89f4ee482f..d1506c3914ca 100644 --- a/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.h +++ b/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.h @@ -4,7 +4,6 @@ #include "library/common/data/utility.h" #include "library/common/extensions/filters/http/platform_bridge/c_types.h" #include "library/common/main_interface.h" -#include "library/common/network/apple_platform_cert_verifier.h" #include "library/common/stats/utility.h" // This file exists in order to expose headers for Envoy's C++ libraries diff --git a/mobile/test/common/extensions/cert_validator/platform_bridge/BUILD b/mobile/test/common/extensions/cert_validator/platform_bridge/BUILD index 9000ab4f2bc9..270bdd75a91e 100644 --- a/mobile/test/common/extensions/cert_validator/platform_bridge/BUILD +++ b/mobile/test/common/extensions/cert_validator/platform_bridge/BUILD @@ -18,6 +18,7 @@ envoy_extension_cc_test( repository = "@envoy", deps = [ "//library/common/extensions/cert_validator/platform_bridge:config", + "//test/common/mocks/common:common_mocks", "@envoy//source/extensions/transport_sockets/tls/cert_validator:cert_validator_lib", "@envoy//test/extensions/transport_sockets/tls:ssl_test_utils", "@envoy//test/extensions/transport_sockets/tls/cert_validator:test_common", diff --git a/mobile/test/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator_test.cc b/mobile/test/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator_test.cc index 41d559fdcecb..76f7b46a6bf2 100644 --- a/mobile/test/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator_test.cc +++ b/mobile/test/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator_test.cc @@ -9,6 +9,7 @@ #include "source/extensions/transport_sockets/tls/cert_validator/san_matcher.h" #include "source/extensions/transport_sockets/tls/stats.h" +#include "test/common/mocks/common/mocks.h" #include "test/common/stats/stat_test_utility.h" #include "test/extensions/transport_sockets/tls/cert_validator/test_common.h" #include "test/extensions/transport_sockets/tls/ssl_test_utility.h" @@ -35,6 +36,7 @@ using testing::NiceMock; using testing::Return; using testing::ReturnRef; using testing::StrEq; +using testing::WithArgs; namespace Envoy { namespace Extensions { @@ -53,7 +55,8 @@ class MockValidateResultCallback : public Ssl::ValidateResultCallback { class MockValidator { public: MOCK_METHOD(void, cleanup, ()); - MOCK_METHOD(envoy_cert_validation_result, validate, (const envoy_data*, uint8_t, const char*)); + MOCK_METHOD(envoy_cert_validation_result, validate, + (const std::vector& certs, absl::string_view hostname)); }; class PlatformBridgeCertValidatorTest @@ -62,12 +65,17 @@ class PlatformBridgeCertValidatorTest PlatformBridgeCertValidatorTest() : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")), stats_(generateSslStats(*test_store_.rootScope())), ssl_ctx_(SSL_CTX_new(TLS_method())), - callback_(std::make_unique()), is_server_(false) { - mock_validator_ = std::make_unique(); - main_thread_id_ = std::this_thread::get_id(); - - platform_validator_.validate_cert = (PlatformBridgeCertValidatorTest::validate); - platform_validator_.validation_cleanup = (PlatformBridgeCertValidatorTest::cleanup); + callback_(std::make_unique()), is_server_(false), + mock_validator_(std::make_unique()), + main_thread_id_(std::this_thread::get_id()), + helper_handle_(test::SystemHelperPeer::replaceSystemHelper()) { + ON_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)) + .WillByDefault(WithArgs<0, 1>(Invoke(this, &PlatformBridgeCertValidatorTest::validate))); + ON_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()) + .WillByDefault(Invoke(this, &PlatformBridgeCertValidatorTest::cleanup)); + + EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)) + .WillRepeatedly(Return(true)); } void initializeConfig() { @@ -93,27 +101,24 @@ class PlatformBridgeCertValidatorTest return GetParam() == CertificateValidationContext::ACCEPT_UNTRUSTED; } - static envoy_cert_validation_result validate(const envoy_data* certs, uint8_t size, - const char* hostname) { + envoy_cert_validation_result validate(const std::vector& certs, + absl::string_view hostname) { // Validate must be called on the worker thread, not the main thread. EXPECT_NE(main_thread_id_, std::this_thread::get_id()); // Make sure the cert was converted correctly. - const Buffer::InstancePtr buffer = Data::Utility::toInternalData(*certs); + const Buffer::InstancePtr buffer(new Buffer::OwnedImpl(certs[0])); const auto digest = Common::Crypto::UtilitySingleton::get().getSha256Digest(*buffer); EXPECT_EQ(TEST_SAN_DNS2_CERT_256_HASH, Hex::encode(digest)); - return mock_validator_->validate(certs, size, hostname); + return mock_validator_->validate(certs, hostname); } - static void cleanup() { + void cleanup() { // Validate must be called on the worker thread, not the main thread. EXPECT_NE(main_thread_id_, std::this_thread::get_id()); mock_validator_->cleanup(); } - static std::unique_ptr mock_validator_; - static std::thread::id main_thread_id_; - Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; Stats::TestUtil::TestStore test_store_; @@ -125,20 +130,18 @@ class PlatformBridgeCertValidatorTest Network::TransportSocketOptionsConstSharedPtr transport_socket_options_; std::unique_ptr callback_; bool is_server_; - envoy_cert_validator platform_validator_; + std::unique_ptr mock_validator_; + std::thread::id main_thread_id_; + std::unique_ptr helper_handle_; }; -std::unique_ptr PlatformBridgeCertValidatorTest::mock_validator_; -std::thread::id PlatformBridgeCertValidatorTest::main_thread_id_; - INSTANTIATE_TEST_SUITE_P(TrustMode, PlatformBridgeCertValidatorTest, testing::ValuesIn({CertificateValidationContext::VERIFY_TRUST_CHAIN, CertificateValidationContext::ACCEPT_UNTRUSTED})); TEST_P(PlatformBridgeCertValidatorTest, NoConfig) { - EXPECT_ENVOY_BUG( - { PlatformBridgeCertValidator validator(nullptr, stats_, &platform_validator_); }, - "Invalid certificate validation context config."); + EXPECT_ENVOY_BUG({ PlatformBridgeCertValidator validator(nullptr, stats_); }, + "Invalid certificate validation context config."); } TEST_P(PlatformBridgeCertValidatorTest, NonEmptyCaCert) { @@ -147,9 +150,8 @@ TEST_P(PlatformBridgeCertValidatorTest, NonEmptyCaCert) { EXPECT_CALL(config_, certificateRevocationList()).WillRepeatedly(ReturnRef(empty_string_)); EXPECT_CALL(config_, trustChainVerification()).WillRepeatedly(Return(GetParam())); - EXPECT_ENVOY_BUG( - { PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); }, - "Invalid certificate validation context config."); + EXPECT_ENVOY_BUG({ PlatformBridgeCertValidator validator(&config_, stats_); }, + "Invalid certificate validation context config."); } TEST_P(PlatformBridgeCertValidatorTest, NonEmptyRevocationList) { @@ -158,14 +160,13 @@ TEST_P(PlatformBridgeCertValidatorTest, NonEmptyRevocationList) { EXPECT_CALL(config_, certificateRevocationList()).WillRepeatedly(ReturnRef(revocation_list)); EXPECT_CALL(config_, trustChainVerification()).WillRepeatedly(Return(GetParam())); - EXPECT_ENVOY_BUG( - { PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); }, - "Invalid certificate validation context config."); + EXPECT_ENVOY_BUG({ PlatformBridgeCertValidator validator(&config_, stats_); }, + "Invalid certificate validation context config."); } TEST_P(PlatformBridgeCertValidatorTest, NoCallback) { initializeConfig(); - PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); + PlatformBridgeCertValidator validator(&config_, stats_); bssl::UniquePtr cert_chain = readCertChainFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns2_cert.pem")); @@ -182,7 +183,7 @@ TEST_P(PlatformBridgeCertValidatorTest, NoCallback) { TEST_P(PlatformBridgeCertValidatorTest, EmptyCertChain) { initializeConfig(); - PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); + PlatformBridgeCertValidator validator(&config_, stats_); bssl::UniquePtr cert_chain(sk_X509_new_null()); std::string hostname = "www.example.com"; @@ -198,14 +199,17 @@ TEST_P(PlatformBridgeCertValidatorTest, EmptyCertChain) { } TEST_P(PlatformBridgeCertValidatorTest, ValidCertificate) { + EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); + EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); + initializeConfig(); - PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); + PlatformBridgeCertValidator validator(&config_, stats_); std::string hostname = "server1.example.com"; bssl::UniquePtr cert_chain = readCertChainFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns2_cert.pem")); envoy_cert_validation_result result = {ENVOY_SUCCESS, 0, NULL}; - EXPECT_CALL(*mock_validator_, validate(_, _, _)).WillOnce(Return(result)); + EXPECT_CALL(*mock_validator_, validate(_, _)).WillOnce(Return(result)); EXPECT_CALL(*mock_validator_, cleanup()); auto& callback_ref = *callback_; EXPECT_CALL(callback_ref, dispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); @@ -225,14 +229,17 @@ TEST_P(PlatformBridgeCertValidatorTest, ValidCertificate) { } TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateEmptySanOverrides) { + EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); + EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); + initializeConfig(); - PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); + PlatformBridgeCertValidator validator(&config_, stats_); std::string hostname = "server1.example.com"; bssl::UniquePtr cert_chain = readCertChainFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns2_cert.pem")); envoy_cert_validation_result result = {ENVOY_SUCCESS, 0, NULL}; - EXPECT_CALL(*mock_validator_, validate(_, _, _)).WillOnce(Return(result)); + EXPECT_CALL(*mock_validator_, validate(_, _)).WillOnce(Return(result)); EXPECT_CALL(*mock_validator_, cleanup()); auto& callback_ref = *callback_; EXPECT_CALL(callback_ref, dispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); @@ -257,14 +264,17 @@ TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateEmptySanOverrides) { } TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateEmptyHostNoOverrides) { + EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); + EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); + initializeConfig(); - PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); + PlatformBridgeCertValidator validator(&config_, stats_); std::string hostname = ""; bssl::UniquePtr cert_chain = readCertChainFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns2_cert.pem")); envoy_cert_validation_result result = {ENVOY_SUCCESS, 0, NULL}; - EXPECT_CALL(*mock_validator_, validate(_, _, _)).WillOnce(Return(result)); + EXPECT_CALL(*mock_validator_, validate(_, _)).WillOnce(Return(result)); EXPECT_CALL(*mock_validator_, cleanup()); auto& callback_ref = *callback_; EXPECT_CALL(callback_ref, dispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); @@ -289,14 +299,17 @@ TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateEmptyHostNoOverrides) { } TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateButInvalidSni) { + EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); + EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); + initializeConfig(); - PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); + PlatformBridgeCertValidator validator(&config_, stats_); std::string hostname = "server2.example.com"; bssl::UniquePtr cert_chain = readCertChainFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns2_cert.pem")); envoy_cert_validation_result result = {ENVOY_SUCCESS, 0, NULL}; - EXPECT_CALL(*mock_validator_, validate(_, _, _)).WillOnce(Return(result)); + EXPECT_CALL(*mock_validator_, validate(_, _)).WillOnce(Return(result)); EXPECT_CALL(*mock_validator_, cleanup()); auto& callback_ref = *callback_; EXPECT_CALL(callback_ref, dispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); @@ -316,8 +329,11 @@ TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateButInvalidSni) { } TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateSniOverride) { + EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); + EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); + initializeConfig(); - PlatformBridgeCertValidator validator(&config_, stats_, &platform_validator_); + PlatformBridgeCertValidator validator(&config_, stats_); std::vector subject_alt_names = {"server1.example.com"}; @@ -325,7 +341,7 @@ TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateSniOverride) { bssl::UniquePtr cert_chain = readCertChainFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns2_cert.pem")); envoy_cert_validation_result result = {ENVOY_SUCCESS, 0, NULL}; - EXPECT_CALL(*mock_validator_, validate(_, _, StrEq(subject_alt_names[0].c_str()))) + EXPECT_CALL(*mock_validator_, validate(_, StrEq(subject_alt_names[0].c_str()))) .WillOnce(Return(result)); EXPECT_CALL(*mock_validator_, cleanup()); auto& callback_ref = *callback_; @@ -346,15 +362,17 @@ TEST_P(PlatformBridgeCertValidatorTest, ValidCertificateSniOverride) { } TEST_P(PlatformBridgeCertValidatorTest, DeletedWithValidationPending) { + EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); + EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); + initializeConfig(); - auto validator = - std::make_unique(&config_, stats_, &platform_validator_); + auto validator = std::make_unique(&config_, stats_); std::string hostname = "server1.example.com"; bssl::UniquePtr cert_chain = readCertChainFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns2_cert.pem")); envoy_cert_validation_result result = {ENVOY_SUCCESS, 0, NULL}; - EXPECT_CALL(*mock_validator_, validate(_, _, _)).WillOnce(Return(result)); + EXPECT_CALL(*mock_validator_, validate(_, _)).WillOnce(Return(result)); EXPECT_CALL(*mock_validator_, cleanup()); auto& callback_ref = *callback_; EXPECT_CALL(callback_ref, dispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index a85f1b0eef7d..f71731f39fce 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -109,6 +109,52 @@ TEST_P(ClientIntegrationTest, ClearTextNotPermitted) { ASSERT_EQ(cc_.on_complete_calls, 1); } +TEST_P(ClientIntegrationTest, BasicHttps) { + EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).Times(0); + EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); + EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); + + builder_.enablePlatformCertificatesValidation(true); + + upstream_tls_ = true; + + initialize(); + default_request_headers_.setScheme("https"); + + Buffer::OwnedImpl request_data = Buffer::OwnedImpl("request body"); + default_request_headers_.addCopy(AutonomousStream::EXPECT_REQUEST_SIZE_BYTES, + std::to_string(request_data.length())); + + stream_prototype_->setOnData([this](envoy_data c_data, bool end_stream) { + if (end_stream) { + EXPECT_EQ(Data::Utility::copyToString(c_data), ""); + } else { + EXPECT_EQ(c_data.length, 10); + } + cc_.on_data_calls++; + release_envoy_data(c_data); + }); + + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), false); + + envoy_data c_data = Data::Utility::toBridgeData(request_data); + stream_->sendData(c_data); + + Platform::RequestTrailersBuilder builder; + std::shared_ptr trailers = + std::make_shared(builder.build()); + stream_->close(trailers); + + terminal_callback_.waitReady(); + + ASSERT_EQ(cc_.on_headers_calls, 1); + ASSERT_EQ(cc_.status, "200"); + ASSERT_EQ(cc_.on_data_calls, 2); + ASSERT_EQ(cc_.on_complete_calls, 1); + ASSERT_EQ(cc_.on_header_consumed_bytes_from_response, 27); + ASSERT_EQ(cc_.on_complete_received_byte_count, 67); +} + TEST_P(ClientIntegrationTest, BasicNon2xx) { initialize(); diff --git a/mobile/test/common/mocks/common/mocks.cc b/mobile/test/common/mocks/common/mocks.cc index 1a394cf0424b..c853b5230e2c 100644 --- a/mobile/test/common/mocks/common/mocks.cc +++ b/mobile/test/common/mocks/common/mocks.cc @@ -4,9 +4,15 @@ namespace Envoy { namespace test { using testing::_; +using testing::Return; MockSystemHelper::MockSystemHelper() { - ON_CALL(*this, isCleartextPermitted(_)).WillByDefault(testing::Return(true)); + ON_CALL(*this, isCleartextPermitted(_)).WillByDefault(Return(true)); + envoy_cert_validation_result success; + success.result = ENVOY_SUCCESS; + success.tls_alert = 0; + success.error_details = ""; + ON_CALL(*this, validateCertificateChain(_, _)).WillByDefault(Return(success)); } } // namespace test diff --git a/mobile/test/common/mocks/common/mocks.h b/mobile/test/common/mocks/common/mocks.h index 8d876bb10101..76ac20556474 100644 --- a/mobile/test/common/mocks/common/mocks.h +++ b/mobile/test/common/mocks/common/mocks.h @@ -11,6 +11,9 @@ class MockSystemHelper : public SystemHelper { // SystemHelper: MOCK_METHOD(bool, isCleartextPermitted, (absl::string_view hostname)); + MOCK_METHOD(envoy_cert_validation_result, validateCertificateChain, + (const std::vector& certs, absl::string_view hostname)); + MOCK_METHOD(void, cleanupAfterCertificateValidation, ()); }; // SystemHelperPeer allows the replacement of the SystemHelper singleton From b896b6dfc266e6a16f7106da2f17f437db1d7f5c Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 9 May 2023 00:06:43 +0100 Subject: [PATCH 156/740] contrib: Shift `vpp` build -> `rules_foreign_cc` (#27243) Signed-off-by: Ryan Northey --- bazel/envoy_build_system.bzl | 9 +- bazel/foreign_cc/vpp_vcl.patch | 158 ++++++++++++++++++++++- bazel/repositories.bzl | 2 +- contrib/vcl/source/BUILD | 71 ++++++---- contrib/vcl/source/vcl_build_launcher.py | 29 ----- contrib/vcl/source/vpp_vcl_build.sh | 51 -------- 6 files changed, 207 insertions(+), 113 deletions(-) delete mode 100644 contrib/vcl/source/vcl_build_launcher.py delete mode 100755 contrib/vcl/source/vpp_vcl_build.sh diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index a64c9babbfea..da025768f034 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -117,14 +117,17 @@ def envoy_cmake( name, cache_entries = {}, debug_cache_entries = {}, + default_cache_entries = {"CMAKE_BUILD_TYPE": "Bazel"}, lib_source = "", postfix_script = "", copy_pdb = False, pdb_name = "", cmake_files_dir = "$BUILD_TMPDIR/CMakeFiles", generate_crosstool_file = False, + generate_args = ["-GNinja"], + targets = ["", "install"], **kwargs): - cache_entries.update({"CMAKE_BUILD_TYPE": "Bazel"}) + cache_entries.update(default_cache_entries) cache_entries_debug = dict(cache_entries) cache_entries_debug.update(debug_cache_entries) @@ -152,8 +155,8 @@ def envoy_cmake( "@envoy//bazel:dbg_build": cache_entries_debug, "//conditions:default": cache_entries, }), - generate_args = ["-GNinja"], - targets = ["", "install"], + generate_args = generate_args, + targets = targets, # TODO: Remove install target and make this work install = False, # TODO(lizan): Make this always true diff --git a/bazel/foreign_cc/vpp_vcl.patch b/bazel/foreign_cc/vpp_vcl.patch index 56635f3bb338..769fc2c877d0 100644 --- a/bazel/foreign_cc/vpp_vcl.patch +++ b/bazel/foreign_cc/vpp_vcl.patch @@ -1,4 +1,5 @@ -# Not a git repo so embed version +diff --git src/CMakeLists.txt src/CMakeLists.txt +index 4be247333..230c667ff 100644 --- src/CMakeLists.txt +++ src/CMakeLists.txt @@ -40,12 +40,8 @@ include(cmake/ccache.cmake) @@ -16,16 +17,18 @@ string(REPLACE "-" ";" VPP_LIB_VERSION ${VPP_VERSION}) list(GET VPP_LIB_VERSION 0 VPP_LIB_VERSION) -@@ -188,7 +184,7 @@ elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") +@@ -188,8 +184,7 @@ elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") find_package(OpenSSL) set(SUBDIRS vppinfra svm vlib vlibmemory vlibapi vnet vpp vat vat2 vcl vpp-api - plugins tools/vppapigen tools/g2 tools/perftool cmake pkg -+ tools/vppapigen tools/g2 tools/perftool cmake pkg - tools/appimage +- tools/appimage ++ tools/vppapigen cmake pkg ) elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") - + set(SUBDIRS vppinfra) +diff --git src/cmake/ccache.cmake src/cmake/ccache.cmake +index 058a0f3d8..30dcb0c15 100644 --- src/cmake/ccache.cmake +++ src/cmake/ccache.cmake @@ -14,7 +14,7 @@ @@ -37,7 +40,8 @@ if(VPP_USE_CCACHE) find_program(CCACHE_FOUND ccache) message(STATUS "Looking for ccache") - +diff --git src/cmake/library.cmake src/cmake/library.cmake +index ad4adfcab..0051bca10 100644 --- src/cmake/library.cmake +++ src/cmake/library.cmake @@ -24,7 +24,7 @@ macro(add_vpp_library lib) @@ -49,3 +53,145 @@ target_sources(${lib} PRIVATE $) if(VPP_LIB_VERSION) +diff --git src/tools/vppapigen/vppapigen.py src/tools/vppapigen/vppapigen.py +index 8415c28fb..a017d9fc6 100755 +--- src/tools/vppapigen/vppapigen.py ++++ src/tools/vppapigen/vppapigen.py +@@ -7,6 +7,13 @@ import logging + import binascii + import os + from subprocess import Popen, PIPE ++ ++# Put ply on the path ... ++plypath = os.path.join( ++ os.environ["EXT_BUILD_ROOT"], ++ os.path.dirname(os.environ["PLYPATHS"].split()[0])) ++sys.path += [plypath] ++ + import ply.lex as lex + import ply.yacc as yacc + +diff --git src/vat/CMakeLists.txt src/vat/CMakeLists.txt +index e5945b20d..3d76f3d88 100644 +--- src/vat/CMakeLists.txt ++++ src/vat/CMakeLists.txt +@@ -19,39 +19,6 @@ add_vpp_library(vatplugin + LINK_LIBRARIES vppinfra + ) + +-############################################################################## +-# vpp_api_test +-############################################################################## +-add_vpp_executable(vpp_api_test ENABLE_EXPORTS +- SOURCES +- api_format.c +- main.c +- plugin.c +- json_format.c +- types.c +- ip_types_api.c +- ip_types.c +- protocols.def +- +- DEPENDS api_headers +- +- LINK_LIBRARIES +- vlibmemoryclient +- svm +- vatplugin +- vppinfra +- Threads::Threads +- dl +-) +- +-############################################################################## +-# vpp_json_test +-############################################################################## +-add_vpp_executable(vpp_json_test ENABLE_EXPORTS NO_INSTALL +- SOURCES json_format.c json_test.c +- LINK_LIBRARIES vppinfra m +-) +- + ############################################################################## + # vat headers + ############################################################################## +diff --git src/vat2/CMakeLists.txt src/vat2/CMakeLists.txt +index 108e184b5..a90107617 100644 +--- src/vat2/CMakeLists.txt ++++ src/vat2/CMakeLists.txt +@@ -14,21 +14,6 @@ + ############################################################################## + # vat2 + ############################################################################## +-add_vpp_executable(vat2 ENABLE_EXPORTS +- SOURCES +- main.c +- plugin.c +- +- DEPENDS api_headers +- +- LINK_LIBRARIES +- vlibmemoryclient +- svm +- vppinfra +- vppapiclient +- Threads::Threads +- dl +-) + + # + # Unit test code. Call generator directly to avoid it being installed +@@ -37,20 +22,6 @@ add_vpp_executable(vat2 ENABLE_EXPORTS + #) + + vpp_generate_api_c_header (test/vat2_test.api) +-add_vpp_executable(test_vat2 ENABLE_EXPORTS NO_INSTALL +- SOURCES +- test/vat2_test.c +- +- DEPENDS api_headers +- +- LINK_LIBRARIES +- vlibmemoryclient +- svm +- vppinfra +- vppapiclient +- Threads::Threads +- dl +-) + #target_link_options(test_vat2 PUBLIC "LINKER:-fsanitize=address") + if(VPP_BUILD_TESTS_WITH_COVERAGE) + set(TARGET_NAME test_vat2) +diff --git src/vpp-api/CMakeLists.txt src/vpp-api/CMakeLists.txt +index 72cc1b29c..0f2510d51 100644 +--- src/vpp-api/CMakeLists.txt ++++ src/vpp-api/CMakeLists.txt +@@ -29,10 +29,5 @@ add_vpp_headers(vpp-api + client/stat_client.h + ) + +-add_vpp_executable(test_vppapiclient NO_INSTALL +- SOURCES client/test.c +- LINK_LIBRARIES vppinfra pthread vppapiclient +-) +- + add_subdirectory(vapi) + add_subdirectory(python) +diff --git src/vpp/CMakeLists.txt src/vpp/CMakeLists.txt +index 1b43b1299..361b981d9 100644 +--- src/vpp/CMakeLists.txt ++++ src/vpp/CMakeLists.txt +@@ -83,13 +83,6 @@ if(VPP_API_TEST_BUILTIN) + add_definitions(-DVPP_API_TEST_BUILTIN=1) + endif() + +-add_vpp_executable(vpp +- ENABLE_EXPORTS +- SOURCES ${VPP_SOURCES} +- LINK_LIBRARIES svm vlib vppinfra vlibmemory vnet Threads::Threads ${CMAKE_DL_LIBS} +- DEPENDS vpp_version_h api_headers +-) +- + add_vpp_headers(vpp + stats/stat_segment.h + stats/stat_segment_shared.h diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 1d8cbb9f33b7..4a4e05b7f41a 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1317,7 +1317,7 @@ filegroup( def _com_github_fdio_vpp_vcl(): external_http_archive( name = "com_github_fdio_vpp_vcl", - build_file_content = BUILD_ALL_CONTENT, + build_file_content = _build_all_content(exclude = ["**/*doc*/**", "**/examples/**", "**/plugins/**"]), patches = ["@envoy//bazel/foreign_cc:vpp_vcl.patch"], ) diff --git a/contrib/vcl/source/BUILD b/contrib/vcl/source/BUILD index 8763c337b3cd..8ed5fb25ea14 100644 --- a/contrib/vcl/source/BUILD +++ b/contrib/vcl/source/BUILD @@ -2,9 +2,9 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_contrib_extension", "envoy_cc_library", + "envoy_cmake", "envoy_contrib_package", ) -load("@rules_python//python:defs.bzl", "py_binary") load("@base_pip3//:requirements.bzl", "requirement") licenses(["notice"]) # Apache 2 @@ -20,6 +20,7 @@ cc_library( "external/libvlibmemoryclient.a", "external/libvppcom.a", "external/libvppinfra.a", + "external/vppcom.h", ], hdrs = ["external/vppcom.h"], defines = ["VPP_VCL"], @@ -28,37 +29,61 @@ cc_library( visibility = ["//visibility:public"], ) -genrule( +envoy_cmake( name = "build", - srcs = [ - "@com_github_fdio_vpp_vcl//:all", + build_data = [requirement("ply")], + cache_entries = { + "CMAKE_BUILD_TYPE": "Release", + "VPP_API_TEST_BUILTIN": "OFF", + "BUILD_SHARED_LIBS": "OFF", + "CMAKE_ENABLE_TESTING": "OFF", + "CMAKE_ENABLE_EXPORTS": "OFF", + }, + copts = ["-Wno-unused-variable"], + default_cache_entries = {}, + env = { + "PLYPATHS": "$(locations %s)" % requirement("ply"), + }, + lib_source = "@com_github_fdio_vpp_vcl//:all", + linkopts = ["-Wno-unused-variable"], + out_static_libs = [ + "libvppcom.a", + "libvppinfra.a", + "libsvm.a", + "libvlibmemoryclient.a", + ], + postfix_script = """ + mkdir -p $INSTALLDIR/lib/external $INSTALLDIR/include/external \ + && find . -name "*.a" | xargs -I{} cp -a {} $INSTALLDIR/lib/ \ + && find . -name "*.h" | xargs -I{} cp -a {} $INSTALLDIR/include + """, + tags = [ + "cpu:16", + "skip_on_windows", ], + targets = [ + "", + "install", + ], + working_directory = "src", +) + +genrule( + name = "build_files", outs = [ - "external/libvppcom.a", - "external/libvppinfra.a", "external/libsvm.a", "external/libvlibmemoryclient.a", + "external/libvppcom.a", + "external/libvppinfra.a", "external/vppcom.h", ], cmd = """ - ./$(location :vcl_build_launcher) vpp_vcl_build.sh $(location external/libvppcom.a) + EXTERNAL_DIR=$$(dirname $(location external/libsvm.a)) \ + && mkdir -p $$EXTERNAL_DIR \ + && find . -name "*.a" | xargs -I{} cp -a {} $$EXTERNAL_DIR \ + && find . -name "vppcom.h" | xargs -I{} cp -a {} $$EXTERNAL_DIR """, - target_compatible_with = [ - "@platforms//os:linux", - ], - tools = [ - ":vcl_build_launcher", - ], -) - -py_binary( - name = "vcl_build_launcher", - srcs = ["vcl_build_launcher.py"], - data = [ - "vpp_vcl_build.sh", - ], - main = "vcl_build_launcher.py", - deps = [requirement("ply")], + exec_tools = [":build"], ) envoy_cc_library( diff --git a/contrib/vcl/source/vcl_build_launcher.py b/contrib/vcl/source/vcl_build_launcher.py deleted file mode 100644 index 28f5cd219b73..000000000000 --- a/contrib/vcl/source/vcl_build_launcher.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/python - -# Launcher for building vcl - -import os -import subprocess -import sys - - -def main(): - """ VCL builder script """ - - # find path to helper script - script_path = os.path.dirname(os.path.abspath(sys.argv[0])) - vcl_build = f"{script_path}/{sys.argv[1]}" - - # find path to vpp/vcl source code - base_path = os.path.dirname(os.path.abspath(sys.argv[1])) - vpp_path = f"{base_path}/external/com_github_fdio_vpp_vcl" - - # find path to dst folder - dst_path = os.path.dirname(os.path.abspath(sys.argv[2])) - - # build vcl - subprocess.run([vcl_build, vpp_path, dst_path]) - - -if __name__ == "__main__": - main() diff --git a/contrib/vcl/source/vpp_vcl_build.sh b/contrib/vcl/source/vpp_vcl_build.sh deleted file mode 100755 index ebafe1029f26..000000000000 --- a/contrib/vcl/source/vpp_vcl_build.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -set -e - -VPP_PATH=$1 -DST_PATH=$2 - -DEBUG_VPP= - -# This works only on Linux. -if [[ $(uname) != "Linux" ]]; then - echo "ERROR: VPP VCL is currently supported only on Linux" - exit 1 -fi - -NINJA_ARGS=() - -# TODO(phlax): quieten this properly once ninja is updated to 1.11+ -# https://github.com/ninja-build/ninja/commit/0a632d11eb1e8b7becbd9756c3e3439777924f6d -# https://discourse.cmake.org/t/making-ninja-build-output-silent/4439 -if [[ -n "$DEBUG_VPP" ]]; then - # Log cmake and ninja versions for later debugging - echo "Building VCL" - echo "running $(cmake --version | head -1)" - echo "ninja version $(ninja --version)" -else - : - # NINJA_ARGS+=(--quiet) -fi - -# Build -pushd "${VPP_PATH}" > /dev/null -mkdir _vcl -cd _vcl - -if [[ -n "$DEBUG_VPP" ]]; then - cmake -G Ninja ../src -DCMAKE_BUILD_TYPE:STRING=release - ninja "${NINJA_ARGS[@]}" -C . vppcom -else - # TODO(phlax): remove redirection once ninja is updated to 1.11+ - cmake -G Ninja ../src -DCMAKE_BUILD_TYPE:STRING=release &> /dev/null - ninja "${NINJA_ARGS[@]}" -C . vppcom > /dev/null -fi - -mv CMakeFiles/vcl/libvppcom.a "${DST_PATH}" -mv CMakeFiles/vppinfra/libvppinfra.a "${DST_PATH}" -mv CMakeFiles/svm/libsvm.a "${DST_PATH}" -mv CMakeFiles/vlibmemory/libvlibmemoryclient.a "${DST_PATH}" -cp ../src/vcl/vppcom.h "${DST_PATH}" - -popd > /dev/null From eee6b6d48d41ac9413c50049106d14537a65d7ad Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Tue, 9 May 2023 07:31:00 +0300 Subject: [PATCH 157/740] stats: allow combining configured tags with custom tags (#27215) Adds a way to create a StatNameTagVector that contains extracted tags names from both the configured and well defined tags, and custom provided tags. Fixes #27030 Risk Level: Low Testing: Unit tests Signed-off-by: ohadvano --- envoy/stats/store.h | 21 ++++++ source/common/stats/isolated_store_impl.h | 8 ++ source/common/stats/thread_local_store.cc | 14 ++++ source/common/stats/thread_local_store.h | 3 + test/common/stats/thread_local_store_test.cc | 78 ++++++++++++++++++++ test/integration/server.h | 8 ++ 6 files changed, 132 insertions(+) diff --git a/envoy/stats/store.h b/envoy/stats/store.h index c44f51debca0..5af35defd9c6 100644 --- a/envoy/stats/store.h +++ b/envoy/stats/store.h @@ -26,6 +26,7 @@ namespace Stats { class Sink; class SinkPredicates; +class StatNamePool; /** * Store keeps track of all Scopes created in it, and the Scopes manage @@ -172,6 +173,26 @@ class Store { * @return a scope of the given name. */ ScopeSharedPtr createScope(const std::string& name) { return rootScope()->createScope(name); } + + /** + * Extracts tags from the name and appends them to the provided StatNameTagVector. + * The StatName for the extracted tags will be saved in the provided pool. + * @param name The stat name. + * @param pool The pool to create the tags in. + * @param stat_tags The stat name tags vector to append the tags to. + */ + virtual void extractAndAppendTags(StatName name, StatNamePool& pool, + StatNameTagVector& stat_tags) PURE; + + /** + * Extracts tags from the name and appends them to the provided StatNameTagVector. + * The StatName for the extracted tags will be saved in the provided pool. + * @param name The stat name. + * @param pool The pool to create the tags in. + * @param stat_tags The stat name tags vector to append the tags to. + */ + virtual void extractAndAppendTags(absl::string_view name, StatNamePool& pool, + StatNameTagVector& stat_tags) PURE; }; using StorePtr = std::unique_ptr; diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index e337e20d67f7..3e642b88888d 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -234,6 +234,14 @@ class IsolatedStoreImpl : public Store { return constRootScope()->iterate(fn); } + void extractAndAppendTags(StatName, StatNamePool&, StatNameTagVector&) override { + IS_ENVOY_BUG("Unexpected call to a function that is not yet implemented"); + } + + void extractAndAppendTags(absl::string_view, StatNamePool&, StatNameTagVector&) override { + IS_ENVOY_BUG("Unexpected call to a function that is not yet implemented"); + } + protected: /** * Provides a hook for sub-classes to define how to create new scopes. When diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index edbb1f595762..cc5658d71ae2 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -1075,5 +1075,19 @@ void ThreadLocalStoreImpl::setSinkPredicates(std::unique_ptr&& s } } +void ThreadLocalStoreImpl::extractAndAppendTags(StatName name, StatNamePool& pool, + StatNameTagVector& stat_tags) { + extractAndAppendTags(symbolTable().toString(name), pool, stat_tags); +} + +void ThreadLocalStoreImpl::extractAndAppendTags(absl::string_view name, StatNamePool& pool, + StatNameTagVector& stat_tags) { + TagVector tags; + tagProducer().produceTags(name, tags); + for (const auto& tag : tags) { + stat_tags.emplace_back(pool.add(tag.name_), pool.add(tag.value_)); + } +} + } // namespace Stats } // namespace Envoy diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index 04d0c216b5d1..a205f6529565 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -211,6 +211,9 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo void releaseHistogramCrossThread(uint64_t histogram_id); const TagProducer& tagProducer() const { return *tag_producer_; } + void extractAndAppendTags(StatName name, StatNamePool& pool, StatNameTagVector& tags) override; + void extractAndAppendTags(absl::string_view name, StatNamePool& pool, + StatNameTagVector& tags) override; private: friend class ThreadLocalStoreTestingPeer; diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 78c2ae0d25bb..1d43ed663eb6 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -775,6 +775,84 @@ TEST_F(StatsThreadLocalStoreTest, SharedScopes) { tls_.shutdownThread(); } +TEST_F(StatsThreadLocalStoreTest, ExtractAndAppendTagsFixedValue) { + store_->initializeThreading(main_thread_dispatcher_, tls_); + + envoy::config::metrics::v3::StatsConfig stats_config; + auto* tag_specifier = stats_config.add_stats_tags(); + tag_specifier->set_tag_name("foo"); + tag_specifier->set_fixed_value("bar"); + + store_->setTagProducer(std::make_unique(stats_config)); + + StatNamePool pool(symbol_table_); + StatNameTagVector tags{{pool.add("a"), pool.add("b")}}; + store_->extractAndAppendTags(pool.add("c1"), pool, tags); + + ASSERT_EQ(2, tags.size()); + EXPECT_EQ("a", symbol_table_.toString(tags[0].first)); + EXPECT_EQ("b", symbol_table_.toString(tags[0].second)); + EXPECT_EQ("foo", symbol_table_.toString(tags[1].first)); + EXPECT_EQ("bar", symbol_table_.toString(tags[1].second)); +} + +TEST_F(StatsThreadLocalStoreTest, ExtractAndAppendTagsRegexValueNoMatch) { + store_->initializeThreading(main_thread_dispatcher_, tls_); + + envoy::config::metrics::v3::StatsConfig stats_config; + auto* tag_specifier = stats_config.add_stats_tags(); + tag_specifier->set_tag_name("foo"); + tag_specifier->set_regex("bar"); + + store_->setTagProducer(std::make_unique(stats_config)); + + StatNamePool pool(symbol_table_); + StatNameTagVector tags{{pool.add("a"), pool.add("b")}}; + store_->extractAndAppendTags(pool.add("c1"), pool, tags); + + ASSERT_EQ(1, tags.size()); + EXPECT_EQ("a", symbol_table_.toString(tags[0].first)); + EXPECT_EQ("b", symbol_table_.toString(tags[0].second)); +} + +TEST_F(StatsThreadLocalStoreTest, ExtractAndAppendTagsRegexValueWithMatch) { + store_->initializeThreading(main_thread_dispatcher_, tls_); + + envoy::config::metrics::v3::StatsConfig stats_config; + auto* tag_specifier = stats_config.add_stats_tags(); + tag_specifier->set_tag_name("foo_tag"); + tag_specifier->set_regex("^foo.(.+)"); + + store_->setTagProducer(std::make_unique(stats_config)); + + StatNamePool pool(symbol_table_); + StatNameTagVector tags{{pool.add("a"), pool.add("b")}}; + store_->extractAndAppendTags(pool.add("foo.bar"), pool, tags); + + ASSERT_EQ(2, tags.size()); + EXPECT_EQ("a", symbol_table_.toString(tags[0].first)); + EXPECT_EQ("b", symbol_table_.toString(tags[0].second)); + EXPECT_EQ("foo_tag", symbol_table_.toString(tags[1].first)); + EXPECT_EQ("bar", symbol_table_.toString(tags[1].second)); +} + +TEST_F(StatsThreadLocalStoreTest, ExtractAndAppendTagsRegexBuiltinExpression) { + store_->initializeThreading(main_thread_dispatcher_, tls_); + + envoy::config::metrics::v3::StatsConfig stats_config; + store_->setTagProducer(std::make_unique(stats_config)); + + StatNamePool pool(symbol_table_); + StatNameTagVector tags{{pool.add("a"), pool.add("b")}}; + store_->extractAndAppendTags(pool.add("cluster.foo.bar"), pool, tags); + + ASSERT_EQ(2, tags.size()); + EXPECT_EQ("a", symbol_table_.toString(tags[0].first)); + EXPECT_EQ("b", symbol_table_.toString(tags[0].second)); + EXPECT_EQ("envoy.cluster_name", symbol_table_.toString(tags[1].first)); + EXPECT_EQ("foo", symbol_table_.toString(tags[1].second)); +} + class LookupWithStatNameTest : public ThreadLocalStoreNoMocksMixin, public testing::Test {}; TEST_F(LookupWithStatNameTest, All) { diff --git a/test/integration/server.h b/test/integration/server.h index 17301e6a5e60..6fefb7f76cba 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -357,6 +357,14 @@ class TestIsolatedStoreImpl : public StoreRoot { bool iterate(const IterateFn& fn) const override { return store_.iterate(fn); } bool iterate(const IterateFn& fn) const override { return store_.iterate(fn); } + void extractAndAppendTags(StatName, StatNamePool&, StatNameTagVector&) override { + IS_ENVOY_BUG("Unexpected call to a function that is not yet implemented"); + }; + + void extractAndAppendTags(absl::string_view, StatNamePool&, StatNameTagVector&) override { + IS_ENVOY_BUG("Unexpected call to a function that is not yet implemented"); + }; + // Stats::StoreRoot void addSink(Sink&) override {} void setTagProducer(TagProducerPtr&&) override {} From 8d4f25beccd90afa05eca9c478c07d2c45b1dd38 Mon Sep 17 00:00:00 2001 From: Vishal Damgude Date: Tue, 9 May 2023 12:40:14 +0530 Subject: [PATCH 158/740] Fix for setting ssl info in stream_info when starttls transport socket is used (#26486) Signed-off-by: VishalDamgude --- envoy/tcp/upstream.h | 7 +++++++ source/common/tcp_proxy/tcp_proxy.cc | 9 ++++++++- source/common/tcp_proxy/upstream.cc | 7 +++++++ source/common/tcp_proxy/upstream.h | 2 ++ .../transport_sockets/starttls/starttls_socket.cc | 1 + .../transport_sockets/starttls/starttls_socket_test.cc | 1 + 6 files changed, 26 insertions(+), 1 deletion(-) diff --git a/envoy/tcp/upstream.h b/envoy/tcp/upstream.h index a46e37d7141a..200ec7fc9ea7 100644 --- a/envoy/tcp/upstream.h +++ b/envoy/tcp/upstream.h @@ -144,6 +144,13 @@ class GenericUpstream { * to secure mode. Implemented only by start_tls transport socket. */ virtual bool startUpstreamSecureTransport() PURE; + + /** + * Called when upstream starttls socket is converted to tls and upstream ssl info + * needs to be set in the connection's stream_info. + * @return the const SSL connection data of upstream. + */ + virtual Ssl::ConnectionInfoConstSharedPtr getUpstreamConnectionSslInfo() PURE; }; using GenericConnPoolPtr = std::unique_ptr; diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index 03519d2a3c8a..bf0e27ba30d0 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -702,7 +702,14 @@ Network::FilterStatus Filter::onNewConnection() { return establishUpstreamConnection(); } -bool Filter::startUpstreamSecureTransport() { return upstream_->startUpstreamSecureTransport(); } +bool Filter::startUpstreamSecureTransport() { + bool switched_to_tls = upstream_->startUpstreamSecureTransport(); + if (switched_to_tls) { + StreamInfo::UpstreamInfo& upstream_info = *getStreamInfo().upstreamInfo(); + upstream_info.setUpstreamSslConnection(upstream_->getUpstreamConnectionSslInfo()); + } + return switched_to_tls; +} void Filter::onDownstreamEvent(Network::ConnectionEvent event) { if (event == Network::ConnectionEvent::LocalClose || diff --git a/source/common/tcp_proxy/upstream.cc b/source/common/tcp_proxy/upstream.cc index a2dca26b31f5..b186b136bfd4 100644 --- a/source/common/tcp_proxy/upstream.cc +++ b/source/common/tcp_proxy/upstream.cc @@ -49,6 +49,13 @@ bool TcpUpstream::startUpstreamSecureTransport() { : upstream_conn_data_->connection().startSecureTransport(); } +Ssl::ConnectionInfoConstSharedPtr TcpUpstream::getUpstreamConnectionSslInfo() { + if (upstream_conn_data_ != nullptr) { + return upstream_conn_data_->connection().ssl(); + } + return nullptr; +} + Tcp::ConnectionPool::ConnectionData* TcpUpstream::onDownstreamEvent(Network::ConnectionEvent event) { if (event == Network::ConnectionEvent::RemoteClose) { diff --git a/source/common/tcp_proxy/upstream.h b/source/common/tcp_proxy/upstream.h index 452df36f376a..2cdf57cc83b0 100644 --- a/source/common/tcp_proxy/upstream.h +++ b/source/common/tcp_proxy/upstream.h @@ -117,6 +117,7 @@ class TcpUpstream : public GenericUpstream { void addBytesSentCallback(Network::Connection::BytesSentCb cb) override; Tcp::ConnectionPool::ConnectionData* onDownstreamEvent(Network::ConnectionEvent event) override; bool startUpstreamSecureTransport() override; + Ssl::ConnectionInfoConstSharedPtr getUpstreamConnectionSslInfo() override; private: Tcp::ConnectionPool::ConnectionDataPtr upstream_conn_data_; @@ -153,6 +154,7 @@ class HttpUpstream : public GenericUpstream, protected Http::StreamCallbacks { void setConnPoolCallbacks(std::unique_ptr&& callbacks) { conn_pool_callbacks_ = std::move(callbacks); } + Ssl::ConnectionInfoConstSharedPtr getUpstreamConnectionSslInfo() override { return nullptr; } protected: HttpUpstream(Tcp::ConnectionPool::UpstreamCallbacks& callbacks, diff --git a/source/extensions/transport_sockets/starttls/starttls_socket.cc b/source/extensions/transport_sockets/starttls/starttls_socket.cc index 38e2448f23e2..241c427500c0 100644 --- a/source/extensions/transport_sockets/starttls/starttls_socket.cc +++ b/source/extensions/transport_sockets/starttls/starttls_socket.cc @@ -17,6 +17,7 @@ bool StartTlsSocket::startSecureTransport() { // does buffering, it should be flushed before destroying or // flush should be called from destructor. active_socket_ = std::move(tls_socket_); + callbacks_.connection().connectionInfoSetter().setSslConnection(active_socket_->ssl()); using_tls_ = true; } return true; diff --git a/test/extensions/transport_sockets/starttls/starttls_socket_test.cc b/test/extensions/transport_sockets/starttls/starttls_socket_test.cc index 33e5757f5d8b..efe9a7656245 100644 --- a/test/extensions/transport_sockets/starttls/starttls_socket_test.cc +++ b/test/extensions/transport_sockets/starttls/starttls_socket_test.cc @@ -72,6 +72,7 @@ TEST(StartTlsTest, BasicSwitch) { // Now switch to Tls. During the switch, the new socket should register for callbacks // and connect. + EXPECT_CALL(*ssl_socket, ssl()); EXPECT_CALL(*ssl_socket, setTransportSocketCallbacks(_)); EXPECT_CALL(*ssl_socket, onConnected); // Make sure that raw socket is destructed. From e08f7e1350c580f40a63fe1a70fe4bf133b05a23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 May 2023 08:35:17 +0100 Subject: [PATCH 159/740] build(deps): bump urllib3 from 1.26.15 to 2.0.2 in /mobile/docs (#27166) Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.15 to 2.0.2. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.15...2.0.2) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mobile/docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/docs/requirements.txt b/mobile/docs/requirements.txt index 7734c1c67156..4f73a83e1b42 100644 --- a/mobile/docs/requirements.txt +++ b/mobile/docs/requirements.txt @@ -197,6 +197,6 @@ sphinxcontrib-qthelp==1.0.3 \ sphinxcontrib-serializinghtml==1.1.5 \ --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \ --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952 -urllib3==1.26.15 \ - --hash=sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305 \ - --hash=sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42 +urllib3==2.0.2 \ + --hash=sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc \ + --hash=sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e From 7657d0b252e698e04d836c4481e71503d792e9c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 May 2023 09:06:29 +0100 Subject: [PATCH 160/740] build(deps): bump certifi from 2022.12.7 to 2023.5.7 in /mobile/docs (#27249) Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.5.7. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.05.07) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mobile/docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/docs/requirements.txt b/mobile/docs/requirements.txt index 4f73a83e1b42..4118d6bee302 100644 --- a/mobile/docs/requirements.txt +++ b/mobile/docs/requirements.txt @@ -4,9 +4,9 @@ alabaster==0.7.13 \ Babel==2.12.1 \ --hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \ --hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455 -certifi==2022.12.7 \ - --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ - --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 +certifi==2023.5.7 \ + --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \ + --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716 charset-normalizer==3.1.0 \ --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ From 3e3f2c773574efd5ea334c4f7800d2781bb714b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 May 2023 10:23:34 +0100 Subject: [PATCH 161/740] build(deps): bump protobuf from 4.22.4 to 4.23.0 in /examples/grpc-bridge/client (#27265) build(deps): bump protobuf in /examples/grpc-bridge/client Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 4.22.4 to 4.23.0. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v4.22.4...v4.23.0) --- updated-dependencies: - dependency-name: protobuf dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/client/requirements.txt | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index 4d7a5033281a..918e6db67ffa 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -112,20 +112,20 @@ idna==3.2 \ --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \ --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3 # via requests -protobuf==4.22.4 \ - --hash=sha256:11b28b4e779d7f275e3ea0efa3938f4d4e8ed3ca818f9fec3b193f8e9ada99fd \ - --hash=sha256:144d5b46df5e44f914f715accaadf88d617242ba5a40cacef4e8de7effa79954 \ - --hash=sha256:21fbaef7f012232eb8d6cb8ba334e931fc6ff8570f5aaedc77d5b22a439aa909 \ - --hash=sha256:3b21074b7fb748d8e123acaef9fa63a84fdc1436dc71199d2317b139f77dd6f4 \ - --hash=sha256:4bfb28d48628deacdb66a95aaa7b6640f3dc82b4edd34db444c7a3cdd90b01fb \ - --hash=sha256:5128b4d5efcaef92189e076077ae389700606ff81d2126b8361dc01f3e026197 \ - --hash=sha256:7b42086d6027be2730151b49f27b2f5be40f3b036adf7b8da5917f4567f268c3 \ - --hash=sha256:8fd329e5dd7b6c4b878cab4b85bb6cec880e2adaf4e8aa2c75944dcbb05e1ff1 \ - --hash=sha256:9537ae27d43318acf8ce27d0359fe28e6ebe4179c3350bc055bb60ff4dc4fcd3 \ - --hash=sha256:a4e661247896c2ffea4b894bca2d8657e752bedb8f3c66d7befa2557291be1e8 \ - --hash=sha256:b7728b5da9eee15c0aa3baaee79e94fa877ddcf7e3d2f34b1eab586cd26eea89 \ - --hash=sha256:e98e26328d7c668541d1052b02de4205b1094ef6b2ce57167440d3e39876db48 \ - --hash=sha256:f4a711588c3a79b6f9c44af4d7f4a2ae868e27063654683932ab6462f90e9656 +protobuf==4.23.0 \ + --hash=sha256:03eee35b60317112a72d19c54d0bff7bc58ff12fea4cd7b018232bd99758ffdf \ + --hash=sha256:2b94bd6df92d71bd1234a2ffe7ce96ddf6d10cf637a18d6b55ad0a89fbb7fc21 \ + --hash=sha256:36f5370a930cb77c8ad2f4135590c672d0d2c72d4a707c7d0058dce4b4b4a598 \ + --hash=sha256:5f1eba1da2a2f3f7df469fccddef3cc060b8a16cfe3cc65961ad36b4dbcf59c5 \ + --hash=sha256:6c16657d6717a0c62d5d740cb354fbad1b0d8cb811669e06fc1caa0ff4799ddd \ + --hash=sha256:6fe180b56e1169d72ecc4acbd39186339aed20af5384531b8e8979b02bbee159 \ + --hash=sha256:7cb5b9a05ce52c6a782bb97de52679bd3438ff2b7460eff5da348db65650f227 \ + --hash=sha256:9744e934ea5855d12191040ea198eaf704ac78665d365a89d9572e3b627c2688 \ + --hash=sha256:9f5a0fbfcdcc364f3986f9ed9f8bb1328fb84114fd790423ff3d7fdb0f85c2d1 \ + --hash=sha256:baca40d067dddd62141a129f244703160d278648b569e90bb0e3753067644711 \ + --hash=sha256:d5a35ff54e3f62e8fc7be02bb0d2fbc212bba1a5a9cc2748090690093996f07b \ + --hash=sha256:e62fb869762b4ba18666370e2f8a18f17f8ab92dd4467295c6d38be6f8fef60b \ + --hash=sha256:ebde3a023b8e11bfa6c890ef34cd6a8b47d586f26135e86c21344fe433daf2e2 # via # -r requirements.in # grpcio-tools From 84c08d7aa3aef96c5476a827dcb7bb0fe4e37738 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 9 May 2023 10:48:08 +0100 Subject: [PATCH 162/740] python: Cleanups and up/downgrade (#27233) Signed-off-by: Ryan Northey --- bazel/repositories_extra.bzl | 8 +++++--- bazel/repository_locations.bzl | 6 +++--- mobile/WORKSPACE | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl index 9d1b31c5d66a..f7ec05fda768 100644 --- a/bazel/repositories_extra.bzl +++ b/bazel/repositories_extra.bzl @@ -5,10 +5,12 @@ load("//bazel/external/cargo:crates.bzl", "raze_fetch_remote_crates") load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies") # Python version for `rules_python` -PYTHON_VERSION = "3.10.2" +PYTHON_VERSION = "3.10.9" # Envoy deps that rely on a first stage of dependency loading in envoy_dependencies(). -def envoy_dependencies_extra(python_version = PYTHON_VERSION): +def envoy_dependencies_extra( + python_version = PYTHON_VERSION, + ignore_root_user_error = False): emsdk_deps() raze_fetch_remote_crates() wasmtime_fetch_remote_crates() @@ -17,7 +19,7 @@ def envoy_dependencies_extra(python_version = PYTHON_VERSION): python_register_toolchains( name = "python%s" % ("_".join(python_version.split(".")[:-1])), python_version = python_version, - ignore_root_user_error = True, + ignore_root_user_error = ignore_root_user_error, ) aspect_bazel_lib_dependencies() diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 0145b5db237b..a7c48902c214 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -883,9 +883,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Python rules for Bazel", project_desc = "Bazel rules for the Python language", project_url = "https://github.com/bazelbuild/rules_python", - version = "0.21.0", - sha256 = "94750828b18044533e98a129003b6a68001204038dc4749f40b195b24c38f49f", - release_date = "2023-04-20", + version = "0.20.0", + sha256 = "a644da969b6824cc87f8fe7b18101a8a6c57da5db39caa6566ec6109f37d2141", + release_date = "2023-03-20", strip_prefix = "rules_python-{version}", urls = ["https://github.com/bazelbuild/rules_python/archive/{version}.tar.gz"], use_category = ["build"], diff --git a/mobile/WORKSPACE b/mobile/WORKSPACE index 6027e1ac218e..2fb069a3f27a 100644 --- a/mobile/WORKSPACE +++ b/mobile/WORKSPACE @@ -52,7 +52,7 @@ envoy_dependencies() load("@envoy//bazel:repositories_extra.bzl", "envoy_dependencies_extra") -envoy_dependencies_extra() +envoy_dependencies_extra(ignore_root_user_error=True) load("@envoy//bazel:python_dependencies.bzl", "envoy_python_dependencies") From 73420f09ba059d29815b4e528171292683693a97 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 9 May 2023 09:05:52 -0400 Subject: [PATCH 163/740] mobile: using envoy_reloadable_features_skip_dns_lookup_for_proxies by default (#27000) Removing the mobile APIs for the flag as we can now use the long term mobile APIs for that. Fixing bug with the implementation uncovered by flipping the default Risk Level: Testing: Docs Changes: Release Notes: Signed-off-by: Alyssa Wilk --- mobile/docs/root/intro/version_history.rst | 1 + mobile/library/cc/engine_builder.cc | 7 ------ mobile/library/cc/engine_builder.h | 2 -- mobile/library/common/jni/jni_interface.cc | 15 +++++-------- .../engine/EnvoyConfiguration.java | 22 +++++++------------ .../envoymobile/engine/JniLibrary.java | 9 ++++---- .../impl/NativeCronvoyEngineBuilderImpl.java | 8 +++---- .../envoyproxy/envoymobile/EngineBuilder.kt | 18 --------------- .../integration/rtds_integration_test.cc | 8 +++---- .../engine/EnvoyConfigurationTest.kt | 9 -------- .../kotlin/apps/experimental/MainActivity.kt | 1 - .../common/upstream/cluster_manager_impl.cc | 3 ++- 12 files changed, 26 insertions(+), 77 deletions(-) diff --git a/mobile/docs/root/intro/version_history.rst b/mobile/docs/root/intro/version_history.rst index 7f4e13f1a52e..4223d672fe0c 100644 --- a/mobile/docs/root/intro/version_history.rst +++ b/mobile/docs/root/intro/version_history.rst @@ -22,6 +22,7 @@ Breaking changes: - api: added ``setRuntimeGuard`` APIs for all languages (:issue: `#25434 <25434>`) - api: added ``setRtdsLayer``, ``addAggregatedDiscoveryService``, ``setNodeId``, ``setNodeLocality`` APIs for all languages - api: added ``setCdsLayer`` APIs for all languages (:issue: `#26122 <26122>`) +- api: removed the ``enableSkipDNSLookup`` API. The runtime guard can still be set via ``setRuntimeGuard``. - clusters: removing the base_h2 cluster. Requests with ``x-envoy-mobile-upstream-protocol`` set to ``http2`` will be sent to the base cluster and use the best available protocol (:issue `#25796 <25796>`). - clusters: only creating the stats cluster if an endpoint is configured. Previously if no domain was configured a cluster would be configured pointed at 127.0.0.1. (:issue: `#25816 <25816>`). - clusters: removing the base_h3 cluster. If HTTP/3 is enabled, the base cluster will use HTTP/3 instead. (:issue: `#25814 <25814>`). diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc index beaa3b2c493c..767132fb2d4f 100644 --- a/mobile/library/cc/engine_builder.cc +++ b/mobile/library/cc/engine_builder.cc @@ -193,11 +193,6 @@ EngineBuilder& EngineBuilder::setForceAlwaysUsev6(bool value) { return *this; } -EngineBuilder& EngineBuilder::setSkipDnsLookupForProxiedRequests(bool value) { - skip_dns_lookups_for_proxied_requests_ = value; - return *this; -} - EngineBuilder& EngineBuilder::enableInterfaceBinding(bool interface_binding_on) { enable_interface_binding_ = interface_binding_on; return *this; @@ -904,8 +899,6 @@ std::unique_ptr EngineBuilder::generate (*flags.mutable_fields())[guard_and_value.first].set_bool_value(guard_and_value.second); } (*flags.mutable_fields())["always_use_v6"].set_bool_value(always_use_v6_); - (*flags.mutable_fields())["skip_dns_lookup_for_proxied_requests"].set_bool_value( - skip_dns_lookups_for_proxied_requests_); (*runtime_values.mutable_fields())["disallow_global_stats"].set_bool_value(true); ProtobufWkt::Struct& overload_values = *(*envoy_layer.mutable_fields())["overload"].mutable_struct_value(); diff --git a/mobile/library/cc/engine_builder.h b/mobile/library/cc/engine_builder.h index 449169ed5ef9..63dcc36374ac 100644 --- a/mobile/library/cc/engine_builder.h +++ b/mobile/library/cc/engine_builder.h @@ -89,7 +89,6 @@ class EngineBuilder { #endif EngineBuilder& enableDnsCache(bool dns_cache_on, int save_interval_seconds = 1); EngineBuilder& setForceAlwaysUsev6(bool value); - EngineBuilder& setSkipDnsLookupForProxiedRequests(bool value); EngineBuilder& addDnsPreresolveHostnames(const std::vector& hostnames); EngineBuilder& addNativeFilter(std::string name, std::string typed_config); #ifdef ENVOY_ADMIN_FUNCTIONALITY @@ -194,7 +193,6 @@ class EngineBuilder { std::vector> runtime_guards_; absl::flat_hash_map string_accessors_; - bool skip_dns_lookups_for_proxied_requests_ = false; }; using EngineBuilderSharedPtr = std::shared_ptr; diff --git a/mobile/library/common/jni/jni_interface.cc b/mobile/library/common/jni/jni_interface.cc index 10795b3fab28..c96f8e687a5e 100644 --- a/mobile/library/common/jni/jni_interface.cc +++ b/mobile/library/common/jni/jni_interface.cc @@ -1208,8 +1208,7 @@ void configureBuilder( jlong stats_flush_seconds, jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id, jboolean trust_chain_verification, jobjectArray filter_chain, jobjectArray stat_sinks, - jboolean enable_platform_certificates_validation, - jboolean enable_skip_dns_lookup_for_proxied_requests, jobjectArray runtime_guards, + jboolean enable_platform_certificates_validation, jobjectArray runtime_guards, jstring rtds_layer_name, jlong rtds_timeout_seconds, jstring ads_address, jlong ads_port, jstring ads_token, jlong ads_token_lifetime, jstring ads_root_certs, jstring node_id, jstring node_region, jstring node_zone, jstring node_sub_zone, jstring cds_resources_locator, @@ -1246,8 +1245,6 @@ void configureBuilder( builder.enforceTrustChainVerification(trust_chain_verification == JNI_TRUE); builder.enablePlatformCertificatesValidation(enable_platform_certificates_validation == JNI_TRUE); builder.setForceAlwaysUsev6(true); - builder.setSkipDnsLookupForProxiedRequests(enable_skip_dns_lookup_for_proxied_requests == - JNI_TRUE); auto guards = javaObjectArrayToStringPairVector(env, runtime_guards); for (std::pair& entry : guards) { @@ -1309,8 +1306,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr jlong stats_flush_seconds, jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id, jboolean trust_chain_verification, jobjectArray filter_chain, jobjectArray stat_sinks, - jboolean enable_platform_certificates_validation, - jboolean enable_skip_dns_lookup_for_proxied_requests, jobjectArray runtime_guards, + jboolean enable_platform_certificates_validation, jobjectArray runtime_guards, jstring rtds_layer_name, jlong rtds_timeout_seconds, jstring ads_address, jlong ads_port, jstring ads_token, jlong ads_token_lifetime, jstring ads_root_certs, jstring node_id, jstring node_region, jstring node_zone, jstring node_sub_zone, jstring cds_resources_locator, @@ -1327,10 +1323,9 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr h2_connection_keepalive_timeout_seconds, max_connections_per_host, stats_flush_seconds, stream_idle_timeout_seconds, per_try_idle_timeout_seconds, app_version, app_id, trust_chain_verification, filter_chain, stat_sinks, enable_platform_certificates_validation, - enable_skip_dns_lookup_for_proxied_requests, runtime_guards, rtds_layer_name, - rtds_timeout_seconds, ads_address, ads_port, ads_token, ads_token_lifetime, ads_root_certs, - node_id, node_region, node_zone, node_sub_zone, cds_resources_locator, cds_timeout_seconds, - enable_cds, builder); + runtime_guards, rtds_layer_name, rtds_timeout_seconds, ads_address, ads_port, ads_token, + ads_token_lifetime, ads_root_certs, node_id, node_region, node_zone, node_sub_zone, + cds_resources_locator, cds_timeout_seconds, enable_cds, builder); return reinterpret_cast(builder.generateBootstrap().release()); } diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java index 0dbc74961b4d..036e5358a0cf 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java @@ -61,7 +61,6 @@ public enum TrustChainVerification { public final List statSinks; public final Map runtimeGuards; public final Boolean enablePlatformCertificatesValidation; - public final Boolean enableSkipDNSLookupForProxiedRequests; public final String rtdsLayerName; public final Integer rtdsTimeoutSeconds; public final String adsAddress; @@ -132,8 +131,6 @@ public enum TrustChainVerification { * @param httpPlatformFilterFactories the configuration for platform filters. * @param stringAccessors platform string accessors to register. * @param keyValueStores platform key-value store implementations. - * @param enableSkipDNSLookupForProxiedRequests whether to skip waiting on DNS response - * for proxied requests. * @param enablePlatformCertificatesValidation whether to use the platform verifier. * @param rtdsLayerName the RTDS layer name for this client. * @param rtdsTimeoutSeconds the timeout for RTDS fetches. @@ -169,12 +166,11 @@ public EnvoyConfiguration( List httpPlatformFilterFactories, Map stringAccessors, Map keyValueStores, List statSinks, - Map runtimeGuards, Boolean enableSkipDNSLookupForProxiedRequests, - boolean enablePlatformCertificatesValidation, String rtdsLayerName, - Integer rtdsTimeoutSeconds, String adsAddress, Integer adsPort, String adsToken, - Integer adsTokenLifetime, String adsRootCerts, String nodeId, String nodeRegion, - String nodeZone, String nodeSubZone, String cdsResourcesLocator, Integer cdsTimeoutSeconds, - boolean enableCds) { + Map runtimeGuards, boolean enablePlatformCertificatesValidation, + String rtdsLayerName, Integer rtdsTimeoutSeconds, String adsAddress, Integer adsPort, + String adsToken, Integer adsTokenLifetime, String adsRootCerts, String nodeId, + String nodeRegion, String nodeZone, String nodeSubZone, String cdsResourcesLocator, + Integer cdsTimeoutSeconds, boolean enableCds) { JniLibrary.load(); this.adminInterfaceEnabled = adminInterfaceEnabled; this.grpcStatsDomain = grpcStatsDomain; @@ -225,7 +221,6 @@ public EnvoyConfiguration( this.runtimeGuards.put(guardAndValue.getKey(), String.valueOf(guardAndValue.getValue())); } this.enablePlatformCertificatesValidation = enablePlatformCertificatesValidation; - this.enableSkipDNSLookupForProxiedRequests = enableSkipDNSLookupForProxiedRequests; this.rtdsLayerName = rtdsLayerName; this.rtdsTimeoutSeconds = rtdsTimeoutSeconds; this.adsAddress = adsAddress; @@ -262,10 +257,9 @@ public long createBootstrap() { h2ConnectionKeepaliveTimeoutSeconds, maxConnectionsPerHost, statsFlushSeconds, streamIdleTimeoutSeconds, perTryIdleTimeoutSeconds, appVersion, appId, enforceTrustChainVerification, filter_chain, stats_sinks, - enablePlatformCertificatesValidation, enableSkipDNSLookupForProxiedRequests, runtime_guards, - rtdsLayerName, rtdsTimeoutSeconds, adsAddress, adsPort, adsToken, adsTokenLifetime, - adsRootCerts, nodeId, nodeRegion, nodeZone, nodeSubZone, cdsResourcesLocator, - cdsTimeoutSeconds, enableCds); + enablePlatformCertificatesValidation, runtime_guards, rtdsLayerName, rtdsTimeoutSeconds, + adsAddress, adsPort, adsToken, adsTokenLifetime, adsRootCerts, nodeId, nodeRegion, nodeZone, + nodeSubZone, cdsResourcesLocator, cdsTimeoutSeconds, enableCds); } static class ConfigurationException extends RuntimeException { diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java index c69667fb72f5..f4b6a1956766 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java @@ -307,9 +307,8 @@ public static native long createBootstrap( long maxConnectionsPerHost, long statsFlushSeconds, long streamIdleTimeoutSeconds, long perTryIdleTimeoutSeconds, String appVersion, String appId, boolean trustChainVerification, byte[][] filterChain, byte[][] statSinks, - boolean enablePlatformCertificatesValidation, boolean enableSkipDNSLookupForProxiedRequests, - byte[][] runtimeGuards, String rtdsLayerName, long rtdsTimeoutSeconds, String adsAddress, - long adsPort, String adsToken, long adsTokenLifetime, String adsRootCerts, String nodeId, - String nodeRegion, String nodeZone, String nodeSubZone, String cdsResourcesLocator, - long cdsTimeoutSeconds, boolean enableCds); + boolean enablePlatformCertificatesValidation, byte[][] runtimeGuards, String rtdsLayerName, + long rtdsTimeoutSeconds, String adsAddress, long adsPort, String adsToken, + long adsTokenLifetime, String adsRootCerts, String nodeId, String nodeRegion, String nodeZone, + String nodeSubZone, String cdsResourcesLocator, long cdsTimeoutSeconds, boolean enableCds); } diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java index ec0cecbd35de..0f59c4ea7cf9 100644 --- a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java @@ -52,7 +52,6 @@ public class NativeCronvoyEngineBuilderImpl extends CronvoyEngineBuilderImpl { private boolean mEnableSocketTag = true; private boolean mEnableInterfaceBinding = false; private boolean mEnableProxying = false; - private boolean mEnableSkipDNSLookupForProxiedRequests = false; private int mH2ConnectionKeepaliveIdleIntervalMilliseconds = 1; private int mH2ConnectionKeepaliveTimeoutSeconds = 10; private int mMaxConnectionsPerHost = 7; @@ -143,9 +142,8 @@ mEnableGzipDecompression, brotliEnabled(), mEnableSocketTag, mEnableInterfaceBin mMaxConnectionsPerHost, mStatsFlushSeconds, mStreamIdleTimeoutSeconds, mPerTryIdleTimeoutSeconds, mAppVersion, mAppId, mTrustChainVerification, nativeFilterChain, platformFilterChain, stringAccessors, keyValueStores, statSinks, runtimeGuards, - mEnableSkipDNSLookupForProxiedRequests, mEnablePlatformCertificatesValidation, - mRtdsLayerName, mRtdsTimeoutSeconds, mAdsAddress, mAdsPort, mAdsToken, mAdsTokenLifetime, - mAdsRootCerts, mNodeId, mNodeRegion, mNodeZone, mNodeSubZone, mCdsResourcesLocator, - mCdsTimeoutSeconds, mEnableCds); + mEnablePlatformCertificatesValidation, mRtdsLayerName, mRtdsTimeoutSeconds, mAdsAddress, + mAdsPort, mAdsToken, mAdsTokenLifetime, mAdsRootCerts, mNodeId, mNodeRegion, mNodeZone, + mNodeSubZone, mCdsResourcesLocator, mCdsTimeoutSeconds, mEnableCds); } } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt index b6e1189ee0bf..2445ec84d16d 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt @@ -38,7 +38,6 @@ open class EngineBuilder( protected var eventTracker: ((Map) -> Unit)? = null protected var enableProxying = false private var runtimeGuards = mutableMapOf() - private var enableSkipDNSLookupForProxiedRequests = false private var engineType: () -> EnvoyEngine = { EnvoyEngineImpl(onEngineRunning, logger, eventTracker) } @@ -306,22 +305,6 @@ open class EngineBuilder( return this } - /** - * Allows Envoy to avoid having to wait on DNS response in the dynamic forward proxy filter - * for requests that are proxied i.e., a proxied request that goes to example.com will - * not have to wait for the DNS resolution for example.com domain if skipping of the DNS lookup - * is enabled. Defaults to false. - * - * @param enableSkipDNSLookup whether to ship waiting for DNS responses in the - * dynamic forward proxy filter for proxied requests. - * - * @return This builder. - */ - fun enableSkipDNSLookupForProxiedRequests(enableSkipDNSLookup: Boolean): EngineBuilder { - this.enableSkipDNSLookupForProxiedRequests = enableSkipDNSLookup - return this - } - /** * Add a rate at which to ping h2 connections on new stream creation if the connection has * sat idle. Defaults to 1 millisecond which effectively enables h2 ping functionality @@ -686,7 +669,6 @@ open class EngineBuilder( keyValueStores, statsSinks, runtimeGuards, - enableSkipDNSLookupForProxiedRequests, enablePlatformCertificatesValidation, rtdsLayerName, rtdsTimeoutSeconds, diff --git a/mobile/test/common/integration/rtds_integration_test.cc b/mobile/test/common/integration/rtds_integration_test.cc index 61e36672552f..658c2c07dedd 100644 --- a/mobile/test/common/integration/rtds_integration_test.cc +++ b/mobile/test/common/integration/rtds_integration_test.cc @@ -52,8 +52,7 @@ TEST_P(RtdsIntegrationTest, RtdsReload) { EXPECT_EQ(cc_.on_header_consumed_bytes_from_response, 27); EXPECT_EQ(cc_.on_complete_received_byte_count, 67); // Check that the Runtime config is from the static layer. - EXPECT_FALSE(Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.skip_dns_lookup_for_proxied_requests")); + EXPECT_FALSE(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); const std::string load_success_counter = "runtime.load_success"; uint64_t load_success_value = getCounterValue(load_success_counter); @@ -63,7 +62,7 @@ TEST_P(RtdsIntegrationTest, RtdsReload) { auto some_rtds_layer = TestUtility::parseYaml(R"EOF( name: some_rtds_layer layer: - envoy.reloadable_features.skip_dns_lookup_for_proxied_requests: True + envoy.reloadable_features.test_feature_false: True )EOF"); sendDiscoveryResponse( Config::TypeUrl::get().Runtime, {some_rtds_layer}, {some_rtds_layer}, {}, "1"); @@ -71,8 +70,7 @@ TEST_P(RtdsIntegrationTest, RtdsReload) { ASSERT_TRUE(waitForCounterGe(load_success_counter, load_success_value + 1)); // Verify that the Runtime config values are from the RTDS response. - EXPECT_TRUE(Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.skip_dns_lookup_for_proxied_requests")); + EXPECT_TRUE(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); } } // namespace diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt index acdb2bd171d7..80969da60de4 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt @@ -97,7 +97,6 @@ class EnvoyConfigurationTest { filterChain: MutableList = mutableListOf(EnvoyNativeFilterConfig("buffer_filter_1", "{'@type': 'type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer'}"), EnvoyNativeFilterConfig("buffer_filter_2", "{'@type': 'type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer'}")), platformFilterFactories: MutableList = mutableListOf(TestEnvoyHTTPFilterFactory("name1"), TestEnvoyHTTPFilterFactory("name2")), runtimeGuards: Map = emptyMap(), - enableSkipDNSLookupForProxiedRequests: Boolean = false, statSinks: MutableList = mutableListOf(), enablePlatformCertificatesValidation: Boolean = false, rtdsLayerName: String = "", @@ -149,7 +148,6 @@ class EnvoyConfigurationTest { emptyMap(), statSinks, runtimeGuards, - enableSkipDNSLookupForProxiedRequests, enablePlatformCertificatesValidation, rtdsLayerName, rtdsTimeoutSeconds, @@ -229,9 +227,6 @@ class EnvoyConfigurationTest { // Cert Validation assertThat(resolvedTemplate).contains("trusted_ca:") - // Proxying - assertThat(resolvedTemplate).contains("skip_dns_lookup_for_proxied_requests: false") - // Validate ordering between filters and platform filters assertThat(resolvedTemplate).matches(Pattern.compile(".*name1.*name2.*buffer_filter_1.*buffer_filter_2.*", Pattern.DOTALL)); // Validate that createYaml doesn't change filter order. @@ -258,7 +253,6 @@ class EnvoyConfigurationTest { enableBrotliDecompression = true, enableSocketTagging = true, enableInterfaceBinding = true, - enableSkipDNSLookupForProxiedRequests = true, enablePlatformCertificatesValidation = true, dnsPreresolveHostnames = mutableListOf(), filterChain = mutableListOf(), @@ -296,9 +290,6 @@ class EnvoyConfigurationTest { // enableInterfaceBinding = true assertThat(resolvedTemplate).contains("enable_interface_binding: true") - // enableSkipDNSLookupForProxiedRequests = true - assertThat(resolvedTemplate).contains("skip_dns_lookup_for_proxied_requests: true") - // enablePlatformCertificatesValidation = true assertThat(resolvedTemplate).doesNotContain("trusted_ca:") diff --git a/mobile/test/kotlin/apps/experimental/MainActivity.kt b/mobile/test/kotlin/apps/experimental/MainActivity.kt index 5d05d7cee9f5..731cb51b00fc 100644 --- a/mobile/test/kotlin/apps/experimental/MainActivity.kt +++ b/mobile/test/kotlin/apps/experimental/MainActivity.kt @@ -64,7 +64,6 @@ class MainActivity : Activity() { .enableInterfaceBinding(true) .enableSocketTagging(true) .enableProxying(true) - .enableSkipDNSLookupForProxiedRequests(true) // TODO: uncomment once platform cert validation is fixed. // .enablePlatformCertificatesValidation(true) .addNativeFilter("envoy.filters.http.buffer", "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}") diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 927c78b45b23..1c9a6509e56e 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -1901,7 +1901,8 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( absl::optional origin = getOrigin(transport_socket_options, host); if (protocols.size() == 3 && - context_.runtime().snapshot().featureEnabled("upstream.use_http3", 100)) { + context_.runtime().snapshot().featureEnabled("upstream.use_http3", 100) && + !transport_socket_options->http11ProxyInfo()) { ASSERT(contains(protocols, {Http::Protocol::Http11, Http::Protocol::Http2, Http::Protocol::Http3})); ASSERT(alternate_protocol_options.has_value()); From 8a533e146103c7b2300088ee7a7634f747b95388 Mon Sep 17 00:00:00 2001 From: code Date: Tue, 9 May 2023 23:36:03 +0800 Subject: [PATCH 164/740] minor optimization: minor optimization to improve the code quality/readability (#27262) Signed-off-by: wbpcode --- source/common/http/conn_manager_impl.cc | 74 ++++++++++++------------- source/common/http/conn_manager_impl.h | 1 + 2 files changed, 35 insertions(+), 40 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 276b255eaebc..f4d9b59abd1e 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -119,6 +119,8 @@ const ResponseHeaderMap& ConnectionManagerImpl::continueHeader() { void ConnectionManagerImpl::initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) { read_callbacks_ = &callbacks; + dispatcher_ = &callbacks.connection().dispatcher(); + stats_.named_.downstream_cx_total_.inc(); stats_.named_.downstream_cx_active_.inc(); if (read_callbacks_->connection().ssl()) { @@ -143,15 +145,15 @@ void ConnectionManagerImpl::initializeReadFilterCallbacks(Network::ReadFilterCal } if (config_.idleTimeout()) { - connection_idle_timer_ = read_callbacks_->connection().dispatcher().createScaledTimer( - Event::ScaledTimerType::HttpDownstreamIdleConnectionTimeout, - [this]() -> void { onIdleTimeout(); }); + connection_idle_timer_ = + dispatcher_->createScaledTimer(Event::ScaledTimerType::HttpDownstreamIdleConnectionTimeout, + [this]() -> void { onIdleTimeout(); }); connection_idle_timer_->enableTimer(config_.idleTimeout().value()); } if (config_.maxConnectionDuration()) { - connection_duration_timer_ = read_callbacks_->connection().dispatcher().createTimer( - [this]() -> void { onConnectionDurationTimeout(); }); + connection_duration_timer_ = + dispatcher_->createTimer([this]() -> void { onConnectionDurationTimeout(); }); connection_duration_timer_->enableTimer(config_.maxConnectionDuration().value()); } @@ -327,7 +329,7 @@ void ConnectionManagerImpl::doDeferredStreamDestroy(ActiveStream& stream) { stream.filter_manager_.destroyFilters(); - read_callbacks_->connection().dispatcher().deferredDelete(stream.removeFromList(streams_)); + dispatcher_->deferredDelete(stream.removeFromList(streams_)); // The response_encoder should never be dangling (unless we're destroying a // stream we are recreating) as the codec level stream will either outlive the @@ -356,7 +358,7 @@ RequestDecoder& ConnectionManagerImpl::newStream(ResponseEncoder& response_encod if (downstream_stream_account == nullptr) { // Create account, wiring the stream to use it for tracking bytes. // If tracking is disabled, the wiring becomes a NOP. - auto& buffer_factory = read_callbacks_->connection().dispatcher().getWatermarkFactory(); + auto& buffer_factory = dispatcher_->getWatermarkFactory(); downstream_stream_account = buffer_factory.createAccount(response_encoder.getStream()); response_encoder.getStream().setAccount(downstream_stream_account); } @@ -663,8 +665,7 @@ void ConnectionManagerImpl::chargeTracingStats(const Tracing::Reason& tracing_re void ConnectionManagerImpl::RdsRouteConfigUpdateRequester::requestRouteConfigUpdate( Http::RouteConfigUpdatedCallbackSharedPtr route_config_updated_cb) { absl::optional route_config = parent_.routeConfig(); - Event::Dispatcher& thread_local_dispatcher = - parent_.connection_manager_.read_callbacks_->connection().dispatcher(); + Event::Dispatcher& thread_local_dispatcher = *parent_.connection_manager_.dispatcher_; if (route_config.has_value() && route_config.value()->usesVhds()) { ASSERT(!parent_.request_headers_->Host()->value().empty()); const auto& host_header = absl::AsciiStrToLower(parent_.request_headers_->getHostValue()); @@ -740,7 +741,7 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect : makeOptRef( *connection_manager_.config_.tracingConfig())), stream_id_(connection_manager.random_generator_.random()), - filter_manager_(*this, connection_manager_.read_callbacks_->connection().dispatcher(), + filter_manager_(*this, *connection_manager_.dispatcher_, connection_manager_.read_callbacks_->connection(), stream_id_, std::move(account), connection_manager_.config_.proxy100Continue(), buffer_limit, connection_manager_.config_.filterFactory(), @@ -799,17 +800,16 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect if (connection_manager_.config_.streamIdleTimeout().count()) { idle_timeout_ms_ = connection_manager_.config_.streamIdleTimeout(); - stream_idle_timer_ = - connection_manager_.read_callbacks_->connection().dispatcher().createScaledTimer( - Event::ScaledTimerType::HttpDownstreamIdleStreamTimeout, - [this]() -> void { onIdleTimeout(); }); + stream_idle_timer_ = connection_manager_.dispatcher_->createScaledTimer( + Event::ScaledTimerType::HttpDownstreamIdleStreamTimeout, + [this]() -> void { onIdleTimeout(); }); resetIdleTimer(); } if (connection_manager_.config_.requestTimeout().count()) { std::chrono::milliseconds request_timeout = connection_manager_.config_.requestTimeout(); - request_timer_ = connection_manager.read_callbacks_->connection().dispatcher().createTimer( - [this]() -> void { onRequestTimeout(); }); + request_timer_ = + connection_manager.dispatcher_->createTimer([this]() -> void { onRequestTimeout(); }); request_timer_->enableTimer(request_timeout, this); } @@ -817,30 +817,27 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect std::chrono::milliseconds request_headers_timeout = connection_manager_.config_.requestHeadersTimeout(); request_header_timer_ = - connection_manager.read_callbacks_->connection().dispatcher().createTimer( - [this]() -> void { onRequestHeaderTimeout(); }); + connection_manager.dispatcher_->createTimer([this]() -> void { onRequestHeaderTimeout(); }); request_header_timer_->enableTimer(request_headers_timeout, this); } const auto max_stream_duration = connection_manager_.config_.maxStreamDuration(); if (max_stream_duration.has_value() && max_stream_duration.value().count()) { - max_stream_duration_timer_ = - connection_manager.read_callbacks_->connection().dispatcher().createTimer( - [this]() -> void { onStreamMaxDurationReached(); }); + max_stream_duration_timer_ = connection_manager.dispatcher_->createTimer( + [this]() -> void { onStreamMaxDurationReached(); }); max_stream_duration_timer_->enableTimer(connection_manager_.config_.maxStreamDuration().value(), this); } if (connection_manager_.config_.accessLogFlushInterval().has_value()) { - access_log_flush_timer_ = - connection_manager.read_callbacks_->connection().dispatcher().createTimer([this]() -> void { - // If the request is complete, we've already done the stream-end access-log, and shouldn't - // do the periodic log. - if (!streamInfo().requestComplete().has_value()) { - filter_manager_.log(AccessLog::AccessLogType::DownstreamPeriodic); - refreshAccessLogFlushTimer(); - } - }); + access_log_flush_timer_ = connection_manager.dispatcher_->createTimer([this]() -> void { + // If the request is complete, we've already done the stream-end access-log, and shouldn't + // do the periodic log. + if (!streamInfo().requestComplete().has_value()) { + filter_manager_.log(AccessLog::AccessLogType::DownstreamPeriodic); + refreshAccessLogFlushTimer(); + } + }); refreshAccessLogFlushTimer(); } } @@ -1053,7 +1050,7 @@ void ConnectionManagerImpl::ActiveStream::maybeEndDecode(bool end_stream) { // If recreateStream is called, the HCM rewinds state and may send more encodeData calls. if (end_stream && !filter_manager_.remoteDecodeComplete()) { filter_manager_.streamInfo().downstreamTiming().onLastDownstreamRxByteReceived( - connection_manager_.read_callbacks_->connection().dispatcher().timeSource()); + connection_manager_.dispatcher_->timeSource()); ENVOY_STREAM_LOG(debug, "request end stream", *this); } } @@ -1403,8 +1400,7 @@ void ConnectionManagerImpl::startDrainSequence() { ASSERT(drain_state_ == DrainState::NotDraining); drain_state_ = DrainState::Draining; codec_->shutdownNotice(); - drain_timer_ = read_callbacks_->connection().dispatcher().createTimer( - [this]() -> void { onDrainTimeout(); }); + drain_timer_ = dispatcher_->createTimer([this]() -> void { onDrainTimeout(); }); drain_timer_->enableTimer(config_.drainTimeout()); } @@ -1505,9 +1501,8 @@ void ConnectionManagerImpl::ActiveStream::refreshDurationTimeout() { // Finally create (if necessary) and enable the timer. if (!max_stream_duration_timer_) { - max_stream_duration_timer_ = - connection_manager_.read_callbacks_->connection().dispatcher().createTimer( - [this]() -> void { onStreamMaxDurationReached(); }); + max_stream_duration_timer_ = connection_manager_.dispatcher_->createTimer( + [this]() -> void { onStreamMaxDurationReached(); }); } max_stream_duration_timer_->enableTimer(timeout); } @@ -1961,10 +1956,9 @@ void ConnectionManagerImpl::ActiveStream::refreshIdleTimeout() { if (idle_timeout_ms_.count()) { // If we have a route-level idle timeout but no global stream idle timeout, create a timer. if (stream_idle_timer_ == nullptr) { - stream_idle_timer_ = - connection_manager_.read_callbacks_->connection().dispatcher().createScaledTimer( - Event::ScaledTimerType::HttpDownstreamIdleStreamTimeout, - [this]() -> void { onIdleTimeout(); }); + stream_idle_timer_ = connection_manager_.dispatcher_->createScaledTimer( + Event::ScaledTimerType::HttpDownstreamIdleStreamTimeout, + [this]() -> void { onIdleTimeout(); }); } } else if (stream_idle_timer_ != nullptr) { // If we had a global stream idle timeout but the route-level idle timeout is set to zero diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 29b8c01b2673..a9f402a94abc 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -568,6 +568,7 @@ class ConnectionManagerImpl : Logger::Loggable, const LocalInfo::LocalInfo& local_info_; Upstream::ClusterManager& cluster_manager_; Network::ReadFilterCallbacks* read_callbacks_{}; + Event::Dispatcher* dispatcher_{}; ConnectionManagerListenerStats& listener_stats_; Server::OverloadManager& overload_manager_; Server::ThreadLocalOverloadState& overload_state_; From 309eec9443b59fb05b129dde5383dc656acdb5c2 Mon Sep 17 00:00:00 2001 From: code Date: Tue, 9 May 2023 23:36:44 +0800 Subject: [PATCH 165/740] minor optimization: minor optimization to setRoute to improve the code quality (#27242) Signed-off-by: wbpcode --- source/common/http/conn_manager_impl.cc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index f4d9b59abd1e..205383045e8c 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1929,19 +1929,21 @@ void ConnectionManagerImpl::ActiveStream::setRoute(Router::RouteConstSharedPtr r return; } - filter_manager_.streamInfo().route_ = route; - setCachedRoute({std::move(route)}); - if (nullptr == filter_manager_.streamInfo().route() || - nullptr == filter_manager_.streamInfo().route()->routeEntry()) { + // Update the cached route. + setCachedRoute({route}); + // Update the cached cluster info based on the new route. + if (nullptr == route || nullptr == route->routeEntry()) { cached_cluster_info_ = nullptr; } else { - Upstream::ThreadLocalCluster* local_cluster = - connection_manager_.cluster_manager_.getThreadLocalCluster( - filter_manager_.streamInfo().route()->routeEntry()->clusterName()); - cached_cluster_info_ = (nullptr == local_cluster) ? nullptr : local_cluster->info(); + auto* cluster = connection_manager_.cluster_manager_.getThreadLocalCluster( + route->routeEntry()->clusterName()); + cached_cluster_info_ = (nullptr == cluster) ? nullptr : cluster->info(); } + // Update route and cluster info in the filter manager's stream info. + filter_manager_.streamInfo().route_ = std::move(route); // Now can move route here safely. filter_manager_.streamInfo().setUpstreamClusterInfo(cached_cluster_info_.value()); + refreshCachedTracingCustomTags(); refreshDurationTimeout(); refreshIdleTimeout(); From d93958dbef1c2b11587bea6f3eb561bd630b212d Mon Sep 17 00:00:00 2001 From: Yousuk Seung Date: Tue, 9 May 2023 08:49:25 -0700 Subject: [PATCH 166/740] api: clarify ranges of LB config parameters (#27192) * api: clarify ranges of LB config parameters Signed-off-by: Yousuk Seung --- .../v3/client_side_weighted_round_robin.proto | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.proto b/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.proto index 4af32eb63504..ae4a99b4517a 100644 --- a/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.proto +++ b/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.proto @@ -6,6 +6,7 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.load_balancing_policies.client_side_weighted_round_robin.v3"; option java_outer_classname = "ClientSideWeightedRoundRobinProto"; @@ -58,10 +59,12 @@ message ClientSideWeightedRoundRobin { // blackout_period applies. Defaults to 3 minutes. google.protobuf.Duration weight_expiration_period = 4; - // How often endpoint weights are recalculated. Default is 1 second. + // How often endpoint weights are recalculated. Values less than 100ms are + // capped at 100ms. Default is 1 second. google.protobuf.Duration weight_update_period = 5; // The multiplier used to adjust endpoint weights with the error rate - // calculated as eps/qps. Default is 1.0. - google.protobuf.FloatValue error_utilization_penalty = 6; + // calculated as eps/qps. Configuration is rejected if this value is negative. + // Default is 1.0. + google.protobuf.FloatValue error_utilization_penalty = 6 [(validate.rules).float = {gte: 0.0}]; } From 20ca3ad7e0b3183d0db6af8fd77a08f83cefcee0 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Tue, 9 May 2023 12:10:54 -0400 Subject: [PATCH 167/740] Reserve more CPU cores for integration tests with high rate of flakes (#27257) Signed-off-by: Yan Avlasov --- .../filters/http/local_ratelimit/BUILD | 3 ++ .../filters/network/local_ratelimit/BUILD | 3 ++ test/integration/BUILD | 46 ++++++++++++++++++- 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/http/local_ratelimit/BUILD b/test/extensions/filters/http/local_ratelimit/BUILD index 3fd8c5261bef..da428f6ec09b 100644 --- a/test/extensions/filters/http/local_ratelimit/BUILD +++ b/test/extensions/filters/http/local_ratelimit/BUILD @@ -40,6 +40,9 @@ envoy_extension_cc_test( size = "large", srcs = ["local_ratelimit_integration_test.cc"], extension_names = ["envoy.filters.http.local_ratelimit"], + tags = [ + "cpu:3", + ], deps = [ "//source/extensions/filters/http/local_ratelimit:config", "//test/integration:http_protocol_integration_lib", diff --git a/test/extensions/filters/network/local_ratelimit/BUILD b/test/extensions/filters/network/local_ratelimit/BUILD index 683887157726..2922f48e8f4e 100644 --- a/test/extensions/filters/network/local_ratelimit/BUILD +++ b/test/extensions/filters/network/local_ratelimit/BUILD @@ -32,6 +32,9 @@ envoy_extension_cc_test( size = "large", srcs = ["local_ratelimit_integration_test.cc"], extension_names = ["envoy.filters.network.local_ratelimit"], + tags = [ + "cpu:3", + ], deps = [ "//source/extensions/filters/network/local_ratelimit:config", "//source/extensions/filters/network/tcp_proxy:config", diff --git a/test/integration/BUILD b/test/integration/BUILD index 95186ba1941d..2be58b4d8685 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -57,6 +57,9 @@ envoy_cc_test( srcs = envoy_select_admin_functionality( ["ads_integration_test.cc"], ), + tags = [ + "cpu:3", + ], deps = [ ":ads_integration_lib", ":http_integration_lib", @@ -415,6 +418,9 @@ envoy_cc_test( "http2_flood_integration_test.cc", ], shard_count = 8, + tags = [ + "cpu:3", + ], deps = [ ":autonomous_upstream_lib", ":http_integration_lib", @@ -441,6 +447,9 @@ envoy_cc_test( "multiplexed_integration_test.cc", ], shard_count = 16, + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", ":socket_interface_swap_lib", @@ -510,6 +519,9 @@ envoy_cc_test( "buffer_accounting_integration_test.cc", ], shard_count = 4, + tags = [ + "cpu:3", + ], deps = [ ":base_overload_integration_test_lib", ":http_integration_lib", @@ -622,6 +634,9 @@ envoy_cc_test( "filter_integration_test.cc", ]), shard_count = 16, + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", ":socket_interface_swap_lib", @@ -696,6 +711,9 @@ envoy_cc_test( # As this test has many H1/H2/v4/v6 tests it takes a while to run. # Shard it enough to bring the run time in line with other integration tests. shard_count = 48, + tags = [ + "cpu:3", + ], deps = [ ":protocol_integration_test_lib", ], @@ -719,6 +737,9 @@ envoy_cc_test( "multiplexed_upstream_integration_test.cc", ], shard_count = 4, + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", "//source/common/http:header_map_lib", @@ -836,6 +857,9 @@ envoy_cc_test( # As this test has many pauses for idle timeouts, it takes a while to run. # Shard it enough to bring the run time in line with other integration tests. shard_count = 8, + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", "//test/integration/filters:backpressure_filter_config_lib", @@ -1160,6 +1184,9 @@ envoy_cc_test( "integration_test.h", ], shard_count = 2, + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//envoy/registry", @@ -1222,6 +1249,9 @@ envoy_cc_test( "websocket_integration_test.h", ], shard_count = 4, + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", "//source/common/http:header_map_lib", @@ -1362,6 +1392,9 @@ envoy_cc_test( size = "large", srcs = ["overload_integration_test.cc"], shard_count = 8, + tags = [ + "cpu:3", + ], deps = [ ":base_overload_integration_test_lib", ":http_protocol_integration_lib", @@ -1536,6 +1569,9 @@ envoy_cc_test( "//test/config/integration/certs", ], shard_count = 2, + tags = [ + "cpu:3", + ], deps = [ ":integration_lib", ":tcp_proxy_integration_proto_cc_proto", @@ -1571,7 +1607,7 @@ envoy_cc_test( ], shard_count = 1, tags = [ - "cpu:4", + "cpu:3", ], deps = [ ":integration_lib", @@ -1603,6 +1639,9 @@ envoy_cc_test( "//test/config/integration/certs", ], shard_count = 16, + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":http_protocol_integration_lib", @@ -2068,6 +2107,7 @@ envoy_cc_test( data = ["//test/config/integration/certs"], shard_count = 12, tags = [ + "cpu:4", "nofips", ], deps = select({ @@ -2105,6 +2145,7 @@ envoy_cc_test( # Each of these tests exceeds 20s; # QuicHttpIntegrationTests/QuicHttpIntegrationTest.MultipleQuicConnections[With|No]BPF* tags = [ + "cpu:3", "fails_on_clang_cl", "fails_on_windows", "nofips", @@ -2192,6 +2233,9 @@ envoy_cc_test( data = [ "//test/config/integration/certs", ], + tags = [ + "cpu:3", + ], deps = [ ":ads_integration_lib", ":fake_upstream_lib", From ec58c465a86afd5b92f8b29654f9605e4c50649a Mon Sep 17 00:00:00 2001 From: David Benjamin Date: Tue, 9 May 2023 12:12:23 -0400 Subject: [PATCH 168/740] tls: Use BoringSSL APIs to bootstrap stat names (#27261) ContextImpl predeclares stat names to avoid needing synchronization and logging stats later. It does this by (1) iterating over SSL_CTX_get_ciphers at SSL_CTX creation time and (2) hard-coding a lists of known TLS 1.3 ciphers, TLS versions, NamedGroups, etc. (1) implicitly assumes cipher suite configuration won't be applied later. The OpenSSL API allows applications to configure ciphers on either the SSL_CTX or the SSL. Envoy has an SslCtxCb which callers could use to either reconfigure the SSL_CTX or install a callback that does the latter. In those two cases, the final cipher list may include a cipher that the initial one does not. (2) breaks whenever BoringSSL adds a new feature. In principle, we could update Envoy when updating BoringSSL, but this is an unresasonable development overhead for just one of many BoringSSL consumers to impose. Such costs are particularly high when considering needing to coordinate updates to Envoy and BoringSSL across different repositories. (The lists are already out of date, as we've replaced our old experimental post-quantum algorithm, CECPQ2, with a new one X25519Kyber768Draft00. That one is also experimental and will be replaced in the future too.) To fix both of these, https://boringssl-review.googlesource.com/c/boringssl/+/59667 adds APIs in BoringSSL to query possible strings that various functions can return. These lists will be a superset of those that any one application may care about (e.g. we may have a deprecated cipher that Envoy no longer needs, it may be a cipher only applicable for some modes, or an cipher that's too experimental for general use) but this is fine provided this is just used to initialize the table. In particular, they are *not* intended to enumerate supported features. Since Envoy has to support a range of BoringSSL versions, the new code is gated behind BORINGSSL_API_VERSION. On older BoringSSLs, the it still uses the old code, with all the issues unchanged. Signed-off-by: David Benjamin --- .../transport_sockets/tls/context_impl.cc | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index 3e1be04fc4ce..ea8dff81ea39 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -303,6 +303,24 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c parsed_alpn_protocols_ = parseAlpnProtocols(config.alpnProtocols()); +#if BORINGSSL_API_VERSION >= 21 + // Register stat names based on lists reported by BoringSSL. + std::vector list(SSL_get_all_cipher_names(nullptr, 0)); + SSL_get_all_cipher_names(list.data(), list.size()); + stat_name_set_->rememberBuiltins(list); + + list.resize(SSL_get_all_curve_names(nullptr, 0)); + SSL_get_all_curve_names(list.data(), list.size()); + stat_name_set_->rememberBuiltins(list); + + list.resize(SSL_get_all_signature_algorithm_names(nullptr, 0)); + SSL_get_all_signature_algorithm_names(list.data(), list.size()); + stat_name_set_->rememberBuiltins(list); + + list.resize(SSL_get_all_version_names(nullptr, 0)); + SSL_get_all_version_names(list.data(), list.size()); + stat_name_set_->rememberBuiltins(list); +#else // Use the SSL library to iterate over the configured ciphers. // // Note that if a negotiated cipher suite is outside of this set, we'll issue an ENVOY_BUG. @@ -312,14 +330,6 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c } } - // As late as possible, run the custom SSL_CTX configuration callback on each - // SSL_CTX, if set. - if (auto sslctx_cb = config.sslctxCb(); sslctx_cb) { - for (TlsContext& ctx : tls_contexts_) { - sslctx_cb(ctx.ssl_ctx_.get()); - } - } - // Add supported cipher suites from the TLS 1.3 spec: // https://tools.ietf.org/html/rfc8446#appendix-B.4 // AES-CCM cipher suites are removed (no BoringSSL support). @@ -358,6 +368,15 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c // // Note that if a negotiated version is outside of this set, we'll issue an ENVOY_BUG. stat_name_set_->rememberBuiltins({"TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"}); +#endif + + // As late as possible, run the custom SSL_CTX configuration callback on each + // SSL_CTX, if set. + if (auto sslctx_cb = config.sslctxCb(); sslctx_cb) { + for (TlsContext& ctx : tls_contexts_) { + sslctx_cb(ctx.ssl_ctx_.get()); + } + } if (!config.tlsKeyLogPath().empty()) { ENVOY_LOG(debug, "Enable tls key log"); From 927d83c7cfdaaf46350a77b8e652dbebf549f471 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 9 May 2023 17:16:50 +0100 Subject: [PATCH 169/740] deps: Bump `rules_rust` -> 0.21.1 (#27279) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index a7c48902c214..62e2c4cfd9e6 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1317,12 +1317,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Bazel rust rules", project_desc = "Bazel rust rules (used by Wasm)", project_url = "https://github.com/bazelbuild/rules_rust", - version = "0.20.0", - sha256 = "950a3ad4166ae60c8ccd628d1a8e64396106e7f98361ebe91b0bcfe60d8e4b60", + version = "0.21.1", + sha256 = "25209daff2ba21e818801c7b2dab0274c43808982d6aea9f796d899db6319146", urls = ["https://github.com/bazelbuild/rules_rust/releases/download/{version}/rules_rust-v{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.wasmtime"], - release_date = "2023-03-28", + release_date = "2023-04-24", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/bazelbuild/rules_rust/blob/{version}/LICENSE.txt", From 626329ffe5810c52e71030632ada0689e669ed22 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 9 May 2023 17:39:21 +0100 Subject: [PATCH 170/740] deps: Bump `aspect_bazel_lib` -> 1.31.1 (#27280) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 62e2c4cfd9e6..c02abaa52ac5 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -136,12 +136,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Aspect Bazel helpers", project_desc = "Base Starlark libraries and basic Bazel rules which are useful for constructing rulesets and BUILD files", project_url = "https://github.com/aspect-build/bazel-lib", - version = "1.27.1", - sha256 = "80897b673c2b506d21f861ae316689aa8abcc3e56947580a41bf9e68ff13af58", + version = "1.31.1", + sha256 = "5f3443b1d98a462a8b7330f4742483afc8b2d17c8555dd97ce4146f43e961718", strip_prefix = "bazel-lib-{version}", urls = ["https://github.com/aspect-build/bazel-lib/archive/v{version}.tar.gz"], use_category = ["build"], - release_date = "2023-02-22", + release_date = "2023-05-05", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/aspect-build/bazel-lib/blob/v{version}/LICENSE", From 8a6f65c37a67c7206c6d6fe19d0f2d6f61b63aa6 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 9 May 2023 17:59:32 +0100 Subject: [PATCH 171/740] deps: Bump `bazel_toolchains` -> 5.1.2 (#27284) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index c02abaa52ac5..926e922d3f10 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -45,13 +45,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "bazel-toolchains", project_desc = "Bazel toolchain configs for RBE", project_url = "https://github.com/bazelbuild/bazel-toolchains", - version = "5.1.1", - sha256 = "e52789d4e89c3e2dc0e3446a9684626a626b6bec3fde787d70bae37c6ebcc47f", + version = "5.1.2", + sha256 = "02e4f3744f1ce3f6e711e261fd322916ddd18cccd38026352f7a4c0351dbda19", strip_prefix = "bazel-toolchains-{version}", urls = [ "https://github.com/bazelbuild/bazel-toolchains/archive/v{version}.tar.gz", ], - release_date = "2021-11-30", + release_date = "2022-08-09", use_category = ["build"], license = "Apache-2.0", license_url = "https://github.com/bazelbuild/bazel-toolchains/blob/v{version}/LICENSE", From e5326d4c7c485d8f978bdfa3b6437a8cdf737f01 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Tue, 9 May 2023 13:11:01 -0400 Subject: [PATCH 172/740] matcher: Generic matching API: configuration validation (#27091) Signed-off-by: tyxia --- envoy/matcher/matcher.h | 26 ++- source/common/matcher/field_matcher.h | 13 +- source/common/matcher/map_matcher.h | 9 +- source/common/matcher/matcher.h | 3 +- .../extensions/common/matcher/trie_matcher.h | 9 +- test/common/matcher/matcher_test.cc | 174 ++++++++++++++++++ test/common/matcher/test_utility.h | 31 +++- .../common/matcher/trie_matcher_test.cc | 39 ++++ tools/code_format/config.yaml | 3 + 9 files changed, 300 insertions(+), 7 deletions(-) diff --git a/envoy/matcher/matcher.h b/envoy/matcher/matcher.h index 8dc9c7f35275..32fdd76e2065 100644 --- a/envoy/matcher/matcher.h +++ b/envoy/matcher/matcher.h @@ -10,6 +10,7 @@ #include "envoy/config/typed_config.h" #include "envoy/protobuf/message_validator.h" +#include "absl/container/flat_hash_set.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "xds/type/matcher/v3/matcher.pb.h" @@ -25,6 +26,7 @@ class ServerFactoryContext; namespace Matcher { using MatchingDataType = absl::variant; +inline constexpr absl::string_view DefaultMatchingDataType = "string"; // This file describes a MatchTree, which traverses a tree of matches until it // either matches (resulting in either an action or a new tree to traverse) or doesn't match. @@ -161,10 +163,22 @@ class InputMatcher { /** * Whether the provided input is a match. - * @param absl::optional the value to match on. Will be absl::nullopt if the + * @param Matcher::MatchingDataType the value to match on. Will be absl::monostate() if the * lookup failed. */ virtual bool match(const Matcher::MatchingDataType& input) PURE; + + /** + * A set of data input types supported by InputMatcher. + * String is default supported data input type because almost all the derived objects support + * string only. The name of core types (e.g., std::string, int) is defined string constrant which + * produces human-readable form (e.g., "string", "int"). + * + * Override this function to provide matcher specific supported data input types. + */ + virtual absl::flat_hash_set supportedDataInputTypes() const { + return absl::flat_hash_set{std::string(DefaultMatchingDataType)}; + } }; using InputMatcherPtr = std::unique_ptr; @@ -237,6 +251,16 @@ template class DataInput { virtual ~DataInput() = default; virtual DataInputGetResult get(const DataType& data) const PURE; + + /** + * Input type of DataInput. + * String is default data input type since nearly all the DataInput's derived objects' input type + * is string. The name of core types (e.g., std::string, int) is defined string constrant which + * produces human-readable form (e.g., "string", "int"). + * + * Override this function to provide matcher specific data input type. + */ + virtual absl::string_view dataInputType() const { return DefaultMatchingDataType; } }; template using DataInputPtr = std::unique_ptr>; diff --git a/source/common/matcher/field_matcher.h b/source/common/matcher/field_matcher.h index c94cf6d18cd9..99c28945ce38 100644 --- a/source/common/matcher/field_matcher.h +++ b/source/common/matcher/field_matcher.h @@ -3,6 +3,8 @@ #include "envoy/matcher/matcher.h" +#include "absl/strings/str_join.h" + namespace Envoy { namespace Matcher { @@ -144,7 +146,16 @@ template class SingleFieldMatcher : public FieldMatcher, Logger::Loggable { public: SingleFieldMatcher(DataInputPtr&& data_input, InputMatcherPtr&& input_matcher) - : data_input_(std::move(data_input)), input_matcher_(std::move(input_matcher)) {} + : data_input_(std::move(data_input)), input_matcher_(std::move(input_matcher)) { + auto supported_input_types = input_matcher_->supportedDataInputTypes(); + if (supported_input_types.find(data_input_->dataInputType()) == supported_input_types.end()) { + std::string supported_types = + absl::StrJoin(supported_input_types.begin(), supported_input_types.end(), ", "); + throw EnvoyException( + absl::StrCat("Unsupported data input type: ", data_input_->dataInputType(), + ". The matcher supports input type: ", supported_types)); + } + } FieldMatchResult match(const DataType& data) override { const auto input = data_input_->get(data); diff --git a/source/common/matcher/map_matcher.h b/source/common/matcher/map_matcher.h index db71454c5645..4cb50e542f99 100644 --- a/source/common/matcher/map_matcher.h +++ b/source/common/matcher/map_matcher.h @@ -17,7 +17,14 @@ template class MapMatcher : public MatchTree, Logger::Loggable { public: MapMatcher(DataInputPtr&& data_input, absl::optional> on_no_match) - : data_input_(std::move(data_input)), on_no_match_(std::move(on_no_match)) {} + : data_input_(std::move(data_input)), on_no_match_(std::move(on_no_match)) { + auto input_type = data_input_->dataInputType(); + if (input_type != DefaultMatchingDataType) { + throw EnvoyException( + absl::StrCat("Unsupported data input type: ", input_type, + ", currently only string type is supported in map matcher")); + } + } // Adds a child to the map. virtual void addChild(std::string value, OnMatch&& on_match) PURE; diff --git a/source/common/matcher/matcher.h b/source/common/matcher/matcher.h index ff83ecfbdb2d..173ed906714b 100644 --- a/source/common/matcher/matcher.h +++ b/source/common/matcher/matcher.h @@ -201,7 +201,6 @@ class MatchTreeFactory : public OnMatchFactory { } auto on_no_match = createOnMatch(config.on_no_match()); - return [matcher_factories, on_no_match]() { auto list_matcher = std::make_unique>( on_no_match ? absl::make_optional((*on_no_match)()) : absl::nullopt); @@ -318,7 +317,7 @@ class MatchTreeFactory : public OnMatchFactory { template absl::optional> createOnMatchBase(const OnMatchType& on_match) { if (on_match.has_matcher()) { - return [matcher_factory = create(on_match.matcher())]() { + return [matcher_factory = std::move(create(on_match.matcher()))]() { return OnMatch{{}, matcher_factory()}; }; } else if (on_match.has_action()) { diff --git a/source/extensions/common/matcher/trie_matcher.h b/source/extensions/common/matcher/trie_matcher.h index fd5ba0226190..b50ae214eac2 100644 --- a/source/extensions/common/matcher/trie_matcher.h +++ b/source/extensions/common/matcher/trie_matcher.h @@ -62,7 +62,14 @@ template class TrieMatcher : public MatchTree { public: TrieMatcher(DataInputPtr&& data_input, absl::optional> on_no_match, const std::shared_ptr>>& trie) - : data_input_(std::move(data_input)), on_no_match_(std::move(on_no_match)), trie_(trie) {} + : data_input_(std::move(data_input)), on_no_match_(std::move(on_no_match)), trie_(trie) { + auto input_type = data_input_->dataInputType(); + if (input_type != Envoy::Matcher::DefaultMatchingDataType) { + throw EnvoyException( + absl::StrCat("Unsupported data input type: ", input_type, + ", currently only string type is supported in trie matcher")); + } + } typename MatchTree::MatchResult match(const DataType& data) override { const auto input = data_input_->get(data); diff --git a/test/common/matcher/matcher_test.cc b/test/common/matcher/matcher_test.cc index 876b79c46646..f3edcd539eb8 100644 --- a/test/common/matcher/matcher_test.cc +++ b/test/common/matcher/matcher_test.cc @@ -134,6 +134,180 @@ TEST_F(MatcherTest, TestPrefixMatcher) { EXPECT_NE(result.on_match_->action_cb_, nullptr); } +TEST_F(MatcherTest, TestInvalidFloatPrefixMapMatcher) { + const std::string yaml = R"EOF( +matcher_tree: + input: + name: outer_input + typed_config: + "@type": type.googleapis.com/google.protobuf.FloatValue + prefix_match_map: + map: + 3.14: + matcher: + matcher_list: + matchers: + - on_match: + action: + name: test_action + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + value: match!! + predicate: + single_predicate: + input: + name: inner_input + typed_config: + "@type": type.googleapis.com/google.protobuf.BoolValue + value_match: + exact: foo + )EOF"; + + envoy::config::common::matcher::v3::Matcher matcher; + MessageUtil::loadFromYaml(yaml, matcher, ProtobufMessage::getStrictValidationVisitor()); + + TestUtility::validate(matcher); + auto outer_input_factory = TestDataInputFloatFactory(3.14); + auto inner_input_factory = TestDataInputBoolFactory("foo"); + + EXPECT_CALL(validation_visitor_, + performDataInputValidation(_, "type.googleapis.com/google.protobuf.BoolValue")); + EXPECT_CALL(validation_visitor_, + performDataInputValidation(_, "type.googleapis.com/google.protobuf.FloatValue")); + + auto match_tree = factory_.create(matcher); + std::string error_message = absl::StrCat( + "Unsupported data input type: float, currently only string type is supported in map matcher"); + + EXPECT_THROW_WITH_MESSAGE(match_tree(), EnvoyException, error_message); +} + +TEST_F(MatcherTest, TestInvalidFloatExactMapMatcher) { + const std::string yaml = R"EOF( +matcher_tree: + input: + name: outer_input + typed_config: + "@type": type.googleapis.com/google.protobuf.FloatValue + exact_match_map: + map: + 3.14: + matcher: + matcher_list: + matchers: + - on_match: + action: + name: test_action + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + value: match!! + predicate: + single_predicate: + input: + name: inner_input + typed_config: + "@type": type.googleapis.com/google.protobuf.BoolValue + value_match: + exact: foo + )EOF"; + + envoy::config::common::matcher::v3::Matcher matcher; + MessageUtil::loadFromYaml(yaml, matcher, ProtobufMessage::getStrictValidationVisitor()); + + TestUtility::validate(matcher); + auto outer_input_factory = TestDataInputFloatFactory(3.14); + auto inner_input_factory = TestDataInputBoolFactory("foo"); + + EXPECT_CALL(validation_visitor_, + performDataInputValidation(_, "type.googleapis.com/google.protobuf.BoolValue")); + EXPECT_CALL(validation_visitor_, + performDataInputValidation(_, "type.googleapis.com/google.protobuf.FloatValue")); + auto match_tree = factory_.create(matcher); + std::string error_message = absl::StrCat( + "Unsupported data input type: float, currently only string type is supported in map matcher"); + EXPECT_THROW_WITH_MESSAGE(match_tree(), EnvoyException, error_message); +} + +TEST_F(MatcherTest, InvalidDataInput) { + const std::string yaml = R"EOF( +matcher_list: + matchers: + - on_match: + action: + name: test_action + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + value: match!! + predicate: + single_predicate: + input: + name: generic + typed_config: + "@type": type.googleapis.com/google.protobuf.FloatValue + value_match: + exact: 3.14 + + )EOF"; + envoy::config::common::matcher::v3::Matcher matcher; + MessageUtil::loadFromYaml(yaml, matcher, ProtobufMessage::getStrictValidationVisitor()); + + TestUtility::validate(matcher); + + auto outer_input_factory = TestDataInputFloatFactory(3.14); + + EXPECT_CALL(validation_visitor_, + performDataInputValidation(_, "type.googleapis.com/google.protobuf.FloatValue")); + auto match_tree = factory_.create(matcher); + std::string error_message = absl::StrCat("Unsupported data input type: float.", + " The matcher supports input type: string"); + EXPECT_THROW_WITH_MESSAGE(match_tree(), EnvoyException, error_message); +} + +TEST_F(MatcherTest, InvalidDataInputInAndMatcher) { + const std::string yaml = R"EOF( + matcher_list: + matchers: + - on_match: + action: + name: test_action + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + value: match!! + predicate: + and_matcher: + predicate: + - single_predicate: + input: + name: inner_input + typed_config: + "@type": type.googleapis.com/google.protobuf.FloatValue + value_match: + exact: 3.14 + - single_predicate: + input: + name: inner_input + typed_config: + "@type": type.googleapis.com/google.protobuf.FloatValue + value_match: + exact: 3.14 + + )EOF"; + envoy::config::common::matcher::v3::Matcher matcher; + MessageUtil::loadFromYaml(yaml, matcher, ProtobufMessage::getStrictValidationVisitor()); + + TestUtility::validate(matcher); + + auto outer_input_factory = TestDataInputFloatFactory(3.14); + + EXPECT_CALL(validation_visitor_, + performDataInputValidation(_, "type.googleapis.com/google.protobuf.FloatValue")) + .Times(2); + + std::string error_message = absl::StrCat("Unsupported data input type: float.", + " The matcher supports input type: string"); + EXPECT_THROW_WITH_MESSAGE(factory_.create(matcher)(), EnvoyException, error_message); +} + TEST_F(MatcherTest, TestAnyMatcher) { const std::string yaml = R"EOF( on_no_match: diff --git a/test/common/matcher/test_utility.h b/test/common/matcher/test_utility.h index 0301d8ab58d4..36acc793d7a2 100644 --- a/test/common/matcher/test_utility.h +++ b/test/common/matcher/test_utility.h @@ -49,7 +49,13 @@ class TestCommonProtocolInputFactory : public CommonProtocolInputFactory { struct TestInput : public DataInput { explicit TestInput(DataInputGetResult result) : result_(result) {} DataInputGetResult get(const TestData&) const override { return result_; } + DataInputGetResult result_; +}; +struct TestFloatInput : public DataInput { + explicit TestFloatInput(DataInputGetResult result) : result_(result) {} + DataInputGetResult get(const TestData&) const override { return result_; } + absl::string_view dataInputType() const override { return "float"; } DataInputGetResult result_; }; @@ -84,6 +90,7 @@ class TestDataInputBoolFactory : public DataInputFactory { {DataInputGetResult::DataAvailability::AllDataAvailable, std::string(data)}) {} DataInputFactoryCb createDataInputFactoryCb(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override { + // Note, here is using `TestInput` same as `TestDataInputStringFactory`. return [&]() { return std::make_unique(result_); }; } @@ -97,12 +104,34 @@ class TestDataInputBoolFactory : public DataInputFactory { Registry::InjectFactory> injection_; }; +class TestDataInputFloatFactory : public DataInputFactory { +public: + TestDataInputFloatFactory(DataInputGetResult result) : result_(result), injection_(*this) {} + TestDataInputFloatFactory(float) + : TestDataInputFloatFactory( + {DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}) {} + DataInputFactoryCb + createDataInputFactoryCb(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override { + return [&]() { return std::make_unique(result_); }; + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + std::string name() const override { return "float"; } + +private: + const DataInputGetResult result_; + Registry::InjectFactory> injection_; +}; + // A matcher that evaluates to the configured value. +// Note, `BoolMatcher` supports string type data input only as `TestDataInputBoolFactory` is using +// `TestInput` same as `TestDataInputStringFactory`. struct BoolMatcher : public InputMatcher { explicit BoolMatcher(bool value) : value_(value) {} bool match(const MatchingDataType&) override { return value_; } - const bool value_; }; diff --git a/test/extensions/common/matcher/trie_matcher_test.cc b/test/extensions/common/matcher/trie_matcher_test.cc index 56b8b3c3a185..7a6be2636bda 100644 --- a/test/extensions/common/matcher/trie_matcher_test.cc +++ b/test/extensions/common/matcher/trie_matcher_test.cc @@ -135,6 +135,45 @@ TEST_F(TrieMatcherTest, TestMatcher) { } } +TEST_F(TrieMatcherTest, TestInvalidMatcher) { + const std::string yaml = R"EOF( +matcher_tree: + input: + name: input + typed_config: + "@type": type.googleapis.com/google.protobuf.FloatValue + custom_match: + name: ip_matcher + typed_config: + "@type": type.googleapis.com/xds.type.matcher.v3.IPMatcher + range_matchers: + - ranges: + - address_prefix: 192.0.0.0 + prefix_len: 2 + on_match: + action: + name: test_action + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + value: foo + - ranges: + - address_prefix: 192.101.0.0 + prefix_len: 10 + on_match: + action: + name: test_action + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + value: bar + )EOF"; + loadConfig(yaml); + auto input_factory = ::Envoy::Matcher::TestDataInputFloatFactory(3.14); + auto match_tree = factory_.create(matcher_); + std::string error_message = absl::StrCat("Unsupported data input type: float, currently only " + "string type is supported in trie matcher"); + EXPECT_THROW_WITH_MESSAGE(match_tree(), EnvoyException, error_message); +} + TEST_F(TrieMatcherTest, TestMatcherOnNoMatch) { const std::string yaml = R"EOF( matcher_tree: diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index f79ef30ac258..d983a4ac0a4a 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -87,6 +87,9 @@ paths: exception: include: - source/common/config/utility.h + - source/common/matcher/map_matcher.h + - source/common/matcher/field_matcher.h + - source/extensions/common/matcher/trie_matcher.h # These files should not throw exceptions. Add HTTP/1 when exceptions removed. exclude: - source/common/http/http2/codec_impl.h From 9a1ccb35f729a9867ec07d1fb58df28a74df360d Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Tue, 9 May 2023 13:50:37 -0400 Subject: [PATCH 173/740] Deflake ManyRequestHeadersTimeout under TSAN (#27259) Signed-off-by: Yan Avlasov --- test/integration/protocol_integration_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 96ca214e296d..8cb09ccc69de 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -2332,7 +2332,7 @@ TEST_P(DownstreamProtocolIntegrationTest, ManyRequestTrailersAccepted) { // time-consuming byte size validations that will cause this test to timeout. TEST_P(DownstreamProtocolIntegrationTest, ManyRequestHeadersTimeout) { // Set timeout for 5 seconds, and ensure that a request with 10k+ headers can be sent. - testManyRequestHeaders(std::chrono::milliseconds(5000)); + testManyRequestHeaders(TestUtility::DefaultTimeout * 5); } TEST_P(DownstreamProtocolIntegrationTest, LargeRequestTrailersAccepted) { From 36866c82b3800aefe68807ddafc5fb2b7beaed0d Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Tue, 9 May 2023 14:17:39 -0400 Subject: [PATCH 174/740] Fix msan error in mobile cds_integration_test (#27287) Signed-off-by: Yan Avlasov --- mobile/test/common/integration/cds_integration_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/test/common/integration/cds_integration_test.cc b/mobile/test/common/integration/cds_integration_test.cc index a0a9b4fd3708..f1a8c794e6d5 100644 --- a/mobile/test/common/integration/cds_integration_test.cc +++ b/mobile/test/common/integration/cds_integration_test.cc @@ -58,7 +58,7 @@ class CdsIntegrationTest : public XdsIntegrationTest { ASSERT_TRUE(waitForGaugeGe("cluster_manager.active_clusters", cluster_count + 1)); } - bool use_xdstp_; + bool use_xdstp_{false}; std::string cds_namespace_; }; From c828800ed2ddef6a22d46256c22b316ec976601b Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 9 May 2023 19:22:25 +0100 Subject: [PATCH 175/740] deps: Bump `com_github_zlib_ng_zlib_ng` -> 2.0.7 (#27286) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 926e922d3f10..d49f97ee543e 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -692,12 +692,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "zlib-ng", project_desc = "zlib fork (higher performance)", project_url = "https://github.com/zlib-ng/zlib-ng", - version = "2.0.6", - sha256 = "8258b75a72303b661a238047cb348203d88d9dddf85d480ed885f375916fcab6", + version = "2.0.7", + sha256 = "6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200", strip_prefix = "zlib-ng-{version}", urls = ["https://github.com/zlib-ng/zlib-ng/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2021-12-24", + release_date = "2023-03-17", cpe = "N/A", license = "zlib", license_url = "https://github.com/zlib-ng/zlib-ng/blob/{version}/LICENSE.md", From e701e7a0e011076666eca810ba485be1fbc3de3c Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 9 May 2023 20:19:49 +0100 Subject: [PATCH 176/740] deps: Bump `com_github_nghttp2_nghttp2` -> 1.52.0 (#27285) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index d49f97ee543e..464051247d0c 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -444,12 +444,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Nghttp2", project_desc = "Implementation of HTTP/2 and its header compression algorithm HPACK in C", project_url = "https://nghttp2.org", - version = "1.51.0", - sha256 = "2a0bef286f65b35c24250432e7ec042441a8157a5b93519412d9055169d9ce54", + version = "1.52.0", + sha256 = "9877caa62bd72dde1331da38ce039dadb049817a01c3bdee809da15b754771b8", strip_prefix = "nghttp2-{version}", urls = ["https://github.com/nghttp2/nghttp2/releases/download/v{version}/nghttp2-{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2022-11-13", + release_date = "2023-02-13", cpe = "cpe:2.3:a:nghttp2:nghttp2:*", license = "MIT", license_url = "https://github.com/nghttp2/nghttp2/blob/v{version}/LICENSE", From 07afb7c4af2fc2290a10a9a251476df3cd6729fd Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 9 May 2023 16:30:53 -0400 Subject: [PATCH 177/740] mobile: adding a test of large response body (#27290) Signed-off-by: Alyssa Wilk --- .../integration/client_integration_test.cc | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index f71731f39fce..b72dfe477a13 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -43,7 +43,6 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ClientIntegrationTest, TestUtility::ipTestParamsToString); void ClientIntegrationTest::basicTest() { - initialize(); Buffer::OwnedImpl request_data = Buffer::OwnedImpl("request body"); default_request_headers_.addCopy(AutonomousStream::EXPECT_REQUEST_SIZE_BYTES, @@ -52,8 +51,6 @@ void ClientIntegrationTest::basicTest() { stream_prototype_->setOnData([this](envoy_data c_data, bool end_stream) { if (end_stream) { EXPECT_EQ(Data::Utility::copyToString(c_data), ""); - } else { - EXPECT_EQ(c_data.length, 10); } cc_.on_data_calls++; release_envoy_data(c_data); @@ -76,10 +73,21 @@ void ClientIntegrationTest::basicTest() { ASSERT_EQ(cc_.on_data_calls, 2); ASSERT_EQ(cc_.on_complete_calls, 1); ASSERT_EQ(cc_.on_header_consumed_bytes_from_response, 27); +} + +TEST_P(ClientIntegrationTest, Basic) { + initialize(); + basicTest(); ASSERT_EQ(cc_.on_complete_received_byte_count, 67); } -TEST_P(ClientIntegrationTest, Basic) { basicTest(); } +TEST_P(ClientIntegrationTest, LargeResponse) { + initialize(); + std::string data(1024 * 32, 'a'); + reinterpret_cast(fake_upstreams_.front().get())->setResponseBody(data); + basicTest(); + ASSERT_EQ(cc_.on_complete_received_byte_count, 32828); +} TEST_P(ClientIntegrationTest, ClearTextNotPermitted) { EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).WillRepeatedly(Return(false)); From 45c442553af62c6093926c7e8e27a7b6e20f4b91 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 9 May 2023 16:04:19 -0700 Subject: [PATCH 178/740] bazel: remove pch dep when disabled (#26872) This should break some unnecessary rebuilding in the case you don't actually use the pch feature. Signed-off-by: Keith Smiley --- bazel/envoy_library.bzl | 5 ++--- bazel/envoy_pch.bzl | 6 ++++++ bazel/envoy_test.bzl | 9 +++------ source/common/common/BUILD | 3 +++ test/tools/schema_validator/BUILD | 1 + test/tools/schema_validator/validator.cc | 1 + 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index f26690c1292f..152302ed6432 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -6,7 +6,7 @@ load( "envoy_external_dep_path", "envoy_linkstatic", ) -load(":envoy_pch.bzl", "envoy_pch_copts") +load(":envoy_pch.bzl", "envoy_pch_copts", "envoy_pch_deps") load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library") load( "@envoy_build_config//:extensions_build_config.bzl", @@ -131,12 +131,11 @@ def envoy_cc_library( deps = deps + [envoy_external_dep_path(dep) for dep in external_deps] + [ repository + "//envoy/common:base_includes", repository + "//source/common/common:fmt_lib", - repository + "//source/common/common:common_pch", envoy_external_dep_path("abseil_flat_hash_map"), envoy_external_dep_path("abseil_flat_hash_set"), envoy_external_dep_path("abseil_strings"), envoy_external_dep_path("fmtlib"), - ], + ] + envoy_pch_deps(repository, "//source/common/common:common_pch"), alwayslink = alwayslink, linkstatic = envoy_linkstatic(), strip_include_prefix = strip_include_prefix, diff --git a/bazel/envoy_pch.bzl b/bazel/envoy_pch.bzl index bb8d4d29c7c5..843937d8dcac 100644 --- a/bazel/envoy_pch.bzl +++ b/bazel/envoy_pch.bzl @@ -8,6 +8,12 @@ load( ) load(":pch.bzl", "pch") +def envoy_pch_deps(repository, target): + return select({ + repository + "//bazel:clang_pch_build": [repository + target], + "//conditions:default": [], + }) + def envoy_pch_copts(repository, target): return select({ repository + "//bazel:clang_pch_build": [ diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index 05f0b3f30abe..ad05121a9922 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -4,7 +4,7 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_test") load("@rules_fuzzing//fuzzing:cc_defs.bzl", "fuzzing_decoration") load(":envoy_binary.bzl", "envoy_cc_binary") load(":envoy_library.bzl", "tcmalloc_external_deps") -load(":envoy_pch.bzl", "envoy_pch_copts") +load(":envoy_pch.bzl", "envoy_pch_copts", "envoy_pch_deps") load( ":envoy_internal.bzl", "envoy_copts", @@ -41,7 +41,7 @@ def _envoy_cc_test_infrastructure_library( if disable_pch: extra_deps = [envoy_external_dep_path("googletest")] else: - extra_deps = [repository + "//test:test_pch"] + extra_deps = envoy_pch_deps(repository, "//test:test_pch") pch_copts = envoy_pch_copts(repository, "//test:test_pch") native.cc_library( @@ -176,10 +176,7 @@ def envoy_cc_test( deps = envoy_stdlib_deps() + deps + [envoy_external_dep_path(dep) for dep in external_deps + ["googletest"]] + [ repository + "//test:main", repository + "//test/test_common:test_version_linkstamp", - ] + select({ - repository + "//bazel:clang_pch_build": [repository + "//test:test_pch"], - "//conditions:default": [], - }), + ] + envoy_pch_deps(repository, "//test:test_pch"), # from https://github.com/google/googletest/blob/6e1970e2376c14bf658eb88f655a054030353f9f/googlemock/src/gmock.cc#L51 # 2 - by default, mocks act as StrictMocks. args = args + ["--gmock_default_mock_behavior=2"], diff --git a/source/common/common/BUILD b/source/common/common/BUILD index 6f0f7a885ee8..2f45fc1c671c 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -221,6 +221,9 @@ envoy_cc_library( name = "base_logger_lib", srcs = ["base_logger.cc"], hdrs = ["base_logger.h"], + external_deps = [ + "spdlog", + ], ) envoy_cc_library( diff --git a/test/tools/schema_validator/BUILD b/test/tools/schema_validator/BUILD index 6e911ac1af88..50dfb2e8fb35 100644 --- a/test/tools/schema_validator/BUILD +++ b/test/tools/schema_validator/BUILD @@ -36,6 +36,7 @@ envoy_cc_test_library( "//test/test_common:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) diff --git a/test/tools/schema_validator/validator.cc b/test/tools/schema_validator/validator.cc index 90aee88c24be..60b5f5637bb9 100644 --- a/test/tools/schema_validator/validator.cc +++ b/test/tools/schema_validator/validator.cc @@ -4,6 +4,7 @@ #include "envoy/config/bootstrap/v3/bootstrap.pb.validate.h" #include "envoy/config/route/v3/route.pb.h" #include "envoy/config/route/v3/route.pb.validate.h" +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" #include "envoy/service/discovery/v3/discovery.pb.h" #include "envoy/service/discovery/v3/discovery.pb.validate.h" From 5e834d0d8d18918beddd4a8f1267f9fda8b49807 Mon Sep 17 00:00:00 2001 From: Davin Kevin Date: Wed, 10 May 2023 04:04:32 +0200 Subject: [PATCH 179/740] doc: `google_re2` for type type.matcher.v3.RegexMatcher is not required any more (#26876) Related to https://github.com/envoyproxy/envoy/issues/26875 Signed-off-by: Davin Kevin --- api/envoy/type/matcher/v3/regex.proto | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/envoy/type/matcher/v3/regex.proto b/api/envoy/type/matcher/v3/regex.proto index 69cb8ba13f20..10b3970ff840 100644 --- a/api/envoy/type/matcher/v3/regex.proto +++ b/api/envoy/type/matcher/v3/regex.proto @@ -57,11 +57,8 @@ message RegexMatcher { oneof engine_type { // Google's RE2 regex engine. - GoogleRE2 google_re2 = 1 [ - deprecated = true, - (validate.rules).message = {required: true}, - (envoy.annotations.deprecated_at_minor_version) = "3.0" - ]; + GoogleRE2 google_re2 = 1 + [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; } // The regex match string. The string must be supported by the configured engine. The regex is matched From 9abb5f159a1b96c5569664f75bafadfdab3d855e Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 10 May 2023 06:47:50 +0100 Subject: [PATCH 180/740] deps: Bump `bazel_gazelle` -> 0.30.0 (#27282) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 464051247d0c..d48acbba5f6d 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -33,10 +33,10 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Gazelle", project_desc = "Bazel BUILD file generator for Go projects", project_url = "https://github.com/bazelbuild/bazel-gazelle", - version = "0.26.0", - sha256 = "501deb3d5695ab658e82f6f6f549ba681ea3ca2a5fb7911154b5aa45596183fa", + version = "0.30.0", + sha256 = "727f3e4edd96ea20c29e8c2ca9e8d2af724d8c7778e7923a854b2c80952bc405", urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v{version}/bazel-gazelle-v{version}.tar.gz"], - release_date = "2022-06-26", + release_date = "2023-03-30", use_category = ["build"], license = "Apache-2.0", license_url = "https://github.com/bazelbuild/bazel-gazelle/blob/v{version}/LICENSE", From 95953c6753325e038c34341778ec6674b4c57624 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Tue, 9 May 2023 23:11:59 -0700 Subject: [PATCH 181/740] contrib: avoid building unnecessary vpp targets (#27264) Signed-off-by: Florin Coras --- bazel/foreign_cc/vpp_vcl.patch | 137 ++++----------------------------- contrib/vcl/source/BUILD | 5 +- 2 files changed, 15 insertions(+), 127 deletions(-) diff --git a/bazel/foreign_cc/vpp_vcl.patch b/bazel/foreign_cc/vpp_vcl.patch index 769fc2c877d0..874fd05c41de 100644 --- a/bazel/foreign_cc/vpp_vcl.patch +++ b/bazel/foreign_cc/vpp_vcl.patch @@ -70,128 +70,17 @@ index 8415c28fb..a017d9fc6 100755 + import ply.lex as lex import ply.yacc as yacc - -diff --git src/vat/CMakeLists.txt src/vat/CMakeLists.txt -index e5945b20d..3d76f3d88 100644 ---- src/vat/CMakeLists.txt -+++ src/vat/CMakeLists.txt -@@ -19,39 +19,6 @@ add_vpp_library(vatplugin - LINK_LIBRARIES vppinfra - ) - --############################################################################## --# vpp_api_test --############################################################################## --add_vpp_executable(vpp_api_test ENABLE_EXPORTS -- SOURCES -- api_format.c -- main.c -- plugin.c -- json_format.c -- types.c -- ip_types_api.c -- ip_types.c -- protocols.def -- -- DEPENDS api_headers -- -- LINK_LIBRARIES -- vlibmemoryclient -- svm -- vatplugin -- vppinfra -- Threads::Threads -- dl --) -- --############################################################################## --# vpp_json_test --############################################################################## --add_vpp_executable(vpp_json_test ENABLE_EXPORTS NO_INSTALL -- SOURCES json_format.c json_test.c -- LINK_LIBRARIES vppinfra m --) -- - ############################################################################## - # vat headers - ############################################################################## -diff --git src/vat2/CMakeLists.txt src/vat2/CMakeLists.txt -index 108e184b5..a90107617 100644 ---- src/vat2/CMakeLists.txt -+++ src/vat2/CMakeLists.txt -@@ -14,21 +14,6 @@ - ############################################################################## - # vat2 - ############################################################################## --add_vpp_executable(vat2 ENABLE_EXPORTS -- SOURCES -- main.c -- plugin.c -- -- DEPENDS api_headers -- -- LINK_LIBRARIES -- vlibmemoryclient -- svm -- vppinfra -- vppapiclient -- Threads::Threads -- dl --) - - # - # Unit test code. Call generator directly to avoid it being installed -@@ -37,20 +22,6 @@ add_vpp_executable(vat2 ENABLE_EXPORTS - #) - - vpp_generate_api_c_header (test/vat2_test.api) --add_vpp_executable(test_vat2 ENABLE_EXPORTS NO_INSTALL -- SOURCES -- test/vat2_test.c -- -- DEPENDS api_headers -- -- LINK_LIBRARIES -- vlibmemoryclient -- svm -- vppinfra -- vppapiclient -- Threads::Threads -- dl --) - #target_link_options(test_vat2 PUBLIC "LINKER:-fsanitize=address") - if(VPP_BUILD_TESTS_WITH_COVERAGE) - set(TARGET_NAME test_vat2) -diff --git src/vpp-api/CMakeLists.txt src/vpp-api/CMakeLists.txt -index 72cc1b29c..0f2510d51 100644 ---- src/vpp-api/CMakeLists.txt -+++ src/vpp-api/CMakeLists.txt -@@ -29,10 +29,5 @@ add_vpp_headers(vpp-api - client/stat_client.h + +diff --git src/vcl/CMakeLists.txt src/vcl/CMakeLists.txt +index e6d8f98ff..600ef4d5f 100644 +--- src/vcl/CMakeLists.txt ++++ src/vcl/CMakeLists.txt +@@ -30,6 +30,8 @@ add_vpp_library(vppcom + api_headers ) - --add_vpp_executable(test_vppapiclient NO_INSTALL -- SOURCES client/test.c -- LINK_LIBRARIES vppinfra pthread vppapiclient --) -- - add_subdirectory(vapi) - add_subdirectory(python) -diff --git src/vpp/CMakeLists.txt src/vpp/CMakeLists.txt -index 1b43b1299..361b981d9 100644 ---- src/vpp/CMakeLists.txt -+++ src/vpp/CMakeLists.txt -@@ -83,13 +83,6 @@ if(VPP_API_TEST_BUILTIN) - add_definitions(-DVPP_API_TEST_BUILTIN=1) - endif() - --add_vpp_executable(vpp -- ENABLE_EXPORTS -- SOURCES ${VPP_SOURCES} -- LINK_LIBRARIES svm vlib vppinfra vlibmemory vnet Threads::Threads ${CMAKE_DL_LIBS} -- DEPENDS vpp_version_h api_headers --) -- - add_vpp_headers(vpp - stats/stat_segment.h - stats/stat_segment_shared.h + ++file(COPY vppcom.h DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) ++ + add_vpp_library(vcl_ldpreload + SOURCES + ldp_socket_wrapper.c diff --git a/contrib/vcl/source/BUILD b/contrib/vcl/source/BUILD index 8ed5fb25ea14..1233b3739b04 100644 --- a/contrib/vcl/source/BUILD +++ b/contrib/vcl/source/BUILD @@ -1,3 +1,4 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load( "//bazel:envoy_build_system.bzl", "envoy_cc_contrib_extension", @@ -36,7 +37,6 @@ envoy_cmake( "CMAKE_BUILD_TYPE": "Release", "VPP_API_TEST_BUILTIN": "OFF", "BUILD_SHARED_LIBS": "OFF", - "CMAKE_ENABLE_TESTING": "OFF", "CMAKE_ENABLE_EXPORTS": "OFF", }, copts = ["-Wno-unused-variable"], @@ -62,8 +62,7 @@ envoy_cmake( "skip_on_windows", ], targets = [ - "", - "install", + "vppcom", ], working_directory = "src", ) From 94af666e8528ae67ab2814cf30ddd3f6c66ac067 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 09:05:38 +0100 Subject: [PATCH 182/740] build(deps): bump pygithub from 1.58.1 to 1.58.2 in /.github/actions/pr_notifier (#27305) build(deps): bump pygithub in /.github/actions/pr_notifier Bumps [pygithub](https://github.com/pygithub/pygithub) from 1.58.1 to 1.58.2. - [Release notes](https://github.com/pygithub/pygithub/releases) - [Changelog](https://github.com/PyGithub/PyGithub/blob/v1.58.2/doc/changes.rst) - [Commits](https://github.com/pygithub/pygithub/compare/v1.58.1...v1.58.2) --- updated-dependencies: - dependency-name: pygithub dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/pr_notifier/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/pr_notifier/requirements.txt b/.github/actions/pr_notifier/requirements.txt index 2d3efaf76d9b..1f2fb83fc37f 100644 --- a/.github/actions/pr_notifier/requirements.txt +++ b/.github/actions/pr_notifier/requirements.txt @@ -102,9 +102,9 @@ pycparser==2.20 \ --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 # via cffi -pygithub==1.58.1 \ - --hash=sha256:4e7fe9c3ec30d5fde5b4fbb97f18821c9dbf372bf6df337fe66f6689a65e0a83 \ - --hash=sha256:7d528b4ad92bc13122129fafd444ce3d04c47d2d801f6446b6e6ee2d410235b3 +pygithub==1.58.2 \ + --hash=sha256:1e6b1b7afe31f75151fb81f7ab6b984a7188a852bdb123dbb9ae90023c3ce60f \ + --hash=sha256:f435884af617c6debaa76cbc355372d1027445a56fbc39972a3b9ed4968badc8 # via -r requirements.in pyjwt[crypto]==2.4.0 \ --hash=sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf \ From c9096862e054fa2f0fe5e7dee5475ec2d8be4728 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 09:07:13 +0100 Subject: [PATCH 183/740] build(deps): bump otel/opentelemetry-collector from `da930e5` to `e5bd89e` in /examples/opentelemetry (#27309) build(deps): bump otel/opentelemetry-collector Bumps otel/opentelemetry-collector from `da930e5` to `e5bd89e`. --- updated-dependencies: - dependency-name: otel/opentelemetry-collector dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/opentelemetry/Dockerfile-opentelemetry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry index 1512b770cd94..71fabfa2aab9 100644 --- a/examples/opentelemetry/Dockerfile-opentelemetry +++ b/examples/opentelemetry/Dockerfile-opentelemetry @@ -1,7 +1,7 @@ FROM alpine:3.17@sha256:124c7d2707904eea7431fffe91522a01e5a861a624ee31d03372cc1d138a3126 as otelc_curl RUN apk --update add curl -FROM otel/opentelemetry-collector:latest@sha256:da930e519a80e34de471794435aa9c24ce8ef84b231fe8f4b9b6f0148797cd60 +FROM otel/opentelemetry-collector:latest@sha256:e5bd89e5ec7cd5bb3ff01fbd811e3730be75d8725405ae0c0d282cc69b9597e0 COPY --from=otelc_curl / / From cb78097b1371bd8cb7517af597db890615cbecf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 10:33:35 +0100 Subject: [PATCH 184/740] build(deps): bump alpine from 3.17 to 3.18 in /examples/opentelemetry (#27308) Bumps alpine from 3.17 to 3.18. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/opentelemetry/Dockerfile-opentelemetry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry index 71fabfa2aab9..4d05281c3000 100644 --- a/examples/opentelemetry/Dockerfile-opentelemetry +++ b/examples/opentelemetry/Dockerfile-opentelemetry @@ -1,4 +1,4 @@ -FROM alpine:3.17@sha256:124c7d2707904eea7431fffe91522a01e5a861a624ee31d03372cc1d138a3126 as otelc_curl +FROM alpine:3.18@sha256:02bb6f428431fbc2809c5d1b41eab5a68350194fb508869a33cb1af4444c9b11 as otelc_curl RUN apk --update add curl FROM otel/opentelemetry-collector:latest@sha256:e5bd89e5ec7cd5bb3ff01fbd811e3730be75d8725405ae0c0d282cc69b9597e0 From 0642d624d2b228ca00f2c2be7f816f920ec4ab8c Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 10 May 2023 12:13:20 +0100 Subject: [PATCH 185/740] github/ci: Dont trigger command workflow for bots (#27272) Signed-off-by: Ryan Northey --- .github/workflows/commands.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml index 82cf93dcfa87..6d50ad40a10e 100644 --- a/.github/workflows/commands.yml +++ b/.github/workflows/commands.yml @@ -5,7 +5,13 @@ on: jobs: retest: - if: github.repository == 'envoyproxy/envoy' + if: | + ${{ + github.event.issue.pull_request + && github.repository == 'envoyproxy/envoy' + && github.actor != 'repokitteh-read-only[bot]' + && github.actor != 'dependabot[bot]' + }} name: Retest runs-on: ubuntu-latest steps: From bedd2b6cc19958eb5c57715722ccaf55b1fc7f87 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 10 May 2023 13:38:03 +0100 Subject: [PATCH 186/740] do/ci: Improve readability (#27268) Signed-off-by: Ryan Northey Signed-off-by: code Co-authored-by: code --- ci/do_ci.sh | 204 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 164 insertions(+), 40 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 27410f17d34f..977bd169dc59 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -32,7 +32,10 @@ fi function collect_build_profile() { declare -g build_profile_count=${build_profile_count:-1} - mv -f "$(bazel info "${BAZEL_BUILD_OPTIONS[@]}" output_base)/command.profile.gz" "${ENVOY_BUILD_PROFILE}/${build_profile_count}-$1.profile.gz" || true + mv -f \ + "$(bazel info "${BAZEL_BUILD_OPTIONS[@]}" output_base)/command.profile.gz" \ + "${ENVOY_BUILD_PROFILE}/${build_profile_count}-$1.profile.gz" \ + || : ((build_profile_count++)) } @@ -218,7 +221,9 @@ case $CI_TARGET in # toolchain is kept consistent. This ifdef is checked in # test/common/stats/stat_test_utility.cc when computing # Stats::TestUtil::MemoryTest::mode(). - [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]] && BAZEL_BUILD_OPTIONS+=("--test_env=ENVOY_MEMORY_TEST_EXACT=true") + if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + BAZEL_BUILD_OPTIONS+=("--test_env=ENVOY_MEMORY_TEST_EXACT=true") + fi setup_clang_toolchain @@ -238,20 +243,31 @@ case $CI_TARGET in echo " build options: ${BAZEL_BUILD_OPTIONS[*]}" echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_minimal "${BAZEL_RELEASE_OPTIONS[@]}" "${TEST_TARGETS[@]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --remote_download_minimal \ + "${BAZEL_RELEASE_OPTIONS[@]}" \ + "${TEST_TARGETS[@]}" # Build release binaries - bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" //distribution/binary:release + bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" \ + //distribution/binary:release # Copy release binaries to binary export directory - cp -a "bazel-bin/distribution/binary/release.tar.zst" "${ENVOY_BINARY_DIR}/release.tar.zst" + cp -a \ + "bazel-bin/distribution/binary/release.tar.zst" \ + "${ENVOY_BINARY_DIR}/release.tar.zst" # Grab the schema_validator_tool # TODO(phlax): bundle this with the release when #26390 is resolved - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel "${BAZEL_RELEASE_OPTIONS[@]}" //test/tools/schema_validator:schema_validator_tool.stripped + bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" \ + --remote_download_toplevel \ + //test/tools/schema_validator:schema_validator_tool.stripped # Copy schema_validator_tool to binary export directory - cp -a bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped "${ENVOY_BINARY_DIR}/schema_validator_tool" + cp -a \ + bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped \ + "${ENVOY_BINARY_DIR}/schema_validator_tool" ;; distribution) @@ -266,10 +282,19 @@ case $CI_TARGET in else ENVOY_RELEASE_TARBALL="/build/bazel.release/arm64/bin/release.tar.zst" fi - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/zstd -- --stdout -d "$ENVOY_RELEASE_TARBALL" | tar xfO - envoy > distribution/custom/envoy + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/zstd \ + -- --stdout \ + -d "$ENVOY_RELEASE_TARBALL" \ + | tar xfO - envoy > distribution/custom/envoy # Build the packages - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c opt --//distribution:envoy-binary=//distribution:custom/envoy //distribution:packages.tar.gz + bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ + --remote_download_toplevel \ + -c opt \ + --//distribution:envoy-binary=//distribution:custom/envoy \ + //distribution:packages.tar.gz + if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.x64.tar.gz" else @@ -289,7 +314,10 @@ case $CI_TARGET in sizeopt) setup_clang_toolchain echo "Testing ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --config=sizeopt "${TEST_TARGETS[@]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --config=sizeopt \ + "${TEST_TARGETS[@]}" echo "bazel size optimized build with tests..." bazel_envoy_binary_build sizeopt @@ -299,7 +327,11 @@ case $CI_TARGET in setup_gcc_toolchain echo "Testing ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild --remote_download_minimal -- "${TEST_TARGETS[@]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + -c fastbuild \ + --remote_download_minimal \ + -- "${TEST_TARGETS[@]}" echo "bazel release build with gcc..." bazel_envoy_binary_build fastbuild @@ -310,7 +342,10 @@ case $CI_TARGET in # Make sure that there are no regressions to building Envoy with autolink disabled. EXTRA_OPTIONS=( "--define" "library_autolink=disabled") - bazel test "${BAZEL_BUILD_OPTIONS[@]}" "${EXTRA_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" + bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ + "${EXTRA_OPTIONS[@]}" \ + -c dbg \ + "${TEST_TARGETS[@]}" echo "bazel debug build with tests..." bazel_envoy_binary_build debug @@ -322,14 +357,20 @@ case $CI_TARGET in ;; asan) setup_clang_toolchain - BAZEL_BUILD_OPTIONS+=(-c dbg "--config=clang-asan" "--build_tests_only" "--remote_download_minimal") + BAZEL_BUILD_OPTIONS+=( + -c dbg + "--config=clang-asan" + "--build_tests_only" + "--remote_download_minimal") echo "bazel ASAN/UBSAN debug build with tests" echo "Building and testing envoy tests ${TEST_TARGETS[*]}" bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" "${TEST_TARGETS[@]}" if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then echo "Building and testing envoy-filter-example tests..." pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" popd fi @@ -349,11 +390,21 @@ case $CI_TARGET in setup_clang_toolchain echo "bazel TSAN debug build with tests" echo "Building and testing envoy tests ${TEST_TARGETS[*]}" - bazel_with_collection test --config=rbe-toolchain-tsan "${BAZEL_BUILD_OPTIONS[@]}" -c dbg --build_tests_only --remote_download_minimal "${TEST_TARGETS[@]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --config=rbe-toolchain-tsan \ + -c dbg \ + --build_tests_only \ + --remote_download_minimal \ + "${TEST_TARGETS[@]}" if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then echo "Building and testing envoy-filter-example tests..." pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c dbg --config=clang-tsan "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + -c dbg \ + --config=clang-tsan \ + "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" popd fi ;; @@ -361,10 +412,17 @@ case $CI_TARGET in ENVOY_STDLIB=libc++ setup_clang_toolchain # rbe-toolchain-msan must comes as first to win library link order. - BAZEL_BUILD_OPTIONS=("--config=rbe-toolchain-msan" "${BAZEL_BUILD_OPTIONS[@]}" "-c" "dbg" "--build_tests_only" "--remote_download_minimal") + BAZEL_BUILD_OPTIONS=( + "--config=rbe-toolchain-msan" + "${BAZEL_BUILD_OPTIONS[@]}" + "-c" "dbg" + "--build_tests_only" + "--remote_download_minimal") echo "bazel MSAN debug build with tests" echo "Building and testing envoy tests ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -- "${TEST_TARGETS[@]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + -- "${TEST_TARGETS[@]}" ;; dev) setup_clang_toolchain @@ -374,7 +432,8 @@ case $CI_TARGET in bazel_envoy_binary_build fastbuild echo "Testing ${TEST_TARGETS[*]}" - bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" + bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ + -c fastbuild "${TEST_TARGETS[@]}" ;; dev.contrib) setup_clang_toolchain @@ -384,7 +443,9 @@ case $CI_TARGET in bazel_contrib_binary_build fastbuild echo "Testing ${TEST_TARGETS[*]}" - bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" + bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ + -c fastbuild \ + "${TEST_TARGETS[@]}" ;; compile_time_options) # Right now, none of the available compile-time options conflict with each other. If this @@ -417,23 +478,65 @@ case $CI_TARGET in # Building all the dependencies from scratch to link them against libc++. echo "Building and testing with wasm=wamr: ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wamr "${COMPILE_TIME_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --define wasm=wamr \ + "${COMPILE_TIME_OPTIONS[@]}" \ + -c fastbuild \ + "${TEST_TARGETS[@]}" \ + --test_tag_filters=-nofips \ + --build_tests_only echo "Building and testing with wasm=wasmtime: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wasmtime --define admin_html=disabled --define admin_functionality=disabled "${COMPILE_TIME_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --define wasm=wasmtime \ + --define admin_html=disabled \ + --define admin_functionality=disabled \ + "${COMPILE_TIME_OPTIONS[@]}" \ + -c fastbuild \ + "${TEST_TARGETS[@]}" \ + --test_tag_filters=-nofips \ + --build_tests_only echo "Building and testing with wasm=wavm: ${TEST_TARGETS[*]}" - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" --test_tag_filters=-nofips --build_tests_only + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --define wasm=wavm \ + "${COMPILE_TIME_OPTIONS[@]}" \ + -c fastbuild \ + "${TEST_TARGETS[@]}" \ + --test_tag_filters=-nofips \ + --build_tests_only # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only # these tests under "-c opt" to save time in CI. - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c opt @envoy//test/common/common:assert_test @envoy//test/server:server_test + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --define wasm=wavm \ + "${COMPILE_TIME_OPTIONS[@]}" \ + -c opt \ + @envoy//test/common/common:assert_test \ + @envoy//test/server:server_test # "--define log_fast_debug_assert_in_release=enabled" must be tested with a release build, so run only these tests under "-c opt" to save time in CI. This option will test only ASSERT()s without SLOW_ASSERT()s, so additionally disable "--define log_debug_assert_in_release" which compiles in both. - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm "${COMPILE_TIME_OPTIONS[@]}" -c opt @envoy//test/common/common:assert_test --define log_fast_debug_assert_in_release=enabled --define log_debug_assert_in_release=disabled + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --define wasm=wavm \ + "${COMPILE_TIME_OPTIONS[@]}" \ + -c opt \ + @envoy//test/common/common:assert_test \ + --define log_fast_debug_assert_in_release=enabled \ + --define log_debug_assert_in_release=disabled echo "Building binary with wasm=wavm... and logging disabled" - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --define wasm=wavm --define enable_logging=disabled "${COMPILE_TIME_OPTIONS[@]}" -c fastbuild @envoy//source/exe:envoy-static --build_tag_filters=-nofips + bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ + --define wasm=wavm \ + --define enable_logging=disabled \ + "${COMPILE_TIME_OPTIONS[@]}" \ + -c fastbuild \ + @envoy//source/exe:envoy-static \ + --build_tag_filters=-nofips collect_build_profile build ;; api) @@ -452,10 +555,16 @@ case $CI_TARGET in echo "Validate Golang protobuf generation..." "${ENVOY_SRCDIR}"/tools/api/generate_go_protobuf.py echo "Testing API..." - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_minimal -c fastbuild @envoy_api//test/... @envoy_api//tools/... \ - @envoy_api//tools:tap2pcap_test + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --remote_download_minimal \ + -c fastbuild \ + @envoy_api//test/... \ + @envoy_api//tools/... \ + @envoy_api//tools:tap2pcap_test echo "Building API..." - bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api//envoy/... + bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ + -c fastbuild @envoy_api//envoy/... ;; api_compat) echo "Checking API for breaking changes to protobuf backwards compatibility..." @@ -463,17 +572,23 @@ case $CI_TARGET in COMMIT_TITLE=$(git log -n 1 --pretty='format:%C(auto)%h (%s, %ad)' "${BASE_BRANCH_REF}") echo -e "\tUsing base commit ${COMMIT_TITLE}" # BAZEL_BUILD_OPTIONS needed for setting the repository_cache param. - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/api_proto_breaking_change_detector:detector_ci "${BASE_BRANCH_REF}" + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/api_proto_breaking_change_detector:detector_ci \ + "${BASE_BRANCH_REF}" ;; coverage|fuzz_coverage) setup_clang_toolchain echo "${CI_TARGET} build with tests ${COVERAGE_TEST_TARGETS[*]}" - [[ "$CI_TARGET" == "fuzz_coverage" ]] && export FUZZ_COVERAGE=true + if [[ "$CI_TARGET" == "fuzz_coverage" ]]; then + export FUZZ_COVERAGE=true + fi # We use custom BAZEL_BUILD_OPTIONS here to cover profiler's code. - BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]} --define tcmalloc=gperftools" "${ENVOY_SRCDIR}"/test/run_envoy_bazel_coverage.sh "${COVERAGE_TEST_TARGETS[@]}" + BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]} --define tcmalloc=gperftools" \ + "${ENVOY_SRCDIR}/test/run_envoy_bazel_coverage.sh" \ + "${COVERAGE_TEST_TARGETS[@]}" collect_build_profile coverage ;; coverage-upload|fuzz_coverage-upload) @@ -513,11 +628,15 @@ case $CI_TARGET in FUZZ_TEST_TARGETS=("$(bazel query "${BAZEL_GLOBAL_OPTIONS[@]}" "attr('tags','fuzzer',${TEST_TARGETS[*]})")") echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS[*]}" echo "Building envoy fuzzers and executing 100 fuzz iterations..." - bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --config=asan-fuzzer "${FUZZ_TEST_TARGETS[@]}" --test_arg="-runs=10" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --config=asan-fuzzer \ + "${FUZZ_TEST_TARGETS[@]}" \ + --test_arg="-runs=10" ;; format) setup_clang_toolchain - "${ENVOY_SRCDIR}"/ci/format_pre.sh + "${ENVOY_SRCDIR}/ci/format_pre.sh" ;; fix_proto_format) # proto_format.sh needs to build protobuf. @@ -538,7 +657,7 @@ case $CI_TARGET in setup_clang_toolchain echo "generating docs..." # Build docs. - "${ENVOY_SRCDIR}"/docs/build.sh + "${ENVOY_SRCDIR}/docs/build.sh" ;; docs-upload) setup_clang_toolchain @@ -548,15 +667,17 @@ case $CI_TARGET in setup_clang_toolchain echo "dependency validate_test..." - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:validate_test + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/dependency:validate_test echo "verifying dependencies..." # Validate dependency relationships between core/extensions and external deps. - time bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:validate + time bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/dependency:validate # Validate repository metadata. echo "check repositories..." - "${ENVOY_SRCDIR}"/tools/check_repositories.sh + "${ENVOY_SRCDIR}/tools/check_repositories.sh" echo "check dependencies..." # Using todays date as an action_env expires the NIST cache daily, which is the update frequency @@ -569,7 +690,8 @@ case $CI_TARGET in # Run dependabot tests echo "Check dependabot ..." - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:dependatool + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/dependency:dependatool ;; verify_examples) run_ci_verify "*" "win32-front-proxy|shared" @@ -580,7 +702,9 @@ case $CI_TARGET in else PACKAGE_BUILD=/build/bazel.distribution/arm64/packages.arm64.tar.gz fi - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //distribution:verify_packages "$PACKAGE_BUILD" + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //distribution:verify_packages \ + "$PACKAGE_BUILD" ;; publish) # If we are on a non-main release branch but the patch version is 0 - then this branch From fb0f3376009991c6303d618f08379a8c2accd8af Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Wed, 10 May 2023 10:10:19 -0400 Subject: [PATCH 187/740] ext_proc filter fuzzer crashed when test cases contains regex config (#27301) Envoy ext_proc filter fuzzer run crashed when test cases contains regex config. Signed-off-by: Yanjun Xiang --- ...h-2f5c31257230464b4b6015ecee3f6f090547fb0a | 43 +++++++++++++++++++ .../unit_test_fuzz/ext_proc_unit_test_fuzz.cc | 5 +++ 2 files changed, 48 insertions(+) create mode 100644 test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-2f5c31257230464b4b6015ecee3f6f090547fb0a diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-2f5c31257230464b4b6015ecee3f6f090547fb0a b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-2f5c31257230464b4b6015ecee3f6f090547fb0a new file mode 100644 index 000000000000..8cb2340a494d --- /dev/null +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-2f5c31257230464b4b6015ecee3f6f090547fb0a @@ -0,0 +1,43 @@ +config { + grpc_service { + envoy_grpc { + cluster_name: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + } + } + response_attributes: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + response_attributes: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + mutation_rules { + allow_expression { + regex: "!" + } + } + max_message_timeout { + } +} +request { +} +response { + request_body { + } + dynamic_metadata { + fields { + key: "" + value { + list_value { + values { + bool_value: false + } + } + } + } + fields { + key: "" + value { + string_value: "" + } + } + } + mode_override { + request_trailer_mode: SEND + } +} diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc index 2c565386cd91..89d6c55c9069 100644 --- a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc @@ -51,6 +51,11 @@ DEFINE_PROTO_FUZZER( input.config(); ExternalProcessing::FilterConfigSharedPtr config; + // Create regex engine which is used by regex matcher code. + Regex::EnginePtr regex_engine = std::make_shared(); + Regex::EngineSingleton::clear(); + Regex::EngineSingleton::initialize(regex_engine.get()); + try { config = std::make_shared( proto_config, std::chrono::milliseconds(200), 200, *stats_store.rootScope(), From 889dd1b9de6b5044ef8cb59f16210c31de905c8a Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Wed, 10 May 2023 10:38:46 -0400 Subject: [PATCH 188/740] Flake: Fix flake OverloadIntegrationTest.NoNewStreamsWhenOverloaded (#27295) Fix rare flake that can occur if the response is encoded before overload state is propagated to the worker. Signed-off-by: Kevin Baichoo --- ...injected_resource_monitor_integration_test.cc | 16 +++++++++++++--- test/integration/server.h | 10 ++++++++-- test/integration/server_stats.h | 10 ++++++++++ test/test_common/utility.cc | 16 +++++++++++++--- test/test_common/utility.h | 14 ++++++++++++++ 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc index a936c6a69876..d0766536b1bb 100644 --- a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc +++ b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc @@ -86,6 +86,7 @@ class OverloadIntegrationTest : public testing::TestWithParam(overload_config); + bootstrap.set_enable_dispatcher_stats(true); }); updateResource(file_updater_1_, 0); @@ -137,9 +138,6 @@ TEST_P(OverloadIntegrationTest, StopAcceptingConnectionsWhenOverloaded) { } TEST_P(OverloadIntegrationTest, NoNewStreamsWhenOverloaded) { - // For https://github.com/envoyproxy/envoy/issues/26264 - // Set to trace since unable to reproduce issue locally. - LogLevelSetter save_levels(spdlog::level::trace); initialize(); updateResource(file_updater_1_, 0.7); @@ -159,6 +157,18 @@ TEST_P(OverloadIntegrationTest, NoNewStreamsWhenOverloaded) { updateResource(file_updater_2_, 0.9); test_server_->waitForGaugeEq("overload.envoy.overload_actions.disable_http_keepalive.active", 1); + // The call to disable keep alive could take some time to be executed on the worker + // even if the stat on the main thread is shows the action is enabled. + // Waiting for additional dispatch loops on the worker so that the overload + // state will have propagated. + auto worker_dispatch_duration_histogram = + test_server_->histogram("listener_manager.worker_0.dispatcher.loop_duration_us"); + const uint64_t num_samples = TestUtility::readSampleCount(test_server_->server().dispatcher(), + *worker_dispatch_duration_histogram); + const uint64_t required_num_samples = num_samples + 2; + test_server_->waitForNumHistogramSamplesGe( + "listener_manager.worker_0.dispatcher.loop_duration_us", required_num_samples); + upstream_request_->encodeHeaders(default_response_headers_, /*end_stream=*/false); upstream_request_->encodeData(10, true); diff --git a/test/integration/server.h b/test/integration/server.h index 6fefb7f76cba..e96335323be2 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -475,8 +475,14 @@ class IntegrationTestServer : public Logger::Loggable, void waitUntilHistogramHasSamples( const std::string& name, std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) override { - ASSERT_TRUE(TestUtility::waitUntilHistogramHasSamples(statStore(), name, time_system_, - server().dispatcher(), timeout)); + waitForNumHistogramSamplesGe(name, 1, timeout); + } + + void waitForNumHistogramSamplesGe( + const std::string& name, uint64_t sample_count, + std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) override { + ASSERT_TRUE(TestUtility::waitForNumHistogramSamplesGe( + statStore(), name, sample_count, time_system_, server().dispatcher(), timeout)); } Stats::CounterSharedPtr counter(const std::string& name) override { diff --git a/test/integration/server_stats.h b/test/integration/server_stats.h index d4520d3456db..04384db7d8be 100644 --- a/test/integration/server_stats.h +++ b/test/integration/server_stats.h @@ -46,6 +46,16 @@ class IntegrationTestServerStats { const std::string& name, std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) PURE; + /** + * Wait until a histogram has at least sample_count samples. + * @param name histogram name. + * @param sample_count the number of samples the histogram should at least + * have. + */ + virtual void waitForNumHistogramSamplesGe( + const std::string& name, uint64_t sample_count, + std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) PURE; + /** * Wait for a gauge to >= a given value. * @param name gauge name. diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index b12bbe3442a1..f090c095bad5 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -236,8 +236,9 @@ AssertionResult TestUtility::waitForGaugeDestroyed(Stats::Store& store, const st return AssertionSuccess(); } -AssertionResult TestUtility::waitUntilHistogramHasSamples(Stats::Store& store, +AssertionResult TestUtility::waitForNumHistogramSamplesGe(Stats::Store& store, const std::string& name, + uint64_t min_sample_count_required, Event::TestTimeSystem& time_system, Event::Dispatcher& main_dispatcher, std::chrono::milliseconds timeout) { @@ -246,7 +247,7 @@ AssertionResult TestUtility::waitUntilHistogramHasSamples(Stats::Store& store, auto histo = findByName(store.histograms(), name); if (histo) { uint64_t sample_count = readSampleCount(main_dispatcher, *histo); - if (sample_count) { + if (sample_count >= min_sample_count_required) { break; } } @@ -254,12 +255,21 @@ AssertionResult TestUtility::waitUntilHistogramHasSamples(Stats::Store& store, time_system.advanceTimeWait(std::chrono::milliseconds(10)); if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { - return AssertionFailure() << fmt::format("timed out waiting for {} to have samples", name); + return AssertionFailure() << fmt::format("timed out waiting for {} to have {} samples", name, + min_sample_count_required); } } return AssertionSuccess(); } +AssertionResult TestUtility::waitUntilHistogramHasSamples(Stats::Store& store, + const std::string& name, + Event::TestTimeSystem& time_system, + Event::Dispatcher& main_dispatcher, + std::chrono::milliseconds timeout) { + return waitForNumHistogramSamplesGe(store, name, 1, time_system, main_dispatcher, timeout); +} + uint64_t TestUtility::readSampleCount(Event::Dispatcher& main_dispatcher, const Stats::ParentHistogram& histogram) { // Note: we need to read the sample count from the main thread, to avoid data races. diff --git a/test/test_common/utility.h b/test/test_common/utility.h index e0b86829fdee..2995e0d7d843 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -306,6 +306,20 @@ class TestUtility { Event::Dispatcher& main_dispatcher, std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()); + /** + * Wait for a histogram to have samples. + * @param store supplies the stats store. + * @param name histogram name. + * @param time_system the time system to use for waiting. + * @param timeout the maximum time to wait before timing out, or 0 for no timeout. + * @return AssertionSuccess() if the histogram was populated within the timeout, else + * AssertionFailure(). + */ + static AssertionResult waitForNumHistogramSamplesGe( + Stats::Store& store, const std::string& name, uint64_t min_sample_count_required, + Event::TestTimeSystem& time_system, Event::Dispatcher& main_dispatcher, + std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()); + /** * Read a histogram's sample count from the main thread. * @param store supplies the stats store. From 0c0c9dc3196eb6bee11f9455273184fd2b31f692 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 10 May 2023 15:45:53 +0100 Subject: [PATCH 189/740] deps: Bump base pip tools (#27310) Signed-off-by: Ryan Northey --- tools/base/requirements.in | 3 + tools/base/requirements.txt | 437 ++++++++++++++++++------------------ 2 files changed, 218 insertions(+), 222 deletions(-) diff --git a/tools/base/requirements.in b/tools/base/requirements.in index c082aa98a5c0..b10adff378ce 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -24,6 +24,9 @@ multidict>=6.0.2 orjson pep8-naming ply +# Upgrading beyond 4.21.x doesnt currently work. 4.23+ might work when the following is resolved +# `MessageFactory class is deprecated. Please use GetMessageClass() instead of MessageFactory.GetPrototype` +protobuf<4.22.0 pygithub pyreadline pyyaml diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index ee17efec730f..fce98079b825 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -200,13 +200,13 @@ async-timeout==4.0.2 \ --hash=sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15 \ --hash=sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c # via aiohttp -attrs==22.2.0 \ - --hash=sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836 \ - --hash=sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99 +attrs==23.1.0 \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 # via aiohttp -babel==2.11.0 \ - --hash=sha256:1ad3eca1c885218f6dce2ab67291178944f810a10a9b5f3cb8382a5a232b64fe \ - --hash=sha256:5ef4b3226b0180dedded4229651c8b0e1a3a6a2837d45a073272f313e4cf97f6 +babel==2.12.1 \ + --hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \ + --hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455 # via sphinx boto==2.49.0 \ --hash=sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8 \ @@ -216,9 +216,9 @@ cachetools==5.3.0 \ --hash=sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14 \ --hash=sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4 # via google-auth -certifi==2022.12.7 \ - --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ - --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 +certifi==2023.5.7 \ + --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \ + --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716 # via requests cffi==1.15.1 \ --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ @@ -289,95 +289,82 @@ cffi==1.15.1 \ # -r requirements.in # cryptography # pynacl -charset-normalizer==3.0.1 \ - --hash=sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b \ - --hash=sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42 \ - --hash=sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d \ - --hash=sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b \ - --hash=sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a \ - --hash=sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59 \ - --hash=sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154 \ - --hash=sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1 \ - --hash=sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c \ - --hash=sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a \ - --hash=sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d \ - --hash=sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6 \ - --hash=sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b \ - --hash=sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b \ - --hash=sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783 \ - --hash=sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5 \ - --hash=sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918 \ - --hash=sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555 \ - --hash=sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639 \ - --hash=sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786 \ - --hash=sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e \ - --hash=sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed \ - --hash=sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820 \ - --hash=sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8 \ - --hash=sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3 \ - --hash=sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541 \ - --hash=sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14 \ - --hash=sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be \ - --hash=sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e \ - --hash=sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76 \ - --hash=sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b \ - --hash=sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c \ - --hash=sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b \ - --hash=sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3 \ - --hash=sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc \ - --hash=sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6 \ - --hash=sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59 \ - --hash=sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4 \ - --hash=sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d \ - --hash=sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d \ - --hash=sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3 \ - --hash=sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a \ - --hash=sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea \ - --hash=sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6 \ - --hash=sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e \ - --hash=sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603 \ - --hash=sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24 \ - --hash=sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a \ - --hash=sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58 \ - --hash=sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678 \ - --hash=sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a \ - --hash=sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c \ - --hash=sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6 \ - --hash=sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18 \ - --hash=sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174 \ - --hash=sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317 \ - --hash=sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f \ - --hash=sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc \ - --hash=sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837 \ - --hash=sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41 \ - --hash=sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c \ - --hash=sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579 \ - --hash=sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753 \ - --hash=sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8 \ - --hash=sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291 \ - --hash=sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087 \ - --hash=sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866 \ - --hash=sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3 \ - --hash=sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d \ - --hash=sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1 \ - --hash=sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca \ - --hash=sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e \ - --hash=sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db \ - --hash=sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72 \ - --hash=sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d \ - --hash=sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc \ - --hash=sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539 \ - --hash=sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d \ - --hash=sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af \ - --hash=sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b \ - --hash=sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602 \ - --hash=sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f \ - --hash=sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478 \ - --hash=sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c \ - --hash=sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e \ - --hash=sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479 \ - --hash=sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7 \ - --hash=sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8 +charset-normalizer==3.1.0 \ + --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ + --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ + --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ + --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ + --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ + --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ + --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ + --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ + --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ + --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ + --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ + --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ + --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ + --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ + --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ + --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ + --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ + --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ + --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ + --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ + --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ + --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ + --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ + --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ + --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ + --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ + --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ + --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ + --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ + --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ + --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ + --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ + --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ + --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ + --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ + --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ + --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ + --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ + --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ + --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ + --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ + --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ + --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ + --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ + --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ + --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ + --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ + --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ + --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ + --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ + --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ + --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ + --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ + --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ + --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ + --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ + --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ + --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ + --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ + --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ + --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ + --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ + --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ + --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ + --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ + --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ + --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ + --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ + --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ + --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ + --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ + --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ + --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ + --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ + --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab # via # aiohttp # requests @@ -396,30 +383,26 @@ coloredlogs==15.0.1 \ crcmod==1.7 \ --hash=sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e # via gsutil -cryptography==39.0.1 \ - --hash=sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4 \ - --hash=sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f \ - --hash=sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885 \ - --hash=sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502 \ - --hash=sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41 \ - --hash=sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965 \ - --hash=sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e \ - --hash=sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc \ - --hash=sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad \ - --hash=sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505 \ - --hash=sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388 \ - --hash=sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6 \ - --hash=sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2 \ - --hash=sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef \ - --hash=sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac \ - --hash=sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695 \ - --hash=sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6 \ - --hash=sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336 \ - --hash=sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0 \ - --hash=sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c \ - --hash=sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106 \ - --hash=sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a \ - --hash=sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8 +cryptography==40.0.2 \ + --hash=sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440 \ + --hash=sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288 \ + --hash=sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b \ + --hash=sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958 \ + --hash=sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b \ + --hash=sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d \ + --hash=sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a \ + --hash=sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404 \ + --hash=sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b \ + --hash=sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e \ + --hash=sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2 \ + --hash=sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c \ + --hash=sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b \ + --hash=sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9 \ + --hash=sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b \ + --hash=sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636 \ + --hash=sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99 \ + --hash=sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e \ + --hash=sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9 # via # pyjwt # pyopenssl @@ -993,9 +976,9 @@ orjson==3.8.12 \ # via # -r requirements.in # envoy-base-utils -packaging==23.0 \ - --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ - --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 +packaging==23.1 \ + --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ + --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f # via # aio-api-github # aio-api-nist @@ -1006,9 +989,9 @@ packaging==23.0 \ # envoy-github-abstract # envoy-github-release # sphinx -pathspec==0.11.0 \ - --hash=sha256:3a66eb970cbac598f9e5ccb5b2cf58930cd8e3ed86d393d541eaf2d8b1705229 \ - --hash=sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc +pathspec==0.11.1 \ + --hash=sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687 \ + --hash=sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293 # via yamllint pep8-naming==0.13.3 \ --hash=sha256:1705f046dfcd851378aac3be1cd1551c7c1e5ff363bacad707d43007877fa971 \ @@ -1034,6 +1017,7 @@ protobuf==4.21.12 \ --hash=sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97 \ --hash=sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574 # via + # -r requirements.in # envoy-base-utils # envoy-docs-sphinx-runner # google-api-core @@ -1063,19 +1047,19 @@ pyflakes==3.0.1 \ --hash=sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf \ --hash=sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd # via flake8 -pygithub==1.58.1 \ - --hash=sha256:4e7fe9c3ec30d5fde5b4fbb97f18821c9dbf372bf6df337fe66f6689a65e0a83 \ - --hash=sha256:7d528b4ad92bc13122129fafd444ce3d04c47d2d801f6446b6e6ee2d410235b3 +pygithub==1.58.2 \ + --hash=sha256:1e6b1b7afe31f75151fb81f7ab6b984a7188a852bdb123dbb9ae90023c3ce60f \ + --hash=sha256:f435884af617c6debaa76cbc355372d1027445a56fbc39972a3b9ed4968badc8 # via -r requirements.in -pygments==2.14.0 \ - --hash=sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297 \ - --hash=sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717 +pygments==2.15.1 \ + --hash=sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c \ + --hash=sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1 # via # envoy-docs-sphinx-runner # sphinx -pyjwt[crypto]==2.6.0 \ - --hash=sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd \ - --hash=sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14 +pyjwt[crypto]==2.7.0 \ + --hash=sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1 \ + --hash=sha256:bd6ca4a3c4285c1a2d4349e5a035fdf8fb94e04ccd0fcbe6ba289dae9cc3e074 # via # gidgethub # pygithub @@ -1108,12 +1092,11 @@ python-gnupg==0.5.0 \ --hash=sha256:345723a03e67b82aba0ea8ae2328b2e4a3906fbe2c18c4082285c3b01068f270 \ --hash=sha256:70758e387fc0e0c4badbcb394f61acbe68b34970a8fed7e0f7c89469fe17912a # via envoy-gpg-identity -pytz==2022.7.1 \ - --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ - --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a +pytz==2023.3 \ + --hash=sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588 \ + --hash=sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb # via # aio-core - # babel # envoy-base-utils pyu2f==0.1.5 \ --hash=sha256:a3caa3a11842fc7d5746376f37195e6af5f17c0a15737538bb1cebf656fb306b @@ -1164,9 +1147,9 @@ pyyaml==6.0 \ # aio-core # envoy-base-utils # yamllint -requests==2.28.2 \ - --hash=sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa \ - --hash=sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf +requests==2.30.0 \ + --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ + --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 # via # google-api-core # google-auth @@ -1218,9 +1201,9 @@ sphinx==7.0.0 \ # sphinx-copybutton # sphinxcontrib-httpdomain # sphinxext-rediraffe -sphinx-copybutton==0.5.1 \ - --hash=sha256:0842851b5955087a7ec7fc870b622cb168618ad408dee42692e9a5c97d071da8 \ - --hash=sha256:366251e28a6f6041514bfb5439425210418d6c750e98d3a695b73e56866a677a +sphinx-copybutton==0.5.2 \ + --hash=sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd \ + --hash=sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e # via envoy-docs-sphinx-runner sphinxcontrib-applehelp==1.0.4 \ --hash=sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228 \ @@ -1274,17 +1257,17 @@ trycast==1.0.0 \ # via # aio-core # envoy-base-utils -typing-extensions==4.4.0 \ - --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ - --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e +typing-extensions==4.5.0 \ + --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \ + --hash=sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4 # via aiodocker uritemplate==4.1.1 \ --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e # via gidgethub -urllib3==1.26.14 \ - --hash=sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72 \ - --hash=sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1 +urllib3==2.0.2 \ + --hash=sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc \ + --hash=sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e # via requests uvloop==0.17.0 \ --hash=sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d \ @@ -1328,75 +1311,86 @@ verboselogs==1.7 \ # envoy-distribution-repo # envoy-github-abstract # envoy-github-release -wrapt==1.14.1 \ - --hash=sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3 \ - --hash=sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b \ - --hash=sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4 \ - --hash=sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2 \ - --hash=sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656 \ - --hash=sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3 \ - --hash=sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff \ - --hash=sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310 \ - --hash=sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a \ - --hash=sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57 \ - --hash=sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069 \ - --hash=sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383 \ - --hash=sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe \ - --hash=sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87 \ - --hash=sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d \ - --hash=sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b \ - --hash=sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907 \ - --hash=sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f \ - --hash=sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0 \ - --hash=sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28 \ - --hash=sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1 \ - --hash=sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853 \ - --hash=sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc \ - --hash=sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3 \ - --hash=sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3 \ - --hash=sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164 \ - --hash=sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1 \ - --hash=sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c \ - --hash=sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1 \ - --hash=sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7 \ - --hash=sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1 \ - --hash=sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320 \ - --hash=sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed \ - --hash=sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1 \ - --hash=sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248 \ - --hash=sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c \ - --hash=sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456 \ - --hash=sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77 \ - --hash=sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef \ - --hash=sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1 \ - --hash=sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7 \ - --hash=sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86 \ - --hash=sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4 \ - --hash=sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d \ - --hash=sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d \ - --hash=sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8 \ - --hash=sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5 \ - --hash=sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471 \ - --hash=sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00 \ - --hash=sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68 \ - --hash=sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3 \ - --hash=sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d \ - --hash=sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735 \ - --hash=sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d \ - --hash=sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569 \ - --hash=sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7 \ - --hash=sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59 \ - --hash=sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5 \ - --hash=sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb \ - --hash=sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b \ - --hash=sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f \ - --hash=sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462 \ - --hash=sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015 \ - --hash=sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af +wrapt==1.15.0 \ + --hash=sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0 \ + --hash=sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420 \ + --hash=sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a \ + --hash=sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c \ + --hash=sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079 \ + --hash=sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923 \ + --hash=sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f \ + --hash=sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1 \ + --hash=sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8 \ + --hash=sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86 \ + --hash=sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0 \ + --hash=sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364 \ + --hash=sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e \ + --hash=sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c \ + --hash=sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e \ + --hash=sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c \ + --hash=sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727 \ + --hash=sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff \ + --hash=sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e \ + --hash=sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29 \ + --hash=sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7 \ + --hash=sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72 \ + --hash=sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475 \ + --hash=sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a \ + --hash=sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317 \ + --hash=sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2 \ + --hash=sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd \ + --hash=sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640 \ + --hash=sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98 \ + --hash=sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248 \ + --hash=sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e \ + --hash=sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d \ + --hash=sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec \ + --hash=sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1 \ + --hash=sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e \ + --hash=sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9 \ + --hash=sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92 \ + --hash=sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb \ + --hash=sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094 \ + --hash=sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46 \ + --hash=sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29 \ + --hash=sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd \ + --hash=sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705 \ + --hash=sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8 \ + --hash=sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975 \ + --hash=sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb \ + --hash=sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e \ + --hash=sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b \ + --hash=sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418 \ + --hash=sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019 \ + --hash=sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1 \ + --hash=sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba \ + --hash=sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6 \ + --hash=sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2 \ + --hash=sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3 \ + --hash=sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7 \ + --hash=sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752 \ + --hash=sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416 \ + --hash=sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f \ + --hash=sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1 \ + --hash=sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc \ + --hash=sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145 \ + --hash=sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee \ + --hash=sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a \ + --hash=sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7 \ + --hash=sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b \ + --hash=sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653 \ + --hash=sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0 \ + --hash=sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90 \ + --hash=sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29 \ + --hash=sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6 \ + --hash=sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034 \ + --hash=sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09 \ + --hash=sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559 \ + --hash=sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639 # via deprecated -yamllint==1.29.0 \ - --hash=sha256:5153bf9f8205aa9dc6af6217e38bd4f5baf09d9a7c6f4ae1e23f90d9c00c49c5 \ - --hash=sha256:66a755d5fbcbb8831f1a9568676329b5bac82c37995bcc9afd048b6459f9fa48 +yamllint==1.31.0 \ + --hash=sha256:15f4bdb645e6a4a0a22fe5415bc38b4a934c51419b30104896d2f3f95e329185 \ + --hash=sha256:2d83f1d12f733e162a87e06b176149d7bb9c5bae4a9e5fce1c771d7f703f7a65 # via envoy-code-check yapf==0.33.0 \ --hash=sha256:4c2b59bd5ffe46f3a7da48df87596877189148226ce267c16e8b44240e51578d \ @@ -1535,4 +1529,3 @@ setuptools==67.7.2 \ # via # -r requirements.in # sphinxcontrib-jquery - # yamllint From 846b648336a1be60690d7ee917df2ca4058d032b Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 10 May 2023 10:47:12 -0400 Subject: [PATCH 190/740] mobile: c++ explicit flow control tests and fixes (#27318) Handling reentrant callbacks for data, and slow uploads for explicit flow control mode properly. Testing: e2e tests Docs Changes: n/a Release Notes:n/a [Optional Runtime guard:] n/a Signed-off-by: Alyssa Wilk --- mobile/library/cc/stream.cc | 5 ++ mobile/library/cc/stream.h | 1 + mobile/library/cc/stream_callbacks.cc | 1 - mobile/library/common/http/client.cc | 7 +-- .../integration/client_integration_test.cc | 53 +++++++++++++++++++ 5 files changed, 63 insertions(+), 4 deletions(-) diff --git a/mobile/library/cc/stream.cc b/mobile/library/cc/stream.cc index c4b6228ef9b1..7e46d0ad38b5 100644 --- a/mobile/library/cc/stream.cc +++ b/mobile/library/cc/stream.cc @@ -21,6 +21,11 @@ Stream& Stream::sendData(envoy_data data) { return *this; } +Stream& Stream::readData(size_t bytes_to_read) { + ::read_data(engine_handle_, handle_, bytes_to_read); + return *this; +} + void Stream::close(RequestTrailersSharedPtr trailers) { envoy_headers raw_headers = rawHeaderMapAsEnvoyHeaders(trailers->allHeaders()); ::send_trailers(engine_handle_, handle_, raw_headers); diff --git a/mobile/library/cc/stream.h b/mobile/library/cc/stream.h index 3ecfe70b7b95..946402cb44cb 100644 --- a/mobile/library/cc/stream.h +++ b/mobile/library/cc/stream.h @@ -19,6 +19,7 @@ class Stream { Stream& sendHeaders(RequestHeadersSharedPtr headers, bool end_stream); Stream& sendData(envoy_data data); + Stream& readData(size_t bytes_to_read); void close(RequestTrailersSharedPtr trailers); void close(envoy_data data); void cancel(); diff --git a/mobile/library/cc/stream_callbacks.cc b/mobile/library/cc/stream_callbacks.cc index 8673094d1ae7..bc73a9bc6cc7 100644 --- a/mobile/library/cc/stream_callbacks.cc +++ b/mobile/library/cc/stream_callbacks.cc @@ -103,7 +103,6 @@ void* c_on_send_window_available(envoy_stream_intel intel, void* context) { auto on_send_window_available = stream_callbacks->on_send_window_available.value(); on_send_window_available(intel); } - delete stream_callbacks_ptr; return nullptr; } diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index f41c432da55b..39c673b08553 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -158,6 +158,10 @@ void Client::DirectStreamCallbacks::sendDataToBridge(Buffer::Instance& data, boo auto callback_time_ms = std::make_unique( http_client_.stats().on_data_callback_latency_, http_client_.timeSource()); + // Make sure that when using explicit flow control this won't send more data until the next call + // to resumeData. Set before on-data to handle reentrant callbacks. + bytes_to_send_ = 0; + bridge_callbacks_.on_data(Data::Utility::toBridgeData(data, bytes_to_send), send_end_stream, streamIntel(), bridge_callbacks_.context); @@ -170,9 +174,6 @@ void Client::DirectStreamCallbacks::sendDataToBridge(Buffer::Instance& data, boo if (send_end_stream) { onComplete(); } - // Make sure that when using explicit flow control this won't send more data until the next call - // to resumeData. - bytes_to_send_ = 0; } void Client::DirectStreamCallbacks::encodeTrailers(const ResponseTrailerMap& trailers) { diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index b72dfe477a13..cedd510f015a 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -33,6 +33,7 @@ class ClientIntegrationTest : public BaseClientIntegrationTest, void TearDown() override { BaseClientIntegrationTest::TearDown(); } void basicTest(); + void trickleTest(); protected: std::unique_ptr helper_handle_; @@ -89,6 +90,58 @@ TEST_P(ClientIntegrationTest, LargeResponse) { ASSERT_EQ(cc_.on_complete_received_byte_count, 32828); } +void ClientIntegrationTest::trickleTest() { + autonomous_upstream_ = false; + + initialize(); + + stream_prototype_->setOnData([this](envoy_data c_data, bool) { + if (explicit_flow_control_) { + // Allow reading up to 100 bytes + stream_->readData(100); + } + cc_.on_data_calls++; + release_envoy_data(c_data); + }); + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), false); + if (explicit_flow_control_) { + // Allow reading up to 100 bytes + stream_->readData(100); + } + Buffer::OwnedImpl request_data = Buffer::OwnedImpl("request body"); + envoy_data c_data = Data::Utility::toBridgeData(request_data); + stream_->sendData(c_data); + Platform::RequestTrailersBuilder builder; + std::shared_ptr trailers = + std::make_shared(builder.build()); + stream_->close(trailers); + + FakeHttpConnectionPtr upstream_connection; + FakeStreamPtr upstream_request; + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*BaseIntegrationTest::dispatcher_, + upstream_connection)); + ASSERT_TRUE( + upstream_connection->waitForNewStream(*BaseIntegrationTest::dispatcher_, upstream_request)); + + upstream_request->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + for (int i = 0; i < 10; ++i) { + upstream_request->encodeData(1, i == 9); + } + + terminal_callback_.waitReady(); +} + +TEST_P(ClientIntegrationTest, Trickle) { + trickleTest(); + ASSERT_LE(cc_.on_data_calls, 11); +} + +TEST_P(ClientIntegrationTest, TrickleExplicitFlowControl) { + explicit_flow_control_ = true; + trickleTest(); + ASSERT_LE(cc_.on_data_calls, 11); +} + TEST_P(ClientIntegrationTest, ClearTextNotPermitted) { EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).WillRepeatedly(Return(false)); From c11f42e0431d7c7e9611b9099f572f2877848311 Mon Sep 17 00:00:00 2001 From: Christoph Pakulski Date: Wed, 10 May 2023 10:48:51 -0400 Subject: [PATCH 191/740] stateful_session: encode expiry time and reject expired cookies (#26761) * Hacked version with expiry added after semicolon. Signed-off-by: Christoph Pakulski * Implemented cookie in JSON format. Signed-off-by: Christoph Pakulski * Added integration tests. Signed-off-by: Christoph Pakulski * Helper class to simplify comparing JSON encoded cookies in tests. Signed-off-by: Christoph Pakulski * Log when old cookie format is detected. Signed-off-by: Christoph Pakulski * Formatting. Signed-off-by: Christoph Pakulski * Updated release notes. Signed-off-by: Christoph Pakulski * Typo. Signed-off-by: Christoph Pakulski * Changed cookie encoding from JSON to protobuf. Signed-off-by: Christoph Pakulski * Fixed clang-tidy. Signed-off-by: Christoph Pakulski * Changed cookie expiry type to uint64. Signed-off-by: Christoph Pakulski * Generate single log regardless of number of old-style cookies. Signed-off-by: Christoph Pakulski * Used ENVOY_LOG_ONCE to supress subsequent logs. Signed-off-by: Christoph Pakulski * Always reply in new format, regardless of received cookie's format. Signed-off-by: Christoph Pakulski * Remove unnecessary intermediate local variables. Signed-off-by: Christoph Pakulski * Kick CI Signed-off-by: Christoph Pakulski --------- Signed-off-by: Christoph Pakulski --- changelogs/current.yaml | 7 + source/common/runtime/runtime_features.cc | 1 + .../http/stateful_session/cookie/BUILD | 7 + .../http/stateful_session/cookie/config.cc | 3 +- .../http/stateful_session/cookie/cookie.cc | 23 +- .../http/stateful_session/cookie/cookie.h | 43 +- .../http/stateful_session/cookie/cookie.proto | 8 + .../stateful_session_integration_test.cc | 790 ++++++++++++------ .../http/stateful_session/cookie/BUILD | 1 + .../stateful_session/cookie/cookie_test.cc | 174 +++- tools/code_format/config.yaml | 1 + 11 files changed, 748 insertions(+), 310 deletions(-) create mode 100644 source/extensions/http/stateful_session/cookie/cookie.proto diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 557090b2e8fd..381512f4f4bc 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -48,6 +48,13 @@ minor_behavior_changes: Added support for fetching credentials from the AWS credentials file, which only happens if credentials cannot be fetched from environment variables. This behavioral change can be reverted by setting runtime guard ``envoy.reloadable_features.enable_aws_credentials_file`` to ``false``. +- area: http cookies + change: | + Changed internal format of http cookie to protobuf and added expiry timestamp. Processing expired cookie + results in selection of a new upstream host and sending a new cookie to the client. Previous format of + the cookie is still accepted, but is planned to be obsoleted in the future. + This behavior change can be reverted by setting + ``envoy.reloadable_features.stateful_session_encode_ttl_in_cookie`` to ``false``. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index ff9d03110511..2eba8476723e 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -62,6 +62,7 @@ RUNTIME_GUARD(envoy_reloadable_features_reject_require_client_certificate_with_q RUNTIME_GUARD(envoy_reloadable_features_sanitize_original_path); RUNTIME_GUARD(envoy_reloadable_features_service_sanitize_non_utf8_strings); RUNTIME_GUARD(envoy_reloadable_features_skip_dns_lookup_for_proxied_requests); +RUNTIME_GUARD(envoy_reloadable_features_stateful_session_encode_ttl_in_cookie); RUNTIME_GUARD(envoy_reloadable_features_successful_active_health_check_uneject_host); RUNTIME_GUARD(envoy_reloadable_features_tcp_pool_idle_timeout); RUNTIME_GUARD(envoy_reloadable_features_test_feature_true); diff --git a/source/extensions/http/stateful_session/cookie/BUILD b/source/extensions/http/stateful_session/cookie/BUILD index ceed1d6ff7d9..0f267e7cc74c 100644 --- a/source/extensions/http/stateful_session/cookie/BUILD +++ b/source/extensions/http/stateful_session/cookie/BUILD @@ -3,6 +3,7 @@ load( "envoy_cc_extension", "envoy_cc_library", "envoy_extension_package", + "envoy_proto_library", ) licenses(["notice"]) # Apache 2 @@ -14,6 +15,7 @@ envoy_cc_library( srcs = ["cookie.cc"], hdrs = ["cookie.h"], deps = [ + ":cookie_encoding_cc_proto", "//envoy/http:stateful_session_interface", "//source/common/common:base64_lib", "//source/common/http:headers_lib", @@ -34,3 +36,8 @@ envoy_cc_extension( "@envoy_api//envoy/extensions/http/stateful_session/cookie/v3:pkg_cc_proto", ], ) + +envoy_proto_library( + name = "cookie_encoding", + srcs = ["cookie.proto"], +) diff --git a/source/extensions/http/stateful_session/cookie/config.cc b/source/extensions/http/stateful_session/cookie/config.cc index 39a0348d2270..e978b2d8a603 100644 --- a/source/extensions/http/stateful_session/cookie/config.cc +++ b/source/extensions/http/stateful_session/cookie/config.cc @@ -14,7 +14,8 @@ CookieBasedSessionStateFactoryConfig::createSessionStateFactory( const auto& proto_config = MessageUtil::downcastAndValidate( config, context.messageValidationVisitor()); - return std::make_shared(proto_config); + return std::make_shared( + proto_config, context.mainThreadDispatcher().timeSource()); } REGISTER_FACTORY(CookieBasedSessionStateFactoryConfig, Envoy::Http::SessionStateFactoryConfig); diff --git a/source/extensions/http/stateful_session/cookie/cookie.cc b/source/extensions/http/stateful_session/cookie/cookie.cc index ade6c4bebcf0..631249b763ea 100644 --- a/source/extensions/http/stateful_session/cookie/cookie.cc +++ b/source/extensions/http/stateful_session/cookie/cookie.cc @@ -1,6 +1,7 @@ #include "source/extensions/http/stateful_session/cookie/cookie.h" #include "source/common/http/headers.h" +#include "source/common/runtime/runtime_features.h" namespace Envoy { namespace Extensions { @@ -11,18 +12,32 @@ namespace Cookie { void CookieBasedSessionStateFactory::SessionStateImpl::onUpdate( const Upstream::HostDescription& host, Envoy::Http::ResponseHeaderMap& headers) { absl::string_view host_address = host.address()->asStringView(); + std::string encoded_address; if (!upstream_address_.has_value() || host_address != upstream_address_.value()) { - const std::string encoded_address = - Envoy::Base64::encode(host_address.data(), host_address.length()); + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.stateful_session_encode_ttl_in_cookie")) { + auto expiry_time = std::chrono::duration_cast( + (time_source_.monotonicTime() + std::chrono::seconds(factory_.ttl_)).time_since_epoch()); + // Build proto message + envoy::Cookie cookie; + cookie.set_address(std::string(host_address)); + cookie.set_expires(expiry_time.count()); + std::string proto_string; + cookie.SerializeToString(&proto_string); + + encoded_address = Envoy::Base64::encode(proto_string.data(), proto_string.length()); + } else { + encoded_address = Envoy::Base64::encode(host_address.data(), host_address.length()); + } headers.addReferenceKey(Envoy::Http::Headers::get().SetCookie, factory_.makeSetCookie(encoded_address)); } } CookieBasedSessionStateFactory::CookieBasedSessionStateFactory( - const CookieBasedSessionStateProto& config) + const CookieBasedSessionStateProto& config, TimeSource& time_source) : name_(config.cookie().name()), ttl_(config.cookie().ttl().seconds()), - path_(config.cookie().path()) { + path_(config.cookie().path()), time_source_(time_source) { if (name_.empty()) { throw EnvoyException("Cookie key cannot be empty for cookie based stateful sessions"); } diff --git a/source/extensions/http/stateful_session/cookie/cookie.h b/source/extensions/http/stateful_session/cookie/cookie.h index ecc883f37638..a03223c5bc8d 100644 --- a/source/extensions/http/stateful_session/cookie/cookie.h +++ b/source/extensions/http/stateful_session/cookie/cookie.h @@ -9,6 +9,7 @@ #include "source/common/common/base64.h" #include "source/common/http/utility.h" +#include "source/extensions/http/stateful_session/cookie/cookie.pb.h" namespace Envoy { namespace Extensions { @@ -24,8 +25,8 @@ class CookieBasedSessionStateFactory : public Envoy::Http::SessionStateFactory { class SessionStateImpl : public Envoy::Http::SessionState { public: SessionStateImpl(absl::optional address, - const CookieBasedSessionStateFactory& factory) - : upstream_address_(std::move(address)), factory_(factory) {} + const CookieBasedSessionStateFactory& factory, TimeSource& time_source) + : upstream_address_(std::move(address)), factory_(factory), time_source_(time_source) {} absl::optional upstreamAddress() const override { return upstream_address_; } void onUpdate(const Upstream::HostDescription& host, @@ -34,16 +35,18 @@ class CookieBasedSessionStateFactory : public Envoy::Http::SessionStateFactory { private: absl::optional upstream_address_; const CookieBasedSessionStateFactory& factory_; + TimeSource& time_source_; }; - CookieBasedSessionStateFactory(const CookieBasedSessionStateProto& config); + CookieBasedSessionStateFactory(const CookieBasedSessionStateProto& config, + TimeSource& time_source); Envoy::Http::SessionStatePtr create(const Envoy::Http::RequestHeaderMap& headers) const override { if (!requestPathMatch(headers.getPathValue())) { return nullptr; } - return std::make_unique(parseAddress(headers), *this); + return std::make_unique(parseAddress(headers), *this, time_source_); } bool requestPathMatch(absl::string_view request_path) const { @@ -54,9 +57,36 @@ class CookieBasedSessionStateFactory : public Envoy::Http::SessionStateFactory { private: absl::optional parseAddress(const Envoy::Http::RequestHeaderMap& headers) const { const std::string cookie_value = Envoy::Http::Utility::parseCookieValue(headers, name_); - std::string address = Envoy::Base64::decode(cookie_value); + if (cookie_value.empty()) { + return absl::nullopt; + } + const std::string decoded_value = Envoy::Base64::decode(cookie_value); + std::string address; + + // Try to interpret the cookie as proto payload. + // Otherwise treat it as "old" style format, which is ip-address:port. + envoy::Cookie cookie; + if (cookie.ParseFromString(decoded_value)) { + address = cookie.address(); + if (address.empty() || (cookie.expires() == 0)) { + return absl::nullopt; + } + + std::chrono::seconds expiry_time(cookie.expires()); + auto now = std::chrono::duration_cast( + (time_source_.monotonicTime()).time_since_epoch()); + if (now > expiry_time) { + // Ignore the address extracted from the cookie. This will cause + // upstream cluster to select a new host and new cookie will be generated. + return absl::nullopt; + } + } else { + ENVOY_LOG_ONCE_MISC( + warn, "Non-proto cookie format detected. This format will be rejected in the future."); + address = decoded_value; + } - return !address.empty() ? absl::make_optional(std::move(address)) : absl::nullopt; + return address.empty() ? absl::nullopt : absl::make_optional(std::move(address)); } std::string makeSetCookie(const std::string& address) const { @@ -66,6 +96,7 @@ class CookieBasedSessionStateFactory : public Envoy::Http::SessionStateFactory { const std::string name_; const std::chrono::seconds ttl_; const std::string path_; + TimeSource& time_source_; std::function path_matcher_; }; diff --git a/source/extensions/http/stateful_session/cookie/cookie.proto b/source/extensions/http/stateful_session/cookie/cookie.proto new file mode 100644 index 000000000000..6bba79664f5b --- /dev/null +++ b/source/extensions/http/stateful_session/cookie/cookie.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package envoy; + +message Cookie { + string address = 1; + uint64 expires = 2; +} diff --git a/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc b/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc index 479cd7875754..6aed8fffc6f3 100644 --- a/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc +++ b/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc @@ -7,6 +7,7 @@ #include "source/common/http/utility.h" #include "source/common/protobuf/protobuf.h" #include "source/extensions/filters/http/stateful_session/stateful_session.h" +#include "source/extensions/http/stateful_session/cookie/cookie.pb.h" #include "test/integration/http_integration.h" @@ -84,6 +85,43 @@ class StatefulSessionIntegrationTest : public Envoy::HttpIntegrationTest, public } }; +// Helper object to parse proto encoded cookie and compare with other object. +// "expire" field is ignored. +class ProtoCookieObject { +public: + ProtoCookieObject() = delete; + // Constructor which parses PROTO object. + ProtoCookieObject(absl::string_view cookie) { + std::vector v = absl::StrSplit(cookie, absl::ByChar(';')); + std::vector sub_v = absl::StrSplit(v[0], absl::MaxSplits('=', 1)); + sub_v[1].remove_prefix(1); + sub_v[1].remove_suffix(1); + const std::string decoded_value = Envoy::Base64::decode(sub_v[1]); + + envoy::Cookie proto_cookie; + if (proto_cookie.ParseFromString(decoded_value)) { + address_ = proto_cookie.address(); + } + max_age_ = absl::StripLeadingAsciiWhitespace(v[1]); + path_ = absl::StripLeadingAsciiWhitespace(v[2]); + http_ = absl::StripLeadingAsciiWhitespace(v[3]); + } + // Constructor which initializes individual fields + ProtoCookieObject(std::string address, uint32_t age, std::string path, std::string http) + : address_(address), max_age_(fmt::format("Max-Age={}", age)), + path_(fmt::format("Path={}", path)), http_(http) {} + bool operator==(const ProtoCookieObject& other) const { + return (address_ == other.address_) && (max_age_ == other.max_age_) && (path_ == other.path_) && + (http_ == other.http_); + } + +private: + std::string address_; + std::string max_age_; + std::string path_; + std::string http_; +}; + static const std::string STATEFUL_SESSION_FILTER = R"EOF( name: envoy.filters.http.stateful_session @@ -145,39 +183,52 @@ static const std::string OVERRIDE_STATEFUL_SESSION_HEADER = TEST_F(StatefulSessionIntegrationTest, NormalStatefulSession) { initializeFilterAndRoute(STATEFUL_SESSION_FILTER, ""); - codec_client_ = makeHttpConnection(lookupPort("http")); - - Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, - {":path", "/test"}, - {":scheme", "http"}, - {":authority", "stateful.session.com"}}; + // Run the test twice. Once with proto cookie encoding and once with "old", non-proto + // encoding. + for (const bool use_proto : std::vector({true, false})) { + Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.stateful_session_encode_ttl_in_cookie", + use_proto); + codec_client_ = makeHttpConnection(lookupPort("http")); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}}; - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - ASSERT(upstream_index.has_value()); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - envoy::config::endpoint::v3::LbEndpoint endpoint; - setUpstreamAddress(upstream_index.value(), endpoint); - const std::string address_string = - fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); - const std::string encoded_address = - Envoy::Base64::encode(address_string.data(), address_string.size()); + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + ASSERT(upstream_index.has_value()); - upstream_request_->encodeHeaders(default_response_headers_, true); + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(upstream_index.value(), endpoint); - ASSERT_TRUE(response->waitForEndStream()); + std::string address_string = + fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); + upstream_request_->encodeHeaders(default_response_headers_, true); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + ASSERT_TRUE(response->waitForEndStream()); - // The selected upstream server address would be selected to the response headers. - EXPECT_EQ( - Envoy::Http::Utility::makeSetCookieValue("global-session-cookie", encoded_address, "/test", - std::chrono::seconds(120), true), - response->headers().get(Http::LowerCaseString("set-cookie"))[0]->value().getStringView()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - cleanupUpstreamAndDownstream(); + // The selected upstream server address would be selected to the response headers. + if (use_proto) { + EXPECT_EQ(ProtoCookieObject(response->headers() + .get(Http::LowerCaseString("set-cookie"))[0] + ->value() + .getStringView()), + ProtoCookieObject(address_string, 120, "/test", "HttpOnly")); + } else { + const std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); + EXPECT_EQ( + Envoy::Http::Utility::makeSetCookieValue("global-session-cookie", encoded_address, + "/test", std::chrono::seconds(120), true), + response->headers().get(Http::LowerCaseString("set-cookie"))[0]->value().getStringView()); + } + cleanupUpstreamAndDownstream(); + } } TEST_F(StatefulSessionIntegrationTest, NormalStatefulSessionHeader) { @@ -220,114 +271,168 @@ TEST_F(StatefulSessionIntegrationTest, NormalStatefulSessionHeader) { TEST_F(StatefulSessionIntegrationTest, DownstreamRequestWithStatefulSessionCookie) { initializeFilterAndRoute(STATEFUL_SESSION_FILTER, ""); - { + // Run the test twice. Once with proto cookie encoding and once with "old", non-proto + // encoding. + for (const bool use_proto : std::vector({true, false})) { + Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.stateful_session_encode_ttl_in_cookie", + use_proto); + { - envoy::config::endpoint::v3::LbEndpoint endpoint; - setUpstreamAddress(1, endpoint); - const std::string address_string = - fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); - const std::string encoded_address = - Envoy::Base64::encode(address_string.data(), address_string.size()); + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(1, endpoint); + + std::string address_string; + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address(std::string(fmt::format( + "127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()))); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&address_string); + } else { + address_string = fmt::format("127.0.0.1:{}", + endpoint.endpoint().address().socket_address().port_value()); + } + const std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); - codec_client_ = makeHttpConnection(lookupPort("http")); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, - {":path", "/test"}, - {":scheme", "http"}, - {":authority", "stateful.session.com"}, - {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - // The upstream with index 1 should be selected. - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - EXPECT_EQ(upstream_index.value(), 1); + // The upstream with index 1 should be selected. + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + EXPECT_EQ(upstream_index.value(), 1); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - // No response header to be added. - EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); - cleanupUpstreamAndDownstream(); - } + cleanupUpstreamAndDownstream(); + } - { - envoy::config::endpoint::v3::LbEndpoint endpoint; - setUpstreamAddress(2, endpoint); - const std::string address_string = - fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); - const std::string encoded_address = - Envoy::Base64::encode(address_string.data(), address_string.size()); + { + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(2, endpoint); + std::string address_string; + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address(std::string(fmt::format( + "127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()))); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&address_string); + } else { + address_string = fmt::format("127.0.0.1:{}", + endpoint.endpoint().address().socket_address().port_value()); + } + const std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); - codec_client_ = makeHttpConnection(lookupPort("http")); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, - {":path", "/test"}, - {":scheme", "http"}, - {":authority", "stateful.session.com"}, - {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - // The upstream with index 2 should be selected. - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - EXPECT_EQ(upstream_index.value(), 2); + // The upstream with index 2 should be selected. + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + EXPECT_EQ(upstream_index.value(), 2); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - // No response header to be added. - EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); - cleanupUpstreamAndDownstream(); - } + cleanupUpstreamAndDownstream(); + } - // Test the case that stateful session cookie with unknown server address. - { - codec_client_ = makeHttpConnection(lookupPort("http")); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, - {":path", "/test"}, - {":scheme", "http"}, - {":authority", "stateful.session.com"}, - {"cookie", fmt::format("global-session-cookie=\"{}\"", - Envoy::Base64::encode("127.0.0.7:50000", 15))}}; + // Test the case that stateful session cookie with unknown server address. + { + std::string address_string; + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address(std::string("127.0.0.1:50000")); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&address_string); + } else { + address_string = "127.0.0.1:50000"; + } + std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - ASSERT(upstream_index.has_value()); + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + ASSERT(upstream_index.has_value()); - envoy::config::endpoint::v3::LbEndpoint endpoint; - setUpstreamAddress(upstream_index.value(), endpoint); - const std::string address_string = - fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); - const std::string encoded_address = - Envoy::Base64::encode(address_string.data(), address_string.size()); + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(upstream_index.value(), endpoint); + address_string = + fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - // The selected upstream server address would be selected to the response headers. - EXPECT_EQ( - Envoy::Http::Utility::makeSetCookieValue("global-session-cookie", encoded_address, "/test", - std::chrono::seconds(120), true), - response->headers().get(Http::LowerCaseString("set-cookie"))[0]->value().getStringView()); + if (use_proto) { + EXPECT_EQ(ProtoCookieObject(response->headers() + .get(Http::LowerCaseString("set-cookie"))[0] + ->value() + .getStringView()), + ProtoCookieObject(address_string, 120, "/test", "HttpOnly")); + } else { + encoded_address = Envoy::Base64::encode(address_string.data(), address_string.size()); + // The selected upstream server address would be selected to the response headers. + EXPECT_EQ(Envoy::Http::Utility::makeSetCookieValue("global-session-cookie", encoded_address, + "/test", std::chrono::seconds(120), + true), + response->headers() + .get(Http::LowerCaseString("set-cookie"))[0] + ->value() + .getStringView()); + } - cleanupUpstreamAndDownstream(); + cleanupUpstreamAndDownstream(); + } } } @@ -445,72 +550,89 @@ TEST_F(StatefulSessionIntegrationTest, DownstreamRequestWithStatefulSessionHeade TEST_F(StatefulSessionIntegrationTest, StatefulSessionDisabledByRoute) { initializeFilterAndRoute(STATEFUL_SESSION_FILTER, DISABLE_STATEFUL_SESSION); - { - uint64_t first_index = 0; - uint64_t second_index = 0; + // Run the test twice. Once with proto cookie encoding and once with "old", non-proto + // encoding. + for (const bool use_proto : std::vector({true, false})) { + Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.stateful_session_encode_ttl_in_cookie", + use_proto); + { + uint64_t first_index = 0; + uint64_t second_index = 0; - envoy::config::endpoint::v3::LbEndpoint endpoint; - setUpstreamAddress(1, endpoint); - const std::string address_string = - fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); - const std::string encoded_address = - Envoy::Base64::encode(address_string.data(), address_string.size()); + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(1, endpoint); + std::string address_string; + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address(std::string(fmt::format( + "127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()))); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&address_string); + } else { + address_string = fmt::format("127.0.0.1:{}", + endpoint.endpoint().address().socket_address().port_value()); + } + const std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, - {":path", "/test"}, - {":scheme", "http"}, - {":authority", "stateful.session.com"}, - {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; - { - codec_client_ = makeHttpConnection(lookupPort("http")); + { + codec_client_ = makeHttpConnection(lookupPort("http")); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - ASSERT(upstream_index.has_value()); - first_index = upstream_index.value(); + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + ASSERT(upstream_index.has_value()); + first_index = upstream_index.value(); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - // No response header to be added. - EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); - cleanupUpstreamAndDownstream(); - } + cleanupUpstreamAndDownstream(); + } - { - codec_client_ = makeHttpConnection(lookupPort("http")); + { + codec_client_ = makeHttpConnection(lookupPort("http")); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - ASSERT(upstream_index.has_value()); - second_index = upstream_index.value(); + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + ASSERT(upstream_index.has_value()); + second_index = upstream_index.value(); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - // No response header to be added. - EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); - cleanupUpstreamAndDownstream(); - } + cleanupUpstreamAndDownstream(); + } - // Choose different upstream servers by default. - EXPECT_NE(first_index, second_index); + // Choose different upstream servers by default. + EXPECT_NE(first_index, second_index); + } } - { uint64_t first_index = 0; uint64_t second_index = 0; @@ -580,81 +702,122 @@ TEST_F(StatefulSessionIntegrationTest, StatefulSessionDisabledByRoute) { TEST_F(StatefulSessionIntegrationTest, CookieStatefulSessionOverriddenByRoute) { initializeFilterAndRoute(STATEFUL_SESSION_FILTER, OVERRIDE_STATEFUL_SESSION); - { - envoy::config::endpoint::v3::LbEndpoint endpoint; - setUpstreamAddress(1, endpoint); - const std::string address_string = - fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); - const std::string encoded_address = - Envoy::Base64::encode(address_string.data(), address_string.size()); + // Run the test twice. Once with proto cookie encoding and once with "old", non-proto + // encoding. + for (const bool use_proto : std::vector({true, false})) { + Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.stateful_session_encode_ttl_in_cookie", + use_proto); + { + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(1, endpoint); + std::string address_string; + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address(std::string(fmt::format( + "127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()))); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&address_string); + } else { + address_string = fmt::format("127.0.0.1:{}", + endpoint.endpoint().address().socket_address().port_value()); + } + const std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); - codec_client_ = makeHttpConnection(lookupPort("http")); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, - {":path", "/test"}, - {":scheme", "http"}, - {":authority", "stateful.session.com"}, - {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - ASSERT(upstream_index.has_value()); + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + ASSERT(upstream_index.has_value()); - envoy::config::endpoint::v3::LbEndpoint route_endpoint; - setUpstreamAddress(upstream_index.value(), route_endpoint); - const std::string route_address_string = fmt::format( - "127.0.0.1:{}", route_endpoint.endpoint().address().socket_address().port_value()); - const std::string route_encoded_address = - Envoy::Base64::encode(route_address_string.data(), route_address_string.size()); + envoy::config::endpoint::v3::LbEndpoint route_endpoint; + setUpstreamAddress(upstream_index.value(), route_endpoint); + const std::string route_address_string = fmt::format( + "127.0.0.1:{}", route_endpoint.endpoint().address().socket_address().port_value()); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - EXPECT_EQ( - Envoy::Http::Utility::makeSetCookieValue("route-session-cookie", route_encoded_address, - "/test", std::chrono::seconds(120), true), - response->headers().get(Http::LowerCaseString("set-cookie"))[0]->value().getStringView()); + if (use_proto) { + EXPECT_EQ(ProtoCookieObject(response->headers() + .get(Http::LowerCaseString("set-cookie"))[0] + ->value() + .getStringView()), + ProtoCookieObject(route_address_string, 120, "/test", "HttpOnly")); + } else { + const std::string route_encoded_address = + Envoy::Base64::encode(route_address_string.data(), route_address_string.size()); + EXPECT_EQ(Envoy::Http::Utility::makeSetCookieValue("route-session-cookie", + route_encoded_address, "/test", + std::chrono::seconds(120), true), + response->headers() + .get(Http::LowerCaseString("set-cookie"))[0] + ->value() + .getStringView()); + } - cleanupUpstreamAndDownstream(); - } - { - envoy::config::endpoint::v3::LbEndpoint endpoint; - setUpstreamAddress(2, endpoint); - const std::string address_string = - fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); - const std::string encoded_address = - Envoy::Base64::encode(address_string.data(), address_string.size()); + cleanupUpstreamAndDownstream(); + } + { + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(2, endpoint); + std::string address_string; + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address(std::string(fmt::format( + "127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()))); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&address_string); + } else { + address_string = fmt::format("127.0.0.1:{}", + endpoint.endpoint().address().socket_address().port_value()); + } + const std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); - codec_client_ = makeHttpConnection(lookupPort("http")); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, - {":path", "/test"}, - {":scheme", "http"}, - {":authority", "stateful.session.com"}, - {"cookie", fmt::format("route-session-cookie=\"{}\"", encoded_address)}}; + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("route-session-cookie=\"{}\"", encoded_address)}}; - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - // Stateful session is overridden and the upstream with index 2 should be selected.. - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - EXPECT_EQ(upstream_index.value(), 2); + // Stateful session is overridden and the upstream with index 2 should be selected.. + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + EXPECT_EQ(upstream_index.value(), 2); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - // No response header to be added. - EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); - cleanupUpstreamAndDownstream(); + cleanupUpstreamAndDownstream(); + } } } @@ -742,75 +905,92 @@ TEST_F(StatefulSessionIntegrationTest, HeaderStatefulSessionOverriddenByRoute) { TEST_F(StatefulSessionIntegrationTest, CookieBasedStatefulSessionDisabledByRequestPath) { initializeFilterAndRoute(STATEFUL_SESSION_FILTER, ""); - { - uint64_t first_index = 0; - uint64_t second_index = 0; + // Run the test twice. Once with proto cookie encoding and once with "old", non-proto + // encoding. + for (const bool use_proto : std::vector({true, false})) { + Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.stateful_session_encode_ttl_in_cookie", + use_proto); + { + uint64_t first_index = 0; + uint64_t second_index = 0; - envoy::config::endpoint::v3::LbEndpoint endpoint; - setUpstreamAddress(1, endpoint); - const std::string address_string = - fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); - const std::string encoded_address = - Envoy::Base64::encode(address_string.data(), address_string.size()); + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(1, endpoint); + std::string address_string; + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address(std::string(fmt::format( + "127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()))); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&address_string); + } else { + address_string = fmt::format("127.0.0.1:{}", + endpoint.endpoint().address().socket_address().port_value()); + } + const std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); - // Request path is not start with cookie path which means that the stateful session cookie in - // the request my not generated by current filter. The stateful session will skip processing - // this request. - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, - {":path", "/path_not_match"}, - {":scheme", "http"}, - {":authority", "stateful.session.com"}, - {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + // Request path is not start with cookie path which means that the stateful session cookie in + // the request my not generated by current filter. The stateful session will skip processing + // this request. + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/path_not_match"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; - { - codec_client_ = makeHttpConnection(lookupPort("http")); + { + codec_client_ = makeHttpConnection(lookupPort("http")); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - ASSERT(upstream_index.has_value()); - first_index = upstream_index.value(); + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + ASSERT(upstream_index.has_value()); + first_index = upstream_index.value(); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - // No response header to be added. - EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); - cleanupUpstreamAndDownstream(); - } + cleanupUpstreamAndDownstream(); + } - { - codec_client_ = makeHttpConnection(lookupPort("http")); + { + codec_client_ = makeHttpConnection(lookupPort("http")); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); - ASSERT(upstream_index.has_value()); - second_index = upstream_index.value(); + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + ASSERT(upstream_index.has_value()); + second_index = upstream_index.value(); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - // No response header to be added. - EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); - cleanupUpstreamAndDownstream(); - } + cleanupUpstreamAndDownstream(); + } - // Choose different upstream servers by default. - EXPECT_NE(first_index, second_index); + // Choose different upstream servers by default. + EXPECT_NE(first_index, second_index); + } } - { uint64_t first_index = 0; uint64_t second_index = 0; @@ -880,6 +1060,98 @@ TEST_F(StatefulSessionIntegrationTest, CookieBasedStatefulSessionDisabledByReque } } +// Test verifies that if a client sends an invalid cookie in "old", non-proto format, the +// reply is in the new proto format. +TEST_F(StatefulSessionIntegrationTest, CookieBasedStatefulSessionBackwardCompatibility) { + initializeFilterAndRoute(STATEFUL_SESSION_FILTER, ""); + Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.stateful_session_encode_ttl_in_cookie", + true); + std::string address_string = "127.0.0.1:50000"; + std::string encoded_address = Envoy::Base64::encode(address_string.data(), address_string.size()); + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + ASSERT(upstream_index.has_value()); + + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(upstream_index.value(), endpoint); + address_string = + fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ( + ProtoCookieObject( + response->headers().get(Http::LowerCaseString("set-cookie"))[0]->value().getStringView()), + ProtoCookieObject(address_string, 120, "/test", "HttpOnly")); + + cleanupUpstreamAndDownstream(); +} + +// Test verifies that sending expired cookie results in selection of new upstream host +// and replying with a new cookie in response headers. +TEST_F(StatefulSessionIntegrationTest, CookieBasedStatefulSessionRejectExpiredCookie) { + initializeFilterAndRoute(STATEFUL_SESSION_FILTER, ""); + Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.stateful_session_encode_ttl_in_cookie", + true); + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(1, endpoint); + // Create already expired cookie. + envoy::Cookie cookie; + cookie.set_address(std::string( + fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()))); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() - + 10); + std::string address_string; + cookie.SerializeToString(&address_string); + std::string encoded_address = Envoy::Base64::encode(address_string.data(), address_string.size()); + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + ASSERT(upstream_index.has_value()); + + setUpstreamAddress(upstream_index.value(), endpoint); + address_string = + fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + // response headers should contain a new cookie. + EXPECT_EQ( + ProtoCookieObject( + response->headers().get(Http::LowerCaseString("set-cookie"))[0]->value().getStringView()), + ProtoCookieObject(address_string, 120, "/test", "HttpOnly")); + + cleanupUpstreamAndDownstream(); +} + } // namespace } // namespace StatefulSession } // namespace HttpFilters diff --git a/test/extensions/http/stateful_session/cookie/BUILD b/test/extensions/http/stateful_session/cookie/BUILD index 491d34207e30..18d2caa24c24 100644 --- a/test/extensions/http/stateful_session/cookie/BUILD +++ b/test/extensions/http/stateful_session/cookie/BUILD @@ -19,6 +19,7 @@ envoy_extension_cc_test( "//source/common/http:utility_lib", "//source/extensions/http/stateful_session/cookie:cookie_lib", "//test/mocks/upstream:host_mocks", + "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/http/stateful_session/cookie/cookie_test.cc b/test/extensions/http/stateful_session/cookie/cookie_test.cc index 81d91bfdb532..4420b98bb856 100644 --- a/test/extensions/http/stateful_session/cookie/cookie_test.cc +++ b/test/extensions/http/stateful_session/cookie/cookie_test.cc @@ -2,6 +2,7 @@ #include "source/extensions/http/stateful_session/cookie/cookie.h" #include "test/mocks/upstream/host.h" +#include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -15,22 +16,25 @@ namespace { TEST(CookieBasedSessionStateFactoryTest, EmptyCookieName) { CookieBasedSessionStateProto config; + Event::SimulatedTimeSystem time_simulator; - EXPECT_THROW_WITH_MESSAGE(std::make_shared(config), - EnvoyException, - "Cookie key cannot be empty for cookie based stateful sessions"); + EXPECT_THROW_WITH_MESSAGE( + std::make_shared(config, time_simulator), EnvoyException, + "Cookie key cannot be empty for cookie based stateful sessions"); config.mutable_cookie()->set_name("override_host"); - EXPECT_NO_THROW(std::make_shared(config)); + EXPECT_NO_THROW(std::make_shared(config, time_simulator)); } TEST(CookieBasedSessionStateFactoryTest, SessionStateTest) { testing::NiceMock mock_host; + Event::SimulatedTimeSystem time_simulator; + time_simulator.setMonotonicTime(std::chrono::seconds(1000)); { CookieBasedSessionStateProto config; config.mutable_cookie()->set_name("override_host"); - CookieBasedSessionStateFactory factory(config); + CookieBasedSessionStateFactory factory(config, time_simulator); // No valid address in the request headers. Envoy::Http::TestRequestHeaderMapImpl request_headers; @@ -38,16 +42,34 @@ TEST(CookieBasedSessionStateFactoryTest, SessionStateTest) { EXPECT_EQ(absl::nullopt, session_state->upstreamAddress()); auto upstream_host = std::make_shared("1.2.3.4", 80); - EXPECT_CALL(mock_host, address()).WillOnce(testing::Return(upstream_host)); - - Envoy::Http::TestResponseHeaderMapImpl response_headers; - session_state->onUpdate(mock_host, response_headers); + EXPECT_CALL(mock_host, address()).Times(2).WillRepeatedly(testing::Return(upstream_host)); // No valid address then update it by set-cookie. - EXPECT_EQ(response_headers.get_("set-cookie"), - Envoy::Http::Utility::makeSetCookieValue("override_host", - Envoy::Base64::encode("1.2.3.4:80", 10), "", - std::chrono::seconds(0), true)); + // Run the test twice: once with proto cookie format and once + // with "old" style plain address. + for (bool use_proto : std::vector({true, false})) { + std::string cookie_content; + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address("1.2.3.4:80"); + cookie.set_expires(1000); + cookie.SerializeToString(&cookie_content); + } else { + cookie_content = "1.2.3.4:80"; + } + Runtime::maybeSetRuntimeGuard( + "envoy.reloadable_features.stateful_session_encode_ttl_in_cookie", use_proto); + + Envoy::Http::TestResponseHeaderMapImpl response_headers; + // Check the format of the cookie sent back to client. + session_state->onUpdate(mock_host, response_headers); + + EXPECT_EQ(response_headers.get_("set-cookie"), + Envoy::Http::Utility::makeSetCookieValue( + "override_host", + Envoy::Base64::encode(cookie_content.c_str(), cookie_content.length()), "", + std::chrono::seconds(0), true)); + } } { @@ -55,43 +77,72 @@ TEST(CookieBasedSessionStateFactoryTest, SessionStateTest) { config.mutable_cookie()->set_name("override_host"); config.mutable_cookie()->set_path("/path"); config.mutable_cookie()->mutable_ttl()->set_seconds(5); - CookieBasedSessionStateFactory factory(config); + CookieBasedSessionStateFactory factory(config, time_simulator); - // Get upstream address from request headers. - Envoy::Http::TestRequestHeaderMapImpl request_headers = { - {":path", "/path"}, {"cookie", "override_host=" + Envoy::Base64::encode("1.2.3.4:80", 10)}}; - auto session_state = factory.create(request_headers); - EXPECT_EQ("1.2.3.4:80", session_state->upstreamAddress().value()); + // Test the following scenario: + // Cookie indicates to route to 1.2.3.4:80 + // Upstream cluster routed to 1.2.3.4.:80. "set-cookie" should not be added to response headers. + // Repeat, but cluster routed to different host 2.3.4.5:80. "set-cookie" should be added to + // response headers. - auto upstream_host = std::make_shared("1.2.3.4", 80); - EXPECT_CALL(mock_host, address()).WillOnce(testing::Return(upstream_host)); + // Run the test twice - once with PROTO style cookie and once with "old" style plain address. + for (bool use_proto : std::vector({true, false})) { + std::string cookie_content; + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address("1.2.3.4:80"); + cookie.set_expires(1005); + cookie.SerializeToString(&cookie_content); + } else { + cookie_content = "1.2.3.4:80"; + } + Runtime::maybeSetRuntimeGuard( + "envoy.reloadable_features.stateful_session_encode_ttl_in_cookie", use_proto); + Envoy::Http::TestRequestHeaderMapImpl request_headers = { + {":path", "/path"}, + {"cookie", "override_host=" + + Envoy::Base64::encode(cookie_content.c_str(), cookie_content.length())}}; + auto session_state = factory.create(request_headers); + EXPECT_EQ("1.2.3.4:80", session_state->upstreamAddress().value()); - Envoy::Http::TestResponseHeaderMapImpl response_headers; - session_state->onUpdate(mock_host, response_headers); + auto upstream_host = std::make_shared("1.2.3.4", 80); + EXPECT_CALL(mock_host, address()).WillOnce(testing::Return(upstream_host)); - // Session state is not updated and then do nothing. - EXPECT_EQ(response_headers.get_("set-cookie"), ""); + Envoy::Http::TestResponseHeaderMapImpl response_headers; + session_state->onUpdate(mock_host, response_headers); - auto upstream_host_2 = std::make_shared("2.3.4.5", 80); - EXPECT_CALL(mock_host, address()).WillOnce(testing::Return(upstream_host_2)); + // Session state is not updated and then do nothing. + EXPECT_EQ(response_headers.get_("set-cookie"), ""); - session_state->onUpdate(mock_host, response_headers); + auto upstream_host_2 = std::make_shared("2.3.4.5", 80); + EXPECT_CALL(mock_host, address()).WillOnce(testing::Return(upstream_host_2)); - // Update session state because the current request is routed to a new upstream host. - EXPECT_EQ(response_headers.get_("set-cookie"), - Envoy::Http::Utility::makeSetCookieValue("override_host", - Envoy::Base64::encode("2.3.4.5:80", 10), - "/path", std::chrono::seconds(5), true)); - } + session_state->onUpdate(mock_host, response_headers); + // Update session state because the current request is routed to a new upstream host. + if (use_proto) { + envoy::Cookie cookie; + cookie.set_address("2.3.4.5:80"); + cookie.set_expires(1005); + cookie.SerializeToString(&cookie_content); + } else { + cookie_content = "2.3.4.5:80"; + } + EXPECT_EQ(response_headers.get_("set-cookie"), + Envoy::Http::Utility::makeSetCookieValue( + "override_host", + Envoy::Base64::encode(cookie_content.c_str(), cookie_content.length()), "/path", + std::chrono::seconds(5), true)); + } + } { CookieBasedSessionStateProto config; config.mutable_cookie()->set_name("override_host"); config.mutable_cookie()->set_path("/path"); config.mutable_cookie()->mutable_ttl()->set_seconds(5); - CookieBasedSessionStateFactory factory(config); + CookieBasedSessionStateFactory factory(config, time_simulator); - // Get upstream address from request headers. + // Get upstream address from request headers' cookie. Envoy::Http::TestRequestHeaderMapImpl request_headers = { {":path", "/not_match_path"}, {"cookie", "override_host=" + Envoy::Base64::encode("1.2.3.4:80", 10)}}; @@ -100,13 +151,56 @@ TEST(CookieBasedSessionStateFactoryTest, SessionStateTest) { } } +TEST(CookieBasedSessionStateFactoryTest, SessionStateProtoCookie) { + CookieBasedSessionStateProto config; + config.mutable_cookie()->set_name("override_host"); + config.mutable_cookie()->set_path("/path"); + config.mutable_cookie()->mutable_ttl()->set_seconds(5); + Event::SimulatedTimeSystem time_simulator; + time_simulator.setMonotonicTime(std::chrono::seconds(1000)); + CookieBasedSessionStateFactory factory(config, time_simulator); + + std::string cookie_content; + envoy::Cookie cookie; + cookie.set_address("2.3.4.5:80"); + cookie.set_expires(1005); + cookie.SerializeToString(&cookie_content); + // PROTO format - expired cookie + time_simulator.setMonotonicTime(std::chrono::seconds(1006)); + Envoy::Http::TestRequestHeaderMapImpl request_headers = { + {":path", "/path"}, + {"cookie", + "override_host=" + Envoy::Base64::encode(cookie_content.c_str(), cookie_content.length())}}; + auto session_state = factory.create(request_headers); + EXPECT_EQ(absl::nullopt, session_state->upstreamAddress()); + + // PROTO format - no "expired field" + cookie.set_expires(0); + cookie.SerializeToString(&cookie_content); + request_headers = {{":path", "/path"}, + {"cookie", "override_host=" + Envoy::Base64::encode(cookie_content.c_str(), + cookie_content.length())}}; + session_state = factory.create(request_headers); + EXPECT_EQ(absl::nullopt, session_state->upstreamAddress()); + + // PROTO format - pass incorrect format. + // The content should be treated as "old" style encoding. + cookie_content = "blahblah"; + request_headers = {{":path", "/path"}, + {"cookie", "override_host=" + Envoy::Base64::encode(cookie_content.c_str(), + cookie_content.length())}}; + session_state = factory.create(request_headers); + EXPECT_EQ("blahblah", session_state->upstreamAddress()); +} + TEST(CookieBasedSessionStateFactoryTest, SessionStatePathMatchTest) { + Event::SimulatedTimeSystem time_simulator; { // Any request path will be accepted for empty cookie path. CookieBasedSessionStateProto config; config.mutable_cookie()->set_name("override_host"); config.mutable_cookie()->mutable_ttl()->set_seconds(5); - CookieBasedSessionStateFactory factory(config); + CookieBasedSessionStateFactory factory(config, time_simulator); EXPECT_TRUE(factory.requestPathMatch("/")); EXPECT_TRUE(factory.requestPathMatch("/foo")); @@ -123,7 +217,7 @@ TEST(CookieBasedSessionStateFactoryTest, SessionStatePathMatchTest) { config.mutable_cookie()->set_name("override_host"); config.mutable_cookie()->set_path("/"); config.mutable_cookie()->mutable_ttl()->set_seconds(5); - CookieBasedSessionStateFactory factory(config); + CookieBasedSessionStateFactory factory(config, time_simulator); EXPECT_TRUE(factory.requestPathMatch("/")); EXPECT_TRUE(factory.requestPathMatch("/foo")); @@ -140,7 +234,7 @@ TEST(CookieBasedSessionStateFactoryTest, SessionStatePathMatchTest) { config.mutable_cookie()->set_name("override_host"); config.mutable_cookie()->set_path("/foo/"); config.mutable_cookie()->mutable_ttl()->set_seconds(5); - CookieBasedSessionStateFactory factory(config); + CookieBasedSessionStateFactory factory(config, time_simulator); EXPECT_FALSE(factory.requestPathMatch("/")); EXPECT_FALSE(factory.requestPathMatch("/foo")); @@ -157,7 +251,7 @@ TEST(CookieBasedSessionStateFactoryTest, SessionStatePathMatchTest) { config.mutable_cookie()->set_name("override_host"); config.mutable_cookie()->set_path("/foo"); config.mutable_cookie()->mutable_ttl()->set_seconds(5); - CookieBasedSessionStateFactory factory(config); + CookieBasedSessionStateFactory factory(config, time_simulator); EXPECT_FALSE(factory.requestPathMatch("/")); EXPECT_TRUE(factory.requestPathMatch("/foo")); diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index d983a4ac0a4a..a556ff5a71ba 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -156,6 +156,7 @@ paths: - test/test_common/test_time.h - test/test_common/utility.cc - test/tools/wee8_compile/wee8_compile.cc + - test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc # Tests in these paths may make use of the Registry::RegisterFactory constructor or the # REGISTER_FACTORY macro. Other locations should use the InjectFactory helper class to From 11ebacd6f1136541896b420022f59a357ebe4629 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 10 May 2023 16:02:52 +0100 Subject: [PATCH 192/740] deps: Bump `bazel` -> 6.2.0 (#27315) Signed-off-by: Ryan Northey --- .azure-pipelines/env.yml | 2 +- .azure-pipelines/pipelines.yml | 2 +- .bazelversion | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/env.yml b/.azure-pipelines/env.yml index aa9d16de691a..82553c615d66 100644 --- a/.azure-pipelines/env.yml +++ b/.azure-pipelines/env.yml @@ -105,7 +105,7 @@ jobs: echo "##vso[task.setvariable variable=docsOnly;isoutput=true]${CHANGED_DOCS_ONLY}" echo "##vso[task.setvariable variable=examplesOnly;isoutput=true]${CHANGED_EXAMPLES_ONLY}" - CHANGED_REQUIREMENT_PATHS="$(git diff --name-only ${CHANGE_TARGET}...HEAD | grep -E '\.*requirements.*\.txt$|go.mod$|\.*\.bzl$|^WORKSPACE$' || echo '')" + CHANGED_REQUIREMENT_PATHS="$(git diff --name-only ${CHANGE_TARGET}...HEAD | grep -E '\.bazelversion|\.*requirements.*\.txt$|go.mod$|\.*\.bzl$|^WORKSPACE$' || echo '')" if [[ -n "$CHANGED_REQUIREMENT_PATHS" ]]; then CHANGED_REQUIREMENTS=true diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index e7bbb8c78070..3c22420d6599 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -48,7 +48,7 @@ variables: - name: cacheKeyBazel value: v0 - name: cacheKeyBazelFiles - value: './WORKSPACE | **/*.bzl, !mobile/**, !envoy-docs/**' + value: '.bazelversion | ./WORKSPACE | **/*.bzl, !mobile/**, !envoy-docs/**' - name: authGithubSSHKeyPublic value: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" diff --git a/.bazelversion b/.bazelversion index dfda3e0b4f01..6abaeb2f9072 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.1.0 +6.2.0 From 1bd16ecb51a641f6823a243ed8e3b516cb7dca0a Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 10 May 2023 11:08:38 -0400 Subject: [PATCH 193/740] Increase CPU for more integration tests (#27300) * Increase CPU for more integration tests Signed-off-by: Yan Avlasov --- test/extensions/filters/http/ext_proc/BUILD | 6 ++++++ test/extensions/filters/http/grpc_json_transcoder/BUILD | 3 +++ test/integration/BUILD | 9 +++++++++ 3 files changed, 18 insertions(+) diff --git a/test/extensions/filters/http/ext_proc/BUILD b/test/extensions/filters/http/ext_proc/BUILD index 26fc4422245b..247a3979fb76 100644 --- a/test/extensions/filters/http/ext_proc/BUILD +++ b/test/extensions/filters/http/ext_proc/BUILD @@ -117,6 +117,9 @@ envoy_extension_cc_test( srcs = ["ext_proc_integration_test.cc"], extension_names = ["envoy.filters.http.ext_proc"], shard_count = 2, + tags = [ + "cpu:3", + ], deps = [ ":utils_lib", "//source/extensions/filters/http/ext_proc:config", @@ -133,6 +136,9 @@ envoy_extension_cc_test( size = "large", srcs = ["streaming_integration_test.cc"], extension_names = ["envoy.filters.http.ext_proc"], + tags = [ + "cpu:3", + ], deps = [ ":test_processor_lib", ":utils_lib", diff --git a/test/extensions/filters/http/grpc_json_transcoder/BUILD b/test/extensions/filters/http/grpc_json_transcoder/BUILD index 2900926a79f6..6124039540a2 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/BUILD +++ b/test/extensions/filters/http/grpc_json_transcoder/BUILD @@ -61,6 +61,9 @@ envoy_extension_cc_test( "//test/proto:bookstore_proto_descriptor", ], extension_names = ["envoy.filters.http.grpc_json_transcoder"], + tags = [ + "cpu:3", + ], deps = [ "//source/common/grpc:codec_lib", "//source/common/http:header_map_lib", diff --git a/test/integration/BUILD b/test/integration/BUILD index 2be58b4d8685..acc5558902c6 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -217,6 +217,9 @@ envoy_cc_test( srcs = [ "upstream_access_log_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":integration_lib", @@ -2016,6 +2019,9 @@ envoy_cc_test( name = "cx_limit_integration_test", size = "large", srcs = ["cx_limit_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//envoy/network:filter_interface", @@ -2031,6 +2037,9 @@ envoy_cc_test( name = "direct_response_integration_test", size = "large", srcs = ["direct_response_integration_test.cc"], + tags = [ + "cpu:2", + ], deps = [ ":http_integration_lib", ], From 1829578f58800dee4b69ebc3b860ccfd95087148 Mon Sep 17 00:00:00 2001 From: code Date: Wed, 10 May 2023 23:45:39 +0800 Subject: [PATCH 194/740] ext_authz: ensure both raw body/body will be used for http authorization request (#27228) * ext_authz: ensure both raw body/body will be used for http authorization request Signed-off-by: wbpcode * add change log Signed-off-by: wbpcode --------- Signed-off-by: wbpcode --- changelogs/current.yaml | 5 ++++ .../common/ext_authz/ext_authz_http_impl.cc | 12 ++++++-- .../ext_authz/ext_authz_http_impl_test.cc | 28 ++++++++++++++++++- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 381512f4f4bc..22530a903b27 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -68,6 +68,11 @@ bug_fixes: - area: tls change: | Fix build FIPS compliance when using both FIPS mode and Wasm extensions (``--define boringssl=fips`` and ``--define wasm=v8``). +- area: ext_authz + change: | + Fix a bug where the ext_authz filter will ignore the request body when the + :ref:`pack_as_bytes ` is set to true and + HTTP authorization service is configured. - area: router change: | Fixed the bug that updating :ref:`scope_key_builder diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc index 51f8ca5541f9..4a519b5050cd 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc @@ -191,7 +191,13 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks, callbacks_ = &callbacks; Http::RequestHeaderMapPtr headers; - const uint64_t request_length = request.attributes().request().http().body().size(); + + const auto& http_request = request.attributes().request().http(); + const auto& http_request_body = + http_request.body().empty() ? http_request.raw_body() : http_request.body(); + + uint64_t request_length = http_request_body.size(); + if (request_length > 0) { headers = Http::createHeaderMap( {{Http::Headers::get().ContentLength, std::to_string(request_length)}}); @@ -199,7 +205,7 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks, headers = Http::createHeaderMap(lengthZeroHeader()); } - for (const auto& header : request.attributes().request().http().headers()) { + for (const auto& header : http_request.headers()) { const Http::LowerCaseString key{header.first}; // Skip setting content-length header since it is already configured at initialization. @@ -219,7 +225,7 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks, Http::RequestMessagePtr message = std::make_unique(std::move(headers)); if (request_length > 0) { - message->body().add(request.attributes().request().http().body()); + message->body().add(http_request_body); } const std::string& cluster = config_->cluster(); diff --git a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc index fbf3225a6f29..3ab57375af02 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc @@ -139,7 +139,9 @@ class ExtAuthzHttpClientTest : public testing::Test { client_->onSuccess(async_request_, std::move(http_response)); } - Http::RequestMessagePtr sendRequest(absl::node_hash_map&& headers) { + Http::RequestMessagePtr sendRequest(absl::node_hash_map&& headers, + const std::string body_content = EMPTY_STRING, + bool use_raw_body = false) { envoy::service::auth::v3::CheckRequest request{}; auto mutable_headers = request.mutable_attributes()->mutable_request()->mutable_http()->mutable_headers(); @@ -147,6 +149,14 @@ class ExtAuthzHttpClientTest : public testing::Test { (*mutable_headers)[header.first] = header.second; } + if (use_raw_body) { + *request.mutable_attributes()->mutable_request()->mutable_http()->mutable_raw_body() = + body_content; + } else { + *request.mutable_attributes()->mutable_request()->mutable_http()->mutable_body() = + body_content; + } + Http::RequestMessagePtr message_ptr; EXPECT_CALL(async_client_, send_(_, _, _)) .WillOnce(Invoke( @@ -237,6 +247,22 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationOkWithPathRewrite) { EXPECT_EQ(message_ptr->headers().getPathValue(), "/bar/foo"); } +// Verify request body is set correctly when the normal body is empty and raw body is set. +TEST_F(ExtAuthzHttpClientTest, AuthorizationOkWithRawBody) { + Http::RequestMessagePtr message_ptr = + sendRequest({{":path", "/foo"}, {"foo", "bar"}}, "raw_body", true); + + EXPECT_EQ(message_ptr->bodyAsString(), "raw_body"); +} + +// Verify request body is set correctly when the normal body is set and raw body is empty. +TEST_F(ExtAuthzHttpClientTest, AuthorizationOkWithBody) { + Http::RequestMessagePtr message_ptr = + sendRequest({{":path", "/foo"}, {"foo", "bar"}}, "body", false); + + EXPECT_EQ(message_ptr->bodyAsString(), "body"); +} + // Test the client when a request contains Content-Length greater than 0. TEST_F(ExtAuthzHttpClientTest, ContentLengthEqualZero) { Http::RequestMessagePtr message_ptr = From 23168f3cc7ea9693e12cfda065268c5e4957c570 Mon Sep 17 00:00:00 2001 From: Chris Schoener Date: Wed, 10 May 2023 12:14:34 -0400 Subject: [PATCH 195/740] [Mobile] Use SystemHelper abstraction for remaining mobile C++ integration tests (#27289) This is a follow-up to #27237 applying the same fix to the remaining 2 tests which needed it. Tested by forcing the system_helper to use android, verifying tests fail, and then fixing it via this change. Signed-off-by: caschoener --- mobile/test/common/BUILD | 1 + mobile/test/common/http/BUILD | 1 + mobile/test/common/http/client_test.cc | 7 ++++++ mobile/test/common/main_interface_test.cc | 29 +++++++++++++++++------ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/mobile/test/common/BUILD b/mobile/test/common/BUILD index 7310e348ad1b..04b501f95f2b 100644 --- a/mobile/test/common/BUILD +++ b/mobile/test/common/BUILD @@ -36,6 +36,7 @@ envoy_cc_test( "//library/common/data:utility_lib", "//library/common/http:header_utility_lib", "//library/common/types:c_types_lib", + "//test/common/mocks/common:common_mocks", "//test/common/mocks/event:event_mocks", "@envoy//test/common/http:common_lib", "@envoy_build_config//:test_extensions", diff --git a/mobile/test/common/http/BUILD b/mobile/test/common/http/BUILD index c190fd0df881..8a76a97f4b32 100644 --- a/mobile/test/common/http/BUILD +++ b/mobile/test/common/http/BUILD @@ -12,6 +12,7 @@ envoy_cc_test( "//library/common/http:client_lib", "//library/common/http:header_utility_lib", "//library/common/types:c_types_lib", + "//test/common/mocks/common:common_mocks", "//test/common/mocks/event:event_mocks", "@envoy//source/common/http:context_lib", "@envoy//source/common/stats:isolated_store_lib", diff --git a/mobile/test/common/http/client_test.cc b/mobile/test/common/http/client_test.cc index 975653514f14..abba6690b940 100644 --- a/mobile/test/common/http/client_test.cc +++ b/mobile/test/common/http/client_test.cc @@ -5,6 +5,7 @@ #include "source/common/stats/isolated_store_impl.h" #include "test/common/http/common.h" +#include "test/common/mocks/common/mocks.h" #include "test/common/mocks/event/mocks.h" #include "test/mocks/buffer/mocks.h" #include "test/mocks/event/mocks.h" @@ -111,6 +112,9 @@ class ClientTest : public testing::TestWithParam { cc->on_trailers_calls++; return nullptr; }; + helper_handle_ = test::SystemHelperPeer::replaceSystemHelper(); + EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)) + .WillRepeatedly(Return(true)); } envoy_headers defaultRequestHeaders() { @@ -155,6 +159,9 @@ class ClientTest : public testing::TestWithParam { bool explicit_flow_control_{GetParam()}; Client http_client_{api_listener_, dispatcher_, *stats_store_.rootScope(), random_}; envoy_stream_t stream_ = 1; + +protected: + std::unique_ptr helper_handle_; }; INSTANTIATE_TEST_SUITE_P(TestModes, ClientTest, ::testing::Bool()); diff --git a/mobile/test/common/main_interface_test.cc b/mobile/test/common/main_interface_test.cc index 372b35c438ae..ba49f648988d 100644 --- a/mobile/test/common/main_interface_test.cc +++ b/mobile/test/common/main_interface_test.cc @@ -1,6 +1,7 @@ #include "source/common/common/assert.h" #include "test/common/http/common.h" +#include "test/common/mocks/common/mocks.h" #include "absl/synchronization/notification.h" #include "gmock/gmock.h" @@ -13,6 +14,8 @@ using testing::_; using testing::HasSubstr; +using testing::Return; +using testing::ReturnRef; namespace Envoy { @@ -131,7 +134,19 @@ Http::ResponseHeaderMapPtr toResponseHeaders(envoy_headers headers) { return transformed_headers; } -TEST(MainInterfaceTest, BasicStream) { +class MainInterfaceTest : public testing::Test { +public: + void SetUp() override { + helper_handle_ = test::SystemHelperPeer::replaceSystemHelper(); + EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)) + .WillRepeatedly(Return(true)); + } + +protected: + std::unique_ptr helper_handle_; +}; + +TEST_F(MainInterfaceTest, BasicStream) { const std::string level = "debug"; engine_test_context engine_cbs_context{}; envoy_engine_callbacks engine_cbs{[](void* context) -> void { @@ -195,7 +210,7 @@ TEST(MainInterfaceTest, BasicStream) { ASSERT_TRUE(engine_cbs_context.on_exit.WaitForNotificationWithTimeout(absl::Seconds(10))); } -TEST(MainInterfaceTest, SendMetadata) { +TEST_F(MainInterfaceTest, SendMetadata) { engine_test_context engine_cbs_context{}; envoy_engine_callbacks engine_cbs{[](void* context) -> void { auto* engine_running = @@ -235,7 +250,7 @@ TEST(MainInterfaceTest, SendMetadata) { ASSERT_TRUE(engine_cbs_context.on_exit.WaitForNotificationWithTimeout(absl::Seconds(10))); } -TEST(MainInterfaceTest, ResetStream) { +TEST_F(MainInterfaceTest, ResetStream) { engine_test_context engine_cbs_context{}; envoy_engine_callbacks engine_cbs{[](void* context) -> void { auto* engine_running = @@ -285,7 +300,7 @@ TEST(MainInterfaceTest, ResetStream) { ASSERT_TRUE(engine_cbs_context.on_exit.WaitForNotificationWithTimeout(absl::Seconds(10))); } -TEST(MainInterfaceTest, UsingMainInterfaceWithoutARunningEngine) { +TEST_F(MainInterfaceTest, UsingMainInterfaceWithoutARunningEngine) { Http::TestRequestHeaderMapImpl headers; HttpTestUtility::addDefaultHeaders(headers); envoy_headers c_headers = Http::Utility::toBridgeHeaders(headers); @@ -307,7 +322,7 @@ TEST(MainInterfaceTest, UsingMainInterfaceWithoutARunningEngine) { release_envoy_headers(c_trailers); } -TEST(MainInterfaceTest, RegisterPlatformApi) { +TEST_F(MainInterfaceTest, RegisterPlatformApi) { engine_test_context engine_cbs_context{}; envoy_engine_callbacks engine_cbs{[](void* context) -> void { auto* engine_running = @@ -335,7 +350,7 @@ TEST(MainInterfaceTest, RegisterPlatformApi) { ASSERT_TRUE(engine_cbs_context.on_exit.WaitForNotificationWithTimeout(absl::Seconds(10))); } -TEST(MainInterfaceTest, PreferredNetwork) { +TEST_F(MainInterfaceTest, PreferredNetwork) { EXPECT_EQ(ENVOY_SUCCESS, set_preferred_network(0, ENVOY_NET_WLAN)); } @@ -557,7 +572,7 @@ TEST(EngineTest, EventTrackerRegistersEnvoyBugRecordAction) { ASSERT_TRUE(test_context.on_exit.WaitForNotificationWithTimeout(absl::Seconds(3))); } -TEST(MainInterfaceTest, ResetConnectivityState) { +TEST_F(MainInterfaceTest, ResetConnectivityState) { engine_test_context test_context{}; envoy_engine_callbacks engine_cbs{[](void* context) -> void { auto* engine_running = From eec3975dc1ed23b1a1d579c29769c44d9e38c73b Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 11 May 2023 06:02:43 +0100 Subject: [PATCH 196/740] deps: Bump `com_github_google_flatbuffers` -> 23.3.3 (#27302) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index d48acbba5f6d..47e7556eaea3 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1107,8 +1107,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "FlatBuffers", project_desc = "Cross platform serialization library architected for maximum memory efficiency", project_url = "https://github.com/google/flatbuffers", - version = "22.12.06", - sha256 = "209823306f2cbedab6ff70997e0d236fcfd1864ca9ad082cbfdb196e7386daed", + version = "23.3.3", + sha256 = "8aff985da30aaab37edf8e5b02fda33ed4cbdd962699a8e2af98fdef306f4e4d", strip_prefix = "flatbuffers-{version}", urls = ["https://github.com/google/flatbuffers/archive/v{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -1124,7 +1124,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.stat_sinks.wasm", "envoy.rbac.matchers.upstream_ip_port", ], - release_date = "2022-12-07", + release_date = "2023-03-03", cpe = "cpe:2.3:a:google:flatbuffers:*", license = "Apache-2.0", license_url = "https://github.com/google/flatbuffers/blob/v{version}/LICENSE.txt", From 3a13e56b69e37e42f29c0897fd0cdc327fabb533 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 11 May 2023 06:03:00 +0100 Subject: [PATCH 197/740] deps: Bump `com_github_gabime_spdlog` -> 1.11.0 (#27304) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 47e7556eaea3..94d73ae07a8c 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -294,12 +294,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "spdlog", project_desc = "Very fast, header-only/compiled, C++ logging library", project_url = "https://github.com/gabime/spdlog", - version = "1.9.2", - sha256 = "6fff9215f5cb81760be4cc16d033526d1080427d236e86d70bb02994f85e3d38", + version = "1.11.0", + sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb", strip_prefix = "spdlog-{version}", urls = ["https://github.com/gabime/spdlog/archive/v{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2021-08-12", + release_date = "2022-11-02", cpe = "N/A", license = "MIT", license_url = "https://github.com/gabime/spdlog/blob/v{version}/LICENSE", From d67f1532df1522a4ed51e27fe2343efaa3456d1d Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 11 May 2023 09:45:37 +0100 Subject: [PATCH 198/740] ci: Improve presubmit caching (#27341) Signed-off-by: Ryan Northey --- .azure-pipelines/bazel.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index f7a768f862ad..660287576ec7 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -162,7 +162,8 @@ steps: AZP_TARGET_BRANCH: "origin/$(System.PullRequest.TargetBranch)" ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: AZP_TARGET_BRANCH: "origin/$(Build.SourceBranchName)" - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # Any PR or CI run in envoy-presubmit uses the fake SCM hash + ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'envoy-presubmit')) }}: # sha1sum of `ENVOY_PULL_REQUEST` BAZEL_FAKE_SCM_REVISION: e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9 ${{ if parameters.rbe }}: From 52ee63e287cc0606ec8c917a49e4c531aaf14d03 Mon Sep 17 00:00:00 2001 From: code Date: Thu, 11 May 2023 17:53:10 +0800 Subject: [PATCH 199/740] =?UTF-8?q?Revert=20"factory=20context=20refactor:?= =?UTF-8?q?=20refactor=20the=20transport=20factory=20cont=E2=80=A6=20(#273?= =?UTF-8?q?48)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert "factory context refactor: refactor the transport factory context (#27081)" This reverts commit 5b5d13955d576369b5593f6ec29e3cfe42ef9fff. --- .../source/cryptomb_private_key_provider.cc | 10 +- .../private_key_providers/test/config_test.cc | 4 +- .../test/fake_factory.cc | 4 +- .../network/test/postgres_integration_test.cc | 2 +- .../source/qat_private_key_provider.cc | 7 +- .../private_key_providers/test/config_test.cc | 4 +- .../private_key_providers/test/ops_test.cc | 5 +- .../filters/network/source/utility.h | 1 - .../filters/network/test/conn_manager_test.cc | 3 +- .../filters/network/test/router_test.cc | 3 +- contrib/sxg/filters/http/test/filter_test.cc | 9 +- envoy/server/transport_socket_config.h | 72 +++++++-- mobile/test/common/integration/test_server.cc | 5 +- .../quic/quic_transport_socket_factory.cc | 8 +- source/common/secret/sds_api.h | 41 ++--- .../upstream/health_discovery_service.cc | 3 +- source/common/upstream/upstream_impl.cc | 16 +- source/extensions/clusters/eds/leds.cc | 2 +- .../listener_manager/listener_impl.cc | 3 +- .../transport_sockets/alts/config.cc | 7 +- .../internal_upstream/config.cc | 2 +- .../proxy_protocol/config.cc | 2 +- .../transport_sockets/tap/config.cc | 17 +-- .../transport_sockets/tcp_stats/config.cc | 2 +- .../transport_sockets/tls/config.cc | 6 +- .../tls/context_config_impl.cc | 5 +- .../tls/context_config_impl.h | 2 +- source/extensions/upstreams/http/config.h | 2 +- source/server/server.h | 6 +- source/server/transport_socket_config_impl.h | 34 +++-- .../grpc_client_integration_test_harness.h | 2 +- .../quic_transport_socket_factory_test.cc | 2 +- .../common/secret/secret_manager_impl_test.cc | 69 +++++---- test/common/upstream/BUILD | 1 - test/common/upstream/hds_test.cc | 4 +- test/extensions/clusters/eds/leds_test.cc | 2 +- .../filters/http/oauth2/filter_test.cc | 9 +- .../alts/alts_integration_test.cc | 2 +- .../transport_sockets/alts/config_test.cc | 6 +- .../upstream_starttls_integration_test.cc | 2 +- test/extensions/transport_sockets/tls/BUILD | 1 - .../tls/context_impl_test.cc | 35 +++-- .../tls/handshaker_factory_test.cc | 12 +- .../transport_sockets/tls/ssl_certs_test.h | 2 +- .../transport_sockets/tls/ssl_socket_test.cc | 52 +++---- .../tls/test_private_key_method_provider.cc | 3 +- test/integration/base_integration_test.cc | 4 +- .../integration/quic_http_integration_test.cc | 4 +- test/integration/ssl_utility.cc | 4 +- test/integration/utility.cc | 4 +- test/integration/xfcc_integration_test.h | 2 +- test/mocks/server/BUILD | 36 ++--- test/mocks/server/instance.cc | 36 ++++- test/mocks/server/instance.h | 134 ++++++++++++++++- test/mocks/server/server_factory_context.cc | 43 ------ test/mocks/server/server_factory_context.h | 140 ------------------ .../transport_socket_factory_context.cc | 12 +- .../server/transport_socket_factory_context.h | 26 +++- 58 files changed, 493 insertions(+), 443 deletions(-) delete mode 100644 test/mocks/server/server_factory_context.cc delete mode 100644 test/mocks/server/server_factory_context.h diff --git a/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc b/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc index f94f37db6323..e1a7fc96d4ae 100644 --- a/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc +++ b/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc @@ -516,10 +516,9 @@ CryptoMbPrivateKeyMethodProvider::CryptoMbPrivateKeyMethodProvider( const envoy::extensions::private_key_providers::cryptomb::v3alpha:: CryptoMbPrivateKeyMethodConfig& conf, Server::Configuration::TransportSocketFactoryContext& factory_context, IppCryptoSharedPtr ipp) - : api_(factory_context.serverFactoryContext().api()), - tls_(ThreadLocal::TypedSlot::makeUnique( - factory_context.serverFactoryContext().threadLocal())), - stats_(generateCryptoMbStats("cryptomb", factory_context.statsScope())) { + : api_(factory_context.api()), + tls_(ThreadLocal::TypedSlot::makeUnique(factory_context.threadLocal())), + stats_(generateCryptoMbStats("cryptomb", factory_context.scope())) { if (!ipp->mbxIsCryptoMbApplicable(0)) { throw EnvoyException("Multi-buffer CPU instructions not available."); @@ -528,7 +527,8 @@ CryptoMbPrivateKeyMethodProvider::CryptoMbPrivateKeyMethodProvider( std::chrono::milliseconds poll_delay = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(conf, poll_delay, 200)); - std::string private_key = Config::DataSource::read(conf.private_key(), false, api_); + std::string private_key = + Config::DataSource::read(conf.private_key(), false, factory_context.api()); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); diff --git a/contrib/cryptomb/private_key_providers/test/config_test.cc b/contrib/cryptomb/private_key_providers/test/config_test.cc index 158936f49037..370ce3953692 100644 --- a/contrib/cryptomb/private_key_providers/test/config_test.cc +++ b/contrib/cryptomb/private_key_providers/test/config_test.cc @@ -35,8 +35,8 @@ parsePrivateKeyProviderFromV3Yaml(const std::string& yaml_string) { class CryptoMbConfigTest : public Event::TestUsingSimulatedTime, public testing::Test { public: CryptoMbConfigTest() : api_(Api::createApiForTest(store_, time_system_)) { - ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); - ON_CALL(factory_context_.server_context_, threadLocal()).WillByDefault(ReturnRef(tls_)); + ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_, threadLocal()).WillByDefault(ReturnRef(tls_)); ON_CALL(factory_context_, sslContextManager()).WillByDefault(ReturnRef(context_manager_)); ON_CALL(context_manager_, privateKeyMethodManager()) .WillByDefault(ReturnRef(private_key_method_manager_)); diff --git a/contrib/cryptomb/private_key_providers/test/fake_factory.cc b/contrib/cryptomb/private_key_providers/test/fake_factory.cc index ddd2a5d363e8..7ae205f52f9e 100644 --- a/contrib/cryptomb/private_key_providers/test/fake_factory.cc +++ b/contrib/cryptomb/private_key_providers/test/fake_factory.cc @@ -135,8 +135,8 @@ FakeCryptoMbPrivateKeyMethodFactory::createPrivateKeyMethodProviderInstance( std::make_shared(supported_instruction_set_); // We need to get more RSA key params in order to be able to use BoringSSL signing functions. - std::string private_key = Config::DataSource::read( - conf.private_key(), false, private_key_provider_context.serverFactoryContext().api()); + std::string private_key = + Config::DataSource::read(conf.private_key(), false, private_key_provider_context.api()); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); diff --git a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc index 4f1e3c829fb8..6fb93661e68c 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc @@ -313,7 +313,7 @@ class UpstreamSSLBaseIntegrationTest : public PostgresBaseIntegrationTest { TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), downstream_tls_context); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(mock_factory_ctx, api()).WillByDefault(testing::ReturnRef(*api_)); auto cfg = std::make_unique( downstream_tls_context, mock_factory_ctx); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); diff --git a/contrib/qat/private_key_providers/source/qat_private_key_provider.cc b/contrib/qat/private_key_providers/source/qat_private_key_provider.cc index 863637604157..6b2d808d254b 100644 --- a/contrib/qat/private_key_providers/source/qat_private_key_provider.cc +++ b/contrib/qat/private_key_providers/source/qat_private_key_provider.cc @@ -347,9 +347,9 @@ QatPrivateKeyMethodProvider::QatPrivateKeyMethodProvider( const envoy::extensions::private_key_providers::qat::v3alpha::QatPrivateKeyMethodConfig& conf, Server::Configuration::TransportSocketFactoryContext& factory_context, LibQatCryptoSharedPtr libqat) - : api_(factory_context.serverFactoryContext().api()), libqat_(libqat) { + : api_(factory_context.api()), libqat_(libqat) { - manager_ = factory_context.serverFactoryContext().singletonManager().getTyped( + manager_ = factory_context.singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(qat_manager), [libqat] { return std::make_shared(libqat); }); @@ -358,7 +358,8 @@ QatPrivateKeyMethodProvider::QatPrivateKeyMethodProvider( std::chrono::milliseconds poll_delay = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(conf, poll_delay, 5)); - std::string private_key = Config::DataSource::read(conf.private_key(), false, api_); + std::string private_key = + Config::DataSource::read(conf.private_key(), false, factory_context.api()); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); diff --git a/contrib/qat/private_key_providers/test/config_test.cc b/contrib/qat/private_key_providers/test/config_test.cc index 518c9ba22d6a..b5527bc97bc8 100644 --- a/contrib/qat/private_key_providers/test/config_test.cc +++ b/contrib/qat/private_key_providers/test/config_test.cc @@ -48,11 +48,11 @@ class QatConfigTest : public Event::TestUsingSimulatedTime, public testing::Test QatConfigTest() : api_(Api::createApiForTest(store_, time_system_)), libqat_(std::make_shared()), fsm_(libqat_) { - ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); ON_CALL(factory_context_, sslContextManager()).WillByDefault(ReturnRef(context_manager_)); ON_CALL(context_manager_, privateKeyMethodManager()) .WillByDefault(ReturnRef(private_key_method_manager_)); - ON_CALL(factory_context_.server_context_, singletonManager()).WillByDefault(ReturnRef(fsm_)); + ON_CALL(factory_context_, singletonManager()).WillByDefault(ReturnRef(fsm_)); } Ssl::PrivateKeyMethodProviderSharedPtr createWithConfig(std::string yaml) { diff --git a/contrib/qat/private_key_providers/test/ops_test.cc b/contrib/qat/private_key_providers/test/ops_test.cc index 68922f08bfef..0e2f77322107 100644 --- a/contrib/qat/private_key_providers/test/ops_test.cc +++ b/contrib/qat/private_key_providers/test/ops_test.cc @@ -59,9 +59,8 @@ class QatProviderTest : public testing::Test { dispatcher_(api_->allocateDispatcher("test_thread")), libqat_(std::make_shared()), fsm_(libqat_) { handle_.setLibqat(libqat_); - ON_CALL(factory_context_.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); - ON_CALL(factory_context_.server_context_, singletonManager()) - .WillByDefault(testing::ReturnRef(fsm_)); + ON_CALL(factory_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(factory_context_, singletonManager()).WillByDefault(testing::ReturnRef(fsm_)); } Stats::TestUtil::TestStore store_; diff --git a/contrib/sip_proxy/filters/network/source/utility.h b/contrib/sip_proxy/filters/network/source/utility.h index 29465c09cab2..1cc4a69872d0 100644 --- a/contrib/sip_proxy/filters/network/source/utility.h +++ b/contrib/sip_proxy/filters/network/source/utility.h @@ -156,7 +156,6 @@ class Utility { public: static const std::string& localAddress(Server::Configuration::FactoryContext& context) { return context.getTransportSocketFactoryContext() - .serverFactoryContext() .localInfo() .address() ->ip() diff --git a/contrib/sip_proxy/filters/network/test/conn_manager_test.cc b/contrib/sip_proxy/filters/network/test/conn_manager_test.cc index 8cf88bed959f..6b687031d9e3 100644 --- a/contrib/sip_proxy/filters/network/test/conn_manager_test.cc +++ b/contrib/sip_proxy/filters/network/test/conn_manager_test.cc @@ -80,8 +80,7 @@ class SipConnectionManagerTest : public testing::Test { EXPECT_CALL(context_, getTransportSocketFactoryContext()) .WillRepeatedly(testing::ReturnRef(factory_context_)); - EXPECT_CALL(factory_context_.server_context_, localInfo()) - .WillRepeatedly(testing::ReturnRef(local_info_)); + EXPECT_CALL(factory_context_, localInfo()).WillRepeatedly(testing::ReturnRef(local_info_)); ON_CALL(random_, random()).WillByDefault(Return(42)); filter_ = std::make_unique( *config_, random_, filter_callbacks_.connection_.dispatcher_.timeSource(), context_, diff --git a/contrib/sip_proxy/filters/network/test/router_test.cc b/contrib/sip_proxy/filters/network/test/router_test.cc index 9964207138da..f50895c58ead 100644 --- a/contrib/sip_proxy/filters/network/test/router_test.cc +++ b/contrib/sip_proxy/filters/network/test/router_test.cc @@ -105,8 +105,7 @@ class SipRouterTest : public testing::Test { EXPECT_CALL(context_, getTransportSocketFactoryContext()) .WillRepeatedly(testing::ReturnRef(factory_context_)); - EXPECT_CALL(factory_context_.server_context_, localInfo()) - .WillRepeatedly(testing::ReturnRef(local_info_)); + EXPECT_CALL(factory_context_, localInfo()).WillRepeatedly(testing::ReturnRef(local_info_)); transaction_infos_ = std::make_shared(); context_.cluster_manager_.initializeThreadLocalClusters({cluster_name_}); diff --git a/contrib/sxg/filters/http/test/filter_test.cc b/contrib/sxg/filters/http/test/filter_test.cc index 92bf1fa92f9e..5a874067c10a 100644 --- a/contrib/sxg/filters/http/test/filter_test.cc +++ b/contrib/sxg/filters/http/test/filter_test.cc @@ -360,13 +360,14 @@ TEST_F(FilterTest, SdsDynamicGenericSecret) { NiceMock secret_context; NiceMock local_info; Api::ApiPtr api = Api::createApiForTest(); + Stats::IsolatedStoreImpl stats; NiceMock init_manager; Init::TargetHandlePtr init_handle; NiceMock dispatcher; - EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); - EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api)); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); EXPECT_CALL(init_manager, add(_)) .WillRepeatedly(Invoke([&init_handle](const Init::Target& target) { diff --git a/envoy/server/transport_socket_config.h b/envoy/server/transport_socket_config.h index c39723b573ca..756c27f64365 100644 --- a/envoy/server/transport_socket_config.h +++ b/envoy/server/transport_socket_config.h @@ -30,23 +30,14 @@ class TransportSocketFactoryContext { virtual ~TransportSocketFactoryContext() = default; /** - * @return ServerFactoryContext& the server factory context. + * @return ServerFactoryContext which lifetime is no shorter than the server. */ - virtual ServerFactoryContext& serverFactoryContext() PURE; + virtual ServerFactoryContext& getServerFactoryContext() PURE; /** - * @return Upstream::ClusterManager& singleton for use by the entire server. - * TODO(wbpcode): clusterManager() of ServerFactoryContext still be invalid when loading - * static cluster. So we need to provide an cluster manager reference here. - * This could be removed after https://github.com/envoyproxy/envoy/issues/26653 is resolved. + * @return OptRef the global HTTP admin endpoint for the server. */ - virtual Upstream::ClusterManager& clusterManager() PURE; - - /** - * @return ProtobufMessage::ValidationVisitor& validation visitor for cluster configuration - * messages. - */ - virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; + virtual OptRef admin() PURE; /** * @return Ssl::ContextManager& the SSL context manager. @@ -56,7 +47,7 @@ class TransportSocketFactoryContext { /** * @return Stats::Scope& the transport socket's stats scope. */ - virtual Stats::Scope& statsScope() PURE; + virtual Stats::Scope& scope() PURE; /** * Return the instance of secret manager. @@ -64,9 +55,60 @@ class TransportSocketFactoryContext { virtual Secret::SecretManager& secretManager() PURE; /** - * @return the init manager of the particular context. + * @return the instance of ClusterManager. + */ + virtual Upstream::ClusterManager& clusterManager() PURE; + + /** + * @return information about the local environment the server is running in. + */ + virtual const LocalInfo::LocalInfo& localInfo() const PURE; + + /** + * @return Event::Dispatcher& the main thread's dispatcher. + */ + virtual Event::Dispatcher& mainThreadDispatcher() PURE; + + /** + * @return Server::Options& the command-line options that Envoy was started with. + */ + virtual const Options& options() PURE; + + /** + * @return the server-wide stats store. + */ + virtual Stats::Store& stats() PURE; + + /** + * @return a reference to the instance of an init manager. */ virtual Init::Manager& initManager() PURE; + + /** + * @return the server's singleton manager. + */ + virtual Singleton::Manager& singletonManager() PURE; + + /** + * @return the server's TLS slot allocator. + */ + virtual ThreadLocal::SlotAllocator& threadLocal() PURE; + + /** + * @return ProtobufMessage::ValidationVisitor& validation visitor for filter configuration + * messages. + */ + virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; + + /** + * @return reference to the Api object + */ + virtual Api::Api& api() PURE; + + /** + * @return reference to the access log manager object + */ + virtual AccessLog::AccessLogManager& accessLogManager() PURE; }; using TransportSocketFactoryContextPtr = std::unique_ptr; diff --git a/mobile/test/common/integration/test_server.cc b/mobile/test/common/integration/test_server.cc index f47ef32aeb00..771360fc3da0 100644 --- a/mobile/test/common/integration/test_server.cc +++ b/mobile/test/common/integration/test_server.cc @@ -53,9 +53,8 @@ Network::DownstreamTransportSocketFactoryPtr TestServer::createUpstreamTlsContex TestServer::TestServer() : api_(Api::createApiForTest(stats_store_, time_system_)), version_(Network::Address::IpVersion::v4), upstream_config_(time_system_), port_(0) { - ON_CALL(factory_context_.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); - ON_CALL(factory_context_, statsScope()) - .WillByDefault(testing::ReturnRef(*stats_store_.rootScope())); + ON_CALL(factory_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(factory_context_, scope()).WillByDefault(testing::ReturnRef(*stats_store_.rootScope())); } void TestServer::startTestServer(bool use_quic) { diff --git a/source/common/quic/quic_transport_socket_factory.cc b/source/common/quic/quic_transport_socket_factory.cc index f7f21ef5db8f..05d5ddc0c0ca 100644 --- a/source/common/quic/quic_transport_socket_factory.cc +++ b/source/common/quic/quic_transport_socket_factory.cc @@ -30,8 +30,8 @@ QuicServerTransportSocketConfigFactory::createTransportSocketFactory( } auto factory = std::make_unique( - PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_transport, enable_early_data, true), - context.statsScope(), std::move(server_config)); + PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_transport, enable_early_data, true), context.scope(), + std::move(server_config)); factory->initialize(); return factory; } @@ -59,9 +59,9 @@ QuicClientTransportSocketConfigFactory::createTransportSocketFactory( QuicClientTransportSocketFactory::QuicClientTransportSocketFactory( Ssl::ClientContextConfigPtr config, Server::Configuration::TransportSocketFactoryContext& factory_context) - : QuicTransportSocketFactoryBase(factory_context.statsScope(), "client"), + : QuicTransportSocketFactoryBase(factory_context.scope(), "client"), fallback_factory_(std::make_unique( - std::move(config), factory_context.sslContextManager(), factory_context.statsScope())) {} + std::move(config), factory_context.sslContextManager(), factory_context.scope())) {} ProtobufTypes::MessagePtr QuicClientTransportSocketConfigFactory::createEmptyConfigProto() { return std::make_unique(); diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index edc5db1e0ad8..a09d87b6ac52 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -137,13 +137,13 @@ class TlsCertificateSdsApi : public SdsApi, public TlsCertificateConfigProvider const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - auto& server_context = secret_provider_context.serverFactoryContext(); - Config::Utility::checkLocalInfo("TlsCertificateSdsApi", server_context.localInfo()); + Config::Utility::checkLocalInfo("TlsCertificateSdsApi", secret_provider_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - server_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), - destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); + secret_provider_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), + destructor_cb, secret_provider_context.mainThreadDispatcher(), + secret_provider_context.api()); } TlsCertificateSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, @@ -223,14 +223,14 @@ class CertificateValidationContextSdsApi : public SdsApi, const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - auto& server_context = secret_provider_context.serverFactoryContext(); Config::Utility::checkLocalInfo("CertificateValidationContextSdsApi", - server_context.localInfo()); + secret_provider_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - server_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), - destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); + secret_provider_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), + destructor_cb, secret_provider_context.mainThreadDispatcher(), + secret_provider_context.api()); } CertificateValidationContextSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, const std::string& sds_config_name, @@ -318,13 +318,14 @@ class TlsSessionTicketKeysSdsApi : public SdsApi, public TlsSessionTicketKeysCon const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - auto& server_context = secret_provider_context.serverFactoryContext(); - Config::Utility::checkLocalInfo("TlsSessionTicketKeysSdsApi", server_context.localInfo()); + Config::Utility::checkLocalInfo("TlsSessionTicketKeysSdsApi", + secret_provider_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - server_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), - destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); + secret_provider_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), + destructor_cb, secret_provider_context.mainThreadDispatcher(), + secret_provider_context.api()); } TlsSessionTicketKeysSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, @@ -391,13 +392,13 @@ class GenericSecretSdsApi : public SdsApi, public GenericSecretConfigProvider { const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - auto& server_context = secret_provider_context.serverFactoryContext(); - Config::Utility::checkLocalInfo("GenericSecretSdsApi", server_context.localInfo()); + Config::Utility::checkLocalInfo("GenericSecretSdsApi", secret_provider_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - server_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), - destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); + secret_provider_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), + destructor_cb, secret_provider_context.mainThreadDispatcher(), + secret_provider_context.api()); } GenericSecretSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index 30f16ac78084..7a8386069184 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -516,7 +516,8 @@ ProdClusterInfoFactory::createClusterInfo(const CreateClusterInfoParams& params) Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( params.server_context_, params.ssl_context_manager_, *scope, - params.server_context_.clusterManager(), params.server_context_.messageValidationVisitor()); + params.server_context_.clusterManager(), params.stats_, + params.server_context_.messageValidationVisitor()); // TODO(JimmyCYJ): Support SDS for HDS cluster. Network::UpstreamTransportSocketFactoryPtr socket_factory = diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 228807771a0c..130404364e3a 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -882,14 +882,11 @@ class FactoryContextImpl : public Server::Configuration::CommonFactoryContext { // other contexts taken from TransportSocketFactoryContext. FactoryContextImpl(Stats::Scope& stats_scope, Envoy::Runtime::Loader& runtime, Server::Configuration::TransportSocketFactoryContext& c) - : admin_(c.serverFactoryContext().admin()), - server_scope_(c.serverFactoryContext().serverScope()), stats_scope_(stats_scope), - cluster_manager_(c.clusterManager()), local_info_(c.serverFactoryContext().localInfo()), - dispatcher_(c.serverFactoryContext().mainThreadDispatcher()), runtime_(runtime), - singleton_manager_(c.serverFactoryContext().singletonManager()), - tls_(c.serverFactoryContext().threadLocal()), api_(c.serverFactoryContext().api()), - options_(c.serverFactoryContext().options()), - message_validation_visitor_(c.messageValidationVisitor()) {} + : admin_(c.admin()), server_scope_(*c.stats().rootScope()), stats_scope_(stats_scope), + cluster_manager_(c.clusterManager()), local_info_(c.localInfo()), + dispatcher_(c.mainThreadDispatcher()), runtime_(runtime), + singleton_manager_(c.singletonManager()), tls_(c.threadLocal()), api_(c.api()), + options_(c.options()), message_validation_visitor_(c.messageValidationVisitor()) {} Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } Event::Dispatcher& mainThreadDispatcher() override { return dispatcher_; } @@ -1377,7 +1374,8 @@ ClusterImplBase::ClusterImplBase(const envoy::config::cluster::v3::Cluster& clus transport_factory_context_ = std::make_unique( server_context, cluster_context.sslContextManager(), *stats_scope, - cluster_context.clusterManager(), cluster_context.messageValidationVisitor()); + cluster_context.clusterManager(), server_context.serverScope().store(), + cluster_context.messageValidationVisitor()); transport_factory_context_->setInitManager(init_manager_); auto socket_factory = createTransportSocketFactory(cluster, *transport_factory_context_); diff --git a/source/extensions/clusters/eds/leds.cc b/source/extensions/clusters/eds/leds.cc index 91c5ad6f32a7..2e2ed0db6941 100644 --- a/source/extensions/clusters/eds/leds.cc +++ b/source/extensions/clusters/eds/leds.cc @@ -17,7 +17,7 @@ LedsSubscription::LedsSubscription( Stats::Scope& cluster_stats_scope, const UpdateCb& callback) : Envoy::Config::SubscriptionBase( factory_context.messageValidationVisitor(), leds_config.leds_collection_name()), - local_info_(factory_context.serverFactoryContext().localInfo()), cluster_name_(cluster_name), + local_info_(factory_context.localInfo()), cluster_name_(cluster_name), stats_scope_(cluster_stats_scope.createScope("leds.")), stats_({ALL_LEDS_STATS(POOL_COUNTER(*stats_scope_))}), callback_(callback) { const xds::core::v3::ResourceLocator leds_resource_locator = diff --git a/source/extensions/listener_managers/listener_manager/listener_impl.cc b/source/extensions/listener_managers/listener_manager/listener_impl.cc index 47909c0de470..94b29a189e71 100644 --- a/source/extensions/listener_managers/listener_manager/listener_impl.cc +++ b/source/extensions/listener_managers/listener_manager/listener_impl.cc @@ -354,7 +354,8 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, transport_factory_context_( std::make_shared( parent_.server_.serverFactoryContext(), parent_.server_.sslContextManager(), - listenerScope(), parent_.server_.clusterManager(), validation_visitor_)), + listenerScope(), parent_.server_.clusterManager(), parent_.server_.stats(), + validation_visitor_)), quic_stat_names_(parent_.quicStatNames()), missing_listener_config_stats_({ALL_MISSING_LISTENER_CONFIG_STATS( POOL_COUNTER(listener_factory_context_->listenerScope()))}) { diff --git a/source/extensions/transport_sockets/alts/config.cc b/source/extensions/transport_sockets/alts/config.cc index 0af3cb780775..7c3d18d27d7a 100644 --- a/source/extensions/transport_sockets/alts/config.cc +++ b/source/extensions/transport_sockets/alts/config.cc @@ -97,10 +97,9 @@ TransportSocketFactoryPtr createTransportSocketFactoryHelper( Server::Configuration::TransportSocketFactoryContext& factory_ctxt) { // A reference to this is held in the factory closure to keep the singleton // instance alive. - auto alts_shared_state = - factory_ctxt.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(alts_shared_state), - [] { return std::make_shared(); }); + auto alts_shared_state = factory_ctxt.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(alts_shared_state), + [] { return std::make_shared(); }); auto config = MessageUtil::downcastAndValidate( message, factory_ctxt.messageValidationVisitor()); diff --git a/source/extensions/transport_sockets/internal_upstream/config.cc b/source/extensions/transport_sockets/internal_upstream/config.cc index 5517c5c34203..2993e34d2815 100644 --- a/source/extensions/transport_sockets/internal_upstream/config.cc +++ b/source/extensions/transport_sockets/internal_upstream/config.cc @@ -104,7 +104,7 @@ InternalSocketFactory::InternalSocketFactory( const envoy::extensions::transport_sockets::internal_upstream::v3::InternalUpstreamTransport& config_proto, Network::UpstreamTransportSocketFactoryPtr&& inner_factory) - : PassthroughFactory(std::move(inner_factory)), config_(config_proto, context.statsScope()) {} + : PassthroughFactory(std::move(inner_factory)), config_(config_proto, context.scope()) {} Network::TransportSocketPtr InternalSocketFactory::createTransportSocket(Network::TransportSocketOptionsConstSharedPtr options, diff --git a/source/extensions/transport_sockets/proxy_protocol/config.cc b/source/extensions/transport_sockets/proxy_protocol/config.cc index bdd57dfdc84f..cf8f1e08b387 100644 --- a/source/extensions/transport_sockets/proxy_protocol/config.cc +++ b/source/extensions/transport_sockets/proxy_protocol/config.cc @@ -27,7 +27,7 @@ UpstreamProxyProtocolSocketConfigFactory::createTransportSocketFactory( auto inner_transport_factory = inner_config_factory.createTransportSocketFactory(*inner_factory_config, context); return std::make_unique( - std::move(inner_transport_factory), outer_config.config(), context.statsScope()); + std::move(inner_transport_factory), outer_config.config(), context.scope()); } ProtobufTypes::MessagePtr UpstreamProxyProtocolSocketConfigFactory::createEmptyConfigProto() { diff --git a/source/extensions/transport_sockets/tap/config.cc b/source/extensions/transport_sockets/tap/config.cc index 3b565bee92e9..b84b173d2d36 100644 --- a/source/extensions/transport_sockets/tap/config.cc +++ b/source/extensions/transport_sockets/tap/config.cc @@ -44,14 +44,11 @@ UpstreamTapSocketConfigFactory::createTransportSocketFactory( outer_config.transport_socket(), context.messageValidationVisitor(), inner_config_factory); auto inner_transport_factory = inner_config_factory.createTransportSocketFactory(*inner_factory_config, context); - - auto& server_context = context.serverFactoryContext(); return std::make_unique( outer_config, - std::make_unique( - server_context.mainThreadDispatcher().timeSource()), - server_context.admin(), server_context.singletonManager(), server_context.threadLocal(), - server_context.mainThreadDispatcher(), std::move(inner_transport_factory)); + std::make_unique(context.mainThreadDispatcher().timeSource()), + context.admin(), context.singletonManager(), context.threadLocal(), + context.mainThreadDispatcher(), std::move(inner_transport_factory)); } Network::DownstreamTransportSocketFactoryPtr @@ -68,13 +65,11 @@ DownstreamTapSocketConfigFactory::createTransportSocketFactory( outer_config.transport_socket(), context.messageValidationVisitor(), inner_config_factory); auto inner_transport_factory = inner_config_factory.createTransportSocketFactory( *inner_factory_config, context, server_names); - auto& server_context = context.serverFactoryContext(); return std::make_unique( outer_config, - std::make_unique( - server_context.mainThreadDispatcher().timeSource()), - server_context.admin(), server_context.singletonManager(), server_context.threadLocal(), - server_context.mainThreadDispatcher(), std::move(inner_transport_factory)); + std::make_unique(context.mainThreadDispatcher().timeSource()), + context.admin(), context.singletonManager(), context.threadLocal(), + context.mainThreadDispatcher(), std::move(inner_transport_factory)); } ProtobufTypes::MessagePtr TapSocketConfigFactory::createEmptyConfigProto() { diff --git a/source/extensions/transport_sockets/tcp_stats/config.cc b/source/extensions/transport_sockets/tcp_stats/config.cc index 0e362f959b1e..aafc940cb01f 100644 --- a/source/extensions/transport_sockets/tcp_stats/config.cc +++ b/source/extensions/transport_sockets/tcp_stats/config.cc @@ -16,7 +16,7 @@ TcpStatsSocketFactory::TcpStatsSocketFactory( Server::Configuration::TransportSocketFactoryContext& context, const envoy::extensions::transport_sockets::tcp_stats::v3::Config& config) { #if defined(__linux__) - config_ = std::make_shared(config, context.statsScope()); + config_ = std::make_shared(config, context.scope()); #else UNREFERENCED_PARAMETER(config); UNREFERENCED_PARAMETER(context); diff --git a/source/extensions/transport_sockets/tls/config.cc b/source/extensions/transport_sockets/tls/config.cc index f2612251584d..75cc7b34a4ba 100644 --- a/source/extensions/transport_sockets/tls/config.cc +++ b/source/extensions/transport_sockets/tls/config.cc @@ -20,8 +20,8 @@ Network::UpstreamTransportSocketFactoryPtr UpstreamSslSocketFactory::createTrans const envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext&>( message, context.messageValidationVisitor()), context); - return std::make_unique( - std::move(client_config), context.sslContextManager(), context.statsScope()); + return std::make_unique(std::move(client_config), + context.sslContextManager(), context.scope()); } ProtobufTypes::MessagePtr UpstreamSslSocketFactory::createEmptyConfigProto() { @@ -41,7 +41,7 @@ DownstreamSslSocketFactory::createTransportSocketFactory( message, context.messageValidationVisitor()), context); return std::make_unique( - std::move(server_config), context.sslContextManager(), context.statsScope(), server_names); + std::move(server_config), context.sslContextManager(), context.scope(), server_names); } ProtobufTypes::MessagePtr DownstreamSslSocketFactory::createEmptyConfigProto() { diff --git a/source/extensions/transport_sockets/tls/context_config_impl.cc b/source/extensions/transport_sockets/tls/context_config_impl.cc index 021c1d881fea..f6a7e30a4f35 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.cc +++ b/source/extensions/transport_sockets/tls/context_config_impl.cc @@ -173,9 +173,8 @@ ContextConfigImpl::ContextConfigImpl( const unsigned default_min_protocol_version, const unsigned default_max_protocol_version, const std::string& default_cipher_suites, const std::string& default_curves, Server::Configuration::TransportSocketFactoryContext& factory_context) - : api_(factory_context.serverFactoryContext().api()), - options_(factory_context.serverFactoryContext().options()), - singleton_manager_(factory_context.serverFactoryContext().singletonManager()), + : api_(factory_context.api()), options_(factory_context.options()), + singleton_manager_(factory_context.singletonManager()), alpn_protocols_(RepeatedPtrUtil::join(config.alpn_protocols(), ",")), cipher_suites_(StringUtil::nonEmptyStringOrDefault( RepeatedPtrUtil::join(config.tls_params().cipher_suites(), ":"), default_cipher_suites)), diff --git a/source/extensions/transport_sockets/tls/context_config_impl.h b/source/extensions/transport_sockets/tls/context_config_impl.h index 23ca329abe8e..da3a526d163c 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.h +++ b/source/extensions/transport_sockets/tls/context_config_impl.h @@ -46,7 +46,7 @@ class ContextConfigImpl : public virtual Ssl::ContextConfig { const Network::Address::IpList& tlsKeyLogRemote() const override { return tls_keylog_remote_; }; const std::string& tlsKeyLogPath() const override { return tls_keylog_path_; }; AccessLog::AccessLogManager& accessLogManager() const override { - return factory_context_.serverFactoryContext().accessLogManager(); + return factory_context_.accessLogManager(); } bool isReady() const override { diff --git a/source/extensions/upstreams/http/config.h b/source/extensions/upstreams/http/config.h index 9e1d4c78283b..93879e9deb6b 100644 --- a/source/extensions/upstreams/http/config.h +++ b/source/extensions/upstreams/http/config.h @@ -71,7 +71,7 @@ class ProtocolOptionsConfigFactory : public Server::Configuration::ProtocolOptio const envoy::extensions::upstreams::http::v3::HttpProtocolOptions&>( config, context.messageValidationVisitor()); return std::make_shared(typed_config, - context.serverFactoryContext()); + context.getServerFactoryContext()); } std::string category() const override { return "envoy.upstream_options"; } std::string name() const override { diff --git a/source/server/server.h b/source/server/server.h index d5140d0e895a..d9c9f175a6dd 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -191,7 +191,6 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, ThreadLocal::Instance& threadLocal() override { return server_.threadLocal(); } OptRef admin() override { return server_.admin(); } TimeSource& timeSource() override { return api().timeSource(); } - AccessLog::AccessLogManager& accessLogManager() override { return server_.accessLogManager(); } Api::Api& api() override { return server_.api(); } Grpc::Context& grpcContext() override { return server_.grpcContext(); } Router::Context& routerContext() override { return server_.routerContext(); } @@ -201,11 +200,12 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return server_.bootstrap(); } // Configuration::TransportSocketFactoryContext - ServerFactoryContext& serverFactoryContext() override { return *this; } + ServerFactoryContext& getServerFactoryContext() override { return *this; }; Ssl::ContextManager& sslContextManager() override { return server_.sslContextManager(); } Secret::SecretManager& secretManager() override { return server_.secretManager(); } - Stats::Scope& statsScope() override { return *server_scope_; } + Stats::Store& stats() override { return server_.stats(); } Init::Manager& initManager() override { return server_.initManager(); } + AccessLog::AccessLogManager& accessLogManager() override { return server_.accessLogManager(); } ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { // Server has two message validation visitors, one for static and // other for dynamic configuration. Choose the dynamic validation diff --git a/source/server/transport_socket_config_impl.h b/source/server/transport_socket_config_impl.h index 87f9aaeac244..edbb65282e19 100644 --- a/source/server/transport_socket_config_impl.h +++ b/source/server/transport_socket_config_impl.h @@ -14,10 +14,11 @@ class TransportSocketFactoryContextImpl : public TransportSocketFactoryContext { public: TransportSocketFactoryContextImpl(Server::Configuration::ServerFactoryContext& server_context, Ssl::ContextManager& context_manager, Stats::Scope& stats_scope, - Upstream::ClusterManager& cm, + Upstream::ClusterManager& cm, Stats::Store& stats, ProtobufMessage::ValidationVisitor& validation_visitor) : server_context_(server_context), context_manager_(context_manager), - stats_scope_(stats_scope), cluster_manager_(cm), validation_visitor_(validation_visitor) {} + stats_scope_(stats_scope), cluster_manager_(cm), stats_(stats), + validation_visitor_(validation_visitor) {} /** * Pass an init manager to register dynamic secret provider. @@ -26,29 +27,42 @@ class TransportSocketFactoryContextImpl : public TransportSocketFactoryContext { void setInitManager(Init::Manager& init_manager) { init_manager_ = &init_manager; } // TransportSocketFactoryContext - ServerFactoryContext& serverFactoryContext() override { return server_context_; } - Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { - return validation_visitor_; - } + ServerFactoryContext& getServerFactoryContext() override { return server_context_; }; + OptRef admin() override { return server_context_.admin(); } Ssl::ContextManager& sslContextManager() override { return context_manager_; } - Stats::Scope& statsScope() override { return stats_scope_; } + Stats::Scope& scope() override { return stats_scope_; } Secret::SecretManager& secretManager() override { return clusterManager().clusterManagerFactory().secretManager(); } + Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } + const LocalInfo::LocalInfo& localInfo() const override { return server_context_.localInfo(); } + Event::Dispatcher& mainThreadDispatcher() override { + return server_context_.mainThreadDispatcher(); + } + Stats::Store& stats() override { return stats_; } Init::Manager& initManager() override { ASSERT(init_manager_ != nullptr); return *init_manager_; } + Singleton::Manager& singletonManager() override { return server_context_.singletonManager(); } + ThreadLocal::SlotAllocator& threadLocal() override { return server_context_.threadLocal(); } + ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { + return validation_visitor_; + } + Api::Api& api() override { return server_context_.api(); } + const Server::Options& options() override { return server_context_.options(); } + AccessLog::AccessLogManager& accessLogManager() override { + return server_context_.accessLogManager(); + } private: Server::Configuration::ServerFactoryContext& server_context_; Ssl::ContextManager& context_manager_; Stats::Scope& stats_scope_; Upstream::ClusterManager& cluster_manager_; - ProtobufMessage::ValidationVisitor& validation_visitor_; - + Stats::Store& stats_; Init::Manager* init_manager_{}; + ProtobufMessage::ValidationVisitor& validation_visitor_; }; using TransportSocketFactoryContextImplPtr = std::unique_ptr; diff --git a/test/common/grpc/grpc_client_integration_test_harness.h b/test/common/grpc/grpc_client_integration_test_harness.h index 7c1baf03a87e..dd3bb0831afe 100644 --- a/test/common/grpc/grpc_client_integration_test_harness.h +++ b/test/common/grpc/grpc_client_integration_test_harness.h @@ -493,7 +493,7 @@ class GrpcClientIntegrationTest : public GrpcClientIntegrationParamTest { class GrpcSslClientIntegrationTest : public GrpcClientIntegrationTest { public: GrpcSslClientIntegrationTest() { - ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); } void TearDown() override { // Reset some state in the superclass before we destruct context_manager_ in our destructor, it diff --git a/test/common/quic/quic_transport_socket_factory_test.cc b/test/common/quic/quic_transport_socket_factory_test.cc index 89aa18174ab1..0b28fc060978 100644 --- a/test/common/quic/quic_transport_socket_factory_test.cc +++ b/test/common/quic/quic_transport_socket_factory_test.cc @@ -17,7 +17,7 @@ class QuicServerTransportSocketFactoryConfigTest : public Event::TestUsingSimula public: QuicServerTransportSocketFactoryConfigTest() : server_api_(Api::createApiForTest(server_stats_store_, simTime())) { - ON_CALL(context_.server_context_, api()).WillByDefault(ReturnRef(*server_api_)); + ON_CALL(context_, api()).WillByDefault(ReturnRef(*server_api_)); } void verifyQuicServerTransportSocketFactory(std::string yaml, bool expect_early_data) { diff --git a/test/common/secret/secret_manager_impl_test.cc b/test/common/secret/secret_manager_impl_test.cc index ead2895c984b..daca620733e7 100644 --- a/test/common/secret/secret_manager_impl_test.cc +++ b/test/common/secret/secret_manager_impl_test.cc @@ -270,6 +270,7 @@ TEST_F(SecretManagerImplTest, DeduplicateDynamicTlsCertificateSecretProvider) { NiceMock local_info; NiceMock dispatcher; NiceMock random; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -277,10 +278,10 @@ TEST_F(SecretManagerImplTest, DeduplicateDynamicTlsCertificateSecretProvider) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); envoy::config::core::v3::ConfigSource config_source; TestUtility::loadFromYaml(R"( @@ -352,6 +353,7 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretUpdateSuccess) { envoy::config::core::v3::ConfigSource config_source; NiceMock local_info; NiceMock random; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -359,11 +361,11 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretUpdateSuccess) { .WillOnce(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); + EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(*dispatcher_)); - EXPECT_CALL(secret_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api_)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); @@ -401,17 +403,18 @@ TEST_F(SecretManagerImplTest, SdsDynamicGenericSecret) { NiceMock secret_context; NiceMock validation_visitor; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock local_info; Init::TargetHandlePtr init_target_handle; NiceMock init_watcher; - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); EXPECT_CALL(secret_context, messageValidationVisitor()).WillOnce(ReturnRef(validation_visitor)); + EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api_)); EXPECT_CALL(init_manager, add(_)) .WillOnce(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); @@ -449,6 +452,7 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandler) { NiceMock local_info; NiceMock dispatcher; NiceMock random; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -456,10 +460,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandler) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); @@ -718,6 +722,7 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerWarmingSecrets) { NiceMock local_info; NiceMock dispatcher; NiceMock random; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -725,10 +730,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerWarmingSecrets) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); @@ -865,6 +870,7 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSecrets) { NiceMock local_info; NiceMock dispatcher; NiceMock random; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -872,10 +878,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSecrets) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); const std::string tls_certificate = R"EOF( @@ -941,6 +947,7 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticValidationContext) { NiceMock local_info; NiceMock dispatcher; NiceMock random; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -948,10 +955,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticValidationContext) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); const std::string validation_context = R"EOF( @@ -988,6 +995,7 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSessionTicketsContext) { NiceMock local_info; NiceMock dispatcher; NiceMock random; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -995,10 +1003,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSessionTicketsContext) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); const std::string stek_context = R"EOF( @@ -1068,6 +1076,7 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretPrivateKeyProviderUpdateSuccess) { envoy::config::core::v3::ConfigSource config_source; NiceMock local_info; NiceMock random; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -1075,11 +1084,11 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretPrivateKeyProviderUpdateSuccess) { .WillOnce(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); + EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(*dispatcher_)); - EXPECT_CALL(secret_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api_)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index d52e677e5fec..8bebd8275dc3 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -60,7 +60,6 @@ envoy_cc_test( envoy_cc_test( name = "cluster_manager_impl_test", - size = "large", srcs = ["cluster_manager_impl_test.cc"], args = [ # Force creation of c-ares DnsResolverImpl when running test on macOS. diff --git a/test/common/upstream/hds_test.cc b/test/common/upstream/hds_test.cc index e3ee0677e7c1..8fdc4eead1d1 100644 --- a/test/common/upstream/hds_test.cc +++ b/test/common/upstream/hds_test.cc @@ -583,7 +583,7 @@ TEST_F(HdsTest, TestSocketContext) { params.stats_.createScope(fmt::format("cluster.{}.", params.cluster_.name())); Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( params.server_context_, params.ssl_context_manager_, *scope, - params.server_context_.clusterManager(), + params.server_context_.clusterManager(), params.stats_, params.server_context_.messageValidationVisitor()); // Create a mock socket_factory for the scope of this unit test. @@ -1072,7 +1072,7 @@ TEST_F(HdsTest, TestUpdateSocketContext) { params.stats_.createScope(fmt::format("cluster.{}.", params.cluster_.name())); Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( params.server_context_, params.ssl_context_manager_, *scope, - params.server_context_.clusterManager(), + params.server_context_.clusterManager(), params.stats_, params.server_context_.messageValidationVisitor()); // Create a mock socket_factory for the scope of this unit test. diff --git a/test/extensions/clusters/eds/leds_test.cc b/test/extensions/clusters/eds/leds_test.cc index 14a0fc5c0958..8e30b3fbeb11 100644 --- a/test/extensions/clusters/eds/leds_test.cc +++ b/test/extensions/clusters/eds/leds_test.cc @@ -90,7 +90,7 @@ class LedsTest : public testing::Test { cluster_scope_ = stats_.createScope("cluster.xds_cluster."); Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( server_context_, ssl_context_manager_, *cluster_scope_, server_context_.cluster_manager_, - validation_visitor_); + stats_, validation_visitor_); // Setup LEDS subscription. EXPECT_CALL(server_context_.cluster_manager_.subscription_factory_, diff --git a/test/extensions/filters/http/oauth2/filter_test.cc b/test/extensions/filters/http/oauth2/filter_test.cc index 970bbf360e81..ddf8f16f4fb5 100644 --- a/test/extensions/filters/http/oauth2/filter_test.cc +++ b/test/extensions/filters/http/oauth2/filter_test.cc @@ -207,13 +207,14 @@ TEST_F(OAuth2Test, SdsDynamicGenericSecret) { NiceMock secret_context; NiceMock local_info; Api::ApiPtr api = Api::createApiForTest(); + Stats::IsolatedStoreImpl stats; NiceMock init_manager; Init::TargetHandlePtr init_handle; NiceMock dispatcher; - EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); - EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api)); - EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api)); + EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); EXPECT_CALL(init_manager, add(_)) .WillRepeatedly(Invoke([&init_handle](const Init::Target& target) { diff --git a/test/extensions/transport_sockets/alts/alts_integration_test.cc b/test/extensions/transport_sockets/alts/alts_integration_test.cc index 4cc0b8f435f7..570646820764 100644 --- a/test/extensions/transport_sockets/alts/alts_integration_test.cc +++ b/test/extensions/transport_sockets/alts/alts_integration_test.cc @@ -160,7 +160,7 @@ class AltsIntegrationTestBase : public Event::TestUsingSimulatedTime, } }; FakeSingletonManager fsm; - ON_CALL(mock_factory_ctx.server_context_, singletonManager()).WillByDefault(ReturnRef(fsm)); + ON_CALL(mock_factory_ctx, singletonManager()).WillByDefault(ReturnRef(fsm)); UpstreamAltsTransportSocketConfigFactory factory; envoy::extensions::transport_sockets::alts::v3::Alts alts_config; diff --git a/test/extensions/transport_sockets/alts/config_test.cc b/test/extensions/transport_sockets/alts/config_test.cc index a848115facf8..625994ba5404 100644 --- a/test/extensions/transport_sockets/alts/config_test.cc +++ b/test/extensions/transport_sockets/alts/config_test.cc @@ -19,8 +19,7 @@ namespace { TEST(UpstreamAltsConfigTest, CreateSocketFactory) { NiceMock factory_context; Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; - EXPECT_CALL(factory_context.server_context_, singletonManager()) - .WillRepeatedly(ReturnRef(singleton_manager)); + EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); UpstreamAltsTransportSocketConfigFactory factory; ProtobufTypes::MessagePtr config = factory.createEmptyConfigProto(); @@ -40,8 +39,7 @@ TEST(UpstreamAltsConfigTest, CreateSocketFactory) { TEST(DownstreamAltsConfigTest, CreateSocketFactory) { NiceMock factory_context; Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; - EXPECT_CALL(factory_context.server_context_, singletonManager()) - .WillRepeatedly(ReturnRef(singleton_manager)); + EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); DownstreamAltsTransportSocketConfigFactory factory; ProtobufTypes::MessagePtr config = factory.createEmptyConfigProto(); diff --git a/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc b/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc index fe298d275640..232b95c1acde 100644 --- a/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc +++ b/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc @@ -269,7 +269,7 @@ void StartTlsIntegrationTest::initialize() { TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), downstream_tls_context); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(mock_factory_ctx, api()).WillByDefault(testing::ReturnRef(*api_)); auto cfg = std::make_unique( downstream_tls_context, mock_factory_ctx); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); diff --git a/test/extensions/transport_sockets/tls/BUILD b/test/extensions/transport_sockets/tls/BUILD index 098ba510db74..375bbb3072e3 100644 --- a/test/extensions/transport_sockets/tls/BUILD +++ b/test/extensions/transport_sockets/tls/BUILD @@ -13,7 +13,6 @@ envoy_package() envoy_cc_test( name = "ssl_socket_test", - size = "large", srcs = [ "ssl_certs_test.h", "ssl_socket_test.cc", diff --git a/test/extensions/transport_sockets/tls/context_impl_test.cc b/test/extensions/transport_sockets/tls/context_impl_test.cc index 8e6bed283c68..4f082a192725 100644 --- a/test/extensions/transport_sockets/tls/context_impl_test.cc +++ b/test/extensions/transport_sockets/tls/context_impl_test.cc @@ -955,12 +955,13 @@ TEST_F(SslServerContextImplTicketTest, TicketKeySdsNotReady) { NiceMock local_info; NiceMock dispatcher; NiceMock random; + Stats::IsolatedStoreImpl stats; NiceMock cluster_manager; NiceMock init_manager; - EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); // EXPECT_CALL(factory_context_, random()).WillOnce(ReturnRef(random)); + EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, clusterManager()).WillOnce(ReturnRef(cluster_manager)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); auto* sds_secret_configs = tls_context.mutable_session_ticket_keys_sds_secret_config(); @@ -1407,12 +1408,13 @@ TEST_F(ClientContextConfigImplTest, TlsCertificatesAndSdsConfig) { TEST_F(ClientContextConfigImplTest, SecretNotReady) { envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; NiceMock local_info; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); @@ -1438,12 +1440,13 @@ TEST_F(ClientContextConfigImplTest, ValidationContextNotReady) { client_cert->mutable_private_key()->set_filename(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_key.pem")); NiceMock local_info; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_validation_context_sds_secret_config(); sds_secret_configs->set_name("abc.com"); @@ -1844,12 +1847,13 @@ TEST_F(ServerContextConfigImplTest, MultiSdsConfig) { TEST_F(ServerContextConfigImplTest, SecretNotReady) { envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; NiceMock local_info; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); @@ -1875,12 +1879,13 @@ TEST_F(ServerContextConfigImplTest, ValidationContextNotReady) { server_cert->mutable_private_key()->set_filename(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_key.pem")); NiceMock local_info; + Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_validation_context_sds_secret_config(); sds_secret_configs->set_name("abc.com"); diff --git a/test/extensions/transport_sockets/tls/handshaker_factory_test.cc b/test/extensions/transport_sockets/tls/handshaker_factory_test.cc index 0221a468d700..64b5e9d0f05d 100644 --- a/test/extensions/transport_sockets/tls/handshaker_factory_test.cc +++ b/test/extensions/transport_sockets/tls/handshaker_factory_test.cc @@ -129,7 +129,7 @@ TEST_F(HandshakerFactoryTest, SetMockFunctionCb) { static_cast(custom_process_object_for_test)); NiceMock mock_factory_ctx; - EXPECT_CALL(mock_factory_ctx.server_context_.api_, processContext()) + EXPECT_CALL(mock_factory_ctx.api_, processContext()) .WillRepeatedly(Return(std::reference_wrapper(*process_context_impl))); Extensions::TransportSockets::Tls::ClientSslSocketFactory socket_factory( @@ -155,7 +155,7 @@ TEST_F(HandshakerFactoryTest, SetSpecificSslCtxOption) { static_cast(custom_process_object_for_test)); NiceMock mock_factory_ctx; - EXPECT_CALL(mock_factory_ctx.server_context_.api_, processContext()) + EXPECT_CALL(mock_factory_ctx.api_, processContext()) .WillRepeatedly(Return(std::reference_wrapper(*process_context_impl))); Extensions::TransportSockets::Tls::ClientSslSocketFactory socket_factory( @@ -181,7 +181,7 @@ TEST_F(HandshakerFactoryTest, HandshakerContextProvidesObjectsFromParentContext) static_cast(custom_process_object_for_test)); NiceMock mock_factory_ctx; - EXPECT_CALL(mock_factory_ctx.server_context_.api_, processContext()) + EXPECT_CALL(mock_factory_ctx.api_, processContext()) .WillRepeatedly(Return(std::reference_wrapper(*process_context_impl))); MockFunction mock_factory_cb; @@ -191,8 +191,8 @@ TEST_F(HandshakerFactoryTest, HandshakerContextProvidesObjectsFromParentContext) .WillOnce(WithArg<1>([&](Ssl::HandshakerFactoryContext& context) { // Check that the objects available via the context are the same ones // provided to the parent context. - EXPECT_THAT(context.api(), Ref(mock_factory_ctx.server_context_.api_)); - EXPECT_THAT(context.options(), Ref(mock_factory_ctx.server_context_.options_)); + EXPECT_THAT(context.api(), Ref(mock_factory_ctx.api_)); + EXPECT_THAT(context.options(), Ref(mock_factory_ctx.options_)); })); Extensions::TransportSockets::Tls::ClientSslSocketFactory socket_factory( @@ -289,7 +289,7 @@ TEST_F(HandshakerFactoryDownstreamTest, ServerHandshakerProvidesCertificates) { static_cast(custom_process_object_for_test)); NiceMock mock_factory_ctx; - EXPECT_CALL(mock_factory_ctx.server_context_.api_, processContext()) + EXPECT_CALL(mock_factory_ctx.api_, processContext()) .WillRepeatedly(Return(std::reference_wrapper(*process_context_impl))); Extensions::TransportSockets::Tls::ServerContextConfigImpl server_context_config( diff --git a/test/extensions/transport_sockets/tls/ssl_certs_test.h b/test/extensions/transport_sockets/tls/ssl_certs_test.h index fca49c721c7d..0fe7d2183b27 100644 --- a/test/extensions/transport_sockets/tls/ssl_certs_test.h +++ b/test/extensions/transport_sockets/tls/ssl_certs_test.h @@ -12,7 +12,7 @@ namespace Envoy { class SslCertsTest : public testing::Test { protected: SslCertsTest() : api_(Api::createApiForTest(store_, time_system_)) { - ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); } Event::SimulatedTimeSystem time_system_; diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index c571da735acc..3a8d1bd817fc 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -329,7 +329,7 @@ void testUtil(const TestUtilOptions& options) { NiceMock runtime; testing::NiceMock server_factory_context; - ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); // For private key method testing. NiceMock context_manager; @@ -370,7 +370,7 @@ void testUtil(const TestUtilOptions& options) { Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_tls_context, client_factory_context); @@ -684,7 +684,7 @@ void testUtilV2(const TestUtilOptionsV2& options) { testing::NiceMock server_factory_context; NiceMock runtime; - ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; const envoy::config::core::v3::TransportSocket& transport_socket = @@ -708,7 +708,7 @@ void testUtilV2(const TestUtilOptionsV2& options) { Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(options.clientCtxProto(), client_factory_context); @@ -3420,7 +3420,7 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, NiceMock runtime; testing::NiceMock server_factory_context; - ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext server_tls_context1; TestUtility::loadFromYaml(TestEnvironment::substitute(server_ctx_yaml1), server_tls_context1); @@ -3454,7 +3454,7 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_tls_context, client_factory_context); @@ -3572,7 +3572,7 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml NiceMock runtime; testing::NiceMock server_factory_context; - ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext server_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(server_ctx_yaml), server_tls_context); @@ -3595,7 +3595,7 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_tls_context, client_factory_context); @@ -4294,7 +4294,7 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya Api::ApiPtr server_api = Api::createApiForTest(server_stats_store, time_system_); testing::NiceMock server_factory_context; - ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext server_ctx_proto; TestUtility::loadFromYaml(TestEnvironment::substitute(server_ctx_yaml), server_ctx_proto); @@ -4321,7 +4321,7 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system_); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_ctx_proto, client_factory_context); @@ -5683,14 +5683,14 @@ TEST_P(SslSocketTest, OverrideApplicationProtocols) { // Validate that if downstream secrets are not yet downloaded from SDS server, Envoy creates // NotReadySslSocket object to handle downstream connection. TEST_P(SslSocketTest, DownstreamNotReadySslSocket) { - testing::NiceMock factory_context; + Stats::TestUtil::TestStore stats_store; NiceMock local_info; + testing::NiceMock factory_context; NiceMock init_manager; NiceMock dispatcher; - - EXPECT_CALL(factory_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context, stats()).WillOnce(ReturnRef(stats_store)); EXPECT_CALL(factory_context, initManager()).WillRepeatedly(ReturnRef(init_manager)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; @@ -5704,8 +5704,7 @@ TEST_P(SslSocketTest, DownstreamNotReadySslSocket) { ContextManagerImpl manager(time_system_); ServerSslSocketFactory server_ssl_socket_factory( - std::move(server_cfg), manager, *factory_context.server_context_.store_.rootScope(), - std::vector{}); + std::move(server_cfg), manager, *stats_store.rootScope(), std::vector{}); auto transport_socket = server_ssl_socket_factory.createDownstreamTransportSocket(); EXPECT_FALSE(transport_socket->startSecureTransport()); // Noop transport_socket->configureInitialCongestionWindow(200, std::chrono::microseconds(223)); // Noop @@ -5723,14 +5722,15 @@ TEST_P(SslSocketTest, DownstreamNotReadySslSocket) { // Validate that if upstream secrets are not yet downloaded from SDS server, Envoy creates // NotReadySslSocket object to handle upstream connection. TEST_P(SslSocketTest, UpstreamNotReadySslSocket) { + Stats::TestUtil::TestStore stats_store; NiceMock local_info; testing::NiceMock factory_context; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context, stats()).WillOnce(ReturnRef(stats_store)); EXPECT_CALL(factory_context, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context.server_context_, mainThreadDispatcher()) - .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; auto sds_secret_configs = @@ -5742,8 +5742,8 @@ TEST_P(SslSocketTest, UpstreamNotReadySslSocket) { EXPECT_FALSE(client_cfg->isReady()); ContextManagerImpl manager(time_system_); - ClientSslSocketFactory client_ssl_socket_factory( - std::move(client_cfg), manager, *factory_context.server_context_.store_.rootScope()); + ClientSslSocketFactory client_ssl_socket_factory(std::move(client_cfg), manager, + *stats_store.rootScope()); auto transport_socket = client_ssl_socket_factory.createTransportSocket(nullptr, nullptr); EXPECT_EQ(EMPTY_STRING, transport_socket->protocol()); EXPECT_EQ(nullptr, transport_socket->ssl()); @@ -5764,15 +5764,17 @@ TEST_P(SslSocketTest, TestTransportSocketCallback) { // Make SslSocket. testing::NiceMock factory_context; + Stats::TestUtil::TestStore stats_store; + ON_CALL(factory_context, stats()).WillByDefault(ReturnRef(stats_store)); NiceMock local_info; - ON_CALL(factory_context.server_context_, localInfo()).WillByDefault(ReturnRef(local_info)); + ON_CALL(factory_context, localInfo()).WillByDefault(ReturnRef(local_info)); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; auto client_cfg = std::make_unique(tls_context, factory_context); ContextManagerImpl manager(time_system_); - ClientSslSocketFactory client_ssl_socket_factory( - std::move(client_cfg), manager, *factory_context.server_context_.store_.rootScope()); + ClientSslSocketFactory client_ssl_socket_factory(std::move(client_cfg), manager, + *stats_store.rootScope()); Network::TransportSocketPtr transport_socket = client_ssl_socket_factory.createTransportSocket(nullptr, nullptr); diff --git a/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc index 1f01175716ea..6cae75d1db06 100644 --- a/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc +++ b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc @@ -350,8 +350,7 @@ TestPrivateKeyMethodProvider::TestPrivateKeyMethodProvider( } } - std::string private_key = - factory_context.serverFactoryContext().api().fileSystem().fileReadToEnd(private_key_path); + std::string private_key = factory_context.api().fileSystem().fileReadToEnd(private_key_path); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); bssl::UniquePtr pkey(PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, nullptr)); diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 4b45cb7e0f39..97f9710ddbda 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -61,8 +61,8 @@ BaseIntegrationTest::BaseIntegrationTest(const InstanceConstSharedPtrFn& upstrea std::function above_overflow) -> Buffer::Instance* { return new Buffer::WatermarkBuffer(below_low, above_high, above_overflow); })); - ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); - ON_CALL(factory_context_, statsScope()).WillByDefault(ReturnRef(*stats_store_.rootScope())); + ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_, scope()).WillByDefault(ReturnRef(*stats_store_.rootScope())); // Allow extension lookup by name in the integration tests. config_helper_.addRuntimeOverride("envoy.reloadable_features.no_extension_lookup_by_name", "false"); diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 3fadbfd2af8b..5a934afd78f4 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -262,8 +262,8 @@ class QuicHttpIntegrationTestBase : public HttpIntegrationTest { // Initialize the transport socket factory using a customized ssl option. ssl_client_option_.setAlpn(true).setSan(san_to_match_).setSni("lyft.com"); NiceMock context; - ON_CALL(context.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); - ON_CALL(context, statsScope()).WillByDefault(testing::ReturnRef(stats_scope_)); + ON_CALL(context, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(context, scope()).WillByDefault(testing::ReturnRef(stats_scope_)); ON_CALL(context, sslContextManager()).WillByDefault(testing::ReturnRef(context_manager_)); envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport quic_transport_socket_config; diff --git a/test/integration/ssl_utility.cc b/test/integration/ssl_utility.cc index af93ec2c3a48..bd8e0057b067 100644 --- a/test/integration/ssl_utility.cc +++ b/test/integration/ssl_utility.cc @@ -117,7 +117,7 @@ createClientSslTransportSocketFactory(const ClientSslTransportOptions& options, initializeUpstreamTlsContextConfig(options, tls_context); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(ReturnRef(api)); + ON_CALL(mock_factory_ctx, api()).WillByDefault(ReturnRef(api)); auto cfg = std::make_unique( tls_context, mock_factory_ctx); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); @@ -132,7 +132,7 @@ createUpstreamSslContext(ContextManager& context_manager, Api::Api& api, bool us ConfigHelper::initializeTls({}, *tls_context.mutable_common_tls_context()); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(ReturnRef(api)); + ON_CALL(mock_factory_ctx, api()).WillByDefault(ReturnRef(api)); auto cfg = std::make_unique( tls_context, mock_factory_ctx); diff --git a/test/integration/utility.cc b/test/integration/utility.cc index b19dd19bd264..db03dded6453 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -135,8 +135,8 @@ IntegrationUtil::createQuicUpstreamTransportSocketFactory(Api::Api& api, Stats:: Ssl::ContextManager& context_manager, const std::string& san_to_match) { NiceMock context; - ON_CALL(context.server_context_, api()).WillByDefault(testing::ReturnRef(api)); - ON_CALL(context, statsScope()).WillByDefault(testing::ReturnRef(*store.rootScope())); + ON_CALL(context, api()).WillByDefault(testing::ReturnRef(api)); + ON_CALL(context, scope()).WillByDefault(testing::ReturnRef(*store.rootScope())); ON_CALL(context, sslContextManager()).WillByDefault(testing::ReturnRef(context_manager)); envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport quic_transport_socket_config; diff --git a/test/integration/xfcc_integration_test.h b/test/integration/xfcc_integration_test.h index c787eaa69497..def7807d94b2 100644 --- a/test/integration/xfcc_integration_test.h +++ b/test/integration/xfcc_integration_test.h @@ -40,7 +40,7 @@ class XfccIntegrationTest : public testing::TestWithParam{admin_})); + ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); + ON_CALL(*this, timeSource()).WillByDefault(ReturnRef(time_system_)); + ON_CALL(*this, messageValidationContext()).WillByDefault(ReturnRef(validation_context_)); + ON_CALL(*this, messageValidationVisitor()) + .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); + ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); + ON_CALL(*this, drainManager()).WillByDefault(ReturnRef(drain_manager_)); + ON_CALL(*this, statsConfig()).WillByDefault(ReturnRef(stats_config_)); + ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); + ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); + ON_CALL(*this, lifecycleNotifier()).WillByDefault(ReturnRef(lifecycle_notifier_)); + ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); +} +MockServerFactoryContext::~MockServerFactoryContext() = default; + +MockStatsConfig::MockStatsConfig() = default; +MockStatsConfig::~MockStatsConfig() = default; + +} // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/instance.h b/test/mocks/server/instance.h index 93b5d4fe43d4..e8d077a1624e 100644 --- a/test/mocks/server/instance.h +++ b/test/mocks/server/instance.h @@ -2,13 +2,44 @@ #include "envoy/server/instance.h" -#include "test/mocks/server/server_factory_context.h" -#include "test/mocks/server/transport_socket_factory_context.h" +#include "source/common/grpc/context_impl.h" +#include "source/common/http/context_impl.h" +#include "source/common/quic/quic_stat_names.h" +#include "source/common/router/context_impl.h" +#include "source/common/stats/symbol_table.h" +#include "source/extensions/transport_sockets/tls/context_manager_impl.h" +#include "test/mocks/access_log/mocks.h" +#include "test/mocks/api/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/secret/mocks.h" +#include "test/mocks/stats/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" + +#include "admin.h" +#include "drain_manager.h" #include "gmock/gmock.h" +#include "hot_restart.h" +#include "listener_manager.h" +#include "options.h" +#include "overload_manager.h" +#include "server_lifecycle_notifier.h" +#include "transport_socket_factory_context.h" namespace Envoy { namespace Server { +namespace Configuration { +class MockServerFactoryContext; +class MockStatsConfig; +} // namespace Configuration class MockInstance : public Instance { public: @@ -102,5 +133,104 @@ class MockInstance : public Instance { transport_socket_factory_context_; }; +namespace Configuration { +class MockStatsConfig : public virtual StatsConfig { +public: + MockStatsConfig(); + ~MockStatsConfig() override; + + MOCK_METHOD(const std::list&, sinks, (), (const)); + MOCK_METHOD(std::chrono::milliseconds, flushInterval, (), (const)); + MOCK_METHOD(bool, flushOnAdmin, (), (const)); + MOCK_METHOD(const Stats::SinkPredicates*, sinkPredicates, (), (const)); +}; + +class MockServerFactoryContext : public virtual ServerFactoryContext { +public: + MockServerFactoryContext(); + ~MockServerFactoryContext() override; + + MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); + MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); + MOCK_METHOD(const Server::Options&, options, ()); + MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); + MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); + MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); + MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(Stats::Scope&, serverScope, ()); + MOCK_METHOD(Singleton::Manager&, singletonManager, ()); + MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); + MOCK_METHOD(OptRef, admin, ()); + MOCK_METHOD(TimeSource&, timeSource, ()); + Event::TestTimeSystem& timeSystem() { return time_system_; } + MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); + MOCK_METHOD(Api::Api&, api, ()); + Grpc::Context& grpcContext() override { return grpc_context_; } + Router::Context& routerContext() override { return router_context_; } + envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return bootstrap_; } + MOCK_METHOD(Server::DrainManager&, drainManager, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); + MOCK_METHOD(StatsConfig&, statsConfig, (), ()); + MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); + + testing::NiceMock cluster_manager_; + testing::NiceMock dispatcher_; + testing::NiceMock drain_manager_; + testing::NiceMock local_info_; + testing::NiceMock runtime_loader_; + testing::NiceMock store_; + testing::NiceMock thread_local_; + testing::NiceMock validation_context_; + testing::NiceMock stats_config_; + testing::NiceMock access_log_manager_; + testing::NiceMock init_manager_; + testing::NiceMock lifecycle_notifier_; + + Singleton::ManagerPtr singleton_manager_; + testing::NiceMock admin_; + Event::GlobalTimeSystem time_system_; + testing::NiceMock api_; + Grpc::ContextImpl grpc_context_; + Router::ContextImpl router_context_; + envoy::config::bootstrap::v3::Bootstrap bootstrap_; + testing::NiceMock options_; +}; + +// Stateless mock ServerFactoryContext for cases where it needs to be used concurrently in different +// threads. Global state in the MockServerFactoryContext causes thread safety issues in this case. +class StatelessMockServerFactoryContext : public virtual ServerFactoryContext { +public: + StatelessMockServerFactoryContext() = default; + ~StatelessMockServerFactoryContext() override = default; + + MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); + MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); + MOCK_METHOD(const Server::Options&, options, ()); + MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); + MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); + MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); + MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(Stats::Scope&, serverScope, ()); + MOCK_METHOD(Singleton::Manager&, singletonManager, ()); + MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); + MOCK_METHOD(OptRef, admin, ()); + MOCK_METHOD(TimeSource&, timeSource, ()); + MOCK_METHOD(Event::TestTimeSystem&, timeSystem, ()); + MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); + MOCK_METHOD(Api::Api&, api, ()); + MOCK_METHOD(Grpc::Context&, grpcContext, ()); + MOCK_METHOD(Router::Context&, routerContext, ()); + MOCK_METHOD(envoy::config::bootstrap::v3::Bootstrap&, bootstrap, ()); + MOCK_METHOD(Server::DrainManager&, drainManager, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); + MOCK_METHOD(StatsConfig&, statsConfig, (), ()); + MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); +}; + +} // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/server_factory_context.cc b/test/mocks/server/server_factory_context.cc deleted file mode 100644 index 0acd1b671908..000000000000 --- a/test/mocks/server/server_factory_context.cc +++ /dev/null @@ -1,43 +0,0 @@ -#include "test/mocks/server/server_factory_context.h" - -namespace Envoy { -namespace Server { -namespace Configuration { - -using ::testing::Return; -using ::testing::ReturnRef; - -MockServerFactoryContext::MockServerFactoryContext() - : singleton_manager_(new Singleton::ManagerImpl(Thread::threadFactoryForTest())), - grpc_context_(store_.symbolTable()), router_context_(store_.symbolTable()) { - ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); - ON_CALL(*this, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); - ON_CALL(*this, drainDecision()).WillByDefault(ReturnRef(drain_manager_)); - ON_CALL(*this, localInfo()).WillByDefault(ReturnRef(local_info_)); - ON_CALL(*this, runtime()).WillByDefault(ReturnRef(runtime_loader_)); - ON_CALL(*this, scope()).WillByDefault(ReturnRef(*store_.rootScope())); - ON_CALL(*this, serverScope()).WillByDefault(ReturnRef(*store_.rootScope())); - ON_CALL(*this, singletonManager()).WillByDefault(ReturnRef(*singleton_manager_)); - ON_CALL(*this, threadLocal()).WillByDefault(ReturnRef(thread_local_)); - ON_CALL(*this, admin()).WillByDefault(Return(OptRef{admin_})); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); - ON_CALL(*this, timeSource()).WillByDefault(ReturnRef(time_system_)); - ON_CALL(*this, messageValidationContext()).WillByDefault(ReturnRef(validation_context_)); - ON_CALL(*this, messageValidationVisitor()) - .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); - ON_CALL(*this, drainManager()).WillByDefault(ReturnRef(drain_manager_)); - ON_CALL(*this, statsConfig()).WillByDefault(ReturnRef(stats_config_)); - ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); - ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); - ON_CALL(*this, lifecycleNotifier()).WillByDefault(ReturnRef(lifecycle_notifier_)); - ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); -} -MockServerFactoryContext::~MockServerFactoryContext() = default; - -MockStatsConfig::MockStatsConfig() = default; -MockStatsConfig::~MockStatsConfig() = default; - -} // namespace Configuration -} // namespace Server -} // namespace Envoy diff --git a/test/mocks/server/server_factory_context.h b/test/mocks/server/server_factory_context.h deleted file mode 100644 index 77a2610680c0..000000000000 --- a/test/mocks/server/server_factory_context.h +++ /dev/null @@ -1,140 +0,0 @@ -#pragma once - -#include "envoy/server/factory_context.h" - -#include "source/common/grpc/context_impl.h" -#include "source/common/http/context_impl.h" -#include "source/common/quic/quic_stat_names.h" -#include "source/common/router/context_impl.h" -#include "source/common/stats/symbol_table.h" -#include "source/extensions/transport_sockets/tls/context_manager_impl.h" - -#include "test/mocks/access_log/mocks.h" -#include "test/mocks/api/mocks.h" -#include "test/mocks/event/mocks.h" -#include "test/mocks/http/mocks.h" -#include "test/mocks/init/mocks.h" -#include "test/mocks/local_info/mocks.h" -#include "test/mocks/network/mocks.h" -#include "test/mocks/protobuf/mocks.h" -#include "test/mocks/runtime/mocks.h" -#include "test/mocks/secret/mocks.h" -#include "test/mocks/server/admin.h" -#include "test/mocks/server/drain_manager.h" -#include "test/mocks/server/hot_restart.h" -#include "test/mocks/server/listener_manager.h" -#include "test/mocks/server/options.h" -#include "test/mocks/server/overload_manager.h" -#include "test/mocks/server/server_lifecycle_notifier.h" -#include "test/mocks/stats/mocks.h" -#include "test/mocks/thread_local/mocks.h" -#include "test/mocks/tracing/mocks.h" -#include "test/mocks/upstream/cluster_manager.h" - -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -namespace Envoy { -namespace Server { -namespace Configuration { - -class MockStatsConfig : public virtual StatsConfig { -public: - MockStatsConfig(); - ~MockStatsConfig() override; - - MOCK_METHOD(const std::list&, sinks, (), (const)); - MOCK_METHOD(std::chrono::milliseconds, flushInterval, (), (const)); - MOCK_METHOD(bool, flushOnAdmin, (), (const)); - MOCK_METHOD(const Stats::SinkPredicates*, sinkPredicates, (), (const)); -}; - -class MockServerFactoryContext : public virtual ServerFactoryContext { -public: - MockServerFactoryContext(); - ~MockServerFactoryContext() override; - - MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); - MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); - MOCK_METHOD(Stats::Scope&, scope, ()); - MOCK_METHOD(Stats::Scope&, serverScope, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); - MOCK_METHOD(OptRef, admin, ()); - MOCK_METHOD(TimeSource&, timeSource, ()); - Event::TestTimeSystem& timeSystem() { return time_system_; } - MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); - Grpc::Context& grpcContext() override { return grpc_context_; } - Router::Context& routerContext() override { return router_context_; } - envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return bootstrap_; } - MOCK_METHOD(Server::DrainManager&, drainManager, ()); - MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); - MOCK_METHOD(StatsConfig&, statsConfig, (), ()); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); - - testing::NiceMock cluster_manager_; - testing::NiceMock dispatcher_; - testing::NiceMock drain_manager_; - testing::NiceMock local_info_; - testing::NiceMock runtime_loader_; - testing::NiceMock store_; - testing::NiceMock thread_local_; - testing::NiceMock validation_context_; - testing::NiceMock stats_config_; - testing::NiceMock access_log_manager_; - testing::NiceMock init_manager_; - testing::NiceMock lifecycle_notifier_; - - Singleton::ManagerPtr singleton_manager_; - testing::NiceMock admin_; - Event::GlobalTimeSystem time_system_; - testing::NiceMock api_; - Grpc::ContextImpl grpc_context_; - Router::ContextImpl router_context_; - envoy::config::bootstrap::v3::Bootstrap bootstrap_; - testing::NiceMock options_; -}; - -// Stateless mock ServerFactoryContext for cases where it needs to be used concurrently in different -// threads. Global state in the MockServerFactoryContext causes thread safety issues in this case. -class StatelessMockServerFactoryContext : public virtual ServerFactoryContext { -public: - StatelessMockServerFactoryContext() = default; - ~StatelessMockServerFactoryContext() override = default; - - MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); - MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); - MOCK_METHOD(Stats::Scope&, scope, ()); - MOCK_METHOD(Stats::Scope&, serverScope, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); - MOCK_METHOD(OptRef, admin, ()); - MOCK_METHOD(TimeSource&, timeSource, ()); - MOCK_METHOD(Event::TestTimeSystem&, timeSystem, ()); - MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); - MOCK_METHOD(Grpc::Context&, grpcContext, ()); - MOCK_METHOD(Router::Context&, routerContext, ()); - MOCK_METHOD(envoy::config::bootstrap::v3::Bootstrap&, bootstrap, ()); - MOCK_METHOD(Server::DrainManager&, drainManager, ()); - MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); - MOCK_METHOD(StatsConfig&, statsConfig, (), ()); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); -}; - -} // namespace Configuration -} // namespace Server -} // namespace Envoy diff --git a/test/mocks/server/transport_socket_factory_context.cc b/test/mocks/server/transport_socket_factory_context.cc index 4de5eff68b6a..2f45814530f8 100644 --- a/test/mocks/server/transport_socket_factory_context.cc +++ b/test/mocks/server/transport_socket_factory_context.cc @@ -1,4 +1,4 @@ -#include "test/mocks/server/transport_socket_factory_context.h" +#include "transport_socket_factory_context.h" #include @@ -12,13 +12,17 @@ namespace Configuration { using ::testing::ReturnRef; MockTransportSocketFactoryContext::MockTransportSocketFactoryContext() - : secret_manager_(std::make_unique(config_tracker_)) { - ON_CALL(*this, serverFactoryContext()).WillByDefault(ReturnRef(server_context_)); + : secret_manager_(std::make_unique(config_tracker_)), + singleton_manager_(Thread::threadFactoryForTest()) { ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); + ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); ON_CALL(*this, messageValidationVisitor()) .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); ON_CALL(*this, sslContextManager()).WillByDefault(ReturnRef(context_manager_)); - ON_CALL(*this, statsScope()).WillByDefault(ReturnRef(*store_.rootScope())); + ON_CALL(*this, scope()).WillByDefault(ReturnRef(*store_.rootScope())); + ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); + ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); + ON_CALL(*this, singletonManager()).WillByDefault(ReturnRef(singleton_manager_)); } MockTransportSocketFactoryContext::~MockTransportSocketFactoryContext() = default; diff --git a/test/mocks/server/transport_socket_factory_context.h b/test/mocks/server/transport_socket_factory_context.h index fc43416a6560..ee23330262d7 100644 --- a/test/mocks/server/transport_socket_factory_context.h +++ b/test/mocks/server/transport_socket_factory_context.h @@ -7,17 +7,16 @@ #include "test/mocks/access_log/mocks.h" #include "test/mocks/api/mocks.h" #include "test/mocks/server/options.h" -#include "test/mocks/server/server_factory_context.h" #include "test/mocks/ssl/mocks.h" #include "test/mocks/stats/mocks.h" #include "test/mocks/upstream/cluster_manager.h" +#include "config_tracker.h" #include "gmock/gmock.h" namespace Envoy { namespace Server { namespace Configuration { - class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { public: MockTransportSocketFactoryContext(); @@ -25,19 +24,32 @@ class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { Secret::SecretManager& secretManager() override { return *(secret_manager_); } - MOCK_METHOD(ServerFactoryContext&, serverFactoryContext, ()); - MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); + MOCK_METHOD(ServerFactoryContext&, getServerFactoryContext, ()); + MOCK_METHOD(OptRef, admin, ()); MOCK_METHOD(Ssl::ContextManager&, sslContextManager, ()); - MOCK_METHOD(Stats::Scope&, statsScope, ()); + MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); + MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); + MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); + MOCK_METHOD(const Server::Options&, options, ()); + MOCK_METHOD(Envoy::Random::RandomGenerator&, random, ()); + MOCK_METHOD(Stats::Store&, stats, ()); MOCK_METHOD(Init::Manager&, initManager, ()); + MOCK_METHOD(Singleton::Manager&, singletonManager, ()); + MOCK_METHOD(ThreadLocal::SlotAllocator&, threadLocal, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); + MOCK_METHOD(Api::Api&, api, ()); + MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, ()); - testing::NiceMock server_context_; testing::NiceMock cluster_manager_; + testing::NiceMock api_; testing::NiceMock config_tracker_; testing::NiceMock context_manager_; testing::NiceMock store_; + testing::NiceMock options_; std::unique_ptr secret_manager_; + testing::NiceMock access_log_manager_; + Singleton::ManagerImpl singleton_manager_; }; } // namespace Configuration } // namespace Server From f63f04e12660cd06ad1d58aed01b017772aa198e Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 11 May 2023 10:53:48 +0100 Subject: [PATCH 200/740] Revert "deps: Bump `bazel` -> 6.2.0 (#27315)" (#27349) This reverts commit 11ebacd6f1136541896b420022f59a357ebe4629. Signed-off-by: Ryan Northey --- .azure-pipelines/env.yml | 2 +- .azure-pipelines/pipelines.yml | 2 +- .bazelversion | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/env.yml b/.azure-pipelines/env.yml index 82553c615d66..aa9d16de691a 100644 --- a/.azure-pipelines/env.yml +++ b/.azure-pipelines/env.yml @@ -105,7 +105,7 @@ jobs: echo "##vso[task.setvariable variable=docsOnly;isoutput=true]${CHANGED_DOCS_ONLY}" echo "##vso[task.setvariable variable=examplesOnly;isoutput=true]${CHANGED_EXAMPLES_ONLY}" - CHANGED_REQUIREMENT_PATHS="$(git diff --name-only ${CHANGE_TARGET}...HEAD | grep -E '\.bazelversion|\.*requirements.*\.txt$|go.mod$|\.*\.bzl$|^WORKSPACE$' || echo '')" + CHANGED_REQUIREMENT_PATHS="$(git diff --name-only ${CHANGE_TARGET}...HEAD | grep -E '\.*requirements.*\.txt$|go.mod$|\.*\.bzl$|^WORKSPACE$' || echo '')" if [[ -n "$CHANGED_REQUIREMENT_PATHS" ]]; then CHANGED_REQUIREMENTS=true diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 3c22420d6599..e7bbb8c78070 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -48,7 +48,7 @@ variables: - name: cacheKeyBazel value: v0 - name: cacheKeyBazelFiles - value: '.bazelversion | ./WORKSPACE | **/*.bzl, !mobile/**, !envoy-docs/**' + value: './WORKSPACE | **/*.bzl, !mobile/**, !envoy-docs/**' - name: authGithubSSHKeyPublic value: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" diff --git a/.bazelversion b/.bazelversion index 6abaeb2f9072..dfda3e0b4f01 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.2.0 +6.1.0 From 6f44a9ba4e7da1e0fb7f5632f6a923b1a201fbd5 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 11 May 2023 11:26:33 +0100 Subject: [PATCH 201/740] ci: Fix test/artefact conditions (#27347) Signed-off-by: Ryan Northey --- .azure-pipelines/bazel.yml | 4 ++-- .azure-pipelines/stage/checks.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index 660287576ec7..804db47035f7 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -217,10 +217,10 @@ steps: testRunTitle: "${{ parameters.ciTarget }}" searchFolder: $(Build.StagingDirectory)/bazel_root timeoutInMinutes: 10 - condition: eq('${{ parameters.publishTestResults }}', 'true') + condition: eq(${{ parameters.publishTestResults }}, 'true') - task: PublishBuildArtifacts@1 inputs: pathtoPublish: "$(Build.StagingDirectory)/envoy" artifactName: ${{ parameters.ciTarget }} timeoutInMinutes: 10 - condition: eq('${{ parameters.publishEnvoy }}', 'true') + condition: eq(${{ parameters.publishEnvoy }}, 'true') diff --git a/.azure-pipelines/stage/checks.yml b/.azure-pipelines/stage/checks.yml index c64ab6bd8422..d82297a46272 100644 --- a/.azure-pipelines/stage/checks.yml +++ b/.azure-pipelines/stage/checks.yml @@ -77,8 +77,8 @@ jobs: cacheTestResults: ${{ parameters.cacheTestResults }} repoFetchDepth: $(REPO_FETCH_DEPTH) repoFetchTags: $(REPO_FETCH_TAGS) - publishTestResults: $(PUBLISH_TEST_RESULTS) - publishEnvoy: $(PUBLISH_ENVOY) + publishTestResults: variables.PUBLISH_TEST_RESULTS + publishEnvoy: variables.PUBLISH_ENVOY stepsPost: # TODO(phlax): consolidate "fix" paths/jobs From 0a2cf15c179379c877ed0825be6c8849b0d9bd20 Mon Sep 17 00:00:00 2001 From: zirain Date: Thu, 11 May 2023 19:22:18 +0800 Subject: [PATCH 202/740] coverage test: make LLVM_VERSION configurable (#27352) Signed-off-by: hejianpeng --- test/run_envoy_bazel_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 859bcd0d6fb6..0ddb64a7e9b6 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -2,7 +2,7 @@ set -e -o pipefail -LLVM_VERSION="14.0.0" +LLVM_VERSION=${LLVM_VERSION:-"14.0.0"} CLANG_VERSION=$(clang --version | grep version | sed -e 's/\ *clang version \(.*\)\ */\1/') LLVM_COV_VERSION=$(llvm-cov --version | grep version | sed -e 's/\ *LLVM version \(.*\)/\1/') LLVM_PROFDATA_VERSION=$(llvm-profdata show --version | grep version | sed -e 's/\ *LLVM version \(.*\)/\1/') From c765bceac290b5bc34bd29786c81e42632ebca50 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 11 May 2023 09:43:00 -0400 Subject: [PATCH 203/740] http: adding test coverage (#27275) Signed-off-by: Alyssa Wilk --- source/common/http/codec_client.cc | 3 +-- source/common/http/codec_wrappers.h | 4 ---- test/common/http/codec_wrappers_test.cc | 22 ++++++++++++++++++++++ test/per_file_coverage.sh | 1 - 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index a6affd33780b..a4dab04c81bb 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -266,13 +266,12 @@ NoConnectCodecClientProd::NoConnectCodecClientProd( host->cluster().maxResponseHeadersCount(), proxied); break; } - case CodecType::HTTP2: { + case CodecType::HTTP2: codec_ = std::make_unique( *connection_, *this, host->cluster().http2CodecStats(), random_generator, host->cluster().http2Options(), Http::DEFAULT_MAX_REQUEST_HEADERS_KB, host->cluster().maxResponseHeadersCount(), Http2::ProdNghttp2SessionFactory::get()); break; - } case CodecType::HTTP3: { #ifdef ENVOY_ENABLE_QUIC auto& quic_session = dynamic_cast(*connection_); diff --git a/source/common/http/codec_wrappers.h b/source/common/http/codec_wrappers.h index 0685757c4ff0..7b98413e1a5a 100644 --- a/source/common/http/codec_wrappers.h +++ b/source/common/http/codec_wrappers.h @@ -19,9 +19,7 @@ class ResponseDecoderWrapper : public ResponseDecoder { if (end_stream) { onPreDecodeComplete(); } - inner_.decodeHeaders(std::move(headers), end_stream); - if (end_stream) { onDecodeComplete(); } @@ -31,9 +29,7 @@ class ResponseDecoderWrapper : public ResponseDecoder { if (end_stream) { onPreDecodeComplete(); } - inner_.decodeData(data, end_stream); - if (end_stream) { onDecodeComplete(); } diff --git a/test/common/http/codec_wrappers_test.cc b/test/common/http/codec_wrappers_test.cc index 58dca68c0ca4..35bf742a03f0 100644 --- a/test/common/http/codec_wrappers_test.cc +++ b/test/common/http/codec_wrappers_test.cc @@ -8,6 +8,25 @@ using testing::_; namespace Envoy { namespace Http { +class MockResponseDecoderWrapper : public ResponseDecoderWrapper { +public: + MockResponseDecoderWrapper() : ResponseDecoderWrapper(inner_decoder_) {} + MockResponseDecoder& innerEncoder() { return inner_decoder_; } + void onDecodeComplete() override {} + void onPreDecodeComplete() override {} + +private: + MockResponseDecoder inner_decoder_; +}; + +TEST(MockResponseDecoderWrapper, dumpState) { + MockResponseDecoderWrapper wrapper; + + std::stringstream os; + EXPECT_CALL(wrapper.innerEncoder(), dumpState(_, _)); + wrapper.dumpState(os, 0); +} + class MockRequestEncoderWrapper : public RequestEncoderWrapper { public: MockRequestEncoderWrapper() : RequestEncoderWrapper(&inner_encoder_) {} @@ -69,6 +88,9 @@ TEST(RequestEncoderWrapper, HeaderAndBodyAndTrailersEncode) { wrapper.encodeData(data, false); EXPECT_FALSE(wrapper.encodeComplete()); + EXPECT_CALL(wrapper.innerEncoder(), http1StreamEncoderOptions()); + wrapper.http1StreamEncoderOptions(); + EXPECT_CALL(wrapper.innerEncoder(), encodeTrailers(_)); wrapper.encodeTrailers(TestRequestTrailerMapImpl{{"trailing", "header"}}); EXPECT_TRUE(wrapper.encodeComplete()); diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index c3c94ccd8be3..31f2a1933732 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -11,7 +11,6 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/crypto:88.1" "source/common/event:95.1" # Emulated edge events guards don't report LCOV "source/common/filesystem/posix:96.2" # FileReadToEndNotReadable fails in some env; createPath can't test all failure branches. -"source/common/http:96.5" "source/common/http/http2:95.2" "source/common/json:93.4" "source/common/matcher:94.6" From b978af64641aa46d3b7012ec1d9f3d7d9fd3b307 Mon Sep 17 00:00:00 2001 From: "Aaron.Zhang" Date: Thu, 11 May 2023 21:52:11 +0800 Subject: [PATCH 204/740] api-description: remove duplicated comments words (#27337) Signed-off-by: bozhang --- api/envoy/config/cluster/v3/cluster.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto index 27b75a87e123..7ecf42425791 100644 --- a/api/envoy/config/cluster/v3/cluster.proto +++ b/api/envoy/config/cluster/v3/cluster.proto @@ -719,7 +719,7 @@ message Cluster { google.protobuf.DoubleValue per_upstream_preconnect_ratio = 1 [(validate.rules).double = {lte: 3.0 gte: 1.0}]; - // Indicates how many many streams (rounded up) can be anticipated across a cluster for each + // Indicates how many streams (rounded up) can be anticipated across a cluster for each // stream, useful for low QPS services. This is currently supported for a subset of // deterministic non-hash-based load-balancing algorithms (weighted round robin, random). // Unlike ``per_upstream_preconnect_ratio`` this preconnects across the upstream instances in a From 7d65b00cc7e414e63bbd782214f7a0c9a4c80210 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 11 May 2023 16:05:24 +0100 Subject: [PATCH 205/740] do/ci: Reorder case statements and improve block spacing (#27271) Signed-off-by: Ryan Northey --- ci/do_ci.sh | 606 +++++++++++++++++++++++++--------------------------- 1 file changed, 295 insertions(+), 311 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 977bd169dc59..9c82fac994f3 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -214,147 +214,45 @@ if [[ "$CI_TARGET" =~ bazel.* ]]; then fi case $CI_TARGET in - release) - # When testing memory consumption, we want to test against exact byte-counts - # where possible. As these differ between platforms and compile options, we - # define the 'release' builds as canonical and test them only in CI, so the - # toolchain is kept consistent. This ifdef is checked in - # test/common/stats/stat_test_utility.cc when computing - # Stats::TestUtil::MemoryTest::mode(). - if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - BAZEL_BUILD_OPTIONS+=("--test_env=ENVOY_MEMORY_TEST_EXACT=true") - fi - + api) + # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib, + # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild. + ENVOY_STDLIB="libstdc++" setup_clang_toolchain - - ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" - mkdir -p "$ENVOY_BINARY_DIR" - - # As the binary build package enforces compiler options, adding here to ensure the tests and distribution build - # reuse settings and any already compiled artefacts, the bundle itself will always be compiled - # `--stripopt=--strip-all -c opt` - BAZEL_RELEASE_OPTIONS=( - --stripopt=--strip-all - -c opt) - - # Run release tests - echo "Testing with:" - echo " targets: ${TEST_TARGETS[*]}" - echo " build options: ${BAZEL_BUILD_OPTIONS[*]}" - echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}" - + export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config + echo "Run protoxform test" + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + --//tools/api_proto_plugin:default_type_db_target=//tools/testdata/protoxform:fix_protos \ + --//tools/api_proto_plugin:extra_args=api_version:3.7 \ + //tools/protoprint:protoprint_test + echo "Validating API structure..." + "${ENVOY_SRCDIR}"/tools/api/validate_structure.py + echo "Validate Golang protobuf generation..." + "${ENVOY_SRCDIR}"/tools/api/generate_go_protobuf.py + echo "Testing API..." bazel_with_collection \ test "${BAZEL_BUILD_OPTIONS[@]}" \ --remote_download_minimal \ - "${BAZEL_RELEASE_OPTIONS[@]}" \ - "${TEST_TARGETS[@]}" - - # Build release binaries - bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" \ - //distribution/binary:release - - # Copy release binaries to binary export directory - cp -a \ - "bazel-bin/distribution/binary/release.tar.zst" \ - "${ENVOY_BINARY_DIR}/release.tar.zst" - - # Grab the schema_validator_tool - # TODO(phlax): bundle this with the release when #26390 is resolved - bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" \ - --remote_download_toplevel \ - //test/tools/schema_validator:schema_validator_tool.stripped - - # Copy schema_validator_tool to binary export directory - cp -a \ - bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped \ - "${ENVOY_BINARY_DIR}/schema_validator_tool" - - ;; - distribution) - echo "Building distro packages..." - - setup_clang_toolchain - - # Extract the Envoy binary from the tarball - mkdir -p distribution/custom - if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - ENVOY_RELEASE_TARBALL="/build/bazel.release/x64/bin/release.tar.zst" - else - ENVOY_RELEASE_TARBALL="/build/bazel.release/arm64/bin/release.tar.zst" - fi - bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ - //tools/zstd \ - -- --stdout \ - -d "$ENVOY_RELEASE_TARBALL" \ - | tar xfO - envoy > distribution/custom/envoy - - # Build the packages + -c fastbuild \ + @envoy_api//test/... \ + @envoy_api//tools/... \ + @envoy_api//tools:tap2pcap_test + echo "Building API..." bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ - --remote_download_toplevel \ - -c opt \ - --//distribution:envoy-binary=//distribution:custom/envoy \ - //distribution:packages.tar.gz - - if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.x64.tar.gz" - else - cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.arm64.tar.gz" - fi - ;; - release.server_only) - setup_clang_toolchain - echo "bazel release build..." - bazel_envoy_binary_build release - ;; - sizeopt.server_only) - setup_clang_toolchain - echo "bazel size optimized build..." - bazel_envoy_binary_build sizeopt - ;; - sizeopt) - setup_clang_toolchain - echo "Testing ${TEST_TARGETS[*]}" - bazel_with_collection \ - test "${BAZEL_BUILD_OPTIONS[@]}" \ - --config=sizeopt \ - "${TEST_TARGETS[@]}" - - echo "bazel size optimized build with tests..." - bazel_envoy_binary_build sizeopt + -c fastbuild @envoy_api//envoy/... ;; - gcc) - BAZEL_BUILD_OPTIONS+=("--test_env=HEAPCHECK=") - setup_gcc_toolchain - - echo "Testing ${TEST_TARGETS[*]}" - bazel_with_collection \ - test "${BAZEL_BUILD_OPTIONS[@]}" \ - -c fastbuild \ - --remote_download_minimal \ - -- "${TEST_TARGETS[@]}" - echo "bazel release build with gcc..." - bazel_envoy_binary_build fastbuild + api_compat) + echo "Checking API for breaking changes to protobuf backwards compatibility..." + BASE_BRANCH_REF=$("${ENVOY_SRCDIR}"/tools/git/last_github_commit.sh) + COMMIT_TITLE=$(git log -n 1 --pretty='format:%C(auto)%h (%s, %ad)' "${BASE_BRANCH_REF}") + echo -e "\tUsing base commit ${COMMIT_TITLE}" + # BAZEL_BUILD_OPTIONS needed for setting the repository_cache param. + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/api_proto_breaking_change_detector:detector_ci \ + "${BASE_BRANCH_REF}" ;; - debug) - setup_clang_toolchain - echo "Testing ${TEST_TARGETS[*]}" - # Make sure that there are no regressions to building Envoy with autolink disabled. - EXTRA_OPTIONS=( - "--define" "library_autolink=disabled") - bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ - "${EXTRA_OPTIONS[@]}" \ - -c dbg \ - "${TEST_TARGETS[@]}" - echo "bazel debug build with tests..." - bazel_envoy_binary_build debug - ;; - debug.server_only) - setup_clang_toolchain - echo "bazel debug build..." - bazel_envoy_binary_build debug - ;; asan) setup_clang_toolchain BAZEL_BUILD_OPTIONS+=( @@ -373,7 +271,6 @@ case $CI_TARGET in "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" popd fi - # TODO(mattklein123): This part of the test is now flaky in CI and it's unclear why, possibly # due to sandboxing issue. Debug and enable it again. # if [ "${CI_SKIP_INTEGRATION_TEST_TRAFFIC_TAPPING}" != "1" ] ; then @@ -386,71 +283,43 @@ case $CI_TARGET in # //test/extensions/transport_sockets/tls/integration:ssl_integration_test # fi ;; - tsan) + + check_and_fix_proto_format) setup_clang_toolchain - echo "bazel TSAN debug build with tests" - echo "Building and testing envoy tests ${TEST_TARGETS[*]}" - bazel_with_collection \ - test "${BAZEL_BUILD_OPTIONS[@]}" \ - --config=rbe-toolchain-tsan \ - -c dbg \ - --build_tests_only \ - --remote_download_minimal \ - "${TEST_TARGETS[@]}" - if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then - echo "Building and testing envoy-filter-example tests..." - pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" - bazel_with_collection \ - test "${BAZEL_BUILD_OPTIONS[@]}" \ - -c dbg \ - --config=clang-tsan \ - "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" - popd - fi + echo "Check and fix proto format ..." + "${ENVOY_SRCDIR}/ci/check_and_fix_format.sh" ;; - msan) - ENVOY_STDLIB=libc++ + + check_proto_format) setup_clang_toolchain - # rbe-toolchain-msan must comes as first to win library link order. - BAZEL_BUILD_OPTIONS=( - "--config=rbe-toolchain-msan" - "${BAZEL_BUILD_OPTIONS[@]}" - "-c" "dbg" - "--build_tests_only" - "--remote_download_minimal") - echo "bazel MSAN debug build with tests" - echo "Building and testing envoy tests ${TEST_TARGETS[*]}" - bazel_with_collection \ - test "${BAZEL_BUILD_OPTIONS[@]}" \ - -- "${TEST_TARGETS[@]}" + echo "Check proto format ..." + "${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" check ;; - dev) - setup_clang_toolchain - # This doesn't go into CI but is available for developer convenience. - echo "bazel fastbuild build with tests..." - echo "Building..." - bazel_envoy_binary_build fastbuild - echo "Testing ${TEST_TARGETS[*]}" - bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ - -c fastbuild "${TEST_TARGETS[@]}" - ;; - dev.contrib) + clang_tidy) + # clang-tidy will warn on standard library issues with libc++ + ENVOY_STDLIB="libstdc++" setup_clang_toolchain - # This doesn't go into CI but is available for developer convenience. - echo "bazel fastbuild build with contrib extensions and tests..." - echo "Building..." - bazel_contrib_binary_build fastbuild - - echo "Testing ${TEST_TARGETS[*]}" - bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ - -c fastbuild \ - "${TEST_TARGETS[@]}" + export CLANG_TIDY_FIX_DIFF="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixed.diff" + export FIX_YAML="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixes.yaml" + export CLANG_TIDY_APPLY_FIXES=1 + mkdir -p "${ENVOY_TEST_TMPDIR}/lint-fixes" + BAZEL_BUILD_OPTIONS+=(--remote_download_minimal) + NUM_CPUS=$NUM_CPUS "${ENVOY_SRCDIR}"/ci/run_clang_tidy.sh "$@" || { + if [[ -s "$FIX_YAML" ]]; then + echo >&2 + echo "Diff/yaml files with (some) fixes will be uploaded. Please check the artefacts for this PR run in the azure pipeline." >&2 + echo >&2 + else + echo "Clang-tidy failed." >&2 + fi + exit 1 + } ;; + compile_time_options) # Right now, none of the available compile-time options conflict with each other. If this # changes, this build type may need to be broken up. - COMPILE_TIME_OPTIONS=( "--define" "admin_html=disabled" "--define" "signal_trace=disabled" @@ -467,15 +336,12 @@ case $CI_TARGET in "--@envoy//source/extensions/filters/http/kill_request:enabled" "--test_env=ENVOY_HAS_EXTRA_EXTENSIONS=true" "--remote_download_minimal") - ENVOY_STDLIB="${ENVOY_STDLIB:-libstdc++}" setup_clang_toolchain # This doesn't go into CI but is available for developer convenience. echo "bazel with different compiletime options build with tests..." - cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" TEST_TARGETS=("${TEST_TARGETS[@]/#\/\//@envoy\/\/}") - # Building all the dependencies from scratch to link them against libc++. echo "Building and testing with wasm=wamr: ${TEST_TARGETS[*]}" bazel_with_collection \ @@ -486,7 +352,6 @@ case $CI_TARGET in "${TEST_TARGETS[@]}" \ --test_tag_filters=-nofips \ --build_tests_only - echo "Building and testing with wasm=wasmtime: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}" bazel_with_collection \ test "${BAZEL_BUILD_OPTIONS[@]}" \ @@ -498,7 +363,6 @@ case $CI_TARGET in "${TEST_TARGETS[@]}" \ --test_tag_filters=-nofips \ --build_tests_only - echo "Building and testing with wasm=wavm: ${TEST_TARGETS[*]}" bazel_with_collection \ test "${BAZEL_BUILD_OPTIONS[@]}" \ @@ -508,7 +372,6 @@ case $CI_TARGET in "${TEST_TARGETS[@]}" \ --test_tag_filters=-nofips \ --build_tests_only - # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only # these tests under "-c opt" to save time in CI. bazel_with_collection \ @@ -518,7 +381,6 @@ case $CI_TARGET in -c opt \ @envoy//test/common/common:assert_test \ @envoy//test/server:server_test - # "--define log_fast_debug_assert_in_release=enabled" must be tested with a release build, so run only these tests under "-c opt" to save time in CI. This option will test only ASSERT()s without SLOW_ASSERT()s, so additionally disable "--define log_debug_assert_in_release" which compiles in both. bazel_with_collection \ test "${BAZEL_BUILD_OPTIONS[@]}" \ @@ -528,7 +390,6 @@ case $CI_TARGET in @envoy//test/common/common:assert_test \ --define log_fast_debug_assert_in_release=enabled \ --define log_debug_assert_in_release=disabled - echo "Building binary with wasm=wavm... and logging disabled" bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ --define wasm=wavm \ @@ -539,146 +400,62 @@ case $CI_TARGET in --build_tag_filters=-nofips collect_build_profile build ;; - api) - # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib, - # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild. - ENVOY_STDLIB="libstdc++" - setup_clang_toolchain - export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config - echo "Run protoxform test" - bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ - --//tools/api_proto_plugin:default_type_db_target=//tools/testdata/protoxform:fix_protos \ - --//tools/api_proto_plugin:extra_args=api_version:3.7 \ - //tools/protoprint:protoprint_test - echo "Validating API structure..." - "${ENVOY_SRCDIR}"/tools/api/validate_structure.py - echo "Validate Golang protobuf generation..." - "${ENVOY_SRCDIR}"/tools/api/generate_go_protobuf.py - echo "Testing API..." - bazel_with_collection \ - test "${BAZEL_BUILD_OPTIONS[@]}" \ - --remote_download_minimal \ - -c fastbuild \ - @envoy_api//test/... \ - @envoy_api//tools/... \ - @envoy_api//tools:tap2pcap_test - echo "Building API..." - bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ - -c fastbuild @envoy_api//envoy/... - ;; - api_compat) - echo "Checking API for breaking changes to protobuf backwards compatibility..." - BASE_BRANCH_REF=$("${ENVOY_SRCDIR}"/tools/git/last_github_commit.sh) - COMMIT_TITLE=$(git log -n 1 --pretty='format:%C(auto)%h (%s, %ad)' "${BASE_BRANCH_REF}") - echo -e "\tUsing base commit ${COMMIT_TITLE}" - # BAZEL_BUILD_OPTIONS needed for setting the repository_cache param. - bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ - //tools/api_proto_breaking_change_detector:detector_ci \ - "${BASE_BRANCH_REF}" - ;; + coverage|fuzz_coverage) setup_clang_toolchain - echo "${CI_TARGET} build with tests ${COVERAGE_TEST_TARGETS[*]}" - if [[ "$CI_TARGET" == "fuzz_coverage" ]]; then export FUZZ_COVERAGE=true fi - # We use custom BAZEL_BUILD_OPTIONS here to cover profiler's code. BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]} --define tcmalloc=gperftools" \ "${ENVOY_SRCDIR}/test/run_envoy_bazel_coverage.sh" \ "${COVERAGE_TEST_TARGETS[@]}" collect_build_profile coverage ;; + coverage-upload|fuzz_coverage-upload) setup_clang_toolchain - if [[ "$CI_TARGET" == "fuzz_coverage-upload" ]]; then TARGET=fuzz_coverage else TARGET=coverage fi - "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" "/source/generated/${TARGET}" "$TARGET" ;; - clang_tidy) - # clang-tidy will warn on standard library issues with libc++ - ENVOY_STDLIB="libstdc++" - setup_clang_toolchain - export CLANG_TIDY_FIX_DIFF="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixed.diff" - export FIX_YAML="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixes.yaml" - export CLANG_TIDY_APPLY_FIXES=1 - mkdir -p "${ENVOY_TEST_TMPDIR}/lint-fixes" - BAZEL_BUILD_OPTIONS+=(--remote_download_minimal) - NUM_CPUS=$NUM_CPUS "${ENVOY_SRCDIR}"/ci/run_clang_tidy.sh "$@" || { - if [[ -s "$FIX_YAML" ]]; then - echo >&2 - echo "Diff/yaml files with (some) fixes will be uploaded. Please check the artefacts for this PR run in the azure pipeline." >&2 - echo >&2 - else - echo "Clang-tidy failed." >&2 - fi - exit 1 - } - ;; - fuzz) - setup_clang_toolchain - FUZZ_TEST_TARGETS=("$(bazel query "${BAZEL_GLOBAL_OPTIONS[@]}" "attr('tags','fuzzer',${TEST_TARGETS[*]})")") - echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS[*]}" - echo "Building envoy fuzzers and executing 100 fuzz iterations..." - bazel_with_collection \ - test "${BAZEL_BUILD_OPTIONS[@]}" \ - --config=asan-fuzzer \ - "${FUZZ_TEST_TARGETS[@]}" \ - --test_arg="-runs=10" - ;; - format) - setup_clang_toolchain - "${ENVOY_SRCDIR}/ci/format_pre.sh" - ;; - fix_proto_format) - # proto_format.sh needs to build protobuf. - setup_clang_toolchain - "${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" fix - ;; - check_proto_format) - setup_clang_toolchain - echo "Check proto format ..." - "${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" check - ;; - check_and_fix_proto_format) - setup_clang_toolchain - echo "Check and fix proto format ..." - "${ENVOY_SRCDIR}/ci/check_and_fix_format.sh" - ;; - docs) + debug) setup_clang_toolchain - echo "generating docs..." - # Build docs. - "${ENVOY_SRCDIR}/docs/build.sh" + echo "Testing ${TEST_TARGETS[*]}" + # Make sure that there are no regressions to building Envoy with autolink disabled. + EXTRA_OPTIONS=( + "--define" "library_autolink=disabled") + bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ + "${EXTRA_OPTIONS[@]}" \ + -c dbg \ + "${TEST_TARGETS[@]}" + echo "bazel debug build with tests..." + bazel_envoy_binary_build debug ;; - docs-upload) + + debug.server_only) setup_clang_toolchain - "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" /source/generated/docs docs + echo "bazel debug build..." + bazel_envoy_binary_build debug ;; + deps) setup_clang_toolchain - echo "dependency validate_test..." bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ //tools/dependency:validate_test - echo "verifying dependencies..." # Validate dependency relationships between core/extensions and external deps. time bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ //tools/dependency:validate - # Validate repository metadata. echo "check repositories..." "${ENVOY_SRCDIR}/tools/check_repositories.sh" - echo "check dependencies..." # Using todays date as an action_env expires the NIST cache daily, which is the update frequency TODAY_DATE=$(date -u -I"date") @@ -687,25 +464,128 @@ case $CI_TARGET in --action_env=TODAY_DATE \ -- -v warn \ -c cves release_dates releases - # Run dependabot tests echo "Check dependabot ..." bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ //tools/dependency:dependatool ;; - verify_examples) - run_ci_verify "*" "win32-front-proxy|shared" + + dev) + setup_clang_toolchain + # This doesn't go into CI but is available for developer convenience. + echo "bazel fastbuild build with tests..." + echo "Building..." + bazel_envoy_binary_build fastbuild + echo "Testing ${TEST_TARGETS[*]}" + bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ + -c fastbuild "${TEST_TARGETS[@]}" ;; - verify_distro) + + dev.contrib) + setup_clang_toolchain + # This doesn't go into CI but is available for developer convenience. + echo "bazel fastbuild build with contrib extensions and tests..." + echo "Building..." + bazel_contrib_binary_build fastbuild + echo "Testing ${TEST_TARGETS[*]}" + bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ + -c fastbuild \ + "${TEST_TARGETS[@]}" + ;; + + distribution) + echo "Building distro packages..." + setup_clang_toolchain + # Extract the Envoy binary from the tarball + mkdir -p distribution/custom if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - PACKAGE_BUILD=/build/bazel.distribution/x64/packages.x64.tar.gz + ENVOY_RELEASE_TARBALL="/build/bazel.release/x64/bin/release.tar.zst" else - PACKAGE_BUILD=/build/bazel.distribution/arm64/packages.arm64.tar.gz + ENVOY_RELEASE_TARBALL="/build/bazel.release/arm64/bin/release.tar.zst" fi bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ - //distribution:verify_packages \ - "$PACKAGE_BUILD" + //tools/zstd \ + -- --stdout \ + -d "$ENVOY_RELEASE_TARBALL" \ + | tar xfO - envoy > distribution/custom/envoy + # Build the packages + bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ + --remote_download_toplevel \ + -c opt \ + --//distribution:envoy-binary=//distribution:custom/envoy \ + //distribution:packages.tar.gz + if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.x64.tar.gz" + else + cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.arm64.tar.gz" + fi + ;; + + docs) + setup_clang_toolchain + echo "generating docs..." + # Build docs. + "${ENVOY_SRCDIR}/docs/build.sh" + ;; + + docs-upload) + setup_clang_toolchain + "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" /source/generated/docs docs + ;; + + fix_proto_format) + # proto_format.sh needs to build protobuf. + setup_clang_toolchain + "${ENVOY_SRCDIR}/tools/proto_format/proto_format.sh" fix + ;; + + format) + setup_clang_toolchain + "${ENVOY_SRCDIR}/ci/format_pre.sh" + ;; + + fuzz) + setup_clang_toolchain + FUZZ_TEST_TARGETS=("$(bazel query "${BAZEL_GLOBAL_OPTIONS[@]}" "attr('tags','fuzzer',${TEST_TARGETS[*]})")") + echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS[*]}" + echo "Building envoy fuzzers and executing 100 fuzz iterations..." + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --config=asan-fuzzer \ + "${FUZZ_TEST_TARGETS[@]}" \ + --test_arg="-runs=10" + ;; + + gcc) + BAZEL_BUILD_OPTIONS+=("--test_env=HEAPCHECK=") + setup_gcc_toolchain + echo "Testing ${TEST_TARGETS[*]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + -c fastbuild \ + --remote_download_minimal \ + -- "${TEST_TARGETS[@]}" + echo "bazel release build with gcc..." + bazel_envoy_binary_build fastbuild + ;; + + msan) + ENVOY_STDLIB=libc++ + setup_clang_toolchain + # rbe-toolchain-msan must comes as first to win library link order. + BAZEL_BUILD_OPTIONS=( + "--config=rbe-toolchain-msan" + "${BAZEL_BUILD_OPTIONS[@]}" + "-c" "dbg" + "--build_tests_only" + "--remote_download_minimal") + echo "bazel MSAN debug build with tests" + echo "Building and testing envoy tests ${TEST_TARGETS[*]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + -- "${TEST_TARGETS[@]}" ;; + publish) # If we are on a non-main release branch but the patch version is 0 - then this branch # has just been created, and the tag/release was cut from `main` - there is no need to @@ -718,27 +598,131 @@ case $CI_TARGET in # from the current commit (as this only happens on non-PRs we are safe from merges) BUILD_SHA="$(git rev-parse HEAD)" bazel run "${BAZEL_BUILD_OPTIONS[@]}" @envoy_repo//:publish -- --publish-commitish="$BUILD_SHA" - # TODO(phlax): move this to pytooling mkdir -p linux/amd64 linux/arm64 publish - # linux/amd64 tar xf /build/bazel.release/release.tar.zst -C ./linux/amd64 cp -a linux/amd64/envoy "publish/envoy-${version}-linux-x86_64" cp -a linux/amd64/envoy-contrib "publish/envoy-contrib-${version}-linux-x86_64" - # linux/arm64 tar xf /build/bazel.release.arm64/release.tar.zst -C ./linux/arm64 cp -a linux/arm64/envoy "publish/envoy-${version}-linux-aarch_64" cp -a linux/arm64/envoy-contrib "publish/envoy-contrib-${version}-linux-aarch_64" - "${ENVOY_SRCDIR}/ci/publish_github_assets.sh" "v${version}" "${PWD}/publish" - exit 0 fi fi echo "Not creating a tag/release for ${version} from ${AZP_BRANCH}" ;; + + release) + # When testing memory consumption, we want to test against exact byte-counts + # where possible. As these differ between platforms and compile options, we + # define the 'release' builds as canonical and test them only in CI, so the + # toolchain is kept consistent. This ifdef is checked in + # test/common/stats/stat_test_utility.cc when computing + # Stats::TestUtil::MemoryTest::mode(). + if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + BAZEL_BUILD_OPTIONS+=("--test_env=ENVOY_MEMORY_TEST_EXACT=true") + fi + setup_clang_toolchain + ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" + mkdir -p "$ENVOY_BINARY_DIR" + # As the binary build package enforces compiler options, adding here to ensure the tests and distribution build + # reuse settings and any already compiled artefacts, the bundle itself will always be compiled + # `--stripopt=--strip-all -c opt` + BAZEL_RELEASE_OPTIONS=( + --stripopt=--strip-all + -c opt) + # Run release tests + echo "Testing with:" + echo " targets: ${TEST_TARGETS[*]}" + echo " build options: ${BAZEL_BUILD_OPTIONS[*]}" + echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --remote_download_minimal \ + "${BAZEL_RELEASE_OPTIONS[@]}" \ + "${TEST_TARGETS[@]}" + # Build release binaries + bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" \ + //distribution/binary:release + # Copy release binaries to binary export directory + cp -a \ + "bazel-bin/distribution/binary/release.tar.zst" \ + "${ENVOY_BINARY_DIR}/release.tar.zst" + # Grab the schema_validator_tool + # TODO(phlax): bundle this with the release when #26390 is resolved + bazel build "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_RELEASE_OPTIONS[@]}" \ + --remote_download_toplevel \ + //test/tools/schema_validator:schema_validator_tool.stripped + # Copy schema_validator_tool to binary export directory + cp -a \ + bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped \ + "${ENVOY_BINARY_DIR}/schema_validator_tool" + ;; + + release.server_only) + setup_clang_toolchain + echo "bazel release build..." + bazel_envoy_binary_build release + ;; + + sizeopt) + setup_clang_toolchain + echo "Testing ${TEST_TARGETS[*]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --config=sizeopt \ + "${TEST_TARGETS[@]}" + echo "bazel size optimized build with tests..." + bazel_envoy_binary_build sizeopt + ;; + + sizeopt.server_only) + setup_clang_toolchain + echo "bazel size optimized build..." + bazel_envoy_binary_build sizeopt + ;; + + tsan) + setup_clang_toolchain + echo "bazel TSAN debug build with tests" + echo "Building and testing envoy tests ${TEST_TARGETS[*]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --config=rbe-toolchain-tsan \ + -c dbg \ + --build_tests_only \ + --remote_download_minimal \ + "${TEST_TARGETS[@]}" + if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then + echo "Building and testing envoy-filter-example tests..." + pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + -c dbg \ + --config=clang-tsan \ + "${ENVOY_FILTER_EXAMPLE_TESTS[@]}" + popd + fi + ;; + + verify_distro) + if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + PACKAGE_BUILD=/build/bazel.distribution/x64/packages.x64.tar.gz + else + PACKAGE_BUILD=/build/bazel.distribution/arm64/packages.arm64.tar.gz + fi + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //distribution:verify_packages \ + "$PACKAGE_BUILD" + ;; + + verify_examples) + run_ci_verify "*" "win32-front-proxy|shared" + ;; + *) echo "Invalid do_ci.sh target (${CI_TARGET}), see ci/README.md for valid targets." exit 1 From 98a2bc24b33c4d9e06e606dc4226b05fb081f806 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 11 May 2023 16:14:01 +0100 Subject: [PATCH 206/740] ci: Dont publish anything if cancelled (#27298) Signed-off-by: Ryan Northey --- .azure-pipelines/bazel.yml | 2 +- .azure-pipelines/stage/checks.yml | 2 +- .azure-pipelines/stage/macos.yml | 2 +- .azure-pipelines/stage/publish.yml | 4 ++-- .azure-pipelines/stage/windows.yml | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index 804db47035f7..79a43fa80107 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -191,7 +191,7 @@ steps: # Cleanup offending files with unicode names rm -rf $(Build.StagingDirectory)/bazel_root/base/external/go_sdk/test/fixedbugs displayName: "Check disk space at end" - condition: always() + condition: not(canceled()) - ${{ each step in parameters.stepsPost }}: - ${{ each pair in step }}: diff --git a/.azure-pipelines/stage/checks.yml b/.azure-pipelines/stage/checks.yml index d82297a46272..3c2ef0e9a434 100644 --- a/.azure-pipelines/stage/checks.yml +++ b/.azure-pipelines/stage/checks.yml @@ -126,7 +126,7 @@ jobs: BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)" ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: BAZEL_REMOTE_INSTANCE_BRANCH: "$(Build.SourceBranchName)" - condition: always() + condition: not(canceled()) - job: complete displayName: "Checks complete" diff --git a/.azure-pipelines/stage/macos.yml b/.azure-pipelines/stage/macos.yml index 8c45e063c972..cb3ecfd38825 100644 --- a/.azure-pipelines/stage/macos.yml +++ b/.azure-pipelines/stage/macos.yml @@ -37,7 +37,7 @@ jobs: testResultsFiles: "**/bazel-testlogs/**/test.xml" testRunTitle: "macOS" timeoutInMinutes: 10 - condition: always() + condition: not(canceled()) - job: tested displayName: Complete diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index 4034eeef16cc..32b2cdb91810 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -126,13 +126,13 @@ jobs: echo "disk space at end of build:" df -h displayName: "Check disk space at end" - condition: always() + condition: not(canceled()) - task: PublishBuildArtifacts@1 inputs: pathtoPublish: "$(Build.StagingDirectory)/build_images" artifactName: docker timeoutInMinutes: 10 - condition: always() + condition: not(canceled()) - script: sudo .azure-pipelines/docker/save_cache.sh /mnt/docker_cache displayName: "Cache/save (publish_docker)" diff --git a/.azure-pipelines/stage/windows.yml b/.azure-pipelines/stage/windows.yml index 5f6563b568c7..6b1062f714b9 100644 --- a/.azure-pipelines/stage/windows.yml +++ b/.azure-pipelines/stage/windows.yml @@ -46,13 +46,13 @@ jobs: testRunTitle: "windows" searchFolder: $(Build.StagingDirectory)/tmp timeoutInMinutes: 10 - condition: always() + condition: not(canceled()) - task: PublishBuildArtifacts@1 inputs: pathtoPublish: "$(Build.StagingDirectory)/envoy" artifactName: windows.release timeoutInMinutes: 10 - condition: always() + condition: not(canceled()) - job: docker displayName: Build Docker image @@ -101,7 +101,7 @@ jobs: pathtoPublish: "$(Build.StagingDirectory)/build_images" artifactName: docker_windows timeoutInMinutes: 10 - condition: always() + condition: not(canceled()) - job: released displayName: Complete From ceda88084402e4ca39b8168ba891b550a06e22a0 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 11 May 2023 13:23:08 -0400 Subject: [PATCH 207/740] mobile: add a min delivery size for C++, Java and Kotlin (#27323) Risk Level: medium for E-M Testing: new integration tests Docs Changes: n/a Release Notes: no Signed-off-by: Alyssa Wilk --- mobile/library/cc/stream_prototype.cc | 4 +- mobile/library/cc/stream_prototype.h | 2 +- mobile/library/common/http/client.cc | 30 +++++--- mobile/library/common/http/client.h | 13 +++- mobile/library/common/jni/jni_interface.cc | 4 +- mobile/library/common/main_interface.cc | 8 ++- mobile/library/common/main_interface.h | 5 +- .../envoymobile/engine/AndroidEngineImpl.java | 5 +- .../envoymobile/engine/EnvoyEngine.java | 5 +- .../envoymobile/engine/EnvoyEngineImpl.java | 9 ++- .../envoymobile/engine/EnvoyHTTPStream.java | 9 ++- .../envoymobile/engine/JniLibrary.java | 4 +- .../net/impl/CronvoyBidirectionalStream.java | 3 +- .../chromium/net/impl/CronvoyUrlRequest.java | 5 +- .../envoyproxy/envoymobile/StreamPrototype.kt | 16 ++++- .../envoymobile/mocks/MockEnvoyEngine.kt | 5 +- .../envoymobile/mocks/MockEnvoyHTTPStream.kt | 5 +- .../envoymobile/mocks/MockStreamPrototype.kt | 2 +- .../library/objective-c/EnvoyHTTPStreamImpl.m | 2 +- mobile/test/common/engine_test.cc | 2 +- mobile/test/common/http/client_test.cc | 6 +- .../base_client_integration_test.cc | 2 +- .../base_client_integration_test.h | 1 + .../integration/client_integration_test.cc | 17 ++++- mobile/test/common/main_interface_test.cc | 6 +- .../net/impl/CancelProofEnvoyStreamTest.java | 2 +- mobile/test/kotlin/integration/BUILD | 18 +++++ .../ReceiveDataWithDeliveryLimitsTest.kt | 68 +++++++++++++++++++ 28 files changed, 208 insertions(+), 50 deletions(-) create mode 100644 mobile/test/kotlin/integration/ReceiveDataWithDeliveryLimitsTest.kt diff --git a/mobile/library/cc/stream_prototype.cc b/mobile/library/cc/stream_prototype.cc index 8a86ee100c3d..7c03da7e9d64 100644 --- a/mobile/library/cc/stream_prototype.cc +++ b/mobile/library/cc/stream_prototype.cc @@ -9,10 +9,10 @@ StreamPrototype::StreamPrototype(EngineSharedPtr engine) : engine_(engine) { callbacks_ = std::make_shared(); } -StreamSharedPtr StreamPrototype::start(bool explicit_flow_control) { +StreamSharedPtr StreamPrototype::start(bool explicit_flow_control, uint64_t min_delivery_size) { auto envoy_stream = init_stream(engine_->engine_); start_stream(engine_->engine_, envoy_stream, callbacks_->asEnvoyHttpCallbacks(), - explicit_flow_control); + explicit_flow_control, min_delivery_size); return std::make_shared(engine_->engine_, envoy_stream); } diff --git a/mobile/library/cc/stream_prototype.h b/mobile/library/cc/stream_prototype.h index 3278a7a2f0de..80185c2aaff7 100644 --- a/mobile/library/cc/stream_prototype.h +++ b/mobile/library/cc/stream_prototype.h @@ -20,7 +20,7 @@ class StreamPrototype { public: StreamPrototype(EngineSharedPtr engine); - StreamSharedPtr start(bool explicit_flow_control = false); + StreamSharedPtr start(bool explicit_flow_control = false, uint64_t min_delivery_size = 0); StreamPrototype& setOnHeaders(OnHeadersCallback closure); StreamPrototype& setOnData(OnDataCallback closure); diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index 39c673b08553..d2efc7d1e8a6 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -39,7 +39,8 @@ Client::DirectStreamCallbacks::DirectStreamCallbacks(DirectStream& direct_stream envoy_http_callbacks bridge_callbacks, Client& http_client) : direct_stream_(direct_stream), bridge_callbacks_(bridge_callbacks), http_client_(http_client), - explicit_flow_control_(direct_stream_.explicit_flow_control_) {} + explicit_flow_control_(direct_stream_.explicit_flow_control_), + min_delivery_size_(direct_stream_.min_delivery_size_) {} void Client::DirectStreamCallbacks::encodeHeaders(const ResponseHeaderMap& headers, bool end_stream) { @@ -114,7 +115,7 @@ void Client::DirectStreamCallbacks::encodeData(Buffer::Instance& data, bool end_ // The response_data_ is systematically assigned here because resumeData can // incur an asynchronous callback to sendDataToBridge. - if (explicit_flow_control_ && !response_data_) { + if ((explicit_flow_control_ || min_delivery_size_ != 0) && !response_data_) { response_data_ = std::make_unique( [this]() -> void { onBufferedDataDrained(); }, [this]() -> void { onHasBufferedData(); }, []() -> void {}); @@ -124,8 +125,10 @@ void Client::DirectStreamCallbacks::encodeData(Buffer::Instance& data, bool end_ response_data_->setWatermarks(1000000); } - // Send data if in default flow control mode, or if resumeData has been called in explicit - // flow control mode. + // Try to send data if + // 1) in default flow control mode + // 2) if resumeData has been called in explicit flow control mode. + // sendDataToBridge will enforce delivery size limits. if (bytes_to_send_ > 0 || !explicit_flow_control_) { ASSERT(!hasBufferedData()); sendDataToBridge(data, end_stream); @@ -133,10 +136,9 @@ void Client::DirectStreamCallbacks::encodeData(Buffer::Instance& data, bool end_ // If not all the bytes have been sent up, buffer any remaining data in response_data. if (data.length() != 0) { - ASSERT(explicit_flow_control_); - ENVOY_LOG( - debug, "[S{}] buffering {} bytes due to explicit flow control. {} total bytes buffered.", - direct_stream_.stream_handle_, data.length(), data.length() + response_data_->length()); + ENVOY_LOG(debug, "[S{}] buffering {} bytes. {} total bytes buffered.", + direct_stream_.stream_handle_, data.length(), + data.length() + response_data_->length()); response_data_->move(data); } } @@ -144,6 +146,15 @@ void Client::DirectStreamCallbacks::encodeData(Buffer::Instance& data, bool end_ void Client::DirectStreamCallbacks::sendDataToBridge(Buffer::Instance& data, bool end_stream) { ASSERT(!explicit_flow_control_ || bytes_to_send_ > 0); + if (min_delivery_size_ > 0 && (data.length() < min_delivery_size_) && !end_stream) { + ENVOY_LOG( + debug, + "[S{}] defering sending {} bytes due to delivery size limits (limit={} end stream={})", + direct_stream_.stream_handle_, data.length(), min_delivery_size_, end_stream); + + return; // Not enough data to justify sending up to the bridge. + } + // Cap by bytes_to_send_ if and only if applying explicit flow control. uint32_t bytes_to_send = calculateBytesToSend(data, bytes_to_send_); // Update the number of bytes consumed by this non terminal callback. @@ -454,10 +465,11 @@ void Client::DirectStream::dumpState(std::ostream&, int indent_level) const { } void Client::startStream(envoy_stream_t new_stream_handle, envoy_http_callbacks bridge_callbacks, - bool explicit_flow_control) { + bool explicit_flow_control, uint64_t min_delivery_size) { ASSERT(dispatcher_.isThreadSafe()); Client::DirectStreamSharedPtr direct_stream{new DirectStream(new_stream_handle, *this)}; direct_stream->explicit_flow_control_ = explicit_flow_control; + direct_stream->min_delivery_size_ = min_delivery_size; direct_stream->callbacks_ = std::make_unique(*direct_stream, bridge_callbacks, *this); diff --git a/mobile/library/common/http/client.h b/mobile/library/common/http/client.h index 9b383041e1a7..3fb622c3ee0d 100644 --- a/mobile/library/common/http/client.h +++ b/mobile/library/common/http/client.h @@ -67,9 +67,11 @@ class Client : public Logger::Loggable { * @param stream, the stream to start. * @param bridge_callbacks, wrapper for callbacks for events on this stream. * @param explicit_flow_control, whether the stream will require explicit flow control. + * @param min_delivery_size, if greater than zero, indicates the smallest number of bytes that + * will be delivered up via the on_data callbacks without end stream. */ void startStream(envoy_stream_t stream, envoy_http_callbacks bridge_callbacks, - bool explicit_flow_control); + bool explicit_flow_control, uint64_t min_delivery_size); /** * Send headers over an open HTTP stream. This method can be invoked once and needs to be called @@ -202,12 +204,15 @@ class Client : public Logger::Loggable { absl::optional error_; bool success_{}; - // Buffered response data when in explicit flow control mode. + // Buffered response data when in explicit flow control or buffering due to min delivery size. Buffer::InstancePtr response_data_; ResponseTrailerMapPtr response_trailers_; // True if the bridge should operate in explicit flow control mode, and only send // data when it is requested by the caller. bool explicit_flow_control_{}; + // If greater than zero, indicates the minimum size of data that should be + // delivered up via on_data without end stream. + uint64_t min_delivery_size_{}; // Set true when the response headers have been forwarded to the bridge. bool response_headers_forwarded_{}; // Called in closeStream() to communicate that the end of the stream has @@ -327,6 +332,10 @@ class Client : public Logger::Loggable { // back, avoids excessive buffering of response bodies if the response body is // read faster than the mobile caller can process it. bool explicit_flow_control_ = false; + // If this is non-zero, Envoy will buffer at the C++ layer until either + // min_delivery_size_ bytes are received or end_stream is received. If this is + // zero, it will deliver data as it arrivies, modulo explicit flow control rules. + uint64_t min_delivery_size_{}; // Latest intel data retrieved from the StreamInfo. envoy_stream_intel stream_intel_{-1, -1, 0, 0}; envoy_final_stream_intel envoy_final_stream_intel_{-1, -1, -1, -1, -1, -1, -1, -1, diff --git a/mobile/library/common/jni/jni_interface.cc b/mobile/library/common/jni/jni_interface.cc index c96f8e687a5e..327377d1de19 100644 --- a/mobile/library/common/jni/jni_interface.cc +++ b/mobile/library/common/jni/jni_interface.cc @@ -914,7 +914,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_startStream( JNIEnv* env, jclass, jlong engine_handle, jlong stream_handle, jobject j_context, - jboolean explicit_flow_control) { + jboolean explicit_flow_control, jlong min_delivery_size) { // TODO: To be truly safe we may need stronger guarantees of operation ordering on this ref. jobject retained_context = env->NewGlobalRef(j_context); @@ -929,7 +929,7 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibra retained_context}; envoy_status_t result = start_stream(static_cast(engine_handle), static_cast(stream_handle), native_callbacks, - explicit_flow_control); + explicit_flow_control, min_delivery_size); if (result != ENVOY_SUCCESS) { env->DeleteGlobalRef(retained_context); // No callbacks are fired and we need to release } diff --git a/mobile/library/common/main_interface.cc b/mobile/library/common/main_interface.cc index 3324db60efce..fbd0654da057 100644 --- a/mobile/library/common/main_interface.cc +++ b/mobile/library/common/main_interface.cc @@ -18,10 +18,12 @@ static std::atomic current_stream_handle_{0}; envoy_stream_t init_stream(envoy_engine_t) { return current_stream_handle_++; } envoy_status_t start_stream(envoy_engine_t engine, envoy_stream_t stream, - envoy_http_callbacks callbacks, bool explicit_flow_control) { + envoy_http_callbacks callbacks, bool explicit_flow_control, + uint64_t min_delivery_size) { return Envoy::EngineHandle::runOnEngineDispatcher( - engine, [stream, callbacks, explicit_flow_control](auto& engine) -> void { - engine.httpClient().startStream(stream, callbacks, explicit_flow_control); + engine, [stream, callbacks, explicit_flow_control, min_delivery_size](auto& engine) -> void { + engine.httpClient().startStream(stream, callbacks, explicit_flow_control, + min_delivery_size); }); } diff --git a/mobile/library/common/main_interface.h b/mobile/library/common/main_interface.h index 6a7ac30c7cdd..0704113ed2ce 100644 --- a/mobile/library/common/main_interface.h +++ b/mobile/library/common/main_interface.h @@ -26,10 +26,13 @@ envoy_stream_t init_stream(envoy_engine_t engine); * @param stream, handle to the stream to be started. * @param callbacks, the callbacks that will run the stream callbacks. * @param explicit_flow_control, whether to enable explicit flow control on the response stream. + * @param min_delivery_size, if non-zero, indicates the smallest non-terminal number of bytes which + * should be delivered via on_data callbacks. * @return envoy_stream, with a stream handle and a success status, or a failure status. */ envoy_status_t start_stream(envoy_engine_t engine, envoy_stream_t stream, - envoy_http_callbacks callbacks, bool explicit_flow_control); + envoy_http_callbacks callbacks, bool explicit_flow_control, + uint64_t min_delivery_size); /** * Send headers over an open HTTP stream. This method can be invoked once and needs to be called diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java index 3fa756ad184e..022181a7d954 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java @@ -34,8 +34,9 @@ public AndroidEngineImpl(Context context, EnvoyOnEngineRunning runningCallback, } @Override - public EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl) { - return envoyEngine.startStream(callbacks, explicitFlowControl); + public EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl, + long minDeliverySize) { + return envoyEngine.startStream(callbacks, explicitFlowControl, minDeliverySize); } @Override diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngine.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngine.java index 7a70a44a6a04..d940d0c1c338 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngine.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngine.java @@ -14,9 +14,12 @@ public interface EnvoyEngine { * * @param callbacks The callbacks for receiving callbacks from the stream. * @param explicitFlowControl Whether explicit flow control will be enabled for this stream. + * @param minDeliverySize If nonzero, indicates the smallest number of response body bytes which + * should be delivered sans end stream. * @return A stream that may be used for sending data. */ - EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl); + EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl, + long minDeliverySize); /** * Terminates the running engine. diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngineImpl.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngineImpl.java index d8e61426260b..1e2a7bf71453 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngineImpl.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngineImpl.java @@ -35,13 +35,16 @@ public EnvoyEngineImpl(EnvoyOnEngineRunning runningCallback, EnvoyLogger logger, * * @param callbacks The callbacks for the stream. * @param explicitFlowControl Whether explicit flow control will be enabled for this stream. + * @param minDeliverySize If nonzero, indicates the smallest number of response body bytes which + * should be delivered sans end stream. * @return A stream that may be used for sending data. */ @Override - public EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl) { + public EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl, + long minDeliverySize) { long streamHandle = JniLibrary.initStream(engineHandle); - EnvoyHTTPStream stream = - new EnvoyHTTPStream(engineHandle, streamHandle, callbacks, explicitFlowControl); + EnvoyHTTPStream stream = new EnvoyHTTPStream(engineHandle, streamHandle, callbacks, + explicitFlowControl, minDeliverySize); stream.start(); return stream; } diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyHTTPStream.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyHTTPStream.java index dfa3e2c371e4..952144f67341 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyHTTPStream.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyHTTPStream.java @@ -12,13 +12,15 @@ public class EnvoyHTTPStream { private final long engineHandle; private final long streamHandle; private final boolean explicitFlowControl; + private final long minDeliverySize; private final JvmCallbackContext callbacksContext; /** * Start the stream via the JNI library. */ void start() { - JniLibrary.startStream(engineHandle, streamHandle, callbacksContext, explicitFlowControl); + JniLibrary.startStream(engineHandle, streamHandle, callbacksContext, explicitFlowControl, + minDeliverySize); } /** @@ -27,12 +29,15 @@ void start() { * @param streamHandle Underlying handle of the HTTP stream owned by an Envoy engine. * @param callbacks The callbacks for the stream. * @param explicitFlowControl Whether explicit flow control will be enabled for this stream. + * @param minDeliverySize If nonzero, indicates the smallest number of response body bytes which + * should be delivered sans end stream. */ public EnvoyHTTPStream(long engineHandle, long streamHandle, EnvoyHTTPCallbacks callbacks, - boolean explicitFlowControl) { + boolean explicitFlowControl, long minDeliverySize) { this.engineHandle = engineHandle; this.streamHandle = streamHandle; this.explicitFlowControl = explicitFlowControl; + this.minDeliverySize = minDeliverySize; callbacksContext = new JvmCallbackContext(callbacks); } diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java index f4b6a1956766..84b54d3e5447 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java @@ -63,11 +63,13 @@ private static class JavaLoader { * callbacks. * @param explicitFlowControl, whether explicit flow control should be enabled * for the stream. + * @param minDeliverySize If nonzero, indicates the smallest number of response body bytes which + * should be delivered sans end stream. * @return envoy_stream, with a stream handle and a success status, or a failure * status. */ protected static native int startStream(long engine, long stream, JvmCallbackContext context, - boolean explicitFlowControl); + boolean explicitFlowControl, long minDeliverySize); /** * Send headers over an open HTTP stream. This method can be invoked once and diff --git a/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java index 01cb9ad32dc9..054e6f8e8050 100644 --- a/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java @@ -319,7 +319,8 @@ public void start() { public void run() { try { mStream.setStream(mRequestContext.getEnvoyEngine().startStream( - CronvoyBidirectionalStream.this, /* explicitFlowCrontrol= */ true)); + CronvoyBidirectionalStream.this, /* explicitFlowCrontrol= */ true, + /* minDeliverySize */ 0)); if (!mDelayRequestHeadersUntilFirstFlush) { mStream.sendHeaders(mEnvoyRequestHeaders, mReadOnly); } diff --git a/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java index e89ee7716c08..e38b0840c416 100644 --- a/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java @@ -541,7 +541,8 @@ private void fireOpenConnection() { mCurrentUrl, mRequestContext.getBuilder().quicEnabled()); mCronvoyCallbacks = new CronvoyHttpCallbacks(); mStream.set(mRequestContext.getEnvoyEngine().startStream(mCronvoyCallbacks, - /* explicitFlowCrontrol= */ true)); + /* explicitFlowCrontrol= */ true, + /* minDeliverySize */ 0)); mStream.get().sendHeaders(envoyRequestHeaders, mUploadDataStream == null); if (mUploadDataStream != null && mUrlChain.size() == 1) { mUploadDataStream.initializeWithRequest(); @@ -984,7 +985,7 @@ public void onSendWindowAvailable(EnvoyStreamIntel streamIntel) { return; } - mUploadDataStream.readDataReady(); // Have the next request body chunk to be sent. + mUploadDataStream.readDataReady(); // Have the next request body delivery to be sent. } @Override diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt index 748351905abf..03f312027177 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt @@ -16,6 +16,7 @@ import java.util.concurrent.Executors open class StreamPrototype(private val engine: EnvoyEngine) { private val callbacks = StreamCallbacks() private var explicitFlowControl = false + private var minDeliverySize: Long = 0 private var useByteBufferPosition = false /** @@ -27,11 +28,24 @@ open class StreamPrototype(private val engine: EnvoyEngine) { open fun start(executor: Executor = Executors.newSingleThreadExecutor()): Stream { val engineStream = engine.startStream( createCallbacks(executor), - explicitFlowControl + explicitFlowControl, + minDeliverySize ) return Stream(engineStream, useByteBufferPosition) } + /** + * Sets min delivery: data will be buffered in the C++ layer until the min delivery length or end stream is read. + * + * @param value set the minimum delivery size fo for this stream + * @return This stream, for chaining syntax. + */ + fun setMinDeliverySize(value: Long): StreamPrototype { + this.minDeliverySize = value + return this + } + + /** * Allows explicit flow control to be enabled. When flow control is enabled, the owner of a stream * is responsible for providing a buffer to receive response body data. If the buffer is smaller diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyEngine.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyEngine.kt index 7fc329f02ea8..27d06c513c98 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyEngine.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyEngine.kt @@ -23,9 +23,10 @@ internal class MockEnvoyEngine : EnvoyEngine { override fun startStream( callbacks: EnvoyHTTPCallbacks?, - explicitFlowControl: Boolean + explicitFlowControl: Boolean, + minDeliverySize: Long ): EnvoyHTTPStream { - return MockEnvoyHTTPStream(callbacks!!, explicitFlowControl) + return MockEnvoyHTTPStream(callbacks!!, explicitFlowControl, minDeliverySize) } override fun terminate() = Unit diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyHTTPStream.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyHTTPStream.kt index 64bd7849e747..978000716102 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyHTTPStream.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyHTTPStream.kt @@ -11,8 +11,9 @@ import java.nio.ByteBuffer */ internal class MockEnvoyHTTPStream( val callbacks: EnvoyHTTPCallbacks, - val explicitFlowControl: Boolean -) : EnvoyHTTPStream(0, 0, callbacks, explicitFlowControl) { + val explicitFlowControl: Boolean, + val minDeliverySize: Long +) : EnvoyHTTPStream(0, 0, callbacks, explicitFlowControl, minDeliverySize) { override fun sendHeaders(headers: MutableMap>?, endStream: Boolean) {} override fun sendData(data: ByteBuffer?, endStream: Boolean) {} diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamPrototype.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamPrototype.kt index 19bf25b4d93b..22188905f54a 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamPrototype.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamPrototype.kt @@ -10,7 +10,7 @@ import java.util.concurrent.Executor class MockStreamPrototype internal constructor(private val onStart: ((stream: MockStream) -> Unit)?) : StreamPrototype(MockEnvoyEngine()) { override fun start(executor: Executor): Stream { val callbacks = createCallbacks(executor) - val stream = MockStream(MockEnvoyHTTPStream(callbacks, false)) + val stream = MockStream(MockEnvoyHTTPStream(callbacks, false, 0)) onStart?.invoke(stream) return stream } diff --git a/mobile/library/objective-c/EnvoyHTTPStreamImpl.m b/mobile/library/objective-c/EnvoyHTTPStreamImpl.m index 16d6f0af0f93..3613164dd6e4 100644 --- a/mobile/library/objective-c/EnvoyHTTPStreamImpl.m +++ b/mobile/library/objective-c/EnvoyHTTPStreamImpl.m @@ -174,7 +174,7 @@ - (instancetype)initWithHandle:(envoy_stream_t)handle // start_stream could result in a reset that would release the native ref. _strongSelf = self; envoy_status_t result = - start_stream(engineHandle, _streamHandle, native_callbacks, explicitFlowControl); + start_stream(engineHandle, _streamHandle, native_callbacks, explicitFlowControl, 0); if (result != ENVOY_SUCCESS) { _strongSelf = nil; return nil; diff --git a/mobile/test/common/engine_test.cc b/mobile/test/common/engine_test.cc index dc1f51ab3b1b..b08f569078ed 100644 --- a/mobile/test/common/engine_test.cc +++ b/mobile/test/common/engine_test.cc @@ -58,7 +58,7 @@ TEST_F(EngineTest, EarlyExit) { ASSERT_EQ(engine_->terminate(), ENVOY_SUCCESS); ASSERT_TRUE(test_context.on_exit.WaitForNotificationWithTimeout(absl::Seconds(10))); - start_stream(handle, 0, {}, false); + start_stream(handle, 0, {}, false, 0); engine_.reset(); } diff --git a/mobile/test/common/http/client_test.cc b/mobile/test/common/http/client_test.cc index abba6690b940..d64546c8985d 100644 --- a/mobile/test/common/http/client_test.cc +++ b/mobile/test/common/http/client_test.cc @@ -136,7 +136,7 @@ class ClientTest : public testing::TestWithParam { response_encoder_ = &encoder; return *request_decoder_; })); - http_client_.startStream(stream_, bridge_callbacks_, explicit_flow_control_); + http_client_.startStream(stream_, bridge_callbacks_, explicit_flow_control_, 0); } void resumeDataIfExplicitFlowControl(int32_t bytes) { @@ -428,7 +428,7 @@ TEST_P(ClientTest, MultipleStreams) { response_encoder2 = &encoder; return request_decoder2; })); - http_client_.startStream(stream2, bridge_callbacks_2, explicit_flow_control_); + http_client_.startStream(stream2, bridge_callbacks_2, explicit_flow_control_, 0); // Send request headers. EXPECT_CALL(dispatcher_, pushTrackedObject(_)); @@ -689,7 +689,7 @@ TEST_P(ClientTest, NullAccessors) { response_encoder_ = &encoder; return *request_decoder_; })); - http_client_.startStream(stream, bridge_callbacks, explicit_flow_control_); + http_client_.startStream(stream, bridge_callbacks, explicit_flow_control_, 0); EXPECT_FALSE(response_encoder_->http1StreamEncoderOptions().has_value()); EXPECT_FALSE(response_encoder_->streamErrorOnInvalidHttpMessage()); diff --git a/mobile/test/common/integration/base_client_integration_test.cc b/mobile/test/common/integration/base_client_integration_test.cc index a382597060ce..dd40c81b12d5 100644 --- a/mobile/test/common/integration/base_client_integration_test.cc +++ b/mobile/test/common/integration/base_client_integration_test.cc @@ -136,7 +136,7 @@ void BaseClientIntegrationTest::initialize() { cc_.terminal_callback->setReady(); }); - stream_ = (*stream_prototype_).start(explicit_flow_control_); + stream_ = (*stream_prototype_).start(explicit_flow_control_, min_delivery_size_); HttpTestUtility::addDefaultHeaders(default_request_headers_); default_request_headers_.setHost(fake_upstreams_[0]->localAddress()->asStringView()); } diff --git a/mobile/test/common/integration/base_client_integration_test.h b/mobile/test/common/integration/base_client_integration_test.h index 0959220aef5a..31143e94c64f 100644 --- a/mobile/test/common/integration/base_client_integration_test.h +++ b/mobile/test/common/integration/base_client_integration_test.h @@ -76,6 +76,7 @@ class BaseClientIntegrationTest : public BaseIntegrationTest { Platform::EngineSharedPtr engine_; Thread::ThreadPtr envoy_thread_; bool explicit_flow_control_ = false; + uint64_t min_delivery_size_ = 10; bool expect_dns_ = true; bool override_builder_config_ = false; // True if data plane requests are expected in the test; false otherwise. diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index cedd510f015a..ac06e467d150 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -131,17 +131,30 @@ void ClientIntegrationTest::trickleTest() { terminal_callback_.waitReady(); } -TEST_P(ClientIntegrationTest, Trickle) { +TEST_P(ClientIntegrationTest, TrickleNoMinDelivery) { + min_delivery_size_ = 0; trickleTest(); ASSERT_LE(cc_.on_data_calls, 11); } -TEST_P(ClientIntegrationTest, TrickleExplicitFlowControl) { +TEST_P(ClientIntegrationTest, TrickleNoNoMinDeliveryExplicitFlowControl) { + min_delivery_size_ = 0; explicit_flow_control_ = true; trickleTest(); ASSERT_LE(cc_.on_data_calls, 11); } +TEST_P(ClientIntegrationTest, TrickleMinDelivery) { + trickleTest(); + ASSERT_EQ(cc_.on_data_calls, 2); +} + +TEST_P(ClientIntegrationTest, TrickleNoMinDeliveryExplicitFlowControl) { + explicit_flow_control_ = true; + trickleTest(); + ASSERT_EQ(cc_.on_data_calls, 2); +} + TEST_P(ClientIntegrationTest, ClearTextNotPermitted) { EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).WillRepeatedly(Return(false)); diff --git a/mobile/test/common/main_interface_test.cc b/mobile/test/common/main_interface_test.cc index ba49f648988d..f790147da438 100644 --- a/mobile/test/common/main_interface_test.cc +++ b/mobile/test/common/main_interface_test.cc @@ -197,7 +197,7 @@ TEST_F(MainInterfaceTest, BasicStream) { envoy_stream_t stream = init_stream(engine_handle); - start_stream(engine_handle, stream, stream_cbs, false); + start_stream(engine_handle, stream, stream_cbs, false, 0); send_headers(engine_handle, stream, c_headers, false); send_data(engine_handle, stream, c_data, false); @@ -241,7 +241,7 @@ TEST_F(MainInterfaceTest, SendMetadata) { envoy_stream_t stream = init_stream(engine_handle); - start_stream(engine_handle, stream, stream_cbs, false); + start_stream(engine_handle, stream, stream_cbs, false, 0); EXPECT_EQ(ENVOY_FAILURE, send_metadata(engine_handle, stream, {})); @@ -289,7 +289,7 @@ TEST_F(MainInterfaceTest, ResetStream) { envoy_stream_t stream = init_stream(engine_handle); - start_stream(engine_handle, stream, stream_cbs, false); + start_stream(engine_handle, stream, stream_cbs, false, 0); reset_stream(engine_handle, stream); diff --git a/mobile/test/java/org/chromium/net/impl/CancelProofEnvoyStreamTest.java b/mobile/test/java/org/chromium/net/impl/CancelProofEnvoyStreamTest.java index 0e477322aa22..6debc9821f76 100644 --- a/mobile/test/java/org/chromium/net/impl/CancelProofEnvoyStreamTest.java +++ b/mobile/test/java/org/chromium/net/impl/CancelProofEnvoyStreamTest.java @@ -357,7 +357,7 @@ public void cancel_manyConcurrentStreamOperationsInFlight() throws Exception { private class MockedStream extends EnvoyHTTPStream { - private MockedStream() { super(0, 0, null, false); } + private MockedStream() { super(0, 0, null, false, 0); } @Override public void sendHeaders(Map> headers, boolean endStream) { diff --git a/mobile/test/kotlin/integration/BUILD b/mobile/test/kotlin/integration/BUILD index cf2a4d97c85c..3b9b4f53621f 100644 --- a/mobile/test/kotlin/integration/BUILD +++ b/mobile/test/kotlin/integration/BUILD @@ -160,6 +160,24 @@ envoy_mobile_jni_kt_test( ], ) +envoy_mobile_jni_kt_test( + name = "receive_data_delivery_limits_test", + srcs = [ + "ReceiveDataWithDeliveryLimitsTest.kt", + ], + exec_properties = { + # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. + "sandboxNetwork": "standard", + }, + native_deps = [ + "//test/common/jni:libenvoy_jni_with_test_extensions.so", + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + deps = [ + "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", + ], +) + envoy_mobile_jni_kt_test( name = "receive_error_test", srcs = [ diff --git a/mobile/test/kotlin/integration/ReceiveDataWithDeliveryLimitsTest.kt b/mobile/test/kotlin/integration/ReceiveDataWithDeliveryLimitsTest.kt new file mode 100644 index 000000000000..454465bb44fe --- /dev/null +++ b/mobile/test/kotlin/integration/ReceiveDataWithDeliveryLimitsTest.kt @@ -0,0 +1,68 @@ +package test.kotlin.integration + +import io.envoyproxy.envoymobile.Standard +import io.envoyproxy.envoymobile.EngineBuilder +import io.envoyproxy.envoymobile.RequestHeadersBuilder +import io.envoyproxy.envoymobile.RequestMethod +import io.envoyproxy.envoymobile.engine.JniLibrary +import java.nio.ByteBuffer +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.fail +import org.junit.Test + +private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" + +class ReceiveDataTest { + + init { + JniLibrary.loadTestLibrary() + } + + @Test + fun `data is received with min delivery size set`() { + + val engine = EngineBuilder(Standard()) + .addNativeFilter("test_remote_response", "{'@type': $testResponseFilterType}") + .build() + val client = engine.streamClient() + + val requestHeaders = RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() + + val headersExpectation = CountDownLatch(1) + val dataExpectation = CountDownLatch(1) + + var status: Int? = null + var body: ByteBuffer? = null + client.newStreamPrototype() + .setOnResponseHeaders { responseHeaders, _, _ -> + status = responseHeaders.httpStatus + headersExpectation.countDown() + } + .setOnResponseData { data, _, _ -> + body = data + dataExpectation.countDown() + } + .setOnError { _, _ -> fail("Unexpected error") } + .setMinDeliverySize(10) + .start() + .sendHeaders(requestHeaders, true) + + headersExpectation.await(10, TimeUnit.SECONDS) + dataExpectation.await(10, TimeUnit.SECONDS) + engine.terminate() + + assertThat(headersExpectation.count).isEqualTo(0) + assertThat(dataExpectation.count).isEqualTo(0) + + assertThat(status).isEqualTo(200) + assertThat(body!!.array().toString(Charsets.UTF_8)).isEqualTo("data") + } +} From dddea445402655f7025270aba41cbfc09c153dbc Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Thu, 11 May 2023 21:39:21 +0300 Subject: [PATCH 208/740] access_log: add AccessLogType filter (#27105) Signed-off-by: ohadvano --- api/envoy/config/accesslog/v3/BUILD | 1 + api/envoy/config/accesslog/v3/accesslog.proto | 17 +++- changelogs/current.yaml | 4 + source/common/access_log/access_log_impl.cc | 17 ++++ source/common/access_log/access_log_impl.h | 18 ++++ .../common/access_log/access_log_impl_test.cc | 88 +++++++++++++++++++ 6 files changed, 144 insertions(+), 1 deletion(-) diff --git a/api/envoy/config/accesslog/v3/BUILD b/api/envoy/config/accesslog/v3/BUILD index 7e5e77fc70f5..e657889317c5 100644 --- a/api/envoy/config/accesslog/v3/BUILD +++ b/api/envoy/config/accesslog/v3/BUILD @@ -8,6 +8,7 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", + "//envoy/data/accesslog/v3:pkg", "//envoy/type/matcher/v3:pkg", "//envoy/type/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", diff --git a/api/envoy/config/accesslog/v3/accesslog.proto b/api/envoy/config/accesslog/v3/accesslog.proto index 1eb348925242..0904b5d055a7 100644 --- a/api/envoy/config/accesslog/v3/accesslog.proto +++ b/api/envoy/config/accesslog/v3/accesslog.proto @@ -4,6 +4,7 @@ package envoy.config.accesslog.v3; import "envoy/config/core/v3/base.proto"; import "envoy/config/route/v3/route_components.proto"; +import "envoy/data/accesslog/v3/accesslog.proto"; import "envoy/type/matcher/v3/metadata.proto"; import "envoy/type/v3/percent.proto"; @@ -43,7 +44,7 @@ message AccessLog { } } -// [#next-free-field: 13] +// [#next-free-field: 14] message AccessLogFilter { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.accesslog.v2.AccessLogFilter"; @@ -87,6 +88,9 @@ message AccessLogFilter { // Metadata Filter MetadataFilter metadata_filter = 12; + + // Log Type Filter + LogTypeFilter log_type_filter = 13; } } @@ -310,6 +314,17 @@ message MetadataFilter { google.protobuf.BoolValue match_if_key_not_found = 2; } +// Filters based on access log type. +message LogTypeFilter { + // Logs only records which their type is one of the types defined in this field. + repeated data.accesslog.v3.AccessLogType types = 1 + [(validate.rules).repeated = {items {enum {defined_only: true}}}]; + + // If this field is set to true, the filter will instead block all records + // with a access log type in types field, and allow all other records. + bool exclude = 2; +} + // Extension filter is statically registered at runtime. message ExtensionFilter { option (udpa.annotations.versioning).previous_message_type = diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 22530a903b27..1b5a30edaf87 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -143,6 +143,10 @@ new_features: added new configuration field :ref:`domain ` to allow for setting rate limit domains on a per-route basis. +- area: access_log + change: | + added access log filter :ref:`log_type_filter ` + to filter access log records based on the type of the record. - area: ext_proc change: | added new configuration field diff --git a/source/common/access_log/access_log_impl.cc b/source/common/access_log/access_log_impl.cc index e3fbe056d352..91bafcdb5ce2 100644 --- a/source/common/access_log/access_log_impl.cc +++ b/source/common/access_log/access_log_impl.cc @@ -80,6 +80,8 @@ FilterPtr FilterFactory::fromProto(const envoy::config::accesslog::v3::AccessLog return FilterPtr{new GrpcStatusFilter(config.grpc_status_filter())}; case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kMetadataFilter: return FilterPtr{new MetadataFilter(config.metadata_filter())}; + case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kLogTypeFilter: + return FilterPtr{new LogTypeFilter(config.log_type_filter())}; case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kExtensionFilter: MessageUtil::validate(config, validation_visitor); { @@ -277,6 +279,21 @@ Grpc::Status::GrpcStatus GrpcStatusFilter::protoToGrpcStatus( return static_cast(status); } +LogTypeFilter::LogTypeFilter(const envoy::config::accesslog::v3::LogTypeFilter& config) { + for (auto type_as_int : config.types()) { + types_.insert(static_cast(type_as_int)); + } + + exclude_ = config.exclude(); +} + +bool LogTypeFilter::evaluate(const StreamInfo::StreamInfo&, const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + AccessLogType access_log_type) const { + const bool found = types_.contains(access_log_type); + return exclude_ ? !found : found; +} + MetadataFilter::MetadataFilter(const envoy::config::accesslog::v3::MetadataFilter& filter_config) : default_match_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(filter_config, match_if_key_not_found, true)), filter_(filter_config.matcher().filter()) { diff --git a/source/common/access_log/access_log_impl.h b/source/common/access_log/access_log_impl.h index eaf15ca75e00..2305181720ef 100644 --- a/source/common/access_log/access_log_impl.h +++ b/source/common/access_log/access_log_impl.h @@ -229,6 +229,24 @@ class GrpcStatusFilter : public Filter { protoToGrpcStatus(envoy::config::accesslog::v3::GrpcStatusFilter::Status status) const; }; +/** + * Filters requests based on access log type + */ +class LogTypeFilter : public Filter { +public: + using LogTypeHashSet = absl::flat_hash_set; + + LogTypeFilter(const envoy::config::accesslog::v3::LogTypeFilter& filter_config); + + bool evaluate(const StreamInfo::StreamInfo&, const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + AccessLogType access_log_type) const override; + +private: + LogTypeHashSet types_; + bool exclude_; +}; + /** * Filters requests based on dynamic metadata */ diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index 96f5b1519351..dc672a741819 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -1439,6 +1439,94 @@ name: accesslog AccessLog::AccessLogType::NotSet); } +TEST_F(AccessLogImplTest, LogTypeFilterUnsupportedValue) { + const std::string yaml = R"EOF( +name: accesslog +filter: + log_type_filter: + types: + - NOT_A_VALID_CODE +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + )EOF"; + + EXPECT_THROW_WITH_REGEX(AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_), + EnvoyException, "NOT_A_VALID_CODE"); +} + +TEST_F(AccessLogImplTest, LogTypeFilterBlock) { + const std::string yaml = R"EOF( +name: accesslog +filter: + log_type_filter: + types: + - TcpUpstreamConnected + - DownstreamEnd +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + )EOF"; + + const InstanceSharedPtr log = + AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); + + EXPECT_CALL(*file_, write(_)).Times(0); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::TcpConnectionEnd); // Blocked +} + +TEST_F(AccessLogImplTest, LogTypeFilterAllow) { + const std::string yaml = R"EOF( +name: accesslog +filter: + log_type_filter: + types: + - TcpUpstreamConnected + - DownstreamEnd +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + )EOF"; + + const InstanceSharedPtr log = + AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); + + EXPECT_CALL(*file_, write(_)).Times(2); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::TcpUpstreamConnected); // Allowed + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::DownstreamEnd); // Allowed + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::UpstreamPoolReady); // Blocked +} + +TEST_F(AccessLogImplTest, LogTypeFilterExclude) { + const std::string yaml = R"EOF( +name: accesslog +filter: + log_type_filter: + exclude: true + types: + - TcpUpstreamConnected + - DownstreamEnd +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + )EOF"; + + const InstanceSharedPtr log = + AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); + + EXPECT_CALL(*file_, write(_)); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::TcpUpstreamConnected); // Blocked + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::DownstreamEnd); // Blocked + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, + AccessLog::AccessLogType::UpstreamPoolReady); // Allowed +} + TEST_F(AccessLogImplTest, MetadataFilter) { const std::string yaml = R"EOF( name: accesslog From 4f3f4870333809529815ca24a891788d05590705 Mon Sep 17 00:00:00 2001 From: DiazAlan <109677874+DiazAlan@users.noreply.github.com> Date: Thu, 11 May 2023 14:40:46 -0400 Subject: [PATCH 209/740] Add common_lb_config_ to ObjectSharedPool (#26648) Signed-off-by: AlanDiaz --- envoy/upstream/cluster_manager.h | 8 ++++++ .../common/upstream/cluster_manager_impl.cc | 6 ++++- source/common/upstream/cluster_manager_impl.h | 14 ++++++++++ source/common/upstream/upstream_impl.cc | 7 ++--- source/common/upstream/upstream_impl.h | 5 ++-- .../upstream/cluster_manager_impl_test.cc | 26 +++++++++++++++++++ test/mocks/upstream/cluster_manager.h | 5 ++++ 7 files changed, 65 insertions(+), 6 deletions(-) diff --git a/envoy/upstream/cluster_manager.h b/envoy/upstream/cluster_manager.h index a28e19c9ac8c..45172c7c66a0 100644 --- a/envoy/upstream/cluster_manager.h +++ b/envoy/upstream/cluster_manager.h @@ -431,6 +431,14 @@ class ClusterManager { allocateOdCdsApi(const envoy::config::core::v3::ConfigSource& odcds_config, OptRef odcds_resources_locator, ProtobufMessage::ValidationVisitor& validation_visitor) PURE; + + /** + * @param common_lb_config The config field to be stored + * @return shared_ptr to the CommonLbConfig + */ + virtual std::shared_ptr + getCommonLbConfigPtr( + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_lb_config) PURE; }; using ClusterManagerPtr = std::unique_ptr; diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 1c9a6509e56e..df6978c8d20e 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -299,7 +299,11 @@ ClusterManagerImpl::ClusterManagerImpl( cluster_load_report_stat_names_(stats.symbolTable()), cluster_circuit_breakers_stat_names_(stats.symbolTable()), cluster_request_response_size_stat_names_(stats.symbolTable()), - cluster_timeout_budget_stat_names_(stats.symbolTable()) { + cluster_timeout_budget_stat_names_(stats.symbolTable()), + common_lb_config_pool_( + std::make_shared>( + main_thread_dispatcher)) { if (admin.has_value()) { config_tracker_entry_ = admin->getConfigTracker().add( "clusters", [this](const Matchers::StringMatcher& name_matcher) { diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index dfe5c693056d..ebb729f752d5 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -349,6 +349,17 @@ class ClusterManagerImpl : public ClusterManager, // Upstream::MissingClusterNotifier void notifyMissingCluster(absl::string_view name) override; + /* + * Return shared_ptr for common_lb_config which is stored in an ObjectSharedPool + * + * @param common_lb_config The config field to be stored in ObjectSharedPool + * @return shared_ptr to the CommonLbConfig in ObjectSharedPool + */ + std::shared_ptr getCommonLbConfigPtr( + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_lb_config) override { + return common_lb_config_pool_->getObject(common_lb_config); + } + protected: virtual void postThreadLocalRemoveHosts(const Cluster& cluster, const HostVector& hosts_removed); @@ -803,6 +814,9 @@ class ClusterManagerImpl : public ClusterManager, ClusterCircuitBreakersStatNames cluster_circuit_breakers_stat_names_; ClusterRequestResponseSizeStatNames cluster_request_response_size_stat_names_; ClusterTimeoutBudgetStatNames cluster_timeout_budget_stat_names_; + std::shared_ptr> + common_lb_config_pool_; std::unique_ptr subscription_factory_; ClusterSet primary_clusters_; diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 130404364e3a..1010fbed6fda 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1064,7 +1064,8 @@ ClusterInfoImpl::ClusterInfoImpl( typed_metadata_(config.has_metadata() ? std::make_unique(config.metadata()) : nullptr), - common_lb_config_(config.common_lb_config()), + common_lb_config_( + factory_context.clusterManager().getCommonLbConfigPtr(config.common_lb_config())), cluster_type_(config.has_cluster_type() ? std::make_unique( config.cluster_type()) @@ -1083,7 +1084,7 @@ ClusterInfoImpl::ClusterInfoImpl( connection_pool_per_downstream_connection_( config.connection_pool_per_downstream_connection()), warm_hosts_(!config.health_checks().empty() && - common_lb_config_.ignore_new_hosts_until_first_hc()), + common_lb_config_->ignore_new_hosts_until_first_hc()), set_local_interface_name_on_upstream_connections_( config.upstream_connection_options().set_local_interface_name_on_upstream_connections()), added_via_api_(added_via_api), has_configured_http_filters_(false) { @@ -1194,7 +1195,7 @@ ClusterInfoImpl::ClusterInfoImpl( // TODO(htuch): Remove this temporary workaround when we have // https://github.com/bufbuild/protoc-gen-validate/issues/97 resolved. This just provides // early validation of sanity of fields that we should catch at config ingestion. - DurationUtil::durationToMilliseconds(common_lb_config_.update_merge_window()); + DurationUtil::durationToMilliseconds(common_lb_config_->update_merge_window()); // Create upstream filter factories const auto& filters = config.filters(); diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 6a9795e0e141..ad6f069c86c5 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -795,7 +795,7 @@ class ClusterInfoImpl : public ClusterInfo, } TypedLoadBalancerFactory* loadBalancerFactory() const override { return load_balancer_factory_; } const envoy::config::cluster::v3::Cluster::CommonLbConfig& lbConfig() const override { - return common_lb_config_; + return *common_lb_config_; } std::chrono::milliseconds connectTimeout() const override { return connect_timeout_; } @@ -1059,7 +1059,8 @@ class ClusterInfoImpl : public ClusterInfo, std::unique_ptr typed_metadata_; ProtobufTypes::MessagePtr load_balancing_policy_; TypedLoadBalancerFactory* load_balancer_factory_ = nullptr; - const envoy::config::cluster::v3::Cluster::CommonLbConfig common_lb_config_; + const std::shared_ptr + common_lb_config_; std::unique_ptr cluster_type_; const std::unique_ptr factory_context_; std::vector filter_factories_; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index cf545458b67d..c55bee1639b4 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -1252,6 +1252,32 @@ TEST_F(ClusterManagerImplTest, ShutdownOrder) { EXPECT_EQ(3U, cluster.info().use_count()); } +TEST_F(ClusterManagerImplTest, TwoEqualCommonLbConfigSharedPool) { + create(defaultConfig()); + + envoy::config::cluster::v3::Cluster::CommonLbConfig config_a; + envoy::config::cluster::v3::Cluster::CommonLbConfig config_b; + + config_a.mutable_healthy_panic_threshold()->set_value(0.3); + config_b.mutable_healthy_panic_threshold()->set_value(0.3); + auto common_config_ptr_a = cluster_manager_->getCommonLbConfigPtr(config_a); + auto common_config_ptr_b = cluster_manager_->getCommonLbConfigPtr(config_b); + EXPECT_EQ(common_config_ptr_a, common_config_ptr_b); +} + +TEST_F(ClusterManagerImplTest, TwoUnequalCommonLbConfigSharedPool) { + create(defaultConfig()); + + envoy::config::cluster::v3::Cluster::CommonLbConfig config_a; + envoy::config::cluster::v3::Cluster::CommonLbConfig config_b; + + config_a.mutable_healthy_panic_threshold()->set_value(0.3); + config_b.mutable_healthy_panic_threshold()->set_value(0.5); + auto common_config_ptr_a = cluster_manager_->getCommonLbConfigPtr(config_a); + auto common_config_ptr_b = cluster_manager_->getCommonLbConfigPtr(config_b); + EXPECT_NE(common_config_ptr_a, common_config_ptr_b); +} + TEST_F(ClusterManagerImplTest, InitializeOrder) { time_system_.setSystemTime(std::chrono::milliseconds(1234567891234)); diff --git a/test/mocks/upstream/cluster_manager.h b/test/mocks/upstream/cluster_manager.h index 50d26fed8317..76c7a24d9cd7 100644 --- a/test/mocks/upstream/cluster_manager.h +++ b/test/mocks/upstream/cluster_manager.h @@ -84,6 +84,11 @@ class MockClusterManager : public ClusterManager { (const envoy::config::core::v3::ConfigSource& odcds_config, OptRef odcds_resources_locator, ProtobufMessage::ValidationVisitor& validation_visitor)); + std::shared_ptr getCommonLbConfigPtr( + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_lb_config) override { + return std::make_shared( + common_lb_config); + } envoy::config::core::v3::BindConfig& mutableBindConfig(); From 4b13460beef02562561f163a493df898f23d9fe2 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 11 May 2023 14:46:21 -0400 Subject: [PATCH 210/740] Runtime: reinstate yaml behavior when using strings instead of structs (#27276) reinstates prior behavior when parsing "buggy" yaml, where fractional structs are encoded as strings. Previously when doing a round trip from json -> proto -> json this formatting would be parsed, so doing some string checks to support it going forward Risk Level: low Testing: new unit tests Docs Changes: n/a Release Notes: n/a Fixes commit #PR: #26451 Signed-off-by: Alyssa Wilk Signed-off-by: alyssawilk Co-authored-by: Greg Greenway --- source/common/runtime/runtime_impl.cc | 35 ++++++++++++++++++++---- test/common/runtime/runtime_impl_test.cc | 34 +++++++++++++++++++++++ 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 0b634d96bee2..423493601583 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -26,6 +26,7 @@ #include "absl/flags/flag.h" #include "absl/strings/match.h" #include "absl/strings/numbers.h" +#include "re2/re2.h" #ifdef ENVOY_ENABLE_QUIC #include "quiche_platform_impl/quiche_flags_impl.h" @@ -271,18 +272,40 @@ void parseEntryDoubleValue(Envoy::Runtime::Snapshot::Entry& entry) { } } -// Handle an absolutely awful corner case where we explicitly shove a yaml percent in a proto string -// value. +// Handle an awful corner case where we explicitly shove a yaml percent in a proto string +// value. Basically due to prior parsing logic we have to handle any combination +// of numerator: #### [denominator Y] with quotes braces etc that could possibly be valid json. +// E.g. "final_value": "{\"numerator\": 10000, \"denominator\": \"TEN_THOUSAND\"}", void parseEntryFractionalPercentValue(Envoy::Runtime::Snapshot::Entry& entry) { - if (!absl::StrContains(entry.raw_string_value_, "numerator:")) { + if (!absl::StrContains(entry.raw_string_value_, "numerator")) { + return; + } + + const re2::RE2 numerator_re(".*numerator[^\\d]+(\\d+)[^\\d]*"); + + std::string match_string; + if (!re2::RE2::FullMatch(entry.raw_string_value_.c_str(), numerator_re, &match_string)) { + return; + } + + uint32_t numerator; + if (!absl::SimpleAtoi(match_string, &numerator)) { return; } envoy::type::v3::FractionalPercent converted_fractional_percent; - TRY_ASSERT_MAIN_THREAD { entry.fractional_percent_value_ = converted_fractional_percent; } - END_TRY - catch (const ProtoValidationException& ex) { + converted_fractional_percent.set_numerator(numerator); + entry.fractional_percent_value_ = converted_fractional_percent; + + if (!absl::StrContains(entry.raw_string_value_, "denominator")) { return; } + if (absl::StrContains(entry.raw_string_value_, "TEN_THOUSAND")) { + entry.fractional_percent_value_->set_denominator( + envoy::type::v3::FractionalPercent::TEN_THOUSAND); + } + if (absl::StrContains(entry.raw_string_value_, "MILLION")) { + entry.fractional_percent_value_->set_denominator(envoy::type::v3::FractionalPercent::MILLION); + } } // Handle corner cases in non-yaml parsing: mixed case strings aren't parsed as booleans. diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index cfc9cebadd89..e4413c09413d 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -740,6 +740,40 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) { ProtobufWkt::Value empty_value; const_cast(dynamic_cast(loader_->snapshot())) .createEntry(empty_value); + + // Make sure the hacky fractional percent function works. + ProtobufWkt::Value fractional_value; + fractional_value.set_string_value(" numerator: 11 "); + auto entry = const_cast(dynamic_cast(loader_->snapshot())) + .createEntry(fractional_value); + ASSERT_TRUE(entry.fractional_percent_value_.has_value()); + EXPECT_EQ(entry.fractional_percent_value_->denominator(), + envoy::type::v3::FractionalPercent::HUNDRED); + EXPECT_EQ(entry.fractional_percent_value_->numerator(), 11); + + // Make sure the hacky percent function works with numerator and denominator + fractional_value.set_string_value("{\"numerator\": 10000, \"denominator\": \"TEN_THOUSAND\"}"); + entry = const_cast(dynamic_cast(loader_->snapshot())) + .createEntry(fractional_value); + ASSERT_TRUE(entry.fractional_percent_value_.has_value()); + EXPECT_EQ(entry.fractional_percent_value_->denominator(), + envoy::type::v3::FractionalPercent::TEN_THOUSAND); + EXPECT_EQ(entry.fractional_percent_value_->numerator(), 10000); + + // Make sure the hacky fractional percent function works with million + fractional_value.set_string_value("{\"numerator\": 10000, \"denominator\": \"MILLION\"}"); + entry = const_cast(dynamic_cast(loader_->snapshot())) + .createEntry(fractional_value); + ASSERT_TRUE(entry.fractional_percent_value_.has_value()); + EXPECT_EQ(entry.fractional_percent_value_->denominator(), + envoy::type::v3::FractionalPercent::MILLION); + EXPECT_EQ(entry.fractional_percent_value_->numerator(), 10000); + + // Test atoi failure for the hacky fractional percent value function. + fractional_value.set_string_value(" numerator: 1.1 "); + entry = const_cast(dynamic_cast(loader_->snapshot())) + .createEntry(fractional_value); + ASSERT_FALSE(entry.fractional_percent_value_.has_value()); } TEST_F(StaticLoaderImplTest, InvalidNumerator) { From 26b259c25024e4c5cf40fd6ad5ae627f7795fba2 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Thu, 11 May 2023 15:13:43 -0400 Subject: [PATCH 211/740] Update QUICHE from 3a06b4d60 to 5569deaa0 (#27325) git log 3a06b4d60..5569deaa0 --date=short --no-merges --format="%ad %al %s" 2023-05-09 asedeno masque_server_session: factor out event handlers for connect-udp and connect-ip 2023-05-08 asedeno masque_*_bin: create quiche event loop after commandline parsing 2023-05-08 asedeno masque_sever_session: unnest different MASQUE implementations 2023-05-08 asedeno masque_*_bin: use actual logging, not std::cerr 2023-05-08 asedeno Implement QUICHE default platform logging using Abseil Logging 2023-05-08 quiche-dev Fix compilation on Chrome OS 2023-05-08 asedeno masque_utils: open tun_fd with O_NONBLOCK 2023-05-05 diannahu Support multiple interim responses in SimpleClient. 2023-05-05 vasilvv Fix open source QUICHE build 2023-05-05 vasilvv Support enough of Windows to make EndToEndTest.SimpleRequestResponse pass. 2023-05-05 elburrito Force the Public Metadata fingerprint value to be passed as big-endian. 2023-05-05 bnc Add quiche::HttpValidationPolicy::validate_transfer_encoding. 2023-05-04 martinduke Record sent ECN marks correctly for coalesced and buffered packets. 2023-05-04 quiche-dev Deprecate unused field `AT_PUBLIC_METADATA_VERIFIED_KEY_TYPE`. Signed-off-by: Yan Avlasov --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 94d73ae07a8c..7dcdaa32470c 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1054,12 +1054,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "3a06b4d609a514aff792d6c3cf033a3dedb42edb", - sha256 = "23dd8b079cf9d21f3fb5a82a44a9241c52973e457a0c89eb46bf521cd41199bd", + version = "5569deaa0b1ed0b4d9bbb84bddb5c8dafa3e2720", + sha256 = "34b564bcd917f9331a8c52e0e8ae027a478ee6e4a7f28c7643de9611e70d7e47", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2023-05-03", + release_date = "2023-05-09", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", From d152699a25fc05dac72d6200f8f27c725cd16a0a Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 11 May 2023 15:32:35 -0400 Subject: [PATCH 212/740] cluster: moving exceptions to higher level of code (#27288) Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: n/a part of envoyproxy/envoy-mobile#176 Signed-off-by: Alyssa Wilk --- envoy/upstream/cluster_factory.h | 6 +-- envoy/upstream/cluster_manager.h | 2 +- .../common/upstream/cluster_factory_impl.cc | 20 +++++---- source/common/upstream/cluster_factory_impl.h | 4 +- .../common/upstream/cluster_manager_impl.cc | 28 +++++++----- source/common/upstream/cluster_manager_impl.h | 2 +- .../upstream/cluster_factory_impl_test.cc | 44 +++++++++---------- test/common/upstream/test_cluster_manager.h | 7 ++- .../dynamic_forward_proxy/cluster_test.cc | 9 +++- test/mocks/upstream/cluster_manager_factory.h | 3 +- 10 files changed, 69 insertions(+), 56 deletions(-) diff --git a/envoy/upstream/cluster_factory.h b/envoy/upstream/cluster_factory.h index 283cda81231d..996abfed6c53 100644 --- a/envoy/upstream/cluster_factory.h +++ b/envoy/upstream/cluster_factory.h @@ -94,10 +94,10 @@ class ClusterFactory : public Config::UntypedFactory { * with the provided parameters, it should throw an EnvoyException in the case of general error. * @param cluster supplies the general protobuf configuration for the cluster. * @param context supplies the cluster's context. - * @return a pair containing the cluster instance as well as an option thread aware load - * balancer if this cluster has an integrated load balancer. + * @return a pair containing the cluster instance as well as an option thread aware load balancer + * if this cluster has an integrated load balancer or an absl::Satus error on failure. */ - virtual std::pair + virtual absl::StatusOr> create(const envoy::config::cluster::v3::Cluster& cluster, ClusterFactoryContext& context) PURE; std::string category() const override { return "envoy.clusters"; } diff --git a/envoy/upstream/cluster_manager.h b/envoy/upstream/cluster_manager.h index 45172c7c66a0..f6598ad612d2 100644 --- a/envoy/upstream/cluster_manager.h +++ b/envoy/upstream/cluster_manager.h @@ -511,7 +511,7 @@ class ClusterManagerFactory { /** * Allocate a cluster from configuration proto. */ - virtual std::pair + virtual absl::StatusOr> clusterFromProto(const envoy::config::cluster::v3::Cluster& cluster, ClusterManager& cm, Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api) PURE; diff --git a/source/common/upstream/cluster_factory_impl.cc b/source/common/upstream/cluster_factory_impl.cc index dc6feb58ffd2..ea782481749b 100644 --- a/source/common/upstream/cluster_factory_impl.cc +++ b/source/common/upstream/cluster_factory_impl.cc @@ -14,11 +14,13 @@ namespace Envoy { namespace Upstream { -std::pair ClusterFactoryImplBase::create( - const envoy::config::cluster::v3::Cluster& cluster, - Server::Configuration::ServerFactoryContext& server_context, ClusterManager& cm, - LazyCreateDnsResolver dns_resolver_fn, Ssl::ContextManager& ssl_context_manager, - Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api) { +absl::StatusOr> +ClusterFactoryImplBase::create(const envoy::config::cluster::v3::Cluster& cluster, + Server::Configuration::ServerFactoryContext& server_context, + ClusterManager& cm, LazyCreateDnsResolver dns_resolver_fn, + Ssl::ContextManager& ssl_context_manager, + Outlier::EventLoggerSharedPtr outlier_event_logger, + bool added_via_api) { std::string cluster_type; if (!cluster.has_cluster_type()) { @@ -47,14 +49,14 @@ std::pair ClusterFactoryImplBase:: if (cluster.common_lb_config().has_consistent_hashing_lb_config() && cluster.common_lb_config().consistent_hashing_lb_config().use_hostname_for_hashing() && cluster.type() != envoy::config::cluster::v3::Cluster::STRICT_DNS) { - throw EnvoyException(fmt::format( + return absl::InvalidArgumentError(fmt::format( "Cannot use hostname for consistent hashing loadbalancing for cluster of type: '{}'", cluster_type)); } ClusterFactory* factory = Registry::FactoryRegistry::getFactory(cluster_type); if (factory == nullptr) { - throw EnvoyException(fmt::format( + return absl::InvalidArgumentError(fmt::format( "Didn't find a registered cluster factory implementation for name: '{}'", cluster_type)); } @@ -89,7 +91,7 @@ ClusterFactoryImplBase::selectDnsResolver(const envoy::config::cluster::v3::Clus return context.dnsResolver(); } -std::pair +absl::StatusOr> ClusterFactoryImplBase::create(const envoy::config::cluster::v3::Cluster& cluster, ClusterFactoryContext& context) { @@ -101,7 +103,7 @@ ClusterFactoryImplBase::create(const envoy::config::cluster::v3::Cluster& cluste if (!cluster.health_checks().empty()) { // TODO(htuch): Need to support multiple health checks in v2. if (cluster.health_checks().size() != 1) { - throw EnvoyException("Multiple health checks not supported"); + return absl::InvalidArgumentError("Multiple health checks not supported"); } else { new_cluster_pair.first->setHealthChecker(HealthCheckerFactory::create( cluster.health_checks()[0], *new_cluster_pair.first, server_context.runtime(), diff --git a/source/common/upstream/cluster_factory_impl.h b/source/common/upstream/cluster_factory_impl.h index 0b729a970518..c523dedc83cd 100644 --- a/source/common/upstream/cluster_factory_impl.h +++ b/source/common/upstream/cluster_factory_impl.h @@ -104,7 +104,7 @@ class ClusterFactoryImplBase : public ClusterFactory { /** * Static method to get the registered cluster factory and create an instance of cluster. */ - static std::pair + static absl::StatusOr> create(const envoy::config::cluster::v3::Cluster& cluster, Server::Configuration::ServerFactoryContext& server_context, ClusterManager& cm, LazyCreateDnsResolver dns_resolver_fn, Ssl::ContextManager& ssl_context_manager, @@ -118,7 +118,7 @@ class ClusterFactoryImplBase : public ClusterFactory { ClusterFactoryContext& context); // Upstream::ClusterFactory - std::pair + absl::StatusOr> create(const envoy::config::cluster::v3::Cluster& cluster, ClusterFactoryContext& context) override; std::string name() const override { return name_; } diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index df6978c8d20e..d0d78fea2aaf 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -843,9 +843,15 @@ ClusterManagerImpl::ClusterDataPtr ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& cluster, const uint64_t cluster_hash, const std::string& version_info, bool added_via_api, ClusterMap& cluster_map) { - std::pair new_cluster_pair = - factory_.clusterFromProto(cluster, *this, outlier_event_logger_, added_via_api); - auto& new_cluster = new_cluster_pair.first; + absl::StatusOr> + new_cluster_pair_or_error = + factory_.clusterFromProto(cluster, *this, outlier_event_logger_, added_via_api); + + if (!new_cluster_pair_or_error.ok()) { + throw EnvoyException(std::string(new_cluster_pair_or_error.status().message())); + } + auto& new_cluster = new_cluster_pair_or_error->first; + auto& lb = new_cluster_pair_or_error->second; Cluster& cluster_reference = *new_cluster; if (!added_via_api) { @@ -855,15 +861,13 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust } } - if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided && - new_cluster_pair.second == nullptr) { + if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided && lb == nullptr) { throw EnvoyException(fmt::format("cluster manager: cluster provided LB specified but cluster " "'{}' did not provide one. Check cluster documentation.", new_cluster->info()->name())); } - if (cluster_reference.info()->lbType() != LoadBalancerType::ClusterProvided && - new_cluster_pair.second != nullptr) { + if (cluster_reference.info()->lbType() != LoadBalancerType::ClusterProvided && lb != nullptr) { throw EnvoyException( fmt::format("cluster manager: cluster provided LB not specified but cluster " "'{}' provided one. Check cluster documentation.", @@ -925,7 +929,7 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust random_, time_source_); } } else if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided) { - cluster_entry_it->second->thread_aware_lb_ = std::move(new_cluster_pair.second); + cluster_entry_it->second->thread_aware_lb_ = std::move(lb); } else if (cluster_reference.info()->lbType() == LoadBalancerType::LoadBalancingPolicyConfig) { TypedLoadBalancerFactory* typed_lb_factory = cluster_reference.info()->loadBalancerFactory(); RELEASE_ASSERT(typed_lb_factory != nullptr, "ClusterInfo should contain a valid factory"); @@ -1976,9 +1980,11 @@ Tcp::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateTcpConnPool( dispatcher, host, priority, options, transport_socket_options, state, tcp_pool_idle_timeout); } -std::pair ProdClusterManagerFactory::clusterFromProto( - const envoy::config::cluster::v3::Cluster& cluster, ClusterManager& cm, - Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api) { +absl::StatusOr> +ProdClusterManagerFactory::clusterFromProto(const envoy::config::cluster::v3::Cluster& cluster, + ClusterManager& cm, + Outlier::EventLoggerSharedPtr outlier_event_logger, + bool added_via_api) { return ClusterFactoryImplBase::create(cluster, context_, cm, dns_resolver_fn_, ssl_context_manager_, outlier_event_logger, added_via_api); } diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index ebb729f752d5..c375e88a5078 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -86,7 +86,7 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { Network::TransportSocketOptionsConstSharedPtr transport_socket_options, ClusterConnectivityState& state, absl::optional tcp_pool_idle_timeout) override; - std::pair + absl::StatusOr> clusterFromProto(const envoy::config::cluster::v3::Cluster& cluster, ClusterManager& cm, Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api) override; CdsApiPtr createCds(const envoy::config::core::v3::ConfigSource& cds_config, diff --git a/test/common/upstream/cluster_factory_impl_test.cc b/test/common/upstream/cluster_factory_impl_test.cc index badd58b397a6..529dec546e36 100644 --- a/test/common/upstream/cluster_factory_impl_test.cc +++ b/test/common/upstream/cluster_factory_impl_test.cc @@ -90,7 +90,7 @@ TEST_F(TestStaticClusterImplTest, CreateWithoutConfig) { auto create_result = ClusterFactoryImplBase::create(cluster_config, server_context_, cm_, dns_resolver_fn_, ssl_context_manager_, std::move(outlier_event_logger_), false); - auto cluster = create_result.first; + auto cluster = create_result->first; cluster->initialize([] {}); EXPECT_EQ(1UL, cluster->prioritySet().hostSetsPerPriority()[1]->healthyHosts().size()); @@ -134,7 +134,7 @@ TEST_F(TestStaticClusterImplTest, CreateWithStructConfig) { auto create_result = ClusterFactoryImplBase::create(cluster_config, server_context_, cm_, dns_resolver_fn_, ssl_context_manager_, std::move(outlier_event_logger_), false); - auto cluster = create_result.first; + auto cluster = create_result->first; cluster->initialize([] {}); EXPECT_EQ(1UL, cluster->prioritySet().hostSetsPerPriority()[10]->healthyHosts().size()); @@ -176,7 +176,7 @@ TEST_F(TestStaticClusterImplTest, CreateWithTypedConfig) { auto create_result = ClusterFactoryImplBase::create(cluster_config, server_context_, cm_, dns_resolver_fn_, ssl_context_manager_, std::move(outlier_event_logger_), false); - auto cluster = create_result.first; + auto cluster = create_result->first; cluster->initialize([] {}); EXPECT_EQ(1UL, cluster->prioritySet().hostSetsPerPriority()[10]->healthyHosts().size()); @@ -211,17 +211,15 @@ TEST_F(TestStaticClusterImplTest, UnsupportedClusterType) { "@type": type.googleapis.com/test.integration.clusters.CustomStaticConfig priority: 10 )EOF"; - // the factory is not registered, expect to throw - EXPECT_THROW_WITH_MESSAGE( - { - const envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); - auto create_result = ClusterFactoryImplBase::create( - cluster_config, server_context_, cm_, dns_resolver_fn_, ssl_context_manager_, - std::move(outlier_event_logger_), false); - }, - EnvoyException, - "Didn't find a registered cluster factory implementation for name: " - "'envoy.clusters.bad_cluster_name'"); + // the factory is not registered, expect to fail + const envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); + auto create_result = + ClusterFactoryImplBase::create(cluster_config, server_context_, cm_, dns_resolver_fn_, + ssl_context_manager_, std::move(outlier_event_logger_), false); + EXPECT_FALSE(create_result.ok()); + EXPECT_EQ(create_result.status().message(), + "Didn't find a registered cluster factory implementation for name: " + "'envoy.clusters.bad_cluster_name'"); } TEST_F(TestStaticClusterImplTest, HostnameWithoutDNS) { @@ -244,16 +242,14 @@ TEST_F(TestStaticClusterImplTest, HostnameWithoutDNS) { name: envoy.clusters.test_static )EOF"; - EXPECT_THROW_WITH_MESSAGE( - { - const envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); - auto create_result = ClusterFactoryImplBase::create( - cluster_config, server_context_, cm_, dns_resolver_fn_, ssl_context_manager_, - std::move(outlier_event_logger_), false); - }, - EnvoyException, - "Cannot use hostname for consistent hashing loadbalancing for cluster of type: " - "'envoy.clusters.test_static'"); + const envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); + auto create_result = + ClusterFactoryImplBase::create(cluster_config, server_context_, cm_, dns_resolver_fn_, + ssl_context_manager_, std::move(outlier_event_logger_), false); + EXPECT_FALSE(create_result.ok()); + EXPECT_EQ(create_result.status().message(), + "Cannot use hostname for consistent hashing loadbalancing for cluster of type: " + "'envoy.clusters.test_static'"); } } // namespace diff --git a/test/common/upstream/test_cluster_manager.h b/test/common/upstream/test_cluster_manager.h index db02fc526e7d..32688a6e9126 100644 --- a/test/common/upstream/test_cluster_manager.h +++ b/test/common/upstream/test_cluster_manager.h @@ -75,7 +75,10 @@ class TestClusterManagerFactory : public ClusterManagerFactory { [this]() -> Network::DnsResolverSharedPtr { return this->dns_resolver_; }, ssl_context_manager_, outlier_event_logger, added_via_api); // Convert from load balancer unique_ptr -> raw pointer -> unique_ptr. - return std::make_pair(result.first, result.second.release()); + if (!result.ok()) { + throw EnvoyException(std::string(result.status().message())); + } + return std::make_pair(result->first, result->second.release()); })); ON_CALL(server_context_, singletonManager()).WillByDefault(ReturnRef(singleton_manager_)); } @@ -102,7 +105,7 @@ class TestClusterManagerFactory : public ClusterManagerFactory { return Tcp::ConnectionPool::InstancePtr{allocateTcpConnPool_(host)}; } - std::pair + absl::StatusOr> clusterFromProto(const envoy::config::cluster::v3::Cluster& cluster, ClusterManager& cm, Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api) override { diff --git a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc index 4f7162fae77f..29c544dc3125 100644 --- a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc +++ b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc @@ -609,8 +609,13 @@ class ClusterFactoryTest : public testing::Test { true); std::unique_ptr cluster_factory = std::make_unique(); - std::tie(cluster_, thread_aware_lb_) = - cluster_factory->create(cluster_config, cluster_factory_context); + auto result = cluster_factory->create(cluster_config, cluster_factory_context); + if (result.ok()) { + cluster_ = result->first; + thread_aware_lb_ = std::move(result->second); + } else { + throw EnvoyException(std::string(result.status().message())); + } } private: diff --git a/test/mocks/upstream/cluster_manager_factory.h b/test/mocks/upstream/cluster_manager_factory.h index 5b98eb1db3d1..9cf678b4f251 100644 --- a/test/mocks/upstream/cluster_manager_factory.h +++ b/test/mocks/upstream/cluster_manager_factory.h @@ -40,7 +40,8 @@ class MockClusterManagerFactory : public ClusterManagerFactory { Network::TransportSocketOptionsConstSharedPtr, ClusterConnectivityState& state, absl::optional tcp_pool_idle_timeout)); - MOCK_METHOD((std::pair), clusterFromProto, + MOCK_METHOD((absl::StatusOr>), + clusterFromProto, (const envoy::config::cluster::v3::Cluster& cluster, ClusterManager& cm, Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api)); From 122661e80f69d0165173020af1286a36b66df292 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 11 May 2023 20:54:48 +0100 Subject: [PATCH 213/740] ci: Upload Docker images to GCS (#27355) Signed-off-by: Ryan Northey --- .azure-pipelines/stage/publish.yml | 23 +++++++++++----- .azure-pipelines/stage/verify.yml | 44 ++++++++++++++++++++++++++---- .azure-pipelines/stages.yml | 1 + ci/do_ci.sh | 30 ++++---------------- 4 files changed, 60 insertions(+), 38 deletions(-) diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index 32b2cdb91810..eebe78b2c281 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -84,7 +84,7 @@ jobs: parameters: name: publish_docker # VERSION.txt is included to refresh Docker images for release - key: "ci/Dockerfile-envoy | VERSION.txt" + key: "ci/Dockerfile-envoy | VERSION.txt| $(cacheKeyBazelFiles)" version: "$(cacheKeyDockerBuild)" path: "" - bash: | @@ -122,17 +122,26 @@ jobs: DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} DOCKER_BUILD_TIMEOUT: ${{ parameters.timeoutDockerBuild }} + - bash: | echo "disk space at end of build:" df -h displayName: "Check disk space at end" condition: not(canceled()) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/build_images" - artifactName: docker - timeoutInMinutes: 10 - condition: not(canceled()) + + # Publish docker + - script: | + ci/run_envoy_docker.sh 'ci/do_ci.sh docker-upload' + displayName: "Upload Docker to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_RBE: "1" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} + GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + - script: sudo .azure-pipelines/docker/save_cache.sh /mnt/docker_cache displayName: "Cache/save (publish_docker)" diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml index d5f514766b4a..5f7ddabe3d02 100644 --- a/.azure-pipelines/stage/verify.yml +++ b/.azure-pipelines/stage/verify.yml @@ -1,6 +1,10 @@ parameters: +- name: bucketGCP + type: string + default: "" + # Auth - name: authGCP type: string @@ -15,12 +19,40 @@ jobs: steps: - bash: .azure-pipelines/cleanup.sh displayName: "Removing tools from agent" - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "docker" - itemPattern: "docker/envoy*.tar" - targetPath: $(Build.StagingDirectory) + - bash: | + set -e + + if [[ "$BUILD_REASON" == "PullRequest" ]]; then + DOWNLOAD_PATH="$(git rev-parse HEAD | head -c7)" + else + DOWNLOAD_PATH="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-${BUILD_SOURCEBRANCHNAME}}" + fi + + tmpdir=$(mktemp -d) + cd "$tmpdir" + images=("" "contrib" "google-vrp") + for image in "${images[@]}"; do + if [[ -n "$image" ]]; then + variant="${image}-dev" + filename="envoy-${image}.tar" + else + variant=dev + filename="envoy.tar" + fi + echo "Download docker image (https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/docker/${filename}) ..." + curl -sLO "https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/docker/${filename}" + echo "Copy oci image: oci-archive:${filename} docker-daemon:envoyproxy/envoy:${variant}" + skopeo copy -q "oci-archive:${filename}" "docker-daemon:envoyproxy/envoy:${variant}" + rm "$filename" + done + docker images | grep envoy + + - bash: | + set -e + export DEBIAN_FRONTEND=noninteractive + sudo apt-get -qq update -y + sudo apt-get -qq install -y --no-install-recommends expect + - bash: ./ci/do_ci.sh verify_examples env: ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index 1023ce7eefd2..05f9d28fa912 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -167,6 +167,7 @@ stages: - template: stage/verify.yml parameters: authGCP: $(GcpServiceAccountKey) + bucketGCP: $(GcsArtifactBucket) - stage: macos displayName: macOS diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 9c82fac994f3..378ad44fb123 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -157,31 +157,6 @@ function bazel_contrib_binary_build() { } function run_ci_verify () { - local images image oci_archive - echo "verify examples..." - - images=("" "contrib" "google-vrp") - - for image in "${images[@]}"; do - if [[ -n "$image" ]]; then - variant="${image}-dev" - filename="envoy-${image}.tar" - else - variant=dev - filename="envoy.tar" - fi - oci_archive="${ENVOY_DOCKER_BUILD_DIR}/docker/${filename}" - - echo "Copy oci image: oci-archive:${oci_archive} docker-daemon:envoyproxy/envoy:${variant}" - skopeo copy -q "oci-archive:${oci_archive}" "docker-daemon:envoyproxy/envoy:${variant}" - rm "$oci_archive" - done - - docker images | grep envoy - - export DEBIAN_FRONTEND=noninteractive - sudo apt-get -qq update -y - sudo apt-get -qq install -y --no-install-recommends expect export DOCKER_NO_PULL=1 export DOCKER_RMI_CLEANUP=1 # This is set to simulate an environment where users have shared home drives protected @@ -533,6 +508,11 @@ case $CI_TARGET in "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" /source/generated/docs docs ;; + docker-upload) + setup_clang_toolchain + "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" "${BUILD_DIR}/build_images" docker + ;; + fix_proto_format) # proto_format.sh needs to build protobuf. setup_clang_toolchain From b12ad779f5ce839d62df50ed55b8631f649bf4d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 09:53:20 +0100 Subject: [PATCH 214/740] build(deps): bump postgres from `78a275d` to `8d45935` in /examples/shared/postgres (#27366) build(deps): bump postgres in /examples/shared/postgres Bumps postgres from `78a275d` to `8d45935`. --- updated-dependencies: - dependency-name: postgres dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile index bc236287dd27..a4ab85b4bc82 100644 --- a/examples/shared/postgres/Dockerfile +++ b/examples/shared/postgres/Dockerfile @@ -1,3 +1,3 @@ -FROM postgres:latest@sha256:78a275d4c891f7b3a33d3f1a78eda9f1d744954d9e20122bfdc97cdda25cddaf +FROM postgres:latest@sha256:8d45935fb783e72c871072e9eb72ee8c817a9eaf25c405b0e62526b14191368d COPY docker-healthcheck.sh /usr/local/bin/ HEALTHCHECK CMD ["docker-healthcheck.sh"] From 05f397381472520837b97e4df723e3c3d3cc59ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 09:53:52 +0100 Subject: [PATCH 215/740] build(deps): bump jaegertracing/all-in-one from `a34185d` to `b49249c` in /examples/shared/jaeger (#27367) build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger Bumps jaegertracing/all-in-one from `a34185d` to `b49249c`. --- updated-dependencies: - dependency-name: jaegertracing/all-in-one dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/jaeger/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile index 1adb0c095895..f58a70da338f 100644 --- a/examples/shared/jaeger/Dockerfile +++ b/examples/shared/jaeger/Dockerfile @@ -1,4 +1,4 @@ -FROM jaegertracing/all-in-one@sha256:a34185d2d040d44788580359ec9bffac794c4ec3e1c83b1da9b1448e9c8185a7 +FROM jaegertracing/all-in-one@sha256:b49249c222993a19c94a55b08174dce1962652224cce5cb0a2b3a31115457703 HEALTHCHECK \ --interval=1s \ --timeout=1s \ From 5e7a4b5f44f398a42897f743a8665ce92fc563a3 Mon Sep 17 00:00:00 2001 From: "Dr. Andre Vehreschild" <101638173+vehre-x41@users.noreply.github.com> Date: Fri, 12 May 2023 14:44:57 +0200 Subject: [PATCH 216/740] [fuzz] Add a concept of a visitor to gen. validated input. (#26141) Signed-off-by: Andre Vehreschild --- source/common/protobuf/BUILD | 10 +- source/common/protobuf/visitor.cc | 59 +- source/common/protobuf/visitor_helper.cc | 30 ++ source/common/protobuf/visitor_helper.h | 48 ++ .../filters/network/common/fuzz/BUILD | 23 + .../fuzz/network_readfilter_fuzz_test.cc | 12 +- .../fuzz/network_writefilter_fuzz_test.cc | 10 +- ...ated_input_generator_any_map_extensions.cc | 89 +++ ...dated_input_generator_any_map_extensions.h | 11 + test/fuzz/BUILD | 28 + test/fuzz/mutable_visitor.cc | 85 +++ test/fuzz/mutable_visitor.h | 33 ++ test/fuzz/validated_input_generator.cc | 509 ++++++++++++++++++ test/fuzz/validated_input_generator.h | 79 +++ tools/code_format/config.yaml | 1 + 15 files changed, 970 insertions(+), 57 deletions(-) create mode 100644 source/common/protobuf/visitor_helper.cc create mode 100644 source/common/protobuf/visitor_helper.h create mode 100644 test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.cc create mode 100644 test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.h create mode 100644 test/fuzz/mutable_visitor.cc create mode 100644 test/fuzz/mutable_visitor.h create mode 100644 test/fuzz/validated_input_generator.cc create mode 100644 test/fuzz/validated_input_generator.h diff --git a/source/common/protobuf/BUILD b/source/common/protobuf/BUILD index 392e855ab02f..3f1c8e3e4ca8 100644 --- a/source/common/protobuf/BUILD +++ b/source/common/protobuf/BUILD @@ -111,8 +111,14 @@ envoy_cc_library( envoy_cc_library( name = "visitor_lib", - srcs = ["visitor.cc"], - hdrs = ["visitor.h"], + srcs = [ + "visitor.cc", + "visitor_helper.cc", + ], + hdrs = [ + "visitor.h", + "visitor_helper.h", + ], deps = [ ":message_validator_lib", ":protobuf", diff --git a/source/common/protobuf/visitor.cc b/source/common/protobuf/visitor.cc index 659196844a60..c5a9b706ada9 100644 --- a/source/common/protobuf/visitor.cc +++ b/source/common/protobuf/visitor.cc @@ -2,8 +2,8 @@ #include -#include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/utility.h" +#include "source/common/protobuf/visitor_helper.h" #include "udpa/type/v1/typed_struct.pb.h" #include "xds/type/v3/typed_struct.pb.h" @@ -12,53 +12,6 @@ namespace Envoy { namespace ProtobufMessage { namespace { -std::unique_ptr typeUrlToMessage(absl::string_view type_url) { - const absl::string_view inner_type_name = TypeUtil::typeUrlToDescriptorFullName(type_url); - const Protobuf::Descriptor* inner_descriptor = - Protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - std::string(inner_type_name)); - if (inner_descriptor == nullptr) { - return nullptr; - } - auto* inner_message_prototype = - Protobuf::MessageFactory::generated_factory()->GetPrototype(inner_descriptor); - return std::unique_ptr(inner_message_prototype->New()); -} - -template -std::pair, absl::string_view> -convertTypedStruct(const Protobuf::Message& message) { - auto* typed_struct = Protobuf::DynamicCastToGenerated(&message); - auto inner_message = typeUrlToMessage(typed_struct->type_url()); - absl::string_view target_type_url = typed_struct->type_url(); - // inner_message might be invalid as we did not previously check type_url during loading. - if (inner_message != nullptr) { -#ifdef ENVOY_ENABLE_YAML - MessageUtil::jsonConvert(typed_struct->value(), ProtobufMessage::getNullValidationVisitor(), - *inner_message); -#else - throw EnvoyException("JSON and YAML support compiled out."); -#endif - } - return {std::move(inner_message), target_type_url}; -} - -/** - * RAII wrapper that push message to parents on construction and pop it on destruction. - */ -struct ScopedMessageParents { - ScopedMessageParents(std::vector& parents, - const Protobuf::Message& message) - : parents_(parents) { - parents_.push_back(&message); - } - - ~ScopedMessageParents() { parents_.pop_back(); } - -private: - std::vector& parents_; -}; - void traverseMessageWorker(ConstProtoVisitor& visitor, const Protobuf::Message& message, std::vector& parents, bool was_any_or_top_level, bool recurse_into_any) { @@ -71,22 +24,22 @@ void traverseMessageWorker(ConstProtoVisitor& visitor, const Protobuf::Message& if (message.GetDescriptor()->full_name() == "google.protobuf.Any") { auto* any_message = Protobuf::DynamicCastToGenerated(&message); - inner_message = typeUrlToMessage(any_message->type_url()); + inner_message = Helper::typeUrlToMessage(any_message->type_url()); target_type_url = any_message->type_url(); // inner_message must be valid as parsing would have already failed to load if there was an // invalid type_url. MessageUtil::unpackTo(*any_message, *inner_message); } else if (message.GetDescriptor()->full_name() == "xds.type.v3.TypedStruct") { std::tie(inner_message, target_type_url) = - convertTypedStruct(message); + Helper::convertTypedStruct(message); } else if (message.GetDescriptor()->full_name() == "udpa.type.v1.TypedStruct") { std::tie(inner_message, target_type_url) = - convertTypedStruct(message); + Helper::convertTypedStruct(message); } if (inner_message != nullptr) { // Push the Any message as a wrapper. - ScopedMessageParents scoped_parents(parents, message); + Helper::ScopedMessageParents scoped_parents(parents, message); traverseMessageWorker(visitor, *inner_message, parents, true, recurse_into_any); return; } else if (!target_type_url.empty()) { @@ -102,7 +55,7 @@ void traverseMessageWorker(ConstProtoVisitor& visitor, const Protobuf::Message& // If this is a message, recurse in to the sub-message. if (field->cpp_type() == Protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { - ScopedMessageParents scoped_parents(parents, message); + Helper::ScopedMessageParents scoped_parents(parents, message); if (field->is_repeated()) { const int size = reflection->FieldSize(message, field); diff --git a/source/common/protobuf/visitor_helper.cc b/source/common/protobuf/visitor_helper.cc new file mode 100644 index 000000000000..53d6f9fd4b11 --- /dev/null +++ b/source/common/protobuf/visitor_helper.cc @@ -0,0 +1,30 @@ +#include "source/common/protobuf/visitor_helper.h" + +namespace Envoy { +namespace ProtobufMessage { +namespace Helper { + +std::unique_ptr typeUrlToMessage(absl::string_view type_url) { + const absl::string_view inner_type_name = TypeUtil::typeUrlToDescriptorFullName(type_url); + const Protobuf::Descriptor* inner_descriptor = + Protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( + std::string(inner_type_name)); + if (inner_descriptor == nullptr) { + return nullptr; + } + auto* inner_message_prototype = + Protobuf::MessageFactory::generated_factory()->GetPrototype(inner_descriptor); + return std::unique_ptr(inner_message_prototype->New()); +} + +ScopedMessageParents::ScopedMessageParents(std::vector& parents, + const Protobuf::Message& message) + : parents_(parents) { + parents_.push_back(&message); +} + +ScopedMessageParents::~ScopedMessageParents() { parents_.pop_back(); } + +} // namespace Helper +} // namespace ProtobufMessage +} // namespace Envoy diff --git a/source/common/protobuf/visitor_helper.h b/source/common/protobuf/visitor_helper.h new file mode 100644 index 000000000000..187648dcf970 --- /dev/null +++ b/source/common/protobuf/visitor_helper.h @@ -0,0 +1,48 @@ +#pragma once + +#include + +#include "source/common/protobuf/message_validator_impl.h" +#include "source/common/protobuf/protobuf.h" +#include "source/common/protobuf/utility.h" + +namespace Envoy { +namespace ProtobufMessage { +namespace Helper { + +std::unique_ptr typeUrlToMessage(absl::string_view type_url); + +template +std::pair, absl::string_view> +convertTypedStruct(const Protobuf::Message& message) { + auto* typed_struct = Protobuf::DynamicCastToGenerated(&message); + auto inner_message = typeUrlToMessage(typed_struct->type_url()); + absl::string_view target_type_url = typed_struct->type_url(); + // inner_message might be invalid as we did not previously check type_url during loading. + if (inner_message != nullptr) { +#ifdef ENVOY_ENABLE_YAML + MessageUtil::jsonConvert(typed_struct->value(), ProtobufMessage::getNullValidationVisitor(), + *inner_message); +#else + throw EnvoyException("JSON and YAML support compiled out."); +#endif + } + return {std::move(inner_message), target_type_url}; +} + +/** + * RAII wrapper that push message to parents on construction and pop it on destruction. + */ +struct ScopedMessageParents { + ScopedMessageParents(std::vector& parents, + const Protobuf::Message& message); + + ~ScopedMessageParents(); + +private: + std::vector& parents_; +}; + +} // namespace Helper +} // namespace ProtobufMessage +} // namespace Envoy diff --git a/test/extensions/filters/network/common/fuzz/BUILD b/test/extensions/filters/network/common/fuzz/BUILD index 43f0e0a0c9cc..7197944a15b2 100644 --- a/test/extensions/filters/network/common/fuzz/BUILD +++ b/test/extensions/filters/network/common/fuzz/BUILD @@ -36,6 +36,27 @@ envoy_proto_library( ], ) +envoy_cc_test_library( + name = "vig_anymap_ext_lib", + srcs = ["validated_input_generator_any_map_extensions.cc"], + hdrs = ["validated_input_generator_any_map_extensions.h"], + deps = [ + "//source/extensions/access_loggers/file:config", + "//source/extensions/http/early_header_mutation/header_mutation:config", + "//source/extensions/http/header_validators/envoy_default:config", + "//source/extensions/http/original_ip_detection/custom_header:config", + "//source/extensions/http/original_ip_detection/xff:config", + "//source/extensions/matching/actions/format_string:config", + "//source/extensions/matching/common_inputs/environment_variable:config", + "//source/extensions/matching/input_matchers/consistent_hashing:config", + "//source/extensions/matching/input_matchers/ip:config", + "//source/extensions/request_id/uuid:config", + "//test/fuzz:validated_input_generator_lib", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + ], +) + envoy_cc_test_library( name = "uber_readfilter_lib", srcs = [ @@ -68,6 +89,7 @@ envoy_cc_fuzz_test( # these up via the NamedNetworkFilterConfigFactory. deps = [ ":uber_readfilter_lib", + ":vig_anymap_ext_lib", "//source/common/config:utility_lib", "//test/config:utility_lib", "//test/test_common:test_runtime_lib", @@ -99,6 +121,7 @@ envoy_cc_fuzz_test( # these up via the NamedNetworkFilterConfigFactory. deps = [ ":uber_writefilter_lib", + ":vig_anymap_ext_lib", "//source/common/config:utility_lib", "//source/extensions/filters/network/mongo_proxy:config", "//source/extensions/filters/network/zookeeper_proxy:config", diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc index 03d6ba34f76c..465d98ab9f20 100644 --- a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc @@ -5,12 +5,14 @@ #include "test/config/utility.h" #include "test/extensions/filters/network/common/fuzz/network_readfilter_fuzz.pb.validate.h" #include "test/extensions/filters/network/common/fuzz/uber_readfilter.h" +#include "test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.h" #include "test/fuzz/fuzz_runner.h" #include "test/test_common/test_runtime.h" namespace Envoy { namespace Extensions { namespace NetworkFilters { + DEFINE_PROTO_FUZZER(const test::extensions::filters::network::FilterFuzzTestCase& input) { TestDeprecatedV2Api _deprecated_v2_api; ABSL_ATTRIBUTE_UNUSED static PostProcessorRegistration reg = { @@ -31,13 +33,21 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::network::FilterFuzzTestCase if (std::find(filter_names.begin(), filter_names.end(), input->config().name()) == std::end(filter_names)) { absl::string_view filter_name = filter_names[seed % filter_names.size()]; - input->mutable_config()->set_name(std::string(filter_name)); + if (filter_name != input->config().name()) { + // Clear old config, or unpacking non-suitable value may crash. + input->mutable_config()->clear_typed_config(); + input->mutable_config()->set_name(std::string(filter_name)); + } } // Set the corresponding type_url for Any. auto& factory = factories.at(input->config().name()); input->mutable_config()->mutable_typed_config()->set_type_url( absl::StrCat("type.googleapis.com/", factory->createEmptyConfigProto()->GetDescriptor()->full_name())); + + ProtobufMessage::ValidatedInputGenerator generator( + seed, ProtobufMessage::composeFiltersAnyMap(), 40); + ProtobufMessage::traverseMessage(generator, *input, true); }}; try { diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc b/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc index e7eb67c58ebb..285160d3e32b 100644 --- a/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc @@ -5,6 +5,7 @@ #include "test/config/utility.h" #include "test/extensions/filters/network/common/fuzz/network_writefilter_fuzz.pb.validate.h" #include "test/extensions/filters/network/common/fuzz/uber_writefilter.h" +#include "test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.h" #include "test/fuzz/fuzz_runner.h" namespace Envoy { @@ -27,13 +28,20 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::network::FilterFuzzTestCase if (std::find(filter_names.begin(), filter_names.end(), input->config().name()) == std::end(filter_names)) { absl::string_view filter_name = filter_names[seed % filter_names.size()]; - input->mutable_config()->set_name(std::string(filter_name)); + if (filter_name != input->config().name()) { + // Clear old config, or unpacking non-suitable value may crash. + input->mutable_config()->clear_typed_config(); + input->mutable_config()->set_name(std::string(filter_name)); + } } // Set the corresponding type_url for Any. auto& factory = factories.at(input->config().name()); input->mutable_config()->mutable_typed_config()->set_type_url( absl::StrCat("type.googleapis.com/", factory->createEmptyConfigProto()->GetDescriptor()->full_name())); + ProtobufMessage::ValidatedInputGenerator generator(seed, + ProtobufMessage::composeFiltersAnyMap()); + ProtobufMessage::traverseMessage(generator, *input, true); }}; try { TestUtility::validate(input); diff --git a/test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.cc b/test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.cc new file mode 100644 index 000000000000..8a5f234df627 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.cc @@ -0,0 +1,89 @@ +#include "validated_input_generator_any_map_extensions.h" + +#include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/config/trace/v3/datadog.pb.h" + +#include "source/extensions/access_loggers/file/config.h" +#include "source/extensions/http/early_header_mutation/header_mutation/config.h" +#include "source/extensions/http/header_validators/envoy_default/config.h" +#include "source/extensions/http/original_ip_detection/custom_header/config.h" +#include "source/extensions/http/original_ip_detection/xff/config.h" +#include "source/extensions/matching/actions/format_string/config.h" +#include "source/extensions/matching/common_inputs/environment_variable/config.h" +#include "source/extensions/matching/input_matchers/consistent_hashing/config.h" +#include "source/extensions/matching/input_matchers/ip/config.h" +#include "source/extensions/request_id/uuid/config.h" + +namespace Envoy { +namespace ProtobufMessage { + +ValidatedInputGenerator::AnyMap composeFiltersAnyMap() { + static const auto dummy_proto_msg = []() -> std::unique_ptr { + return std::make_unique(); + }; + + static ValidatedInputGenerator::AnyMap any_map; + if (any_map.empty()) { + any_map = ValidatedInputGenerator::getDefaultAnyMap(); + any_map.insert( + {{"envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + {{"typed_header_validation_config", + {{"envoy.extensions.http.header_validators.envoy_default.v3.HeaderValidatorConfig", + []() -> std::unique_ptr { + return Envoy::Extensions::Http::HeaderValidators::EnvoyDefault:: + HeaderValidatorFactoryConfig() + .createEmptyConfigProto(); + }}}}, + {"early_header_mutation_extensions", + {{"envoy.extensions.http.early_header_mutation.header_mutation.v3.HeaderMutation", + []() -> std::unique_ptr { + return Envoy::Extensions::Http::EarlyHeaderMutation::HeaderMutation::Factory() + .createEmptyConfigProto(); + }}}}, + {"original_ip_detection_extensions", + {{"envoy.extensions.http.original_ip_detection.custom_header.v3.CustomHeaderConfig", + []() -> std::unique_ptr { + return Envoy::Extensions::Http::OriginalIPDetection::CustomHeader:: + CustomHeaderIPDetectionFactory() + .createEmptyConfigProto(); + }}, + {"envoy.extensions.http.original_ip_detection.xff.v3.XffConfig", + []() -> std::unique_ptr { + return Envoy::Extensions::Http::OriginalIPDetection::Xff::XffIPDetectionFactory() + .createEmptyConfigProto(); + }}}}, + {"access_log", + {{"envoy.config.accesslog.v3.AccessLog.File", + []() -> std::unique_ptr { + return Envoy::Extensions::AccessLoggers::File::FileAccessLogFactory() + .createEmptyConfigProto(); + }}}}, + {"request_id_extension", + {{"envoy.extensions.request_id.uuid.v3.UuidRequestIdConfig", + []() -> std::unique_ptr { + return Envoy::Extensions::RequestId::UUIDRequestIDExtensionFactory() + .createEmptyConfigProto(); + }}}}}}, + {"envoy.config.core.v3.SubstitutionFormatString", + {{"formatters", + {{"envoy.extensions.formatter.metadata.v3.Metadata", dummy_proto_msg}, + {"envoy.extensions.formatter.req_without_query.v3.ReqWithoutQuery", + dummy_proto_msg}}}}}, + {"envoy.config.route.v3.RouteConfiguration", + {{"cluster_specifier_plugins", + {{"envoy.config.route.v3.ClusterSpecifierPlugin", + []() -> std::unique_ptr { + return std::make_unique(); + }}}}, + {"typed_per_filter_config", {{"envoy.config.route.v3.FilterConfig", dummy_proto_msg}}}}}, + {"envoy.config.trace.v3.Tracing.Http", + {{"typed_config", + {{"envoy.config.trace.v3.DatadogConfig", []() -> std::unique_ptr { + return std::make_unique(); + }}}}}}}); + } + return any_map; +} + +} // namespace ProtobufMessage +} // namespace Envoy diff --git a/test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.h b/test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.h new file mode 100644 index 000000000000..14c885e0c4eb --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/validated_input_generator_any_map_extensions.h @@ -0,0 +1,11 @@ +#pragma once + +#include "test/fuzz/validated_input_generator.h" + +namespace Envoy { +namespace ProtobufMessage { + +ValidatedInputGenerator::AnyMap composeFiltersAnyMap(); + +} // namespace ProtobufMessage +} // namespace Envoy diff --git a/test/fuzz/BUILD b/test/fuzz/BUILD index 51b17c68f410..129b42d04bdb 100644 --- a/test/fuzz/BUILD +++ b/test/fuzz/BUILD @@ -78,6 +78,34 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "mutable_visitor_lib", + srcs = ["mutable_visitor.cc"], + hdrs = ["mutable_visitor.h"], + deps = [ + "//source/common/protobuf", + "//source/common/protobuf:message_validator_lib", + "//source/common/protobuf:utility_lib_header", + "//source/common/protobuf:visitor_lib", + "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto", + "@com_github_cncf_udpa//xds/type/v3:pkg_cc_proto", + ], +) + +envoy_cc_test_library( + name = "validated_input_generator_lib", + srcs = ["validated_input_generator.cc"], + hdrs = ["validated_input_generator.h"], + deps = [ + "//test/fuzz:mutable_visitor_lib", + "//test/fuzz:random_lib", + "@com_envoyproxy_protoc_gen_validate//validate:cc_validate", + "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto", + "@com_github_cncf_udpa//xds/type/v3:pkg_cc_proto", + "@com_github_google_libprotobuf_mutator//:libprotobuf_mutator", + ], +) + envoy_cc_test_library( name = "random_lib", hdrs = ["random.h"], diff --git a/test/fuzz/mutable_visitor.cc b/test/fuzz/mutable_visitor.cc new file mode 100644 index 000000000000..4ee7f29b29e8 --- /dev/null +++ b/test/fuzz/mutable_visitor.cc @@ -0,0 +1,85 @@ +#include "test/fuzz/mutable_visitor.h" + +#include + +#include "source/common/protobuf/utility.h" +#include "source/common/protobuf/visitor_helper.h" + +#include "udpa/type/v1/typed_struct.pb.h" +#include "xds/type/v3/typed_struct.pb.h" + +namespace Envoy { +namespace ProtobufMessage { +namespace { + +void traverseMessageWorkerExt(ProtoVisitor& visitor, Protobuf::Message& message, + std::vector& parents, + bool was_any_or_top_level, bool recurse_into_any, + absl::string_view const& field_name) { + visitor.onEnterMessage(message, parents, was_any_or_top_level, field_name); + + // If told to recurse into Any messages, do that here and skip the rest of the function. + if (recurse_into_any) { + std::unique_ptr inner_message; + absl::string_view target_type_url; + + if (message.GetDescriptor()->full_name() == "google.protobuf.Any") { + auto* any_message = Protobuf::DynamicCastToGenerated(&message); + inner_message = Helper::typeUrlToMessage(any_message->type_url()); + target_type_url = any_message->type_url(); + if (inner_message) { + MessageUtil::unpackTo(*any_message, *inner_message); + } + } else if (message.GetDescriptor()->full_name() == "xds.type.v3.TypedStruct") { + std::tie(inner_message, target_type_url) = + Helper::convertTypedStruct(message); + } else if (message.GetDescriptor()->full_name() == "udpa.type.v1.TypedStruct") { + std::tie(inner_message, target_type_url) = + Helper::convertTypedStruct(message); + } + + if (inner_message != nullptr) { + // Push the Any message as a wrapper. + Helper::ScopedMessageParents scoped_parents(parents, message); + traverseMessageWorkerExt(visitor, *inner_message, parents, true, recurse_into_any, + absl::string_view()); + return; + } else if (!target_type_url.empty()) { + throw EnvoyException(fmt::format("Invalid type_url '{}' during traversal", target_type_url)); + } + } + + const Protobuf::Descriptor* descriptor = message.GetDescriptor(); + const Protobuf::Reflection* reflection = message.GetReflection(); + for (int i = 0; i < descriptor->field_count(); ++i) { + const Protobuf::FieldDescriptor* field = descriptor->field(i); + visitor.onField(message, *field, parents); + + // If this is a message, recurse in to the sub-message. + if (field->cpp_type() == Protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + Helper::ScopedMessageParents scoped_parents(parents, message); + + if (field->is_repeated()) { + const int size = reflection->FieldSize(message, field); + for (int j = 0; j < size; ++j) { + traverseMessageWorkerExt(visitor, *reflection->MutableRepeatedMessage(&message, field, j), + parents, false, recurse_into_any, field->name()); + } + } else if (reflection->HasField(message, field)) { + traverseMessageWorkerExt(visitor, *reflection->MutableMessage(&message, field), parents, + false, recurse_into_any, field->name()); + } + } + } + visitor.onLeaveMessage(message, parents, was_any_or_top_level, field_name); +} + +} // namespace + +void traverseMessage(ProtoVisitor& visitor, Protobuf::Message& message, bool recurse_into_any) { + std::vector parents; + traverseMessageWorkerExt(visitor, message, parents, true, recurse_into_any, "envoy"); +} + +} // namespace ProtobufMessage +} // namespace Envoy diff --git a/test/fuzz/mutable_visitor.h b/test/fuzz/mutable_visitor.h new file mode 100644 index 000000000000..789c1dfbd8de --- /dev/null +++ b/test/fuzz/mutable_visitor.h @@ -0,0 +1,33 @@ +#pragma once + +#include "envoy/common/pure.h" + +#include "source/common/protobuf/protobuf.h" + +#include "absl/types/span.h" + +namespace Envoy { +namespace ProtobufMessage { + +class ProtoVisitor { +public: + virtual ~ProtoVisitor() = default; + + // Invoked when a field is visited, with the message, and field descriptor. + virtual void onField(Protobuf::Message&, const Protobuf::FieldDescriptor&, + const absl::Span) PURE; + + // Invoked when a message is visited, with the message and visited parents. + // @param was_any_or_top_level supplies whether the message was either the top level message or an + // Any before being unpacked for further recursion. The latter can + // only be achieved by using recurse_into_any. + virtual void onEnterMessage(Protobuf::Message&, absl::Span, + bool was_any_or_top_level, absl::string_view const& field_name) PURE; + virtual void onLeaveMessage(Protobuf::Message&, absl::Span, + bool was_any_or_top_level, absl::string_view const& field_name) PURE; +}; + +void traverseMessage(ProtoVisitor& visitor, Protobuf::Message& message, bool recurse_into_any); + +} // namespace ProtobufMessage +} // namespace Envoy diff --git a/test/fuzz/validated_input_generator.cc b/test/fuzz/validated_input_generator.cc new file mode 100644 index 000000000000..0eb859d10859 --- /dev/null +++ b/test/fuzz/validated_input_generator.cc @@ -0,0 +1,509 @@ +#include "validated_input_generator.h" + +#include "source/common/protobuf/visitor_helper.h" + +#include "xds/type/matcher/v3/cel.pb.h" +#include "xds/type/matcher/v3/domain.pb.h" +#include "xds/type/matcher/v3/http_inputs.pb.h" +#include "xds/type/matcher/v3/ip.pb.h" +#include "xds/type/matcher/v3/matcher.pb.h" +#include "xds/type/matcher/v3/range.pb.h" +#include "xds/type/matcher/v3/regex.pb.h" +#include "xds/type/matcher/v3/string.pb.h" + +namespace Envoy { +namespace ProtobufMessage { + +const std::string ValidatedInputGenerator::kAny = "google.protobuf.Any"; + +ValidatedInputGenerator::ValidatedInputGenerator(unsigned int seed, AnyMap&& default_any_map, + unsigned int max_depth) + : current_depth_(0), max_depth_(max_depth), any_map_(std::move(default_any_map)) { + random_.initializeSeed(seed); + mutator_.Seed(seed); +} + +template static bool handleNumericRules(T& number, const R& number_rules) { + if (number_rules.has_ignore_empty() && number_rules.ignore_empty()) { + return false; + } + if (number_rules.has_const_() && number != number_rules.const_()) { + number = number_rules.const_(); + } + if (number_rules.has_lt() && number >= number_rules.lt()) { + number = number_rules.lt() - 1; + } + if (number_rules.has_lte() && number > number_rules.lte()) { + number = number_rules.lte(); + } + if (number_rules.has_gt() && number <= number_rules.gt()) { + number = number_rules.gt() + 1; + } + if (number_rules.has_gte() && number < number_rules.gte()) { + number = number_rules.gte(); + } + if (number_rules.in_size() > 0 || number_rules.not_in_size() > 0) { + ENVOY_LOG_MISC(info, "pgv::U32IntRules::in|not_in rule found, not handling yet"); + } + return true; +} + +static void handleStringRules(std::string& str, const validate::StringRules& string_rules) { + // Multiple rules could be present, therefore use an if and not a switch. + // Go by ascending order of proto-field-index. + if (string_rules.has_const_()) { + str = string_rules.const_(); + } + const size_t c_len = + string_rules.has_min_len() || string_rules.has_max_len() ? pgv::Utf8Len(str) : 0; + if (string_rules.has_min_len() && c_len < string_rules.min_len()) { + // just fill up with 'a's for simplicity + str += std::string(string_rules.min_len() - c_len, 'a'); + } + if (string_rules.has_max_len() && c_len > string_rules.max_len()) { + const char* codepoint_ptr = str.c_str(); + ptrdiff_t byte_len = str.length(); + size_t unicode_len = 0; + int char_len = 0; + while (byte_len > 0 && unicode_len < string_rules.max_len()) { + char_len = Protobuf::UTF8FirstLetterNumBytes(codepoint_ptr, byte_len); + codepoint_ptr += char_len; + byte_len -= char_len; + ++unicode_len; + } + str = std::string(str.c_str(), codepoint_ptr); + } + if (string_rules.has_min_bytes() && str.length() < string_rules.min_bytes()) { + // just fill up with 'a's for simplicity + str += std::string(string_rules.min_bytes() - str.length(), 'a'); + } + if (string_rules.has_max_bytes() && str.length() > string_rules.max_bytes()) { + str = str.substr(0, string_rules.max_bytes()); + } + if (string_rules.has_pattern()) { + ENVOY_LOG_MISC(info, "pgv::StringRules::pattern '{}' found, not handling yet", + string_rules.pattern()); + } + if (string_rules.has_prefix() && + str.substr(0, string_rules.prefix().length()) != string_rules.prefix()) { + if (str.length() < string_rules.prefix().length()) { + str = string_rules.prefix(); + } else { + str.replace(0, string_rules.prefix().length(), string_rules.prefix()); + } + } + if (string_rules.has_suffix() && str.length() >= string_rules.suffix().length() && + str.substr(str.length() - string_rules.suffix().length(), string_rules.suffix().length()) != + string_rules.suffix()) { + if (str.length() < string_rules.suffix().length()) { + str = string_rules.suffix(); + } else { + str.replace(str.length() - string_rules.suffix().length(), string_rules.suffix().length(), + string_rules.suffix()); + } + } + if (string_rules.has_contains() && + str.find_first_of(string_rules.contains(), 0) == std::string::npos) { + str += string_rules.contains(); + } + if (string_rules.in_size() != 0 || string_rules.not_in_size() != 0) { + ENVOY_LOG_MISC(info, "pgv::StringRules::in|not_in rule found, not handling yet"); + } + if (string_rules.has_email()) { + ENVOY_LOG_MISC(info, "pgv::StringRules::email rule found, not handling yet"); + } + if (string_rules.has_hostname() && !pgv::IsHostname(str)) { + str = "localhost.localdomain"; + } + if ((string_rules.has_ip() && !pgv::IsIp(str)) || + (string_rules.has_ipv4() && !pgv::IsIpv4(str))) { + str = "127.0.0.1"; + } + if (string_rules.has_ipv6() && !pgv::IsIpv6(str)) { + str = "::1"; + } + if (string_rules.has_uri()) { + ENVOY_LOG_MISC(info, "pgv::StringRules::uri rule found, not handling yet"); + } + if (string_rules.has_uri_ref()) { + ENVOY_LOG_MISC(info, "pgv::StringRules::uri_ref rule found, not handling yet"); + } + if (string_rules.has_len()) { + ENVOY_LOG_MISC(info, "pgv::StringRules::len rule found, not handling yet"); + } + if (string_rules.has_len_bytes() && str.length() != string_rules.len_bytes()) { + + if (str.length() < string_rules.len_bytes()) { + str += std::string(string_rules.len_bytes() - str.length(), 'a'); + } else { + str = str.substr(0, string_rules.len_bytes()); + } + } + + if (string_rules.has_address()) { + ENVOY_LOG_MISC(info, "pgv::StringRules::address rule found, not handling yet"); + } + if (string_rules.has_uuid()) { + ENVOY_LOG_MISC(info, "pgv::StringRules::uuid rule found, not handling yet"); + } + { + std::string::size_type found_at; + if (string_rules.has_not_contains() && + ((found_at = str.find_first_of(string_rules.not_contains(), 0))) != std::string::npos) { + str[found_at] += 1; // just increase the first character's byte. Yes, quite ugly. + } + } + if (string_rules.has_well_known_regex()) { + ENVOY_LOG_MISC(info, "pgv::StringRules::well_known_regex rule found, not handling yet"); + } +} + +void ValidatedInputGenerator::handleAnyRules( + Protobuf::Message* msg, const validate::AnyRules& any_rules, + const absl::Span& parents) { + if (any_rules.has_required() && any_rules.required()) { + // Stop creating any message when a certain depth is reached + if (max_depth_ > 0 && current_depth_ > max_depth_) { + auto* any_message = Protobuf::DynamicCastToGenerated(msg); + any_message->PackFrom(ProtobufWkt::Struct()); + return; + } + const Protobuf::Descriptor* descriptor = msg->GetDescriptor(); + std::unique_ptr inner_message; + if (descriptor->full_name() == kAny) { + const std::string class_name = parents.back()->GetDescriptor()->full_name(); + AnyMap::const_iterator any_map_cand = any_map_.find(class_name); + if (any_map_cand != any_map_.end()) { + const FieldToTypeUrls& field_to_typeurls = any_map_cand->second; + const std::string field_name = std::string(message_path_.back()); + FieldToTypeUrls::const_iterator field_to_typeurls_cand = field_to_typeurls.find(field_name); + if (field_to_typeurls_cand != any_map_cand->second.end()) { + auto* any_message = Protobuf::DynamicCastToGenerated(msg); + inner_message = ProtobufMessage::Helper::typeUrlToMessage(any_message->type_url()); + if (!inner_message || !any_message->UnpackTo(inner_message.get())) { + const TypeUrlAndFactory& randomed_typeurl = field_to_typeurls_cand->second.at( + random_() % field_to_typeurls_cand->second.size()); + any_message->set_type_url(absl::StrCat("type.googleapis.com/", randomed_typeurl.first)); + auto prototype = randomed_typeurl.second(); + ASSERT(prototype); + any_message->PackFrom(*prototype); + } + return; + } + } + ENVOY_LOG_MISC(info, "!!!! NOT FOUND parent of Any is: {}..{}", + parents[parents.size() - 2]->GetDescriptor()->full_name(), + message_path_[message_path_.size() - 2]); + const Protobuf::Descriptor* par_desc = parents.back()->GetDescriptor(); + ENVOY_LOG_MISC(info, "!!!! NOT FOUND in class {}", par_desc->full_name()); + } + } + if (any_rules.in_size() > 0 || any_rules.not_in_size() > 0) { + ENVOY_LOG_MISC(info, "pgv::AnyRules::in|not_in rule found, not handling yet"); + } +} + +void ValidatedInputGenerator::handleMessageTypedField( + Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, + const Protobuf::Reflection* reflection, const validate::FieldRules& rules, + const absl::Span& parents, const bool force_create) { + + if (field.is_repeated()) { + const validate::RepeatedRules& repeated_rules = rules.repeated(); + std::uint64_t repeat_total64 = static_cast(reflection->FieldSize(msg, &field)); + const bool ignore_empty = repeated_rules.has_ignore_empty() && repeated_rules.ignore_empty(); + if (ignore_empty && repeat_total64 == 0) { + return; + } + if (repeated_rules.has_min_items() && repeated_rules.min_items() > repeat_total64) { + for (; repeated_rules.min_items() > repeat_total64; ++repeat_total64) { + reflection->AddMessage(&msg, &field); + } + } + if (repeated_rules.has_max_items() && repeated_rules.max_items() <= repeat_total64) { + for (; repeat_total64 > repeated_rules.max_items(); --repeat_total64) { + reflection->RemoveLast(&msg, &field); + } + } + if (repeated_rules.has_unique()) { + ENVOY_LOG_MISC(debug, + "repeated protobuf validation rule 'unique' found, but not supported yet"); + } + // The visitor will traverse over all repeated entries anyway. Therefore no + // need to traverse here. + } else { + if (force_create || reflection->HasField(msg, &field) || + (rules.message().has_required() && rules.message().required()) || + (rules.message().IsInitialized())) { + // Enforce that the msg for the given field follows the validation rules, if + // - the member is set already, or + // - needs to be set, but is not, or + // - there are rules to apply. + Protobuf::Message* value = reflection->MutableMessage(&msg, &field); + if (value->GetDescriptor()->options().HasExtension(validate::disabled) && + value->GetDescriptor()->options().GetExtension(validate::disabled)) { + return; + } + switch (rules.type_case()) { + case validate::FieldRules::kAny: { + handleAnyRules(value, rules.any(), parents); + break; + } + default: + break; + } + } + } +} + +// Handle all validation rules for intrinsic types like int, uint and string. +// Messages are more complicated to handle and can not be handled here. +template +void ValidatedInputGenerator::handleIntrinsicTypedField(Protobuf::Message& msg, + const Protobuf::FieldDescriptor& field, + const Protobuf::Reflection* reflection, + const validate::FieldRules& rules, + const bool force) { + + if (field.is_repeated()) { + const validate::RepeatedRules& repeated_rules = rules.repeated(); + std::uint64_t repeat_total64 = static_cast(reflection->FieldSize(msg, &field)); + const bool ignore_empty = repeated_rules.has_ignore_empty() && repeated_rules.ignore_empty(); + if (ignore_empty && repeat_total64 == 0) { + return; + } + if (repeated_rules.has_min_items() && repeated_rules.min_items() > repeat_total64) { + for (; repeated_rules.min_items() > repeat_total64; ++repeat_total64) { + const T value{}; + (*reflection.*FIELDADDER)(&msg, &field, value); + } + } + if (repeated_rules.has_max_items() && repeated_rules.max_items() <= repeat_total64) { + for (; repeat_total64 > repeated_rules.max_items(); --repeat_total64) { + reflection->RemoveLast(&msg, &field); + } + } + if (repeated_rules.has_unique()) { + ENVOY_LOG_MISC(debug, + "repeated protobuf validation rule 'unique' found, but not supported yet"); + } + if (repeated_rules.has_items()) { + const int repeat_total = reflection->FieldSize(msg, &field); + for (int repeat_cnt = 0; repeat_cnt < repeat_total; ++repeat_cnt) { + T value = (*reflection.*REPGETTER)(msg, &field, repeat_cnt); + TYPEHANDLER(value, (repeated_rules.items().*RULEGETTER)()); + (*reflection.*REPSETTER)(&msg, &field, repeat_cnt, value); + } + } + } else { + if (force || reflection->HasField(msg, &field) || + (rules.message().has_required() && rules.message().required()) || + ((rules.*RULEGETTER)().IsInitialized())) { + // Enforce that the msg for the given field follows the validation rules, if + // - the member is set already, or + // - needs to be set, but is not, or + // - there are rules to apply. + T value = (*reflection.*FIELDGETTER)(msg, &field); + TYPEHANDLER(value, (rules.*RULEGETTER)()); + (*reflection.*FIELDSETTER)(&msg, &field, value); + } + } +} + +void ValidatedInputGenerator::onField(Protobuf::Message& msg, + const Protobuf::FieldDescriptor& field, + const absl::Span parents) { + onField(msg, field, parents, false); +} + +void ValidatedInputGenerator::onField(Protobuf::Message& msg, + const Protobuf::FieldDescriptor& field, + const absl::Span parents, + const bool force_create) { + const Protobuf::Reflection* reflection = msg.GetReflection(); + + if (!field.options().HasExtension(validate::rules) && !force_create) { + return; + } + const validate::FieldRules& rules = field.options().GetExtension(validate::rules); + if (rules.message().has_skip() && rules.message().skip()) { + return; + } + + switch (field.cpp_type()) { + case Protobuf::FieldDescriptor::CPPTYPE_INT32: { + handleIntrinsicTypedField< + std::int32_t, &Protobuf::Reflection::GetInt32, &Protobuf::Reflection::SetInt32, + &Protobuf::Reflection::GetRepeatedInt32, &Protobuf::Reflection::SetRepeatedInt32, + &Protobuf::Reflection::AddInt32, &validate::FieldRules::int32>(msg, field, reflection, + rules, force_create); + break; + } + case Protobuf::FieldDescriptor::CPPTYPE_INT64: { + handleIntrinsicTypedField< + std::int64_t, &Protobuf::Reflection::GetInt64, &Protobuf::Reflection::SetInt64, + &Protobuf::Reflection::GetRepeatedInt64, &Protobuf::Reflection::SetRepeatedInt64, + &Protobuf::Reflection::AddInt64, &validate::FieldRules::int64>(msg, field, reflection, + rules, force_create); + break; + } + case Protobuf::FieldDescriptor::CPPTYPE_UINT32: { + handleIntrinsicTypedField< + std::uint32_t, &Protobuf::Reflection::GetUInt32, &Protobuf::Reflection::SetUInt32, + &Protobuf::Reflection::GetRepeatedUInt32, &Protobuf::Reflection::SetRepeatedUInt32, + &Protobuf::Reflection::AddUInt32, &validate::FieldRules::uint32>(msg, field, reflection, + rules, force_create); + break; + } + case Protobuf::FieldDescriptor::CPPTYPE_UINT64: { + handleIntrinsicTypedField< + std::uint64_t, &Protobuf::Reflection::GetUInt64, &Protobuf::Reflection::SetUInt64, + &Protobuf::Reflection::GetRepeatedUInt64, &Protobuf::Reflection::SetRepeatedUInt64, + &Protobuf::Reflection::AddUInt64, &validate::FieldRules::uint64>(msg, field, reflection, + rules, force_create); + break; + } + case Protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + handleIntrinsicTypedField< + double, &Protobuf::Reflection::GetDouble, &Protobuf::Reflection::SetDouble, + &Protobuf::Reflection::GetRepeatedDouble, &Protobuf::Reflection::SetRepeatedDouble, + &Protobuf::Reflection::AddDouble, &validate::FieldRules::double_>(msg, field, reflection, + rules, force_create); + break; + } + case Protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + handleIntrinsicTypedField< + float, &Protobuf::Reflection::GetFloat, &Protobuf::Reflection::SetFloat, + &Protobuf::Reflection::GetRepeatedFloat, &Protobuf::Reflection::SetRepeatedFloat, + &Protobuf::Reflection::AddFloat, &validate::FieldRules::float_>(msg, field, reflection, + rules, force_create); + break; + } + case Protobuf::FieldDescriptor::CPPTYPE_BOOL: + break; + case Protobuf::FieldDescriptor::CPPTYPE_ENUM: + break; + case Protobuf::FieldDescriptor::CPPTYPE_STRING: { + handleIntrinsicTypedField< + std::string, &Protobuf::Reflection::GetString, &Protobuf::Reflection::SetString, + &Protobuf::Reflection::GetRepeatedString, &Protobuf::Reflection::SetRepeatedString, + &Protobuf::Reflection::AddString, &validate::FieldRules::string, &handleStringRules>( + msg, field, reflection, rules, force_create); + break; + } + case Protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + handleMessageTypedField(msg, field, reflection, rules, parents, force_create); + break; + } + default: + break; + } +} + +void ValidatedInputGenerator::onEnterMessage(Protobuf::Message& msg, + absl::Span parents, + bool, absl::string_view const& field_name) { + ++current_depth_; + const Protobuf::Reflection* reflection = msg.GetReflection(); + const Protobuf::Descriptor* descriptor = msg.GetDescriptor(); + message_path_.push_back(field_name); + if (descriptor->full_name() == kAny) { + auto* any_message = Protobuf::DynamicCastToGenerated(&msg); + std::unique_ptr inner_message = + ProtobufMessage::Helper::typeUrlToMessage(any_message->type_url()); + if (!inner_message || !any_message->UnpackTo(inner_message.get())) { + any_message->Clear(); + } + } + for (int oneof_index = 0; oneof_index < descriptor->oneof_decl_count(); ++oneof_index) { + const Protobuf::OneofDescriptor* oneof_desc = descriptor->oneof_decl(oneof_index); + if (oneof_desc->options().HasExtension(validate::required) && + oneof_desc->options().GetExtension(validate::required) && + !reflection->HasOneof(msg, descriptor->oneof_decl(oneof_index))) { + // No required member in one of set, so create one. + for (int index = 0; index < oneof_desc->field_count(); ++index) { + const std::string parents_class_name = parents.back()->GetDescriptor()->full_name(); + // Treat matchers special, because in their oneof they reference themselves, which may + // create long chains. Prefer the first alternative, which does not reference itself. + // Nevertheless do it randomly to allow for some nesting. + if ((max_depth_ > 0 && current_depth_ > max_depth_) || + ((parents_class_name == "xds.type.matcher.v3.Matcher.MatcherList.Predicate" || + parents_class_name == + "xds.type.matcher.v3.Matcher.MatcherList.Predicate.SinglePredicate") && + (random_() % 200) > 0)) { + onField(msg, *oneof_desc->field(0), parents, true); + } else { + // Do not use the first available alternative all the time, because of cyclic + // dependencies. + const int rnd_index = random_() % oneof_desc->field_count(); + onField(msg, *oneof_desc->field(rnd_index), parents, true); + } + // Check if for the above field an entry could be created and quit the inner loop if so. + // It might not be possible, when the datatype is not supported (yet). + if (reflection->HasOneof(msg, descriptor->oneof_decl(oneof_index))) { + break; + } + } + } + } +} + +void ValidatedInputGenerator::onLeaveMessage(Protobuf::Message&, + absl::Span, bool, + absl::string_view const&) { + message_path_.pop_back(); + --current_depth_; +} + +ValidatedInputGenerator::AnyMap ValidatedInputGenerator::getDefaultAnyMap() { + + static const auto dummy_proto_msg = []() -> std::unique_ptr { + return std::make_unique(); + }; + + static const ValidatedInputGenerator::ListOfTypeUrlAndFactory matchers = { + {"xds.type.matcher.v3.CelMatcher", + []() -> std::unique_ptr { + return std::make_unique(); + }}, + {"xds.type.matcher.v3.ServerNameMatcher", + []() -> std::unique_ptr { + return std::make_unique(); + }}, + {"xds.type.matcher.v3.HttpAttributesCelMatchInput", + []() -> std::unique_ptr { + return std::make_unique(); + }}, + {"xds.type.matcher.v3.IPMatcher", + []() -> std::unique_ptr { + return std::make_unique(); + }}, + {"xds.type.matcher.v3.RegexMatcher", + []() -> std::unique_ptr { + return std::make_unique(); + }}, + {"xds.type.matcher.v3.StringMatcher", []() -> std::unique_ptr { + return std::make_unique(); + }}}; + + static const ValidatedInputGenerator::ListOfTypeUrlAndFactory input_matchers = { + {"xds.type.matcher.v3.StringMatcher", []() -> std::unique_ptr { + return std::make_unique(); + }}}; + + static const ValidatedInputGenerator::ListOfTypeUrlAndFactory actions = { + {"envoy.config.core.v3.SubstitutionFormatString", dummy_proto_msg}}; + + static const ValidatedInputGenerator::AnyMap any_map = { + {"xds.type.matcher.v3.Matcher", {{"on_no_match", matchers}}}, + {"xds.type.matcher.v3.Matcher.OnMatch", {{"action", actions}}}, + {"xds.type.matcher.v3.Matcher.MatcherTree", + {{"input", input_matchers}, {"custom_match", input_matchers}}}, + {"xds.type.matcher.v3.Matcher.MatcherList.Predicate.SinglePredicate", + {{"custom_match", input_matchers}, {"input", input_matchers}}}}; + return any_map; +} + +} // namespace ProtobufMessage +} // namespace Envoy diff --git a/test/fuzz/validated_input_generator.h b/test/fuzz/validated_input_generator.h new file mode 100644 index 000000000000..d7844217bbfe --- /dev/null +++ b/test/fuzz/validated_input_generator.h @@ -0,0 +1,79 @@ +#pragma once + +#include "test/fuzz/mutable_visitor.h" +#include "test/fuzz/random.h" + +#include "src/libfuzzer/libfuzzer_mutator.h" +#include "validate/validate.h" +#include "validate/validate.pb.h" + +namespace Envoy { +namespace ProtobufMessage { + +template static bool handleNumericRules(T& number, const R& number_rules); + +class ValidatedInputGenerator : public ProtobufMessage::ProtoVisitor, private pgv::BaseValidator { +public: + static const std::string kAny; + + using SimpleProtoFactory = std::function()>; + using TypeUrlAndFactory = + std::pair; + using ListOfTypeUrlAndFactory = std::vector; + using FieldToTypeUrls = + std::map; + using AnyMap = std::map; + + static AnyMap getDefaultAnyMap(); + + class Mutator : public protobuf_mutator::libfuzzer::Mutator { + public: + using protobuf_mutator::libfuzzer::Mutator::Mutator; + + using protobuf_mutator::libfuzzer::Mutator::MutateString; + }; + ValidatedInputGenerator(unsigned int seed, AnyMap&& default_any_map = getDefaultAnyMap(), + unsigned int max_depth = 0); + + void handleAnyRules(Protobuf::Message* msg, const validate::AnyRules& any_rules, + const absl::Span& parents); + + void handleMessageTypedField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, + const Protobuf::Reflection* reflection, + const validate::FieldRules& rules, + const absl::Span& parents, + const bool force_create); + + // Handle all validation rules for intrinsic types like int, uint and string. + // Messages are more complicated to handle and can not be handled here. + template ::type>> + void handleIntrinsicTypedField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, + const Protobuf::Reflection* reflection, + const validate::FieldRules& rules, const bool force); + + void onField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, + const absl::Span parents) override; + + void onField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, + const absl::Span parents, const bool force_create); + + void onEnterMessage(Protobuf::Message& msg, absl::Span parents, + bool, absl::string_view const& field_name) override; + + void onLeaveMessage(Protobuf::Message&, absl::Span, bool, + absl::string_view const&) override; + +private: + Mutator mutator_; + Random::PsuedoRandomGenerator64 random_; + std::deque message_path_; + unsigned int current_depth_; + + const unsigned int max_depth_; + const AnyMap any_map_; +}; +} // namespace ProtobufMessage +} // namespace Envoy diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index a556ff5a71ba..fc94f107465b 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -89,6 +89,7 @@ paths: - source/common/config/utility.h - source/common/matcher/map_matcher.h - source/common/matcher/field_matcher.h + - source/common/protobuf/visitor_helper.h - source/extensions/common/matcher/trie_matcher.h # These files should not throw exceptions. Add HTTP/1 when exceptions removed. exclude: From b14b3e875db1436d2239c0affb8f9429196b13dd Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Fri, 12 May 2023 11:42:29 -0400 Subject: [PATCH 217/740] Envoy ext_proc filter throw exception when received response timeout Duration is too large (#27260) * Adding ext_proc filter config and response timeout Duration PGVs to avoid ext_proc filter fuzzer crash due to duration config out-of-bounds. Signed-off-by: Yanjun Xiang --- .../filters/http/ext_proc/v3/ext_proc.proto | 23 ++++++---- source/common/protobuf/utility.cc | 31 +++++++++++-- source/common/protobuf/utility.h | 9 ++++ .../filters/http/ext_proc/ext_proc.cc | 18 ++++++-- .../filters/http/ext_proc/ext_proc.h | 2 +- test/common/protobuf/utility_test.cc | 46 +++++++++++++++++++ .../ext_proc/ext_proc_integration_test.cc | 13 +++++- ...h-caae576f1c5a5c4bd6f831dd7d159b2504f476b0 | 36 +++++++++++++++ 8 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-caae576f1c5a5c4bd6f831dd7d159b2504f476b0 diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto index b916f874f30b..a924b7625922 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto @@ -139,13 +139,16 @@ message ExternalProcessor { repeated string response_attributes = 6; // Specifies the timeout for each individual message sent on the stream and - // when the filter is running in synchronous mode. Whenever - // the proxy sends a message on the stream that requires a response, it will - // reset this timer, and will stop processing and return an error (subject - // to the processing mode) if the timer expires before a matching response - // is received. There is no timeout when the filter is running in asynchronous - // mode. Default is 200 milliseconds. - google.protobuf.Duration message_timeout = 7; + // when the filter is running in synchronous mode. Whenever the proxy sends + // a message on the stream that requires a response, it will reset this timer, + // and will stop processing and return an error (subject to the processing mode) + // if the timer expires before a matching response is received. There is no + // timeout when the filter is running in asynchronous mode. The + // ``message_timeout`` range is >= 0s and <= 3600s. Default is 200 milliseconds. + google.protobuf.Duration message_timeout = 7 [(validate.rules).duration = { + lte {seconds: 3600} + gte {} + }]; // Optional additional prefix to use when emitting statistics. This allows to distinguish // emitted statistics between configured *ext_proc* filters in an HTTP filter chain. @@ -166,8 +169,12 @@ message ExternalProcessor { // Specify the upper bound of // :ref:`override_message_timeout ` + // The ``max_message_timeout`` range is >= 0s and <= 3600s. // If not specified, by default it is 0, which will effectively disable the ``override_message_timeout`` API. - google.protobuf.Duration max_message_timeout = 10; + google.protobuf.Duration max_message_timeout = 10 [(validate.rules).duration = { + lte {seconds: 3600} + gte {} + }]; // Prevents clearing the route-cache when the // :ref:`clear_route_cache ` diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index 7cdc7d5042a8..ec8a55a36efb 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -645,14 +645,22 @@ ProtobufWkt::Value ValueUtil::listValue(const std::vector& v namespace { -void validateDuration(const ProtobufWkt::Duration& duration, int64_t max_seconds_value) { +absl::Status validateDurationNoThrow(const ProtobufWkt::Duration& duration, + int64_t max_seconds_value) { if (duration.seconds() < 0 || duration.nanos() < 0) { - throw DurationUtil::OutOfRangeException( + return absl::OutOfRangeError( fmt::format("Expected positive duration: {}", duration.DebugString())); } if (duration.nanos() > 999999999 || duration.seconds() > max_seconds_value) { - throw DurationUtil::OutOfRangeException( - fmt::format("Duration out-of-range: {}", duration.DebugString())); + return absl::OutOfRangeError(fmt::format("Duration out-of-range: {}", duration.DebugString())); + } + return absl::OkStatus(); +} + +void validateDuration(const ProtobufWkt::Duration& duration, int64_t max_seconds_value) { + const auto result = validateDurationNoThrow(duration, max_seconds_value); + if (!result.ok()) { + throw DurationUtil::OutOfRangeException(std::string(result.message())); } } @@ -669,6 +677,12 @@ void validateDurationAsMilliseconds(const ProtobufWkt::Duration& duration) { validateDuration(duration, kMaxInt64Nanoseconds); } +absl::Status validateDurationAsMillisecondsNoThrow(const ProtobufWkt::Duration& duration) { + constexpr int64_t kMaxInt64Nanoseconds = + std::numeric_limits::max() / (1000 * 1000 * 1000); + return validateDurationNoThrow(duration, kMaxInt64Nanoseconds); +} + } // namespace uint64_t DurationUtil::durationToMilliseconds(const ProtobufWkt::Duration& duration) { @@ -676,6 +690,15 @@ uint64_t DurationUtil::durationToMilliseconds(const ProtobufWkt::Duration& durat return Protobuf::util::TimeUtil::DurationToMilliseconds(duration); } +absl::StatusOr +DurationUtil::durationToMillisecondsNoThrow(const ProtobufWkt::Duration& duration) { + const auto result = validateDurationAsMillisecondsNoThrow(duration); + if (!result.ok()) { + return result; + } + return Protobuf::util::TimeUtil::DurationToMilliseconds(duration); +} + uint64_t DurationUtil::durationToSeconds(const ProtobufWkt::Duration& duration) { validateDuration(duration); return Protobuf::util::TimeUtil::DurationToSeconds(duration); diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 7f5847e310a7..ad746301d01e 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -15,6 +15,7 @@ #include "source/common/singleton/const_singleton.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_join.h" // Obtain the value of a wrapped field (e.g. google.protobuf.UInt32Value) if set. Otherwise, return @@ -683,6 +684,14 @@ class DurationUtil { */ static uint64_t durationToMilliseconds(const ProtobufWkt::Duration& duration); + /** + * Same as DurationUtil::durationToMilliseconds but does not throw an exception. + * @param duration protobuf. + * @return duration in milliseconds or an error status. + */ + static absl::StatusOr + durationToMillisecondsNoThrow(const ProtobufWkt::Duration& duration); + /** * Same as Protobuf::util::TimeUtil::DurationToSeconds but with extra validation logic. * Specifically, we ensure that the duration is positive. diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index f849c0fb4794..34c4ce953a9d 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -527,12 +527,20 @@ void Filter::sendTrailers(ProcessorState& state, const Http::HeaderMap& trailers stats_.stream_msgs_sent_.inc(); } -void Filter::onNewTimeout(const uint32_t message_timeout_ms) { +void Filter::onNewTimeout(const ProtobufWkt::Duration& override_message_timeout) { + const auto result = DurationUtil::durationToMillisecondsNoThrow(override_message_timeout); + if (!result.ok()) { + ENVOY_LOG(warn, "Ext_proc server new timeout setting is out of duration range. " + "Ignoring the message."); + stats_.override_message_timeout_ignored_.inc(); + return; + } + const auto message_timeout_ms = result.value(); // The new timeout has to be >=1ms and <= max_message_timeout configured in filter. - const uint32_t min_timeout_ms = 1; - const uint32_t max_timeout_ms = config_->maxMessageTimeout(); + const uint64_t min_timeout_ms = 1; + const uint64_t max_timeout_ms = config_->maxMessageTimeout(); if (message_timeout_ms < min_timeout_ms || message_timeout_ms > max_timeout_ms) { - ENVOY_LOG(warn, "Ext_proc server new timeout setting is out of range. " + ENVOY_LOG(warn, "Ext_proc server new timeout setting is out of config range. " "Ignoring the message."); stats_.override_message_timeout_ignored_.inc(); return; @@ -559,7 +567,7 @@ void Filter::onReceiveMessage(std::unique_ptr&& r) { // Check whether the server is asking to extend the timer. if (response->has_override_message_timeout()) { - onNewTimeout(DurationUtil::durationToMilliseconds(response->override_message_timeout())); + onNewTimeout(response->override_message_timeout()); return; } diff --git a/source/extensions/filters/http/ext_proc/ext_proc.h b/source/extensions/filters/http/ext_proc/ext_proc.h index 0bc052997e88..2824cb4d1944 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.h +++ b/source/extensions/filters/http/ext_proc/ext_proc.h @@ -191,7 +191,7 @@ class Filter : public Logger::Loggable, void onGrpcClose() override; void onMessageTimeout(); - void onNewTimeout(const uint32_t message_timeout_ms); + void onNewTimeout(const ProtobufWkt::Duration& override_message_timeout); void sendBufferedData(ProcessorState& state, ProcessorState::CallbackState new_state, bool end_stream) { diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 030ff3f0ba0e..4e2edbf3e687 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -1705,6 +1705,52 @@ TEST(DurationUtilTest, OutOfRange) { } } +TEST(DurationUtilTest, NoThrow) { + { + // In range test + ProtobufWkt::Duration duration; + duration.set_seconds(5); + duration.set_nanos(10000000); + const auto result = DurationUtil::durationToMillisecondsNoThrow(duration); + EXPECT_TRUE(result.ok()); + EXPECT_TRUE(result.value() == 5010); + } + + // Below are out-of-range tests + { + ProtobufWkt::Duration duration; + duration.set_seconds(-1); + const auto result = DurationUtil::durationToMillisecondsNoThrow(duration); + EXPECT_FALSE(result.ok()); + } + { + ProtobufWkt::Duration duration; + duration.set_nanos(-1); + const auto result = DurationUtil::durationToMillisecondsNoThrow(duration); + EXPECT_FALSE(result.ok()); + } + { + ProtobufWkt::Duration duration; + duration.set_nanos(1000000000); + const auto result = DurationUtil::durationToMillisecondsNoThrow(duration); + EXPECT_FALSE(result.ok()); + } + { + ProtobufWkt::Duration duration; + duration.set_seconds(Protobuf::util::TimeUtil::kDurationMaxSeconds + 1); + const auto result = DurationUtil::durationToMillisecondsNoThrow(duration); + EXPECT_FALSE(result.ok()); + } + { + ProtobufWkt::Duration duration; + constexpr int64_t kMaxInt64Nanoseconds = + std::numeric_limits::max() / (1000 * 1000 * 1000); + duration.set_seconds(kMaxInt64Nanoseconds + 1); + const auto result = DurationUtil::durationToMillisecondsNoThrow(duration); + EXPECT_FALSE(result.ok()); + } +} + // Verify WIP accounting of the file based annotations. This test uses the strict validator to test // that code path. TEST_F(ProtobufUtilityTest, MessageInWipFile) { diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc index 55fb05d10d0f..1d4565a56470 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -315,7 +315,7 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, // ext_proc server sends back a response to tell Envoy to stop the // original timer and start a new timer. - void serverSendNewTimeout(const uint32_t timeout_ms) { + void serverSendNewTimeout(const uint64_t timeout_ms) { ProcessingResponse response; if (timeout_ms < 1000) { response.mutable_override_message_timeout()->set_nanos(timeout_ms * 1000000); @@ -327,7 +327,7 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, // The new timeout message is ignored by Envoy due to different reasons, like // new_timeout setting is out-of-range, or max_message_timeout is not configured. - void newTimeoutWrongConfigTest(const uint32_t timeout_ms) { + void newTimeoutWrongConfigTest(const uint64_t timeout_ms) { // Set envoy filter timeout to be 200ms. proto_config_.mutable_message_timeout()->set_nanos(200000000); // Config max_message_timeout proto to enable the new timeout API. @@ -1960,4 +1960,13 @@ TEST_P(ExtProcIntegrationTest, RequestMessageNewTimeoutNegativeTestTimeoutNotAcc newTimeoutWrongConfigTest(500); } +// Send the new timeout to be an extremely large number, which is out-of-range of duration. +// Verify the code appropriately handled it. +TEST_P(ExtProcIntegrationTest, RequestMessageNewTimeoutOutOfBounds) { + // Config max_message_timeout proto to 100ms to enable the new timeout API. + max_message_timeout_ms_ = 100; + const uint64_t override_message_timeout = 1000000000000000; + newTimeoutWrongConfigTest(override_message_timeout); +} + } // namespace Envoy diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-caae576f1c5a5c4bd6f831dd7d159b2504f476b0 b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-caae576f1c5a5c4bd6f831dd7d159b2504f476b0 new file mode 100644 index 000000000000..ec0f8a51e546 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-caae576f1c5a5c4bd6f831dd7d159b2504f476b0 @@ -0,0 +1,36 @@ +config { + grpc_service { + envoy_grpc { + cluster_name: "#" + retry_policy { + retry_back_off { + base_interval { + seconds: 137438953472 + } + } + } + } + timeout { + seconds: 137438953472 + nanos: 655360 + } + } + request_attributes: "#" + message_timeout { + seconds: 137438953472 + } + max_message_timeout { + seconds: 137438953472 + } + disable_clear_route_cache: true +} +request { +} +response { + request_headers { + } + override_message_timeout { + seconds: 137438953472 + nanos: 655360 + } +} From 1fe98932d6be17ce4a38dd9a3a224db3375b98fa Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Fri, 12 May 2023 13:47:23 -0400 Subject: [PATCH 218/740] grpc: moving more code to extensions (#27116) moves the grpc mux and new grpc mux code into extensions, so builds which do not want xDS do not need to compile that code. Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk Signed-off-by: alyssawilk Co-authored-by: Adi (Suissa) Peleg --- envoy/common/backoff_strategy.h | 5 ++ envoy/config/BUILD | 10 +++ .../config/custom_config_validators.h | 2 +- envoy/config/subscription_factory.h | 20 ++++++ .../integration/xds_integration_test.cc | 4 ++ source/common/common/backoff_strategy.h | 4 +- source/common/config/BUILD | 59 ++--------------- .../config/custom_config_validators_impl.h | 4 +- source/common/config/grpc_stream.h | 4 +- source/common/config/null_grpc_mux_impl.h | 38 +++++++++++ .../config/subscription_factory_impl.cc | 1 - source/common/config/watch_map.h | 2 +- source/common/config/xds_mux/grpc_mux_impl.h | 2 +- source/common/upstream/BUILD | 2 +- .../common/upstream/cluster_manager_impl.cc | 39 ++++++----- source/common/upstream/cluster_manager_impl.h | 1 - .../extensions/config_subscription/grpc/BUILD | 53 ++++++++++++++- .../grpc_collection_subscription_factory.cc | 4 +- .../grpc}/grpc_mux_impl.cc | 30 ++++++++- .../config_subscription/grpc}/grpc_mux_impl.h | 34 ++-------- .../grpc/grpc_subscription_factory.cc | 4 +- .../grpc}/new_grpc_mux_impl.cc | 28 +++++++- .../grpc}/new_grpc_mux_impl.h | 6 +- source/server/BUILD | 2 - source/server/server.cc | 14 ++-- test/common/config/BUILD | 63 ------------------ .../config/delta_subscription_test_harness.h | 2 +- .../config/grpc_subscription_test_harness.h | 2 +- test/common/router/BUILD | 2 +- test/common/router/scoped_rds_test.cc | 2 +- test/extensions/clusters/eds/BUILD | 1 - .../extensions/clusters/eds/eds_speed_test.cc | 2 +- .../extensions/config_subscription/grpc/BUILD | 65 +++++++++++++++++++ .../grpc}/grpc_mux_impl_test.cc | 2 +- .../grpc}/new_grpc_mux_impl_test.cc | 2 +- test/mocks/config/BUILD | 2 +- test/mocks/config/custom_config_validators.h | 2 +- test/per_file_coverage.sh | 2 + 38 files changed, 315 insertions(+), 206 deletions(-) rename {source/common => envoy}/config/custom_config_validators.h (97%) create mode 100644 source/common/config/null_grpc_mux_impl.h rename source/{common/config => extensions/config_subscription/grpc}/grpc_mux_impl.cc (93%) rename source/{common/config => extensions/config_subscription/grpc}/grpc_mux_impl.h (88%) rename source/{common/config => extensions/config_subscription/grpc}/new_grpc_mux_impl.cc (90%) rename source/{common/config => extensions/config_subscription/grpc}/new_grpc_mux_impl.h (98%) create mode 100644 test/extensions/config_subscription/grpc/BUILD rename test/{common/config => extensions/config_subscription/grpc}/grpc_mux_impl_test.cc (99%) rename test/{common/config => extensions/config_subscription/grpc}/new_grpc_mux_impl_test.cc (99%) diff --git a/envoy/common/backoff_strategy.h b/envoy/common/backoff_strategy.h index 2288d93beb7b..976d707873a8 100644 --- a/envoy/common/backoff_strategy.h +++ b/envoy/common/backoff_strategy.h @@ -28,6 +28,11 @@ class BackOffStrategy { * @param base_interval the new base interval for the backoff strategy. */ virtual void reset(uint64_t base_interval) PURE; + + /** + * @return if the time interval exceeds any configured cap on next backoff value. + */ + virtual bool isOverTimeLimit(uint64_t interval_ms) const PURE; }; using BackOffStrategyPtr = std::unique_ptr; diff --git a/envoy/config/BUILD b/envoy/config/BUILD index d527fea857bf..23fd71a5c19a 100644 --- a/envoy/config/BUILD +++ b/envoy/config/BUILD @@ -67,11 +67,21 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "custom_config_validators_interface", + hdrs = ["custom_config_validators.h"], + deps = [ + ":subscription_interface", + ], +) + envoy_cc_library( name = "subscription_factory_interface", hdrs = ["subscription_factory.h"], deps = [ + ":custom_config_validators_interface", ":subscription_interface", + "//envoy/common:backoff_strategy_interface", "@com_github_cncf_udpa//xds/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/source/common/config/custom_config_validators.h b/envoy/config/custom_config_validators.h similarity index 97% rename from source/common/config/custom_config_validators.h rename to envoy/config/custom_config_validators.h index 3d1d5a3ebd4e..d7fdd1fac3f8 100644 --- a/source/common/config/custom_config_validators.h +++ b/envoy/config/custom_config_validators.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/config/config_validator.h" +#include "envoy/config/subscription.h" namespace Envoy { namespace Config { diff --git a/envoy/config/subscription_factory.h b/envoy/config/subscription_factory.h index e2042fa8322b..c59a648a44d3 100644 --- a/envoy/config/subscription_factory.h +++ b/envoy/config/subscription_factory.h @@ -1,7 +1,10 @@ #pragma once #include "envoy/api/api.h" +#include "envoy/common/backoff_strategy.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/custom_config_validators.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/config/typed_config.h" #include "envoy/local_info/local_info.h" @@ -15,6 +18,9 @@ namespace Envoy { namespace Server { class Instance; } // namespace Server +namespace Grpc { +class RawAsyncClient; +} // namespace Grpc namespace Config { class XdsResourcesDelegate; @@ -104,5 +110,19 @@ class ConfigSubscriptionFactory : public Config::UntypedFactory { virtual SubscriptionPtr create(SubscriptionData& data) PURE; }; +class MuxFactory : public Config::UntypedFactory { +public: + std::string category() const override { return "envoy.config_mux"; } + virtual void shutdownAll() PURE; + virtual std::shared_ptr + create(std::unique_ptr&& async_client, Event::Dispatcher& dispatcher, + Random::RandomGenerator& random, Stats::Scope& scope, + const envoy::config::core::v3::ApiConfigSource& ads_config, + const LocalInfo::LocalInfo& local_info, + std::unique_ptr&& config_validators, + BackOffStrategyPtr&& backoff_strategy, OptRef xds_config_tracker, + OptRef xds_resources_delegate) PURE; +}; + } // namespace Config } // namespace Envoy diff --git a/mobile/test/common/integration/xds_integration_test.cc b/mobile/test/common/integration/xds_integration_test.cc index 232bbfe09a88..06eb684552cb 100644 --- a/mobile/test/common/integration/xds_integration_test.cc +++ b/mobile/test/common/integration/xds_integration_test.cc @@ -5,7 +5,9 @@ #include "source/common/grpc/google_grpc_creds_impl.h" #include "source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_factory.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" #include "test/common/grpc/grpc_client_integration.h" #include "test/common/integration/base_client_integration_test.h" @@ -28,6 +30,8 @@ XdsIntegrationTest::XdsIntegrationTest() : BaseClientIntegrationTest(ipVersion() Config::forceRegisterDeltaGrpcCollectionConfigSubscriptionFactory(); Config::forceRegisterAggregatedGrpcCollectionConfigSubscriptionFactory(); Config::forceRegisterAdsCollectionConfigSubscriptionFactory(); + Config::forceRegisterGrpcMuxFactory(); + Config::forceRegisterNewGrpcMuxFactory(); override_builder_config_ = false; expect_dns_ = false; // doesn't use DFP. diff --git a/source/common/common/backoff_strategy.h b/source/common/common/backoff_strategy.h index f0bd09e058d2..9e854a32a061 100644 --- a/source/common/common/backoff_strategy.h +++ b/source/common/common/backoff_strategy.h @@ -40,7 +40,7 @@ class JitteredExponentialBackOffStrategy : public BackOffStrategy { * @param interval time interval to be checked. * @return returns true if interval is greater than the maximum time interval */ - bool isOverTimeLimit(uint64_t interval_ms) const { return interval_ms > max_interval_; } + bool isOverTimeLimit(uint64_t interval_ms) const override { return interval_ms > max_interval_; } private: uint64_t base_interval_; @@ -69,6 +69,7 @@ class JitteredLowerBoundBackOffStrategy : public BackOffStrategy { uint64_t nextBackOffMs() override; void reset() override {} void reset(uint64_t min_interval) override { min_interval_ = min_interval; } + bool isOverTimeLimit(uint64_t) const override { return false; } // no max interval. private: uint64_t min_interval_; @@ -91,6 +92,7 @@ class FixedBackOffStrategy : public BackOffStrategy { uint64_t nextBackOffMs() override; void reset() override {} void reset(uint64_t interval_ms) override { interval_ms_ = interval_ms; } + bool isOverTimeLimit(uint64_t) const override { return false; } // no max interval. private: uint64_t interval_ms_; diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 6d62b4c7f491..08a1ef61ebf5 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -133,51 +133,10 @@ envoy_cc_library( ) envoy_cc_library( - name = "grpc_mux_lib", - srcs = ["grpc_mux_impl.cc"], - hdrs = ["grpc_mux_impl.h"], + name = "null_grpc_mux_lib", + hdrs = ["null_grpc_mux_impl.h"], deps = [ - ":api_version_lib", - ":custom_config_validators_interface", - ":decoded_resource_lib", - ":grpc_stream_lib", - ":ttl_lib", - ":utility_lib", - ":xds_context_params_lib", - ":xds_resource_lib", - ":xds_source_id_lib", "//envoy/config:grpc_mux_interface", - "//envoy/config:subscription_interface", - "//envoy/config:xds_config_tracker_interface", - "//envoy/config:xds_resources_delegate_interface", - "//envoy/upstream:cluster_manager_interface", - "//source/common/common:cleanup_lib", - "//source/common/common:minimal_logger_lib", - "//source/common/common:utility_lib", - "//source/common/memory:utils_lib", - "//source/common/protobuf", - "@com_google_absl//absl/container:btree", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - -envoy_cc_library( - name = "new_grpc_mux_lib", - srcs = ["new_grpc_mux_impl.cc"], - hdrs = ["new_grpc_mux_impl.h"], - deps = [ - ":custom_config_validators_interface", - ":delta_subscription_state_lib", - ":grpc_stream_lib", - ":pausable_ack_queue_lib", - ":watch_map_lib", - ":xds_context_params_lib", - ":xds_resource_lib", - "//envoy/config:xds_config_tracker_interface", - "//envoy/event:dispatcher_interface", - "//envoy/grpc:async_client_interface", - "//source/common/memory:utils_lib", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) @@ -267,7 +226,6 @@ envoy_cc_library( hdrs = ["subscription_factory_impl.h"], deps = [ ":custom_config_validators_lib", - ":new_grpc_mux_lib", ":type_to_endpoint_lib", ":utility_lib", ":xds_resource_lib", @@ -374,10 +332,10 @@ envoy_cc_library( srcs = ["watch_map.cc"], hdrs = ["watch_map.h"], deps = [ - ":custom_config_validators_interface", ":decoded_resource_lib", ":utility_lib", ":xds_resource_lib", + "//envoy/config:custom_config_validators_interface", "//envoy/config:subscription_interface", "//envoy/config:xds_config_tracker_interface", "//source/common/common:assert_lib", @@ -399,22 +357,15 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "custom_config_validators_interface", - hdrs = ["custom_config_validators.h"], - deps = [ - "//envoy/config:config_validator_interface", - ], -) - envoy_cc_library( name = "custom_config_validators_lib", srcs = ["custom_config_validators_impl.cc"], hdrs = ["custom_config_validators_impl.h"], deps = [ - ":custom_config_validators_interface", ":opaque_resource_decoder_lib", ":resource_name_lib", + "//envoy/config:config_validator_interface", + "//envoy/config:custom_config_validators_interface", "//envoy/server:instance_interface", "//source/common/config:utility_lib", ], diff --git a/source/common/config/custom_config_validators_impl.h b/source/common/config/custom_config_validators_impl.h index 0d764922beef..0b31c29fef28 100644 --- a/source/common/config/custom_config_validators_impl.h +++ b/source/common/config/custom_config_validators_impl.h @@ -1,9 +1,9 @@ #pragma once +#include "envoy/config/config_validator.h" +#include "envoy/config/custom_config_validators.h" #include "envoy/server/instance.h" -#include "source/common/config/custom_config_validators.h" - namespace Envoy { namespace Config { diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index b46d1d9b718b..91449aefe090 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -26,7 +26,7 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, public: GrpcStream(GrpcStreamCallbacks* callbacks, Grpc::RawAsyncClientPtr async_client, const Protobuf::MethodDescriptor& service_method, Event::Dispatcher& dispatcher, - Stats::Scope& scope, JitteredExponentialBackOffStrategyPtr backoff_strategy, + Stats::Scope& scope, BackOffStrategyPtr backoff_strategy, const RateLimitSettings& rate_limit_settings) : callbacks_(callbacks), async_client_(std::move(async_client)), service_method_(service_method), @@ -228,7 +228,7 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, // Reestablishes the gRPC channel when necessary, with some backoff politeness. Event::TimerPtr retry_timer_; TimeSource& time_source_; - JitteredExponentialBackOffStrategyPtr backoff_strategy_; + BackOffStrategyPtr backoff_strategy_; // Prevents the Envoy from making too many requests. TokenBucketPtr limit_request_; diff --git a/source/common/config/null_grpc_mux_impl.h b/source/common/config/null_grpc_mux_impl.h new file mode 100644 index 000000000000..b4211e3a21ec --- /dev/null +++ b/source/common/config/null_grpc_mux_impl.h @@ -0,0 +1,38 @@ +#pragma once + +#include "envoy/config/grpc_mux.h" + +namespace Envoy { +namespace Config { + +// A placeholder class returned if no ADS is configured. +class NullGrpcMuxImpl : public GrpcMux, + GrpcStreamCallbacks { +public: + void start() override {} + ScopedResume pause(const std::string&) override { + return std::make_unique([] {}); + } + ScopedResume pause(const std::vector) override { + return std::make_unique([] {}); + } + + GrpcMuxWatchPtr addWatch(const std::string&, const absl::flat_hash_set&, + SubscriptionCallbacks&, OpaqueResourceDecoderSharedPtr, + const SubscriptionOptions&) override { + ExceptionUtil::throwEnvoyException("ADS must be configured to support an ADS config source"); + } + + void requestOnDemandUpdate(const std::string&, const absl::flat_hash_set&) override { + ENVOY_BUG(false, "unexpected request for on demand update"); + } + + void onWriteable() override {} + void onStreamEstablished() override {} + void onEstablishmentFailure() override {} + void onDiscoveryResponse(std::unique_ptr&&, + ControlPlaneStats&) override {} +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 38b744147185..20448c94a375 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -4,7 +4,6 @@ #include "envoy/config/xds_resources_delegate.h" #include "source/common/config/custom_config_validators_impl.h" -#include "source/common/config/new_grpc_mux_impl.h" #include "source/common/config/type_to_endpoint.h" #include "source/common/config/utility.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index 7c0da75df7f5..cd5c1973fd02 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -4,12 +4,12 @@ #include #include +#include "envoy/config/custom_config_validators.h" #include "envoy/config/subscription.h" #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/common/assert.h" #include "source/common/common/logger.h" -#include "source/common/config/custom_config_validators.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" diff --git a/source/common/config/xds_mux/grpc_mux_impl.h b/source/common/config/xds_mux/grpc_mux_impl.h index d91639847372..03c6a9867141 100644 --- a/source/common/config/xds_mux/grpc_mux_impl.h +++ b/source/common/config/xds_mux/grpc_mux_impl.h @@ -7,6 +7,7 @@ #include "envoy/common/random_generator.h" #include "envoy/common/time.h" #include "envoy/common/token_bucket.h" +#include "envoy/config/custom_config_validators.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/config/xds_config_tracker.h" @@ -19,7 +20,6 @@ #include "source/common/common/logger.h" #include "source/common/common/utility.h" #include "source/common/config/api_version.h" -#include "source/common/config/custom_config_validators.h" #include "source/common/config/grpc_stream.h" #include "source/common/config/pausable_ack_queue.h" #include "source/common/config/watch_map.h" diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 337b8f539648..eed605278f73 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -80,6 +80,7 @@ envoy_cc_library( hdrs = ["cluster_manager_impl.h"], deps = [ "//source/extensions/filters/network/http_connection_manager:config", + "//source/common/config:null_grpc_mux_lib", ":cds_api_lib", ":cluster_discovery_manager_lib", ":load_balancer_lib", @@ -101,7 +102,6 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/common:utility_lib", "//source/common/config:custom_config_validators_lib", - "//source/common/config:grpc_mux_lib", "//source/common/config/xds_mux:grpc_mux_lib", "//source/common/config:subscription_factory_lib", "//source/common/config:utility_lib", diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index d0d78fea2aaf..1df9665ce0bc 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -26,7 +26,7 @@ #include "source/common/common/fmt.h" #include "source/common/common/utility.h" #include "source/common/config/custom_config_validators_impl.h" -#include "source/common/config/new_grpc_mux_impl.h" +#include "source/common/config/null_grpc_mux_impl.h" #include "source/common/config/utility.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/config/xds_resource.h" @@ -412,17 +412,18 @@ ClusterManagerImpl::ClusterManagerImpl( std::move(custom_config_validators), std::move(backoff_strategy), makeOptRefFromPtr(xds_config_tracker_.get())); } else { - ads_mux_ = std::make_shared( + auto* factory = Config::Utility::getFactoryByName( + "envoy.config_mux.new_grpc_mux_factory"); + if (!factory) { + throw EnvoyException("envoy.config_mux.new_grpc_mux_factory factory not found"); + } + ads_mux_ = factory->create( Config::Utility::factoryForGrpcApiConfigSource( *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) ->createUncachedRawAsyncClient(), - main_thread_dispatcher, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.discovery.v3.AggregatedDiscoveryService.DeltaAggregatedResources"), - *stats_.rootScope(), - Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info, - std::move(custom_config_validators), std::move(backoff_strategy), - makeOptRefFromPtr(xds_config_tracker_.get())); + main_thread_dispatcher, random_, *stats_.rootScope(), dyn_resources.ads_config(), + local_info, std::move(custom_config_validators), std::move(backoff_strategy), + makeOptRefFromPtr(xds_config_tracker_.get()), {}); } } else { Config::Utility::checkTransportVersion(dyn_resources.ads_config()); @@ -446,20 +447,18 @@ ClusterManagerImpl::ClusterManagerImpl( makeOptRefFromPtr(xds_config_tracker_.get()), xds_delegate_opt_ref, target_xds_authority); } else { - ads_mux_ = std::make_shared( - local_info, + auto* factory = Config::Utility::getFactoryByName( + "envoy.config_mux.grpc_mux_factory"); + if (!factory) { + throw EnvoyException("envoy.config_mux.grpc_mux_factory factory not found"); + } + ads_mux_ = factory->create( Config::Utility::factoryForGrpcApiConfigSource( *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) ->createUncachedRawAsyncClient(), - main_thread_dispatcher, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"), - *stats_.rootScope(), - Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), - bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only(), - std::move(custom_config_validators), std::move(backoff_strategy), - makeOptRefFromPtr(xds_config_tracker_.get()), xds_delegate_opt_ref, - target_xds_authority); + main_thread_dispatcher, random_, *stats_.rootScope(), dyn_resources.ads_config(), + local_info, std::move(custom_config_validators), std::move(backoff_strategy), + makeOptRefFromPtr(xds_config_tracker_.get()), xds_delegate_opt_ref); } } } else { diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index c375e88a5078..77d8b2d1b4bc 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -29,7 +29,6 @@ #include "envoy/upstream/cluster_manager.h" #include "source/common/common/cleanup.h" -#include "source/common/config/grpc_mux_impl.h" #include "source/common/config/subscription_factory_impl.h" #include "source/common/http/async_client_impl.h" #include "source/common/http/http_server_properties_cache_impl.h" diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD index 19d01c1deded..05d85d93ac21 100644 --- a/source/extensions/config_subscription/grpc/BUILD +++ b/source/extensions/config_subscription/grpc/BUILD @@ -9,17 +9,66 @@ licenses(["notice"]) # Apache 2 envoy_extension_package() +envoy_cc_library( + name = "grpc_mux_lib", + srcs = ["grpc_mux_impl.cc"], + hdrs = ["grpc_mux_impl.h"], + deps = [ + "//envoy/config:custom_config_validators_interface", + "//envoy/config:grpc_mux_interface", + "//envoy/config:subscription_interface", + "//envoy/config:xds_config_tracker_interface", + "//envoy/config:xds_resources_delegate_interface", + "//envoy/upstream:cluster_manager_interface", + "//source/common/common:cleanup_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/common:utility_lib", + "//source/common/config:api_version_lib", + "//source/common/config:decoded_resource_lib", + "//source/common/config:grpc_stream_lib", + "//source/common/config:ttl_lib", + "//source/common/config:utility_lib", + "//source/common/config:xds_context_params_lib", + "//source/common/config:xds_resource_lib", + "//source/common/config:xds_source_id_lib", + "//source/common/memory:utils_lib", + "//source/common/protobuf", + "@com_google_absl//absl/container:btree", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "new_grpc_mux_lib", + srcs = ["new_grpc_mux_impl.cc"], + hdrs = ["new_grpc_mux_impl.h"], + deps = [ + "//envoy/config:custom_config_validators_interface", + "//envoy/config:xds_config_tracker_interface", + "//envoy/event:dispatcher_interface", + "//envoy/grpc:async_client_interface", + "//source/common/config:delta_subscription_state_lib", + "//source/common/config:grpc_stream_lib", + "//source/common/config:pausable_ack_queue_lib", + "//source/common/config:watch_map_lib", + "//source/common/config:xds_context_params_lib", + "//source/common/config:xds_resource_lib", + "//source/common/memory:utils_lib", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "grpc_subscription_impl_lib", srcs = ["grpc_subscription_impl.cc"], hdrs = ["grpc_subscription_impl.h"], deps = [ + ":grpc_mux_lib", + ":new_grpc_mux_lib", "//envoy/config:subscription_interface", "//envoy/event:dispatcher_interface", "//envoy/grpc:async_client_interface", "//source/common/config:custom_config_validators_lib", - "//source/common/config:grpc_mux_lib", - "//source/common/config:new_grpc_mux_lib", "//source/common/config:type_to_endpoint_lib", "//source/common/config:xds_resource_lib", "//source/common/config/xds_mux:grpc_mux_lib", diff --git a/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc index 7531b1c3f994..e6b89fc1da05 100644 --- a/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc +++ b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc @@ -1,11 +1,11 @@ #include "source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h" #include "source/common/config/custom_config_validators_impl.h" -#include "source/common/config/grpc_mux_impl.h" -#include "source/common/config/new_grpc_mux_impl.h" #include "source/common/config/type_to_endpoint.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" namespace Envoy { namespace Config { diff --git a/source/common/config/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc similarity index 93% rename from source/common/config/grpc_mux_impl.cc rename to source/extensions/config_subscription/grpc/grpc_mux_impl.cc index e35114c721b8..3dce1268b18d 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc @@ -1,4 +1,4 @@ -#include "source/common/config/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "envoy/service/discovery/v3/discovery.pb.h" @@ -61,7 +61,7 @@ GrpcMuxImpl::GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node, CustomConfigValidatorsPtr&& config_validators, - JitteredExponentialBackOffStrategyPtr backoff_strategy, + BackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, XdsResourcesDelegateOptRef xds_resources_delegate, const std::string& target_xds_authority) @@ -520,5 +520,31 @@ void GrpcMuxImpl::drainRequests() { grpc_stream_.maybeUpdateQueueSizeStat(request_queue_->size()); } +// A factory class for creating GrpcMuxImpl so it does not have to be +// hard-compiled into cluster_manager_impl.cc +class GrpcMuxFactory : public MuxFactory { +public: + std::string name() const override { return "envoy.config_mux.grpc_mux_factory"; } + void shutdownAll() override { return GrpcMuxImpl::shutdownAll(); } + std::shared_ptr + create(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + Random::RandomGenerator&, Stats::Scope& scope, + const envoy::config::core::v3::ApiConfigSource& ads_config, + const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, + BackOffStrategyPtr&& backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, + XdsResourcesDelegateOptRef xds_resources_delegate) override { + return std::make_shared( + local_info, std::move(async_client), dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v3.AggregatedDiscoveryService.StreamAggregatedResources"), + scope, Utility::parseRateLimitSettings(ads_config), + ads_config.set_node_on_first_message_only(), std::move(config_validators), + std::move(backoff_strategy), xds_config_tracker, xds_resources_delegate, + Config::Utility::getGrpcControlPlane(ads_config).value_or("")); + } +}; + +REGISTER_FACTORY(GrpcMuxFactory, MuxFactory); + } // namespace Config } // namespace Envoy diff --git a/source/common/config/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/grpc_mux_impl.h similarity index 88% rename from source/common/config/grpc_mux_impl.h rename to source/extensions/config_subscription/grpc/grpc_mux_impl.h index 28ebf809a925..280a37af6e87 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.h @@ -6,6 +6,7 @@ #include "envoy/common/random_generator.h" #include "envoy/common/time.h" +#include "envoy/config/custom_config_validators.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/config/xds_config_tracker.h" @@ -19,7 +20,6 @@ #include "source/common/common/logger.h" #include "source/common/common/utility.h" #include "source/common/config/api_version.h" -#include "source/common/config/custom_config_validators.h" #include "source/common/config/grpc_stream.h" #include "source/common/config/ttl.h" #include "source/common/config/utility.h" @@ -42,8 +42,7 @@ class GrpcMuxImpl : public GrpcMux, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node, CustomConfigValidatorsPtr&& config_validators, - JitteredExponentialBackOffStrategyPtr backoff_strategy, - XdsConfigTrackerOptRef xds_config_tracker, + BackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, XdsResourcesDelegateOptRef xds_resources_delegate, const std::string& target_xds_authority); @@ -241,33 +240,8 @@ class GrpcMuxImpl : public GrpcMux, using GrpcMuxImplPtr = std::unique_ptr; using GrpcMuxImplSharedPtr = std::shared_ptr; -class NullGrpcMuxImpl : public GrpcMux, - GrpcStreamCallbacks { -public: - void start() override {} - ScopedResume pause(const std::string&) override { - return std::make_unique([] {}); - } - ScopedResume pause(const std::vector) override { - return std::make_unique([] {}); - } - - GrpcMuxWatchPtr addWatch(const std::string&, const absl::flat_hash_set&, - SubscriptionCallbacks&, OpaqueResourceDecoderSharedPtr, - const SubscriptionOptions&) override { - ExceptionUtil::throwEnvoyException("ADS must be configured to support an ADS config source"); - } - - void requestOnDemandUpdate(const std::string&, const absl::flat_hash_set&) override { - ENVOY_BUG(false, "unexpected request for on demand update"); - } - - void onWriteable() override {} - void onStreamEstablished() override {} - void onEstablishmentFailure() override {} - void onDiscoveryResponse(std::unique_ptr&&, - ControlPlaneStats&) override {} -}; +class GrpcMuxFactory; +DECLARE_FACTORY(GrpcMuxFactory); } // namespace Config } // namespace Envoy diff --git a/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc b/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc index 2d703316cd37..82d633e1c2c9 100644 --- a/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc +++ b/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc @@ -1,11 +1,11 @@ #include "source/extensions/config_subscription/grpc/grpc_subscription_factory.h" #include "source/common/config/custom_config_validators_impl.h" -#include "source/common/config/grpc_mux_impl.h" -#include "source/common/config/new_grpc_mux_impl.h" #include "source/common/config/type_to_endpoint.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" namespace Envoy { namespace Config { diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc similarity index 90% rename from source/common/config/new_grpc_mux_impl.cc rename to source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc index 7f0133846a2a..2bf989b4c4b1 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc @@ -1,4 +1,4 @@ -#include "source/common/config/new_grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" #include "envoy/service/discovery/v3/discovery.pb.h" @@ -40,7 +40,7 @@ NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, - JitteredExponentialBackOffStrategyPtr backoff_strategy, + BackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker) : grpc_stream_(this, std::move(async_client), service_method, dispatcher, scope, std::move(backoff_strategy), rate_limit_settings), @@ -328,5 +328,29 @@ absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { return absl::nullopt; } +// A factory class for creating NewGrpcMuxImpl so it does not have to be +// hard-compiled into cluster_manager_impl.cc +class NewGrpcMuxFactory : public MuxFactory { +public: + std::string name() const override { return "envoy.config_mux.new_grpc_mux_factory"; } + void shutdownAll() override { return NewGrpcMuxImpl::shutdownAll(); } + std::shared_ptr + create(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + Random::RandomGenerator&, Stats::Scope& scope, + const envoy::config::core::v3::ApiConfigSource& ads_config, + const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, + BackOffStrategyPtr&& backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, + OptRef) override { + return std::make_shared( + std::move(async_client), dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v3.AggregatedDiscoveryService.DeltaAggregatedResources"), + scope, Utility::parseRateLimitSettings(ads_config), local_info, + std::move(config_validators), std::move(backoff_strategy), xds_config_tracker); + } +}; + +REGISTER_FACTORY(NewGrpcMuxFactory, MuxFactory); + } // namespace Config } // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.h b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h similarity index 98% rename from source/common/config/new_grpc_mux_impl.h rename to source/extensions/config_subscription/grpc/new_grpc_mux_impl.h index e6a9c13aa358..a434a487b8b2 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h @@ -35,8 +35,7 @@ class NewGrpcMuxImpl const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, - CustomConfigValidatorsPtr&& config_validators, - JitteredExponentialBackOffStrategyPtr backoff_strategy, + CustomConfigValidatorsPtr&& config_validators, BackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker); ~NewGrpcMuxImpl() override; @@ -192,5 +191,8 @@ class NewGrpcMuxImpl using NewGrpcMuxImplPtr = std::unique_ptr; using NewGrpcMuxImplSharedPtr = std::shared_ptr; +class NewGrpcMuxFactory; +DECLARE_FACTORY(NewGrpcMuxFactory); + } // namespace Config } // namespace Envoy diff --git a/source/server/BUILD b/source/server/BUILD index 58d3d1a129a5..2d01b38df922 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -414,8 +414,6 @@ envoy_cc_library( "//source/common/common:mutex_tracer_lib", "//source/common/common:perf_tracing_lib", "//source/common/common:utility_lib", - "//source/common/config:grpc_mux_lib", - "//source/common/config:new_grpc_mux_lib", "//source/common/config:utility_lib", "//source/common/config:xds_resource_lib", "//source/common/config/xds_mux:grpc_mux_lib", diff --git a/source/server/server.cc b/source/server/server.cc index 1e2feade4745..2559c6213c35 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -29,8 +29,6 @@ #include "source/common/common/enum_to_int.h" #include "source/common/common/mutex_tracer_impl.h" #include "source/common/common/utility.h" -#include "source/common/config/grpc_mux_impl.h" -#include "source/common/config/new_grpc_mux_impl.h" #include "source/common/config/utility.h" #include "source/common/config/well_known_names.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" @@ -957,8 +955,16 @@ void InstanceImpl::terminate() { stats_store_.shutdownThreading(); // TODO: figure out the correct fix: https://github.com/envoyproxy/envoy/issues/15072. - Config::GrpcMuxImpl::shutdownAll(); - Config::NewGrpcMuxImpl::shutdownAll(); + auto* factory = Config::Utility::getFactoryByName( + "envoy.config_mux.new_grpc_mux_factory"); + if (factory) { + factory->shutdownAll(); + } + factory = + Config::Utility::getFactoryByName("envoy.config_mux.grpc_mux_factory"); + if (factory) { + factory->shutdownAll(); + } Config::XdsMux::GrpcMuxSotw::shutdownAll(); Config::XdsMux::GrpcMuxDelta::shutdownAll(); diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 65295b3142fd..45a96d491346 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -38,7 +38,6 @@ envoy_cc_test( ":delta_subscription_test_harness", "//envoy/config:xds_config_tracker_interface", "//source/common/config:api_version_lib", - "//source/common/config:new_grpc_mux_lib", "//source/common/stats:isolated_store_lib", "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//test/mocks:common_lib", @@ -59,7 +58,6 @@ envoy_cc_test( srcs = ["delta_subscription_state_test.cc"], deps = [ "//source/common/config:delta_subscription_state_lib", - "//source/common/config:new_grpc_mux_lib", "//source/common/config/xds_mux:delta_subscription_state_lib", "//source/common/stats:isolated_store_lib", "//source/extensions/config_subscription/grpc:grpc_subscription_lib", @@ -81,7 +79,6 @@ envoy_cc_test( srcs = ["delta_subscription_state_old_test.cc"], deps = [ "//source/common/config:delta_subscription_state_lib", - "//source/common/config:new_grpc_mux_lib", "//source/common/stats:isolated_store_lib", "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//test/mocks:common_lib", @@ -117,62 +114,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "grpc_mux_impl_test", - srcs = ["grpc_mux_impl_test.cc"], - deps = [ - "//envoy/config:xds_config_tracker_interface", - "//envoy/config:xds_resources_delegate_interface", - "//source/common/config:api_version_lib", - "//source/common/config:grpc_mux_lib", - "//source/common/config:protobuf_link_hacks", - "//source/common/protobuf", - "//source/common/stats:isolated_store_lib", - "//test/common/stats:stat_test_utility_lib", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/config:custom_config_validators_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - "//test/test_common:resources_lib", - "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - -envoy_cc_test( - name = "new_grpc_mux_impl_test", - srcs = ["new_grpc_mux_impl_test.cc"], - deps = [ - "//source/common/config:new_grpc_mux_lib", - "//source/common/config:protobuf_link_hacks", - "//source/common/config/xds_mux:grpc_mux_lib", - "//source/common/protobuf", - "//test/common/stats:stat_test_utility_lib", - "//test/config:v2_link_hacks", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/config:custom_config_validators_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - "//test/test_common:resources_lib", - "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "xds_grpc_mux_impl_test", srcs = ["xds_grpc_mux_impl_test.cc"], @@ -235,8 +176,6 @@ envoy_cc_test_library( "//envoy/config:xds_resources_delegate_interface", "//source/common/common:hash_lib", "//source/common/config:api_version_lib", - "//source/common/config:grpc_mux_lib", - "//source/common/config/xds_mux:grpc_mux_lib", "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//test/mocks/config:config_mocks", "//test/mocks/config:custom_config_validators_mocks", @@ -259,8 +198,6 @@ envoy_cc_test_library( ":subscription_test_harness", "//envoy/config:xds_config_tracker_interface", "//source/common/common:utility_lib", - "//source/common/config:new_grpc_mux_lib", - "//source/common/config/xds_mux:grpc_mux_lib", "//source/common/grpc:common_lib", "//test/mocks/config:config_mocks", "//test/mocks/config:custom_config_validators_mocks", diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 9a2422dd50c2..956af6412b57 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -8,10 +8,10 @@ #include "envoy/config/xds_config_tracker.h" #include "envoy/service/discovery/v3/discovery.pb.h" -#include "source/common/config/new_grpc_mux_impl.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/grpc/common.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" #include "test/common/config/subscription_test_harness.h" #include "test/mocks/common.h" diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 5f90f6cae80b..e6e0d890163f 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -11,8 +11,8 @@ #include "source/common/common/hash.h" #include "source/common/config/api_version.h" -#include "source/common/config/grpc_mux_impl.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "test/common/config/subscription_test_harness.h" diff --git a/test/common/router/BUILD b/test/common/router/BUILD index 1e2f7ddca4fa..a01f8c91bf96 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -156,7 +156,7 @@ envoy_cc_test( "//envoy/config:subscription_interface", "//envoy/init:manager_interface", "//source/common/config:api_version_lib", - "//source/common/config:grpc_mux_lib", + "//source/common/config:null_grpc_mux_lib", "//source/common/config:utility_lib", "//source/common/http:message_lib", "//source/common/json:json_loader_lib", diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index eaf598d89631..d4b8b16f0b5d 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -14,7 +14,7 @@ #include "source/common/config/api_version.h" #include "source/common/config/config_provider_impl.h" -#include "source/common/config/grpc_mux_impl.h" +#include "source/common/config/null_grpc_mux_impl.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/router/scoped_rds.h" diff --git a/test/extensions/clusters/eds/BUILD b/test/extensions/clusters/eds/BUILD index 89070a08523c..f970a51b670b 100644 --- a/test/extensions/clusters/eds/BUILD +++ b/test/extensions/clusters/eds/BUILD @@ -47,7 +47,6 @@ envoy_cc_benchmark_binary( ], deps = [ "//envoy/config:xds_resources_delegate_interface", - "//source/common/config:grpc_mux_lib", "//source/common/config:protobuf_link_hacks", "//source/common/config:utility_lib", "//source/common/config/xds_mux:grpc_mux_lib", diff --git a/test/extensions/clusters/eds/eds_speed_test.cc b/test/extensions/clusters/eds/eds_speed_test.cc index 602792695bfc..98737b2b79b4 100644 --- a/test/extensions/clusters/eds/eds_speed_test.cc +++ b/test/extensions/clusters/eds/eds_speed_test.cc @@ -10,12 +10,12 @@ #include "envoy/service/discovery/v3/discovery.pb.h" #include "envoy/stats/scope.h" -#include "source/common/config/grpc_mux_impl.h" #include "source/common/config/protobuf_link_hacks.h" #include "source/common/config/utility.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/singleton/manager_impl.h" #include "source/extensions/clusters/eds/eds.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/server/transport_socket_config_impl.h" diff --git a/test/extensions/config_subscription/grpc/BUILD b/test/extensions/config_subscription/grpc/BUILD new file mode 100644 index 000000000000..6eafc78a4ca5 --- /dev/null +++ b/test/extensions/config_subscription/grpc/BUILD @@ -0,0 +1,65 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "new_grpc_mux_impl_test", + srcs = ["new_grpc_mux_impl_test.cc"], + deps = [ + "//source/common/config:protobuf_link_hacks", + "//source/common/config/xds_mux:grpc_mux_lib", + "//source/common/protobuf", + "//source/extensions/config_subscription/grpc:new_grpc_mux_lib", + "//test/common/stats:stat_test_utility_lib", + "//test/config:v2_link_hacks", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/config:custom_config_validators_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:resources_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + +envoy_cc_test( + name = "grpc_mux_impl_test", + srcs = ["grpc_mux_impl_test.cc"], + deps = [ + "//envoy/config:xds_config_tracker_interface", + "//envoy/config:xds_resources_delegate_interface", + "//source/common/config:api_version_lib", + "//source/common/config:protobuf_link_hacks", + "//source/common/protobuf", + "//source/common/stats:isolated_store_lib", + "//source/extensions/config_subscription/grpc:grpc_mux_lib", + "//test/common/stats:stat_test_utility_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/config:custom_config_validators_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:resources_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc similarity index 99% rename from test/common/config/grpc_mux_impl_test.cc rename to test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc index e8b41a7cc08c..af24bf376a88 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc @@ -8,11 +8,11 @@ #include "source/common/common/empty_string.h" #include "source/common/config/api_version.h" -#include "source/common/config/grpc_mux_impl.h" #include "source/common/config/protobuf_link_hacks.h" #include "source/common/config/utility.h" #include "source/common/protobuf/protobuf.h" #include "source/common/stats/isolated_store_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "test/common/stats/stat_test_utility.h" #include "test/mocks/common.h" diff --git a/test/common/config/new_grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc similarity index 99% rename from test/common/config/new_grpc_mux_impl_test.cc rename to test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc index 0abf1f935841..7fcee773e465 100644 --- a/test/common/config/new_grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc @@ -7,11 +7,11 @@ #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/common/empty_string.h" -#include "source/common/config/new_grpc_mux_impl.h" #include "source/common/config/protobuf_link_hacks.h" #include "source/common/config/utility.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/protobuf/protobuf.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" #include "test/common/stats/stat_test_utility.h" #include "test/config/v2_link_hacks.h" diff --git a/test/mocks/config/BUILD b/test/mocks/config/BUILD index dd41f9d8b9bd..ea9057e1aed7 100644 --- a/test/mocks/config/BUILD +++ b/test/mocks/config/BUILD @@ -32,6 +32,6 @@ envoy_cc_mock( srcs = ["custom_config_validators.cc"], hdrs = ["custom_config_validators.h"], deps = [ - "//source/common/config:custom_config_validators_interface", + "//envoy/config:config_validator_interface", ], ) diff --git a/test/mocks/config/custom_config_validators.h b/test/mocks/config/custom_config_validators.h index 0d4842e8d230..78241c9db670 100644 --- a/test/mocks/config/custom_config_validators.h +++ b/test/mocks/config/custom_config_validators.h @@ -1,6 +1,6 @@ #pragma once -#include "source/common/config/custom_config_validators.h" +#include "envoy/config/config_validator.h" #include "gmock/gmock.h" diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 31f2a1933732..badf302abe26 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -80,6 +80,8 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/health_checkers/grpc:92.0" "source/extensions/load_balancing_policies:95.5" "source/extensions/load_balancing_policies/subset:94.3" +"source/extensions/config_subscription:94.8" +"source/extensions/config_subscription/grpc:94.0" ) [[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" From e8576f77b7d1082bd5c8f19801da1c4c5ad6a97a Mon Sep 17 00:00:00 2001 From: Raven Black Date: Fri, 12 May 2023 12:08:37 -0700 Subject: [PATCH 219/740] [rbac] Correct deprecation comment (#27374) Change rbac deprecation comment Signed-off-by: Raven Black --- api/envoy/config/rbac/v3/rbac.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/envoy/config/rbac/v3/rbac.proto b/api/envoy/config/rbac/v3/rbac.proto index ce32ea7d6b59..3a9271c0015a 100644 --- a/api/envoy/config/rbac/v3/rbac.proto +++ b/api/envoy/config/rbac/v3/rbac.proto @@ -321,10 +321,10 @@ message Principal { // A CIDR block that describes the downstream IP. // This address will honor proxy protocol, but will not honor XFF. // - // This field is deprecated; either use :ref:`direct_remote_ip - // ` for the same + // This field is deprecated; either use :ref:`remote_ip + // ` for the same // behavior, or use - // :ref:`remote_ip `. + // :ref:`direct_remote_ip `. core.v3.CidrRange source_ip = 5 [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; From a466221dfb6a03ef5faa40646014a7cf4d0bbeda Mon Sep 17 00:00:00 2001 From: Ryan Hamilton Date: Fri, 12 May 2023 15:36:25 -0700 Subject: [PATCH 220/740] Bump coverage for source/extensions/transport_sockets down to 95.6 (#27382) Bump coverage for source/extensions/transport_sockets down to 95.6 Looks like post-submit coverage broke after #27116 ``` Code coverage for source/extensions/transport_sockets is lower than limit of 95.7 (95.6) ``` https://dev.azure.com/cncf/envoy/_build/results?buildId=137095&view=logs&j=bbe4b42d-86e6-5e9c-8a0b-fea01d818a24&t=299ff00a-2cef-5552-88aa-ea9cf69316f0 Signed-off-by: Ryan Hamilton --- test/per_file_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index badf302abe26..4742e08e893f 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -61,7 +61,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/tracers/common/ot:71.7" "source/extensions/tracers/opencensus:93.2" "source/extensions/tracers/zipkin:95.8" -"source/extensions/transport_sockets:95.7" +"source/extensions/transport_sockets:95.6" "source/extensions/transport_sockets/tls:94.9" "source/extensions/transport_sockets/tls/cert_validator:95.1" "source/extensions/transport_sockets/tls/private_key:88.9" From bf96fc81a475f1b0fc7187b0ed91e7f562812faf Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Sat, 13 May 2023 23:53:42 -0400 Subject: [PATCH 221/740] cluster-manager: allow cluster update callback removal during iteration (#27335) Signed-off-by: Adi Suissa-Peleg --- .../common/upstream/cluster_manager_impl.cc | 18 ++++-- .../upstream/cluster_manager_impl_test.cc | 59 +++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 1df9665ce0bc..90105f2daee1 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -812,8 +812,13 @@ bool ClusterManagerImpl::removeCluster(const std::string& cluster_name) { tls_.runOnAllThreads([cluster_name](OptRef cluster_manager) { ASSERT(cluster_manager->thread_local_clusters_.count(cluster_name) == 1); ENVOY_LOG(debug, "removing TLS cluster {}", cluster_name); - for (auto& cb : cluster_manager->update_callbacks_) { - cb->onClusterRemoval(cluster_name); + for (auto cb_it = cluster_manager->update_callbacks_.begin(); + cb_it != cluster_manager->update_callbacks_.end();) { + // The current callback may remove itself from the list, so a handle for + // the next item is fetched before calling the callback. + auto curr_cb_it = cb_it; + ++cb_it; + (*curr_cb_it)->onClusterRemoval(cluster_name); } cluster_manager->thread_local_clusters_.erase(cluster_name); }); @@ -1145,8 +1150,13 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_ } if (new_cluster != nullptr) { - for (auto& cb : cluster_manager->update_callbacks_) { - cb->onClusterAddOrUpdate(*new_cluster); + for (auto cb_it = cluster_manager->update_callbacks_.begin(); + cb_it != cluster_manager->update_callbacks_.end();) { + // The current callback may remove itself from the list, so a handle for + // the next item is fetched before calling the callback. + auto curr_cb_it = cb_it; + ++cb_it; + (*curr_cb_it)->onClusterAddOrUpdate(*new_cluster); } } }); diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index c55bee1639b4..97a7283a3d00 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -1983,6 +1983,65 @@ TEST_F(ClusterManagerImplTest, DynamicAddRemove) { EXPECT_TRUE(Mock::VerifyAndClearExpectations(callbacks.get())); } +// Validates that a callback can remove itself from the callbacks list. +TEST_F(ClusterManagerImplTest, ClusterAddOrUpdateCallbackRemovalDuringIteration) { + create(defaultConfig()); + + InSequence s; + ReadyWatcher initialized; + EXPECT_CALL(initialized, ready()); + cluster_manager_->setInitializedCb([&]() -> void { initialized.ready(); }); + + std::unique_ptr callbacks(new NiceMock()); + ClusterUpdateCallbacksHandlePtr cb = + cluster_manager_->addThreadLocalClusterUpdateCallbacks(*callbacks); + + std::shared_ptr cluster1(new NiceMock()); + EXPECT_CALL(factory_, clusterFromProto_(_, _, _, _)) + .WillOnce(Return(std::make_pair(cluster1, nullptr))); + EXPECT_CALL(*cluster1, initializePhase()).Times(0); + EXPECT_CALL(*cluster1, initialize(_)); + EXPECT_CALL(*callbacks, onClusterAddOrUpdate(_)).WillOnce(Invoke([&cb](ThreadLocalCluster&) { + // This call will remove the callback from the list. + cb.reset(); + })); + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "")); + checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 0 /*active*/, 1 /*warming*/); + EXPECT_EQ(1, cluster_manager_->warmingClusterCount()); + EXPECT_EQ(nullptr, cluster_manager_->getThreadLocalCluster("fake_cluster")); + cluster1->initialize_callback_(); + + EXPECT_EQ(cluster1->info_, cluster_manager_->getThreadLocalCluster("fake_cluster")->info()); + checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 1 /*active*/, 0 /*warming*/); + EXPECT_EQ(0, cluster_manager_->warmingClusterCount()); + + // Now do it again with a different hash. + auto update_cluster = defaultStaticCluster("fake_cluster"); + update_cluster.mutable_per_connection_buffer_limit_bytes()->set_value(12345); + + std::shared_ptr cluster2(new NiceMock()); + cluster2->prioritySet().getMockHostSet(0)->hosts_ = { + makeTestHost(cluster2->info_, "tcp://127.0.0.1:80", time_system_)}; + EXPECT_CALL(factory_, clusterFromProto_(_, _, _, _)) + .WillOnce(Return(std::make_pair(cluster2, nullptr))); + EXPECT_CALL(*cluster2, initializePhase()).Times(0); + EXPECT_CALL(*cluster2, initialize(_)) + .WillOnce(Invoke([cluster1](std::function initialize_callback) { + // Test inline init. + initialize_callback(); + })); + // There shouldn't be a call to onClusterAddOrUpdate on the callbacks as the + // handler was removed. + EXPECT_CALL(*callbacks, onClusterAddOrUpdate(_)).Times(0); + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(update_cluster, "")); + + checkStats(1 /*added*/, 1 /*modified*/, 0 /*removed*/, 1 /*active*/, 0 /*warming*/); + + EXPECT_TRUE(Mock::VerifyAndClearExpectations(cluster1.get())); + EXPECT_TRUE(Mock::VerifyAndClearExpectations(cluster2.get())); + EXPECT_TRUE(Mock::VerifyAndClearExpectations(callbacks.get())); +} + TEST_F(ClusterManagerImplTest, AddOrUpdateClusterStaticExists) { const std::string json = fmt::sprintf("{\"static_resources\":{%s}}", clustersJson({defaultStaticClusterJson("fake_cluster")})); From 1e514959d4b7d9bf8d63aeb9fbee08b621c04c20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 10:31:31 +0100 Subject: [PATCH 222/740] build(deps): bump sphinx from 7.0.0 to 7.0.1 in /tools/base (#27394) Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 7.0.0 to 7.0.1. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.0.0...v7.0.1) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index fce98079b825..03539bbf0380 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -1192,9 +1192,9 @@ snowballstemmer==2.2.0 \ --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a # via sphinx -sphinx==7.0.0 \ - --hash=sha256:283c44aa28922bb4223777b44ac0d59af50a279ac7690dfe945bb2b9575dc41b \ - --hash=sha256:3cfc1c6756ef1b132687b813ec6ea2214cb7a7e5d1dcb2772006cb895a0fa469 +sphinx==7.0.1 \ + --hash=sha256:60c5e04756c1709a98845ed27a2eed7a556af3993afb66e77fec48189f742616 \ + --hash=sha256:61e025f788c5977d9412587e733733a289e2b9fdc2fef8868ddfbfc4ccfe881d # via # -r requirements.in # envoy-docs-sphinx-runner From 52a9f0731a0adc9176b8c8c79bb7bac2cd1d0ddd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 10:32:04 +0100 Subject: [PATCH 223/740] build(deps): bump grpcio-tools from 1.54.0 to 1.54.2 in /examples/grpc-bridge/client (#27393) build(deps): bump grpcio-tools in /examples/grpc-bridge/client Bumps [grpcio-tools](https://github.com/grpc/grpc) from 1.54.0 to 1.54.2. - [Release notes](https://github.com/grpc/grpc/releases) - [Changelog](https://github.com/grpc/grpc/blob/master/doc/grpc_release_schedule.md) - [Commits](https://github.com/grpc/grpc/compare/v1.54.0...v1.54.2) --- updated-dependencies: - dependency-name: grpcio-tools dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/client/requirements.txt | 184 +++++++++---------- 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index 918e6db67ffa..d1e3fa54972a 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -12,101 +12,101 @@ charset-normalizer==2.0.6 \ --hash=sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6 \ --hash=sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f # via requests -grpcio==1.54.0 \ - --hash=sha256:02000b005bc8b72ff50c477b6431e8886b29961159e8b8d03c00b3dd9139baed \ - --hash=sha256:031bbd26656e0739e4b2c81c172155fb26e274b8d0312d67aefc730bcba915b6 \ - --hash=sha256:1209d6b002b26e939e4c8ea37a3d5b4028eb9555394ea69fb1adbd4b61a10bb8 \ - --hash=sha256:125ed35aa3868efa82eabffece6264bf638cfdc9f0cd58ddb17936684aafd0f8 \ - --hash=sha256:1382bc499af92901c2240c4d540c74eae8a671e4fe9839bfeefdfcc3a106b5e2 \ - --hash=sha256:16bca8092dd994f2864fdab278ae052fad4913f36f35238b2dd11af2d55a87db \ - --hash=sha256:1c59d899ee7160638613a452f9a4931de22623e7ba17897d8e3e348c2e9d8d0b \ - --hash=sha256:1d109df30641d050e009105f9c9ca5a35d01e34d2ee2a4e9c0984d392fd6d704 \ - --hash=sha256:1fa7d6ddd33abbd3c8b3d7d07c56c40ea3d1891ce3cd2aa9fa73105ed5331866 \ - --hash=sha256:21c4a1aae861748d6393a3ff7867473996c139a77f90326d9f4104bebb22d8b8 \ - --hash=sha256:224166f06ccdaf884bf35690bf4272997c1405de3035d61384ccb5b25a4c1ca8 \ - --hash=sha256:2262bd3512ba9e9f0e91d287393df6f33c18999317de45629b7bd46c40f16ba9 \ - --hash=sha256:2585b3c294631a39b33f9f967a59b0fad23b1a71a212eba6bc1e3ca6e6eec9ee \ - --hash=sha256:27fb030a4589d2536daec5ff5ba2a128f4f155149efab578fe2de2cb21596d3d \ - --hash=sha256:30fbbce11ffeb4f9f91c13fe04899aaf3e9a81708bedf267bf447596b95df26b \ - --hash=sha256:3930669c9e6f08a2eed824738c3d5699d11cd47a0ecc13b68ed11595710b1133 \ - --hash=sha256:3b170e441e91e4f321e46d3cc95a01cb307a4596da54aca59eb78ab0fc03754d \ - --hash=sha256:3db71c6f1ab688d8dfc102271cedc9828beac335a3a4372ec54b8bf11b43fd29 \ - --hash=sha256:48cb7af77238ba16c77879009003f6b22c23425e5ee59cb2c4c103ec040638a5 \ - --hash=sha256:49eace8ea55fbc42c733defbda1e4feb6d3844ecd875b01bb8b923709e0f5ec8 \ - --hash=sha256:533eaf5b2a79a3c6f35cbd6a095ae99cac7f4f9c0e08bdcf86c130efd3c32adf \ - --hash=sha256:5942a3e05630e1ef5b7b5752e5da6582460a2e4431dae603de89fc45f9ec5aa9 \ - --hash=sha256:62117486460c83acd3b5d85c12edd5fe20a374630475388cfc89829831d3eb79 \ - --hash=sha256:650f5f2c9ab1275b4006707411bb6d6bc927886874a287661c3c6f332d4c068b \ - --hash=sha256:6dc1e2c9ac292c9a484ef900c568ccb2d6b4dfe26dfa0163d5bc815bb836c78d \ - --hash=sha256:73c238ef6e4b64272df7eec976bb016c73d3ab5a6c7e9cd906ab700523d312f3 \ - --hash=sha256:775a2f70501370e5ba54e1ee3464413bff9bd85bd9a0b25c989698c44a6fb52f \ - --hash=sha256:860fcd6db7dce80d0a673a1cc898ce6bc3d4783d195bbe0e911bf8a62c93ff3f \ - --hash=sha256:87f47bf9520bba4083d65ab911f8f4c0ac3efa8241993edd74c8dd08ae87552f \ - --hash=sha256:960b176e0bb2b4afeaa1cd2002db1e82ae54c9b6e27ea93570a42316524e77cf \ - --hash=sha256:a7caf553ccaf715ec05b28c9b2ab2ee3fdb4036626d779aa09cf7cbf54b71445 \ - --hash=sha256:a947d5298a0bbdd4d15671024bf33e2b7da79a70de600ed29ba7e0fef0539ebb \ - --hash=sha256:a97b0d01ae595c997c1d9d8249e2d2da829c2d8a4bdc29bb8f76c11a94915c9a \ - --hash=sha256:b7655f809e3420f80ce3bf89737169a9dce73238af594049754a1128132c0da4 \ - --hash=sha256:c33744d0d1a7322da445c0fe726ea6d4e3ef2dfb0539eadf23dce366f52f546c \ - --hash=sha256:c55a9cf5cba80fb88c850915c865b8ed78d5e46e1f2ec1b27692f3eaaf0dca7e \ - --hash=sha256:d2f62fb1c914a038921677cfa536d645cb80e3dd07dc4859a3c92d75407b90a5 \ - --hash=sha256:d8ae6e0df3a608e99ee1acafaafd7db0830106394d54571c1ece57f650124ce9 \ - --hash=sha256:e355ee9da9c1c03f174efea59292b17a95e0b7b4d7d2a389265f731a9887d5a9 \ - --hash=sha256:e3e526062c690517b42bba66ffe38aaf8bc99a180a78212e7b22baa86902f690 \ - --hash=sha256:eb0807323572642ab73fd86fe53d88d843ce617dd1ddf430351ad0759809a0ae \ - --hash=sha256:ebff0738be0499d7db74d20dca9f22a7b27deae31e1bf92ea44924fd69eb6251 \ - --hash=sha256:ed36e854449ff6c2f8ee145f94851fe171298e1e793f44d4f672c4a0d78064e7 \ - --hash=sha256:ed3d458ded32ff3a58f157b60cc140c88f7ac8c506a1c567b2a9ee8a2fd2ce54 \ - --hash=sha256:f4a7dca8ccd8023d916b900aa3c626f1bd181bd5b70159479b142f957ff420e4 +grpcio==1.54.2 \ + --hash=sha256:0212e2f7fdf7592e4b9d365087da30cb4d71e16a6f213120c89b4f8fb35a3ab3 \ + --hash=sha256:09d4bfd84686cd36fd11fd45a0732c7628308d094b14d28ea74a81db0bce2ed3 \ + --hash=sha256:1e623e0cf99a0ac114f091b3083a1848dbc64b0b99e181473b5a4a68d4f6f821 \ + --hash=sha256:2288d76e4d4aa7ef3fe7a73c1c470b66ea68e7969930e746a8cd8eca6ef2a2ea \ + --hash=sha256:2296356b5c9605b73ed6a52660b538787094dae13786ba53080595d52df13a98 \ + --hash=sha256:2a1e601ee31ef30a9e2c601d0867e236ac54c922d32ed9f727b70dd5d82600d5 \ + --hash=sha256:2be88c081e33f20630ac3343d8ad9f1125f32987968e9c8c75c051c9800896e8 \ + --hash=sha256:33d40954199bddbb6a78f8f6f2b2082660f381cd2583ec860a6c2fa7c8400c08 \ + --hash=sha256:40e1cbf69d6741b40f750f3cccc64326f927ac6145a9914d33879e586002350c \ + --hash=sha256:46a057329938b08e5f0e12ea3d7aed3ecb20a0c34c4a324ef34e00cecdb88a12 \ + --hash=sha256:4864f99aac207e3e45c5e26c6cbb0ad82917869abc2f156283be86c05286485c \ + --hash=sha256:4c44e1a765b31e175c391f22e8fc73b2a2ece0e5e6ff042743d8109b5d2eff9f \ + --hash=sha256:4cb283f630624ebb16c834e5ac3d7880831b07cbe76cb08ab7a271eeaeb8943e \ + --hash=sha256:5008964885e8d23313c8e5ea0d44433be9bfd7e24482574e8cc43c02c02fc796 \ + --hash=sha256:50a9f075eeda5097aa9a182bb3877fe1272875e45370368ac0ee16ab9e22d019 \ + --hash=sha256:51630c92591d6d3fe488a7c706bd30a61594d144bac7dee20c8e1ce78294f474 \ + --hash=sha256:5cc928cfe6c360c1df636cf7991ab96f059666ac7b40b75a769410cc6217df9c \ + --hash=sha256:61f7203e2767800edee7a1e1040aaaf124a35ce0c7fe0883965c6b762defe598 \ + --hash=sha256:66233ccd2a9371158d96e05d082043d47dadb18cbb294dc5accfdafc2e6b02a7 \ + --hash=sha256:70fcac7b94f4c904152809a050164650ac81c08e62c27aa9f156ac518029ebbe \ + --hash=sha256:714242ad0afa63a2e6dabd522ae22e1d76e07060b5af2ddda5474ba4f14c2c94 \ + --hash=sha256:782f4f8662a2157c4190d0f99eaaebc602899e84fb1e562a944e5025929e351c \ + --hash=sha256:7fc2b4edb938c8faa4b3c3ea90ca0dd89b7565a049e8e4e11b77e60e4ed2cc05 \ + --hash=sha256:881d058c5ccbea7cc2c92085a11947b572498a27ef37d3eef4887f499054dca8 \ + --hash=sha256:89dde0ac72a858a44a2feb8e43dc68c0c66f7857a23f806e81e1b7cc7044c9cf \ + --hash=sha256:8cdbcbd687e576d48f7886157c95052825ca9948c0ed2afdc0134305067be88b \ + --hash=sha256:8d6192c37a30a115f4663592861f50e130caed33efc4eec24d92ec881c92d771 \ + --hash=sha256:96a41817d2c763b1d0b32675abeb9179aa2371c72aefdf74b2d2b99a1b92417b \ + --hash=sha256:9bdbb7624d65dc0ed2ed8e954e79ab1724526f09b1efa88dcd9a1815bf28be5f \ + --hash=sha256:9bf88004fe086c786dc56ef8dd6cb49c026833fdd6f42cb853008bce3f907148 \ + --hash=sha256:a08920fa1a97d4b8ee5db2f31195de4a9def1a91bc003544eb3c9e6b8977960a \ + --hash=sha256:a2f5a1f1080ccdc7cbaf1171b2cf384d852496fe81ddedeb882d42b85727f610 \ + --hash=sha256:b04202453941a63b36876a7172b45366dc0cde10d5fd7855c0f4a4e673c0357a \ + --hash=sha256:b38b3de8cff5bc70f8f9c615f51b48eff7313fc9aca354f09f81b73036e7ddfa \ + --hash=sha256:b52d00d1793d290c81ad6a27058f5224a7d5f527867e5b580742e1bd211afeee \ + --hash=sha256:b74ae837368cfffeb3f6b498688a123e6b960951be4dec0e869de77e7fa0439e \ + --hash=sha256:be48496b0e00460717225e7680de57c38be1d8629dc09dadcd1b3389d70d942b \ + --hash=sha256:c0e3155fc5335ec7b3b70f15230234e529ca3607b20a562b6c75fb1b1218874c \ + --hash=sha256:c2392f5b5d84b71d853918687d806c1aa4308109e5ca158a16e16a6be71041eb \ + --hash=sha256:c72956972e4b508dd39fdc7646637a791a9665b478e768ffa5f4fe42123d5de1 \ + --hash=sha256:dc80c9c6b608bf98066a038e0172013a49cfa9a08d53335aefefda2c64fc68f4 \ + --hash=sha256:e416c8baf925b5a1aff31f7f5aecc0060b25d50cce3a5a7255dc5cf2f1d4e5eb \ + --hash=sha256:f8da84bbc61a4e92af54dc96344f328e5822d574f767e9b08e1602bb5ddc254a \ + --hash=sha256:f900ed4ad7a0f1f05d35f955e0943944d5a75f607a836958c6b8ab2a81730ef2 \ + --hash=sha256:fd6c6c29717724acf9fc1847c4515d57e4dc12762452457b9cb37461f30a81bb # via # -r requirements.in # grpcio-tools -grpcio-tools==1.54.0 \ - --hash=sha256:072279c394086ec07af8736f94aa6be671000af143df5f3040500e8cd5de484d \ - --hash=sha256:12c3091ef09df47c20d698dc74a373c1674c67ac5ac12ef24433002165129556 \ - --hash=sha256:1e15aa21a68cdb66b38db645924a09196cbbf4ad3ce1cf9dbcdf7b245292e380 \ - --hash=sha256:22b826b1d0d98a3bb80094928e21c48e23d022168e91d466a21389d1cda87177 \ - --hash=sha256:22eecfbb85da93405e056f073e339c34f881a3b410800902ae533e475688b1f1 \ - --hash=sha256:23474a820bdf8126af93724c7f6b23b16d9f3ad4c4bdcb40936ecb7f2be6bcc7 \ - --hash=sha256:235ea036d56ab9b6e4f235c554cf4680c42bf79352046e8a25829f597ca0d56b \ - --hash=sha256:24158db60cdda7c9eb803a9ed033a23b4420fd13ba4c54392d6b396589afb969 \ - --hash=sha256:292c0838d4a52ca53a941d197de57efc7f9df548d0619e14680c3e1ebeb33752 \ - --hash=sha256:42e8509ae024e8fb4eeab5ed9518c8e88d7f46a56015ae940796fe708d781e9f \ - --hash=sha256:43339122c891110e9ed020974c0fe2bd773d96c184203a62d0973f4cfb007f80 \ - --hash=sha256:436335857731e8f0bc5d345804d7dcd1bc677a1f793ea2b22e0627870ea31df6 \ - --hash=sha256:473dbd7005d059d1dc84490eec35699029d629048e7fdd76e31c14fc58a603e5 \ - --hash=sha256:4b32eaf44939c3dfa127bcb6bcf685188a05c1e0f54a7c8ed10770cbe177ca15 \ - --hash=sha256:52b6f78f5601cd080e5456ec24fd960328f9962e9aa4f9f556ec6fd5e4a391bf \ - --hash=sha256:536c108ee8fa46b1ca8d03f4033cdf282741978f3f83dcc8b2a9e7a2b8197271 \ - --hash=sha256:53dc65731a46bc065ad8d717d75d8659763126d1626eacc6f225c0b779ccc350 \ - --hash=sha256:58d2cbbf4e40cecd09f4fa6a2c15f7a95ec90f492b4e31469d8c9db981426106 \ - --hash=sha256:66738bc9d7b5db1a86832fa930cdd8faaf0c68bf70751d930c686d6bb309fad0 \ - --hash=sha256:6dd57137706257c39255020fb04322eef6c4043a11fc86042ae22b05167603ca \ - --hash=sha256:6ff318cd73b5cbdb16616d093c1c7b340d8d13b5bada2df7fbd873e899eab162 \ - --hash=sha256:7615b824132fcea4769d0cd99bdeffe25b096b48215c36675f37f7b57dc09635 \ - --hash=sha256:7ab23a94a8a7aa97ad8f46ba39ade4462c19833aa4815780db2cec647df7efef \ - --hash=sha256:7abdefb364de75d13d929093e571b84c4ea2580dbf5e4b6f5099ac9fa1a93f29 \ - --hash=sha256:86169076170f835139db7ec8362124cabe8f01050c2a717500a0fcdeb71dc537 \ - --hash=sha256:8a91f973b44fc90c32b7df79daec78907f0721e65d85fdff9133e05fee1f6e56 \ - --hash=sha256:8d2d5522d2222879c161479b5c91661cf788a3178f0b7532fd4c3e918e3390c4 \ - --hash=sha256:9d9a8d35a898f5c160fdb1ba0f58ada80e8e291cdffd6025f6043228d0832b2f \ - --hash=sha256:9fa42e3851c3c795228e2699c68ff7db14789991da4b53d3634360a361e0e420 \ - --hash=sha256:a61f367a153e3e604b1b59cec2f5e61255ffb6328d7348a53531e90ad98cc4fd \ - --hash=sha256:a66baf974aa00542ba4b6be499266ef8d63aa1abe7c78abab168bdf2fcd7f216 \ - --hash=sha256:ab45bda18b4931b08d89e490fd920bf654109d1c2e80b037fe9b60ab30fb4d9d \ - --hash=sha256:adc93f29353377182556e37120985f69f0fccc79ab65a0f24c1cc53075344dd7 \ - --hash=sha256:b1a3aad8a85d408ec925ee63668c2e4f075c96f67fcc3c43366a28f60e25873c \ - --hash=sha256:b944c1654b5f0710e782c0e3462d7acca900cbf2fb874d62e51a5279293151a9 \ - --hash=sha256:c03ceec6441365ec56c7fc6f38386917814ad02144dc1ecf03e13cee45f26389 \ - --hash=sha256:cca182c9a33dbf52c61a13fd1b98af535e7563bcd9e02e60e9de742d385f2ffb \ - --hash=sha256:ce20c3523a4bdb85531e762f45f7c5f5a915e5a8f25db3d58ebb6a8a37290a75 \ - --hash=sha256:d58528d6ea1835c0d4c34b60152dd29b3c9ab02ad0a8e734de240d54786d8b1e \ - --hash=sha256:d8ed6146e08b82caff467ac2af6c41f092212e3b5795687b76dd9219ac8b7186 \ - --hash=sha256:de07f7441e2f318a68631626d4d99c22244901ad7451a3e9fa08e5601a54772f \ - --hash=sha256:df79acbf59997018e131713b716a2fddb5556e1840e9fb9de4ca73bf2059590e \ - --hash=sha256:e0ba9344855b0c617d6df6bfb46ffbc574815c7152afa023e443cce4d5402f55 \ - --hash=sha256:f5aaa7e90e0947aec936e02fb85f312ca03c2258f3ee78403dcc27389fc62838 \ - --hash=sha256:fb6774907a708079afb3deecdd8e053d2b2b066c279a5d8c4957687db3f809ce +grpcio-tools==1.54.2 \ + --hash=sha256:0239b929eb8b3b30b2397eef3b9abb245087754d77c3721e3be43c44796de87d \ + --hash=sha256:0ab1b323905d449298523db5d34fa5bf5fffd645bd872b25598e2f8a01f0ea39 \ + --hash=sha256:0de05c7698c655e9a240dc34ae91d6017b93143ac89e5b20046d7ca3bd09c27c \ + --hash=sha256:0f952c8a5c47e9204fe8959f7e9add149e660f6579d67cf65024c32736d34caf \ + --hash=sha256:10dd41862f579d185c60f629b5ee89103e216f63b576079d258d974d980bad87 \ + --hash=sha256:11939c9a8a39bd4815c7e88cb2fee48e1948775b59dbb06de8fcae5991e84f9e \ + --hash=sha256:129de5579f95d6a55dde185f188b4cbe19d1e2f1471425431d9930c31d300d70 \ + --hash=sha256:1b8ee3099c51ce987fa8a08e6b93fc342b10228415dd96b5c0caa0387f636a6f \ + --hash=sha256:21b1467e31e44429d2a78b50135c9cdbd4b8f6d3b5cd548bc98985d3bdc352d0 \ + --hash=sha256:21b9d2dee80f3f77e4097252e7f0db89772335a7300b72ab3d2e5c280872b1db \ + --hash=sha256:27671c68c7e0e3c5ff9967f5500799f65a04e7b153b8ce10243c87c43199039d \ + --hash=sha256:2b96f5f17d3156058be247fd25b062b4768138665694c00b056659618b8fb418 \ + --hash=sha256:30a49b8b168aced2a4ff40959e6c4383ad6cfd7a20839a47a215e9837eb722dc \ + --hash=sha256:3237149beec39e897fd62cef4aa1e1cd9422d7a95661d24bd0a79200b167e730 \ + --hash=sha256:37393ef90674964175923afe3859fc5a208e1ece565f642b4f76a8c0224a0993 \ + --hash=sha256:39fd530cfdf58dc05125775cc233b05554d553d27478f14ae5fd8a6306f0cb28 \ + --hash=sha256:3bb9ec4aea0f2b3006fb002fa59e5c10f92b48fc374619fbffd14d2b0e388c3e \ + --hash=sha256:49c2846dcc4803476e839d8bd4db8845e928f19130e0ea86121f2d1f43d2b452 \ + --hash=sha256:4abfc1892380abe6cef381eab86f9350cbd703bfe5d834095aa66fd91c886b6d \ + --hash=sha256:4f285f8ef3de422717a36bd372239ae778b8cc112ce780ca3c7fe266dadc49fb \ + --hash=sha256:503ef1351c62fb1d6747eaf74932b609d8fdd4345b3591ef910adef8fa9969d0 \ + --hash=sha256:5ef30c2dbc63c1e0a462423ca4f95001814d26ef4fe66208e53fcf220ea3b717 \ + --hash=sha256:6037f123905dc0141f7c8383ca616ef0195e79cd3b4d82faaee789d4045e891b \ + --hash=sha256:72d15de4c4b6a764a76c4ae69d99c35f7a0751223688c3f7e62dfa95eb4f61be \ + --hash=sha256:7b24fbab9e7598518ce4549e066df00aab79c2bf9bedcdde23fb5ef6a3cf532f \ + --hash=sha256:7baa210c20f71a242d9ae0e02734628f6948e8bee3bf538647894af427d28800 \ + --hash=sha256:7d7e6e8d62967b3f037f952620cb7381cc39a4bd31790c75fcfba56cc975d70b \ + --hash=sha256:7f4624ef2e76a3a5313c4e61a81be38bcc16b59a68a85d30758b84cd2102b161 \ + --hash=sha256:8742122782953d2fd038f0a199f047a24e941cc9718b1aac90876dbdb7167739 \ + --hash=sha256:8e4531267736d88fde1022b36dd42ed8163e3575bcbd12bfed96662872aa93fe \ + --hash=sha256:8e4c5a48f7b2e8798ce381498ee7b9a83c65b87ae66ee5022387394e5eb51771 \ + --hash=sha256:9acf443dcf6f68fbea3b7fb519e1716e014db1a561939f5aecc4abda74e4015d \ + --hash=sha256:a0b7049814442f918b522d66b1d015286afbeb9e6d141af54bbfafe31710a3c8 \ + --hash=sha256:a3ce0b98fb581c471424d2cda45120f57658ed97677c6fec4d6decf5d7c1b976 \ + --hash=sha256:b80585e06c4f0082327eb5c9ad96fbdb2b0e7c14971ea5099fe78c22f4608451 \ + --hash=sha256:b82ca472db9c914c44e39a41e9e8bd3ed724523dd7aff5ce37592b8d16920ed9 \ + --hash=sha256:c4128c01cd6f5ea8f7c2db405dbfd8582cd967d36e6fa0952565436633b0e591 \ + --hash=sha256:d512de051342a576bb89777476d13c5266d9334cf4badb6468aed9dc8f5bdec1 \ + --hash=sha256:df079479fb1b9e488334312e35ebbf30cbf5ecad6c56599f1a961800b33ab7c1 \ + --hash=sha256:e11c2c2aee53f340992e8e4d6a59172cbbbd0193f1351de98c4f810a5041d5ca \ + --hash=sha256:e3d0e5188ff8dbaddac2ee44731d36f09c4eccd3eac7328e547862c44f75cacd \ + --hash=sha256:e543f457935ba7b763b121f1bf893974393b4d30065042f947f85a8d81081b80 \ + --hash=sha256:e5c7292dd899ad8fa09a2be96719648cee37b17909fe8c12007e3bff58ebee61 \ + --hash=sha256:f39d8e8806b8857fb473ca6a9c7bd800b0673dfdb7283ff569af0345a222f32c \ + --hash=sha256:f6787d07fdab31a32c433c1ba34883dea6559d8a3fbe08fb93d834ca34136b71 # via -r requirements.in idna==3.2 \ --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \ From 43d6954a8b2a40aecc4903d1b6125593626f536b Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 15 May 2023 11:18:45 +0100 Subject: [PATCH 224/740] mobile: Fix kotlin lint problem (#27399) Signed-off-by: Ryan Northey --- .../kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt index 03f312027177..15f4f68b225f 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt @@ -35,7 +35,8 @@ open class StreamPrototype(private val engine: EnvoyEngine) { } /** - * Sets min delivery: data will be buffered in the C++ layer until the min delivery length or end stream is read. + * Sets min delivery: data will be buffered in the C++ layer until the min + * delivery length or end stream is read. * * @param value set the minimum delivery size fo for this stream * @return This stream, for chaining syntax. From 06f5991bb171b1f79c8488f2e0ef49708439ae62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Neum=C3=BCller?= Date: Mon, 15 May 2023 13:51:19 +0200 Subject: [PATCH 225/740] Opentracing dynamic: Commit generated files to avoid cmake. (#27400) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian Neumüller --- bazel/io_opentracing_cpp.patch | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/bazel/io_opentracing_cpp.patch b/bazel/io_opentracing_cpp.patch index a0760ee1e86c..498a056fa7ef 100644 --- a/bazel/io_opentracing_cpp.patch +++ b/bazel/io_opentracing_cpp.patch @@ -57,3 +57,76 @@ index 1721fb3..3873b3a 100644 endif() # ============================================================================== +diff --git a/include/opentracing/config.h b/include/opentracing/config.h +new file mode 100755 +index 0000000..cae3e61 +--- /dev/null ++++ b/include/opentracing/config.h +@@ -0,0 +1,3 @@ ++#pragma once ++ ++#define OPENTRACING_BUILD_DYNAMIC_LOADING +diff --git a/include/opentracing/version.h b/include/opentracing/version.h +new file mode 100755 +index 0000000..25b9945 +--- /dev/null ++++ b/include/opentracing/version.h +@@ -0,0 +1,14 @@ ++#ifndef OPENTRACING_VERSION_H ++#define OPENTRACING_VERSION_H ++ ++#define OPENTRACING_VERSION "1.5.1" ++#define OPENTRACING_ABI_VERSION "2" ++ ++// clang-format off ++#define BEGIN_OPENTRACING_ABI_NAMESPACE \ ++ inline namespace v2 { ++#define END_OPENTRACING_ABI_NAMESPACE \ ++ } // namespace v2 ++// clang-format on ++ ++#endif // OPENTRACING_VERSION_H +diff --git a/BUILD.bazel b/BUILD.bazel +index c57dc9f..587bf0e 100644 +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -1,10 +1,7 @@ + cc_library( + name = "opentracing", + srcs = glob(["src/**/*.cpp"], exclude=["src/dynamic_load_unsupported.cpp", "src/dynamic_load_windows.cpp"]), +- hdrs = glob(["include/opentracing/**/*.h"]) + [ +- ":include/opentracing/config.h", +- ":include/opentracing/version.h", +- ], ++ hdrs = glob(["include/opentracing/**/*.h"]), + strip_include_prefix = "include", + visibility = ["//visibility:public"], + deps = [ +@@ -15,27 +12,3 @@ cc_library( + "-ldl", + ], + ) +- +-genrule( +- name = "generate_version_h", +- srcs = glob([ +- "*", +- "cmake/*", +- "src/**/*.cpp", +- ]), +- outs = [ +- "include/opentracing/config.h", +- "include/opentracing/version.h" +- ], +- cmd = """ +- TEMP_DIR=$$(mktemp -d) +- CONFIG_H_OUT=$${PWD}/$(location :include/opentracing/config.h) +- VERSION_H_OUT=$${PWD}/$(location :include/opentracing/version.h) +- OPENTRACING_ROOT=$$(dirname $${PWD}/$(location :CMakeLists.txt)) +- cd $$TEMP_DIR +- cmake -DBUILD_TESTING=OFF -DBUILD_MOCKTRACER=OFF -L $$OPENTRACING_ROOT +- mv include/opentracing/config.h $$CONFIG_H_OUT +- mv include/opentracing/version.h $$VERSION_H_OUT +- rm -rf $$TEMP_DIR +- """, +-) From 6aa3fa96240b79655652cce4918a7a14867a531b Mon Sep 17 00:00:00 2001 From: River <6375745+RiverPhillips@users.noreply.github.com> Date: Mon, 15 May 2023 15:41:03 +0100 Subject: [PATCH 226/740] http: Only append to xfh if it was not the last value appended (#26417) Signed-off-by: River phillips --- .../config/route/v3/route_components.proto | 3 ++- changelogs/current.yaml | 5 ++++ source/common/http/utility.cc | 23 +++++++++++++++++-- source/common/runtime/runtime_features.cc | 1 + test/common/http/utility_test.cc | 20 ++++++++++++++++ 5 files changed, 49 insertions(+), 3 deletions(-) diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 24533788e233..a38bdccac97b 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -1188,7 +1188,8 @@ message RouteAction { // :ref:`host_rewrite_header `, or // :ref:`host_rewrite_path_regex `) // causes the original value of the host header, if any, to be appended to the - // :ref:`config_http_conn_man_headers_x-forwarded-host` HTTP header. + // :ref:`config_http_conn_man_headers_x-forwarded-host` HTTP header if it is different to the last value appended. + // This can be disabled by setting the runtime guard `envoy_reloadable_features_append_xfh_idempotent` to false. bool append_x_forwarded_host = 38; // Specifies the upstream timeout for the route. If not specified, the default is 15s. This diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 1b5a30edaf87..52fa5a3c53b2 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -10,6 +10,11 @@ behavior_changes: change: | Moved xDS code extensions. If you use the xDS and override extensions_build_config.bzl you will need to include the new config_subscriptions explicitly. +- area: http + change: | + When ``append_x_forwarded_host`` is enabled for a given route action it is now only appended iff it is different from the last + value in the list. This resolves issues where a retry caused the same value to be appended multiple times. This + behavioral change can be temporarily reverted by setting runtime guard ``envoy_reloadable_features_append_xfh_idempotent`` to false. minor_behavior_changes: # *Changes that may cause incompatibilities for some users, but should not for most* diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc index d9bee4ffa8f0..b805297d5e14 100644 --- a/source/common/http/utility.cc +++ b/source/common/http/utility.cc @@ -463,9 +463,28 @@ void Utility::appendVia(RequestOrResponseHeaderMap& headers, const std::string& void Utility::updateAuthority(RequestHeaderMap& headers, absl::string_view hostname, const bool append_xfh) { - if (append_xfh && !headers.getHostValue().empty()) { - headers.appendForwardedHost(headers.getHostValue(), ","); + const auto host = headers.getHostValue(); + + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.append_xfh_idempotent")) { + // Only append to x-forwarded-host if the value was not the last value appended. + const auto xfh = headers.getForwardedHostValue(); + + if (append_xfh && !host.empty()) { + if (!xfh.empty()) { + const auto xfh_split = StringUtil::splitToken(xfh, ","); + if (!xfh_split.empty() && xfh_split.back() != host) { + headers.appendForwardedHost(host, ","); + } + } else { + headers.appendForwardedHost(host, ","); + } + } + } else { + if (append_xfh && !host.empty()) { + headers.appendForwardedHost(host, ","); + } } + headers.setHost(hostname); } diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 2eba8476723e..55c7e468324a 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -31,6 +31,7 @@ // problem of the bugs being found after the old code path has been removed. RUNTIME_GUARD(envoy_reloadable_features_allow_compact_maglev); RUNTIME_GUARD(envoy_reloadable_features_append_query_parameters_path_rewriter); +RUNTIME_GUARD(envoy_reloadable_features_append_xfh_idempotent); RUNTIME_GUARD(envoy_reloadable_features_conn_pool_delete_when_idle); RUNTIME_GUARD(envoy_reloadable_features_delta_xds_subscription_state_tracking_fix); RUNTIME_GUARD(envoy_reloadable_features_do_not_count_mapped_pages_as_free); diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index aeecabe7f94e..43f8bb41e6af 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -490,6 +490,26 @@ TEST(HttpUtility, updateAuthority) { EXPECT_EQ("dns.name", headers.get_(":authority")); EXPECT_EQ("host.com", headers.get_("x-forwarded-host")); } + + // Test that we only append to x-forwarded-host if it is not already present. + { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({{"envoy.reloadable_features.append_xfh_idempotent", "true"}}); + TestRequestHeaderMapImpl headers{{":authority", "dns.name"}, + {"x-forwarded-host", "host.com,dns.name"}}; + Utility::updateAuthority(headers, "newhost.com", true); + EXPECT_EQ("newhost.com", headers.get_(":authority")); + EXPECT_EQ("host.com,dns.name", headers.get_("x-forwarded-host")); + } + { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({{"envoy.reloadable_features.append_xfh_idempotent", "false"}}); + TestRequestHeaderMapImpl headers{{":authority", "dns.name"}, + {"x-forwarded-host", "host.com,dns.name"}}; + Utility::updateAuthority(headers, "newhost.com", true); + EXPECT_EQ("newhost.com", headers.get_(":authority")); + EXPECT_EQ("host.com,dns.name,dns.name", headers.get_("x-forwarded-host")); + } } TEST(HttpUtility, createSslRedirectPath) { From 3c8481889cd1ed804121a0eccabc2286b58c9aaf Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 15 May 2023 10:50:17 -0400 Subject: [PATCH 227/740] Add more CPU for integration tests (#27379) * Add more CPU for integration tests Signed-off-by: Yan Avlasov --- .../generic_proxy/filters/network/test/BUILD | 3 ++ .../filters/http/file_system_buffer/BUILD | 5 +++- test/extensions/filters/http/on_demand/BUILD | 3 ++ test/extensions/filters/network/echo/BUILD | 1 + test/integration/BUILD | 30 +++++++++++++++++++ 5 files changed, 41 insertions(+), 1 deletion(-) diff --git a/contrib/generic_proxy/filters/network/test/BUILD b/contrib/generic_proxy/filters/network/test/BUILD index 08a4ed870bd2..afd72e504d0c 100644 --- a/contrib/generic_proxy/filters/network/test/BUILD +++ b/contrib/generic_proxy/filters/network/test/BUILD @@ -83,6 +83,9 @@ envoy_cc_test( srcs = [ "integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":fake_codec_lib", "//contrib/generic_proxy/filters/network/source:config", diff --git a/test/extensions/filters/http/file_system_buffer/BUILD b/test/extensions/filters/http/file_system_buffer/BUILD index 68c869da8890..5c46acb03b8d 100644 --- a/test/extensions/filters/http/file_system_buffer/BUILD +++ b/test/extensions/filters/http/file_system_buffer/BUILD @@ -48,7 +48,10 @@ envoy_extension_cc_test( ], extension_names = ["envoy.filters.http.file_system_buffer"], shard_count = 2, - tags = ["skip_on_windows"], + tags = [ + "cpu:3", + "skip_on_windows", + ], deps = [ "//source/extensions/filters/http/file_system_buffer:config", "//test/integration:http_protocol_integration_lib", diff --git a/test/extensions/filters/http/on_demand/BUILD b/test/extensions/filters/http/on_demand/BUILD index c9288218f6f2..4de58cb23e4e 100644 --- a/test/extensions/filters/http/on_demand/BUILD +++ b/test/extensions/filters/http/on_demand/BUILD @@ -32,6 +32,9 @@ envoy_extension_cc_test( size = "large", srcs = ["on_demand_integration_test.cc"], extension_names = ["envoy.filters.http.on_demand"], + tags = [ + "cpu:3", + ], deps = [ "//source/extensions/filters/http/on_demand:config", "//source/extensions/filters/http/on_demand:on_demand_update_lib", diff --git a/test/extensions/filters/network/echo/BUILD b/test/extensions/filters/network/echo/BUILD index d84aeebaac91..28eaa8397261 100644 --- a/test/extensions/filters/network/echo/BUILD +++ b/test/extensions/filters/network/echo/BUILD @@ -17,6 +17,7 @@ envoy_cc_test( # Uncomment this line to run this test repeatedly in exclusive mode if not using docker-sandbox, # or RBE, see comments in AddRemoveListener. # "exclusive", + "cpu:2", ], deps = [ "//source/extensions/filters/network/echo:config", diff --git a/test/integration/BUILD b/test/integration/BUILD index acc5558902c6..fbcbb3b2922b 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -82,6 +82,9 @@ envoy_cc_test( name = "alpn_integration_test", size = "large", srcs = ["alpn_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [":http_integration_lib"], ) @@ -89,6 +92,9 @@ envoy_cc_test( name = "api_listener_integration_test", size = "large", srcs = ["api_listener_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//test/mocks/http:stream_encoder_mock", @@ -125,6 +131,9 @@ envoy_cc_test( "//test/config/integration/certs", ], shard_count = 4, + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/common/config:protobuf_link_hacks", @@ -144,6 +153,9 @@ envoy_cc_test( name = "eds_integration_test", size = "large", srcs = ["eds_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/common/upstream:load_balancer_lib", @@ -163,6 +175,9 @@ envoy_cc_test( name = "leds_integration_test", size = "large", srcs = ["leds_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//test/config:utility_lib", @@ -186,6 +201,9 @@ envoy_cc_test( srcs = [ "filter_manager_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":filter_manager_integration_proto_cc_proto", ":http_integration_lib", @@ -270,6 +288,9 @@ envoy_cc_test( data = [ "//test/config/integration/certs", ], + tags = [ + "cpu:3", + ], deps = [ ":vhds_lib", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", @@ -300,6 +321,9 @@ envoy_cc_test( "drain_close_integration_test.cc", ]), shard_count = 2, + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", "//test/test_common:utility_lib", @@ -359,6 +383,9 @@ envoy_cc_test( data = [ "//test/config/integration/certs", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//test/test_common:utility_lib", @@ -374,6 +401,9 @@ envoy_cc_test( srcs = [ "header_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":http_protocol_integration_lib", From 2ede9735ab136791f7b9b349d771f1ac9b588612 Mon Sep 17 00:00:00 2001 From: code Date: Mon, 15 May 2023 23:07:33 +0800 Subject: [PATCH 228/740] tracing: inject the tracing header before the upstream filter chain is started (#27269) * tracing: inject the tracing header before the upstream filter chain is started Signed-off-by: wbpcode * fix test Signed-off-by: wbpcode * update comment Signed-off-by: wbpcode * update comment Signed-off-by: wbpcode --------- Signed-off-by: wbpcode --- source/common/router/upstream_request.cc | 21 +++++++++-------- test/common/router/router_2_test.cc | 29 +++++++++++++++--------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index c8e764e6b3cf..2d5f67f8a39d 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -106,6 +106,18 @@ UpstreamRequest::UpstreamRequest(RouterFilterInterface& parent, } } } + + // The router checks that the connection pool is non-null before creating the upstream request. + auto upstream_host = conn_pool_->host(); + if (span_ != nullptr) { + span_->injectContext(*parent_.downstreamHeaders(), upstream_host); + } else { + // No independent child span for current upstream request then inject the parent span's tracing + // context into the request headers. + // The injectContext() of the parent span may be called repeatedly when the request is retried. + parent_.callbacks()->activeSpan().injectContext(*parent_.downstreamHeaders(), upstream_host); + } + stream_info_.setUpstreamInfo(std::make_shared()); stream_info_.route_ = parent.callbacks()->route(); parent_.callbacks()->streamInfo().setUpstreamInfo(stream_info_.upstreamInfo()); @@ -633,15 +645,6 @@ void UpstreamRequest::onPoolReady(std::unique_ptr&& upstream, route_entry->appendXfh()); } - if (span_ != nullptr) { - span_->injectContext(*parent_.downstreamHeaders(), host); - } else { - // No independent child span for current upstream request then inject the parent span's tracing - // context into the request headers. - // The injectContext() of the parent span may be called repeatedly when the request is retried. - parent_.callbacks()->activeSpan().injectContext(*parent_.downstreamHeaders(), host); - } - stream_info_.setRequestHeaders(*parent_.downstreamHeaders()); if (parent_.config().flush_upstream_log_on_upstream_stream_) { diff --git a/test/common/router/router_2_test.cc b/test/common/router/router_2_test.cc index be34f0c13951..4186a2f13607 100644 --- a/test/common/router/router_2_test.cc +++ b/test/common/router/router_2_test.cc @@ -411,14 +411,13 @@ TEST_F(RouterTestChildSpan, BasicFlow) { NiceMock encoder; Http::ResponseDecoder* response_decoder = nullptr; - Tracing::MockSpan* child_span{new Tracing::MockSpan()}; + EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) .WillOnce( Invoke([&](Http::ResponseDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks, const Http::ConnectionPool::Instance::StreamOptions&) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - EXPECT_CALL(*child_span, injectContext(_, _)); callbacks.onPoolReady(encoder, cm_.thread_local_cluster_.conn_pool_.host_, upstream_stream_info_, Http::Protocol::Http10); return nullptr; @@ -426,6 +425,9 @@ TEST_F(RouterTestChildSpan, BasicFlow) { Http::TestRequestHeaderMapImpl headers; HttpTestUtility::addDefaultHeaders(headers); + + Tracing::MockSpan* child_span{new Tracing::MockSpan()}; + EXPECT_CALL(*child_span, injectContext(_, _)); EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router observability_name egress", _)) .WillOnce(Return(child_span)); EXPECT_CALL(callbacks_, tracingConfig()).Times(2); @@ -460,14 +462,13 @@ TEST_F(RouterTestChildSpan, ResetFlow) { NiceMock encoder; Http::ResponseDecoder* response_decoder = nullptr; - Tracing::MockSpan* child_span{new Tracing::MockSpan()}; + EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) .WillOnce( Invoke([&](Http::ResponseDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks, const Http::ConnectionPool::Instance::StreamOptions&) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - EXPECT_CALL(*child_span, injectContext(_, _)); callbacks.onPoolReady(encoder, cm_.thread_local_cluster_.conn_pool_.host_, upstream_stream_info_, Http::Protocol::Http10); return nullptr; @@ -475,6 +476,9 @@ TEST_F(RouterTestChildSpan, ResetFlow) { Http::TestRequestHeaderMapImpl headers; HttpTestUtility::addDefaultHeaders(headers); + + Tracing::MockSpan* child_span{new Tracing::MockSpan()}; + EXPECT_CALL(*child_span, injectContext(_, _)); EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router observability_name egress", _)) .WillOnce(Return(child_span)); EXPECT_CALL(callbacks_, tracingConfig()).Times(2); @@ -515,12 +519,11 @@ TEST_F(RouterTestChildSpan, CancelFlow) { EXPECT_CALL(callbacks_.dispatcher_, createTimer_(_)).Times(0); NiceMock encoder; - Tracing::MockSpan* child_span{new Tracing::MockSpan()}; + EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) .WillOnce(Invoke([&](Http::StreamDecoder&, Http::ConnectionPool::Callbacks& callbacks, const Http::ConnectionPool::Instance::StreamOptions&) -> Http::ConnectionPool::Cancellable* { - EXPECT_CALL(*child_span, injectContext(_, _)); callbacks.onPoolReady(encoder, cm_.thread_local_cluster_.conn_pool_.host_, upstream_stream_info_, Http::Protocol::Http10); return nullptr; @@ -528,6 +531,8 @@ TEST_F(RouterTestChildSpan, CancelFlow) { Http::TestRequestHeaderMapImpl headers; HttpTestUtility::addDefaultHeaders(headers); + Tracing::MockSpan* child_span{new Tracing::MockSpan()}; + EXPECT_CALL(*child_span, injectContext(_, _)); EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router observability_name egress", _)) .WillOnce(Return(child_span)); EXPECT_CALL(callbacks_, tracingConfig()).Times(2); @@ -560,14 +565,13 @@ TEST_F(RouterTestChildSpan, CancelFlow) { TEST_F(RouterTestChildSpan, ResetRetryFlow) { NiceMock encoder1; Http::ResponseDecoder* response_decoder = nullptr; - Tracing::MockSpan* child_span_1{new Tracing::MockSpan()}; + EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) .WillOnce( Invoke([&](Http::ResponseDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks, const Http::ConnectionPool::Instance::StreamOptions&) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - EXPECT_CALL(*child_span_1, injectContext(_, _)); callbacks.onPoolReady(encoder1, cm_.thread_local_cluster_.conn_pool_.host_, upstream_stream_info_, Http::Protocol::Http10); return nullptr; @@ -577,6 +581,9 @@ TEST_F(RouterTestChildSpan, ResetRetryFlow) { // Upstream responds back to envoy simulating an upstream reset. Http::TestRequestHeaderMapImpl headers{{"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}}; HttpTestUtility::addDefaultHeaders(headers); + + Tracing::MockSpan* child_span_1{new Tracing::MockSpan()}; + EXPECT_CALL(*child_span_1, injectContext(_, _)); EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router observability_name egress", _)) .WillOnce(Return(child_span_1)); EXPECT_CALL(callbacks_, tracingConfig()).Times(2); @@ -605,20 +612,20 @@ TEST_F(RouterTestChildSpan, ResetRetryFlow) { // We expect this reset to kick off a new request. NiceMock encoder2; - Tracing::MockSpan* child_span_2{new Tracing::MockSpan()}; EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) .WillOnce( Invoke([&](Http::ResponseDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks, const Http::ConnectionPool::Instance::StreamOptions&) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - EXPECT_CALL(*child_span_2, injectContext(_, _)); EXPECT_CALL(*router_->retry_state_, onHostAttempted(_)); callbacks.onPoolReady(encoder2, cm_.thread_local_cluster_.conn_pool_.host_, upstream_stream_info_, Http::Protocol::Http10); return nullptr; })); + Tracing::MockSpan* child_span_2{new Tracing::MockSpan()}; + EXPECT_CALL(*child_span_2, injectContext(_, _)); EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router observability_name egress", _)) .WillOnce(Return(child_span_2)); EXPECT_CALL(callbacks_, tracingConfig()).Times(2); @@ -665,7 +672,6 @@ TEST_F(RouterTestNoChildSpan, BasicFlow) { const Http::ConnectionPool::Instance::StreamOptions&) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - EXPECT_CALL(callbacks_.active_span_, injectContext(_, _)); callbacks.onPoolReady(encoder, cm_.thread_local_cluster_.conn_pool_.host_, upstream_stream_info_, Http::Protocol::Http10); return nullptr; @@ -673,6 +679,7 @@ TEST_F(RouterTestNoChildSpan, BasicFlow) { Http::TestRequestHeaderMapImpl headers; HttpTestUtility::addDefaultHeaders(headers); + EXPECT_CALL(callbacks_.active_span_, injectContext(_, _)); router_->decodeHeaders(headers, true); EXPECT_EQ(1U, callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); From 39b25c0bec2a9a6c3e8e776ca0effd742d68d82f Mon Sep 17 00:00:00 2001 From: code Date: Mon, 15 May 2023 23:08:04 +0800 Subject: [PATCH 229/740] api: new subset lb extension api (#27256) * api: new subset lb extension api Signed-off-by: wbpcode * fix proto format Signed-off-by: wbpcode * address comment Signed-off-by: wbpcode * add previous message type Signed-off-by: wbpcode --------- Signed-off-by: wbpcode --- api/BUILD | 1 + .../load_balancing_policies/subset/v3/BUILD | 12 + .../subset/v3/subset.proto | 220 ++++++++++++++++++ api/versioning/BUILD | 1 + 4 files changed, 234 insertions(+) create mode 100644 api/envoy/extensions/load_balancing_policies/subset/v3/BUILD create mode 100644 api/envoy/extensions/load_balancing_policies/subset/v3/subset.proto diff --git a/api/BUILD b/api/BUILD index a40ea1204b00..36f2dbe78e06 100644 --- a/api/BUILD +++ b/api/BUILD @@ -258,6 +258,7 @@ proto_library( "//envoy/extensions/load_balancing_policies/random/v3:pkg", "//envoy/extensions/load_balancing_policies/ring_hash/v3:pkg", "//envoy/extensions/load_balancing_policies/round_robin/v3:pkg", + "//envoy/extensions/load_balancing_policies/subset/v3:pkg", "//envoy/extensions/load_balancing_policies/wrr_locality/v3:pkg", "//envoy/extensions/matching/common_inputs/environment_variable/v3:pkg", "//envoy/extensions/matching/common_inputs/network/v3:pkg", diff --git a/api/envoy/extensions/load_balancing_policies/subset/v3/BUILD b/api/envoy/extensions/load_balancing_policies/subset/v3/BUILD new file mode 100644 index 000000000000..b49ae9078cfc --- /dev/null +++ b/api/envoy/extensions/load_balancing_policies/subset/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/cluster/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/load_balancing_policies/subset/v3/subset.proto b/api/envoy/extensions/load_balancing_policies/subset/v3/subset.proto new file mode 100644 index 000000000000..27610c053e13 --- /dev/null +++ b/api/envoy/extensions/load_balancing_policies/subset/v3/subset.proto @@ -0,0 +1,220 @@ +syntax = "proto3"; + +package envoy.extensions.load_balancing_policies.subset.v3; + +import "envoy/config/cluster/v3/cluster.proto"; + +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.load_balancing_policies.subset.v3"; +option java_outer_classname = "SubsetProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/subset/v3;subsetv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Subset Load Balancing Policy] +// [#extension: envoy.load_balancing_policies.subset] +// [#not-implemented-hide:] + +// Optionally divide the endpoints in this cluster into subsets defined by +// endpoint metadata and selected by route and weighted cluster metadata. +// [#next-free-field: 10] +message Subset { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.cluster.v3.LbSubsetConfig"; + + // If NO_FALLBACK is selected, a result + // equivalent to no healthy hosts is reported. If ANY_ENDPOINT is selected, + // any cluster endpoint may be returned (subject to policy, health checks, + // etc). If DEFAULT_SUBSET is selected, load balancing is performed over the + // endpoints matching the values from the default_subset field. + enum LbSubsetFallbackPolicy { + NO_FALLBACK = 0; + ANY_ENDPOINT = 1; + DEFAULT_SUBSET = 2; + } + + enum LbSubsetMetadataFallbackPolicy { + // No fallback. Route metadata will be used as-is. + METADATA_NO_FALLBACK = 0; + + // A special metadata key ``fallback_list`` will be used to provide variants of metadata to try. + // Value of ``fallback_list`` key has to be a list. Every list element has to be a struct - it will + // be merged with route metadata, overriding keys that appear in both places. + // ``fallback_list`` entries will be used in order until a host is found. + // + // ``fallback_list`` key itself is removed from metadata before subset load balancing is performed. + // + // Example: + // + // for metadata: + // + // .. code-block:: yaml + // + // version: 1.0 + // fallback_list: + // - version: 2.0 + // hardware: c64 + // - hardware: c32 + // - version: 3.0 + // + // at first, metadata: + // + // .. code-block:: json + // + // {"version": "2.0", "hardware": "c64"} + // + // will be used for load balancing. If no host is found, metadata: + // + // .. code-block:: json + // + // {"version": "1.0", "hardware": "c32"} + // + // is next to try. If it still results in no host, finally metadata: + // + // .. code-block:: json + // + // {"version": "3.0"} + // + // is used. + FALLBACK_LIST = 1; + } + + // Specifications for subsets. + message LbSubsetSelector { + // Allows to override top level fallback policy per selector. + enum LbSubsetSelectorFallbackPolicy { + // If NOT_DEFINED top level config fallback policy is used instead. + NOT_DEFINED = 0; + + // If NO_FALLBACK is selected, a result equivalent to no healthy hosts is reported. + NO_FALLBACK = 1; + + // If ANY_ENDPOINT is selected, any cluster endpoint may be returned + // (subject to policy, health checks, etc). + ANY_ENDPOINT = 2; + + // If DEFAULT_SUBSET is selected, load balancing is performed over the + // endpoints matching the values from the default_subset field. + DEFAULT_SUBSET = 3; + + // If KEYS_SUBSET is selected, subset selector matching is performed again with metadata + // keys reduced to + // :ref:`fallback_keys_subset`. + // It allows for a fallback to a different, less specific selector if some of the keys of + // the selector are considered optional. + KEYS_SUBSET = 4; + } + + // List of keys to match with the weighted cluster metadata. + repeated string keys = 1; + + // Selects a mode of operation in which each subset has only one host. This mode uses the same rules for + // choosing a host, but updating hosts is faster, especially for large numbers of hosts. + // + // If a match is found to a host, that host will be used regardless of priority levels. + // + // When this mode is enabled, configurations that contain more than one host with the same metadata value for the single key in ``keys`` + // will use only one of the hosts with the given key; no requests will be routed to the others. The cluster gauge + // :ref:`lb_subsets_single_host_per_subset_duplicate` indicates how many duplicates are + // present in the current configuration. + bool single_host_per_subset = 4; + + // The behavior used when no endpoint subset matches the selected route's + // metadata. + LbSubsetSelectorFallbackPolicy fallback_policy = 2 + [(validate.rules).enum = {defined_only: true}]; + + // Subset of + // :ref:`keys` used by + // :ref:`KEYS_SUBSET` + // fallback policy. + // It has to be a non empty list if KEYS_SUBSET fallback policy is selected. + // For any other fallback policy the parameter is not used and should not be set. + // Only values also present in + // :ref:`keys` are allowed, but + // ``fallback_keys_subset`` cannot be equal to ``keys``. + repeated string fallback_keys_subset = 3; + } + + // The behavior used when no endpoint subset matches the selected route's + // metadata. The value defaults to + // :ref:`NO_FALLBACK`. + LbSubsetFallbackPolicy fallback_policy = 1 [(validate.rules).enum = {defined_only: true}]; + + // Specifies the default subset of endpoints used during fallback if + // fallback_policy is + // :ref:`DEFAULT_SUBSET`. + // Each field in default_subset is + // compared to the matching LbEndpoint.Metadata under the ``envoy.lb`` + // namespace. It is valid for no hosts to match, in which case the behavior + // is the same as a fallback_policy of + // :ref:`NO_FALLBACK`. + google.protobuf.Struct default_subset = 2; + + // For each entry, LbEndpoint.Metadata's + // ``envoy.lb`` namespace is traversed and a subset is created for each unique + // combination of key and value. For example: + // + // .. code-block:: json + // + // { "subset_selectors": [ + // { "keys": [ "version" ] }, + // { "keys": [ "stage", "hardware_type" ] } + // ]} + // + // A subset is matched when the metadata from the selected route and + // weighted cluster contains the same keys and values as the subset's + // metadata. The same host may appear in multiple subsets. + repeated LbSubsetSelector subset_selectors = 3; + + // If true, routing to subsets will take into account the localities and locality weights of the + // endpoints when making the routing decision. + // + // There are some potential pitfalls associated with enabling this feature, as the resulting + // traffic split after applying both a subset match and locality weights might be undesirable. + // + // Consider for example a situation in which you have 50/50 split across two localities X/Y + // which have 100 hosts each without subsetting. If the subset LB results in X having only 1 + // host selected but Y having 100, then a lot more load is being dumped on the single host in X + // than originally anticipated in the load balancing assignment delivered via EDS. + bool locality_weight_aware = 4; + + // When used with locality_weight_aware, scales the weight of each locality by the ratio + // of hosts in the subset vs hosts in the original subset. This aims to even out the load + // going to an individual locality if said locality is disproportionately affected by the + // subset predicate. + bool scale_locality_weight = 5; + + // If true, when a fallback policy is configured and its corresponding subset fails to find + // a host this will cause any host to be selected instead. + // + // This is useful when using the default subset as the fallback policy, given the default + // subset might become empty. With this option enabled, if that happens the LB will attempt + // to select a host from the entire cluster. + bool panic_mode_any = 6; + + // If true, metadata specified for a metadata key will be matched against the corresponding + // endpoint metadata if the endpoint metadata matches the value exactly OR it is a list value + // and any of the elements in the list matches the criteria. + bool list_as_any = 7; + + // Fallback mechanism that allows to try different route metadata until a host is found. + // If load balancing process, including all its mechanisms (like + // :ref:`fallback_policy`) + // fails to select a host, this policy decides if and how the process is repeated using another metadata. + // + // The value defaults to + // :ref:`METADATA_NO_FALLBACK + // `. + LbSubsetMetadataFallbackPolicy metadata_fallback_policy = 8 + [(validate.rules).enum = {defined_only: true}]; + + // The child LB policy to create for endpoint-picking within the chosen subset. + config.cluster.v3.LoadBalancingPolicy subset_lb_policy = 9 + [(validate.rules).message = {required: true}]; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 82f27feda2b5..2f201794cf05 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -197,6 +197,7 @@ proto_library( "//envoy/extensions/load_balancing_policies/random/v3:pkg", "//envoy/extensions/load_balancing_policies/ring_hash/v3:pkg", "//envoy/extensions/load_balancing_policies/round_robin/v3:pkg", + "//envoy/extensions/load_balancing_policies/subset/v3:pkg", "//envoy/extensions/load_balancing_policies/wrr_locality/v3:pkg", "//envoy/extensions/matching/common_inputs/environment_variable/v3:pkg", "//envoy/extensions/matching/common_inputs/network/v3:pkg", From 491269b6f6ac1dc70711c47d7004f135d8711614 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Mon, 15 May 2023 08:31:47 -0700 Subject: [PATCH 230/740] http: Add HCM log on successful tunnel establishment. (#27292) * Add HCM log on successful tunnel establishment. Signed-off-by: Paul Gallagher --- api/envoy/data/accesslog/v3/accesslog.proto | 1 + .../v3/http_connection_manager.proto | 5 ++ changelogs/current.yaml | 6 ++ .../observability/access_log/usage.rst | 2 + source/common/http/conn_manager_config.h | 2 + source/common/http/conn_manager_impl.cc | 4 ++ .../network/http_connection_manager/config.cc | 2 + .../network/http_connection_manager/config.h | 4 ++ source/server/admin/admin.h | 1 + .../http/conn_manager_impl_fuzz_test.cc | 4 ++ test/common/http/conn_manager_impl_test.cc | 61 +++++++++++++++++++ .../common/http/conn_manager_impl_test_base.h | 4 ++ test/integration/protocol_integration_test.cc | 2 +- .../tcp_tunneling_integration_test.cc | 18 +++++- test/mocks/http/mocks.h | 1 + 15 files changed, 115 insertions(+), 2 deletions(-) diff --git a/api/envoy/data/accesslog/v3/accesslog.proto b/api/envoy/data/accesslog/v3/accesslog.proto index d069f4b771cf..56d9e7b8c4ca 100644 --- a/api/envoy/data/accesslog/v3/accesslog.proto +++ b/api/envoy/data/accesslog/v3/accesslog.proto @@ -43,6 +43,7 @@ enum AccessLogType { UpstreamPoolReady = 7; UpstreamPeriodic = 8; UpstreamEnd = 9; + DownstreamTunnelSuccessfullyEstablished = 10; } message TCPAccessLogEntry { diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 3daddff1393a..041db6ff60bf 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -372,6 +372,11 @@ message HttpConnectionManager { // This log record, if enabled, does not depend on periodic log records or request completion log. // Details related to upstream cluster, such as upstream host, will not be available for this log. bool flush_access_log_on_new_request = 2; + + // If true, the HCM will flush an access log when a tunnel is successfully established. For example, + // this could be when an upstream has successfully returned 101 Switching Protocols, or when the proxy + // has returned 200 to a CONNECT request. + bool flush_log_on_tunnel_successfully_established = 3; } reserved 27, 11; diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 52fa5a3c53b2..a2dd3015d18c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -163,6 +163,12 @@ new_features: added new field :ref:`connection_rate_limit ` to limit reconnection rate to redis server to avoid reconnection storm. +- area: access_log + change: | + added additional HCM access log option :ref:`flush_log_on_tunnel_successfully_established + `. + Enabling this option will write a log to all access loggers when HTTP tunnels (e.g. Websocket and CONNECT) + are successfully established. deprecated: - area: access_log diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index 491aecf0aa24..7b707de2bf53 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -1061,6 +1061,8 @@ The following command operators are supported: * TcpPeriodic - On any TCP Proxy filter periodic log record. * TcpConnectionEnd - When a TCP connection is ended on TCP Proxy filter. * DownstreamStart - When HTTP Connection Manager filter receives a new HTTP request. + * DownstreamTunnelSuccessfullyEstablished - When the HTTP Connection Manager sends response headers + indicating a successful HTTP tunnel. * DownstreamPeriodic - On any HTTP Connection Manager periodic log record. * DownstreamEnd - When an HTTP stream is ended on HTTP Connection Manager filter. * UpstreamPoolReady - When a new HTTP request is received by the HTTP Router filter. diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index f084b0d95836..52f8188c205c 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -222,6 +222,8 @@ class ConnectionManagerConfig { // headers have been evaluated, and before attempting to establish a connection with the upstream. virtual bool flushAccessLogOnNewRequest() PURE; + virtual bool flushAccessLogOnTunnelSuccessfullyEstablished() const PURE; + /** * Called to create a codec for the connection manager. This function will be called when the * first byte of application data is received. This is done to support handling of ALPN, protocol diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 205383045e8c..d658f0a7f4c7 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1715,6 +1715,10 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& heade chargeStats(headers); + if (state_.is_tunneling_ && + connection_manager_.config_.flushAccessLogOnTunnelSuccessfullyEstablished()) { + filter_manager_.log(AccessLog::AccessLogType::DownstreamTunnelSuccessfullyEstablished); + } ENVOY_STREAM_LOG(debug, "encoding headers via codec (end_stream={}):\n{}", *this, end_stream, headers); diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 6b5626c8de22..3fabdb579b3a 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -564,6 +564,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( flush_access_log_on_new_request_ = config.access_log_options().flush_access_log_on_new_request(); + flush_log_on_tunnel_successfully_established_ = + config.access_log_options().flush_log_on_tunnel_successfully_established(); if (config.access_log_options().has_access_log_flush_interval()) { access_log_flush_interval_ = std::chrono::milliseconds(DurationUtil::durationToMilliseconds( diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index cb65ca146721..da8126fb2aea 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -153,6 +153,9 @@ class HttpConnectionManagerConfig : Logger::Loggable, } const std::list& accessLogs() override { return access_logs_; } bool flushAccessLogOnNewRequest() override { return flush_access_log_on_new_request_; } + bool flushAccessLogOnTunnelSuccessfullyEstablished() const override { + return flush_log_on_tunnel_successfully_established_; + } const absl::optional& accessLogFlushInterval() override { return access_log_flush_interval_; } @@ -283,6 +286,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, std::list access_logs_; bool flush_access_log_on_new_request_; absl::optional access_log_flush_interval_; + bool flush_log_on_tunnel_successfully_established_{false}; const std::string stats_prefix_; Http::ConnectionManagerStats stats_; mutable Http::Http1::CodecStats::AtomicPtr http1_codec_stats_; diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 8706b6d47fca..240b970af7e0 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -124,6 +124,7 @@ class AdminImpl : public Admin, } const std::list& accessLogs() override { return access_logs_; } bool flushAccessLogOnNewRequest() override { return flush_access_log_on_new_request_; } + bool flushAccessLogOnTunnelSuccessfullyEstablished() const override { return false; } const absl::optional& accessLogFlushInterval() override { return null_access_log_flush_interval_; } diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 4bc08de034aa..e783f2a184df 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -129,6 +129,9 @@ class FuzzConfig : public ConnectionManagerConfig { const RequestIDExtensionSharedPtr& requestIDExtension() override { return request_id_extension_; } const std::list& accessLogs() override { return access_logs_; } bool flushAccessLogOnNewRequest() override { return flush_access_log_on_new_request_; } + bool flushAccessLogOnTunnelSuccessfullyEstablished() const override { + return flush_access_log_on_tunnel_successfully_established_; + } const absl::optional& accessLogFlushInterval() override { return access_log_flush_interval_; } @@ -245,6 +248,7 @@ class FuzzConfig : public ConnectionManagerConfig { RequestIDExtensionSharedPtr request_id_extension_; std::list access_logs_; bool flush_access_log_on_new_request_ = false; + bool flush_access_log_on_tunnel_successfully_established_ = false; absl::optional access_log_flush_interval_; MockServerConnection* codec_{}; MockStreamDecoderFilter* decoder_filter_{}; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index c83cbac73bda..d33ab4243850 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -2471,6 +2471,67 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogOnNewRequest) { conn_manager_->onData(fake_input, false); } +TEST_F(HttpConnectionManagerImplTest, TestAccessLogOnTunnelEstablished) { + setup(false, ""); + + std::shared_ptr filter(new NiceMock()); + std::shared_ptr handler(new NiceMock()); + + EXPECT_CALL(filter_factory_, createUpgradeFilterChain("CONNECT", _, _)) + .WillOnce(Invoke([&](absl::string_view, const FilterChainFactory::UpgradeMap*, + FilterChainManager& manager) -> bool { + FilterFactoryCb filter_factory = createDecoderFilterFactoryCb(filter); + FilterFactoryCb handler_factory = createLogHandlerFactoryCb(handler); + manager.applyFilterFactoryCb({}, filter_factory); + manager.applyFilterFactoryCb({}, handler_factory); + return true; + })); + + flush_log_on_tunnel_successfully_established_ = true; + + EXPECT_CALL(*handler, log(_, _, _, _, _)) + .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { + // First call to log() is made when a new HTTP tunnel has been established. + EXPECT_EQ(access_log_type, + AccessLog::AccessLogType::DownstreamTunnelSuccessfullyEstablished); + EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); + })) + .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, + const StreamInfo::StreamInfo& stream_info, + AccessLog::AccessLogType access_log_type) { + // Second call to log() is made when the request is completed, so it is expected + // that the response code is available and matches the response headers. + EXPECT_EQ(AccessLog::AccessLogType::DownstreamEnd, access_log_type); + EXPECT_TRUE(stream_info.responseCode()); + EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().remoteAddress()); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().directRemoteAddress()); + EXPECT_NE(nullptr, stream_info.route()); + })); + + EXPECT_CALL(*codec_, dispatch(_)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data) -> Http::Status { + decoder_ = &conn_manager_->newStream(response_encoder_); + + RequestHeaderMapPtr headers{ + new TestRequestHeaderMapImpl{{":method", "CONNECT"}, {":authority", "host"}}}; + decoder_->decodeHeaders(std::move(headers), true); + + filter->callbacks_->streamInfo().setResponseCodeDetails(""); + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + filter->callbacks_->encodeHeaders(std::move(response_headers), true, "details"); + + data.drain(4); + return Http::okStatus(); + })); + + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); +} + TEST_F(HttpConnectionManagerImplTest, TestPeriodicAccessLogging) { access_log_flush_interval_ = std::chrono::milliseconds(30 * 1000); stream_idle_timeout_ = std::chrono::milliseconds(0); diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index 1a00ab4548f9..eddca70cb681 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -62,6 +62,9 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { // Http::ConnectionManagerConfig const std::list& accessLogs() override { return access_logs_; } bool flushAccessLogOnNewRequest() override { return flush_access_log_on_new_request_; } + bool flushAccessLogOnTunnelSuccessfullyEstablished() const override { + return flush_log_on_tunnel_successfully_established_; + } const absl::optional& accessLogFlushInterval() override { return access_log_flush_interval_; } @@ -211,6 +214,7 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { std::string access_log_path_; std::list access_logs_; bool flush_access_log_on_new_request_ = false; + bool flush_log_on_tunnel_successfully_established_ = false; absl::optional access_log_flush_interval_; NiceMock filter_callbacks_; MockServerConnection* codec_; diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 8cb09ccc69de..ac73af5f40ed 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -406,7 +406,7 @@ name: add-trailers-filter } } -TEST_P(ProtocolIntegrationTest, AccessLogTest) { +TEST_P(ProtocolIntegrationTest, AccessLogOnRequestStartTest) { if (upstreamProtocol() == Http::CodecType::HTTP3) { return; } diff --git a/test/integration/tcp_tunneling_integration_test.cc b/test/integration/tcp_tunneling_integration_test.cc index b824696f3ba5..742e34eb17d5 100644 --- a/test/integration/tcp_tunneling_integration_test.cc +++ b/test/integration/tcp_tunneling_integration_test.cc @@ -21,7 +21,7 @@ class ConnectTerminationIntegrationTest : public HttpProtocolIntegrationTest { void initialize() override { useAccessLog("%UPSTREAM_WIRE_BYTES_SENT% %UPSTREAM_WIRE_BYTES_RECEIVED% " - "%UPSTREAM_HEADER_BYTES_SENT% %UPSTREAM_HEADER_BYTES_RECEIVED%"); + "%UPSTREAM_HEADER_BYTES_SENT% %UPSTREAM_HEADER_BYTES_RECEIVED% %ACCESS_LOG_TYPE%"); config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { @@ -167,6 +167,22 @@ TEST_P(ConnectTerminationIntegrationTest, Basic) { sendBidirectionalDataAndCleanShutdown(); } +TEST_P(ConnectTerminationIntegrationTest, LogOnSuccessfulTunnel) { + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { + hcm.mutable_access_log_options()->set_flush_log_on_tunnel_successfully_established(true); + }); + + initialize(); + + setUpConnection(); + std::string log = waitForAccessLog(access_log_name_, access_log_entry_); + EXPECT_THAT(log, testing::HasSubstr("DownstreamTunnelSuccessfullyEstablished")); + ++access_log_entry_; + sendBidirectionalDataAndCleanShutdown(); +} + TEST_P(ConnectTerminationIntegrationTest, BasicWithClusterconfig) { terminate_via_cluster_config_ = true; config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 44ce6bc4b7b1..fdd09f7f31a8 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -597,6 +597,7 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_METHOD(const RequestIDExtensionSharedPtr&, requestIDExtension, ()); MOCK_METHOD(const std::list&, accessLogs, ()); MOCK_METHOD(bool, flushAccessLogOnNewRequest, ()); + MOCK_METHOD(bool, flushAccessLogOnTunnelSuccessfullyEstablished, (), (const)); MOCK_METHOD(const absl::optional&, accessLogFlushInterval, ()); MOCK_METHOD(ServerConnection*, createCodec_, (Network::Connection&, const Buffer::Instance&, ServerConnectionCallbacks&, From 09361b458d91378f963e4b49ce0b659150e25eed Mon Sep 17 00:00:00 2001 From: Loong Dai Date: Mon, 15 May 2023 23:34:03 +0800 Subject: [PATCH 231/740] previous_hosts: remove useless assign (#27389) Signed-off-by: Loong --- source/extensions/retry/host/previous_hosts/config.h | 4 ++-- source/extensions/retry/host/previous_hosts/previous_hosts.h | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/source/extensions/retry/host/previous_hosts/config.h b/source/extensions/retry/host/previous_hosts/config.h index b57dc5732847..4f133a434c57 100644 --- a/source/extensions/retry/host/previous_hosts/config.h +++ b/source/extensions/retry/host/previous_hosts/config.h @@ -13,8 +13,8 @@ namespace Host { class PreviousHostsRetryPredicateFactory : public Upstream::RetryHostPredicateFactory { public: Upstream::RetryHostPredicateSharedPtr createHostPredicate(const Protobuf::Message&, - uint32_t retry_count) override { - return std::make_shared(retry_count); + uint32_t) override { + return std::make_shared(); } std::string name() const override { return "envoy.retry_host_predicates.previous_hosts"; } diff --git a/source/extensions/retry/host/previous_hosts/previous_hosts.h b/source/extensions/retry/host/previous_hosts/previous_hosts.h index b7e6748aea0a..097060d01455 100644 --- a/source/extensions/retry/host/previous_hosts/previous_hosts.h +++ b/source/extensions/retry/host/previous_hosts/previous_hosts.h @@ -6,8 +6,6 @@ namespace Envoy { class PreviousHostsRetryPredicate : public Upstream::RetryHostPredicate { public: - PreviousHostsRetryPredicate(uint32_t retry_count) : attempted_hosts_(retry_count) {} - bool shouldSelectAnotherHost(const Upstream::Host& candidate_host) override { return std::find(attempted_hosts_.begin(), attempted_hosts_.end(), &candidate_host) != attempted_hosts_.end(); From 8cfeabd5bd0b15b5e6ef029bf9fef4dcaa1c8254 Mon Sep 17 00:00:00 2001 From: zirain Date: Tue, 16 May 2023 00:14:06 +0800 Subject: [PATCH 232/740] accesslog: support CEL command (#26428) Commit Message: accesslog: support CEL command in the formatter Risk Level: low Testing: unit Docs Changes: none Release Notes: yes --- CODEOWNERS | 1 + api/BUILD | 1 + api/envoy/extensions/formatter/cel/v3/BUILD | 9 ++ .../extensions/formatter/cel/v3/cel.proto | 34 ++++ api/versioning/BUILD | 1 + bazel/repository_locations.bzl | 2 + changelogs/current.yaml | 3 + source/extensions/extensions_build_config.bzl | 1 + source/extensions/extensions_metadata.yaml | 7 + source/extensions/formatter/cel/BUILD | 58 +++++++ source/extensions/formatter/cel/cel.cc | 80 ++++++++++ source/extensions/formatter/cel/cel.h | 47 ++++++ source/extensions/formatter/cel/config.cc | 30 ++++ source/extensions/formatter/cel/config.h | 20 +++ test/extensions/formatter/cel/BUILD | 41 +++++ test/extensions/formatter/cel/cel_test.cc | 147 ++++++++++++++++++ 16 files changed, 482 insertions(+) create mode 100644 api/envoy/extensions/formatter/cel/v3/BUILD create mode 100644 api/envoy/extensions/formatter/cel/v3/cel.proto create mode 100644 source/extensions/formatter/cel/BUILD create mode 100644 source/extensions/formatter/cel/cel.cc create mode 100644 source/extensions/formatter/cel/cel.h create mode 100644 source/extensions/formatter/cel/config.cc create mode 100644 source/extensions/formatter/cel/config.h create mode 100644 test/extensions/formatter/cel/BUILD create mode 100644 test/extensions/formatter/cel/cel_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index 28d0a8b456fa..adc46f60230d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -193,6 +193,7 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/filters/http/set_metadata @aguinet @snowp # Formatters /*/extensions/formatter/metadata @cpakulski @lizan +/*/extensions/formatter/cel @kyessenov @zirain # IP address input matcher /*/extensions/matching/input_matchers/ip @aguinet @snowp # Key Value store diff --git a/api/BUILD b/api/BUILD index 36f2dbe78e06..898f318b5795 100644 --- a/api/BUILD +++ b/api/BUILD @@ -232,6 +232,7 @@ proto_library( "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg", "//envoy/extensions/filters/udp/dns_filter/v3:pkg", "//envoy/extensions/filters/udp/udp_proxy/v3:pkg", + "//envoy/extensions/formatter/cel/v3:pkg", "//envoy/extensions/formatter/metadata/v3:pkg", "//envoy/extensions/formatter/req_without_query/v3:pkg", "//envoy/extensions/health_checkers/redis/v3:pkg", diff --git a/api/envoy/extensions/formatter/cel/v3/BUILD b/api/envoy/extensions/formatter/cel/v3/BUILD new file mode 100644 index 000000000000..ee92fb652582 --- /dev/null +++ b/api/envoy/extensions/formatter/cel/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/formatter/cel/v3/cel.proto b/api/envoy/extensions/formatter/cel/v3/cel.proto new file mode 100644 index 000000000000..ca9d01dedb4d --- /dev/null +++ b/api/envoy/extensions/formatter/cel/v3/cel.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package envoy.extensions.formatter.cel.v3; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.formatter.cel.v3"; +option java_outer_classname = "CelProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/formatter/cel/v3;celv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Formatter extension for printing CEL expressions] +// [#extension: envoy.formatter.cel] + +// CEL formatter extension implements CEL command operator that evaluates configured +// symbolic Common Expression Language expressions to generate an access log. +// +// See :ref:`here ` for more information on access log configuration. + +// %CEL(EXPRESSION):Z% +// Expressions are based on the set of Envoy :ref:`attributes `. +// Expression errors are considered `-`. Z is an optional parameter denoting string +// truncation up to Z characters long. +// +// Examples: +// - ``%CEL(response.code)%`` +// - ``%CEL(connection.mtls)%`` +// - ``%CEL(request.headers['x-envoy-original-path']):10%`` +// - ``%CEL(request.headers['x-log-mtls'] || request.url_path.contains('v1beta3'))%`` + +// Configuration for the CEL formatter. +message Cel { +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 2f201794cf05..ad7aa0d75b47 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -170,6 +170,7 @@ proto_library( "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg", "//envoy/extensions/filters/udp/dns_filter/v3:pkg", "//envoy/extensions/filters/udp/udp_proxy/v3:pkg", + "//envoy/extensions/formatter/cel/v3:pkg", "//envoy/extensions/formatter/metadata/v3:pkg", "//envoy/extensions/formatter/req_without_query/v3:pkg", "//envoy/extensions/health_checkers/redis/v3:pkg", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 7dcdaa32470c..474a02f9d3e8 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1099,6 +1099,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", "envoy.rbac.matchers.upstream_ip_port", + "envoy.formatter.cel", ], release_date = "2022-09-01", cpe = "N/A", @@ -1115,6 +1116,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( extensions = [ "envoy.access_loggers.extension_filters.cel", "envoy.access_loggers.wasm", + "envoy.formatter.cel", "envoy.bootstrap.wasm", "envoy.rate_limit_descriptors.expr", "envoy.filters.http.rbac", diff --git a/changelogs/current.yaml b/changelogs/current.yaml index a2dd3015d18c..a401b5dee946 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -113,6 +113,9 @@ new_features: change: | added %ACCESS_LOG_TYPE% substitution string, to help distinguishing between access log records and when they are being recorded. Please refer to the access log configuration documentation for more information. +- area: access_log + change: | + added :ref:`CEL ` access log formatter to print CEL expression. - area: http change: | added Runtime feature ``envoy.reloadable_features.max_request_headers_size_kb`` to override the default value of diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 6655438230f0..47499a67d2d4 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -369,6 +369,7 @@ EXTENSIONS = { # Formatter # + "envoy.formatter.cel": "//source/extensions/formatter/cel:config", "envoy.formatter.metadata": "//source/extensions/formatter/metadata:config", "envoy.formatter.req_without_query": "//source/extensions/formatter/req_without_query:config", diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 3da991ca9960..faa4c7ef4ae4 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -728,6 +728,13 @@ envoy.filters.udp_listener.udp_proxy: status: stable type_urls: - envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig +envoy.formatter.cel: + categories: + - envoy.formatter + security_posture: robust_to_untrusted_downstream_and_upstream + status: alpha + type_urls: + - envoy.extensions.formatter.cel.v3.Cel envoy.formatter.metadata: categories: - envoy.formatter diff --git a/source/extensions/formatter/cel/BUILD b/source/extensions/formatter/cel/BUILD new file mode 100644 index 000000000000..4a2061662717 --- /dev/null +++ b/source/extensions/formatter/cel/BUILD @@ -0,0 +1,58 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "cel_lib", + srcs = ["cel.cc"], + hdrs = ["cel.h"], + copts = select({ + "//bazel:windows_x86_64": [], # TODO: fix the windows ANTLR build + "//conditions:default": [ + "-DUSE_CEL_PARSER", + ], + }), + deps = [ + "//source/common/formatter:substitution_formatter_lib", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/common/expr:evaluator_lib", + ] + select( + { + "//bazel:windows_x86_64": [], + "//conditions:default": [ + "@com_google_cel_cpp//parser", + ], + }, + ), +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + copts = select({ + "//bazel:windows_x86_64": [], # TODO: fix the windows ANTLR build + "//conditions:default": [ + "-DUSE_CEL_PARSER", + ], + }), + deps = [ + "//envoy/registry", + "//source/extensions/formatter/cel:cel_lib", + "@envoy_api//envoy/extensions/formatter/cel/v3:pkg_cc_proto", + ] + select( + { + "//bazel:windows_x86_64": [], + "//conditions:default": [ + "@com_google_cel_cpp//parser", + ], + }, + ), +) diff --git a/source/extensions/formatter/cel/cel.cc b/source/extensions/formatter/cel/cel.cc new file mode 100644 index 000000000000..b0802e81139c --- /dev/null +++ b/source/extensions/formatter/cel/cel.cc @@ -0,0 +1,80 @@ +#include "source/extensions/formatter/cel/cel.h" + +#include "source/common/config/metadata.h" +#include "source/common/formatter/substitution_formatter.h" +#include "source/common/http/utility.h" +#include "source/common/protobuf/utility.h" + +#if defined(USE_CEL_PARSER) +#include "parser/parser.h" +#endif + +namespace Envoy { +namespace Extensions { +namespace Formatter { + +namespace Expr = Filters::Common::Expr; + +CELFormatter::CELFormatter(Expr::Builder& builder, + const google::api::expr::v1alpha1::Expr& input_expr, + absl::optional& max_length) + : parsed_expr_(input_expr), max_length_(max_length) { + compiled_expr_ = Expr::createExpression(builder, parsed_expr_); +} + +absl::optional CELFormatter::format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + absl::string_view, + AccessLog::AccessLogType) const { + Protobuf::Arena arena; + auto eval_status = Expr::evaluate(*compiled_expr_, arena, stream_info, &request_headers, + &response_headers, &response_trailers); + if (!eval_status.has_value() || eval_status.value().IsError()) { + return absl::nullopt; + } + const auto result = Expr::print(eval_status.value()); + if (max_length_) { + return result.substr(0, max_length_.value()); + } + + return result; +} + +ProtobufWkt::Value CELFormatter::formatValue(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo& stream_info, + absl::string_view, AccessLog::AccessLogType) const { + auto result = format(request_headers, response_headers, response_trailers, stream_info, "", + AccessLog::AccessLogType::NotSet); + if (!result.has_value()) { + return ValueUtil::nullValue(); + } + return ValueUtil::stringValue(result.value()); +} + +::Envoy::Formatter::FormatterProviderPtr +CELFormatterCommandParser::parse(const std::string& command, const std::string& subcommand, + absl::optional& max_length) const { +#if defined(USE_CEL_PARSER) + if (command == "CEL") { + auto parse_status = google::api::expr::parser::Parse(subcommand); + if (!parse_status.ok()) { + throw EnvoyException("Not able to parse filter expression: " + + parse_status.status().ToString()); + } + + return std::make_unique(*expr_builder_, parse_status.value().expr(), max_length); + } + + return nullptr; +#else + throw EnvoyException("CEL is not available for use in this environment."); +#endif +} + +} // namespace Formatter +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/formatter/cel/cel.h b/source/extensions/formatter/cel/cel.h new file mode 100644 index 000000000000..d54e881d6c26 --- /dev/null +++ b/source/extensions/formatter/cel/cel.h @@ -0,0 +1,47 @@ +#pragma once + +#include + +#include "envoy/config/typed_config.h" +#include "envoy/registry/registry.h" + +#include "source/common/formatter/substitution_formatter.h" +#include "source/extensions/filters/common/expr/evaluator.h" + +namespace Envoy { +namespace Extensions { +namespace Formatter { + +class CELFormatter : public ::Envoy::Formatter::FormatterProvider { +public: + CELFormatter(Extensions::Filters::Common::Expr::Builder&, + const google::api::expr::v1alpha1::Expr&, absl::optional&); + + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view, AccessLog::AccessLogType) const override; + +private: + const google::api::expr::v1alpha1::Expr parsed_expr_; + const absl::optional max_length_; + Extensions::Filters::Common::Expr::ExpressionPtr compiled_expr_; +}; + +class CELFormatterCommandParser : public ::Envoy::Formatter::CommandParser { +public: + CELFormatterCommandParser() + : expr_builder_(Extensions::Filters::Common::Expr::createBuilder(nullptr)){}; + ::Envoy::Formatter::FormatterProviderPtr parse(const std::string& command, + const std::string& subcommand, + absl::optional& max_length) const override; + +private: + Extensions::Filters::Common::Expr::BuilderPtr expr_builder_; +}; + +} // namespace Formatter +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/formatter/cel/config.cc b/source/extensions/formatter/cel/config.cc new file mode 100644 index 000000000000..79534aed08d8 --- /dev/null +++ b/source/extensions/formatter/cel/config.cc @@ -0,0 +1,30 @@ +#include "source/extensions/formatter/cel/config.h" + +#include "envoy/extensions/formatter/cel/v3/cel.pb.h" + +#include "source/extensions/formatter/cel/cel.h" + +namespace Envoy { +namespace Extensions { +namespace Formatter { + +::Envoy::Formatter::CommandParserPtr +CELFormatterFactory::createCommandParserFromProto(const Protobuf::Message&) { +#if defined(USE_CEL_PARSER) + return std::make_unique(); +#else + throw EnvoyException("CEL is not available for use in this environment."); +#endif +} + +ProtobufTypes::MessagePtr CELFormatterFactory::createEmptyConfigProto() { + return std::make_unique(); +} + +std::string CELFormatterFactory::name() const { return "envoy.formatter.cel"; } + +REGISTER_FACTORY(CELFormatterFactory, CELFormatterFactory::CommandParserFactory); + +} // namespace Formatter +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/formatter/cel/config.h b/source/extensions/formatter/cel/config.h new file mode 100644 index 000000000000..4016ade6b0c6 --- /dev/null +++ b/source/extensions/formatter/cel/config.h @@ -0,0 +1,20 @@ +#pragma once + +#include "source/common/formatter/substitution_formatter.h" +#include "source/extensions/filters/common/expr/evaluator.h" + +namespace Envoy { +namespace Extensions { +namespace Formatter { + +class CELFormatterFactory : public ::Envoy::Formatter::CommandParserFactory { +public: + ::Envoy::Formatter::CommandParserPtr + createCommandParserFromProto(const Protobuf::Message&) override; + ProtobufTypes::MessagePtr createEmptyConfigProto() override; + std::string name() const override; +}; + +} // namespace Formatter +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/formatter/cel/BUILD b/test/extensions/formatter/cel/BUILD new file mode 100644 index 000000000000..d1160a3b0188 --- /dev/null +++ b/test/extensions/formatter/cel/BUILD @@ -0,0 +1,41 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "cel_test", + srcs = ["cel_test.cc"], + copts = select({ + "//bazel:windows_x86_64": [], # TODO: fix the windows ANTLR build + "//conditions:default": [ + "-DUSE_CEL_PARSER", + ], + }), + extension_names = ["envoy.formatter.cel"], + deps = [ + "//source/common/formatter:substitution_formatter_lib", + "//source/common/json:json_loader_lib", + "//source/extensions/formatter/cel:cel_lib", + "//source/extensions/formatter/cel:config", + "//test/mocks/server:factory_context_mocks", + "//test/mocks/stream_info:stream_info_mocks", + "//test/test_common:test_runtime_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ] + select( + { + "//bazel:windows_x86_64": [], + "//conditions:default": [ + "@com_google_cel_cpp//parser", + ], + }, + ), +) diff --git a/test/extensions/formatter/cel/cel_test.cc b/test/extensions/formatter/cel/cel_test.cc new file mode 100644 index 000000000000..e97a30fe1b3a --- /dev/null +++ b/test/extensions/formatter/cel/cel_test.cc @@ -0,0 +1,147 @@ +#include "envoy/config/core/v3/substitution_format_string.pb.validate.h" + +#include "source/common/formatter/substitution_format_string.h" +#include "source/common/formatter/substitution_formatter.h" +#include "source/extensions/formatter/cel/cel.h" + +#include "test/mocks/server/factory_context.h" +#include "test/mocks/stream_info/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Formatter { + +class CELFormatterTest : public ::testing::Test { +public: + Http::TestRequestHeaderMapImpl request_headers_{ + {":method", "GET"}, + {":path", "/request/path?secret=parameter"}, + {"x-envoy-original-path", "/original/path?secret=parameter"}}; + Http::TestResponseHeaderMapImpl response_headers_; + Http::TestResponseTrailerMapImpl response_trailers_; + StreamInfo::MockStreamInfo stream_info_; + std::string body_; + + envoy::config::core::v3::SubstitutionFormatString config_; + NiceMock context_; +}; + +#ifdef USE_CEL_PARSER +TEST_F(CELFormatterTest, TestFormatValue) { + auto cel_parser = std::make_unique(); + absl::optional max_length = absl::nullopt; + auto formatter = cel_parser->parse("CEL", "request.headers[':method']", max_length); + EXPECT_THAT(formatter->formatValue(request_headers_, response_headers_, response_trailers_, + stream_info_, body_, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("GET"))); +} + +TEST_F(CELFormatterTest, TestParseFail) { + auto cel_parser = std::make_unique(); + absl::optional max_length = absl::nullopt; + EXPECT_EQ(nullptr, + cel_parser->parse("INVALID_CMD", "requests.headers['missing_headers']", max_length)); +} + +TEST_F(CELFormatterTest, TestNullFormatValue) { + auto cel_parser = std::make_unique(); + absl::optional max_length = absl::nullopt; + auto formatter = cel_parser->parse("CEL", "requests.headers['missing_headers']", max_length); + EXPECT_THAT(formatter->formatValue(request_headers_, response_headers_, response_trailers_, + stream_info_, body_, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); +} + +TEST_F(CELFormatterTest, TestRequestHeader) { + const std::string yaml = R"EOF( + text_format_source: + inline_string: "%CEL(request.headers[':method'])%" + formatters: + - name: envoy.formatter.cel + typed_config: + "@type": type.googleapis.com/envoy.extensions.formatter.cel.v3.Cel +)EOF"; + TestUtility::loadFromYaml(yaml, config_); + + auto formatter = + Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + EXPECT_EQ("GET", formatter->format(request_headers_, response_headers_, response_trailers_, + stream_info_, body_, AccessLog::AccessLogType::NotSet)); +} + +TEST_F(CELFormatterTest, TestMissingRequestHeader) { + const std::string yaml = R"EOF( + text_format_source: + inline_string: "%CEL(request.headers['missing-header'])%" + formatters: + - name: envoy.formatter.cel + typed_config: + "@type": type.googleapis.com/envoy.extensions.formatter.cel.v3.Cel +)EOF"; + TestUtility::loadFromYaml(yaml, config_); + + auto formatter = + Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + EXPECT_EQ("-", formatter->format(request_headers_, response_headers_, response_trailers_, + stream_info_, body_, AccessLog::AccessLogType::NotSet)); +} + +TEST_F(CELFormatterTest, TestWithoutMaxLength) { + const std::string yaml = R"EOF( + text_format_source: + inline_string: "%CEL(request.headers['x-envoy-original-path'])%" + formatters: + - name: envoy.formatter.cel + typed_config: + "@type": type.googleapis.com/envoy.extensions.formatter.cel.v3.Cel +)EOF"; + TestUtility::loadFromYaml(yaml, config_); + + auto formatter = + Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + EXPECT_EQ("/original/path?secret=parameter", + formatter->format(request_headers_, response_headers_, response_trailers_, stream_info_, + body_, AccessLog::AccessLogType::NotSet)); +} + +TEST_F(CELFormatterTest, TestMaxLength) { + const std::string yaml = R"EOF( + text_format_source: + inline_string: "%CEL(request.headers['x-envoy-original-path']):9%" + formatters: + - name: envoy.formatter.cel + typed_config: + "@type": type.googleapis.com/envoy.extensions.formatter.cel.v3.Cel +)EOF"; + TestUtility::loadFromYaml(yaml, config_); + + auto formatter = + Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + EXPECT_EQ("/original", formatter->format(request_headers_, response_headers_, response_trailers_, + stream_info_, body_, AccessLog::AccessLogType::NotSet)); +} + +TEST_F(CELFormatterTest, TestInvalidExpression) { + const std::string yaml = R"EOF( + text_format_source: + inline_string: "%CEL(+++++)%" + formatters: + - name: envoy.formatter.cel + typed_config: + "@type": type.googleapis.com/envoy.extensions.formatter.cel.v3.Cel +)EOF"; + TestUtility::loadFromYaml(yaml, config_); + + EXPECT_THROW_WITH_REGEX( + Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_), + EnvoyException, "Not able to parse filter expression: .*"); +} +#endif + +} // namespace Formatter +} // namespace Extensions +} // namespace Envoy From 5cfc4fa0abce76c6964d7ab33b754068c251232b Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 15 May 2023 13:31:56 -0400 Subject: [PATCH 233/740] Harmonize transfer-encoding and content-length checks for UHV (#27378) * Harmonize transfer-encoding and content-length checks for UHV Signed-off-by: Yan Avlasov --- envoy/http/header_validator_errors.h | 9 ++ source/common/http/conn_manager_impl.cc | 4 +- source/common/http/http1/balsa_parser.cc | 4 + source/common/http/http1/codec_impl.cc | 2 +- .../envoy_default/http1_header_validator.cc | 91 ++++++++------ .../envoy_default/http1_header_validator.h | 31 ++++- test/common/http/http1/codec_impl_test.cc | 112 +++++++++++++++--- .../http1_header_validator_test.cc | 36 +++++- test/integration/integration_test.cc | 53 +++++++-- 9 files changed, 267 insertions(+), 75 deletions(-) diff --git a/envoy/http/header_validator_errors.h b/envoy/http/header_validator_errors.h index e2ba33b2dfe2..2b047a876638 100644 --- a/envoy/http/header_validator_errors.h +++ b/envoy/http/header_validator_errors.h @@ -24,5 +24,14 @@ struct UhvResponseCodeDetailValues { using UhvResponseCodeDetail = ConstSingleton; +struct Http1ResponseCodeDetailValues { + const std::string InvalidTransferEncoding = "http1.invalid_transfer_encoding"; + const std::string TransferEncodingNotAllowed = "uhv.http1.transfer_encoding_not_allowed"; + const std::string ContentLengthNotAllowed = "uhv.http1.content_length_not_allowed"; + const std::string ChunkedContentLength = "http1.content_length_and_chunked_not_allowed"; +}; + +using Http1ResponseCodeDetail = ConstSingleton; + } // namespace Http } // namespace Envoy diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index d658f0a7f4c7..d833a4e3e787 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -970,7 +970,9 @@ bool ConnectionManagerImpl::ActiveStream::validateHeaders() { } if (failure) { std::function modify_headers; - Code response_code = Code::BadRequest; + Code response_code = failure_details == Http1ResponseCodeDetail::get().InvalidTransferEncoding + ? Code::NotImplemented + : Code::BadRequest; absl::optional grpc_status; bool is_grpc = Grpc::Common::hasGrpcContentType(*request_headers_); if (redirect && !is_grpc) { diff --git a/source/common/http/http1/balsa_parser.cc b/source/common/http/http1/balsa_parser.cc index a274a76e6723..8826994d7bc3 100644 --- a/source/common/http/http1/balsa_parser.cc +++ b/source/common/http/http1/balsa_parser.cc @@ -150,6 +150,10 @@ BalsaParser::BalsaParser(MessageType type, ParserCallbacks* connection, size_t m http_validation_policy.require_header_colon = true; http_validation_policy.disallow_multiple_content_length = false; http_validation_policy.disallow_transfer_encoding_with_content_length = false; +#ifdef ENVOY_ENABLE_UHV + // UHV - disable transfer-encoding validations in Balsa + http_validation_policy.validate_transfer_encoding = false; +#endif framer_.set_http_validation_policy(http_validation_policy); framer_.set_balsa_headers(&headers_); diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index c9f06c1acf55..db93c5ee68e7 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -910,7 +910,6 @@ StatusOr ConnectionImpl::onHeadersCompleteImpl() { "http/1.1 protocol error: both 'Content-Length' and 'Transfer-Encoding' are set."); } } -#endif // Per https://tools.ietf.org/html/rfc7230#section-3.3.1 Envoy should reject // transfer-codings it does not understand. @@ -925,6 +924,7 @@ StatusOr ConnectionImpl::onHeadersCompleteImpl() { return codecProtocolError("http/1.1 protocol error: unsupported transfer encoding"); } } +#endif auto statusor = onHeadersCompleteBase(); if (!statusor.ok()) { diff --git a/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc b/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc index bc512e77a064..fc54b156454a 100644 --- a/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc @@ -23,15 +23,7 @@ using ::Envoy::Http::Protocol; using ::Envoy::Http::RequestHeaderMap; using ::Envoy::Http::UhvResponseCodeDetail; using ValidationResult = ::Envoy::Http::HeaderValidator::ValidationResult; - -struct Http1ResponseCodeDetailValues { - const std::string InvalidTransferEncoding = "uhv.http1.invalid_transfer_encoding"; - const std::string TransferEncodingNotAllowed = "uhv.http1.transfer_encoding_not_allowed"; - const std::string ContentLengthNotAllowed = "uhv.http1.content_length_not_allowed"; - const std::string ChunkedContentLength = "http1.content_length_and_chunked_not_allowed"; -}; - -using Http1ResponseCodeDetail = ConstSingleton; +using Http1ResponseCodeDetail = ::Envoy::Http::Http1ResponseCodeDetail; /* * Header validation implementation for the Http/1 codec. This class follows guidance from @@ -99,6 +91,30 @@ Http1HeaderValidator::validateResponseHeaderEntry(const HeaderString& key, return validateGenericHeaderValue(value); } +ValidationResult Http1HeaderValidator::validateContentLengthAndTransferEncoding( + const ::Envoy::Http::RequestOrResponseHeaderMap& header_map) { + /** + * Validate Transfer-Encoding and Content-Length headers. + * HTTP/1.1 disallows a Transfer-Encoding and Content-Length headers, + * https://www.rfc-editor.org/rfc/rfc9112.html#section-6.2: + * + * A sender MUST NOT send a Content-Length header field in any message that + * contains a Transfer-Encoding header field. + * + * The http1_protocol_options.allow_chunked_length config setting can + * override the RFC compliance to allow a Transfer-Encoding of "chunked" with + * a Content-Length set. In this exception case, we remove the Content-Length + * header in the transform[Request/Response]Headers() method. + */ + if (header_map.TransferEncoding() && header_map.ContentLength() && + hasChunkedTransferEncoding(header_map.TransferEncoding()->value()) && + !config_.http1_protocol_options().allow_chunked_length()) { + // Configuration does not allow chunked encoding and content-length, reject the request + return {ValidationResult::Action::Reject, Http1ResponseCodeDetail::get().ChunkedContentLength}; + } + return ValidationResult::success(); +} + ValidationResult Http1HeaderValidator::validateRequestHeaders(const RequestHeaderMap& header_map) { absl::string_view path = header_map.getPathValue(); absl::string_view host = header_map.getHostValue(); @@ -166,16 +182,6 @@ ValidationResult Http1HeaderValidator::validateRequestHeaders(const RequestHeade } // Step 2: Validate Transfer-Encoding and Content-Length headers. - // HTTP/1.1 disallows a Transfer-Encoding and Content-Length headers, - // https://www.rfc-editor.org/rfc/rfc9112.html#section-6.2: - // - // A sender MUST NOT send a Content-Length header field in any message that - // contains a Transfer-Encoding header field. - // - // The http1_protocol_options.allow_chunked_length config setting can - // override the RFC compliance to allow a Transfer-Encoding of "chunked" with - // a Content-Length set. In this exception case, we remove the Content-Length - // header. if (header_map.TransferEncoding()) { // CONNECT methods must not contain any content so reject the request if Transfer-Encoding or // Content-Length is provided, per RFC 9110, @@ -188,12 +194,9 @@ ValidationResult Http1HeaderValidator::validateRequestHeaders(const RequestHeade Http1ResponseCodeDetail::get().TransferEncodingNotAllowed}; } - if (header_map.ContentLength() && - hasChunkedTransferEncoding(header_map.TransferEncoding()->value()) && - !config_.http1_protocol_options().allow_chunked_length()) { - // Configuration does not allow chunked length, reject the request - return {ValidationResult::Action::Reject, - Http1ResponseCodeDetail::get().ChunkedContentLength}; + ValidationResult result = validateContentLengthAndTransferEncoding(header_map); + if (!result.ok()) { + return result; } } @@ -275,23 +278,28 @@ ValidationResult Http1HeaderValidator::validateRequestHeaders(const RequestHeade return ValidationResult::success(); } -void Http1HeaderValidator::sanitizeContentLength(::Envoy::Http::RequestHeaderMap& header_map) { +void Http1HeaderValidator::sanitizeContentLength( + ::Envoy::Http::RequestOrResponseHeaderMap& header_map) { // The http1_protocol_options.allow_chunked_length config setting can // override the RFC compliance to allow a Transfer-Encoding of "chunked" with // a Content-Length set. In this exception case, we remove the Content-Length // header. - if (header_map.TransferEncoding()) { - if (header_map.ContentLength() && - hasChunkedTransferEncoding(header_map.TransferEncoding()->value())) { - if (config_.http1_protocol_options().allow_chunked_length()) { - // Allow a chunked transfer encoding and remove the content length. - header_map.removeContentLength(); - } - } - } else if (header_map.ContentLength() && header_map.getContentLengthValue() == "0" && - ::Envoy::Http::HeaderUtility::isConnect(header_map)) { + if (header_map.TransferEncoding() && header_map.ContentLength() && + hasChunkedTransferEncoding(header_map.TransferEncoding()->value()) && + config_.http1_protocol_options().allow_chunked_length()) { + // Allow a chunked transfer encoding and remove the content length. + header_map.removeContentLength(); + } +} + +void ServerHttp1HeaderValidator::sanitizeContentLength( + ::Envoy::Http::RequestHeaderMap& header_map) { + if (header_map.ContentLength() && header_map.getContentLengthValue() == "0" && + ::Envoy::Http::HeaderUtility::isConnect(header_map)) { // Remove a 0 content length from a CONNECT request header_map.removeContentLength(); + } else { + Http1HeaderValidator::sanitizeContentLength(header_map); } } @@ -308,6 +316,12 @@ ServerHttp1HeaderValidator::transformRequestHeaders(::Envoy::Http::RequestHeader return ::Envoy::Http::ServerHeaderValidator::RequestHeadersTransformationResult::success(); } +::Envoy::Http::HeaderValidator::TransformationResult +ClientHttp1HeaderValidator::transformResponseHeaders(::Envoy::Http::ResponseHeaderMap& header_map) { + sanitizeContentLength(header_map); + return TransformationResult::success(); +} + ValidationResult Http1HeaderValidator::validateResponseHeaders(const ::Envoy::Http::ResponseHeaderMap& header_map) { // Step 1: verify that required pseudo headers are present @@ -333,6 +347,11 @@ Http1HeaderValidator::validateResponseHeaders(const ::Envoy::Http::ResponseHeade Http1ResponseCodeDetail::get().TransferEncodingNotAllowed}; } + ValidationResult result = validateContentLengthAndTransferEncoding(header_map); + if (!result.ok()) { + return result; + } + // Step 3: Verify each response header std::string reject_details; header_map.iterate([this, &reject_details](const ::Envoy::Http::HeaderEntry& header_entry) diff --git a/source/extensions/http/header_validators/envoy_default/http1_header_validator.h b/source/extensions/http/header_validators/envoy_default/http1_header_validator.h index 9134619d0800..a9368d70a974 100644 --- a/source/extensions/http/header_validators/envoy_default/http1_header_validator.h +++ b/source/extensions/http/header_validators/envoy_default/http1_header_validator.h @@ -34,10 +34,24 @@ class Http1HeaderValidator : public HeaderValidator { * config options is true. * If the http1_protocol_options.allow_chunked_length is false a request with both * Transfer-Encoding and Content-Length headers is rejected in the validateRequestHeaders method. - * Additionally if request is CONNECT and Content-Length is 0, the Content-Length header is - * removed. */ - void sanitizeContentLength(::Envoy::Http::RequestHeaderMap& header_map); + void sanitizeContentLength(::Envoy::Http::RequestOrResponseHeaderMap& header_map); + + /** + * Validate Transfer-Encoding and Content-Length headers. + * HTTP/1.1 disallows a Transfer-Encoding and Content-Length headers, + * https://www.rfc-editor.org/rfc/rfc9112.html#section-6.2: + * + * A sender MUST NOT send a Content-Length header field in any message that + * contains a Transfer-Encoding header field. + * + * The http1_protocol_options.allow_chunked_length config setting can + * override the RFC compliance to allow a Transfer-Encoding of "chunked" with + * a Content-Length set. In this exception case, we remove the Content-Length + * header in the transform[Request/Response]Headers() method. + */ + ::Envoy::Http::HeaderValidator::ValidationResult validateContentLengthAndTransferEncoding( + const ::Envoy::Http::RequestOrResponseHeaderMap& header_map); private: /* @@ -94,6 +108,13 @@ class ServerHttp1HeaderValidator : public Http1HeaderValidator, transformResponseHeaders(const ::Envoy::Http::ResponseHeaderMap&) override { return ResponseHeadersTransformationResult::success(); } + +private: + /** + * If request is CONNECT and Content-Length is 0, the Content-Length header is removed. + * Otherwise calls Http1HeaderValidator::sanitizeContentLength() + */ + void sanitizeContentLength(::Envoy::Http::RequestHeaderMap& header_map); }; class ClientHttp1HeaderValidator : public Http1HeaderValidator, @@ -130,9 +151,7 @@ class ClientHttp1HeaderValidator : public Http1HeaderValidator, return RequestHeadersTransformationResult::success(); } - TransformationResult transformResponseHeaders(::Envoy::Http::ResponseHeaderMap&) override { - return TransformationResult::success(); - } + TransformationResult transformResponseHeaders(::Envoy::Http::ResponseHeaderMap&) override; }; using ServerHttp1HeaderValidatorPtr = std::unique_ptr; diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 821e446959d9..04b7b9ce46be 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -7,6 +7,7 @@ #include "envoy/event/dispatcher.h" #include "envoy/extensions/http/header_validators/envoy_default/v3/header_validator.pb.h" #include "envoy/http/codec.h" +#include "envoy/http/header_validator_errors.h" #include "source/common/buffer/buffer_impl.h" #include "source/common/common/utility.h" @@ -38,6 +39,7 @@ using testing::InvokeWithoutArgs; using testing::NiceMock; using testing::Return; using testing::ReturnRef; +using testing::StartsWith; using testing::StrictMock; namespace Envoy { @@ -120,9 +122,14 @@ class MockRequestDecoderShimWithUhv : public Http::MockRequestDecoder { } failure_details = transformation_result.details(); } - sendLocalReply(Http::Code::BadRequest, Http::CodeUtility::toString(Http::Code::BadRequest), - nullptr, absl::nullopt, failure_details); - response_encoder_->getStream().resetStream(Http::StreamResetReason::LocalReset); + Code response_code = failure_details == Http1ResponseCodeDetail::get().InvalidTransferEncoding + ? Code::NotImplemented + : Code::BadRequest; + sendLocalReply(response_code, Http::CodeUtility::toString(response_code), nullptr, + absl::nullopt, failure_details); + if (response_encoder_) { + response_encoder_->getStream().resetStream(Http::StreamResetReason::LocalReset); + } connection_.state_ = Network::Connection::State::Closing; } else { MockRequestDecoder::decodeHeaders(std::move(headers), end_stream); @@ -500,33 +507,67 @@ TEST_P(Http1ServerConnectionImplTest, EmptyHeader) { // We support the identity encoding, but because it does not end in chunked encoding we reject it // per RFC 7230 Section 3.3.3 TEST_P(Http1ServerConnectionImplTest, IdentityEncodingNoChunked) { +#ifdef ENVOY_ENABLE_UHV + // TODO(#27377): http-parser will not be used together with UHV and triggers an internal + // transfer-encoding check preventing UHV to be called. + if (parser_impl_ == Http1ParserImpl::HttpParser) { + return; + } +#endif initialize(); InSequence sequence; - MockRequestDecoder decoder; + MockRequestDecoderShimWithUhv decoder(header_validator_.get(), connection_); EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); - Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\ntransfer-encoding: identity\r\n\r\n"); - EXPECT_CALL(decoder, sendLocalReply(_, _, _, _, _)); + Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\nHost: host\r\ntransfer-encoding: identity\r\n\r\n"); + EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, _, _, _, + "http1.invalid_transfer_encoding")); auto status = codec_->dispatch(buffer); +#ifdef ENVOY_ENABLE_UHV + EXPECT_TRUE(status.ok()); +#else EXPECT_TRUE(isCodecProtocolError(status)); EXPECT_EQ(status.message(), "http/1.1 protocol error: unsupported transfer encoding"); +#endif } TEST_P(Http1ServerConnectionImplTest, UnsupportedEncoding) { +#ifdef ENVOY_ENABLE_UHV + // TODO(#27377): http-parser will not be used together with UHV and triggers an internal + // transfer-encoding check preventing UHV to be called. + if (parser_impl_ == Http1ParserImpl::HttpParser) { + return; + } +#endif initialize(); InSequence sequence; - MockRequestDecoder decoder; + MockRequestDecoderShimWithUhv decoder(header_validator_.get(), connection_); EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); - Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\ntransfer-encoding: gzip\r\n\r\n"); - EXPECT_CALL(decoder, sendLocalReply(_, _, _, _, _)); + Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\nHost: host\r\ntransfer-encoding: gzip\r\n\r\n"); +#ifdef ENVOY_ENABLE_UHV + EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, _, _, _, + "http1.invalid_transfer_encoding")); +#else + if (parser_impl_ == Http1ParserImpl::HttpParser) { + EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, _, _, _, + "http1.invalid_transfer_encoding")); + } else { + // TODO(#27375): Balsa codec produces invalid response in non UHV mode + EXPECT_CALL(decoder, sendLocalReply(Http::Code::BadRequest, _, _, _, "http1.codec_error")); + } +#endif auto status = codec_->dispatch(buffer); +#ifdef ENVOY_ENABLE_UHV + EXPECT_TRUE(status.ok()); +#else EXPECT_TRUE(isCodecProtocolError(status)); EXPECT_EQ(status.message(), "http/1.1 protocol error: unsupported transfer encoding"); +#endif } // Verify that the optimization which moves large slices of body instead of copying them is working. @@ -738,6 +779,13 @@ TEST_P(Http1ServerConnectionImplTest, InvalidChunkHeader) { } TEST_P(Http1ServerConnectionImplTest, IdentityAndChunkedBody) { +#ifdef ENVOY_ENABLE_UHV + // TODO(#27377): http-parser will not be used together with UHV and triggers an internal + // transfer-encoding check preventing UHV to be called. + if (parser_impl_ == Http1ParserImpl::HttpParser) { + return; + } +#endif initialize(); InSequence sequence; @@ -745,13 +793,19 @@ TEST_P(Http1ServerConnectionImplTest, IdentityAndChunkedBody) { MockRequestDecoder decoder; EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); - Buffer::OwnedImpl buffer("POST / HTTP/1.1\r\ntransfer-encoding: " + Buffer::OwnedImpl buffer("POST / HTTP/1.1\r\nHost: host\r\ntransfer-encoding: " "identity,chunked\r\n\r\nb\r\nHello World\r\n0\r\n\r\n"); - EXPECT_CALL(decoder, sendLocalReply(_, _, _, _, _)); + if (parser_impl_ == Http1ParserImpl::HttpParser) { + EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, _, _, _, + "http1.invalid_transfer_encoding")); + } else { + // TODO(#27375): Balsa codec produces invalid response in non UHV mode + EXPECT_CALL(decoder, sendLocalReply(Http::Code::BadRequest, _, _, _, "http1.codec_error")); + } auto status = codec_->dispatch(buffer); EXPECT_TRUE(isCodecProtocolError(status)); - EXPECT_EQ(status.message(), "http/1.1 protocol error: unsupported transfer encoding"); + EXPECT_THAT(status.message(), StartsWith("http/1.1 protocol error")); } TEST_P(Http1ServerConnectionImplTest, HostWithLWS) { @@ -2232,17 +2286,30 @@ TEST_P(Http1ServerConnectionImplTest, ConnectRequestWithTEChunked) { initialize(); InSequence sequence; - NiceMock decoder; - EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + MockRequestDecoderShimWithUhv decoder(header_validator_.get(), connection_); + EXPECT_CALL(callbacks_, newStream(_, _)) + .WillOnce(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoder& { + decoder.setResponseEncoder(&encoder); + return decoder; + })); // Per https://tools.ietf.org/html/rfc7231#section-4.3.6 CONNECT with body has no defined // semantics: Envoy will reject chunked CONNECT requests. - EXPECT_CALL(decoder, sendLocalReply(_, _, _, _, _)); +#ifdef ENVOY_ENABLE_UHV + EXPECT_CALL(decoder, sendLocalReply(Http::Code::BadRequest, _, _, _, _)); +#else + EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, _, _, _, + "http1.invalid_transfer_encoding")); +#endif Buffer::OwnedImpl buffer( "CONNECT host:80 HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n12345abcd"); auto status = codec_->dispatch(buffer); +#ifdef ENVOY_ENABLE_UHV + EXPECT_TRUE(status.ok()); +#else EXPECT_TRUE(isCodecProtocolError(status)); EXPECT_EQ(status.message(), "http/1.1 protocol error: unsupported transfer encoding"); +#endif } TEST_P(Http1ServerConnectionImplTest, ConnectRequestWithNonZeroContentLength) { @@ -4435,13 +4502,18 @@ TEST_P(Http1ServerConnectionImplTest, MultipleTransferEncoding) { initialize(); InSequence s; - StrictMock decoder; + MockRequestDecoderShimWithUhv decoder(header_validator_.get(), connection_); Http::ResponseEncoder* response_encoder = nullptr; EXPECT_CALL(callbacks_, newStream(_, _)) .WillOnce(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoder& { response_encoder = &encoder; + decoder.setResponseEncoder(&encoder); return decoder; })); +#ifdef ENVOY_ENABLE_UHV + EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, "Not Implemented", _, _, + "http1.invalid_transfer_encoding")); +#else if (parser_impl_ == Http1ParserImpl::BalsaParser) { EXPECT_CALL(decoder, sendLocalReply(Http::Code::BadRequest, "Bad Request", _, _, "http1.codec_error")); @@ -4449,14 +4521,17 @@ TEST_P(Http1ServerConnectionImplTest, MultipleTransferEncoding) { EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, "Not Implemented", _, _, "http1.invalid_transfer_encoding")); } - - Buffer::OwnedImpl buffer("POST / HTTP/1.1\r\n" +#endif + Buffer::OwnedImpl buffer("POST / HTTP/1.1\r\nHost: foo.bar\r\n" "Transfer-Encoding: chunked\r\n" "Transfer-Encoding: chunked\r\n" "\r\n"); auto status = codec_->dispatch(buffer); +#ifdef ENVOY_ENABLE_UHV + EXPECT_TRUE(status.ok()); +#else EXPECT_TRUE(isCodecProtocolError(status)); if (parser_impl_ == Http1ParserImpl::BalsaParser) { @@ -4466,6 +4541,7 @@ TEST_P(Http1ServerConnectionImplTest, MultipleTransferEncoding) { EXPECT_EQ("http1.invalid_transfer_encoding", response_encoder->getStream().responseDetails()); EXPECT_EQ(status.message(), "http/1.1 protocol error: unsupported transfer encoding"); } +#endif } } // namespace Http diff --git a/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc b/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc index ba3a38119dad..0dc4d5d7fe3d 100644 --- a/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc @@ -66,7 +66,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateTransferEncodingInRequest) { request_headers.setCopy(LowerCaseString("transfer-encoding"), "gzip"); EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(request_headers), - "uhv.http1.invalid_transfer_encoding"); + "http1.invalid_transfer_encoding"); } TEST_F(Http1HeaderValidatorTest, ValidateTransferEncodingInResponse) { @@ -78,7 +78,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateTransferEncodingInResponse) { request_headers.setCopy(LowerCaseString("transfer-encoding"), "gzip"); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(request_headers), - "uhv.http1.invalid_transfer_encoding"); + "http1.invalid_transfer_encoding"); } TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderEntryCustomMethod) { @@ -357,6 +357,36 @@ TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderMapContentLengthConnectRej "uhv.http1.content_length_not_allowed"); } +TEST_F(Http1HeaderValidatorTest, ResponseTransferEncodingContentLengthReject) { + ::Envoy::Http::TestResponseHeaderMapImpl headers{ + {":status", "200"}, {"transfer-encoding", "chunked"}, {"content-length", "10"}}; + auto uhv = createH1Client(empty_config); + + EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), + "http1.content_length_and_chunked_not_allowed"); +} + +TEST_F(Http1HeaderValidatorTest, ResponseTransferEncodingContentLengthAllowed) { + ::Envoy::Http::TestResponseHeaderMapImpl headers{ + {":status", "200"}, {"transfer-encoding", "chunked"}, {"content-length", "10"}}; + auto uhv = createH1Client(allow_chunked_length_config); + + EXPECT_ACCEPT(uhv->validateResponseHeaders(headers)); + // The transform method should remove content-length + EXPECT_ACCEPT(uhv->transformResponseHeaders(headers)); + EXPECT_EQ(headers.ContentLength(), nullptr); +} + +TEST_F(Http1HeaderValidatorTest, ResponseContentLengthNoTransferEncoding) { + ::Envoy::Http::TestResponseHeaderMapImpl headers{{":status", "200"}, {"content-length", "10"}}; + auto uhv = createH1Client(allow_chunked_length_config); + + EXPECT_ACCEPT(uhv->validateResponseHeaders(headers)); + // The transform method should keep content-length + EXPECT_ACCEPT(uhv->transformResponseHeaders(headers)); + EXPECT_EQ(headers.getContentLengthValue(), "10"); +} + TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderMapConnectRegNameMissingPort) { ::Envoy::Http::TestRequestHeaderMapImpl headers{ {":scheme", "https"}, {":method", "CONNECT"}, {":authority", "envoy.com"}}; @@ -561,7 +591,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateResponseHeaderMapInvaidTransferEncoding auto uhv = createH1Client(empty_config); EXPECT_REJECT_WITH_DETAILS(uhv->validateResponseHeaders(headers), - "uhv.http1.invalid_transfer_encoding"); + "http1.invalid_transfer_encoding"); } TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderMapNormalizePath) { diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index f4e601051946..618bf95aac2a 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -854,11 +854,6 @@ TEST_P(IntegrationTest, UpstreamDisconnectWithTwoRequests) { } TEST_P(IntegrationTest, TestSmuggling) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23289) - uniform handling of Transfer-Encoding validation between codec and UHV - return; -#endif - config_helper_.disableDelayClose(); initialize(); @@ -919,6 +914,28 @@ TEST_P(IntegrationTest, TestSmuggling) { } } +TEST_P(IntegrationTest, TestInvalidTransferEncoding) { + config_helper_.disableDelayClose(); + initialize(); + + // Verify that sending `Transfer-Encoding: chunked` as a second header is detected and triggers + // the "bad Transfer-Encoding" check. + std::string response; + const std::string request = "GET / HTTP/1.1\r\nHost: host\r\ntransfer-encoding: " + "identity\r\ntransfer-encoding: chunked \r\n\r\n"; + sendRawHttpAndWaitForResponse(lookupPort("http"), request.c_str(), &response, false); +#ifdef ENVOY_ENABLE_UHV + EXPECT_THAT(response, StartsWith("HTTP/1.1 501 Not Implemented\r\n")); +#else + if (http1_implementation_ == Http1ParserImpl::BalsaParser) { + // TODO(#27375): Balsa codec produces invalid response in non UHV mode + EXPECT_THAT(response, StartsWith("HTTP/1.1 400 Bad Request\r\n")); + } else { + EXPECT_THAT(response, StartsWith("HTTP/1.1 501 Not Implemented\r\n")); + } +#endif +} + TEST_P(IntegrationTest, TestPipelinedResponses) { initialize(); auto tcp_client = makeTcpConnection(lookupPort("http")); @@ -986,11 +1003,6 @@ TEST_P(IntegrationTest, TestServerAllowChunkedLength) { } TEST_P(IntegrationTest, TestClientAllowChunkedLength) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23289) - uniform handling of Transfer-Encoding validation between codec and UHV - return; -#endif - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() == 1, ""); if (fake_upstreams_[0]->httpType() == Http::CodecType::HTTP1) { @@ -2163,6 +2175,27 @@ TEST_P(IntegrationTest, ConnectWithChunkedBody) { ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); } +TEST_P(IntegrationTest, ConnectWithTEChunked) { + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { ConfigHelper::setConnectConfig(hcm, false, false); }); + initialize(); + + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("http")); + ASSERT_TRUE(tcp_client->write( + "CONNECT host.com:80 HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\npayload", false)); + + tcp_client->waitForData("\r\n\r\n", false); +#ifdef ENVOY_ENABLE_UHV + EXPECT_TRUE(absl::StartsWith(tcp_client->data(), "HTTP/1.1 400 Bad Request\r\n")) + << tcp_client->data(); +#else + EXPECT_TRUE(absl::StartsWith(tcp_client->data(), "HTTP/1.1 501 Not Implemented\r\n")) + << tcp_client->data(); +#endif + tcp_client->close(); +} + // Verifies that a 204 response returns without a body TEST_P(IntegrationTest, Response204WithBody) { initialize(); From b59e2980831edd85382f49f0dd98f3fdb403af33 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 15 May 2023 14:40:23 -0400 Subject: [PATCH 234/740] Deflake file_system_buffer_integration_test under TSAN (#27405) Signed-off-by: Yan Avlasov --- .../filters/http/file_system_buffer/filter_integration_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/filters/http/file_system_buffer/filter_integration_test.cc b/test/extensions/filters/http/file_system_buffer/filter_integration_test.cc index 579b267be212..46fdd93a94ba 100644 --- a/test/extensions/filters/http/file_system_buffer/filter_integration_test.cc +++ b/test/extensions/filters/http/file_system_buffer/filter_integration_test.cc @@ -75,7 +75,7 @@ TEST_P(FileSystemBufferIntegrationTest, RequestAndResponseWithGiantBodyBuffer) { // Not quite as giant as the memory buffer's integration test uses - with // disk operations involved that size risks timing out the test. testRouterRequestAndResponseWithBody(1024 * 1024, 1024 * 1024, false, false, nullptr, - std::chrono::milliseconds(25000)); + std::chrono::milliseconds(25000) * TSAN_TIMEOUT_FACTOR); } TEST_P(FileSystemBufferIntegrationTest, HeaderOnlyRequestAndResponseBuffer) { From 54daa8cd0eef38d869a27ae8506ca8de14d07efc Mon Sep 17 00:00:00 2001 From: Chase <70911039+pchaseh@users.noreply.github.com> Date: Mon, 15 May 2023 15:02:29 -0400 Subject: [PATCH 235/740] docs: Fix type URI for custom_response example (#27329) Signed-off-by: pchaseh --- .../filters/http/custom_response/v3/custom_response.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/envoy/extensions/filters/http/custom_response/v3/custom_response.proto b/api/envoy/extensions/filters/http/custom_response/v3/custom_response.proto index 8f2c1c2bd25c..cd28640fefda 100644 --- a/api/envoy/extensions/filters/http/custom_response/v3/custom_response.proto +++ b/api/envoy/extensions/filters/http/custom_response/v3/custom_response.proto @@ -48,7 +48,7 @@ message CustomResponse { // action: // name: action // typed_config: - // "@type": type.googleapis.com/envoy.extensions.filters.http.custom_response.v3.LocalResponsePolicy + // "@type": type.googleapis.com/envoy.extensions.http.custom_response.local_response_policy.v3.LocalResponsePolicy // status_code: 499 // body: // inline_string: "not allowed" @@ -89,7 +89,7 @@ message CustomResponse { // action: // name: action // typed_config: - // "@type": type.googleapis.com/envoy.extensions.filters.http.custom_response.v3.RedirectPolicy + // "@type": type.googleapis.com/envoy.extensions.http.custom_response.redirect_policy.v3.RedirectPolicy // status_code: 299 // uri: "https://foo.example/gateway_error" // response_headers_to_add: From 15790e2113c920ffd19d7a8f5bc978ecb77c2c67 Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Mon, 15 May 2023 15:12:48 -0400 Subject: [PATCH 236/740] sub-formatter: fix wrong handling of UPSTREAM_WIRE_BYTES_RECEIVED (#27281) Signed-off-by: Adi Suissa-Peleg --- .../formatter/substitution_formatter.cc | 3 ++- .../wrong_UPSTREAM_WIRE_BYTES_RECEIVED | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 test/common/router/route_corpus/wrong_UPSTREAM_WIRE_BYTES_RECEIVED diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index 3937d2ffbee6..087d3a9d2f06 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -974,7 +974,8 @@ const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnow [](const std::string&, const absl::optional&) { return std::make_unique( [](const StreamInfo::StreamInfo& stream_info) { - return stream_info.getUpstreamBytesMeter()->wireBytesReceived(); + auto bytes_meter = stream_info.getUpstreamBytesMeter(); + return bytes_meter ? bytes_meter->wireBytesReceived() : 0; }); }}}, {"UPSTREAM_HEADER_BYTES_RECEIVED", diff --git a/test/common/router/route_corpus/wrong_UPSTREAM_WIRE_BYTES_RECEIVED b/test/common/router/route_corpus/wrong_UPSTREAM_WIRE_BYTES_RECEIVED new file mode 100644 index 000000000000..63a22eb8104e --- /dev/null +++ b/test/common/router/route_corpus/wrong_UPSTREAM_WIRE_BYTES_RECEIVED @@ -0,0 +1,27 @@ +config { + virtual_hosts { + name: "eMn" + domains: "*" + matcher { + on_no_match { + action { + name: "%" + typed_config { + type_url: "/envoy.config.route.v3.Route" + value: "\n\002\n\000\022\003\n\001~J\217\007\n\214\007\n\001~\022\206\007`%START_TIME()%\00184\177\177\177%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%UPSTREAM_WIRE_BYTES_RECEIVED%%%%UPSTREAM_TLS_SESSION_ID%%%%%%%%%%%%%%%%%%%%%%%%REQUEST_TX_DURATION%%%%%%UPSTREAM_WIRE_BYTES_RECEIVED%%%%UPSTREAM_TLS_SESSION_ID%%%%%%UPSTREAM_TLS_SESSION_ID%%%%%%%%%%%%%%%%%%%%%%%%%%%%%VTSTNAME%START_TIME(/TIME%_%%%%%%%%%%%%%%%%%%%%%%%%%%_)%TA\177\177?\017%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%REQUEST_TX_DURATION%%%%%%UPSTREAM_WIRE_BYTES_RECEIVED%%%%UPSTREAM_TLS_SESSION_ID%%%%%%UPSTREAM_TLS_SESSION_ID%%%%%%%%%%%%%%%%%%%%%%%%%%%%%VTSTNAM%%%%%%%%%%%%%%%%%%%%%%%%%UPSTREAM_WIRE_BYTES_RECEIVED%%%%UPSTREAM_TLS_SESSION_ID%%%%%%%%%%%%%%%%%%%%%%%%REQUEST_TX_DURATION%%%%%%UPSTREAM_WIRE_BYTES_RECEIVED%%%%UPSTREAM_TLS_SESSION_ID%%%%%%UPSTREAM_TLS_SESSION_ID%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%E%START_TIME(/TIME%_%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%SSION_ID(%%%%%%%%%SSION_ID(%%%%_)%TA" + } + } + } + } + } + ignore_port_in_host_matching: true +} +headers { + headers { + key: ":path" + } + headers { + key: "x-forwarded-proto" + value: "2" + } +} From 4b473e8adb3de5b327b1e930efec26137fc23780 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 15 May 2023 15:30:03 -0400 Subject: [PATCH 237/740] grpc: moving files to extensions (#27402) Signed-off-by: Alyssa Wilk --- source/common/config/BUILD | 30 ------- .../extensions/config_subscription/grpc/BUILD | 32 ++++++- .../grpc}/delta_subscription_state.cc | 2 +- .../grpc}/delta_subscription_state.h | 4 +- .../grpc}/new_delta_subscription_state.cc | 2 +- .../grpc}/new_delta_subscription_state.h | 0 .../grpc/new_grpc_mux_impl.h | 2 +- .../grpc}/old_delta_subscription_state.cc | 2 +- .../grpc}/old_delta_subscription_state.h | 0 test/common/config/BUILD | 83 ------------------ .../config_subscription/common/BUILD | 2 +- .../common/subscription_impl_test.cc | 2 +- .../extensions/config_subscription/grpc/BUILD | 84 +++++++++++++++++++ .../grpc}/delta_subscription_impl_test.cc | 2 +- .../delta_subscription_state_old_test.cc | 2 +- .../grpc}/delta_subscription_state_test.cc | 2 +- .../grpc}/delta_subscription_test_harness.h | 0 17 files changed, 126 insertions(+), 125 deletions(-) rename source/{common/config => extensions/config_subscription/grpc}/delta_subscription_state.cc (98%) rename source/{common/config => extensions/config_subscription/grpc}/delta_subscription_state.h (90%) rename source/{common/config => extensions/config_subscription/grpc}/new_delta_subscription_state.cc (99%) rename source/{common/config => extensions/config_subscription/grpc}/new_delta_subscription_state.h (100%) rename source/{common/config => extensions/config_subscription/grpc}/old_delta_subscription_state.cc (99%) rename source/{common/config => extensions/config_subscription/grpc}/old_delta_subscription_state.h (100%) rename test/{common/config => extensions/config_subscription/grpc}/delta_subscription_impl_test.cc (99%) rename test/{common/config => extensions/config_subscription/grpc}/delta_subscription_state_old_test.cc (99%) rename test/{common/config => extensions/config_subscription/grpc}/delta_subscription_state_test.cc (99%) rename test/{common/config => extensions/config_subscription/grpc}/delta_subscription_test_harness.h (100%) diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 08a1ef61ebf5..3b9dc30c03f7 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -85,36 +85,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "delta_subscription_state_lib", - srcs = [ - "delta_subscription_state.cc", - "new_delta_subscription_state.cc", - "old_delta_subscription_state.cc", - ], - hdrs = [ - "delta_subscription_state.h", - "new_delta_subscription_state.h", - "old_delta_subscription_state.h", - ], - deps = [ - ":api_version_lib", - ":pausable_ack_queue_lib", - ":ttl_lib", - ":utility_lib", - ":watch_map_lib", - "//envoy/config:subscription_interface", - "//envoy/event:dispatcher_interface", - "//source/common/common:assert_lib", - "//source/common/common:backoff_lib", - "//source/common/common:minimal_logger_lib", - "//source/common/common:token_bucket_impl_lib", - "//source/common/grpc:common_lib", - "//source/common/protobuf", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_cc_library( name = "grpc_stream_lib", hdrs = ["grpc_stream.h"], diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD index 05d85d93ac21..ade30d4cb80f 100644 --- a/source/extensions/config_subscription/grpc/BUILD +++ b/source/extensions/config_subscription/grpc/BUILD @@ -43,11 +43,11 @@ envoy_cc_library( srcs = ["new_grpc_mux_impl.cc"], hdrs = ["new_grpc_mux_impl.h"], deps = [ + ":delta_subscription_state_lib", "//envoy/config:custom_config_validators_interface", "//envoy/config:xds_config_tracker_interface", "//envoy/event:dispatcher_interface", "//envoy/grpc:async_client_interface", - "//source/common/config:delta_subscription_state_lib", "//source/common/config:grpc_stream_lib", "//source/common/config:pausable_ack_queue_lib", "//source/common/config:watch_map_lib", @@ -124,3 +124,33 @@ envoy_cc_extension( "//source/common/protobuf:utility_lib", ], ) + +envoy_cc_library( + name = "delta_subscription_state_lib", + srcs = [ + "delta_subscription_state.cc", + "new_delta_subscription_state.cc", + "old_delta_subscription_state.cc", + ], + hdrs = [ + "delta_subscription_state.h", + "new_delta_subscription_state.h", + "old_delta_subscription_state.h", + ], + deps = [ + "//envoy/config:subscription_interface", + "//envoy/event:dispatcher_interface", + "//source/common/common:assert_lib", + "//source/common/common:backoff_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/common:token_bucket_impl_lib", + "//source/common/config:api_version_lib", + "//source/common/config:pausable_ack_queue_lib", + "//source/common/config:ttl_lib", + "//source/common/config:utility_lib", + "//source/common/config:watch_map_lib", + "//source/common/grpc:common_lib", + "//source/common/protobuf", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) diff --git a/source/common/config/delta_subscription_state.cc b/source/extensions/config_subscription/grpc/delta_subscription_state.cc similarity index 98% rename from source/common/config/delta_subscription_state.cc rename to source/extensions/config_subscription/grpc/delta_subscription_state.cc index 850dd565238d..85ca6425e3e2 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/extensions/config_subscription/grpc/delta_subscription_state.cc @@ -1,4 +1,4 @@ -#include "source/common/config/delta_subscription_state.h" +#include "source/extensions/config_subscription/grpc/delta_subscription_state.h" #include "source/common/runtime/runtime_features.h" diff --git a/source/common/config/delta_subscription_state.h b/source/extensions/config_subscription/grpc/delta_subscription_state.h similarity index 90% rename from source/common/config/delta_subscription_state.h rename to source/extensions/config_subscription/grpc/delta_subscription_state.h index ec57f555280c..ec0237fe51e2 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/extensions/config_subscription/grpc/delta_subscription_state.h @@ -6,8 +6,8 @@ #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/common/logger.h" -#include "source/common/config/new_delta_subscription_state.h" -#include "source/common/config/old_delta_subscription_state.h" +#include "source/extensions/config_subscription/grpc/new_delta_subscription_state.h" +#include "source/extensions/config_subscription/grpc/old_delta_subscription_state.h" #include "absl/container/flat_hash_set.h" #include "absl/types/variant.h" diff --git a/source/common/config/new_delta_subscription_state.cc b/source/extensions/config_subscription/grpc/new_delta_subscription_state.cc similarity index 99% rename from source/common/config/new_delta_subscription_state.cc rename to source/extensions/config_subscription/grpc/new_delta_subscription_state.cc index a9734dd08ff1..7798351f873a 100644 --- a/source/common/config/new_delta_subscription_state.cc +++ b/source/extensions/config_subscription/grpc/new_delta_subscription_state.cc @@ -1,4 +1,4 @@ -#include "source/common/config/new_delta_subscription_state.h" +#include "source/extensions/config_subscription/grpc/new_delta_subscription_state.h" #include "envoy/event/dispatcher.h" #include "envoy/service/discovery/v3/discovery.pb.h" diff --git a/source/common/config/new_delta_subscription_state.h b/source/extensions/config_subscription/grpc/new_delta_subscription_state.h similarity index 100% rename from source/common/config/new_delta_subscription_state.h rename to source/extensions/config_subscription/grpc/new_delta_subscription_state.h diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h index a434a487b8b2..fd694c542102 100644 --- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h @@ -11,12 +11,12 @@ #include "source/common/common/logger.h" #include "source/common/config/api_version.h" -#include "source/common/config/delta_subscription_state.h" #include "source/common/config/grpc_stream.h" #include "source/common/config/pausable_ack_queue.h" #include "source/common/config/watch_map.h" #include "source/common/grpc/common.h" #include "source/common/runtime/runtime_features.h" +#include "source/extensions/config_subscription/grpc/delta_subscription_state.h" namespace Envoy { namespace Config { diff --git a/source/common/config/old_delta_subscription_state.cc b/source/extensions/config_subscription/grpc/old_delta_subscription_state.cc similarity index 99% rename from source/common/config/old_delta_subscription_state.cc rename to source/extensions/config_subscription/grpc/old_delta_subscription_state.cc index 69461ae9158b..bd7cf9d68ca3 100644 --- a/source/common/config/old_delta_subscription_state.cc +++ b/source/extensions/config_subscription/grpc/old_delta_subscription_state.cc @@ -1,4 +1,4 @@ -#include "source/common/config/old_delta_subscription_state.h" +#include "source/extensions/config_subscription/grpc/old_delta_subscription_state.h" #include "envoy/event/dispatcher.h" #include "envoy/service/discovery/v3/discovery.pb.h" diff --git a/source/common/config/old_delta_subscription_state.h b/source/extensions/config_subscription/grpc/old_delta_subscription_state.h similarity index 100% rename from source/common/config/old_delta_subscription_state.h rename to source/extensions/config_subscription/grpc/old_delta_subscription_state.h diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 45a96d491346..48adc3bb329e 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -31,69 +31,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "delta_subscription_impl_test", - srcs = ["delta_subscription_impl_test.cc"], - deps = [ - ":delta_subscription_test_harness", - "//envoy/config:xds_config_tracker_interface", - "//source/common/config:api_version_lib", - "//source/common/stats:isolated_store_lib", - "//source/extensions/config_subscription/grpc:grpc_subscription_lib", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - -envoy_cc_test( - name = "delta_subscription_state_test", - srcs = ["delta_subscription_state_test.cc"], - deps = [ - "//source/common/config:delta_subscription_state_lib", - "//source/common/config/xds_mux:delta_subscription_state_lib", - "//source/common/stats:isolated_store_lib", - "//source/extensions/config_subscription/grpc:grpc_subscription_lib", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - "//test/test_common:test_runtime_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - -envoy_cc_test( - name = "delta_subscription_state_old_test", - srcs = ["delta_subscription_state_old_test.cc"], - deps = [ - "//source/common/config:delta_subscription_state_lib", - "//source/common/stats:isolated_store_lib", - "//source/extensions/config_subscription/grpc:grpc_subscription_lib", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - "//test/test_common:test_runtime_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "sotw_subscription_state_test", srcs = ["sotw_subscription_state_test.cc"], @@ -191,26 +128,6 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( - name = "delta_subscription_test_harness", - hdrs = ["delta_subscription_test_harness.h"], - deps = [ - ":subscription_test_harness", - "//envoy/config:xds_config_tracker_interface", - "//source/common/common:utility_lib", - "//source/common/grpc:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/config:custom_config_validators_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "opaque_resource_decoder_impl_test", srcs = ["opaque_resource_decoder_impl_test.cc"], diff --git a/test/extensions/config_subscription/common/BUILD b/test/extensions/config_subscription/common/BUILD index 6807a30b3676..183dd27d4301 100644 --- a/test/extensions/config_subscription/common/BUILD +++ b/test/extensions/config_subscription/common/BUILD @@ -43,10 +43,10 @@ envoy_cc_test( name = "subscription_impl_test", srcs = ["subscription_impl_test.cc"], deps = [ - "//test/common/config:delta_subscription_test_harness", "//test/common/config:grpc_subscription_test_harness", "//test/common/config:subscription_test_harness", "//test/extensions/config_subscription/filesystem:filesystem_subscription_test_harness", + "//test/extensions/config_subscription/grpc:delta_subscription_test_harness", "//test/extensions/config_subscription/rest:http_subscription_test_harness", ], ) diff --git a/test/extensions/config_subscription/common/subscription_impl_test.cc b/test/extensions/config_subscription/common/subscription_impl_test.cc index 1585d222abf0..329d14be13a9 100644 --- a/test/extensions/config_subscription/common/subscription_impl_test.cc +++ b/test/extensions/config_subscription/common/subscription_impl_test.cc @@ -1,10 +1,10 @@ #include #include -#include "test/common/config/delta_subscription_test_harness.h" #include "test/common/config/grpc_subscription_test_harness.h" #include "test/common/config/subscription_test_harness.h" #include "test/extensions/config_subscription/filesystem/filesystem_subscription_test_harness.h" +#include "test/extensions/config_subscription/grpc/delta_subscription_test_harness.h" #include "test/extensions/config_subscription/rest/http_subscription_test_harness.h" using testing::InSequence; diff --git a/test/extensions/config_subscription/grpc/BUILD b/test/extensions/config_subscription/grpc/BUILD index 6eafc78a4ca5..5ba88b5748df 100644 --- a/test/extensions/config_subscription/grpc/BUILD +++ b/test/extensions/config_subscription/grpc/BUILD @@ -1,6 +1,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_test", + "envoy_cc_test_library", "envoy_package", ) @@ -63,3 +64,86 @@ envoy_cc_test( "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) + +envoy_cc_test( + name = "delta_subscription_impl_test", + srcs = ["delta_subscription_impl_test.cc"], + deps = [ + ":delta_subscription_test_harness", + "//envoy/config:xds_config_tracker_interface", + "//source/common/config:api_version_lib", + "//source/common/stats:isolated_store_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + +envoy_cc_test( + name = "delta_subscription_state_test", + srcs = ["delta_subscription_state_test.cc"], + deps = [ + "//source/common/config/xds_mux:delta_subscription_state_lib", + "//source/common/stats:isolated_store_lib", + "//source/extensions/config_subscription/grpc:delta_subscription_state_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:test_runtime_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + +envoy_cc_test( + name = "delta_subscription_state_old_test", + srcs = ["delta_subscription_state_old_test.cc"], + deps = [ + "//source/common/stats:isolated_store_lib", + "//source/extensions/config_subscription/grpc:delta_subscription_state_lib", + "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:test_runtime_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + +envoy_cc_test_library( + name = "delta_subscription_test_harness", + hdrs = ["delta_subscription_test_harness.h"], + deps = [ + "//envoy/config:xds_config_tracker_interface", + "//source/common/common:utility_lib", + "//source/common/grpc:common_lib", + "//test/common/config:subscription_test_harness", + "//test/mocks/config:config_mocks", + "//test/mocks/config:custom_config_validators_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/extensions/config_subscription/grpc/delta_subscription_impl_test.cc similarity index 99% rename from test/common/config/delta_subscription_impl_test.cc rename to test/extensions/config_subscription/grpc/delta_subscription_impl_test.cc index 48bd147168e4..2b435f510aac 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/extensions/config_subscription/grpc/delta_subscription_impl_test.cc @@ -6,7 +6,7 @@ #include "source/common/buffer/zero_copy_input_stream_impl.h" #include "source/common/config/api_version.h" -#include "test/common/config/delta_subscription_test_harness.h" +#include "test/extensions/config_subscription/grpc/delta_subscription_test_harness.h" namespace Envoy { namespace Config { diff --git a/test/common/config/delta_subscription_state_old_test.cc b/test/extensions/config_subscription/grpc/delta_subscription_state_old_test.cc similarity index 99% rename from test/common/config/delta_subscription_state_old_test.cc rename to test/extensions/config_subscription/grpc/delta_subscription_state_old_test.cc index f5fa768901c3..0206162d55b0 100644 --- a/test/common/config/delta_subscription_state_old_test.cc +++ b/test/extensions/config_subscription/grpc/delta_subscription_state_old_test.cc @@ -4,9 +4,9 @@ #include "envoy/config/xds_config_tracker.h" #include "envoy/service/discovery/v3/discovery.pb.h" -#include "source/common/config/delta_subscription_state.h" #include "source/common/config/utility.h" #include "source/common/stats/isolated_store_impl.h" +#include "source/extensions/config_subscription/grpc/delta_subscription_state.h" #include "test/mocks/config/mocks.h" #include "test/mocks/event/mocks.h" diff --git a/test/common/config/delta_subscription_state_test.cc b/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc similarity index 99% rename from test/common/config/delta_subscription_state_test.cc rename to test/extensions/config_subscription/grpc/delta_subscription_state_test.cc index 4f113cb57762..899fb1e55dab 100644 --- a/test/common/config/delta_subscription_state_test.cc +++ b/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc @@ -4,10 +4,10 @@ #include "envoy/config/xds_config_tracker.h" #include "envoy/service/discovery/v3/discovery.pb.h" -#include "source/common/config/delta_subscription_state.h" #include "source/common/config/utility.h" #include "source/common/config/xds_mux/delta_subscription_state.h" #include "source/common/stats/isolated_store_impl.h" +#include "source/extensions/config_subscription/grpc/delta_subscription_state.h" #include "test/mocks/config/mocks.h" #include "test/mocks/event/mocks.h" diff --git a/test/common/config/delta_subscription_test_harness.h b/test/extensions/config_subscription/grpc/delta_subscription_test_harness.h similarity index 100% rename from test/common/config/delta_subscription_test_harness.h rename to test/extensions/config_subscription/grpc/delta_subscription_test_harness.h From fef1d5ae37e88795c8bc06b7252d8f9b931284b2 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 15 May 2023 16:59:29 -0400 Subject: [PATCH 238/740] flags: updating comment (#27408) Signed-off-by: Alyssa Wilk --- source/common/runtime/runtime_features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 55c7e468324a..a796495ea3c2 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -96,7 +96,7 @@ FALSE_RUNTIME_GUARD(envoy_reloadable_features_unified_mux); FALSE_RUNTIME_GUARD(envoy_reloadable_features_defer_processing_backedup_streams); // TODO(birenroy) flip after a burn-in period FALSE_RUNTIME_GUARD(envoy_reloadable_features_http2_use_oghttp2); -// TODO(bencebeky): Finish BalsaParser implementation, then enable by default. See issue #21245. +// TODO(bencebeky): Flip true after sufficient canarying. FALSE_RUNTIME_GUARD(envoy_reloadable_features_http1_use_balsa_parser); // Used to track if runtime is initialized. FALSE_RUNTIME_GUARD(envoy_reloadable_features_runtime_initialized); From 32accf029a7193c5fdd9523ea1953a61d7203aec Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 15 May 2023 19:37:44 -0400 Subject: [PATCH 239/740] UHV: allow validation of HTTP/1.0 requests (#27403) Signed-off-by: Yan Avlasov --- source/common/http/conn_manager_impl.cc | 44 +++++++++---------- .../http_grpc_access_log_integration_test.cc | 5 --- .../filters/http/lua/lua_integration_test.cc | 3 -- test/integration/integration_test.cc | 30 ------------- test/integration/xds_integration_test.cc | 5 --- 5 files changed, 22 insertions(+), 65 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index d833a4e3e787..551c64212041 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1088,6 +1088,28 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt // We end the decode here to mark that the downstream stream is complete. maybeEndDecode(end_stream); + // Make sure we are getting a codec version we support. + if (protocol == Protocol::Http10) { + // Assume this is HTTP/1.0. This is fine for HTTP/0.9 but this code will also affect any + // requests with non-standard version numbers (0.9, 1.3), basically anything which is not + // HTTP/1.1. + // + // The protocol may have shifted in the HTTP/1.0 case so reset it. + filter_manager_.streamInfo().protocol(protocol); + if (!connection_manager_.config_.http1Settings().accept_http_10_) { + // Send "Upgrade Required" if HTTP/1.0 support is not explicitly configured on. + sendLocalReply(Code::UpgradeRequired, "", nullptr, absl::nullopt, + StreamInfo::ResponseCodeDetails::get().LowVersion); + return; + } + if (!request_headers_->Host() && + !connection_manager_.config_.http1Settings().default_host_for_http_10_.empty()) { + // Add a default host if configured to do so. + request_headers_->setHost( + connection_manager_.config_.http1Settings().default_host_for_http_10_); + } + } + if (!validateHeaders()) { ENVOY_STREAM_LOG(debug, "request headers validation failed\n{}", *this, *request_headers_); return; @@ -1141,28 +1163,6 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt connection_manager_.stats_.prefixStatName(), connection_manager_.stats_.scope_); - // Make sure we are getting a codec version we support. - if (protocol == Protocol::Http10) { - // Assume this is HTTP/1.0. This is fine for HTTP/0.9 but this code will also affect any - // requests with non-standard version numbers (0.9, 1.3), basically anything which is not - // HTTP/1.1. - // - // The protocol may have shifted in the HTTP/1.0 case so reset it. - filter_manager_.streamInfo().protocol(protocol); - if (!connection_manager_.config_.http1Settings().accept_http_10_) { - // Send "Upgrade Required" if HTTP/1.0 support is not explicitly configured on. - sendLocalReply(Code::UpgradeRequired, "", nullptr, absl::nullopt, - StreamInfo::ResponseCodeDetails::get().LowVersion); - return; - } - if (!request_headers_->Host() && - !connection_manager_.config_.http1Settings().default_host_for_http_10_.empty()) { - // Add a default host if configured to do so. - request_headers_->setHost( - connection_manager_.config_.http1Settings().default_host_for_http_10_); - } - } - if (!request_headers_->Host()) { // Require host header. For HTTP/1.1 Host has already been translated to :authority. sendLocalReply(Code::BadRequest, "", nullptr, absl::nullopt, diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index d871b2eed496..07e6d7a5191a 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -293,11 +293,6 @@ name: router // Verify the grpc cached logger is available after the initial logger filter is destroyed. // Regression test for https://github.com/envoyproxy/envoy/issues/18066 TEST_P(AccessLogIntegrationTest, GrpcLoggerSurvivesAfterReloadConfig) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23287) - Determine HTTP/0.9 and HTTP/1.0 support within UHV - return; -#endif - config_helper_.disableDelayClose(); autonomous_upstream_ = true; // The grpc access logger connection never closes. It's ok to see an incomplete logging stream. diff --git a/test/extensions/filters/http/lua/lua_integration_test.cc b/test/extensions/filters/http/lua/lua_integration_test.cc index 5a2b6b4b6521..f60ac3d40118 100644 --- a/test/extensions/filters/http/lua/lua_integration_test.cc +++ b/test/extensions/filters/http/lua/lua_integration_test.cc @@ -252,11 +252,8 @@ name: lua initializeFilter(FILTER_AND_CODE, "foo"); std::string response; -#ifndef ENVOY_ENABLE_UHV - // TODO(#23287) - Determine HTTP/0.9 and HTTP/1.0 support within UHV sendRawHttpAndWaitForResponse(lookupPort("http"), "GET / HTTP/1.0\r\n\r\n", &response, true); EXPECT_TRUE(response.find("HTTP/1.1 426 Upgrade Required\r\n") == 0); -#endif response = ""; sendRawHttpAndWaitForResponse(lookupPort("http"), "GET / HTTP/1.1\r\n\r\n", &response, true); diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index 618bf95aac2a..8967cabaf55a 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -1121,11 +1121,6 @@ TEST_P(IntegrationTest, BadHeader) { } TEST_P(IntegrationTest, Http10Disabled) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23287) - Determine HTTP/0.9 and HTTP/1.0 support within UHV - return; -#endif - initialize(); std::string response; sendRawHttpAndWaitForResponse(lookupPort("http"), "GET / HTTP/1.0\r\n\r\n", &response, true); @@ -1133,11 +1128,6 @@ TEST_P(IntegrationTest, Http10Disabled) { } TEST_P(IntegrationTest, Http10DisabledWithUpgrade) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23287) - Determine HTTP/0.9 and HTTP/1.0 support within UHV - return; -#endif - initialize(); std::string response; sendRawHttpAndWaitForResponse(lookupPort("http"), "GET / HTTP/1.0\r\nUpgrade: h2c\r\n\r\n", @@ -1147,11 +1137,6 @@ TEST_P(IntegrationTest, Http10DisabledWithUpgrade) { // Turn HTTP/1.0 support on and verify 09 style requests work. TEST_P(IntegrationTest, Http09Enabled) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23287) - Determine HTTP/0.9 and HTTP/1.0 support within UHV - return; -#endif - useAccessLog(); autonomous_upstream_ = true; config_helper_.addConfigModifier(&setAllowHttp10WithDefaultHost); @@ -1177,11 +1162,6 @@ TEST_P(IntegrationTest, Http09WithKeepalive) { return; } -#ifdef ENVOY_ENABLE_UHV - // TODO(#23287) - Determine HTTP/0.9 and HTTP/1.0 support within UHV - return; -#endif - useAccessLog(); autonomous_upstream_ = true; config_helper_.addConfigModifier(&setAllowHttp10WithDefaultHost); @@ -1198,11 +1178,6 @@ TEST_P(IntegrationTest, Http09WithKeepalive) { // Turn HTTP/1.0 support on and verify the request is proxied and the default host is sent upstream. TEST_P(IntegrationTest, Http10Enabled) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23287) - Determine HTTP/0.9 and HTTP/1.0 support within UHV - return; -#endif - autonomous_upstream_ = true; config_helper_.addConfigModifier(&setAllowHttp10WithDefaultHost); initialize(); @@ -1404,11 +1379,6 @@ TEST_P(IntegrationTest, PipelineWithTrailers) { // an inline sendLocalReply to make sure the "kick" works under the call stack // of dispatch as well as when a response is proxied from upstream. TEST_P(IntegrationTest, PipelineInline) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23287) - Determine HTTP/0.9 and HTTP/1.0 support within UHV - return; -#endif - config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { hcm.mutable_stream_error_on_invalid_http_message()->set_value(true); }); diff --git a/test/integration/xds_integration_test.cc b/test/integration/xds_integration_test.cc index f433b916fb69..41ee5759b0ef 100644 --- a/test/integration/xds_integration_test.cc +++ b/test/integration/xds_integration_test.cc @@ -667,11 +667,6 @@ INSTANTIATE_TEST_SUITE_P(Protocols, LdsIntegrationTest, // Sample test making sure our config framework correctly reloads listeners. TEST_P(LdsIntegrationTest, ReloadConfig) { -#ifdef ENVOY_ENABLE_UHV - // TODO(#23287) - Determine HTTP/0.9 and HTTP/1.0 support within UHV - return; -#endif - config_helper_.disableDelayClose(); autonomous_upstream_ = true; initialize(); From a6f31aaaaa75c6d22f73dcc65f650ce557a51746 Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Tue, 16 May 2023 05:43:28 +0300 Subject: [PATCH 240/740] stats: change extractAndAppendTags() implementation of IsolatedStatsStore to no-op (#27388) Signed-off-by: ohadvano --- source/common/stats/isolated_store_impl.h | 9 ++------- test/integration/server.h | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index 3e642b88888d..1d29b90b08e9 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -234,13 +234,8 @@ class IsolatedStoreImpl : public Store { return constRootScope()->iterate(fn); } - void extractAndAppendTags(StatName, StatNamePool&, StatNameTagVector&) override { - IS_ENVOY_BUG("Unexpected call to a function that is not yet implemented"); - } - - void extractAndAppendTags(absl::string_view, StatNamePool&, StatNameTagVector&) override { - IS_ENVOY_BUG("Unexpected call to a function that is not yet implemented"); - } + void extractAndAppendTags(StatName, StatNamePool&, StatNameTagVector&) override {} + void extractAndAppendTags(absl::string_view, StatNamePool&, StatNameTagVector&) override {} protected: /** diff --git a/test/integration/server.h b/test/integration/server.h index e96335323be2..657396ccf4eb 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -357,13 +357,8 @@ class TestIsolatedStoreImpl : public StoreRoot { bool iterate(const IterateFn& fn) const override { return store_.iterate(fn); } bool iterate(const IterateFn& fn) const override { return store_.iterate(fn); } - void extractAndAppendTags(StatName, StatNamePool&, StatNameTagVector&) override { - IS_ENVOY_BUG("Unexpected call to a function that is not yet implemented"); - }; - - void extractAndAppendTags(absl::string_view, StatNamePool&, StatNameTagVector&) override { - IS_ENVOY_BUG("Unexpected call to a function that is not yet implemented"); - }; + void extractAndAppendTags(StatName, StatNamePool&, StatNameTagVector&) override{}; + void extractAndAppendTags(absl::string_view, StatNamePool&, StatNameTagVector&) override{}; // Stats::StoreRoot void addSink(Sink&) override {} From 911fd1642607781b80d8876baf96196b5f034506 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 16 May 2023 05:24:57 +0100 Subject: [PATCH 241/740] deps: Bump `python` -> 3.11.1 (#25359) Signed-off-by: Ryan Northey --- api/tools/generate_listeners_test.py | 4 ++++ api/tools/tap2pcap_test.py | 3 +++ bazel/python_dependencies.bzl | 2 +- bazel/repositories_extra.bzl | 8 ++++++-- bazel/repository_locations.bzl | 2 ++ bazel/rules_fuzzing.patch | 18 +++++++++++++++++- bazel/v8.patch | 13 +++++++++++++ tools/gsutil/BUILD | 3 ++- ..._crcfunext.cpython-310-x86_64-linux-gnu.so | Bin 53824 -> 0 bytes ..._crcfunext.cpython-311-x86_64-linux-gnu.so | Bin 0 -> 55536 bytes tools/gsutil/vendor_util.sh | 11 ++++++----- 11 files changed, 54 insertions(+), 10 deletions(-) delete mode 100755 tools/gsutil/crcmod/_crcfunext.cpython-310-x86_64-linux-gnu.so create mode 100755 tools/gsutil/crcmod/_crcfunext.cpython-311-x86_64-linux-gnu.so diff --git a/api/tools/generate_listeners_test.py b/api/tools/generate_listeners_test.py index 1defb3f66698..5e214c1172de 100644 --- a/api/tools/generate_listeners_test.py +++ b/api/tools/generate_listeners_test.py @@ -1,6 +1,10 @@ """Tests for generate_listeners.""" import os +import sys + +# Workaround for https://github.com/bazelbuild/rules_python/issues/1221 +sys.path += [os.path.dirname(__file__)] import generate_listeners diff --git a/api/tools/tap2pcap_test.py b/api/tools/tap2pcap_test.py index c0151846f5e1..504b49aa4335 100644 --- a/api/tools/tap2pcap_test.py +++ b/api/tools/tap2pcap_test.py @@ -5,6 +5,9 @@ import subprocess as sp import sys +# Workaround for https://github.com/bazelbuild/rules_python/issues/1221 +sys.path += [os.path.dirname(__file__)] + import tap2pcap # Validate that the tapped trace when run through tap2cap | tshark matches diff --git a/bazel/python_dependencies.bzl b/bazel/python_dependencies.bzl index 37c0183664f5..0033a5364547 100644 --- a/bazel/python_dependencies.bzl +++ b/bazel/python_dependencies.bzl @@ -1,5 +1,5 @@ load("@rules_python//python:pip.bzl", "pip_parse") -load("@python3_10//:defs.bzl", "interpreter") +load("@python3_11//:defs.bzl", "interpreter") def envoy_python_dependencies(): pip_parse( diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl index f7ec05fda768..877392ebf473 100644 --- a/bazel/repositories_extra.bzl +++ b/bazel/repositories_extra.bzl @@ -4,8 +4,12 @@ load("@proxy_wasm_cpp_host//bazel/cargo/wasmtime:crates.bzl", "wasmtime_fetch_re load("//bazel/external/cargo:crates.bzl", "raze_fetch_remote_crates") load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies") +def _python_minor_version(python_version): + return "_".join(python_version.split(".")[:-1]) + # Python version for `rules_python` -PYTHON_VERSION = "3.10.9" +PYTHON_VERSION = "3.11.1" +PYTHON_MINOR_VERSION = _python_minor_version(PYTHON_VERSION) # Envoy deps that rely on a first stage of dependency loading in envoy_dependencies(). def envoy_dependencies_extra( @@ -17,7 +21,7 @@ def envoy_dependencies_extra( # Registers underscored Python minor version - eg `python3_10` python_register_toolchains( - name = "python%s" % ("_".join(python_version.split(".")[:-1])), + name = "python%s" % _python_minor_version(python_version), python_version = python_version, ignore_root_user_error = ignore_root_user_error, ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 474a02f9d3e8..be93fc612f36 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -72,6 +72,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Fuzzing Rules for Bazel", project_desc = "Bazel rules for fuzz tests", project_url = "https://github.com/bazelbuild/rules_fuzzing", + # Patch contains workaround for https://github.com/bazelbuild/rules_python/issues/1221 version = "0.3.2", sha256 = "f85dc70bb9672af0e350686461fe6fdd0d61e10e75645f9e44fedf549b21e369", strip_prefix = "rules_fuzzing-{version}", @@ -1026,6 +1027,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_desc = "Google’s open source high-performance JavaScript and WebAssembly engine, written in C++", project_url = "https://v8.dev", # NOTE: Update together with com_googlesource_chromium_base_trace_event_common. + # Patch contains workaround for https://github.com/bazelbuild/rules_python/issues/1221 version = "10.7.193.13", # Static snapshot created using https://storage.googleapis.com/envoyproxy-wee8/wee8-fetch-deps.sh. sha256 = "2170df76ce5d7ecd7fb8d131370d210152f200273cba126f06d8b88fb53c9fbc", diff --git a/bazel/rules_fuzzing.patch b/bazel/rules_fuzzing.patch index eca1b56e4d52..3c1efa1b77e5 100644 --- a/bazel/rules_fuzzing.patch +++ b/bazel/rules_fuzzing.patch @@ -10,4 +10,20 @@ index e5e9dc4..a3bb1b8 100644 + tar -czhf "{output}" -C "$STAGING_DIR" . """.format( base_name = ctx.attr.base_name, - binary_path = binary_info.binary_file.path, \ No newline at end of file + binary_path = binary_info.binary_file.path, +diff --git a/fuzzing/tools/validate_dict.py b/fuzzing/tools/validate_dict.py +index d561e68..24e3adc 100644 +--- a/fuzzing/tools/validate_dict.py ++++ b/fuzzing/tools/validate_dict.py +@@ -19,6 +19,11 @@ Validates and merges a set of fuzzing dictionary files into a single output. + + from absl import app + from absl import flags ++ ++import os ++import sys ++sys.path += [os.path.dirname(__file__)] ++ + from dict_validation import validate_line + from sys import stderr + diff --git a/bazel/v8.patch b/bazel/v8.patch index 78a8411ad46e..9805eb749589 100644 --- a/bazel/v8.patch +++ b/bazel/v8.patch @@ -93,3 +93,16 @@ index ce3f569fd5..dc8a4c4f6a 100644 } // extern "C" + +#endif +diff --git a/third_party/inspector_protocol/code_generator.py b/third_party/inspector_protocol/code_generator.py +index c3768b8..d4a1dda 100644 +--- a/third_party/inspector_protocol/code_generator.py ++++ b/third_party/inspector_protocol/code_generator.py +@@ -16,6 +16,8 @@ try: + except ImportError: + import simplejson as json + ++sys.path += [os.path.dirname(__file__)] ++ + import pdl + + try: diff --git a/tools/gsutil/BUILD b/tools/gsutil/BUILD index 6f265485df9c..c02f0f643451 100644 --- a/tools/gsutil/BUILD +++ b/tools/gsutil/BUILD @@ -1,4 +1,5 @@ load("//bazel:envoy_build_system.bzl", "envoy_package") +load("//bazel:repositories_extra.bzl", "PYTHON_MINOR_VERSION") load("@base_pip3//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library") @@ -11,7 +12,7 @@ envoy_package() # TODO(phlax): figure a way to keep this updated (eg requirements.txt) py_library( name = "crcmod", - data = ["crcmod/_crcfunext.cpython-310-x86_64-linux-gnu.so"], + data = ["crcmod/_crcfunext.cpython-%s-x86_64-linux-gnu.so" % PYTHON_MINOR_VERSION.replace("_", "")], ) py_binary( diff --git a/tools/gsutil/crcmod/_crcfunext.cpython-310-x86_64-linux-gnu.so b/tools/gsutil/crcmod/_crcfunext.cpython-310-x86_64-linux-gnu.so deleted file mode 100755 index 9ed6a477524d2cc3f3734a038f892fbbfbac3340..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53824 zcmeHw3wTu3wf{LYnPd(LlgSIhJ4(=iCM1D`paet0BPfp$5=EMavdDOw-Vp)0l5iZc~n6>Qq79^9n#xXVe&(_+M;H zW!ki4(Gci?QG(K`;gvkK;UZAwQ&X2}U#Gg9>T!GJR2f%F+UpdwPVM#5QFOFc%8xD| zoNiT}xIuSD$>jhU)pO4mdiRRK=^mvzow}u%Wh7P)5QAaTBE}kqPE+eWr|PAq#j2i8 zwYaO_h`0Ye?KP=--U?xynqH>b)2VK673z^*{@P2MxmmS0k{#Tt-#S(5WqlG4@|h(o zS`CoxAz75UxjbLxsr=}juP*4TdaxiedgYwPyQHqnPR$xwwh1$PX?>f&Cp% z8NQ`0%YE&^Xs|OBiwC1E%Nx2Q;b4outve`X!e`Td#%FdheD%?!+PZD((zlz z&2l|xbbJ;vec<)%DH<~5YdJ8CecM5)!8qh^4__d0@Bn5vUPx#iQ;G>GZF$G_j zEA-E;l$7QM%LLRK^s2PMt|6b_uBABZTK7;zOYF$ z)TVu071{85ZY0VQ8=lu)Qr#%G;blIQf*Knh12J*c+3;#<$7PK+d{%FMIMI8?SoX-MHQL^D_*iam${#W8j6x ztDbg$j~g1OdL8H^1JizuBzuZM^j%bPf$S;C_C83-zmt-GL+2ZJ)P7tCz3*@xp=9pS zcmYgNhGH^>HEB@g9W8@GJ~fgZ+^6ppj0y>WZmv=ZjJuy{kop2i)^irtOdUSEc^ zaM_N9#omK2p@}VfvakLPQC)diaiIZ|W^Wmw?Dl!ZjoX*FKiu>cg`Pzd{6j6=U2`n*{W_QHEgdr_z2YkpKFm;?D^eO3A&Kb z)hjwxepjM=fs}Vn6h0So{X>_s&cWF=UCQ^V@-wCUGble;2Dqc6EPKk;5HB%am3?xk z!e3{fYVaZZyiL7K+9w017-FAXvdkeq{K}O1mC|CyfTr8>Gye|2QxJ zG13lZfirQkkDVPPftck6*kkEj5~!+8>2V{f5{qV}7&rS8`mPYVQHoBL-&TCc8EP_ z936ZFZBmWLi6q4y&!UX&@dXr$J;H;>OsUNtKZZSIxL~yGeXCFg%dE6p#3b)ZJad32 zo1`B=s!47nA=o5(p4sktcn%3x^#f7^^!fTxvCHE~VV5Tpdc95PqKZzH-u%-engdTl=Ab)E*AnGF`liV3V-c*X$L3L_FL4^qE?uyk|7@B$Erpjc2u3H^miK7#Q4cY3OhZXu*17mO!tE^)a4poetV+)c`5%xfs`Nl zkm{?s9yMGKOZl0q{(dPxhwQW)u<)_Pv&V?hUu&PI!QpSSPgKNf|PZTNalbml>3Ed1ur^=J_&2%YW znJ@Nf=K8kzCP&I2ROOXYzJ}~G2JjI3Od`S&?3124-xPzxVEbT^pS|T6<7fZ-Qe!y# zyos_A?4#CfRXl*QA6;0SPHVR0l18uFI%MMMn3cA&14Ar5v2Gg=-r0{{R-AsY4p6Fz zhFP~gU&;1)9Vdn~F8=tQ*k?ad*eAJedsyhMRdlL6xo+Dg<=@B^K96vH+q&%%DIcZk z-zepuA^WTYJj6bqB}RX(eXa(Fzs)|Cs>X2kS&Xs~?4#CfHFo>_IBB26Jk+uGEw>CL znCKj3qIdg#MXta8h&IukfKp9#T_Vh3UPIiOUcvUMS9(2cj0lfC5_>w_Crjp5b@_oIyMGiZIl(+-cH=i^u(#EZSzyB8K`G@36a?NjNXZBO>@ zJ=wd{(zjCS+#mG4Mf!aGFpRSMvf_+`Zg6kJBt~nttZB598+O05b0+KaK~e+8%|6s8 zKDQ$sG;aEZu36Ek^5nQ#CFSdJ+)I2ma(&ylnJeY{Rrz^R&f{i2;Gye-L+4;iXxpDY z^#KkbuyK9}j1HZJD(Df-Q&{l0F~HPG2aw0F;~F@wf#VuDu7TqkIIe-?8aS?j;~F@w zf#VuDu7Tqk`2SP`E6<8eHA3Nac(Jv;sG+%`D9*3Ku3QTA%0~KEkAJooKnbj2L`{^F zifWZewsJOSD`!h~RwZX=RZ6y6N~=*SS8F&Fh_nYwBmClQtf;3i7B6ZG76rqEZU@*A zi3W=T{_a3uw?B^8U5orAr#6SwD~k2?_C}&y9PbJiwehRBMGKmi7Da=X^o63qc2z;% zqfImu1HhPUwg9{1|AierI4nzbi?&9|5-sI2`C7zm99*xCV}E;J5~kYv8yBj%(n!299gs z|5O9|^K$z0aagMIMW4Z=EAbf~wTaN5-8ovs#RguY<4b=I4=*|LMXzSlg_rc?QlH18 zy$fBs{DV&hB9uR@_B}4j;5vZU=IGL&MJl9s$8n94#JEz)r$0M|m(KW7Wca|5%a=dfL zd963q{Qr5PKhK^jNV&3mol2WjdXY+dRk}%~H>h-%N*_|`9+jr@*ZECN8X7*;`W(JA ziPfLm)EL-(5i_&rhb7f11{pG&G!6 zRMOhk7moK8%`KZ7rb#Qc*(Gq!Ks`4mi_aCw)S4edyx`N z+~%LM5z=Ph&Bfn>V7Q8oGDJA%i(}`a-09Akh;iyn%Qyi^S{+a>x6^e07lK3@#WSZP z<6$s#n(03Q%IR>;o{Wf<<_d-5hT}AZmb4jTsFu_1dK-b!@c^P}TImGh?f74GA??<2 zn+ZPT1$^5ChEGBJX?q#YaDSI7o9Qo6g9N+ZMErW$RpD+%ZR;>dU9&Pco>=rcw5!_s zHc?qmgUmIDGhXXGaCg;krqH?-Z#B8*a;C`gAey>P<4lS57W56Et?YVdxY>oa3-M9Hnz5$ezR zfb}3Ex$8n}8R|S_?Z**~%QvbNna8a4)a{E{wmnuEP_8!jePHmk;dl$|(lf^W5Xeyr z;f1bKJeh5HFD=8}0fG!S6=adYNUqUjZ^B-POlB3t%5d)>@DJdU;l33h)BdNX%>(vX z)JP+YqZC%oSf2MaAUpx8vCZ=m@EPm_o|EB!8P_t#(+dM+Y!~>O=%agl7Wqcjw?Uo> zJ!NAUAn}|CqLzB+5w4Knp)17uT+|9cYB_5ZPS!Izh zF1R#n-!wd{u(HU$+e5E_c~+sB>|N6QJy37ffz z_HV%d4@j#N|Gax#mmq=e&_(aw95QF2btQb+yU$DV z7g-^oy!$3oro=iQ|Gm$8mtl=jZe_qvyx(WC8jGa!K9@sZg-~ZbNUWdF`4TdX)} z-X9eF5G^!WPeV=KA9?npWR10r7`!m%MU-4*T}%?ZSokv}UDiYld++|SuOQQFEr!c` zU-G<)IvcFJU@z~_awwW@vVKgNmzmpEs{$7D{yg`3klkR7gKoUP$bA->TdjU#{Yw6? zk?gV_z9C=4kG%#Nbckb|Pb#xrfn$_s`jPp;fn;dx!}Cl3P#8Fmuz$Dxc)8K(5ft?IigI z#71Tw>C{(-?3W?gi_E+?sb6l%cA0q_NS&5sd(FIeFe<&H zOm>$IX5OP@$I)hP9lE;7%=V zmV0YhH$i?!cODhEF{a5lj{ih0M~{;nJHOBov&gI2!yR$Y0>FlK5WRKuF$=HZm{ugCua(m&l}=pT^8A&p>1J5=cS_Ly%0&TxMR|CyG&k!huZ z!%3u%kNG82y{4(imy%S9bRxXT@ty2+C?mt2h5rQ+G&uo)A~=J({=}G*nV^&iXeiE0 z61+kpJz3Dm1S^PuByH0KV~E^vTn+U)zUv7yRh+1(ogIUyRExhLP3`sE#-w*^(#H~{ zNzFLE51({=FZ*dG{edQZbr7lHc#SOdboN_Jby!o;I1_a&gCvII2$|uTY%kCm?lJh! z5|Wb&6>}BR3JeU#bKcXLbOHX4rHC{}3eSbtIiB~f;lc~4kb;ux+hg!h#}5knxbRZ^ zXJt{$kVYsb8;)s2^}_=agIMA2s?gU zKu+cO2C@4|!CTCNhIqk`K}pd>RoqKdFBD{=sSI}x{V||1KbxbCeSNQ<39p3}?sx2z&?0@lrvQo7te7p&&7a22-*v!%>T2@Az54 zolJV4CVezP%1-5RkN|%gPb1J%k1&E+kl||)V;z5C*y|zq0>>B{L*B1ohIOo!Ora}{ zy5zXj;Ta2o$Grjj8{?ltAiQGS6Ywk3nD9j!99KA;AD~_LPDtP6t?=#z@kBf}HN2IX zZ1IVBvTAr|Nz|O!9Sj@Z*)qvSqPEr*&Zx0X_PKAkf)){~IUJC8GU)*On2wJbp+tVRlP;H@%{&}hV>!MgJI6Ntwl7`L^$KMeoC`lFJ}s^ zN2p7eaHhz5lctI&PrfDA&(W?omi<|9E4O|K!+PVKsj*(7p7e31&iXRk+`CR@7ps`& z&h=cA4Z zr;k~8!nWRPxMYvDkb3$#&OB#bpN-5`&b(lKmFjHc%*)o7$UUy*4Aug~;PafpT7Y!D zgENP%Eb8`koH?3XLG$JHoWWXP6`5ft_W^4GdOGj@0+(Se&_TVpiF1YcKB{eqINqC` z=R;HZP2O9qpOLknqjqi`O;d?s`Dh^A#uePw1j^i=OLK_Vx|5vZOPndR7Lr2lU?mn= z)zsvj1vHbCSpP#Nxr0x~z?)ru&s#M@WW$S8}CM=7Bt#lrTltkc0k*^Vg{CIFIUlrSMeL&u~}be*uMz zvj8N6))>m7n1W?~k@stC9P0$C`E}N!VQr_(H$W&rwA+x5&FqZ zA91ciF5m0@ruAI_*7Y>{zLo#qXxOlRMRwa=m4MjOq-Lx#U7ND7&;#^vE8`d8wGcD&-gbla#doav7DJMdBnjTt0p(FL$}-m(m8P8xm!5H~#5aC`<1 zk+mVb`Pl-0h_>>*0#86Z$j=dYGc25+D=@axMt+{aV<2aKzQ9i+*5wxn{2q*x zKStml#La>+GKsGMzF;iFy!rnM%CZd)ZKTKK=k7wzGY@(hJ3+`Z5EsWz6!-v|DvSuc z4fw)dVbcoz(J~L){NEz+ZpQE!SD1DJ;oO&kS<(1b_o?V3FWtwVtv09s3oVU5C-Wi{ z6dIn((C-OZ!jPPx5oor>fZ_RX4DbmnC(yx)<5yIxH1|Qso7DnDucm|G)4*fmC=u{p z=x$;LkyaQx^(63p}T;6!?7Dduo%wwTLZKR|z~F@=skYFzqd-wg?;`KCKD{eX(k{ zmte2p-++FLa|B*Rc%_qd&{eV0$(!)qX-W^DMSrJd3jJ=#cG99k3e}$V(C10Z1pfkL zIBB`S{|+0Sv_jxp&{mFzVMjaYbMjvV@L< zYMqaMmMqKv4(b@zIB+Ogo=sCiiM0(qDp|pqa_dP9>yni$O^x**lxgxdp-!FkH5j*K z)kI>}Xk7_Qlr(#3bJb*-B=qV`+IX(9ZbK|6Y007u)kP5g`>5;r8EiGZN_3o#rl+?H zd>{O4dQjlsq23JTn(gHk52NBUDtKkuwdwHLUl*s;Z7NZfgeydDVh(r=Y}Z)k40ih52WIL7`QK zidBo$uH!CMfXYbJ?vE@v*Xj>4&{&f`p%h402Ts(fR{ zp`Bjqd5rn0z*ySX++e*&;slxOR_jS>r;}ydW!+Di5SKh)-AA3fm=Dt)vVMwbv#OiP z9<%DermBa@_E?{zcCN{y{rYoO2r;edb37DYu*MSEHm>us^)&T#JJ&g2If?b>Idjk& zK#Z%pjx&d?Tw-v&ht>^8r-s^%sv8-`lKHpP?VGZmK~*=F#jnF$v&ZDOAzSPV#SGd2 z8damSV?boM`|+RmWVZki7bJ^~iGYZ+hWmHm=dGIT{Vpi19i)=!EUIDAK|s}v0xF<0 zC}QFGDF#s0DNcgj5ooEXs??i@{#lhI)@+s@8M4Og@f1Pa)}Kf*vrnRo*UH0~sH&bw zRcM{}BJpe_o;6N_Gu-5n8SIG}Zt7!(n=FT;df+_}$f%ww-n8m8-;>gg)h}+39_-&+QZu z|BS&lM+U2BDF#u^DT03;L!jnVfiHo6YDxuOhR8E_%mjMK;MtAtou+1?)!=!$gb>eu zsBGR5iC}xcr?y^Zr+x&9+6I9`P)DtrwYDHg)GiTxANpI{DDYVDtX(SbTTo8znF6;! z#kI=>{t@Aq3*1TcD+KO?fVC?HJ`o;Y+a&N2czW$BfggpOwao%Afo^MC1YSyVo-J^U z@aG8pI`mMxM&Pegzs?o-V(6syJb}v*rE5PU@SAYl+Vcgz7k*oNp}=RtcC{A?9DvSi z{Q{?v%xwbyhTwp}FJi3KwhMeG^(83qV%W5{L*N}GXQ#j`NrznmAErKr1pY75`NaZ% zKyt1XI7IsI7I-i5?-BT6k}WK7Gxg;XflG-#D)26XV*-yQI4< zU~<^nEecO^ZWovyBiDXj;4x&Q9RlBhaZ!7nz-Lk2>jnNF!rvh9e1w47odW-e+PzWW zi;2${1ilGlr1mC(&!f6G3;aFunJ)@F0AHG~*H(~cz8Xg!*kZn{5WV{WpJAFjs7|1; z>Uj`S)#c{B4FyDW(C!p2TWo8iSAvAnp0KH zx#(Vf@%S$jmGwK=xqcc)d#{nUmg~1t7d|4YQ;HvD3^DP{2c{uILe0Oy+zo0?VL?R= z5~i|fsJ;tgwNXiXjVfLa{Venr5k8A}=N2Ui9DqGP#GqR^=>)=ij=*^qPG;C~3o4q^ z#;!me$K`M&^Q6M_2;PcdVxByvj^N3V#4MS(8h)HbtKmqIVm`{4wus7V9r-@sET?|) zWZK8XYMEmKFJGOzn!pCvNju>t<~a_NBK(0O#u;G9mwCtAMOb!U1Ev2NK*mQB1U~iX0=27|i$xx-!Q~8~UXO{tyF*%{!J)#WHdM+5|}+ ziGjhu*D3E~xBxkokPSf9;y#tmgUxXco^YKz4_|&Nofezp9oclw>fCw56{q5pLHPWP zBWraLA&y>NdXzLW(UCSr-SMYObT;w&um2IJ^=UNE^kHVhX9_( zZB%SjoFc%UkBZQ$^F6Z#`m~(-Cq<^4s~qc5(oB7sZmuD@rkm%IG)5)u+v&O#3@V;g znXSlN!`xaN>riu*S#DHJ#vDS|%_zJ}nnIybS)%aw5{C;_=IhA(KsZ>=!{t)bsOYfq zcs2u6RKnR_av&_Z!0d+#jZs0v$o{n+g2JBZIDnowtDs!{>)%R2(`PGNg z6#LbGDLngCx(X@eS0}>x?M@~C3saT*u55|4^9TeWj_wWMFIuoZ96#Yw*k+lp_| z#Yw-~sEXOIW)x63*e$-KUqa&#kU^d19Y;>qzO>uq81)v^{3dSkrQIgSsOf1qSivp6 z^qBUgqbf#yJJpvSH?x)$QJ{HW#i;LyFI@^AeYnM!o)9nj5fHR_6<>PN98EGMW7JbA zzVx!{osj^vFa5}LefK2gOVeC92E)z1bS;3bxY?H;0q_uRqv8q0iG0aKn|5Cst41ke zj6^2{srQEjsoyIO>`bG^h#=%lr!YI4Gkr!Ck|Q)oTkK5j3jaOeE54{Qsxg?K@r)<^k*VU5F^s24+8C{%orcZ3eztP1>XQGo1x>S%Nv(ES!6TG=aTuQ(9 zn4I<&&Co8jMZ(jo(DpBIi%V^h@H8nM2d}urr9PMHQd?79YFnyHT`Mj{Th9JjdC=Tlv3M~X{zsXiJBfV+%Ne#vy@PFF59E(51_xY?yH2XGl~cB%UT&_he3 zVwvJZF4cmHf4fUvt~jtu-L5j^QkOG3n@jy(6_PuAQQBgc`bgo~rOL1=p^IERweW@hKaMGh4)&^h>tM zx$eeOwKKIyeA*B4ALABhYLWPqgRM&@ZgHlwQ=RFYRA*X~>P+W~GX=oo0^H(E=ZTwq z6NseQ2)~xL*qQE7_@4k@K~Ko(B4^5mVNxc$ zS8Y5_=3<%bKCx9>tg9uRX)-oZnrl!OC!J}&t@vtPoOGr$Rk1kJW{ONx#hLUQ-AQNq zRhf3CsS=s!eVBc?#hIo`WO9KAz2zd#G%eMcPD*vAlT)3kM4agy@L7dhoN2na3H?mp zy|~4hW~4gPDJjnMfa;x*07O@YIoEW3wNyEinQ0glli8U%kZH%w&U77s&*L^KKCd{D zGyNVF|8{43N^xLk`i07nGZjLua0U2_T9KFG#Dgw!rgAQ1N2sh-cy=aw;6fKU)A_=| z9+@t;@rWmRRNQ5&_61!n=}b@AiXYL%NoRW1R{WwaPCC2YXzH}2W*kD*$;Z2B1a_i$J7*6stfvpafu74PgmOf0~wcw_fb z*6BsG@cHoaYTnl!<>mKTS@_BT+||6T`0YqzheWEB-(iCtc`rTX8-V&p|NhLf5KdwO%5V&T7EeH21M1>5r)-9qHm( z+L8A0X720I^@F&@k@oRk?j7WRi(4G&=@dtLCdHASO>v~}vm?=~9dmJuBR$7Xas%?$ z;uc4GKE;uKkZ`0LIvhBtdX@xO`RLlqt~r&;j}C#`Yq;5u=pBG%xY>`^0f^%^Dt@FG zkstM=;@|E^pC}ION8@oIM;H0geaz10NBydheBe9M7W>gv3eSGDTV=?Pek&a8e)N=$ z$3T)t#jCbzlVL^ngrpyRVk@rI#YsP!jPouJTBniatr&!JPs|l{*c`=jX7WeIB(XK}Ra@^v+T}+pZgQ3ChyHzRFNC2dLT;_V$hqIOY zz5;IlgPYyA9;bVCxY>Q_TZ_AJ8x`G(5xH+o)nNBMp`2O56_VS=kA{T7yA%g@-^W#k z+_#O{+1%GtB>c#IH%eRVzJ&_U?z=)|$dm3D4tDoF+s5OmB#(-9wra2FYDxF~qOJJP zx;W{+yKTi2&~Qo^+^34!eQW&&j&}Du%m`8^9kR~91^Yo@PM|k<=YIn4My6^miqCkL zGCy_j(^;c#I*uafKI8Smy6=p^7{+}@NzQ`LgNp7&8D`FcUjm?ekq4RAuvDje(PA^V zZW5+6x+nMx>oy~3&|Nz^xBh7~Pj}r-;79j_yu!NcKuGt5+(N_5F{gR*GM%0Yo*Xkb zgLb)&F^1D~TIT7Qb2I6&m9yB$%$yGjoa>%s5XO}a3_ic43LrxUVW`NRXF5HZnNr&X z!WwO3R)bnNJtkfH5xa<5bZ|66Zz7-*{E5vZLR^&CC zkJV?AKx;D3b)Ca1p6N0w@)V=;*-5xU#H(OmsYJ%e0Rt>OlyLCbA;Qg8!m&Rqo>fXX z!9h5DpO|*brMlrcgK!|KNtHHJq@6psb@GwZ?2Y0%w!J;fO?J*0se@Hp#44C?*C~R5 zs-j%1uFN3k4|}xKcmy{H1v)$lOQJAzCSz)H8csNhY5oZ_t%(b0+-7foyekr(H8X}M zFEjgM(V3xepu4XoRbHAB(rQ2Ri^G z$TDw$*!RTZ(LVgn^5Uh-76Ad(Bq7uYMS#TlTC6|j3r3^e_^D^9hD=zM^@V|L7dpJs z8|jVv0{Esmp(7sQcxfO0g$2{wPv0*G7&7(v<6TlQj_-{VC?PW@jGiE@(w@MeMtd+8 zh}sdo{ayYTF+pK(e|xa4uM-*6ZR?K*{n4nu-%dRgsVHN1D)flHGw)a^RO`lXN~2rR zU~g2KCxpFDTO`tL&v$f3{Bb)X5b42BUT?6YqW*B_poV*bJ(1|3KE&yFvh8)qD)xMP zC@?4=8H9l3!Ej%Xy#n67wr5kcjtV{eOVxH7cv--n4|MxugLK#t8Z6|XJXrSj2mHM; z{QkA*j`S^d>+SD|`q>#^(RkD!2)5xDw}mR!9SRKUdLY<6NW~%i61iQh-hTXEcU-lR z%%J_wVAxKGY%m(amm`M!gSxRk=wBPfHzy7nNHk(%!AnG@FoccuLQFeNZz!;~8@`yV zh$g9zftb|l?Z*g;L{%;j3CHo9;8Fx-t1n^Hq!gN>eKda9$&*d2tf--fa74^S(}8Ti2&%b}lTJ8wxd(@obXr?uH-I@ia~bh*;s%i}BGH3}te zcLz{gU2edx^ta432k5?Rf-Zk-e!7`vI`?0WnKI3h?#|_84gRO4Z^r+0_gC@X9d}Kq z1l1|YbB(4XJr9R)NY4>$`niWX{B|_qj%& ziK25|nb|DWg;%?>0GM|(vy60S%>5hi$Z(S^86=AHF8b$QHa~F`Av$$%wva{2ZuQ;)nRrf9fn-> zA5zQL{vNeFYFEpif3aHFU1qYo%yUhETHr39rj{fBxLQ7S91Y_x2km;f{%_PvHu+F` zfdrT(I>4y7qry=7d9Hj##^kMdRD)V@QpUnlDAke3O5gQ&I91e9{0|K_55mw_{~oWp z&1oBKuU_`A9c+KKSG|!KZ2wn5=y|(dzWy)O3uevav^AQiEgow-Q$x@{ch34dBJ&^Z zO8E=E6k~+B{GSk+p_IYig1P+Drz|(M{11iZzg+%z=;fDbwn22qzgjQsEHi11ldo1c zLnf<#rlVv|B_(X*Mh|(Kp!u|X~}Q+Y&${2*B9=K z1>5n(Nz)&Z@4dp5>oAJJ>$k z-&XE#tMu0d{MFU|n%U)*6_wNv+J;Gk5DdE?5w5Af-X9CDXf#&_QL1>pp=Z!=<{Uh*Pkz196-vf&(mG>q?uC+Tcb z!IrVmM(qKMRxTFRgu{1boi*R>{k9ZO}m^=83s9!Z)Gn|Bp}@4ZhxoY>y8Ad ziN&;m*Gdxa_BBLdydXKmGHm~wY5Tqu@+rp?bl={NoK;4K_u#H53LKe2Pv;@tE??uX^4cwK~_$4u!9Dq?ilqmS-7AL z=GTS%;{?m_T@=D%R%r0X$wXd20K8pl=t$!rHVg0dM^&~*eMi;CUg)C7UI+?h~bS9p?N{W ztRo+M2`47LK-XGdhd@9I;tidGETB^m75l}_MZUBB zQ3-6Mrse*yzcbig&p!|$90ER!TXoPws`Ilg*hy#vL2NxhaY*Qo!rOz?6&O21-F(`E7=ymT*r3O-(gvXr zEk=@jqJ*Ddgcp2iX*kx4@x~E{G%>4E#mhK+H)ix=&f^FX=t7?RmY63SaIz$= z@fhprW$F9yLmZu9{PYKEP#+Se6w>yK+cwOa}h7jRa^wGeOF&ZQ3Rg~t4-VqApsE+5320D4vg^4vl506q*Tg-zJ3j`4kfma;1 zIoKT}I((M7!(`$XIHckGAAci8>9tN0oE|O5%T%fIC~8CaBtfE zalZsxUY1A+oXw93T!#eeqIN=VFX%4wjL}; zNFW%F7HRBgXr2ycr9p|LiX#?a2x{Fy{3uLf>S=@R)c7KWO883{j;iCAZ+d+69UyHT zv>0q-iy7;9GNJ&$jiLZSLj-^V8Y0oiAiGc`=hIJL9LMmq+KEw@p#ddxgE&o6Y798U z6pBZfHNYNL!+a8DZ-5W86hTZXcq+n!l%GXF4~m~^-L{Tk0x)6%k_XG)uGPUn6vm3% z=N4Md@-&5+cU`dEh=j3J=ng^*ObW1`kF)|Vv{a+v74xqP>exeuYe8U-^u=Xxk?Yfp zD>jumB^mYAlqgKiEOC4oQ9Z3Hw9uM|Za&)v z0Xzsc5$5m;oX%57j8{|eH z6fG^GJD-o9C{@qq7^5OD6u5zR7{s_a7>o2p1BvMlh2dZ)=F4@YQ`oQx%PpIzYU-6} zA>$C^3jLEAUq+`B@u3%uTge_W6z}6cv=P6Agc5g_8odyh!BrhH7XeOlys@N?F#uc-kZY(hU zoiQ5P;oy3sBe5m31ywDVTE!$|)avfYASZ^ggLKq3OYxRR6}qw_LMv$=MTylBmq|3mQzRZ6D!8gh=kpPX&o?MUpeI=I^+oB& zwR~Mz-9{sk9)tIEMTwanKCMHCz#1U?4!G#ut$oiv!u39{)F0GzjRNATkR#q03(%k`_1=g0rPH~RXrLZ=JE(+FT z|GExo)6xd~=apL*iZE@J_J!9YQkC*nBz7zw!kiv4cIhUd1k?z0*zyMEaqRFAkm2m- zE?Km~x2U;!WivfPM9h$Q3O9#FXrD!!L5?5fE3{dVc-oU#0Gt=<)!O2%esWw<4Ooh| z8ZaXQJk5j=gZ0S9g%b~+`GiwI4pJ}=lgp6fqq#_Lzv1h_0*)Lp$U|R681U}lMGqno znw0$wuQ;g$4N45f6pD4cL{Y)e>H$G8h&?Pe=`^BNujf;QmA#ZZN1kc=ICQW-Q=p?? zl1q#c-E%c$T4;->aTH#8Ck#FEZd^A_E0m;`5OFcs@tbovtT6f#Q$FqdF~z>8OXAR?8}Nnnb~s&qfl_m1r0vVo~p#;G5X1sLm+oXkOIP z+Ps3igLMwq4PX}I2tlJUx!NF4fm2~v)9>H$VGH6A*2@OsC#_PbOZ;qi9iwb9i4IXV zX!T$oOln7_Ijk55$(=MBB*0<|qZDgkbeyIgOlepyQU?AO3Ji`MTK9{Hv_fKmAS>Q!fm^dr>fuNGFq|HD!UHk2wsi;jY%)XwjIivqPmZ+i z!$4mU?ZjBe+DXo4S3{32gy&V{!cb>OLIrsn<_dmg1?Pg$bIi_6jz%eNB$w-4MaOxp ztFZkNU#AlnrDs}^&Z?A!ee$pfuGQZi>g+;j5RXPsv#*pO3v5I2MajvL5LoKFI-4c$ zvyX3(t`6ElEo7yIG{1!+V&VKrX_M5Vj*=JPR4k$nd(Z&yPN;FPqct|puGj%o+jf-< zI);P?r7p%)$YU9m065M>17J-XGrTc z`BkqXl5g-zE4dzP$6VWm{X$<9GeLJ>PZ)8O&lhZiPC8D3z~1ZRbWDqPikZo28NJm9 zzR45JI3I2L@WoQtc3=sP#Q}l^R$b@}dn{J7?Gc<^^E#O=3O;!G2XgU-l2nC>r!C@- zwlAeUP_z$=S62xNDubYMYODUbaFf&BU_2g%rv=~*J#KG$r<&<_A8svRJ5s3oS;6qR@pyRf(7PeIv3Kj zJgVwtCMI8wX*jM*PJHrQDAA1VK~D=UZ+Mg5heJuN{=}A15HMp0?_5Rp^YGT+LOdj) zy^-*Qa@e==z8e(44umr=4g*@<$;B5`h4}ze5f35~RyRvNf@Ndk@e1O!=1(h66~$5x z{t|;qKWrgA>J6k?Zi(FVjDw>yS((Qx8G|;?c%~^2gw*ad=>{Q?(!9X4j<9J8Mtef= zUEO)abQ*b5sDm5-473POVMD?@#yw=y$7J~FXH&jgZtQnG)zj0UpUVx`cg5-o^gzn089g;PDvt zPik4eV$qrwb=b*Eo#gbVZOZ0>SWuC;8eWD>T=*;pU#8KjLH!Y^G<@|+>)i>;VFA2g z&X;MlXmD-<)v7@|$ClwZ;3$iXW0R-G)xnjJn3+Ard)<(50i1^r;Uh_DO~W?NV4>3X^M)zAKlm zW0_o6|At}%eQNOM2~?{F?Horkc^%&FrHj4{oi5X8*5EpYavISq?$#OlvK~bzK=Qws zTr<=uMARVt{1shna-F$UAwSjRAE*wFQWeOS1xjaAR0@-8kUrN%7m;bWRN+Q6xo$(J zBbi*6>-1PAuTzaSCS+{Y;1BRw2)d4Ca$TK%NP{luQ-hZ#P^}uYa~#R!zfsi&64h5} z(4z$6QLZ!N71A_Ti3rL6Vsgz;rw~zt^zCMJvB`BNPci&dli#Xpe@&%i%X?MjZzQB@ z(%`ohipVtlq{5{e_!_Wbhq-P1m+SOcHh)32`f@_ZRt@T(lsuNrb@g8*s<&t` z6Q7o$3xPv}c8(+2Jg%x;m#E&X!Fv>nhq=ytElDTsCI5@fHA9`k<{I3RV2H@BGxsax zr`o(k5!R}dj9H6M1JSiar5NTKT%}M%rr}nF8`%?RfBepBiWq3PKd4_C#pAVkUq9d z7mstDc_&FHrOE#?&NV}wVw`L6nFK?ObDeopAwSjTU5fCtDkWoXQ97gFo}vrmT!Zv$ z0CW+VhVN9kk!`LU(dkGw*X24rmdzhhtv;3zvQ>j;;?tjW9n0pr`gar6TQo@DuSyrZ zNP~8cBiXzKUjRbadX-|FYmmMzpDrHfI&*W9PD+#i#par!PGNHm($D45#q+t&T&Iwq zYICZe*M5Mszxo&2_m>k7e^F)#{prkgXbg z3!mwy>sU6|)qRQTEgIaOK(%Vn&T%B0zptv%cX`rFJ~u4hX&fFFpJo)Q4b5OaF5`e& z@DIkP8+wU97@uKmRSTxUc(?J`uy_l<;cBN8*l8dd<6CJ`FDHHpqi9%smhr-{_|e9J zVeuYgQ%)*(nnJRTqea8v!K8dRe2(F*7!IFn+%POY&p5h#IC^}=?uy~?1%`WA{21f8 zmBZ2F`!1NufFXrrO0Wh?~5CPSKm@L1h2lgY6xC^Rn!ptloX6) zokr0RJRQBH8Sf6H7Eo++7(?w#gO~6RrP%!hg+H8vp98#UB=w_?dW^-4A8M~2#iu+M zfbHrBY#KxDek<_9$^T{GNlyKdbbUzrHHANbFBPFnf1%np0ds!~7;WgUI(|aoQ@^kF zdkUZW)uTUD_|$Ka+^_Jd-w5~%g-`v0x7QTDT$xhK|67HBp$GsjJ?wuE*u*cX=3AYxfn+W6f-qEj7=lZe_;gtSAZW* zo@anRaTqy`Kr|kU<5>~j0l`b1!A?Bx#q+y3y-|nPRq>b_?-Tht@pT=3yf%gR{PAJ8 zz72S^Oz(%|<&(0S^6HA=iSQz0IOO9mbHPh@alC`Yudevo`Gt49ElDqUN)8Xq@w&8~ zbkNHw`b9TM`2}Bovmfuj;q7n2(#Jg{<%B;I1T;RDLf9Jlk7v#LRekjxHS)HT&$qaF z!SY4EMJpEK?M)V&dK8P`z2$H_UTX0zJa@%{gtv(TUy`Xt1O#6%!5gKIBI*-PT0}YQ~iiEAqxRuAkRKi-e>OY zp|^SQaUi_q4jMdS#|Lo|AFIRTzG2_$8Twv)g8h=j`^_W0Im8+qR)2WWeuJOv#-A=p zcE7T$3cvLh>+ivHOr-Isq+J>c8G|G+aO( z;aBU*f?aBWbhU$<&Iu10DN;kjXv*Ic0yD5C&XAH`(JO0<#f&lxS9+%lIQnz|s?Y}> zftM`5aifdc+UU`*%F#zSH4LR`kUk9gPr7J@MhA}wPkib9fliAQUMK3TOJ|KTK=%52 zAEDFY1f>9}+k*u_e>gA@(e3N~hE8>TT`#r$vrwP*<+{G!m+164Q`)7ocmj2Od@UPa ztMQ+B<=5@#l-4hw*y`*3kWR~0{Z#oC z8c%*{eUp@5@0)bWol`VaC$;}qsW($pP* z)`MbeT$55?@7HwtkS0`V3VUN`N`1Yr)2XJ{Y;<0yx2M!UpbB)lOIK8>uBX%cQtIn{ zqE7qt3k@mtHQWD4sjv5wI#uTnnw;&B$N>0;s$Z#*um}CHDG> z{dk>J=}OcTK-bgh$3WZb>wU6JQk){uSF z(BlqW|C9u!0J^?TM+4BXo%AcJeeKS=ER}ZR2=#xd>i=)Gt~t1UjV&3W{-d-J$Mv>W zjAVz%T0KJj`#vibPD~m9sq)vW`l3MQSa!)~dDEN2_Sn3$(R}FZwe7-&$+$nUhI^_4eNHe&7G& ze3|UE*Is+=wbxpE?Z-K1X8rs{Rd$=r&{vi*%b?tb0>SuI!J%I%0EyqIG;;BOj4_&N zv(rVrpnC@jN~eZf^3+BSfhwPw=BV~{s>`V!w^vS;ab=|KS%TK7wVne-M`lX-k;OgJ zO{x>ybZ3-Yc92m$=PaRjF6o)>R;tsfQ;JzeVs$Su=p*$p)-ZIMS#PhZmzj=H^>nJm zUGZMB@xRkvt*Ym)5XPD5C8|B0>h_kQ9_i&@y|kH^tM>Y{gH!cer%Ju7PvSv7vtUWR z0kYjBi!wKt7ppv#|LDnSs}7C`{b9%ldHEaj>~DVVd-+8aQ$3>)AL1+WgNsjaKp883 z^vyRY?e8KPe0>z&1Dx)owfLrf@O%2f=Rh!$FY|gCh`#uL+z-xr^@e&|8o zXMbz^;ZxrapDX&od-~yXNu7Hc2UY-GXVi8rZMZNTN(Sb(ghQ+6bTl=E<7woAa59AwAh0~#8V)AHpji})HV3NW zv9|hXBGMcUHxhzAD4X9I3P^?d@pz0lA`PqvCs!onkrYK;S9>^Bjv8kMTRRjYK-~|m z4uo1(2bzMB*8bYF;#J!<(FoKQiiesyqT$XYC>F;WJJ92~@o+F1o@lg28bYOsSm{(_ z!J?Wua|07gCzei0!IQNaD5bI2*Oi6QXhUhx;tL%m5tByuYK1F~LF_#+m4fd%ial?=6M z9alaJUeAq0Io5*bb(d5($}M=A52c{eg2zBiU49E*E$z6h+JetZQ5#0B1&;xox>j27 zt`ycVR$1`*7W^^rmhEct7chcVdLiLqT zJAc9rbx(N*=)>J7`~gY+NQ3BGsO0d0x4XN$w{e~(%EJdo+hNj2d+!yX(Bp&;POFZk*sRlGu1DCo-77~ z!HR*idtVdFrR$TuHq6|Jx<<#z_0?C-JWK~7)tlc-=2dT*8AY}IKm5GAdw(O8`Ap9F zptGHix~cZ5{(-e9RPs>1zIyX#5a=N+N#Qsf8mqTtpD>oW&KuKNvAcTfqA||u&F?Hi z>Rq&T-Wd1(SJ1@9-T7C&OH`L%GRAAbr1=}WDZ6FHnCdMHoS$yE4>@=K#)0sz{To$< zCx~Ke>lh!ARlHQa`Gftpq3FsZ-T7DDg0d?w88gqeK3=^gIi|dNOC9JoewwVP-g3zp zclDNe5X1J0=p%pE!0Ida*eaf?ermrnf7i3sld^wRy_qPcAMSXkdh=iQzx2=UZmQ_q z{}@BbF`bRolO}V|@;6>bf?WBh4d?$v_(A8F&mh^R+(v5HQn~+Oss%pRAg$Q_$0t*C z5uux-=v4V#sq)!U-aK6RT*&ngT+BKL=bUj;zDJcWl=5>?zP}W3Q&VaF$g3b;YP>4@ z6i|hK%|27XhwSqw>SfwK17M0?_9-O$y!Q{O@KdVL(>{Mg5!>hMC=>gvQ}$UT_L*w6 z&+|$ToA*>-`IN2tseR7srw&$5T-Kex>m?ZJx5`K#cKn`{I|0naNJR|wG}4VEjW&{~ z1p0gJRFZMZVeE*=P6MeTSiSRiw$qPNc6dtYZc}us{PtA&ty2D<*eMa8JzW1lXUa~O zOL>W^{|zZ$Np`v#u<)^rpH0N5ANyoiZ><~Sw2Yt|!GSFFddfoCG=e^%EIDnDAw=V9 z{~T50VImO|flkKInJ8l$Z9$>fsHx`|8qYSu5X#07LOe)~9~nNAvcHDGlTCx?JyPaP zf9vjs7}@*tz?nE%$Ie!gK+LiY_NbXo0!?X9dR&L9#9}#8jGH|veOCzGKt-p@zmY08 zrMzq?si1QIM5%v8s{F%BZs!_RUdZJK0%VU*|JsK=j$ljsFWVy<+{hm9P~UjOus!}s z*Y!H!%cSgE#3Ua| zJhOu*o1_asrb(_RA=o5(p4pP~&@>Wk%Fjs+(C0fx#4e8`gGf8jiz_-+erKxu zA}PORi14|M>mOK`8YgE<`C(PQPRh?AyIcUc-*~o?D*UVCr4gJ++XtzmY5UM|Yj6AH zlYI(RjXizX=R}mTeQr86yC(jgw>$DE2jY@Kqv(%-366XT~4DeUxk$_^h+VY;7tsLPeQ{I*p2vr_(- z!BT$sQ>w4#deq2yNXi$g`Y%cOS!AbOfQ64Go;^m4{#E-t4Gv$pnb3__bgDc( z-;9&;rNv^OB0+JqtINUhsSz&roZOU5|% z`vGN|sE>8qvlH1q@8HCc#>L-068pS_6!uB4+a40SYZRR-Pp{iHOZkI^!slVGZ&|lp zB;^BD{p+RtT(Zv^z`g8qDKYw2?Q<14d^!6}R5kjtPZi4gv5#7_Ra))yeA+&#d8p~B zgH9PpFwt4cL?3p%MXta0ur|@1fHF<=)l`_nyoR_lu7d3|N9pw(ssbhxk-|PtrQ%nC z&>bug{j2i3Q{@NBnNPkdFO>S1r^;WH@(We@;SV|fT|oAE1#mC>yyNcQJ_9nx&%5A2 z_IZao$MX;7nEZ`LSsz@lYV@@}xCdoypC0Q2o_2WrJe$P&AUVdJziZx@0oAq_)ApHY zr)^LEuHE^&vK^bKbo$Rb4w63KIRvBZx@63N{Z4SN#w12-w!9N)C)ekG=ejc1=XXgB z7&m)RpZMH{w8yyV61wGzPL-#}%`z!pi{oD6vySUq#?5poe@T^}Bjr49W&!TKJ~(g| zwuF}b>64$}00JB5onUm}3{*jnXr97?$B6-^PC9@*b{*5eF%2Bkz%dOR)4(we9Mixt z4II0y8d!QpVzd#7uEB$?jlQ|d=lYWTDD2W2pqEzDzd874c>t8a zN=8)1IjNYc63I@S%-M;PB|B*%XD3aR>{Kb8ic-1iqmfXoF+3s056&ihZ5@fEuOaLU zM+w~suqhS~`$ECiP)BPpiN{@iL6TFOL+bew9qsM0I2R{d!oCK6^wu}Kw#F9^U(^wa zhZ|J|d5$*KOacsB!@kByTNuyX#-fI8Sk`HmfR>&~bHCNyy&dUpq`Q&6fb>|o=P-hIVxPNQ3zuDcrlPFy7DwlU* ze(qZ52IKUSQ^!vlb3CD`{ycn+fDQ*Psf+OE;d2Q19Vski0i49Aa$k4%^=QYK?{Z&l zU*g(kv;TRZ%RSrWT#!pNq=!e)@7sFP92S~|O!H&Vl>ec-`Y(7aMqpjxYTl9v*V!iyqCU3lHhZ zrQXM*y$fBs{FBeSW0XIn_B}b2!L<*M&C#XbMe@?K`cAbz z`z}pm^?REd1AFl0Tc$Dpy|kiS>go46Gv(3mbH3mcy3BI@{-%CkGqYU3uc_b9%q-XM zXX^JcGt2e+nEL(8%yRwyrGDQsvpl5wr{Ay4EMKk4_4|~W<@$X}{r+TT`4?$@!Q8p0 z`o`8bbVQRKzUiftOUoxzba1ZXvWexT<&#S*#tGgy1-xWf%iz>a{=oYFpryUqQ`J{R z7*5*$ zaYU7~jS~BSTfp4mT#uscHY&F{E`@~IoeVpZ1YafiHGu7ViSWcBjI!ta!w1wsgoW(! z9^!8Q7!qbr7)FiTpFw}Kw+~%OupQ5hW#2lC-~mqiH4r!ZDb6{Z)RO&LU~*cEeg_yG zcpK6>8xiGRbyPSvfsMH(A5u;lFdyZH`3B;#W2(6yu+y9aGRHK|xXnB?dcaUq%deoXVN8=0U`1$7!4?HwO`e8Jwv!XHlJ*obj6*F^@R>oT)Z{L5b2O$2$Flq@b}~c0P-9uXzczbAkCf>g+TVBz0im1Y{mFw^6rO zv244|QlK0S&ey=;X~W(Eb^`_srEggo_($s8$*$aQV?uH`?*M_rNdD-a_{7zD^lZxFgLX8rX>o1rF^TtTMHdrxlqVT=~TFaXzut{U|G=}Xz2Zur4qFaz; z{|G$of5HeER8st1Anc!FgbW&1yo=y(Lr~Y1Lq8z+An>m1ir)o&R(AF)&~TPBdlo9X zpFv4}P8RddBk4|p`?98OhU-7D_HcjQH3TS^4gGWP5P43*fN|eluoC31ZxhWuLX$-_ z4+zaTqIr;M+y}5GaTnUSz!^t}TM8CByTR4WLydwnirbJF<|C+5uqeL`@L2Oq)G1iZ znR2rQeJNPtBKa%LE8r9bOWov`e)A%XoPyfHU0_gc4j=~0-1Cssnk7(R!SdWwG5A-S zkE5}I6?rj~tTOCBLQ*(rR4zn!66`LR;hcnwxfa9)GtK#sz%WO^Oa-%YUkB_o?0&3s z3SGlbr#b}til#a*r#e?dj-qLWB(-4%;crEi1&5$or@0xdi>4P4H@BG}veTH%YuFzG z@521yTZtRNV~ca0zaws&K~|hsz}yO&!%3rfP~HpZy|W-e8CTx#kZ~6@#zIDMK9hM( zH{7Gx&4T(2`_o`k=pMcwNiMCaoGK(c7Dg zt_LxCdouJ@STuYB$ei~;DZatWo%K{_Hfg2x!vZu(P=YE+`(k9KbrR3S@0c{_|0XA8lJlf-aw|>tb%8FzCJhr>RK}fMS1Ra6=E!|G~Xcx z|KSNES!LFe1owDXBhzBOgV^o)_K=H^X*XX+B=OwmIu3O@%@fcc&-V)Q(fti(J!S4^ zZkx=7a4pXRg}*|h+stQSYtMs)4%FFh28i{;#cRQBhdB%C@%*4@I^@0AjKecLk8-zn znq#TXj|$0AA2T0-iabvaB1heAUJS*1_7t2BZhOs1ROhGM?H9~nz{Nd#3ob;-tL6gA zJX^R1nSJIxB+he%G$8hy*~I#ng;yeT$gCi;=L^ZHjufpxg!a6^7B*}}yQtqU=68_2 zZACkY@TI~lz|U)-&S-N+2%FwWwot{;<@K_ZoSr4^egJX8=PBdE4r2p z_dCw5vK4Km{`{VEEw-Y+kVf9*T)VA^me8KJ*he~TMJwTko_7jqsj$ITw1}jBm&-QU zijJqU_lh5fxVUFzzZ}D$aPWvc&^QU+J*3%r13Jc@I;459xc6o#3>i`pCVa@DGCuR~ z5JP3gn)_flWagvYa&rWR5i;LIX0n+tbtp5{yb1pmS*1A&f~d?4a|V_ID&sdVfx1*? zp80JkO=YUh-O!TCEHdwha#g0*yfH07otebIQ6($QEs$Mh&NKgl=fshr{;V>8md>=8 z8{pllq}`k=5)fIZ8HW$6k`3k#X_QJfnGxwJWwx2Cq*2OjH`ifMqYlj`JIr!u9vRZk zz2+0b?R8{!ny-`3tDZ zECGLSeKF0%hIxSxnKL=#G~b3y-m``_Qj9d;gz~)Sb1?Use?%+ZRh$`aeih@v+rSy0 zxgM_X4RL0Sc@^y9ZRE^Y(~qXT5zd@qK1U@Na;DteN)a#Wz7XwHnscb6z0eCk@|(?6 zauH{$%>|gxy>a%|T5~PT=S^^CrTGK`jdu-aR+%1>Z7pY7%(u})ZzpHk%^To>-b;tH zqn%Fkha}E=F4nBBcsan#vv4y4RAoY`T%4wHMY#U%wLjh-{Z`y z<~-8q1Dx4sE`T$5A9THnWPd@V(eOUZ@FDXFYV(nTICvf@`XRNst9T9ihCo7JndW`e z{TAd!APE!i$2f;T@>{C*GtcLwU|Z29((upopGFQ}wPk+*&QLhC__yFTI1ot~p4y?s zCi!6jZgm?&2XjA+0^AKZhI&{)XF+Q?Y78C15zTF$3GEN{a>i>$(D$Jw++CkJ2rUgA z#+fnZ&5&&9@LY20vF5Wd-_Q{S8_-O->4vaFkK;_Gxf41bI+8Pf^Fe4}sE;M6#^UeK z@Xl-!&Ecdu&i)!)cj&0B-$8T-&#oS?z0IW9!*R$Sbx{q&d;o^G$3$Giei<}nZ_nBe zGW(qn%Ko(HW#;)h{tqGc2k`L{VAw05I{QyOpCIFK{vH3BfTlDiP*gX=f9%hAh5_wx zj-p^fwP`|1Qz_CQ3~m2eeif50B2pUJ=W0^JemXU~*VD`dZA3tQy(CTGpsqhVcoP$R zl?bS@Z)gH9FGuW!(7pW^t_PUv5u&1Ye%XUciL`^%^GnxjO!}54J(40N_KNfdIE(#P z`PuMuhjSpsIpTU8fOJOHcH^Ba5VcXgMYwb}(7leu%X3YW{z5jXzTj>LRU9(lr$G73_rhq&WMla6ad_z9KZwh=I zwfELwayk1aM6*xih+;I_-xj!p60$q8g4}OLzXij3))0b??YxebK78^52y01;CPEe z3?i%Jl5OS^DmjzuY&Xw@sXS+KW{0_mdb*M`_nPOC)Ms;Mr@4a^at`WIrD67?t{}-6eHEQaGBdyR7$;waL$W)u$l1QM!AC1v{NQlNO9S1ZY8H^=Zx3fP71k*mFP3+0LBv^OwoC)c^;+- zPl8Lz&8dh1o}`hiYEQS~Sf0)bv%NQuA37bt7lo=EG>ybG7Rc;%L~%klbI%dJj2|+q2nR zL7dK`(RWR8Ga5F`?~uE0@oq;3JLYrAaMxy)fz08gW*tr%a1JLK!r`QDJDkMC!AA`a z-tJ>IfVSP!NU`Ur=(CD6b*^Oi#N-iF$~WBqAp9y@!FQ1LxJypUc@Kbj7#b>>QA7dS zFh4}VESbp}C(-zG2A~%OIH@j~T}aC@z!gTx9EL&REt#KlGAPVB(0WPL5P}t#1WuW3Of$a)8ed1qD4AZwU{iFBQ8H`5QxJO8 zXgFWV+yT^qQ8!T2)dNV-QDdOdl0^fkIcfM2K-};U97$jZmn_brwq1|Ih$Tyoqhi-# z;D<4|$5O9VI`e8=Pp{4+FoLZQG5p|cZ;1kNSA>i0EZcB0}xi{Lz= z`53~VG(_MD@cEOx4Cm39e%c~mHS|1I@!9}=jujocZh`H_dZhmIs535K;63oRadQOz z81&=j3OonO9XC(lci=PQmI~~F?Z(v#{5V3xxMc#r4V{f!A@IxarEzrvk0(C$0v`bV zc-8Kw1iJN$|r&FN2Q%tG59&(Ye| z%`?0gDVUHYdy!7GTu_?di~_fL5B#s7jE`l#=4QxRP(FyvvyI z;8S1M26VSRR%BLuF1nkPyZ9nOD323{}~z*sVEybeZ&D-N!clmndyMGU!C6!A8c0-(th8=ut z<#IuosU|O2UH?E=rm6sO5FDnCkXkQcuA8R(@cU47rBm?tBPdjgU)eD%46iGH3|!yg z9FPCJV66rq%KAA(@y;hAvajJO_l`7MQEE1#$6j9%!B}+=)hI7@+zCE(1Z+=&wfFce zV&&Y55$g4x=>9&{oB?sY6@%zj1}0}JnKbG#qB75;lF37!M8<989uKQHoK(f(ybS;C z%w~lg=(2&gCej(^O;niK2TiR5X3Mz*oWc(@z!nF2vjE3j_|rkET}(yc{;1 zUL)|W@a5?X8MfbmUKBeAe+HEOe)PE5%-_$Fe1QLK!vPo`Vnec=Y&~78AWN|dJ3oL} z#cI6GfGEXkG<<}P6{~PM5aYdAh31J+VzGo~*X^)P@lX-#Iq*DnhQPHj%W2|TuD2k- z3>Bt^!5L<#aDO^9Geex&MQ0i_J-!4)alH>GpQZ43KuxC~mT|rxVa8w0U2(kxH}=;E zd_3y<7YaNI>hLcT_#RBn{v`qrhiv|(0)H3${j~z$1TFcO2|N-i_OB3lE#d0~{u|NP z3mkw>{AUV$F8bm>OW?o5iTx`D-V0aupDpkLbjW|Mz~3O5&li|F>c2o>@@#)V;8&=R zs|4Of{R#^F2FcSPa2v@J5_m35?hgw*1h(@x3;b8;+}|SbAE{pvf$3JM|3ZP6!kPW6 z1%8M6(kk#Lq^mZ8&w#D{QGq{&t^F~9ABSE1?E>FMI=@KZyQtl`z}Jxe69PM^?@56d zAfWj>1bz*E;_noA3DI{6{1n0K1g<3bVu61{dcH*9#|XYu;Ae>EWdc8mG3Z|}@JpoU z4FZ2gG?xqfW9rKl0$)sexKiMD>dRFE=aHT_3VZ|n-+#5h?~=~92>eab=d}XUZ%z2G z7x)r{U;hmP-$?v#6!;Kk1OH6|pGiiUjSR)LQYpW6gpf${FY zUEl|Z=N$sm`JVqyfh%Ft*(%gO0C{GsF}Q&6YUwrvcA2x9$Jg-`-CcBteQsgVDaTXP z8ixO_T&Qhc=Wsf6AF>}LX94M@+v@VK4JgC47WsKygF`5DIA`I%>j7wN{+v;MlsZ>I zLU)zdc_#|^K4;ZXz8z+mUMxeZN(yLE;WS(?fUauz$nP+h4*Yj+1IAZ1()oF+rcc$R zd8cYrN!Dmkam21Vo~Kf`X~GUwqd8M-CeZP!G2DRH$bOJHZy|~2LQ_@a#|%b&1Zm)1 zmtwpv7$EVBzBp`w(%tbGlM5u4<*h)~rKrMNlWrsXbil5kK%v!cAK~+ece{mP6%lO5 z$gCbQitw(-eSnW+*#0|Iw4K0Dz1VZ$ZMGA=O{i}l2Jf_;B#*vyA)4664*whQd6S@x zUuxz9jnUtwvYGbcCxA1v^kZG4uO(KC?87dBBFwBEwWx|~Oad0Mon^PtG_cRdI0NSA z%XY`YF<79t!W`!SIPnvo;NP5%Ww@myqo^l4`4AM82F zw#>d3CCjNVC)rk#TqoJiCTWcF`xJvu!JzCJm9d{{81FK-I{O;bTxKgb%8rA>(&a+o zKcp!X8s%dZz6AKP3smMrWX3`VEXbj9scDooS$O!lxSn{FU2Ulr(A9RQs?iN;x`^v# zx;Q6Q{EVgePF=h~7ni@JiYKBrM&(vm)!`)Dj_oGjecvX&tEl+(G!WYE$Qq~p>U~=w zxzJ}&`A7JOU%hYRP03U{bbyce)qY!sUmdVz_|-vMhF|^B#(qUlj$MI|_|>0m>{s+y z6FnOxe)VTthF^UkR4Koj0`H^CrUA(U@ef%+S^SFP99`sBb#c98yC&uVvzZ&~A);G3Y;!FAw zPkucU6t+8RPu9M)%f>P4Ak_RmKH^KeY#gJ;A##nzM||lq?Mp{gjQU}wFFkI{`;XBS zXg*dk>PO;B^w38KKH^J17B6`Yh-dH-UwXneh-6B~s3$Xg=~dM`n+CYA)S2gOIU~j^ zUpgTNr%L#+FI@v*6F%%q4+GeVk5Tqx#ff|=fQr@_HAIb4#2ATA2vQ&S3Q`{^4(v<= zF+I^m&a|1?S)A!yRY;C-x3tC1)Tr>M0bh2r%8)btQaD(h=`IV8_tHGdp0rfUhVD4F zrJdQXi;7a;cA{Eq1BT6rNqG6svc-$fbNROvb!7RTUGDN-map?@UXz zvvjqzORcdKuhGS6m%7+%<1NewDotx=Q=VdsP zbEs(CrUA*KlNZ`@o+wq$Gzd{S2OoB(IDmG1*qLqvupJ+x>`zKhXIe^;X|y<#eqSQ(Ofx5HXBsV$=^5yG z4?f~dqa`xs41~IY6lXdi)0s}pbf%LsooTE%(^=rN3?Ff(apETQBbeX7N1SPVrZb(K z;Y|0c-q|#ea;E9FoPia}nQZ9wU%`}}sR@}zeAt=33gB9NjIwJLCvv7msQBfb=}E@3byisKTx$eF$>ZE<9psqpMfYgC5J{Eg6A9aP2aOchfwXW2gDMYw)9C+$lcCSy+I&D}@5Zh9PA z-i6OZ-rIe|%cj2~{}Dcucx(5GT7e(AY!dJ6K20saC-KJav%Fh*&GhMIlX+ivgqPoE z@g(I! zpMc$;@nIJ_XAmAG#)n<#3IO!{mr-_#;zTa=HB_{^(C|LiOP}^yFI}KGunVnI8FHZ? zGCPY4y`l=qgO#k?GTer`L$Yk)F+Pq@Sl8 zX(}BK&_O$0HVsG?h*xuNnxg!OZV0}O5Bt%Qe8d`j*pJo#NaACZJ*OCvAN>{;zq}uP zt~jtCmEaJTF7hKAl-Xlm>r#c}14Fq`=Cvyop8aT-%8(z;7YJ8~Hcz5%&#cxNl>I`-WM}tm{5gaj=O+ zn&gI4;Un(b%%WX|{AKuv`?fG$Iu1s9y6+~XP@4u=`_#3yIdzrFed)Qn-{8aUOV2y| z@nQG95I_q)Mp>(3MDBYtDq7ulRJm}qZW}-A6$bB89N2vyR~d5O-OSDs23|s05G?33bWDLif|Nc(PcHCp z!>W!xLqmmr2RP7Y)I6{MR3C-~KK?<4v%T0h(#O99%;+J_2$H;P>ntNLA^xT>v2s*~d&7B1bnBzo)FgXf?U4$bag@nU9T)CJR zoJF=QS8lGi{CV7Xa)G3 zbPe9gvwN?b0~TXTVL0hXF9zds#1z~I`;bTJ_1f`ZN;(%D7@XWgC?Ps!gsq}zx$Utz zxg^la+_Q7eVl_?3F)E4_qteM~xJ<;WU{5GV#wY*-EIE{L@W@`mO;p0MwoHoPw9aJ(bP<7b4A)|JzFO`S6UmzLd4o02A`EPZl(8Bt&deO!>Uun2~|b8 z$*i7DN)^1rn09jv=~9E-KkCvw!&Wzq!brIjaU#nI(6Zn5ch?~Nr^iAKd~EGq$(C4j zQdt7GV9GiY@v=xX)Y{P)E>i`i6%~TVk71WJheBnYl~V&#Cr@aNL_0bsG)Ftirt2y_ z>-VB-h~OQd9!>NjNpyV?1u-d0;^)A7687Yq7QHMaFwQ}-Qjp*IF2%hj!@z<9zh++A zV&Jepk!)-XHvvY-W!?aBZmU74Es>0O;J44KY8K5$8KP(!Lgh#dNF3cj{6IJ!Z^h5A z3kPJPs;nalY@^WOtp1_ zk~G9(t=4=~Yb=7zy>r$9fYeEn&UK}bU^5xz)?840a!g+HZfz>fSPr2Mhx*b~ z$x94~chJ~kN2QP_Sx~okwMLr4p{|hDP^3+bJrp*y;)_>hu<6Pn8fSep1kMJ2Gs?8< z8|GQJWZN7#&B`*b$gyRuO)kvIaeP$7S8>ijlsKJDKyAu#0?u(9v}N0~UcF=?Xg|#c zqIgKL!PG&WMGqoTe{Sj(s%A^lxeBf%V_}0_9wh={Y~V z6uRx{IeYt8&f}2q-}Ib!GaP3IJI)L^&Ka4Gb6whT$YuX6)!hH(RdcYnYHs;oRt>vN z8N1DloMBK6+~$i^bNJs^%@>ZTUT$;yWxd?y{Y&~CQ+>KkKKYVkumqe1I^d`{q{5Qp zoSb4s#^m+5M}%4{6*WADveYg`CeyLwzu{bQ`$%<<{K1#h;Fk;8 z@8*;s8}t2dd*?~lMeH8 z7KUR^&KX`>S2?jAr(IMc*qStIyB36_fjX=(0S0(^P8$FtVA)$5fsSZLBHW0dAE32< zG-lu*Q`L6O?+gVN^WJbtELj!nh&IlrRWg_n&p>rBQ56m*JL1HiXwL|D2|=2O$;}iAl_8 zdn-0D;Mk5{8R$bWfxj+NSZ&vA0-(;b;=y+8jL!_VcBt0S8*HlMkF&?_(5t9BwDaZr`r)=*dALOjvK+YKro7p2zoac zdncupNHmg+U=y?szttd$sfjj*qfocCH>wu_D0fXHjvs6Y!A|Hme!!w5Y$PJ>hi_oxQ0*$IV^OshM=183Ocq|0`(iMT_;b0?ms0w~Y z^3=v)LUe@-YohHP$?8~awc=5;RQuC{u-Lwd)ZPGtC5%8I*3^`;eO<6YXkm&KN$fG_ zwsb^SLuu4)YI`y7buHeLAYtKh%9jI6+i~ClR8?!R*$A}8LL@~MZS1vMh^q(Y#$o>O zMA(OSfq^v;ANy?b=9mm0SdKQ2Bx_?hh)pdUGfMm0HNiMUr%jz~M@gGFJ&46grxDaa zUOQTAq9xLl#E*(VLD68VQp(&|G>XF{=x%wqiAJW@5qWL8aCQT{MHljKku1X3n20|t zU7<{q2(&|0!Db%+OJenK`4t$A^YA+#IB1A>fqoH=K4f(CYTw%MoOoG|yDAaYqJ(@}fluJss`hOHc|LZ8bQ+2M`&l`6aQa3hK>~R&)=J#WNN>2=ov} z+mNOgv7mf)QTu$(m@Q9mAdD9%+SsX$USwQ3tMq|5bE^r-FSF z114d#V|pgN!&Q*yzN_gRGZ7-6NWde#t(|++fnNn`j^cNSP=orF@&_Spys)8j(xe~? zV2XB3yclG(ENP=j7OL!Mz^or>*E4-hVs3+<^+351{8VVNpv`Kvm0OSs-d7?hr zKr2I{ma9xVKOXwo$fI?BG9C?z_N}=SPVsX82GoK zv@o*nP%ER_F46|60!uF42#t*uI9(67aj$5E%9OwhCndw+ggvG$IutkHF>4~R4j!!3!Fe5RZ8VL{l~Z9YO8|D@K8F#FaZ7`p4?U&G zNjC=YhcU~`26#I!E0mqUi#kGUJ#-_HRG~i|>Dw_1L5o=0hLsq}sU~MN(XXImXOV-H zsD5<|PA`nHfJ8KH4dZvLQqyMxEUZQ*saxW1%B;i%Ki1V2pl{i1Xrh&^^vzhqvnYiT zZW{#%J#iV~G*IHP9#+Gs3-EbwAc@-xw7|oXJkf$RL@S12fTl)9LHtOxDMnfqACul7 zMp1mj)E0=f2QR`Z4h_P}nEc{#gHOw$38jTV-PyJC!9fXNSr;)OYHBfQ8Q-!Z96~6- zC_*@1y*3tY?XoV`^@4>K?Qn#680&~NSbfH#N_I3vb5kIFUyP^bL~u=5hcYs19j2LB zM^eTmjTTxm$gIh~DVCm8LcF@G?NUnzQ2^aqVyV?qmNztXzkztnquWKYwu8O!^J%iQ z31BR?)vMq{YaRNajKm>_M;gXjA{Jc`k6}$vmTYS;Ymc-~Kom^Et`jOFs{?^`YcDJV zQ^fK>V^eTSdE<24=BQ|xJbCKm%F4>=(<`S>nN;3T5o`$3gag-wAw^`G-DvLRWg)Fx z>9)iQ{L&hJwk?c7LMwpAsz|t1FXfj2A^M?s+8k^HLmKPC&eA8VK><;O`Xhv zP)!*yRVVM(sI$v)U#ue@N=@@9jE0-BPQWsX1j9;;mle7pwd9acCP4y&9$KJZvrBJ> zX0O1qL+T`2V~~R)B(2=xil7hhDqQ@Ewn4ZboLVHerf#~?MkgVgKpof8As#yig=|Zt zl-h(vJOyel(jsd*rTaDtWLUiOOe!h~%xwvWR!cOd6&U<+9j1JxV@yq*Dw>krXi^kM zp{YuYqC+_?D7byuS1t%A6;o8?Wf6z8wpOhDNl?VAXn3vBl-ey@CJ>RnO}Ceh$16ym zFq0mA@SU*UMZf|OD6oRobEGO7{_I<9@3~cK_XYXMPnOfINt!LR{@Sn)3pdJahTFwq z#1Hs@p06lQ1rwo2B(Syx0WE<&L>PfT){J!mq*;Txl}3zgSVeBc6WJ1p!f2zR*OL^v zIS`0}Y56d_u@NyO!D69dY-;Fs8V0!7rpb5z`zEv{JEGy4U`gwBWP|Zg%S=2QF=6Ut z--KrUw{)U!0)3j;Stp>kNuJ9lz>DB8C|HYgjx|VYYv$rVuaa9(gsFc*M|3Sh_XJ+Q zCyvHLnA1Jx7Tp9if*PSFOWwd*NC%ACvlq-?5}3by`O@Vy70Je+y|p8O1%D&Pk#)X| z!sTiD??FeSmf{W=axf(CqdlQaLv5+0);W=O?O1$plpY;a1IFbYB`gom=AFO6+Gc;i z_~^Nc5KbXEVZ+jf9FN8nnv1n}839Zj6iTpqrD&wWC3v^-G85vWN!iBox}Sniy65C? zZE;!+P#DMR1bZ_bjC##t6q*Ao*76b6(sm=t@jZ~_ZE*nKy^7+qS%jXBkLAgH>Nd;7>2NfpC`pJnbDD2YEiG@ zo~zmC(;Gh?$kYwBRN|jm1q+;_pgL_}DwQtOVpknW(j>vBaw-;xz8E7S63<%1OKe9~ z#|&||<@4+6moFjbV*6k)(0LBWBvO8Qm8a%41QPnG)GitGsFO4h>1lOHU80kr&>BYZ z6q1evavVV}E3(zl`bVn|i-NSqWX{LhzDHl6dReQe5QJEOT??4e9E|xL3wz4IUn8NO zQ8Ga4O{j>>5l+qELdFvmon-LyD6&yP_=;nRjFSVI2YHjI78d9&xh{fSu#uuD$wS>I z<6z~bhm0O9b7g}m+X=OW6L*pUP{$^wZb*dHLMbir*@vhJ@g3XI)Uq=`3r1>O%TBQ% zthFTP3wk*7$igJdLSdC3RO6r4PmEJii_owmM-7>?$mekw7luMYY9T;u>9H*tyExbr zSwv(+hu-^FV$#Rq3*;-^9NN>t_u3$~it6DzdZ4f@y}7_cchcAS za<)x-IprWQ7z+eh@%$Br2Q6CVyosM6Ls(D7ajr!BdC=%FPg`=__h^PwV+GOBip@Sm zh{$|QOQ>G^28`@DQwbLDvNfWWaMubcsP{JAU98z-&qL55{D77vhDbn);Y{CyR z!g8<#)XJUZ7z`{aK;3{Kf3QXoiZo$MkjKHMQeZmZwc@~?B~vHeq$713Wj2m#W9obn z4e_RwgaBJoHI7T!O_U5)mGwAHho5DxO4YiZWtHw>O@jeS%XQk43ohK*&I>AG zS8baT>V6>va){eF)LNgyVSw>^3Pw^fd zCA-RAPe%+`Nv{zoG*LWAuNB}(dOwmrUB~VXdlhsZAqLAxR&-kQ|us8e+kCBcDyiJFs-;=#4qML+PLL9GK}82#>sCwa@#_I20P? z^%tEC!=2hOl2f+Or}K2E)cYJvB09N8U>;`uv*>;Tc@ItG?1Yv+(?CyPR>ta@mUlXg z(8+}A!))9piZq~4shAlGhFWwFttZfEPw$T9;5(6y=qRhNtE$9ikWPDA8}-UV^t2Rf z4h(~}(u74k<(sIDDAIRtW%MN5mPo{L^N%b=w$mgY1-v8BR| zNj^@fQu|LCYwW4~kM=x0e0&~jS^6POMD?8)**Z@IJ?3gU=hMpqi+^HL=1_&%BRv7i z-OLnE%T38TS{somR&?MDT7*MT=qW9RdXqX`AfG%3DBtvlMw2~TK(9W;oWbZgXp zF&4O}BiPEJ2j`hQywMD6IlYR5jxZI_BuiQ#!LDfLNt8XrQdVZuI);SO2#326szv-3%>%ndR- zQCP(~41xw~nxlB+-LeAUu_~MFw4YUV(DMm8w9zWV2(00ip&n9};gfkAU!r~_#KYV| zm$D+sTZLO!DWA12HPr56$^4ad>d2c{Md>+ETc1q^(V!u5P(1say71UAUpAv&gZj0? zY&)-d zbP7r(wk*6Gkz%+*;US_1A6BR=Bf0+TI`fJ`4lu6UvEETJyqBEx%rsr~k{B1N)NXV* z+vpV)Tx(ToGio(>xk3?{hBqo)jJKV5(5D7pOQGsDXyw?S$^A;B^aYo6*^K2HT%%A~ zMtuF(b%wqGOVJ6C{x2ri40Q?-HMl&*5GL1|ixu(*`BW)$25PaSBzh zK`Y1pO#ZH_)}5-pOoJ{Z5RY=5DN#t9u}s8G{}+>MhB}4GHAvqWNEeTCohecbztH5{ zRqgv#O1Auls{CL|s#*>HP@#xS!%rxj!@yVT^*PLSBRcKR=DJ*`N3;10s?}FhLe^_g z{|NHYY_6-nm8xE+!CbstMi&By2CW?Xvw2cg`)aECat+?CP&~|a=KeIDw3q%bHrEVw z3Y%+iV~Qb$xz609kY8x?v5Jtsbe=9U=1jamN7n+CVwh`inL-hnhUxcR=<46*x)GiB zXLDVy)1%qEUA5Yo60%-{1J&$uG@I+{>r>V1G7;_?zny}vR}q|;B3!P)vJ^wi6*_Z{>UA1CBZaEh zpp|2PHm9%XqU-ro_2n9TTcLQI>&&0ibW$<>U&gs+s8ft{4gNI65aV2D-dD&kv^o8v z6kYW5&~%Y8H!7WdO{Ey;8oX7Zh)lzGDqR0I*Ny13Kbz}vogU5RJ5{TXrG%{4;6l8< zPuJ0GuB-oNs(PIU|CmD6YtYKEKbzBUEzz}Br5NWL+^kSM&UNOdG@VpT{}-EUhB}4K zHQ1D52%GE7R~7OLZSGWrBUDPpJVEJlF zrW0|~|HbB-p-y3Q4LVW`VRN0SP{=Q|`MpX~`XxxZ$e4RneE_MxOoLA;6pwSA`DL0;Dy095%{4=v!ih9^Pl_StbDeocA-~Y()r#f zS30AgiKGkTT!R-W6p?ATL*e?jxo$+K{n=cX>-1c@`XcZ=d)q z<4~XYY{RRzGd=m_82i+6zbD>d=mmaH`~YKfR=7rxNg)+fHmII_4e z`eJ;WQD6AMhOJLzud$&|{7_@V^1kRxdLx0!GSoLPW|U?b>U;ls z;ng?6^}?$!U+aY*nSqgPmf`D#r-PSl2i=67tquJD;(nfSKCmn&0h`TtAd zU+@9IrHB0ofNl7tL;cGTEnJFUW>$FpLlUc$0VwH9Z`mlY8HGmcB*nmz^hI9?dV3$^ z9Y12p@??&e6ByrnJeC5FZ=UWI2WD$N8RKg)<9m0KL_`=OZR~znRB2C66ZP8E^kMow|>(okd$C2N*GD=gZh7P<| zIRVdtG66qSR@&GV1v7(FJXpac^eiN9=Jv=0P=+T>k=-3hAi7&lB(5X-1 zb)rtXbk-;ZWUa6F5jq`{q7)!=doUa5PrJKgx_!Oh(5bGk>t(io2I|wUT-Vq85}od~ zNxO6sPoS=kFFfRH1^&}kM%UN-7oF-mtwJq4G)(Ea_;g$3*ZUluo~Vem{JI^T()#6d zOMSf`(rLM>pDDjW*@5H8TIu(QKudHafXcgn(cQo>g)ZaPSxpyCTBaOG5{V_^(Sg%xdK$`R{pPx zwf!~vQB`02qu!tD^b%cBrJ45oX-0j$kJafu-H@uU+t+M=1%UQiGzRI?``sfmC7(&J z^E!P4WTawUKeaEO^{>^ZeFpiiuCMpcPE~)mlAp#OUAkRu|C@j#5?x>Kv-h4()cC4Y zNmBoHeVraciM4)eKkk<*EvcFU=z2Q+E6~>ZdLOdEuPSQ8s;p0?I-w0Q@zV0^{rv_N z5Oh15Ua3Q84cSNaKJL){6-qhE|MXl39OhKlKM*E*0p9qODhE{W+?BWpH3kM;*Wf6~4@Lq)*gN4b8%iSc#@^UU^1 SyuRvR_q^0!nNcAlYy3Zz{!GmP literal 0 HcmV?d00001 diff --git a/tools/gsutil/vendor_util.sh b/tools/gsutil/vendor_util.sh index 720eae042051..6f709291ac0e 100755 --- a/tools/gsutil/vendor_util.sh +++ b/tools/gsutil/vendor_util.sh @@ -15,6 +15,7 @@ fi ARCH=x86_64 HOST_UID="$(id -u)" HOST_GID="$(id -g)" +PYTHON_VERSION=3.11 docker run --rm \ -v "$PWD/tools/gsutil/crcmod:/output" \ @@ -24,10 +25,10 @@ docker run --rm \ && apt-get -qq install -y ca-certificates software-properties-common \ && add-apt-repository ppa:deadsnakes/ppa \ && apt-get -qq update -y \ - && apt-get -qq install -y build-essential curl python3.10 python3.10-dev \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 \ + && apt-get -qq install -y build-essential curl python${PYTHON_VERSION} python${PYTHON_VERSION}-dev \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ && update-alternatives --install /usr/bin/${ARCH}-linux-gnu-gcc ${ARCH}-linux-gnu-gcc /usr/bin/${ARCH}-linux-gnu-gcc-9 1 \ && pip install crcmod \ - && chown -R ${HOST_UID}:${HOST_GID} /usr/local/lib/python3.10/dist-packages/crcmod/ \ - && cp -a /usr/local/lib/python3.10/dist-packages/crcmod/*.so /output/" + && chown -R ${HOST_UID}:${HOST_GID} /usr/local/lib/python${PYTHON_VERSION}/dist-packages/crcmod/ \ + && cp -a /usr/local/lib/python${PYTHON_VERSION}/dist-packages/crcmod/*.so /output/" From 6e3b48dd9507693dd01881cec32ff20ce2bc9e96 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 16 May 2023 09:25:45 +0100 Subject: [PATCH 242/740] python: Workaround for template macro (#27418) Signed-off-by: Ryan Northey --- tools/base/envoy_python.bzl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/base/envoy_python.bzl b/tools/base/envoy_python.bzl index c92c761a5743..f88150fffc14 100644 --- a/tools/base/envoy_python.bzl +++ b/tools/base/envoy_python.bzl @@ -400,10 +400,11 @@ def envoy_gencontent( name = "%s_generate_content_py" % name, cmd = """ echo "import pathlib" > $@ \ + && echo "import os" >> $@ \ && echo "import sys" >> $@ \ - && echo "sys.path.append(pathlib.Path(\\"$(location :%s)\\").parent)" >> $@ \ + && echo "sys.path.append(str(pathlib.Path(os.environ.get('RUNFILES_DIR', '.')).joinpath(\\"$(rlocationpath :%s)\\").parent))" >> $@ \ && echo "from %s import data" >> $@ \ - && echo "sys.path.append(pathlib.Path(\\"$(location :%s)\\").parent)" >> $@ \ + && echo "sys.path.append(str(pathlib.Path(\\"$(rlocationpath :%s)\\").parent))" >> $@ \ && echo "from %s import env" >> $@ \ && echo "print(env.get_template(\\"%s\\").render(**data))" >> $@ """ % (name_data, name_data, name_tpl, name_tpl, template_name), From aa9f970162a21c86d8bb97ff5cb50c1eb6853d9d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 May 2023 09:45:19 +0100 Subject: [PATCH 243/740] build(deps): bump jaegertracing/all-in-one from `b49249c` to `308b14d` in /examples/shared/jaeger (#27417) build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger Bumps jaegertracing/all-in-one from `b49249c` to `308b14d`. --- updated-dependencies: - dependency-name: jaegertracing/all-in-one dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/jaeger/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile index f58a70da338f..2c87ffde174e 100644 --- a/examples/shared/jaeger/Dockerfile +++ b/examples/shared/jaeger/Dockerfile @@ -1,4 +1,4 @@ -FROM jaegertracing/all-in-one@sha256:b49249c222993a19c94a55b08174dce1962652224cce5cb0a2b3a31115457703 +FROM jaegertracing/all-in-one@sha256:308b14daf2ddbc171bd5ce882dd5b12c1bac6446913fbb60b69424f4a34b7ade HEALTHCHECK \ --interval=1s \ --timeout=1s \ From a6703d5cf446d209a7d9b450ab2b6c23d376b6ab Mon Sep 17 00:00:00 2001 From: DiazAlan <109677874+DiazAlan@users.noreply.github.com> Date: Tue, 16 May 2023 08:26:41 -0400 Subject: [PATCH 244/740] Fixing TCMalloc mapped pages bug in fixed heap monitor (#27359) * Discount mapped but unused TCMalloc pages from counting against memory usage for the fixed heap resource monitor. Signed-off-by: AlanDiaz --- changelogs/current.yaml | 5 ++ source/common/runtime/runtime_features.cc | 2 +- .../fixed_heap/fixed_heap_monitor.cc | 13 ++--- .../fixed_heap/fixed_heap_monitor_test.cc | 47 ++++++++----------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index a401b5dee946..682a596a9d8e 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -60,6 +60,11 @@ minor_behavior_changes: the cookie is still accepted, but is planned to be obsoleted in the future. This behavior change can be reverted by setting ``envoy.reloadable_features.stateful_session_encode_ttl_in_cookie`` to ``false``. +- area: resource_monitors + change: | + Changed behavior of the fixed heap monitor to count unused mapped pages as + free memory. This change can be reverted temporarily by setting the runtime guard + ``envoy.reloadable_features.count_unused_mapped_pages_as_free`` to false. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index a796495ea3c2..09b771d8baff 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -33,8 +33,8 @@ RUNTIME_GUARD(envoy_reloadable_features_allow_compact_maglev); RUNTIME_GUARD(envoy_reloadable_features_append_query_parameters_path_rewriter); RUNTIME_GUARD(envoy_reloadable_features_append_xfh_idempotent); RUNTIME_GUARD(envoy_reloadable_features_conn_pool_delete_when_idle); +RUNTIME_GUARD(envoy_reloadable_features_count_unused_mapped_pages_as_free); RUNTIME_GUARD(envoy_reloadable_features_delta_xds_subscription_state_tracking_fix); -RUNTIME_GUARD(envoy_reloadable_features_do_not_count_mapped_pages_as_free); RUNTIME_GUARD(envoy_reloadable_features_enable_aws_credentials_file); RUNTIME_GUARD(envoy_reloadable_features_enable_compression_bomb_protection); RUNTIME_GUARD(envoy_reloadable_features_enable_intermediate_ca); diff --git a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc index 5c6d4fab762e..c556ee258433 100644 --- a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc +++ b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc @@ -27,18 +27,19 @@ FixedHeapMonitor::FixedHeapMonitor( void FixedHeapMonitor::updateResourceUsage(Server::ResourceUpdateCallbacks& callbacks) { auto computeUsedMemory = [this]() -> size_t { + // TODO(Diazalan): Remove if statement once runtime guard is deprecated if (!Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.do_not_count_mapped_pages_as_free")) { + "envoy.reloadable_features.count_unused_mapped_pages_as_free")) { const size_t physical = stats_->reservedHeapBytes(); const size_t unmapped = stats_->unmappedHeapBytes(); - const size_t free_mapped = stats_->freeMappedHeapBytes(); - ASSERT(physical >= (unmapped + free_mapped)); - return physical - unmapped - free_mapped; + ASSERT(physical >= unmapped); + return physical - unmapped; } else { const size_t physical = stats_->reservedHeapBytes(); const size_t unmapped = stats_->unmappedHeapBytes(); - ASSERT(physical >= unmapped); - return physical - unmapped; + const size_t free_mapped = stats_->freeMappedHeapBytes(); + ASSERT(physical >= (unmapped + free_mapped)); + return physical - unmapped - free_mapped; } }; diff --git a/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc b/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc index d7a55f9b426f..5c5e558993c2 100644 --- a/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc +++ b/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc @@ -45,10 +45,6 @@ class ResourcePressure : public Server::ResourceUpdateCallbacks { TEST(FixedHeapMonitorTest, ComputesCorrectUsage) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.do_not_count_mapped_pages_as_free", "false"}}); - envoy::extensions::resource_monitors::fixed_heap::v3::FixedHeapConfig config; config.set_max_heap_size_bytes(1000); auto stats_reader = std::make_unique(); @@ -64,52 +60,49 @@ TEST(FixedHeapMonitorTest, ComputesCorrectUsage) { EXPECT_EQ(resource.pressure(), 0.5); } -TEST(FixedHeapMonitorTest, LegacyComputesCorrectUsage) { - - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.do_not_count_mapped_pages_as_free", "true"}}); +TEST(FixedHeapMonitorTest, ComputeUsageWithRealMemoryStats) { envoy::extensions::resource_monitors::fixed_heap::v3::FixedHeapConfig config; - config.set_max_heap_size_bytes(1000); - auto stats_reader = std::make_unique(); - EXPECT_CALL(*stats_reader, reservedHeapBytes()).WillOnce(Return(800)); - EXPECT_CALL(*stats_reader, unmappedHeapBytes()).WillOnce(Return(100)); + const uint64_t max_heap = 1024 * 1024 * 1024; + config.set_max_heap_size_bytes(max_heap); + auto stats_reader = std::make_unique(); + const double expected_usage = + (stats_reader->reservedHeapBytes() - stats_reader->unmappedHeapBytes() - + stats_reader->freeMappedHeapBytes()) / + static_cast(max_heap); auto monitor = std::make_unique(config, std::move(stats_reader)); ResourcePressure resource; monitor->updateResourceUsage(resource); - EXPECT_TRUE(resource.hasPressure()); - EXPECT_FALSE(resource.hasError()); - EXPECT_EQ(resource.pressure(), 0.7); + EXPECT_NEAR(resource.pressure(), expected_usage, 0.0005); } -TEST(FixedHeapMonitorTest, ComputeUsageWithRealMemoryStats) { +//// TODO(Diazalan): Remove two legacy tests after flag is deprecated +TEST(FixedHeapMonitorTest, LegacyComputesCorrectUsage) { TestScopedRuntime scoped_runtime; scoped_runtime.mergeValues( - {{"envoy.reloadable_features.do_not_count_mapped_pages_as_free", "false"}}); + {{"envoy.reloadable_features.count_unused_mapped_pages_as_free", "false"}}); envoy::extensions::resource_monitors::fixed_heap::v3::FixedHeapConfig config; - const uint64_t max_heap = 1024 * 1024 * 1024; - config.set_max_heap_size_bytes(max_heap); - auto stats_reader = std::make_unique(); - const double expected_usage = - (stats_reader->reservedHeapBytes() - stats_reader->unmappedHeapBytes() - - stats_reader->freeMappedHeapBytes()) / - static_cast(max_heap); + config.set_max_heap_size_bytes(1000); + auto stats_reader = std::make_unique(); + EXPECT_CALL(*stats_reader, reservedHeapBytes()).WillOnce(Return(800)); + EXPECT_CALL(*stats_reader, unmappedHeapBytes()).WillOnce(Return(100)); auto monitor = std::make_unique(config, std::move(stats_reader)); ResourcePressure resource; monitor->updateResourceUsage(resource); - EXPECT_NEAR(resource.pressure(), expected_usage, 0.0005); + EXPECT_TRUE(resource.hasPressure()); + EXPECT_FALSE(resource.hasError()); + EXPECT_EQ(resource.pressure(), 0.7); } TEST(FixedHeapMonitorTest, LegacyComputeUsageWithRealMemoryStats) { TestScopedRuntime scoped_runtime; scoped_runtime.mergeValues( - {{"envoy.reloadable_features.do_not_count_mapped_pages_as_free", "true"}}); + {{"envoy.reloadable_features.count_unused_mapped_pages_as_free", "false"}}); envoy::extensions::resource_monitors::fixed_heap::v3::FixedHeapConfig config; const uint64_t max_heap = 1024 * 1024 * 1024; From 10d55f2ed0c6b46a53f44fab73f5348e756d6a80 Mon Sep 17 00:00:00 2001 From: StarryNight Date: Tue, 16 May 2023 22:34:45 +0800 Subject: [PATCH 245/740] go extension trailer add api (#27350) Signed-off-by: wangkai19 --- contrib/golang/filters/http/source/cgo.cc | 14 ++++----- .../filters/http/source/go/pkg/api/api.h | 2 +- .../filters/http/source/go/pkg/api/capi.go | 2 +- .../http/source/go/pkg/http/capi_impl.go | 10 +++++-- .../filters/http/source/go/pkg/http/type.go | 22 +++++++++----- .../filters/http/source/golang_filter.cc | 30 ++++++++++++++----- .../filters/http/source/golang_filter.h | 2 +- .../http/test/golang_integration_test.cc | 18 ++++++++++- .../http/test/test_data/basic/filter.go | 3 ++ 9 files changed, 76 insertions(+), 27 deletions(-) diff --git a/contrib/golang/filters/http/source/cgo.cc b/contrib/golang/filters/http/source/cgo.cc index ea570145e15d..2529958a58e7 100644 --- a/contrib/golang/filters/http/source/cgo.cc +++ b/contrib/golang/filters/http/source/cgo.cc @@ -140,13 +140,13 @@ CAPIStatus envoyGoFilterHttpCopyTrailers(void* r, void* strs, void* buf) { }); } -CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key, void* value) { - return envoyGoFilterHandlerWrapper(r, - [key, value](std::shared_ptr& filter) -> CAPIStatus { - auto key_str = referGoString(key); - auto value_str = referGoString(value); - return filter->setTrailer(key_str, value_str); - }); +CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key, void* value, headerAction act) { + return envoyGoFilterHandlerWrapper( + r, [key, value, act](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = referGoString(key); + auto value_str = referGoString(value); + return filter->setTrailer(key_str, value_str, act); + }); } CAPIStatus envoyGoFilterHttpGetStringValue(void* r, int id, void* value) { diff --git a/contrib/golang/filters/http/source/go/pkg/api/api.h b/contrib/golang/filters/http/source/go/pkg/api/api.h index e2a9677d94bf..3ce0714a1415 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/api.h +++ b/contrib/golang/filters/http/source/go/pkg/api/api.h @@ -55,7 +55,7 @@ CAPIStatus envoyGoFilterHttpSetBufferHelper(void* r, unsigned long long int buff int length, bufferAction action); CAPIStatus envoyGoFilterHttpCopyTrailers(void* r, void* strs, void* buf); -CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key, void* value); +CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key, void* value, headerAction action); CAPIStatus envoyGoFilterHttpGetStringValue(void* r, int id, void* value); CAPIStatus envoyGoFilterHttpGetIntegerValue(void* r, int id, void* value); diff --git a/contrib/golang/filters/http/source/go/pkg/api/capi.go b/contrib/golang/filters/http/source/go/pkg/api/capi.go index 084d85341130..2cc1d2a273c5 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/capi.go +++ b/contrib/golang/filters/http/source/go/pkg/api/capi.go @@ -36,7 +36,7 @@ type HttpCAPI interface { HttpSetBufferHelper(r unsafe.Pointer, bufferPtr uint64, value string, action BufferAction) HttpCopyTrailers(r unsafe.Pointer, num uint64, bytes uint64) map[string][]string - HttpSetTrailer(r unsafe.Pointer, key *string, value *string) + HttpSetTrailer(r unsafe.Pointer, key *string, value *string, add bool) HttpGetStringValue(r unsafe.Pointer, id int) (string, bool) HttpGetIntegerValue(r unsafe.Pointer, id int) (uint64, bool) diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go index a4d49551022c..f89dc9f7175d 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go +++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go @@ -199,8 +199,14 @@ func (c *httpCApiImpl) HttpCopyTrailers(r unsafe.Pointer, num uint64, bytes uint return m } -func (c *httpCApiImpl) HttpSetTrailer(r unsafe.Pointer, key *string, value *string) { - res := C.envoyGoFilterHttpSetTrailer(r, unsafe.Pointer(key), unsafe.Pointer(value)) +func (c *httpCApiImpl) HttpSetTrailer(r unsafe.Pointer, key *string, value *string, add bool) { + var act C.headerAction + if add { + act = C.HeaderAdd + } else { + act = C.HeaderSet + } + res := C.envoyGoFilterHttpSetTrailer(r, unsafe.Pointer(key), unsafe.Pointer(value), act) handleCApiStatus(res) } diff --git a/contrib/golang/filters/http/source/go/pkg/http/type.go b/contrib/golang/filters/http/source/go/pkg/http/type.go index 72e77dbe37e5..0131fdcb677b 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/type.go +++ b/contrib/golang/filters/http/source/go/pkg/http/type.go @@ -173,7 +173,7 @@ type requestOrResponseTrailerMapImpl struct { headerMapImpl } -func (h *requestOrResponseTrailerMapImpl) initHeaders() { +func (h *requestOrResponseTrailerMapImpl) initTrailers() { if h.headers == nil { h.headers = cAPI.HttpCopyTrailers(unsafe.Pointer(h.request.req), h.headerNum, h.headerBytes) } @@ -184,7 +184,7 @@ func (h *requestOrResponseTrailerMapImpl) GetRaw(key string) string { } func (h *requestOrResponseTrailerMapImpl) Get(key string) (string, bool) { - h.initHeaders() + h.initTrailers() value, ok := h.headers[key] if !ok { return "", false @@ -193,7 +193,7 @@ func (h *requestOrResponseTrailerMapImpl) Get(key string) (string, bool) { } func (h *requestOrResponseTrailerMapImpl) Values(key string) []string { - h.initHeaders() + h.initTrailers() value, ok := h.headers[key] if !ok { return nil @@ -205,16 +205,24 @@ func (h *requestOrResponseTrailerMapImpl) Set(key, value string) { // Get all header values first before setting a value, since the set operation may not take affects immediately // when it's invoked in a Go thread, instead, it will post a callback to run in the envoy worker thread. // Otherwise, we may get outdated values in a following Get call. - h.initHeaders() + h.initTrailers() if h.headers != nil { h.headers[key] = []string{value} } - cAPI.HttpSetTrailer(unsafe.Pointer(h.request.req), &key, &value) + cAPI.HttpSetTrailer(unsafe.Pointer(h.request.req), &key, &value, false) } func (h *requestOrResponseTrailerMapImpl) Add(key, value string) { - panic("unsupported yet") + h.initTrailers() + if h.headers != nil { + if trailers, found := h.headers[key]; found { + h.headers[key] = append(trailers, value) + } else { + h.headers[key] = []string{value} + } + } + cAPI.HttpSetTrailer(unsafe.Pointer(h.request.req), &key, &value, true) } func (h *requestOrResponseTrailerMapImpl) Del(key string) { @@ -222,7 +230,7 @@ func (h *requestOrResponseTrailerMapImpl) Del(key string) { } func (h *requestOrResponseTrailerMapImpl) Range(f func(key, value string) bool) { - h.initHeaders() + h.initTrailers() for key, values := range h.headers { for _, value := range values { if !f(key, value) { diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc index ba5bb9b6fcf3..bb210a5f6816 100644 --- a/contrib/golang/filters/http/source/golang_filter.cc +++ b/contrib/golang/filters/http/source/golang_filter.cc @@ -332,6 +332,7 @@ bool Filter::doTrailer(ProcessorState& state, Http::HeaderMap& trailers) { } bool done = false; + Buffer::OwnedImpl body; switch (state.state()) { case FilterState::WaitingTrailer: done = doTrailerGo(state, trailers); @@ -344,11 +345,15 @@ bool Filter::doTrailer(ProcessorState& state, Http::HeaderMap& trailers) { // do data first if (!state.isBufferDataEmpty()) { done = doDataGo(state, state.getBufferData(), false); - } - // NP: can not use done as condition here, since done will be false - // maybe we can remove the done variable totally? by using state_ only? - // continue trailers - if (state.state() == FilterState::WaitingTrailer) { + // NP: can not use done as condition here, since done will be false + // maybe we can remove the done variable totally? by using state_ only? + // continue trailers + if (state.state() == FilterState::WaitingTrailer) { + state.continueDoData(); + done = doTrailerGo(state, trailers); + } + } else { + state.continueDoData(); done = doTrailerGo(state, trailers); } break; @@ -820,7 +825,7 @@ CAPIStatus Filter::copyTrailers(GoString* go_strs, char* go_buf) { return CAPIStatus::CAPIOK; } -CAPIStatus Filter::setTrailer(absl::string_view key, absl::string_view value) { +CAPIStatus Filter::setTrailer(absl::string_view key, absl::string_view value, headerAction act) { Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -835,7 +840,18 @@ CAPIStatus Filter::setTrailer(absl::string_view key, absl::string_view value) { ENVOY_LOG(debug, "invoking cgo api at invalid phase: {}", __func__); return CAPIStatus::CAPIInvalidPhase; } - trailers_->setCopy(Http::LowerCaseString(key), value); + switch (act) { + case HeaderAdd: + trailers_->addCopy(Http::LowerCaseString(key), value); + break; + + case HeaderSet: + trailers_->setCopy(Http::LowerCaseString(key), value); + break; + + default: + RELEASE_ASSERT(false, absl::StrCat("unknown header action: ", act)); + } return CAPIStatus::CAPIOK; } diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h index c9d9c7f14d47..bf456e43ee8f 100644 --- a/contrib/golang/filters/http/source/golang_filter.h +++ b/contrib/golang/filters/http/source/golang_filter.h @@ -174,7 +174,7 @@ class Filter : public Http::StreamFilter, CAPIStatus setBufferHelper(Buffer::Instance* buffer, absl::string_view& value, bufferAction action); CAPIStatus copyTrailers(GoString* go_strs, char* go_buf); - CAPIStatus setTrailer(absl::string_view key, absl::string_view value); + CAPIStatus setTrailer(absl::string_view key, absl::string_view value, headerAction act); CAPIStatus getStringValue(int id, GoString* value_str); CAPIStatus getIntegerValue(int id, uint64_t* value); CAPIStatus setDynamicMetadata(std::string filter_name, std::string key, absl::string_view buf); diff --git a/contrib/golang/filters/http/test/golang_integration_test.cc b/contrib/golang/filters/http/test/golang_integration_test.cc index 31348b07aa68..0b3efc96a2f8 100644 --- a/contrib/golang/filters/http/test/golang_integration_test.cc +++ b/contrib/golang/filters/http/test/golang_integration_test.cc @@ -156,6 +156,8 @@ name: golang ->mutable_route() ->set_cluster("cluster_0"); }); + config_helper_.addConfigModifier(setEnableDownstreamTrailersHttp1()); + config_helper_.addConfigModifier(setEnableUpstreamTrailersHttp1()); initialize(); } @@ -236,7 +238,11 @@ name: golang Http::RequestEncoder& request_encoder = encoder_decoder.first; auto response = std::move(encoder_decoder.second); codec_client_->sendData(request_encoder, "helloworld", false); - codec_client_->sendData(request_encoder, "", true); + codec_client_->sendData(request_encoder, "", false); + + Http::TestRequestTrailerMapImpl request_trailers{{"x-test-trailer-0", "foo"}, + {"existed-trailer", "foo"}}; + codec_client_->sendTrailers(request_encoder, request_trailers); waitForNextUpstreamRequest(); // original header: x-test-header-0 @@ -276,6 +282,16 @@ name: golang // only match the prefix since data buffer may be combined into a single. EXPECT_EQ(expected, upstream_request_->body().toString()); + // check trailer value which is appended in golang: existed-trailer + entries = upstream_request_->trailers()->get(Http::LowerCaseString("existed-trailer")); + EXPECT_EQ(2, entries.size()); + EXPECT_EQ("foo", entries[0]->value().getStringView()); + EXPECT_EQ("bar", entries[1]->value().getStringView()); + + // check trailer value which set in golang: x-test-trailer-0 + entries = upstream_request_->trailers()->get(Http::LowerCaseString("x-test-trailer-0")); + EXPECT_EQ("bar", entries[0]->value().getStringView()); + Http::TestResponseHeaderMapImpl response_headers{ {":status", "200"}, {"x-test-header-0", "foo"}, diff --git a/contrib/golang/filters/http/test/test_data/basic/filter.go b/contrib/golang/filters/http/test/test_data/basic/filter.go index cda1699228e2..baaf13f7f0ee 100644 --- a/contrib/golang/filters/http/test/test_data/basic/filter.go +++ b/contrib/golang/filters/http/test/test_data/basic/filter.go @@ -189,6 +189,9 @@ func (f *filter) decodeTrailers(trailers api.RequestTrailerMap) api.StatusType { return f.sendLocalReply("decode-trailer") } + trailers.Add("existed-trailer", "bar") + trailers.Set("x-test-trailer-0", "bar") + if f.panic == "decode-trailer" { badcode() } From 0a41170f0eec59f284e658b689eeabe83f20a48b Mon Sep 17 00:00:00 2001 From: doujiang24 Date: Tue, 16 May 2023 07:35:53 -0700 Subject: [PATCH 246/740] refactor DFP cluster to support STRICT_DNS (#26420) Signed-off-by: doujiang24 --- .../clusters/dynamic_forward_proxy/v3/BUILD | 2 + .../dynamic_forward_proxy/v3/cluster.proto | 50 +++- .../v3/dynamic_forward_proxy.proto | 22 +- changelogs/current.yaml | 5 + envoy/upstream/cluster_manager.h | 5 + .../common/upstream/cluster_manager_impl.cc | 3 +- source/common/upstream/cluster_manager_impl.h | 5 + .../clusters/dynamic_forward_proxy/BUILD | 1 + .../clusters/dynamic_forward_proxy/cluster.cc | 181 ++++++++++++- .../clusters/dynamic_forward_proxy/cluster.h | 36 +++ .../common/dynamic_forward_proxy/BUILD | 12 + .../dynamic_forward_proxy/cluster_store.cc | 41 +++ .../dynamic_forward_proxy/cluster_store.h | 79 ++++++ .../filters/http/dynamic_forward_proxy/BUILD | 3 + .../http/dynamic_forward_proxy/config.cc | 3 +- .../dynamic_forward_proxy/proxy_filter.cc | 202 +++++++++++++-- .../http/dynamic_forward_proxy/proxy_filter.h | 83 +++++- .../dynamic_forward_proxy/cluster_test.cc | 29 +++ .../common/dynamic_forward_proxy/BUILD | 1 + .../common/dynamic_forward_proxy/mocks.h | 13 + .../filters/http/dynamic_forward_proxy/BUILD | 2 + .../proxy_filter_integration_test.cc | 173 ++++++++++++- .../proxy_filter_test.cc | 237 ++++++++++++++---- test/mocks/upstream/cluster_manager.h | 1 + 24 files changed, 1100 insertions(+), 89 deletions(-) create mode 100644 source/extensions/common/dynamic_forward_proxy/cluster_store.cc create mode 100644 source/extensions/common/dynamic_forward_proxy/cluster_store.h diff --git a/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/BUILD b/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/BUILD index 05f25a2fe5d9..00ebd40c5d65 100644 --- a/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/BUILD +++ b/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/BUILD @@ -6,6 +6,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/cluster/v3:pkg", + "//envoy/config/core/v3:pkg", "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], diff --git a/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.proto b/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.proto index 9c711ae13f60..8cb7183fc4d9 100644 --- a/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.proto +++ b/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.proto @@ -2,8 +2,13 @@ syntax = "proto3"; package envoy.extensions.clusters.dynamic_forward_proxy.v3; +import "envoy/config/cluster/v3/cluster.proto"; +import "envoy/config/core/v3/address.proto"; import "envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -23,11 +28,27 @@ message ClusterConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.cluster.dynamic_forward_proxy.v2alpha.ClusterConfig"; - // The DNS cache configuration that the cluster will attach to. Note this configuration must - // match that of associated :ref:`dynamic forward proxy HTTP filter configuration - // `. - common.dynamic_forward_proxy.v3.DnsCacheConfig dns_cache_config = 1 - [(validate.rules).message = {required: true}]; + oneof cluster_implementation_specifier { + // The DNS cache configuration that the cluster will attach to. Note this configuration must + // match that of associated :ref:`dynamic forward proxy HTTP filter configuration + // `. + common.dynamic_forward_proxy.v3.DnsCacheConfig dns_cache_config = 1; + + // Configuration for sub clusters, when this configuration is enabled, + // Envoy will create an independent sub cluster dynamically for each host:port. + // Most of the configuration of a sub cluster is inherited from the current cluster, + // i.e. health_checks, dns_resolvers and etc. + // And the load_assignment will be set to the only one endpoint, host:port. + // + // Compared to the dns_cache_config, it has the following advantages: + // + // 1. sub clusters will be created with the STRICT_DNS DiscoveryType, + // so that Envoy will use all of the IPs resolved from the host. + // + // 2. each sub cluster is full featured cluster, with lb_policy and health check and etc enabled. + // + SubClustersConfig sub_clusters_config = 4; + } // If true allow the cluster configuration to disable the auto_sni and auto_san_validation options // in the :ref:`cluster's upstream_http_protocol_options @@ -54,3 +75,22 @@ message ClusterConfig { // bool allow_coalesced_connections = 3; } + +// Configuration for sub clusters. Hard code STRICT_DNS cluster type now. +message SubClustersConfig { + // The :ref:`load balancer type ` to use + // when picking a host in a sub cluster. Note that CLUSTER_PROVIDED is not allowed here. + config.cluster.v3.Cluster.LbPolicy lb_policy = 1 [(validate.rules).enum = {defined_only: true}]; + + // The maximum number of sub clusters that the DFP cluster will hold. If not specified defaults to 1024. + google.protobuf.UInt32Value max_sub_clusters = 2 [(validate.rules).uint32 = {gt: 0}]; + + // The TTL for sub clusters that are unused. Sub clusters that have not been used in the configured time + // interval will be purged. If not specified defaults to 5m. + google.protobuf.Duration sub_cluster_ttl = 3 [(validate.rules).duration = {gt {}}]; + + // Sub clusters that should be created & warmed upon creation. This might provide a + // performance improvement, in the form of cache hits, for sub clusters that are going to be + // warmed during steady state and are known at config load time. + repeated config.core.v3.SocketAddress preresolve_clusters = 4; +} diff --git a/api/envoy/extensions/filters/http/dynamic_forward_proxy/v3/dynamic_forward_proxy.proto b/api/envoy/extensions/filters/http/dynamic_forward_proxy/v3/dynamic_forward_proxy.proto index 50bbea629d55..1ef5e024273b 100644 --- a/api/envoy/extensions/filters/http/dynamic_forward_proxy/v3/dynamic_forward_proxy.proto +++ b/api/envoy/extensions/filters/http/dynamic_forward_proxy/v3/dynamic_forward_proxy.proto @@ -4,6 +4,8 @@ package envoy.extensions.filters.http.dynamic_forward_proxy.v3; import "envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto"; +import "google/protobuf/duration.proto"; + import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -23,11 +25,16 @@ message FilterConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.dynamic_forward_proxy.v2alpha.FilterConfig"; - // The DNS cache configuration that the filter will attach to. Note this configuration must - // match that of associated :ref:`dynamic forward proxy cluster configuration - // `. - common.dynamic_forward_proxy.v3.DnsCacheConfig dns_cache_config = 1 - [(validate.rules).message = {required: true}]; + oneof implementation_specifier { + // The DNS cache configuration that the filter will attach to. Note this configuration must + // match that of associated :ref:`dynamic forward proxy cluster configuration + // `. + common.dynamic_forward_proxy.v3.DnsCacheConfig dns_cache_config = 1; + + // The configuration that the filter will use, when the related dynamic forward proxy cluster enabled + // sub clusters. + SubClusterConfig sub_cluster_config = 3; + } // When this flag is set, the filter will add the resolved upstream address in the filter // state. The state should be saved with key @@ -69,3 +76,8 @@ message PerRouteConfig { string host_rewrite_header = 2; } } + +message SubClusterConfig { + // The timeout used for sub cluster initialization. Defaults to 5s if not set. + google.protobuf.Duration cluster_init_timeout = 3 [(validate.rules).duration = {gt {}}]; +} diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 682a596a9d8e..2083295a0972 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -121,6 +121,11 @@ new_features: - area: access_log change: | added :ref:`CEL ` access log formatter to print CEL expression. +- area: dynamic_forward_proxy + change: | + added :ref:`sub_clusters_config + ` to enable + independent sub cluster for each host:port, with STRICT_DNS cluster type. - area: http change: | added Runtime feature ``envoy.reloadable_features.max_request_headers_size_kb`` to override the default value of diff --git a/envoy/upstream/cluster_manager.h b/envoy/upstream/cluster_manager.h index f6598ad612d2..319cbec9c754 100644 --- a/envoy/upstream/cluster_manager.h +++ b/envoy/upstream/cluster_manager.h @@ -315,6 +315,11 @@ class ClusterManager { */ virtual void shutdown() PURE; + /** + * @return whether the shutdown method has been called. + */ + virtual bool isShutdown() PURE; + /** * @return cluster manager wide bind configuration for new upstream connections. */ diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 90105f2daee1..fd24da4c9770 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -303,7 +303,8 @@ ClusterManagerImpl::ClusterManagerImpl( common_lb_config_pool_( std::make_shared>( - main_thread_dispatcher)) { + main_thread_dispatcher)), + shutdown_(false) { if (admin.has_value()) { config_tracker_entry_ = admin->getConfigTracker().add( "clusters", [this](const Matchers::StringMatcher& name_matcher) { diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 77d8b2d1b4bc..11c44a7365b1 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -280,6 +280,7 @@ class ClusterManagerImpl : public ClusterManager, bool removeCluster(const std::string& cluster) override; void shutdown() override { + shutdown_ = true; if (resume_cds_ != nullptr) { resume_cds_->cancel(); } @@ -291,6 +292,8 @@ class ClusterManagerImpl : public ClusterManager, updateClusterCounts(); } + bool isShutdown() override { return shutdown_; } + const absl::optional& bindConfig() const override { return bind_config_; } @@ -822,6 +825,8 @@ class ClusterManagerImpl : public ClusterManager, std::unique_ptr xds_resources_delegate_; std::unique_ptr xds_config_tracker_; + + std::atomic shutdown_{}; }; } // namespace Upstream diff --git a/source/extensions/clusters/dynamic_forward_proxy/BUILD b/source/extensions/clusters/dynamic_forward_proxy/BUILD index a5e377f9b83a..22d3ceacdfdb 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/BUILD +++ b/source/extensions/clusters/dynamic_forward_proxy/BUILD @@ -17,6 +17,7 @@ envoy_cc_extension( "//source/common/network:transport_socket_options_lib", "//source/common/upstream:cluster_factory_lib", "//source/extensions/clusters/common:logical_host_lib", + "//source/extensions/common/dynamic_forward_proxy:cluster_store", "//source/extensions/common/dynamic_forward_proxy:dns_cache_interface", "//source/extensions/common/dynamic_forward_proxy:dns_cache_manager_impl", "//source/extensions/transport_sockets/tls/cert_validator:cert_validator_lib", diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc index d4a211a8348c..3d400213b7c4 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -28,7 +28,44 @@ Cluster::Cluster( dns_cache_(dns_cache_manager_->getCache(config.dns_cache_config())), update_callbacks_handle_(dns_cache_->addUpdateCallbacks(*this)), local_info_(context.serverFactoryContext().localInfo()), - allow_coalesced_connections_(config.allow_coalesced_connections()) {} + main_thread_dispatcher_(context.serverFactoryContext().mainThreadDispatcher()), + orig_cluster_config_(cluster), + allow_coalesced_connections_(config.allow_coalesced_connections()), + cm_(context.clusterManager()), max_sub_clusters_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( + config.sub_clusters_config(), max_sub_clusters, 1024)), + sub_cluster_ttl_( + PROTOBUF_GET_MS_OR_DEFAULT(config.sub_clusters_config(), sub_cluster_ttl, 300000)), + sub_cluster_lb_policy_(config.sub_clusters_config().lb_policy()), + enable_sub_cluster_(config.has_sub_clusters_config()) { + + if (enable_sub_cluster_) { + if (sub_cluster_lb_policy_ == + envoy::config::cluster::v3::Cluster_LbPolicy::Cluster_LbPolicy_CLUSTER_PROVIDED) { + throw EnvoyException("unsupported lb_policy 'CLUSTER_PROVIDED' in sub_cluster_config"); + } + idle_timer_ = main_thread_dispatcher_.createTimer([this]() { checkIdleSubCluster(); }); + idle_timer_->enableTimer(sub_cluster_ttl_); + } +} + +Cluster::~Cluster() { + if (enable_sub_cluster_) { + idle_timer_->disableTimer(); + idle_timer_.reset(); + } + if (cm_.isShutdown()) { + return; + } + // Should remove all sub clusters, otherwise, might be memory leaking. + // This lock is useless, just make compiler happy. + absl::WriterMutexLock lock{&cluster_map_lock_}; + for (auto it = cluster_map_.cbegin(); it != cluster_map_.cend();) { + auto cluster_name = it->first; + ENVOY_LOG(debug, "cluster='{}' removing from cluster_map & cluster manager", cluster_name); + cluster_map_.erase(it++); + cm_.removeCluster(cluster_name); + } +} void Cluster::startPreInit() { // If we are attaching to a pre-populated cache we need to initialize our hosts. @@ -43,6 +80,132 @@ void Cluster::startPreInit() { onPreInitComplete(); } +bool Cluster::touch(const std::string& cluster_name) { + absl::ReaderMutexLock lock{&cluster_map_lock_}; + const auto cluster_it = cluster_map_.find(cluster_name); + if (cluster_it != cluster_map_.end()) { + cluster_it->second->touch(); + return true; + } + ENVOY_LOG(debug, "cluster='{}' has been removed while touching", cluster_name); + return false; +} + +void Cluster::checkIdleSubCluster() { + ASSERT(main_thread_dispatcher_.isThreadSafe()); + { + // TODO: try read lock first. + absl::WriterMutexLock lock{&cluster_map_lock_}; + for (auto it = cluster_map_.cbegin(); it != cluster_map_.cend();) { + if (it->second->checkIdle()) { + auto cluster_name = it->first; + ENVOY_LOG(debug, "cluster='{}' removing from cluster_map & cluster manager", cluster_name); + cluster_map_.erase(it++); + cm_.removeCluster(cluster_name); + } else { + ++it; + } + } + } + idle_timer_->enableTimer(sub_cluster_ttl_); +} + +std::pair> +Cluster::createSubClusterConfig(const std::string& cluster_name, const std::string& host, + const int port) { + { + absl::WriterMutexLock lock{&cluster_map_lock_}; + const auto cluster_it = cluster_map_.find(cluster_name); + if (cluster_it != cluster_map_.end()) { + cluster_it->second->touch(); + return std::make_pair(true, absl::nullopt); + } + if (cluster_map_.size() >= max_sub_clusters_) { + ENVOY_LOG(debug, "cluster='{}' create failed due to max sub cluster limitation", + cluster_name); + return std::make_pair(false, absl::nullopt); + } + cluster_map_.emplace(cluster_name, std::make_shared(cluster_name, *this)); + } + + // Inherit configuration from the parent DFP cluster. + envoy::config::cluster::v3::Cluster config = orig_cluster_config_; + + // Overwrite the type. + config.set_name(cluster_name); + config.clear_cluster_type(); + config.set_lb_policy(sub_cluster_lb_policy_); + config.set_type( + envoy::config::cluster::v3::Cluster_DiscoveryType::Cluster_DiscoveryType_STRICT_DNS); + + // Set endpoint. + auto load_assignments = config.mutable_load_assignment(); + load_assignments->set_cluster_name(cluster_name); + load_assignments->clear_endpoints(); + + auto socket_address = load_assignments->add_endpoints() + ->add_lb_endpoints() + ->mutable_endpoint() + ->mutable_address() + ->mutable_socket_address(); + socket_address->set_address(host); + socket_address->set_port_value(port); + + return std::make_pair(true, absl::make_optional(config)); +} + +Upstream::HostConstSharedPtr Cluster::chooseHost(absl::string_view host, + Upstream::LoadBalancerContext* context) const { + uint16_t default_port = 80; + if (info_->transportSocketMatcher().resolve(nullptr).factory_.implementsSecureTransport()) { + default_port = 443; + } + + const auto host_attributes = Http::Utility::parseAuthority(host); + auto dynamic_host = std::string(host_attributes.host_); + auto port = host_attributes.port_.value_or(default_port); + + // cluster name is prefix + host + port + auto cluster_name = "DFPCluster:" + dynamic_host + ":" + std::to_string(port); + + // try again to get the sub cluster. + auto cluster = cm_.getThreadLocalCluster(cluster_name); + if (cluster == nullptr) { + ENVOY_LOG(debug, "cluster='{}' get thread local failed, too short ttl?", cluster_name); + return nullptr; + } + + return cluster->loadBalancer().chooseHost(context); +} + +Cluster::ClusterInfo::ClusterInfo(std::string cluster_name, Cluster& parent) + : cluster_name_(cluster_name), parent_(parent) { + ENVOY_LOG(debug, "cluster='{}' ClusterInfo created", cluster_name_); + touch(); +} + +void Cluster::ClusterInfo::touch() { + ENVOY_LOG(debug, "cluster='{}' updating last used time", cluster_name_); + last_used_time_ = parent_.time_source_.monotonicTime().time_since_epoch(); +} + +// checkIdle run in the main thread. +bool Cluster::ClusterInfo::checkIdle() { + ASSERT(parent_.main_thread_dispatcher_.isThreadSafe()); + + const std::chrono::steady_clock::duration now_duration = + parent_.main_thread_dispatcher_.timeSource().monotonicTime().time_since_epoch(); + auto last_used_time = last_used_time_.load(); + ENVOY_LOG(debug, "cluster='{}' TTL check: now={} last_used={} TTL {}", cluster_name_, + now_duration.count(), last_used_time.count(), parent_.sub_cluster_ttl_.count()); + + if ((now_duration - last_used_time) > parent_.sub_cluster_ttl_) { + ENVOY_LOG(debug, "cluster='{}' TTL expired", cluster_name_); + return true; + } + return false; +} + void Cluster::addOrUpdateHost( absl::string_view host, const Extensions::Common::DynamicForwardProxy::DnsHostInfoSharedPtr& host_info, @@ -53,9 +216,10 @@ void Cluster::addOrUpdateHost( // NOTE: Right now we allow a DNS cache to be shared between multiple clusters. Though we have // connection/request circuit breakers on the cluster, we don't have any way to control the - // maximum hosts on a cluster. We currently assume that host data shared via shared pointer is a - // marginal memory cost above that already used by connections and requests, so relying on - // connection/request circuit breakers is sufficient. We may have to revisit this in the future. + // maximum hosts on a cluster. We currently assume that host data shared via shared pointer is + // a marginal memory cost above that already used by connections and requests, so relying on + // connection/request circuit breakers is sufficient. We may have to revisit this in the + // future. const auto host_map_it = host_map_.find(host); if (host_map_it != host_map_.end()) { // If we only have an address change, we can do that swap inline without any other updates. @@ -170,6 +334,11 @@ Cluster::LoadBalancer::chooseHost(Upstream::LoadBalancerContext* context) { if (host.empty()) { return nullptr; } + + if (cluster_.enableSubCluster()) { + return cluster_.chooseHost(host, context); + } + { absl::ReaderMutexLock lock{&cluster_.host_map_lock_}; const auto host_it = cluster_.host_map_.find(host); @@ -281,6 +450,10 @@ ClusterFactory::createClusterWithConfig( auto new_cluster = std::make_shared(cluster_config, proto_config, context, cache_manager_factory); + Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( + factory_context_base); + cluster_store_factory.get()->save(new_cluster->info()->name(), new_cluster); + auto& options = new_cluster->info()->upstreamHttpProtocolOptions(); if (!proto_config.allow_insecure_cluster_options()) { diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.h b/source/extensions/clusters/dynamic_forward_proxy/cluster.h index 9a7d4d51aa97..9a38eb362abe 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.h +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.h @@ -8,6 +8,7 @@ #include "source/common/upstream/cluster_factory_impl.h" #include "source/extensions/clusters/common/logical_host.h" +#include "source/extensions/common/dynamic_forward_proxy/cluster_store.h" #include "source/extensions/common/dynamic_forward_proxy/dns_cache.h" namespace Envoy { @@ -16,12 +17,14 @@ namespace Clusters { namespace DynamicForwardProxy { class Cluster : public Upstream::BaseDynamicClusterImpl, + public Extensions::Common::DynamicForwardProxy::DfpCluster, public Extensions::Common::DynamicForwardProxy::DnsCache::UpdateCallbacks { public: Cluster(const envoy::config::cluster::v3::Cluster& cluster, const envoy::extensions::clusters::dynamic_forward_proxy::v3::ClusterConfig& config, Upstream::ClusterFactoryContext& context, Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory); + ~Cluster() override; // Upstream::Cluster Upstream::Cluster::InitializePhase initializePhase() const override { @@ -41,8 +44,28 @@ class Cluster : public Upstream::BaseDynamicClusterImpl, Network::DnsResolver::ResolutionStatus) override {} bool allowCoalescedConnections() const { return allow_coalesced_connections_; } + bool enableSubCluster() const override { return enable_sub_cluster_; } + Upstream::HostConstSharedPtr chooseHost(absl::string_view host, + Upstream::LoadBalancerContext* context) const; + std::pair> + createSubClusterConfig(const std::string& cluster_name, const std::string& host, + const int port) override; + bool touch(const std::string& cluster_name) override; + void checkIdleSubCluster(); private: + struct ClusterInfo { + ClusterInfo(std::string cluster_name, Cluster& parent); + void touch(); + bool checkIdle(); + + std::string cluster_name_; + Cluster& parent_; + std::atomic last_used_time_; + }; + + using ClusterInfoMap = absl::flat_hash_map>; + struct HostInfo { HostInfo(const Extensions::Common::DynamicForwardProxy::DnsHostInfoSharedPtr& shared_host_info, const Upstream::LogicalHostSharedPtr& logical_host) @@ -145,6 +168,10 @@ class Cluster : public Upstream::BaseDynamicClusterImpl, const envoy::config::endpoint::v3::LocalityLbEndpoints dummy_locality_lb_endpoint_; const envoy::config::endpoint::v3::LbEndpoint dummy_lb_endpoint_; const LocalInfo::LocalInfo& local_info_; + Event::Dispatcher& main_thread_dispatcher_; + const envoy::config::cluster::v3::Cluster orig_cluster_config_; + + Event::TimerPtr idle_timer_; // True if H2 and H3 connections may be reused across different origins. const bool allow_coalesced_connections_; @@ -152,6 +179,15 @@ class Cluster : public Upstream::BaseDynamicClusterImpl, mutable absl::Mutex host_map_lock_; HostInfoMap host_map_ ABSL_GUARDED_BY(host_map_lock_); + mutable absl::Mutex cluster_map_lock_; + ClusterInfoMap cluster_map_ ABSL_GUARDED_BY(cluster_map_lock_); + + Upstream::ClusterManager& cm_; + const size_t max_sub_clusters_; + const std::chrono::milliseconds sub_cluster_ttl_; + const envoy::config::cluster::v3::Cluster_LbPolicy sub_cluster_lb_policy_; + const bool enable_sub_cluster_; + friend class ClusterFactory; friend class ClusterTest; }; diff --git a/source/extensions/common/dynamic_forward_proxy/BUILD b/source/extensions/common/dynamic_forward_proxy/BUILD index ceb356d83150..df95f5254c54 100644 --- a/source/extensions/common/dynamic_forward_proxy/BUILD +++ b/source/extensions/common/dynamic_forward_proxy/BUILD @@ -69,3 +69,15 @@ envoy_cc_library( "@envoy_api//envoy/extensions/common/dynamic_forward_proxy/v3:pkg_cc_proto", ], ) + +envoy_cc_library( + name = "cluster_store", + srcs = ["cluster_store.cc"], + hdrs = ["cluster_store.h"], + deps = [ + "//envoy/server:factory_context_interface", + "//envoy/upstream:upstream_interface", + "//source/server:factory_context_base_impl_lib", + "@com_google_absl//absl/container:flat_hash_set", + ], +) diff --git a/source/extensions/common/dynamic_forward_proxy/cluster_store.cc b/source/extensions/common/dynamic_forward_proxy/cluster_store.cc new file mode 100644 index 000000000000..96ba5c151ad4 --- /dev/null +++ b/source/extensions/common/dynamic_forward_proxy/cluster_store.cc @@ -0,0 +1,41 @@ +#include "source/extensions/common/dynamic_forward_proxy/cluster_store.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { + +SINGLETON_MANAGER_REGISTRATION(dynamic_forward_proxy_cluster_store); + +DfpClusterSharedPtr DFPClusterStore::load(const std::string cluster_name) { + ClusterStoreType& clusterStore = getClusterStore(); + absl::ReaderMutexLock lock(&clusterStore.mutex_); + auto it = clusterStore.map_.find(cluster_name); + if (it != clusterStore.map_.end()) { + return it->second.lock(); + } + return nullptr; +} + +void DFPClusterStore::save(const std::string cluster_name, DfpClusterSharedPtr cluster) { + ClusterStoreType& clusterStore = getClusterStore(); + absl::WriterMutexLock lock(&clusterStore.mutex_); + clusterStore.map_[cluster_name] = std::move(cluster); +} + +void DFPClusterStore::remove(const std::string cluster_name) { + ClusterStoreType& clusterStore = getClusterStore(); + absl::WriterMutexLock lock(&clusterStore.mutex_); + clusterStore.map_.erase(cluster_name); +} + +DFPClusterStoreSharedPtr DFPClusterStoreFactory::get() { + return context_.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(dynamic_forward_proxy_cluster_store), + []() { return std::make_shared(); }); +} + +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/dynamic_forward_proxy/cluster_store.h b/source/extensions/common/dynamic_forward_proxy/cluster_store.h new file mode 100644 index 000000000000..47798d8416d0 --- /dev/null +++ b/source/extensions/common/dynamic_forward_proxy/cluster_store.h @@ -0,0 +1,79 @@ +#pragma once + +#include "envoy/server/factory_context.h" +#include "envoy/upstream/upstream.h" + +#include "source/server/factory_context_base_impl.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { + +class DfpCluster { +public: + virtual ~DfpCluster() = default; + + /** + * @return the cluster enabled subCluster configuration or not. + */ + virtual bool enableSubCluster() const PURE; + + /** + * Create a full cluster config for the cluster_name, with the specified host and port. + * @return true and nullptr when the subCluster with the specified cluster_name already + * created, or true and the created cluster config when it not exists and not + * reach the limitation of max_sub_clusters, otherwise, return false and nullptr. + */ + virtual std::pair> + createSubClusterConfig(const std::string& cluster_name, const std::string& host, + const int port) PURE; + + /** + * Update the last used time of the subCluster with the specified cluster_name. + * @return true if the subCluster is existing. + */ + virtual bool touch(const std::string& cluster_name) PURE; +}; + +using DfpClusterSharedPtr = std::shared_ptr; +using DfpClusterWeakPtr = std::weak_ptr; + +class DFPClusterStore : public Singleton::Instance { +public: + // Load the dynamic forward proxy cluster from this store. + DfpClusterSharedPtr load(std::string cluster_name); + + // Save the dynamic forward proxy cluster into this store. + void save(const std::string cluster_name, DfpClusterSharedPtr cluster); + + // Remove the dynamic forward proxy cluster from this store. + void remove(std::string cluster_name); + +private: + using ClusterMapType = absl::flat_hash_map; + struct ClusterStoreType { + ClusterMapType map_ ABSL_GUARDED_BY(mutex_); + absl::Mutex mutex_; + }; + + ClusterStoreType& getClusterStore() { MUTABLE_CONSTRUCT_ON_FIRST_USE(ClusterStoreType); } +}; + +using DFPClusterStoreSharedPtr = std::shared_ptr; + +class DFPClusterStoreFactory { +public: + DFPClusterStoreFactory(Server::Configuration::FactoryContextBase& context) : context_(context) {} + DFPClusterStoreSharedPtr get(); + +private: + Server::FactoryContextBaseImpl context_; +}; + +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/dynamic_forward_proxy/BUILD b/source/extensions/filters/http/dynamic_forward_proxy/BUILD index df611c38d9d0..f254c534d286 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/BUILD +++ b/source/extensions/filters/http/dynamic_forward_proxy/BUILD @@ -18,10 +18,13 @@ envoy_cc_library( "//source/common/http:header_utility_lib", "//source/common/network:filter_state_proxy_info_lib", "//source/common/stream_info:upstream_address_lib", + "//source/common/upstream:upstream_includes", + "//source/extensions/clusters/dynamic_forward_proxy:cluster", "//source/extensions/common/dynamic_forward_proxy:dns_cache_interface", "//source/extensions/filters/http/common:pass_through_filter_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/dynamic_forward_proxy/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/dynamic_forward_proxy/config.cc b/source/extensions/filters/http/dynamic_forward_proxy/config.cc index 75b69d19876d..3717a38fdda5 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/config.cc +++ b/source/extensions/filters/http/dynamic_forward_proxy/config.cc @@ -16,8 +16,9 @@ Http::FilterFactoryCb DynamicForwardProxyFilterFactory::createFilterFactoryFromP const std::string&, Server::Configuration::FactoryContext& context) { Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( context); + Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory(context); ProxyFilterConfigSharedPtr filter_config(std::make_shared( - proto_config, cache_manager_factory, context.clusterManager())); + proto_config, cache_manager_factory, cluster_store_factory, context)); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(filter_config)); }; diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc index cb6adc38fb95..7b851c589df2 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc @@ -2,13 +2,15 @@ #include "envoy/config/cluster/v3/cluster.pb.h" #include "envoy/config/core/v3/base.pb.h" +#include "envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.pb.h" #include "envoy/extensions/filters/http/dynamic_forward_proxy/v3/dynamic_forward_proxy.pb.h" #include "source/common/http/utility.h" #include "source/common/network/filter_state_proxy_info.h" +#include "source/common/protobuf/protobuf.h" #include "source/common/runtime/runtime_features.h" #include "source/common/stream_info/upstream_address.h" -#include "source/extensions/common/dynamic_forward_proxy/dns_cache.h" +#include "source/extensions/common/dynamic_forward_proxy/cluster_store.h" namespace Envoy { namespace Extensions { @@ -27,12 +29,18 @@ struct ResponseStringValues { const std::string DnsCacheOverflow = "DNS cache overflow"; const std::string PendingRequestOverflow = "Dynamic forward proxy pending request overflow"; const std::string DnsResolutionFailure = "DNS resolution failure"; + const std::string SubClusterOverflow = "Sub cluster overflow"; + const std::string SubClusterWarmingTimeout = "Sub cluster warming timeout"; + const std::string DFPClusterIsGone = "Dynamic forward proxy cluster is gone"; }; struct RcDetailsValues { const std::string DnsCacheOverflow = "dns_cache_overflow"; const std::string PendingRequestOverflow = "dynamic_forward_proxy_pending_request_overflow"; const std::string DnsResolutionFailure = "dns_resolution_failure"; + const std::string SubClusterOverflow = "sub_cluster_overflow"; + const std::string SubClusterWarmingTimeout = "sub_cluster_warming_timeout"; + const std::string DFPClusterIsGone = "dynamic_forward_proxy_cluster_is_gone"; }; using CustomClusterType = envoy::config::cluster::v3::Cluster::CustomClusterType; @@ -45,11 +53,84 @@ using LoadDnsCacheEntryStatus = Common::DynamicForwardProxy::DnsCache::LoadDnsCa ProxyFilterConfig::ProxyFilterConfig( const envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig& proto_config, Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, - Upstream::ClusterManager& cluster_manager) - : dns_cache_manager_(cache_manager_factory.get()), + Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory& cluster_store_factory, + Server::Configuration::FactoryContext& context) + : cluster_store_(cluster_store_factory.get()), dns_cache_manager_(cache_manager_factory.get()), dns_cache_(dns_cache_manager_->getCache(proto_config.dns_cache_config())), - cluster_manager_(cluster_manager), - save_upstream_address_(proto_config.save_upstream_address()) {} + cluster_manager_(context.clusterManager()), + main_thread_dispatcher_(context.mainThreadDispatcher()), tls_slot_(context.threadLocal()), + cluster_init_timeout_(PROTOBUF_GET_MS_OR_DEFAULT(proto_config.sub_cluster_config(), + cluster_init_timeout, 5000)), + save_upstream_address_(proto_config.save_upstream_address()) { + tls_slot_.set( + [&](Event::Dispatcher&) { return std::make_shared(*this); }); +} + +LoadClusterEntryHandlePtr ProxyFilterConfig::addDynamicCluster( + Extensions::Common::DynamicForwardProxy::DfpClusterSharedPtr cluster, + const std::string& cluster_name, const std::string& host, const int port, + LoadClusterEntryCallbacks& callbacks) { + std::pair> sub_cluster_pair = + cluster->createSubClusterConfig(cluster_name, host, port); + + if (!sub_cluster_pair.first) { + ENVOY_LOG(debug, "cluster='{}' create failed due to max sub cluster limitation", cluster_name); + return nullptr; + } + + if (sub_cluster_pair.second.has_value()) { + auto cluster = sub_cluster_pair.second.value(); + // TODO: a meaningful version_info. + std::string version_info = ""; + ENVOY_LOG(debug, "deliver dynamic cluster {} creation to main thread", cluster_name); + main_thread_dispatcher_.post([this, cluster, version_info]() { + ENVOY_LOG(debug, "initilizing dynamic cluster {} creation in main thread", cluster.name()); + cluster_manager_.addOrUpdateCluster(cluster, version_info); + }); + } else { + ENVOY_LOG(debug, "cluster='{}' already created, waiting it warming", cluster_name); + } + + // register a callback that will continue the request when the created cluster is ready. + ENVOY_LOG(debug, "adding pending cluster for: {}", cluster_name); + ThreadLocalClusterInfo& tls_cluster_info = *tls_slot_; + return std::make_unique(tls_cluster_info.pending_clusters_, + cluster_name, callbacks); +} + +Upstream::ClusterUpdateCallbacksHandlePtr +ProxyFilterConfig::addThreadLocalClusterUpdateCallbacks() { + return cluster_manager_.addThreadLocalClusterUpdateCallbacks(*this); +} + +ProxyFilterConfig::ThreadLocalClusterInfo::~ThreadLocalClusterInfo() { + for (const auto& it : pending_clusters_) { + for (auto cluster : it.second) { + cluster->cancel(); + } + } +} + +void ProxyFilterConfig::onClusterAddOrUpdate(Upstream::ThreadLocalCluster& cluster) { + const std::string& cluster_name = cluster.info()->name(); + ENVOY_LOG(debug, "thread local cluster {} added or updated", cluster_name); + ThreadLocalClusterInfo& tls_cluster_info = *tls_slot_; + auto it = tls_cluster_info.pending_clusters_.find(cluster_name); + if (it != tls_cluster_info.pending_clusters_.end()) { + for (auto* cluster : it->second) { + auto& callbacks = cluster->callbacks_; + cluster->cancel(); + callbacks.onLoadClusterComplete(); + } + tls_cluster_info.pending_clusters_.erase(it); + } else { + ENVOY_LOG(debug, "but not pending request waiting on {}", cluster_name); + } +} + +void ProxyFilterConfig::onClusterRemoval(const std::string&) { + // do nothing, should have no pending clusters. +} ProxyPerRouteConfig::ProxyPerRouteConfig( const envoy::extensions::filters::http::dynamic_forward_proxy::v3::PerRouteConfig& config) @@ -57,10 +138,15 @@ ProxyPerRouteConfig::ProxyPerRouteConfig( host_rewrite_header_(Http::LowerCaseString(config.host_rewrite_header())) {} void ProxyFilter::onDestroy() { - // Make sure we destroy any active cache load handle in case we are getting reset and deferred - // deleted. + // Make sure we destroy any active cache/cluster load handle in case we are getting reset and + // deferred deleted. cache_load_handle_.reset(); circuit_breaker_.reset(); + cluster_load_handle_.reset(); + if (cluster_init_timer_) { + cluster_init_timer_->disableTimer(); + cluster_init_timer_.reset(); + } } bool ProxyFilter::isProxying() { @@ -101,16 +187,6 @@ Http::FilterHeadersStatus ProxyFilter::decodeHeaders(Http::RequestHeaderMap& hea return Http::FilterHeadersStatus::Continue; } - circuit_breaker_ = config_->cache().canCreateDnsRequest(); - - if (circuit_breaker_ == nullptr) { - ENVOY_STREAM_LOG(debug, "pending request overflow", *this->decoder_callbacks_); - this->decoder_callbacks_->sendLocalReply( - Http::Code::ServiceUnavailable, ResponseStrings::get().PendingRequestOverflow, nullptr, - absl::nullopt, RcDetails::get().PendingRequestOverflow); - return Http::FilterHeadersStatus::StopIteration; - } - uint16_t default_port = 80; if (cluster_info_->transportSocketMatcher() .resolve(nullptr) @@ -140,8 +216,37 @@ Http::FilterHeadersStatus ProxyFilter::decodeHeaders(Http::RequestHeaderMap& hea } } + Extensions::Common::DynamicForwardProxy::DfpClusterSharedPtr dfp_cluster = + config_->clusterStore()->load(cluster_info_->name()); + if (!dfp_cluster) { + // This could happen in a very small race when users remove the DFP cluster and a route still + // using it, which is not a good usage, will end with ServiceUnavailable. + // Thread local cluster is existing due to the thread local cache, and the main thread notify + // work thread is on the way. + ENVOY_STREAM_LOG(debug, "dynamic foward cluster is gone", *this->decoder_callbacks_); + this->decoder_callbacks_->sendLocalReply(Http::Code::ServiceUnavailable, + ResponseStrings::get().DFPClusterIsGone, nullptr, + absl::nullopt, RcDetails::get().DFPClusterIsGone); + return Http::FilterHeadersStatus::StopIteration; + } + + if (dfp_cluster->enableSubCluster()) { + return loadDynamicCluster(dfp_cluster, headers, default_port); + } + + circuit_breaker_ = config_->cache().canCreateDnsRequest(); + + if (circuit_breaker_ == nullptr) { + ENVOY_STREAM_LOG(debug, "pending request overflow", *this->decoder_callbacks_); + this->decoder_callbacks_->sendLocalReply( + Http::Code::ServiceUnavailable, ResponseStrings::get().PendingRequestOverflow, nullptr, + absl::nullopt, RcDetails::get().PendingRequestOverflow); + return Http::FilterHeadersStatus::StopIteration; + } + latchTime(decoder_callbacks_, DNS_START); - // See the comments in dns_cache.h for how loadDnsCacheEntry() handles hosts with embedded ports. + // See the comments in dns_cache.h for how loadDnsCacheEntry() handles hosts with embedded + // ports. // TODO(mattklein123): Because the filter and cluster have independent configuration, it is // not obvious to the user if something is misconfigured. We should see if // we can do better here, perhaps by checking the cache to see if anything @@ -183,6 +288,48 @@ Http::FilterHeadersStatus ProxyFilter::decodeHeaders(Http::RequestHeaderMap& hea PANIC_DUE_TO_CORRUPT_ENUM; } +Http::FilterHeadersStatus ProxyFilter::loadDynamicCluster( + Extensions::Common::DynamicForwardProxy::DfpClusterSharedPtr cluster, + Http::RequestHeaderMap& headers, uint16_t default_port) { + const auto host_attributes = Http::Utility::parseAuthority(headers.getHostValue()); + auto host = std::string(host_attributes.host_); + auto port = host_attributes.port_.value_or(default_port); + + latchTime(decoder_callbacks_, DNS_START); + + // cluster name is prefix + host + port + auto cluster_name = "DFPCluster:" + host + ":" + std::to_string(port); + Upstream::ThreadLocalCluster* local_cluster = + config_->clusterManager().getThreadLocalCluster(cluster_name); + if (local_cluster && cluster->touch(cluster_name)) { + ENVOY_STREAM_LOG(debug, "using the thread local cluster after touch success", + *decoder_callbacks_); + latchTime(decoder_callbacks_, DNS_END); + return Http::FilterHeadersStatus::Continue; + } + + // Still need to add dynamic cluster again even the thread local cluster exists while touch + // failed, that means the cluster is removed in main thread due to ttl reached. + // Otherwise, we may not be able to get the thread local cluster in router. + + // Create a new cluster & register a callback to tls + cluster_load_handle_ = config_->addDynamicCluster(cluster, cluster_name, host, port, *this); + if (!cluster_load_handle_) { + ENVOY_STREAM_LOG(debug, "sub clusters overflow", *this->decoder_callbacks_); + this->decoder_callbacks_->sendLocalReply(Http::Code::ServiceUnavailable, + ResponseStrings::get().SubClusterOverflow, nullptr, + absl::nullopt, RcDetails::get().SubClusterOverflow); + return Http::FilterHeadersStatus::StopIteration; + } + + cluster_init_timer_ = + decoder_callbacks_->dispatcher().createTimer([this]() { onClusterInitTimeout(); }); + cluster_init_timer_->enableTimer(config_->clusterInitTimeout()); + + ENVOY_STREAM_LOG(debug, "waiting to load cluster entry", *decoder_callbacks_); + return Http::FilterHeadersStatus::StopAllIterationAndWatermark; +} + void ProxyFilter::addHostAddressToFilterState( const Network::Address::InstanceConstSharedPtr& address) { ASSERT(address); // null pointer checks must be done before calling this function. @@ -205,6 +352,25 @@ void ProxyFilter::addHostAddressToFilterState( StreamInfo::FilterState::LifeSpan::Request); } +void ProxyFilter::onLoadClusterComplete() { + ASSERT(cluster_init_timer_); + cluster_init_timer_->disableTimer(); + cluster_init_timer_.reset(); + + latchTime(decoder_callbacks_, DNS_END); + ENVOY_STREAM_LOG(debug, "load cluster complete, continuing", *decoder_callbacks_); + decoder_callbacks_->continueDecoding(); +} + +void ProxyFilter::onClusterInitTimeout() { + latchTime(decoder_callbacks_, DNS_END); + ENVOY_STREAM_LOG(debug, "load cluster failed, aborting", *decoder_callbacks_); + cluster_load_handle_.reset(); + decoder_callbacks_->sendLocalReply(Http::Code::ServiceUnavailable, + ResponseStrings::get().SubClusterWarmingTimeout, nullptr, + absl::nullopt, RcDetails::get().SubClusterWarmingTimeout); +} + void ProxyFilter::onDnsResolutionFail() { if (isProxying()) { decoder_callbacks_->continueDecoding(); diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h index d3149a320df8..db9a417a2ad9 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h @@ -1,8 +1,13 @@ #pragma once #include "envoy/extensions/filters/http/dynamic_forward_proxy/v3/dynamic_forward_proxy.pb.h" +#include "envoy/server/factory_context.h" +#include "envoy/thread_local/thread_local.h" #include "envoy/upstream/cluster_manager.h" +#include "source/common/upstream/upstream_impl.h" +#include "source/extensions/clusters/dynamic_forward_proxy/cluster.h" +#include "source/extensions/common/dynamic_forward_proxy/cluster_store.h" #include "source/extensions/common/dynamic_forward_proxy/dns_cache.h" #include "source/extensions/filters/http/common/pass_through_filter.h" @@ -11,21 +16,83 @@ namespace Extensions { namespace HttpFilters { namespace DynamicForwardProxy { -class ProxyFilterConfig { +/** + * Handle returned from addDynamicCluster(). Destruction of the handle will cancel any future + * callback. + */ +class LoadClusterEntryHandle { +public: + virtual ~LoadClusterEntryHandle() = default; +}; + +using LoadClusterEntryHandlePtr = std::unique_ptr; + +class LoadClusterEntryCallbacks { +public: + virtual ~LoadClusterEntryCallbacks() = default; + + virtual void onLoadClusterComplete() PURE; +}; + +class ProxyFilterConfig : public Upstream::ClusterUpdateCallbacks, + Logger::Loggable { public: ProxyFilterConfig( const envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig& proto_config, Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, - Upstream::ClusterManager& cluster_manager); + Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory& cluster_store_factory, + Server::Configuration::FactoryContext& context); + Extensions::Common::DynamicForwardProxy::DFPClusterStoreSharedPtr clusterStore() { + return cluster_store_; + } Extensions::Common::DynamicForwardProxy::DnsCache& cache() { return *dns_cache_; } Upstream::ClusterManager& clusterManager() { return cluster_manager_; } bool saveUpstreamAddress() const { return save_upstream_address_; }; + const std::chrono::milliseconds clusterInitTimeout() const { return cluster_init_timeout_; }; + + LoadClusterEntryHandlePtr + addDynamicCluster(Extensions::Common::DynamicForwardProxy::DfpClusterSharedPtr cluster, + const std::string& cluster_name, const std::string& host, const int port, + LoadClusterEntryCallbacks& callback); + // run in each worker thread. + Upstream::ClusterUpdateCallbacksHandlePtr addThreadLocalClusterUpdateCallbacks(); + + // Upstream::ClusterUpdateCallbacks + void onClusterAddOrUpdate(Upstream::ThreadLocalCluster& cluster) override; + void onClusterRemoval(const std::string&) override; private: + struct LoadClusterEntryHandleImpl + : public LoadClusterEntryHandle, + RaiiMapOfListElement { + LoadClusterEntryHandleImpl( + absl::flat_hash_map>& parent, + absl::string_view host, LoadClusterEntryCallbacks& callbacks) + : RaiiMapOfListElement(parent, host, this), + callbacks_(callbacks) {} + + LoadClusterEntryCallbacks& callbacks_; + }; + + // Per-thread cluster info including pending callbacks. + struct ThreadLocalClusterInfo : public ThreadLocal::ThreadLocalObject { + ThreadLocalClusterInfo(ProxyFilterConfig& parent) : parent_{parent} { + handle_ = parent.addThreadLocalClusterUpdateCallbacks(); + } + ~ThreadLocalClusterInfo() override; + absl::flat_hash_map> pending_clusters_; + ProxyFilterConfig& parent_; + Upstream::ClusterUpdateCallbacksHandlePtr handle_; + }; + + Extensions::Common::DynamicForwardProxy::DFPClusterStoreSharedPtr cluster_store_; const Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr dns_cache_manager_; const Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr dns_cache_; Upstream::ClusterManager& cluster_manager_; + Event::Dispatcher& main_thread_dispatcher_; + ThreadLocal::TypedSlot tls_slot_; + const std::chrono::milliseconds cluster_init_timeout_; const bool save_upstream_address_; }; @@ -47,6 +114,7 @@ class ProxyPerRouteConfig : public ::Envoy::Router::RouteSpecificFilterConfig { class ProxyFilter : public Http::PassThroughDecoderFilter, public Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryCallbacks, + public LoadClusterEntryCallbacks, Logger::Loggable { public: ProxyFilter(const ProxyFilterConfigSharedPtr& config) : config_(config) {} @@ -59,10 +127,19 @@ class ProxyFilter bool end_stream) override; void onDestroy() override; + Http::FilterHeadersStatus + loadDynamicCluster(Extensions::Common::DynamicForwardProxy::DfpClusterSharedPtr cluster, + Http::RequestHeaderMap& headers, uint16_t default_port); + // Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryCallbacks void onLoadDnsCacheComplete( const Extensions::Common::DynamicForwardProxy::DnsHostInfoSharedPtr&) override; + // LoadClusterEntryCallbacks + void onLoadClusterComplete() override; + + void onClusterInitTimeout(); + private: void addHostAddressToFilterState(const Network::Address::InstanceConstSharedPtr& address); void onDnsResolutionFail(); @@ -72,6 +149,8 @@ class ProxyFilter Upstream::ClusterInfoConstSharedPtr cluster_info_; Upstream::ResourceAutoIncDecPtr circuit_breaker_; Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryHandlePtr cache_load_handle_; + LoadClusterEntryHandlePtr cluster_load_handle_; + Event::TimerPtr cluster_init_timer_; }; } // namespace DynamicForwardProxy diff --git a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc index 29c544dc3125..55495f4175f3 100644 --- a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc +++ b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc @@ -170,6 +170,17 @@ class ClusterTest : public testing::Test, NiceMock stream_info_; NiceMock connection_; + const std::string sub_cluster_yaml_config_ = R"EOF( +name: name +connect_timeout: 0.25s +cluster_type: + name: dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + sub_clusters_config: + max_sub_clusters: 1024 +)EOF"; + const std::string default_yaml_config_ = R"EOF( name: name connect_timeout: 0.25s @@ -196,6 +207,24 @@ connect_timeout: 0.25s )EOF"; }; +// createSubClusterConfig twice. +TEST_F(ClusterTest, CreateSubClusterConfig) { + initialize(sub_cluster_yaml_config_, false); + + const std::string cluster_name = "fake_cluster_name"; + const std::string host = "localhost"; + const int port = 80; + std::pair> sub_cluster_pair = + cluster_->createSubClusterConfig(cluster_name, host, port); + EXPECT_EQ(true, sub_cluster_pair.first); + EXPECT_EQ(true, sub_cluster_pair.second.has_value()); + + // create again, already exists + sub_cluster_pair = cluster_->createSubClusterConfig(cluster_name, host, port); + EXPECT_EQ(true, sub_cluster_pair.first); + EXPECT_EQ(false, sub_cluster_pair.second.has_value()); +} + // Basic flow of the cluster including adding hosts and removing them. TEST_F(ClusterTest, BasicFlow) { initialize(default_yaml_config_, false); diff --git a/test/extensions/common/dynamic_forward_proxy/BUILD b/test/extensions/common/dynamic_forward_proxy/BUILD index 1e1f39afe27f..5bbcf74c2823 100644 --- a/test/extensions/common/dynamic_forward_proxy/BUILD +++ b/test/extensions/common/dynamic_forward_proxy/BUILD @@ -56,6 +56,7 @@ envoy_cc_mock( srcs = ["mocks.cc"], hdrs = ["mocks.h"], deps = [ + "//source/extensions/common/dynamic_forward_proxy:cluster_store", "//source/extensions/common/dynamic_forward_proxy:dns_cache_impl", "//test/mocks/upstream:basic_resource_limit_mocks", "@envoy_api//envoy/extensions/common/dynamic_forward_proxy/v3:pkg_cc_proto", diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.h b/test/extensions/common/dynamic_forward_proxy/mocks.h index ec8bd5566fbc..1a613271efdf 100644 --- a/test/extensions/common/dynamic_forward_proxy/mocks.h +++ b/test/extensions/common/dynamic_forward_proxy/mocks.h @@ -2,6 +2,7 @@ #include "envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.pb.h" +#include "source/extensions/common/dynamic_forward_proxy/cluster_store.h" #include "source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h" #include "test/mocks/upstream/basic_resource_limit.h" @@ -121,6 +122,18 @@ class MockLoadDnsCacheEntryCallbacks : public DnsCache::LoadDnsCacheEntryCallbac MOCK_METHOD(void, onLoadDnsCacheComplete, (const DnsHostInfoSharedPtr&)); }; +class MockDfpCluster : public DfpCluster { +public: + MockDfpCluster() = default; + ~MockDfpCluster() override = default; + + // Extensions::Common::DynamicForwardProxy::DfpCluster + MOCK_METHOD(bool, enableSubCluster, (), (const)); + MOCK_METHOD((std::pair>), + createSubClusterConfig, (const std::string&, const std::string&, const int)); + MOCK_METHOD(bool, touch, (const std::string&)); +}; + } // namespace DynamicForwardProxy } // namespace Common } // namespace Extensions diff --git a/test/extensions/filters/http/dynamic_forward_proxy/BUILD b/test/extensions/filters/http/dynamic_forward_proxy/BUILD index 336b812abc7f..fd7e9980745e 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/BUILD +++ b/test/extensions/filters/http/dynamic_forward_proxy/BUILD @@ -33,6 +33,7 @@ envoy_extension_cc_test( "//source/extensions/filters/http/dynamic_forward_proxy:config", "//test/extensions/common/dynamic_forward_proxy:mocks", "//test/mocks/http:http_mocks", + "//test/mocks/server:factory_context_mocks", "//test/mocks/upstream:basic_resource_limit_mocks", "//test/mocks/upstream:cluster_manager_mocks", "//test/mocks/upstream:transport_socket_match_mocks", @@ -55,6 +56,7 @@ envoy_extension_cc_test( tags = ["fails_on_clang_cl"], deps = [ "//source/extensions/clusters/dynamic_forward_proxy:cluster", + "//source/extensions/clusters/strict_dns:strict_dns_cluster_lib", "//source/extensions/filters/http/dynamic_forward_proxy:config", "//source/extensions/key_value/file_based:config_lib", "//source/extensions/network/dns_resolver/getaddrinfo:config", diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 58140bca2649..38ee775b6174 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -33,10 +33,18 @@ class ProxyFilterIntegrationTest : public testing::TestWithParamset_auto_sni(true); @@ -100,7 +112,16 @@ name: stream-info-to-headers-filter cluster_.mutable_transport_socket()->mutable_typed_config()->PackFrom(tls_context); } - const std::string cluster_type_config = fmt::format( + const std::string cluster_type_config_use_sub_cluster = fmt::format( + R"EOF( +name: envoy.clusters.dynamic_forward_proxy +typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + sub_clusters_config: + max_sub_clusters: {} +)EOF", + max_hosts); + const std::string cluster_type_config_use_dns_cache = fmt::format( R"EOF( name: envoy.clusters.dynamic_forward_proxy typed_config: @@ -115,7 +136,9 @@ name: envoy.clusters.dynamic_forward_proxy Network::Test::ipVersionToDnsFamily(GetParam()), max_hosts, max_pending_requests, key_value_config_, typed_dns_resolver_config); - TestUtility::loadFromYaml(cluster_type_config, *cluster_.mutable_cluster_type()); + TestUtility::loadFromYaml(use_sub_cluster ? cluster_type_config_use_sub_cluster + : cluster_type_config_use_dns_cache, + *cluster_.mutable_cluster_type()); cluster_.mutable_circuit_breakers() ->add_thresholds() ->mutable_max_pending_requests() @@ -748,5 +771,145 @@ TEST_P(ProxyFilterIntegrationTest, TestQueueingBasedOnCircuitBreakers) { EXPECT_EQ("200", response2->headers().getStatusValue()); } +TEST_P(ProxyFilterIntegrationTest, SubClusterWithUnknownDomain) { + key_value_config_ = ""; + + useAccessLog("%RESPONSE_CODE_DETAILS%"); + initializeWithArgs(1024, 1024, "", "", true); + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "doesnotexist.example.com"}}; + + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("503", response->headers().getStatusValue()); + EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("no_healthy_upstream")); + + response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("503", response->headers().getStatusValue()); + EXPECT_THAT(waitForAccessLog(access_log_name_, 1), HasSubstr("no_healthy_upstream")); +} + +// Verify that removed all sub cluster when dfp cluster is removed/updated. +TEST_P(ProxyFilterIntegrationTest, SubClusterReloadCluster) { + initializeWithArgs(1024, 1024, "", "", true); + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = + sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); + checkSimpleRequestSuccess(1024, 1024, response.get()); + // one more sub cluster + test_server_->waitForCounterEq("cluster_manager.cluster_added", 2); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + test_server_->waitForCounterEq("cluster_manager.cluster_modified", 0); + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 0); + + // Cause a cluster reload via CDS. + cluster_.mutable_circuit_breakers()->add_thresholds()->mutable_max_connections()->set_value(100); + cds_helper_.setCds({cluster_}); + // sub cluster is removed + test_server_->waitForCounterEq("cluster_manager.cluster_added", 2); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + test_server_->waitForCounterEq("cluster_manager.cluster_modified", 1); + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 1); + + // We need to wait until the workers have gotten the new cluster update. The only way we can + // know this currently is when the connection pools drain and terminate. + AssertionResult result = fake_upstream_connection_->waitForDisconnect(); + RELEASE_ASSERT(result, result.message()); + fake_upstream_connection_.reset(); + + // Now send another request. This should create a new sub cluster. + response = sendRequestAndWaitForResponse(request_headers, 512, default_response_headers_, 512); + checkSimpleRequestSuccess(512, 512, response.get()); + test_server_->waitForCounterEq("cluster_manager.cluster_added", 3); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); +} + +// Verify that we expire sub clusters. +TEST_P(ProxyFilterWithSimtimeIntegrationTest, RemoveSubClusterViaTTL) { + initializeWithArgs(1024, 1024, "", "", true); + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = + sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); + checkSimpleRequestSuccess(1024, 1024, response.get()); + // one more cluster + test_server_->waitForCounterEq("cluster_manager.cluster_added", 2); + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 0); + cleanupUpstreamAndDownstream(); + + // > 5m + simTime().advanceTimeWait(std::chrono::milliseconds(300001)); + test_server_->waitForCounterEq("cluster_manager.cluster_added", 2); + test_server_->waitForCounterEq("cluster_manager.cluster_removed", 1); +} + +// Test sub clusters overflow. +TEST_P(ProxyFilterIntegrationTest, SubClusterOverflow) { + initializeWithArgs(1, 1024, "", "", true); + + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = + sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); + checkSimpleRequestSuccess(1024, 1024, response.get()); + + // Send another request, this should lead to a response directly from the filter. + const Http::TestRequestHeaderMapImpl request_headers2{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", fmt::format("localhost2", fake_upstreams_[0]->localAddress()->ip()->port())}}; + response = codec_client_->makeHeaderOnlyRequest(request_headers2); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("503", response->headers().getStatusValue()); +} + +TEST_P(ProxyFilterIntegrationTest, SubClusterWithIpHost) { + upstream_tls_ = true; + initializeWithArgs(1024, 1024, "", "", true); + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", fake_upstreams_[0]->localAddress()->asString()}}; + + auto response = codec_client_->makeHeaderOnlyRequest(request_headers); + waitForNextUpstreamRequest(); + + // No SNI for IP hosts. + const Extensions::TransportSockets::Tls::SslHandshakerImpl* ssl_socket = + dynamic_cast( + fake_upstream_connection_->connection().ssl().get()); + EXPECT_STREQ(nullptr, SSL_get_servername(ssl_socket->ssl(), TLSEXT_NAMETYPE_host_name)); + + upstream_request_->encodeHeaders(default_response_headers_, true); + ASSERT_TRUE(response->waitForEndStream()); + checkSimpleRequestSuccess(0, 0, response.get()); +} + } // namespace } // namespace Envoy diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc index b6ae74d6d93a..d4db4d0a1c08 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc @@ -2,11 +2,13 @@ #include "envoy/extensions/filters/http/dynamic_forward_proxy/v3/dynamic_forward_proxy.pb.h" #include "source/common/stream_info/upstream_address.h" +#include "source/extensions/common/dynamic_forward_proxy/cluster_store.h" #include "source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h" #include "source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h" #include "test/extensions/common/dynamic_forward_proxy/mocks.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/server/factory_context.h" #include "test/mocks/upstream/basic_resource_limit.h" #include "test/mocks/upstream/cluster_manager.h" #include "test/mocks/upstream/transport_socket_match.h" @@ -39,18 +41,28 @@ class ProxyFilterTest : public testing::Test, } void setupSocketMatcher() { - cm_.initializeThreadLocalClusters({"fake_cluster"}); + factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); transport_socket_match_ = new NiceMock( Network::UpstreamTransportSocketFactoryPtr(transport_socket_factory_)); - cm_.thread_local_cluster_.cluster_.info_->transport_socket_matcher_.reset( - transport_socket_match_); + factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->transport_socket_matcher_.reset(transport_socket_match_); + dfp_cluster_ = + std::make_shared>(); + auto cluster = std::dynamic_pointer_cast( + dfp_cluster_); + Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( + factory_context_); + cluster_store_factory.get()->save("fake_cluster", cluster); } virtual void setupFilter() { EXPECT_CALL(*dns_cache_manager_, getCache(_)); + Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( + factory_context_); envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig proto_config; - filter_config_ = std::make_shared(proto_config, *this, cm_); + filter_config_ = std::make_shared(proto_config, *this, cluster_store_factory, + factory_context_); filter_ = std::make_unique(filter_config_); filter_->setDecoderFilterCallbacks(callbacks_); @@ -65,17 +77,20 @@ class ProxyFilterTest : public testing::Test, // kind we need to do DNS entries for. CustomClusterType cluster_type; cluster_type.set_name("envoy.clusters.dynamic_forward_proxy"); - cm_.thread_local_cluster_.cluster_.info_->cluster_type_ = + factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->cluster_type_ = std::make_unique( cluster_type); // Configure max pending to 1 so we can test circuit breaking. - cm_.thread_local_cluster_.cluster_.info_->resetResourceManager(0, 1, 0, 0, 0, 100); + factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager( + 0, 1, 0, 0, 0, 100); } ~ProxyFilterTest() override { EXPECT_TRUE( - cm_.thread_local_cluster_.cluster_.info_->resource_manager_->pendingRequests().canCreate()); + factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->resource_manager_ + ->pendingRequests() + .canCreate()); } Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr get() override { @@ -87,7 +102,8 @@ class ProxyFilterTest : public testing::Test, Network::MockTransportSocketFactory* transport_socket_factory_{ new Network::MockTransportSocketFactory()}; NiceMock* transport_socket_match_; - Upstream::MockClusterManager cm_; + NiceMock factory_context_; + std::shared_ptr> dfp_cluster_; ProxyFilterConfigSharedPtr filter_config_; std::unique_ptr filter_; NiceMock callbacks_; @@ -102,11 +118,11 @@ TEST_F(ProxyFilterTest, HttpDefaultPort) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(callbacks_, streamInfo()); EXPECT_CALL(callbacks_, dispatcher()); EXPECT_CALL(callbacks_, streamInfo()); @@ -130,11 +146,11 @@ TEST_F(ProxyFilterTest, HttpsDefaultPort) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(callbacks_, streamInfo()); EXPECT_CALL(callbacks_, dispatcher()); EXPECT_CALL(callbacks_, streamInfo()); @@ -158,11 +174,11 @@ TEST_F(ProxyFilterTest, CacheOverflow) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(callbacks_, streamInfo()); EXPECT_CALL(callbacks_, dispatcher()); EXPECT_CALL(callbacks_, streamInfo()); @@ -187,11 +203,11 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflow) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(callbacks_, streamInfo()); EXPECT_CALL(callbacks_, dispatcher()); EXPECT_CALL(callbacks_, streamInfo()); @@ -207,7 +223,9 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflow) { auto filter2 = std::make_unique(filter_config_); filter2->setDecoderFilterCallbacks(callbacks_); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); + EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()); EXPECT_CALL(callbacks_, sendLocalReply(Http::Code::ServiceUnavailable, Eq("Dynamic forward proxy pending request overflow"), _, _, @@ -229,13 +247,13 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflowWithDnsCacheResourceManager) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(callbacks_, streamInfo()); EXPECT_CALL(callbacks_, dispatcher()); EXPECT_CALL(callbacks_, streamInfo()); @@ -249,7 +267,9 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflowWithDnsCacheResourceManager) { auto filter2 = std::make_unique(filter_config_); filter2->setDecoderFilterCallbacks(callbacks_); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); + EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()); EXPECT_CALL(callbacks_, sendLocalReply(Http::Code::ServiceUnavailable, Eq("Dynamic forward proxy pending request overflow"), _, _, @@ -260,8 +280,8 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflowWithDnsCacheResourceManager) { filter2->decodeHeaders(request_headers_, false)); // Cluster circuit breaker overflow counter won't be incremented. - EXPECT_EQ(0, cm_.thread_local_cluster_.cluster_.info_->traffic_stats_ - ->upstream_rq_pending_overflow_.value()); + EXPECT_EQ(0, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->traffic_stats_->upstream_rq_pending_overflow_.value()); filter2->onDestroy(); EXPECT_CALL(*handle, onDestroy()); filter_->onDestroy(); @@ -280,18 +300,19 @@ TEST_F(ProxyFilterTest, NoCluster) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)) + .WillOnce(Return(nullptr)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); } // No cluster type leads to skipping DNS lookups. TEST_F(ProxyFilterTest, NoClusterType) { - cm_.thread_local_cluster_.cluster_.info_->cluster_type_ = nullptr; + factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->cluster_type_ = nullptr; InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); } @@ -299,13 +320,13 @@ TEST_F(ProxyFilterTest, NoClusterType) { TEST_F(ProxyFilterTest, NonDynamicForwardProxy) { CustomClusterType cluster_type; cluster_type.set_name("envoy.cluster.static"); - cm_.thread_local_cluster_.cluster_.info_->cluster_type_ = + factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->cluster_type_ = std::make_unique(cluster_type); InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); } @@ -319,14 +340,14 @@ TEST_F(ProxyFilterTest, HostRewrite) { ProxyPerRouteConfig config(proto_config); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*callbacks_.route_, mostSpecificPerFilterConfig(_)).WillOnce(Return(&config)); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(callbacks_, streamInfo()); EXPECT_CALL(callbacks_, dispatcher()); EXPECT_CALL(callbacks_, streamInfo()); @@ -350,14 +371,14 @@ TEST_F(ProxyFilterTest, HostRewriteViaHeader) { ProxyPerRouteConfig config(proto_config); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*callbacks_.route_, mostSpecificPerFilterConfig(_)).WillOnce(Return(&config)); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(callbacks_, streamInfo()); EXPECT_CALL(callbacks_, dispatcher()); EXPECT_CALL(callbacks_, streamInfo()); @@ -373,6 +394,123 @@ TEST_F(ProxyFilterTest, HostRewriteViaHeader) { filter_->onDestroy(); } +// Thread local cluster not exists. +TEST_F(ProxyFilterTest, SubClusterNotExists) { + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).WillOnce(Return(true)); + // get DFPCluster, not exists. + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(Eq("DFPCluster:foo:80"))); + // "true" means another thread already created it. + EXPECT_CALL(*(dfp_cluster_.get()), createSubClusterConfig(_, _, _)) + .WillOnce(Return(std::make_pair(true, absl::nullopt))); + + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + filter_->onDestroy(); +} + +// Thread local cluster exists. +TEST_F(ProxyFilterTest, SubClusterExists) { + factory_context_.cluster_manager_.initializeThreadLocalClusters({"DFPCluster:foo:80"}); + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).WillOnce(Return(true)); + // get DFPCluster, exists. + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(Eq("DFPCluster:foo:80"))); + EXPECT_CALL(*(dfp_cluster_.get()), touch(_)).WillOnce(Return(true)); + // should not create. + EXPECT_CALL(*(dfp_cluster_.get()), createSubClusterConfig(_, _, _)).Times(0); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); + + filter_->onDestroy(); +} + +// Sub cluster overflow. +TEST_F(ProxyFilterTest, SubClusterOverflow) { + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).WillOnce(Return(true)); + // get DFPCluster + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(Eq("DFPCluster:foo:80"))); + // reach the max_sub_clusters limitation. + EXPECT_CALL(*(dfp_cluster_.get()), createSubClusterConfig(_, _, _)) + .WillOnce(Return(std::make_pair(false, absl::nullopt))); + + EXPECT_CALL(callbacks_, sendLocalReply(Http::Code::ServiceUnavailable, Eq("Sub cluster overflow"), + _, _, Eq("sub_cluster_overflow"))); + EXPECT_CALL(callbacks_, encodeHeaders_(_, false)); + EXPECT_CALL(callbacks_, encodeData(_, true)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_, false)); + + filter_->onDestroy(); +} + +// DFP cluster is removed early. +TEST_F(ProxyFilterTest, DFPClusterIsGone) { + Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( + factory_context_); + cluster_store_factory.get()->remove("fake_cluster"); + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).Times(0); + EXPECT_CALL(callbacks_, sendLocalReply(Http::Code::ServiceUnavailable, + Eq("Dynamic forward proxy cluster is gone"), _, _, + Eq("dynamic_forward_proxy_cluster_is_gone"))); + EXPECT_CALL(callbacks_, encodeHeaders_(_, false)); + EXPECT_CALL(callbacks_, encodeData(_, true)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_, false)); + + filter_->onDestroy(); +} + +// Sub cluster init timeout +TEST_F(ProxyFilterTest, SubClusterInitTimeout) { + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).WillOnce(Return(true)); + // get DFPCluster, not exists. + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(Eq("DFPCluster:foo:80"))); + // "true" means another thread already created it. + EXPECT_CALL(*(dfp_cluster_.get()), createSubClusterConfig(_, _, _)) + .WillOnce(Return(std::make_pair(true, absl::nullopt))); + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + EXPECT_CALL(callbacks_, + sendLocalReply(Http::Code::ServiceUnavailable, Eq("Sub cluster warming timeout"), _, + _, Eq("sub_cluster_warming_timeout"))); + EXPECT_CALL(callbacks_, encodeHeaders_(_, false)); + EXPECT_CALL(callbacks_, encodeData(_, true)); + + filter_->onClusterInitTimeout(); + filter_->onDestroy(); +} + class UpstreamResolvedHostFilterStateHelper : public ProxyFilterTest { public: void setupFilter() override { @@ -381,7 +519,10 @@ class UpstreamResolvedHostFilterStateHelper : public ProxyFilterTest { envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig proto_config; proto_config.set_save_upstream_address(true); - filter_config_ = std::make_shared(proto_config, *this, cm_); + Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( + factory_context_); + filter_config_ = std::make_shared(proto_config, *this, cluster_store_factory, + factory_context_); filter_ = std::make_unique(filter_config_); filter_->setDecoderFilterCallbacks(callbacks_); @@ -405,11 +546,11 @@ TEST_F(UpstreamResolvedHostFilterStateHelper, AddResolvedHostFilterStateMetadata host_info->address_ = Network::Utility::parseInternetAddress("1.2.3.4", 80); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(callbacks_, streamInfo()); EXPECT_CALL(callbacks_, dispatcher()); @@ -463,11 +604,11 @@ TEST_F(UpstreamResolvedHostFilterStateHelper, UpdateResolvedHostFilterStateMetad host_info->address_ = Network::Utility::parseInternetAddress("1.2.3.4", 80); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(callbacks_, streamInfo()); EXPECT_CALL(callbacks_, dispatcher()); @@ -521,11 +662,11 @@ TEST_F(UpstreamResolvedHostFilterStateHelper, IgnoreFilterStateMetadataNullAddre host_info->address_ = nullptr; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(cm_, getThreadLocalCluster(_)); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) - .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("foo"), 80, _, _)) .WillOnce(Invoke([&](absl::string_view, uint16_t, bool, ProxyFilter::LoadDnsCacheEntryCallbacks&) { diff --git a/test/mocks/upstream/cluster_manager.h b/test/mocks/upstream/cluster_manager.h index 76c7a24d9cd7..977e11a45ff4 100644 --- a/test/mocks/upstream/cluster_manager.h +++ b/test/mocks/upstream/cluster_manager.h @@ -48,6 +48,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD(ThreadLocalCluster*, getThreadLocalCluster, (absl::string_view cluster)); MOCK_METHOD(bool, removeCluster, (const std::string& cluster)); MOCK_METHOD(void, shutdown, ()); + MOCK_METHOD(bool, isShutdown, ()); MOCK_METHOD(const absl::optional&, bindConfig, (), (const)); MOCK_METHOD(Config::GrpcMuxSharedPtr, adsMux, ()); MOCK_METHOD(Grpc::AsyncClientManager&, grpcAsyncClientManager, ()); From 700d815801306c6144282b198be2ad40a84b12c9 Mon Sep 17 00:00:00 2001 From: Christoph Pakulski Date: Tue, 16 May 2023 10:54:22 -0400 Subject: [PATCH 247/740] envoy.reloadable_features.unified_header_formatter runtime guard deprecation (#27360) * Removed envoy.reloadable_features.unified_header_formatter runtime guard. Signed-off-by: Christoph Pakulski --- changelogs/current.yaml | 3 + source/common/grpc/async_client_impl.cc | 5 +- .../common/grpc/google_async_client_impl.cc | 5 +- source/common/router/BUILD | 1 - source/common/router/header_formatter.cc | 441 ------- source/common/router/header_formatter.h | 95 -- source/common/router/header_parser.cc | 213 +--- source/common/runtime/runtime_features.cc | 1 - test/common/json/json_loader_test.cc | 12 + test/common/router/config_impl_test.cc | 13 +- test/common/router/header_formatter_test.cc | 1066 +---------------- test/per_file_coverage.sh | 2 +- 12 files changed, 48 insertions(+), 1809 deletions(-) delete mode 100644 source/common/router/header_formatter.cc diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 2083295a0972..a18a8932b525 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -112,6 +112,9 @@ removed_config_or_runtime: - area: udp change: | removed runtime key ``envoy.reloadable_features.udp_proxy_connect`` and legacy code paths. +- area: header_formatters + change: | + removed runtime key ``envoy.reloadable_features.unified_header_formatter`` and legacy code paths. new_features: - area: access_log diff --git a/source/common/grpc/async_client_impl.cc b/source/common/grpc/async_client_impl.cc index 77aaccc2b2eb..f6f087ee38c1 100644 --- a/source/common/grpc/async_client_impl.cc +++ b/source/common/grpc/async_client_impl.cc @@ -96,9 +96,8 @@ void AsyncStreamImpl::initialize(bool buffer_body_for_retry) { parent_.host_name_.empty() ? parent_.remote_cluster_name_ : parent_.host_name_, service_full_name_, method_name_, options_.timeout); // Fill service-wide initial metadata. - // TODO(cpakulski): Find a better way to access requestHeaders after runtime guard - // envoy_reloadable_features_unified_header_formatter runtime guard is deprecated and - // request headers are not stored in stream_info. + // TODO(cpakulski): Find a better way to access requestHeaders + // request headers should not be stored in stream_info. // Maybe put it to parent_context? // Since request headers may be empty, consider using Envoy::OptRef. parent_.metadata_parser_->evaluateHeaders(headers_message_->headers(), diff --git a/source/common/grpc/google_async_client_impl.cc b/source/common/grpc/google_async_client_impl.cc index 1b453f017916..18b95f22aa89 100644 --- a/source/common/grpc/google_async_client_impl.cc +++ b/source/common/grpc/google_async_client_impl.cc @@ -182,9 +182,8 @@ void GoogleAsyncStreamImpl::initialize(bool /*buffer_body_for_retry*/) { ctxt_.set_deadline(abs_deadline); // Fill service-wide initial metadata. auto initial_metadata = Http::RequestHeaderMapImpl::create(); - // TODO(cpakulski): Find a better way to access requestHeaders after runtime guard - // envoy_reloadable_features_unified_header_formatter runtime guard is deprecated - // and request headers are not stored in stream_info. + // TODO(cpakulski): Find a better way to access requestHeaders + // request headers should not be stored in stream_info. // Maybe put it to parent_context? parent_.metadata_parser_->evaluateHeaders(*initial_metadata, options_.parent_context.stream_info); callbacks_.onCreateInitialMetadata(*initial_metadata); diff --git a/source/common/router/BUILD b/source/common/router/BUILD index bbff165c3c65..d881a5d1ab00 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -431,7 +431,6 @@ envoy_cc_library( envoy_cc_library( name = "header_formatter_lib", - srcs = ["header_formatter.cc"], hdrs = ["header_formatter.h"], external_deps = ["abseil_optional"], deps = [ diff --git a/source/common/router/header_formatter.cc b/source/common/router/header_formatter.cc deleted file mode 100644 index 9a59834872e6..000000000000 --- a/source/common/router/header_formatter.cc +++ /dev/null @@ -1,441 +0,0 @@ -#include "source/common/router/header_formatter.h" - -#include - -#include "envoy/router/string_accessor.h" - -#include "source/common/common/fmt.h" -#include "source/common/common/logger.h" -#include "source/common/common/thread.h" -#include "source/common/common/utility.h" -#include "source/common/config/metadata.h" -#include "source/common/formatter/substitution_formatter.h" -#include "source/common/http/header_map_impl.h" -#include "source/common/json/json_loader.h" -#include "source/common/stream_info/utility.h" - -#include "absl/strings/match.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" -#include "absl/types/optional.h" - -namespace Envoy { -namespace Router { - -namespace { - -std::string formatUpstreamMetadataParseException(absl::string_view params, - const EnvoyException* cause = nullptr) { - std::string reason; - if (cause != nullptr) { - reason = absl::StrCat(", because ", cause->what()); - } - - return absl::StrCat("Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA", - params, reason); -} - -std::string formatPerRequestStateParseException(absl::string_view params) { - return absl::StrCat("Invalid header configuration. Expected format " - "PER_REQUEST_STATE(), actual format " - "PER_REQUEST_STATE", - params); -} - -// Parses a substitution format field and returns a function that formats it. -std::function -parseSubstitutionFormatField(absl::string_view field_name, - StreamInfoHeaderFormatter::FormatterPtrMap& formatter_map) { - const std::string pattern = fmt::format("%{}%", field_name); - if (formatter_map.find(pattern) == formatter_map.end()) { - formatter_map.emplace( - std::make_pair(pattern, Formatter::FormatterPtr(new Formatter::FormatterImpl( - pattern, /*omit_empty_values=*/true)))); - } - return [&formatter_map, pattern](const Envoy::StreamInfo::StreamInfo& stream_info) { - const auto& formatter = formatter_map.at(pattern); - return formatter->format(*Http::StaticEmptyHeaders::get().request_headers, - *Http::StaticEmptyHeaders::get().response_headers, - *Http::StaticEmptyHeaders::get().response_trailers, stream_info, - absl::string_view(), AccessLog::AccessLogType::NotSet); - }; -} - -// Parses the parameters for UPSTREAM_METADATA and returns a function suitable for accessing the -// specified metadata from an StreamInfo::StreamInfo. Expects a string formatted as: -// (["a", "b", "c"]) -// There must be at least 2 array elements (a metadata namespace and at least 1 key). -std::function -parseMetadataField(absl::string_view params_str, bool upstream = true) { - params_str = StringUtil::trim(params_str); - if (params_str.empty() || params_str.front() != '(' || params_str.back() != ')') { - throw EnvoyException(formatUpstreamMetadataParseException(params_str)); - } - - absl::string_view json = params_str.substr(1, params_str.size() - 2); // trim parens - - std::vector params; - TRY_ASSERT_MAIN_THREAD { - Json::ObjectSharedPtr parsed_params = Json::Factory::loadFromString(std::string(json)); - - // The given json string may be an invalid object. - if (!parsed_params) { - throw EnvoyException(formatUpstreamMetadataParseException(json)); - } - - for (const auto& param : parsed_params->asObjectArray()) { - params.emplace_back(param->asString()); - } - } - END_TRY - catch (Json::Exception& e) { - throw EnvoyException(formatUpstreamMetadataParseException(params_str, &e)); - } - - // Minimum parameters are a metadata namespace (e.g. "envoy.lb") and a metadata key. - if (params.size() < 2) { - throw EnvoyException(formatUpstreamMetadataParseException(params_str)); - } - - return [upstream, params](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - const envoy::config::core::v3::Metadata* metadata = nullptr; - if (upstream && stream_info.upstreamInfo()) { - Upstream::HostDescriptionConstSharedPtr host = stream_info.upstreamInfo()->upstreamHost(); - if (!host) { - return {}; - } - metadata = host->metadata().get(); - } else { - metadata = &(stream_info.dynamicMetadata()); - } - - const ProtobufWkt::Value* value = - &::Envoy::Config::Metadata::metadataValue(metadata, params[0], params[1]); - if (value->kind_case() == ProtobufWkt::Value::KIND_NOT_SET) { - // No kind indicates default ProtobufWkt::Value which means namespace or key not - // found. - return {}; - } - - size_t i = 2; - while (i < params.size()) { - if (!value->has_struct_value()) { - break; - } - - const auto field_it = value->struct_value().fields().find(params[i]); - if (field_it == value->struct_value().fields().end()) { - return {}; - } - - value = &field_it->second; - i++; - } - - if (i < params.size()) { - // Didn't find all the keys. - return {}; - } - - switch (value->kind_case()) { - case ProtobufWkt::Value::kNumberValue: - return fmt::format("{:g}", value->number_value()); - - case ProtobufWkt::Value::kStringValue: - return value->string_value(); - - case ProtobufWkt::Value::kBoolValue: - return value->bool_value() ? "true" : "false"; - - default: - // Unsupported type or null value. - ENVOY_LOG_MISC(debug, "unsupported value type for metadata [{}]", - absl::StrJoin(params, ", ")); - return {}; - } - }; -} - -// Parses the parameters for PER_REQUEST_STATE and returns a function suitable for accessing the -// specified metadata from an StreamInfo::StreamInfo. Expects a string formatted as: -// () -// The state name is expected to be in reverse DNS format, though this is not enforced by -// this function. -std::function -parsePerRequestStateField(absl::string_view param_str) { - absl::string_view modified_param_str = StringUtil::trim(param_str); - if (modified_param_str.empty() || modified_param_str.front() != '(' || - modified_param_str.back() != ')') { - throw EnvoyException(formatPerRequestStateParseException(param_str)); - } - modified_param_str = modified_param_str.substr(1, modified_param_str.size() - 2); // trim parens - if (modified_param_str.empty()) { - throw EnvoyException(formatPerRequestStateParseException(param_str)); - } - - std::string param(modified_param_str); - return [param](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - const Envoy::StreamInfo::FilterState& filter_state = stream_info.filterState(); - - auto typed_state = filter_state.getDataReadOnly(param); - - // Value exists but isn't string accessible is a contract violation; throw an error. - if (typed_state == nullptr) { - ENVOY_LOG_MISC(debug, - "Invalid header information: PER_REQUEST_STATE value \"{}\" " - "exists but is not string accessible", - param); - return {}; - } - - return std::string(typed_state->asString()); - }; -} - -// Parses the parameter for REQ and returns a function suitable for accessing the specified -// request header from an StreamInfo::StreamInfo. Expects a string formatted as: -// () -std::function -parseRequestHeader(absl::string_view param) { - param = StringUtil::trim(param); - if (param.empty() || param.front() != '(') { - throw EnvoyException(fmt::format("Invalid header configuration. Expected format " - "REQ(), actual format REQ{}", - param)); - } - ASSERT(param.back() == ')'); // Ensured by header_parser. - param = param.substr(1, param.size() - 2); // Trim parens. - Http::LowerCaseString header_name{std::string(param)}; - return [header_name](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - if (const auto* request_headers = stream_info.getRequestHeaders()) { - const auto entry = request_headers->get(header_name); - if (!entry.empty()) { - // TODO(https://github.com/envoyproxy/envoy/issues/13454): Potentially use all header - // values. - return std::string(entry[0]->value().getStringView()); - } - } - return {}; - }; -} - -// Helper that handles the case when the ConnectionInfo is missing or if the desired value is -// empty. -StreamInfoHeaderFormatter::FieldExtractor sslConnectionInfoStringHeaderExtractor( - std::function string_extractor) { - return [string_extractor](const StreamInfo::StreamInfo& stream_info) { - if (stream_info.downstreamAddressProvider().sslConnection() == nullptr) { - return std::string(); - } - - return string_extractor(*stream_info.downstreamAddressProvider().sslConnection()); - }; -} - -} // namespace - -StreamInfoHeaderFormatter::StreamInfoHeaderFormatter(absl::string_view field_name) { - if (field_name == "PROTOCOL") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) { - return Envoy::Formatter::SubstitutionFormatUtils::protocolToStringOrDefault( - stream_info.protocol()); - }; - } else if (field_name == "REQUESTED_SERVER_NAME") { - field_extractor_ = [](const StreamInfo::StreamInfo& stream_info) -> std::string { - return std::string(stream_info.downstreamAddressProvider().requestedServerName()); - }; - } else if (field_name == "VIRTUAL_CLUSTER_NAME") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - return stream_info.virtualClusterName().value_or(""); - }; - } else if (field_name == "DOWNSTREAM_REMOTE_ADDRESS") { - field_extractor_ = [](const StreamInfo::StreamInfo& stream_info) { - return stream_info.downstreamAddressProvider().remoteAddress()->asString(); - }; - } else if (field_name == "DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) { - return StreamInfo::Utility::formatDownstreamAddressNoPort( - *stream_info.downstreamAddressProvider().remoteAddress()); - }; - } else if (field_name == "DOWNSTREAM_REMOTE_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) { - return StreamInfo::Utility::formatDownstreamAddressJustPort( - *stream_info.downstreamAddressProvider().remoteAddress()); - }; - } else if (field_name == "DOWNSTREAM_DIRECT_REMOTE_ADDRESS") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) { - return stream_info.downstreamAddressProvider().directRemoteAddress()->asString(); - }; - } else if (field_name == "DOWNSTREAM_DIRECT_REMOTE_ADDRESS_WITHOUT_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) { - return StreamInfo::Utility::formatDownstreamAddressNoPort( - *stream_info.downstreamAddressProvider().directRemoteAddress()); - }; - } else if (field_name == "DOWNSTREAM_DIRECT_REMOTE_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) { - return StreamInfo::Utility::formatDownstreamAddressJustPort( - *stream_info.downstreamAddressProvider().directRemoteAddress()); - }; - } else if (field_name == "DOWNSTREAM_LOCAL_ADDRESS") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) { - return stream_info.downstreamAddressProvider().localAddress()->asString(); - }; - } else if (field_name == "DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) { - return StreamInfo::Utility::formatDownstreamAddressNoPort( - *stream_info.downstreamAddressProvider().localAddress()); - }; - } else if (field_name == "DOWNSTREAM_LOCAL_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) { - return StreamInfo::Utility::formatDownstreamAddressJustPort( - *stream_info.downstreamAddressProvider().localAddress()); - }; - } else if (field_name == "DOWNSTREAM_PEER_URI_SAN") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return absl::StrJoin(connection_info.uriSanPeerCertificate(), ","); - }); - } else if (field_name == "DOWNSTREAM_LOCAL_URI_SAN") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return absl::StrJoin(connection_info.uriSanLocalCertificate(), ","); - }); - } else if (field_name == "DOWNSTREAM_PEER_ISSUER") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return connection_info.issuerPeerCertificate(); - }); - } else if (field_name == "DOWNSTREAM_PEER_SUBJECT") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return connection_info.subjectPeerCertificate(); - }); - } else if (field_name == "DOWNSTREAM_LOCAL_SUBJECT") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return connection_info.subjectLocalCertificate(); - }); - } else if (field_name == "DOWNSTREAM_TLS_SESSION_ID") { - field_extractor_ = sslConnectionInfoStringHeaderExtractor( - [](const Ssl::ConnectionInfo& connection_info) { return connection_info.sessionId(); }); - } else if (field_name == "DOWNSTREAM_TLS_CIPHER") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return connection_info.ciphersuiteString(); - }); - } else if (field_name == "DOWNSTREAM_TLS_VERSION") { - field_extractor_ = sslConnectionInfoStringHeaderExtractor( - [](const Ssl::ConnectionInfo& connection_info) { return connection_info.tlsVersion(); }); - } else if (field_name == "DOWNSTREAM_PEER_FINGERPRINT_256") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return connection_info.sha256PeerCertificateDigest(); - }); - } else if (field_name == "DOWNSTREAM_PEER_FINGERPRINT_1") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return connection_info.sha1PeerCertificateDigest(); - }); - } else if (field_name == "DOWNSTREAM_PEER_SERIAL") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return connection_info.serialNumberPeerCertificate(); - }); - } else if (field_name == "DOWNSTREAM_PEER_CERT") { - field_extractor_ = - sslConnectionInfoStringHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { - return connection_info.urlEncodedPemEncodedPeerCertificate(); - }); - } else if (absl::StartsWith(field_name, "DOWNSTREAM_PEER_CERT_V_START")) { - field_extractor_ = parseSubstitutionFormatField(field_name, formatter_map_); - } else if (absl::StartsWith(field_name, "DOWNSTREAM_PEER_CERT_V_END")) { - field_extractor_ = parseSubstitutionFormatField(field_name, formatter_map_); - } else if (field_name == "UPSTREAM_LOCAL_ADDRESS") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - if (stream_info.upstreamInfo().has_value() && - stream_info.upstreamInfo()->upstreamLocalAddress()) { - return stream_info.upstreamInfo()->upstreamLocalAddress()->asString(); - } - return ""; - }; - } else if (field_name == "UPSTREAM_LOCAL_ADDRESS_WITHOUT_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - if (stream_info.upstreamInfo().has_value() && - stream_info.upstreamInfo()->upstreamLocalAddress()) { - return StreamInfo::Utility::formatDownstreamAddressNoPort( - *stream_info.upstreamInfo()->upstreamLocalAddress()); - } - return ""; - }; - } else if (field_name == "UPSTREAM_LOCAL_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - if (stream_info.upstreamInfo().has_value() && - stream_info.upstreamInfo()->upstreamLocalAddress()) { - return StreamInfo::Utility::formatDownstreamAddressJustPort( - *stream_info.upstreamInfo()->upstreamLocalAddress()); - } - return ""; - }; - } else if (field_name == "UPSTREAM_REMOTE_ADDRESS") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - if (stream_info.upstreamInfo() && stream_info.upstreamInfo()->upstreamRemoteAddress()) { - return stream_info.upstreamInfo()->upstreamRemoteAddress()->asString(); - } - return ""; - }; - } else if (field_name == "UPSTREAM_REMOTE_ADDRESS_WITHOUT_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - if (stream_info.upstreamInfo() && stream_info.upstreamInfo()->upstreamRemoteAddress()) { - return StreamInfo::Utility::formatDownstreamAddressNoPort( - *stream_info.upstreamInfo()->upstreamRemoteAddress()); - } - return ""; - }; - } else if (field_name == "UPSTREAM_REMOTE_PORT") { - field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - if (stream_info.upstreamInfo() && stream_info.upstreamInfo()->upstreamHost()) { - return StreamInfo::Utility::formatDownstreamAddressJustPort( - *stream_info.upstreamInfo()->upstreamHost()->address()); - } - return ""; - }; - } else if (absl::StartsWith(field_name, "START_TIME")) { - field_extractor_ = parseSubstitutionFormatField(field_name, formatter_map_); - } else if (absl::StartsWith(field_name, "UPSTREAM_METADATA")) { - field_extractor_ = parseMetadataField(field_name.substr(STATIC_STRLEN("UPSTREAM_METADATA"))); - } else if (absl::StartsWith(field_name, "DYNAMIC_METADATA")) { - field_extractor_ = - parseMetadataField(field_name.substr(STATIC_STRLEN("DYNAMIC_METADATA")), false); - } else if (absl::StartsWith(field_name, "PER_REQUEST_STATE")) { - field_extractor_ = - parsePerRequestStateField(field_name.substr(STATIC_STRLEN("PER_REQUEST_STATE"))); - } else if (absl::StartsWith(field_name, "REQ")) { - field_extractor_ = parseRequestHeader(field_name.substr(STATIC_STRLEN("REQ"))); - } else if (field_name == "HOSTNAME") { - std::string hostname = Envoy::Formatter::SubstitutionFormatUtils::getHostnameOrDefault(); - field_extractor_ = [hostname](const StreamInfo::StreamInfo&) { return hostname; }; - } else if (field_name == "RESPONSE_FLAGS") { - field_extractor_ = [](const StreamInfo::StreamInfo& stream_info) { - return StreamInfo::ResponseFlagUtils::toShortString(stream_info); - }; - } else if (field_name == "RESPONSE_CODE_DETAILS") { - field_extractor_ = [](const StreamInfo::StreamInfo& stream_info) -> std::string { - if (stream_info.responseCodeDetails().has_value()) { - return stream_info.responseCodeDetails().value(); - } - return ""; - }; - } else { - throw EnvoyException(fmt::format("field '{}' not supported as custom header", field_name)); - } -} - -const std::string -StreamInfoHeaderFormatter::format(const Envoy::StreamInfo::StreamInfo& stream_info) const { - return field_extractor_(stream_info); -} - -} // namespace Router -} // namespace Envoy diff --git a/source/common/router/header_formatter.h b/source/common/router/header_formatter.h index 1f11fecf51e7..29ce01460214 100644 --- a/source/common/router/header_formatter.h +++ b/source/common/router/header_formatter.h @@ -14,80 +14,6 @@ namespace Envoy { namespace Router { -/** - * Interface for all types of header formatters used for custom request headers. - */ -class HeaderFormatter { -public: - virtual ~HeaderFormatter() = default; - - virtual const std::string format(const Envoy::StreamInfo::StreamInfo& stream_info) const PURE; -}; - -using HeaderFormatterPtr = std::unique_ptr; - -/** - * A formatter that expands the request header variable to a value based on info in StreamInfo. - */ -class StreamInfoHeaderFormatter : public HeaderFormatter { -public: - StreamInfoHeaderFormatter(absl::string_view field_name); - - // HeaderFormatter::format - const std::string format(const Envoy::StreamInfo::StreamInfo& stream_info) const override; - - using FieldExtractor = std::function; - using FormatterPtrMap = absl::node_hash_map; - -private: - FieldExtractor field_extractor_; - - // Maps a string format pattern (including field name and any command operators between - // parenthesis) to the list of FormatterProviderPtrs that are capable of formatting that pattern. - // We use a map here to make sure that we only create a single parser for a given format pattern - // even if it appears multiple times in the larger formatting context (e.g. it shows up multiple - // times in a format string). - FormatterPtrMap formatter_map_; -}; - -/** - * A formatter that returns back the same static header value. - */ -class PlainHeaderFormatter : public HeaderFormatter { -public: - PlainHeaderFormatter(const std::string& static_header_value) - : static_value_(static_header_value) {} - - // HeaderFormatter::format - const std::string format(const Envoy::StreamInfo::StreamInfo&) const override { - return static_value_; - }; - -private: - const std::string static_value_; -}; - -/** - * A formatter that produces a value by concatenating the results of multiple HeaderFormatters. - */ -class CompoundHeaderFormatter : public HeaderFormatter { -public: - CompoundHeaderFormatter(std::vector&& formatters) - : formatters_(std::move(formatters)) {} - - // HeaderFormatter::format - const std::string format(const Envoy::StreamInfo::StreamInfo& stream_info) const override { - std::string buf; - for (const auto& formatter : formatters_) { - buf += formatter->format(stream_info); - } - return buf; - }; - -private: - const std::vector formatters_; -}; - /** * HttpHeaderFormatter is used by HTTP headers manipulators. **/ @@ -126,26 +52,5 @@ class HttpHeaderFormatterImpl : public HttpHeaderFormatter { const Formatter::FormatterPtr formatter_; }; -// TODO(cpakulski). -// This class is used only when runtime guard envoy_reloadable_features_unified_header_formatter -// is false and should be removed when the guard is removed. See issue 20389. -// This is basically a bridge between "new" header formatters which take request_headers and -// response_headers as parameters and "old" header formatters which take only stream_info as -// parameter. -class HttpHeaderFormatterBridge : public HttpHeaderFormatter { -public: - HttpHeaderFormatterBridge() = delete; - HttpHeaderFormatterBridge(HeaderFormatterPtr&& header_formatter) - : header_formatter_(std::move(header_formatter)) {} - - const std::string format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Envoy::StreamInfo::StreamInfo& stream_info) const override { - return header_formatter_->format(stream_info); - } - -private: - const HeaderFormatterPtr header_formatter_; -}; - } // namespace Router } // namespace Envoy diff --git a/source/common/router/header_parser.cc b/source/common/router/header_parser.cc index 70857a1b9b23..f753ad8ad2b0 100644 --- a/source/common/router/header_parser.cc +++ b/source/common/router/header_parser.cc @@ -22,19 +22,6 @@ namespace Router { namespace { -enum class ParserState { - Literal, // processing literal data - VariableName, // consuming a %VAR% name - ExpectArray, // expect starting [ in %VAR([...])% - ExpectString, // expect starting " in array of strings - String, // consuming an array element string - ExpectArrayDelimiterOrEnd, // expect array delimiter (,) or end of array (]) - ExpectArgsEnd, // expect closing ) in %VAR(...)% - ExpectVariableEnd // expect closing % in %VAR(...)% -}; - -std::string unescape(absl::string_view sv) { return absl::StrReplaceAll(sv, {{"%%", "%"}}); } - HttpHeaderFormatterPtr parseHttpHeaderFormatter(const envoy::config::core::v3::HeaderValue& header_value) { const std::string& key = header_value.key(); @@ -62,191 +49,6 @@ parseHttpHeaderFormatter(const envoy::config::core::v3::HeaderValue& header_valu std::make_unique(final_header_value, true)); } -// Implements a state machine to parse custom headers. Each character of the custom header format -// is either literal text (with % escaped as %%) or part of a %VAR% or %VAR(["args"])% expression. -// The statement machine does minimal validation of the arguments (if any) and does not know the -// names of valid variables. Interpretation of the variable name and arguments is delegated to -// StreamInfoHeaderFormatter. -// TODO(cpakulski): parseInternal function is executed only when -// envoy_reloadable_features_unified_header_formatter runtime guard is false. When the guard is -// deprecated, parseInternal function is not needed anymore. -HeaderFormatterPtr parseInternal(const envoy::config::core::v3::HeaderValue& header_value) { - const std::string& key = header_value.key(); - // PGV constraints provide this guarantee. - ASSERT(!key.empty()); - // We reject :path/:authority rewriting, there is already a well defined mechanism to - // perform this in the RouteAction, and doing this via request_headers_to_add - // will cause us to have to worry about interaction with other aspects of the - // RouteAction, e.g. prefix rewriting. We also reject other :-prefixed - // headers, since it seems dangerous and there doesn't appear a use case. - // Host is disallowed as it created confusing and inconsistent behaviors for - // HTTP/1 and HTTP/2. It could arguably be allowed on the response path. - if (!Http::HeaderUtility::isModifiableHeader(key)) { - throw EnvoyException(":-prefixed or host headers may not be modified"); - } - - absl::string_view format(header_value.value()); - if (format.empty()) { - return std::make_unique(""); - } - - std::vector formatters; - - size_t pos = 0, start = 0; - ParserState state = ParserState::Literal; - do { - const char ch = format[pos]; - const bool has_next_ch = (pos + 1) < format.size(); - - switch (state) { - case ParserState::Literal: - // Searching for start of %VARIABLE% expression. - if (ch != '%') { - break; - } - - if (!has_next_ch) { - throw EnvoyException( - fmt::format("Invalid header configuration. Un-escaped % at position {}", pos)); - } - - if (format[pos + 1] == '%') { - // Escaped %, skip next character. - pos++; - break; - } - - // Un-escaped %: start of variable name. Create a formatter for preceding characters, if - // any. - state = ParserState::VariableName; - if (pos > start) { - absl::string_view literal = format.substr(start, pos - start); - formatters.emplace_back(new PlainHeaderFormatter(unescape(literal))); - } - start = pos + 1; - break; - - case ParserState::VariableName: - // Consume "VAR" from "%VAR%" or "%VAR(...)%" - if (ch == '%') { - // Found complete variable name, add formatter. - formatters.emplace_back(new StreamInfoHeaderFormatter(format.substr(start, pos - start))); - start = pos + 1; - state = ParserState::Literal; - break; - } - - if (ch == '(') { - // Variable with arguments, search for start of arg array. - state = ParserState::ExpectArray; - } - break; - - case ParserState::ExpectArray: - // Skip over whitespace searching for the start of JSON array args. - if (ch == '[') { - // Search for first argument string - state = ParserState::ExpectString; - } else if (!isspace(ch)) { - // Consume it as a string argument. - state = ParserState::String; - } - break; - - case ParserState::ExpectArrayDelimiterOrEnd: - // Skip over whitespace searching for a comma or close bracket. - if (ch == ',') { - state = ParserState::ExpectString; - } else if (ch == ']') { - state = ParserState::ExpectArgsEnd; - } else if (!isspace(ch)) { - throw EnvoyException(fmt::format( - "Invalid header configuration. Expecting ',', ']', or whitespace after '{}', but " - "found '{}'", - absl::StrCat(format.substr(start, pos - start)), ch)); - } - break; - - case ParserState::ExpectString: - // Skip over whitespace looking for the starting quote of a JSON string. - if (ch == '"') { - state = ParserState::String; - } else if (!isspace(ch)) { - throw EnvoyException(fmt::format( - "Invalid header configuration. Expecting '\"' or whitespace after '{}', but found '{}'", - absl::StrCat(format.substr(start, pos - start)), ch)); - } - break; - - case ParserState::String: - // Consume a JSON string (ignoring backslash-escaped chars). - if (ch == '\\') { - if (!has_next_ch) { - throw EnvoyException(fmt::format( - "Invalid header configuration. Un-terminated backslash in JSON string after '{}'", - absl::StrCat(format.substr(start, pos - start)))); - } - - // Skip escaped char. - pos++; - } else if (ch == ')') { - state = ParserState::ExpectVariableEnd; - } else if (ch == '"') { - state = ParserState::ExpectArrayDelimiterOrEnd; - } - break; - - case ParserState::ExpectArgsEnd: - // Search for the closing paren of a %VAR(...)% expression. - if (ch == ')') { - state = ParserState::ExpectVariableEnd; - } else if (!isspace(ch)) { - throw EnvoyException(fmt::format( - "Invalid header configuration. Expecting ')' or whitespace after '{}', but found '{}'", - absl::StrCat(format.substr(start, pos - start)), ch)); - } - break; - - case ParserState::ExpectVariableEnd: - // Search for closing % of a %VAR(...)% expression - if (ch == '%') { - formatters.emplace_back(new StreamInfoHeaderFormatter(format.substr(start, pos - start))); - start = pos + 1; - state = ParserState::Literal; - break; - } - - if (!isspace(ch)) { - throw EnvoyException(fmt::format( - "Invalid header configuration. Expecting '%' or whitespace after '{}', but found '{}'", - absl::StrCat(format.substr(start, pos - start)), ch)); - } - break; - } - } while (++pos < format.size()); - - if (state != ParserState::Literal) { - // Parsing terminated mid-variable. - throw EnvoyException( - fmt::format("Invalid header configuration. Un-terminated variable expression '{}'", - absl::StrCat(format.substr(start, pos - start)))); - } - - if (pos > start) { - // Trailing constant data. - absl::string_view literal = format.substr(start, pos - start); - formatters.emplace_back(new PlainHeaderFormatter(unescape(literal))); - } - - ASSERT(!formatters.empty()); - - if (formatters.size() == 1) { - return std::move(formatters[0]); - } - - return std::make_unique(std::move(formatters)); -} - } // namespace HeadersToAddEntry::HeadersToAddEntry(const HeaderValueOption& header_value_option) @@ -266,24 +68,13 @@ HeadersToAddEntry::HeadersToAddEntry(const HeaderValueOption& header_value_optio append_action_ = header_value_option.append_action(); } - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_header_formatter")) { - formatter_ = parseHttpHeaderFormatter(header_value_option.header()); - } else { - // Use "old" implementation of header formatters. - formatter_ = - std::make_unique(parseInternal(header_value_option.header())); - } + formatter_ = parseHttpHeaderFormatter(header_value_option.header()); } HeadersToAddEntry::HeadersToAddEntry(const HeaderValue& header_value, HeaderAppendAction append_action) : original_value_(header_value.value()), append_action_(append_action) { - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_header_formatter")) { - formatter_ = parseHttpHeaderFormatter(header_value); - } else { - // Use "old" implementation of header formatters. - formatter_ = std::make_unique(parseInternal(header_value)); - } + formatter_ = parseHttpHeaderFormatter(header_value); } HeaderParserPtr diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 09b771d8baff..721b8d9b0795 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -73,7 +73,6 @@ RUNTIME_GUARD(envoy_reloadable_features_tls_async_cert_validation); RUNTIME_GUARD(envoy_reloadable_features_uhv_allow_malformed_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_uhv_preserve_url_encoded_case); RUNTIME_GUARD(envoy_reloadable_features_uhv_translate_backslash_to_slash); -RUNTIME_GUARD(envoy_reloadable_features_unified_header_formatter); RUNTIME_GUARD(envoy_reloadable_features_upstream_wait_for_response_headers_before_disabling_read); RUNTIME_GUARD(envoy_reloadable_features_use_http3_header_normalisation); RUNTIME_GUARD(envoy_reloadable_features_validate_connect); diff --git a/test/common/json/json_loader_test.cc b/test/common/json/json_loader_test.cc index 0befc9d19959..a95341847a8c 100644 --- a/test/common/json/json_loader_test.cc +++ b/test/common/json/json_loader_test.cc @@ -249,6 +249,18 @@ TEST_F(JsonLoaderTest, Integer) { ObjectSharedPtr json = Factory::loadFromString("{\"val\":-9223372036854775809}"); EXPECT_THROW(json->getInteger("val"), EnvoyException); } + // Number overflow exception. + { + EXPECT_THROW(Factory::loadFromString("-" + "521111111111111111111111111111111111111111111111111111111" + "111111111111111111111111111111111111111111111111111111111" + "111111111111111111111111111111111111111111111111111111111" + "111111111111111111111111111111111111111111111111111111111" + "111111111111111111111111111111111111111111111111111111111" + "111111111111111111111111111111111111111111111111111111111" + "1111111111111111111111111111111111111111111111111"), + EnvoyException); + } } TEST_F(JsonLoaderTest, Double) { diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 2b848d3d9319..122c00f7ccc5 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -7238,17 +7238,8 @@ TEST_F(CustomRequestHeadersTest, CustomHeaderWrongFormat) { value: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT" )EOF"; NiceMock stream_info; - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_header_formatter")) { - EXPECT_THROW_WITH_MESSAGE( - TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true), - EnvoyException, - "Invalid header configuration. Un-terminated variable expression " - "'DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT'"); - } else { - EXPECT_THROW( - TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true), - EnvoyException); - } + EXPECT_THROW(TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true), + EnvoyException); } TEST(MetadataMatchCriteriaImpl, Create) { diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 86de6e661f10..3e2998d49f3a 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -45,956 +45,7 @@ static envoy::config::route::v3::Route parseRouteFromV3Yaml(const std::string& y return route; } -class StreamInfoHeaderFormatterTest : public testing::Test { -public: - void testFormatting(const Envoy::StreamInfo::MockStreamInfo& stream_info, - const std::string& variable, const std::string& expected_output) { - { - auto f = StreamInfoHeaderFormatter(variable); - const std::string formatted_string = f.format(stream_info); - EXPECT_EQ(expected_output, formatted_string); - } - if (test_with_and_without_runtime_) { - TestScopedRuntime runtime_; - auto f = StreamInfoHeaderFormatter(variable); - const std::string formatted_string = f.format(stream_info); - EXPECT_EQ(expected_output, formatted_string); - } - } - - void testFormatting(const std::string& variable, const std::string& expected_output) { - NiceMock stream_info; - testFormatting(stream_info, variable, expected_output); - } - - void testInvalidFormat(const std::string& variable) { - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter{variable}, EnvoyException, - fmt::format("field '{}' not supported as custom header", variable)); - } - bool test_with_and_without_runtime_ = false; -}; - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamRemoteAddressVariable) { - testFormatting("DOWNSTREAM_REMOTE_ADDRESS", "127.0.0.1:0"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamRemoteAddressWithoutPortVariable) { - testFormatting("DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT", "127.0.0.1"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamRemotePortVariable) { - testFormatting("DOWNSTREAM_REMOTE_PORT", "0"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamDirectRemoteAddressVariable) { - testFormatting("DOWNSTREAM_DIRECT_REMOTE_ADDRESS", "127.0.0.3:63443"); -} - -TEST_F(StreamInfoHeaderFormatterTest, - TestFormatWithDownstreamDirectRemoteAddressWithoutPortVariable) { - testFormatting("DOWNSTREAM_DIRECT_REMOTE_ADDRESS_WITHOUT_PORT", "127.0.0.3"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamDirectRemotePortVariable) { - testFormatting("DOWNSTREAM_DIRECT_REMOTE_PORT", "63443"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalAddressVariable) { - testFormatting("DOWNSTREAM_LOCAL_ADDRESS", "127.0.0.2:0"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalAddressVariableVariants) { - NiceMock stream_info; - // Validate for IPv4 address - auto address = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv4Instance("127.1.2.3", 8443)}; - stream_info.downstream_connection_info_provider_->setLocalAddress(address); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_ADDRESS", "127.1.2.3:8443"); - - // Validate for IPv6 address - address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv6Instance("::1", 9443)}; - stream_info.downstream_connection_info_provider_->setLocalAddress(address); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_ADDRESS", "[::1]:9443"); - - // Validate for Pipe - address = Network::Address::InstanceConstSharedPtr{new Network::Address::PipeInstance("/foo")}; - stream_info.downstream_connection_info_provider_->setLocalAddress(address); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_ADDRESS", "/foo"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalPortVariable) { - testFormatting("DOWNSTREAM_LOCAL_PORT", "0"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalPortVariableVariants) { - NiceMock stream_info; - // Validate for IPv4 address - auto address = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv4Instance("127.1.2.3", 8443)}; - stream_info.downstream_connection_info_provider_->setLocalAddress(address); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_PORT", "8443"); - - // Validate for IPv6 address - address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv6Instance("::1", 9443)}; - stream_info.downstream_connection_info_provider_->setLocalAddress(address); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_PORT", "9443"); - - // Validate for Pipe - address = Network::Address::InstanceConstSharedPtr{new Network::Address::PipeInstance("/foo")}; - stream_info.downstream_connection_info_provider_->setLocalAddress(address); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_PORT", ""); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalAddressWithoutPortVariable) { - testFormatting("DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT", "127.0.0.2"); -} - -TEST_F(StreamInfoHeaderFormatterTest, - TestFormatWithDownstreamLocalAddressWithoutPortVariableVariants) { - NiceMock stream_info; - // Validate for IPv4 address - auto address = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv4Instance("127.1.2.3", 8443)}; - stream_info.downstream_connection_info_provider_->setLocalAddress(address); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT", "127.1.2.3"); - - // Validate for IPv6 address - address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv6Instance("::1", 9443)}; - stream_info.downstream_connection_info_provider_->setLocalAddress(address); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT", "::1"); - - // Validate for Pipe - address = Network::Address::InstanceConstSharedPtr{new Network::Address::PipeInstance("/foo")}; - stream_info.downstream_connection_info_provider_->setLocalAddress(address); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT", "/foo"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestformatWithUpstreamRemoteAddressVariable) { - test_with_and_without_runtime_ = true; - testFormatting("UPSTREAM_REMOTE_ADDRESS", "10.0.0.1:443"); - - NiceMock stream_info; - stream_info.upstreamInfo()->setUpstreamHost(nullptr); - Network::Address::InstanceConstSharedPtr nullptr_address; - EXPECT_CALL(*dynamic_cast(stream_info.upstream_info_.get()), - upstreamRemoteAddress()) - .WillRepeatedly(ReturnRef(nullptr_address)); - testFormatting(stream_info, "UPSTREAM_REMOTE_ADDRESS", ""); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestformatWithUpstreamRemotePortVariable) { - test_with_and_without_runtime_ = true; - testFormatting("UPSTREAM_REMOTE_PORT", "443"); - - NiceMock stream_info; - stream_info.upstreamInfo()->setUpstreamHost(nullptr); - Network::Address::InstanceConstSharedPtr nullptr_address; - EXPECT_CALL(*dynamic_cast(stream_info.upstream_info_.get()), - upstreamRemoteAddress()) - .WillRepeatedly(ReturnRef(nullptr_address)); - testFormatting(stream_info, "UPSTREAM_REMOTE_PORT", ""); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestformatWithUpstreamRemoteAddressWithoutPortVariable) { - test_with_and_without_runtime_ = true; - testFormatting("UPSTREAM_REMOTE_ADDRESS_WITHOUT_PORT", "10.0.0.1"); - - NiceMock stream_info; - stream_info.upstreamInfo()->setUpstreamHost(nullptr); - Network::Address::InstanceConstSharedPtr nullptr_address; - EXPECT_CALL(*dynamic_cast(stream_info.upstream_info_.get()), - upstreamRemoteAddress()) - .WillRepeatedly(ReturnRef(nullptr_address)); - testFormatting(stream_info, "UPSTREAM_REMOTE_ADDRESS_WITHOUT_PORT", ""); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestformatWithUpstreamLocalAddressVariable) { - testFormatting("UPSTREAM_LOCAL_ADDRESS", "127.1.2.3:58443"); - - NiceMock stream_info; - stream_info.upstreamInfo()->setUpstreamHost(nullptr); - stream_info.upstreamInfo()->setUpstreamLocalAddress(nullptr); - testFormatting(stream_info, "UPSTREAM_LOCAL_ADDRESS", ""); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestformatWithUpstreamLocalPortVariable) { - testFormatting("UPSTREAM_LOCAL_PORT", "58443"); - - NiceMock stream_info; - stream_info.upstreamInfo()->setUpstreamHost(nullptr); - stream_info.upstreamInfo()->setUpstreamLocalAddress(nullptr); - testFormatting(stream_info, "UPSTREAM_LOCAL_PORT", ""); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestformatWithUpstreamLocalAddressWithoutPortVariable) { - testFormatting("UPSTREAM_LOCAL_ADDRESS_WITHOUT_PORT", "127.1.2.3"); - - NiceMock stream_info; - stream_info.upstreamInfo()->setUpstreamHost(nullptr); - stream_info.upstreamInfo()->setUpstreamLocalAddress(nullptr); - testFormatting(stream_info, "UPSTREAM_LOCAL_ADDRESS_WITHOUT_PORT", ""); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestformatWithHostnameVariable) { - { - NiceMock os_sys_calls; - TestThreadsafeSingletonInjector os_calls(&os_sys_calls); - EXPECT_CALL(os_sys_calls, gethostname(_, _)) - .WillOnce(Invoke([](char*, size_t) -> Api::SysCallIntResult { - return {-1, ENAMETOOLONG}; - })); - testFormatting("HOSTNAME", "-"); - } - - { - NiceMock os_sys_calls; - TestThreadsafeSingletonInjector os_calls(&os_sys_calls); - EXPECT_CALL(os_sys_calls, gethostname(_, _)) - .WillOnce(Invoke([](char* name, size_t) -> Api::SysCallIntResult { - StringUtil::strlcpy(name, "myhostname", 11); - return {0, 0}; - })); - testFormatting("HOSTNAME", "myhostname"); - } -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithProtocolVariable) { - NiceMock stream_info; - absl::optional protocol = Envoy::Http::Protocol::Http11; - ON_CALL(stream_info, protocol()).WillByDefault(ReturnPointee(&protocol)); - - testFormatting(stream_info, "PROTOCOL", "HTTP/1.1"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithRequestedServerNameVariable) { - NiceMock stream_info; - // Validate for empty Request Server Name - testFormatting(stream_info, "REQUESTED_SERVER_NAME", ""); - - // Validate for a valid Request Server Name - const std::string requested_server_name = "foo.bar"; - stream_info.downstream_connection_info_provider_->setRequestedServerName(requested_server_name); - testFormatting(stream_info, "REQUESTED_SERVER_NAME", requested_server_name); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithVirtualClusterNameVariable) { - NiceMock stream_info; - // Validate for empty VC - testFormatting(stream_info, "VIRTUAL_CLUSTER_NAME", ""); - - // Validate for a valid VC - const std::string virtual_cluster_name = "authN"; - stream_info.setVirtualClusterName(virtual_cluster_name); - testFormatting(stream_info, "VIRTUAL_CLUSTER_NAME", virtual_cluster_name); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerUriSanVariableSingleSan) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::vector sans{"san"}; - ON_CALL(*connection_info, uriSanPeerCertificate()).WillByDefault(Return(sans)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_URI_SAN", "san"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerUriSanVariableMultipleSans) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::vector sans{"san1", "san2"}; - ON_CALL(*connection_info, uriSanPeerCertificate()).WillByDefault(Return(sans)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_URI_SAN", "san1,san2"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerUriSanEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, uriSanPeerCertificate()) - .WillByDefault(Return(std::vector())); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_URI_SAN", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_PEER_URI_SAN", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalUriSanVariableSingleSan) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::vector sans{"san"}; - ON_CALL(*connection_info, uriSanLocalCertificate()).WillByDefault(Return(sans)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_URI_SAN", "san"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalUriSanVariableMultipleSans) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::vector sans{"san1", "san2"}; - ON_CALL(*connection_info, uriSanLocalCertificate()).WillByDefault(Return(sans)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_URI_SAN", "san1,san2"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalUriSanVariableNoSans) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, uriSanLocalCertificate()) - .WillByDefault(Return(std::vector())); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_URI_SAN", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalUriSanNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_URI_SAN", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubject) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string subject = "subject"; - ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(ReturnRef(subject)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_SUBJECT", "subject"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubjectEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string subject; - ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(ReturnRef(subject)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_SUBJECT", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubjectNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_LOCAL_SUBJECT", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsSessionId) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string session_id = "deadbeef"; - ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(session_id)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_TLS_SESSION_ID", "deadbeef"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsSessionIdEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string session_id; - ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(session_id)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_TLS_SESSION_ID", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsSessionIdNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_TLS_SESSION_ID", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsCipher) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, ciphersuiteString()) - .WillByDefault(Return("TLS_DHE_RSA_WITH_AES_256_GCM_SHA384")); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_TLS_CIPHER", "TLS_DHE_RSA_WITH_AES_256_GCM_SHA384"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsCipherEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, ciphersuiteString()).WillByDefault(Return("")); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_TLS_CIPHER", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsCipherNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_TLS_CIPHER", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsVersion) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string tls_version = "TLSv1.2"; - ON_CALL(*connection_info, tlsVersion()).WillByDefault(ReturnRef(tls_version)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_TLS_VERSION", "TLSv1.2"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsVersionEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, tlsVersion()).WillByDefault(ReturnRef(EMPTY_STRING)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_TLS_VERSION", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsVersionNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_TLS_VERSION", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSha256Fingerprint) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string expected_sha = "685a2db593d5f86d346cb1a297009c3b467ad77f1944aa799039a2fb3d531f3f"; - ON_CALL(*connection_info, sha256PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_FINGERPRINT_256", - "685a2db593d5f86d346cb1a297009c3b467ad77f1944aa799039a2fb3d531f3f"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSha256FingerprintEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string expected_sha; - ON_CALL(*connection_info, sha256PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_FINGERPRINT_256", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSha256FingerprintNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_PEER_FINGERPRINT_256", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSha1Fingerprint) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string expected_sha = "685a2db593d5f86d346cb1a297009c3b467ad77f1944aa799039a2fb3d531f3f"; - ON_CALL(*connection_info, sha1PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_FINGERPRINT_1", - "685a2db593d5f86d346cb1a297009c3b467ad77f1944aa799039a2fb3d531f3f"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSha1FingerprintEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string expected_sha; - ON_CALL(*connection_info, sha1PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_FINGERPRINT_1", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSha1FingerprintNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_PEER_FINGERPRINT_1", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerial) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::string serial_number = "b8b5ecc898f2124a"; - ON_CALL(*connection_info, serialNumberPeerCertificate()).WillByDefault(ReturnRef(serial_number)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_SERIAL", "b8b5ecc898f2124a"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerialEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::string serial_number; - ON_CALL(*connection_info, serialNumberPeerCertificate()).WillByDefault(ReturnRef(serial_number)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_SERIAL", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerialNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_PEER_SERIAL", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuer) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::string issuer_peer = - "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"; - ON_CALL(*connection_info, issuerPeerCertificate()).WillByDefault(ReturnRef(issuer_peer)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_ISSUER", - "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuerEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::string issuer_peer; - ON_CALL(*connection_info, issuerPeerCertificate()).WillByDefault(ReturnRef(issuer_peer)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_ISSUER", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuerNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_PEER_ISSUER", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSubject) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::string subject_peer = - "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"; - ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(ReturnRef(subject_peer)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_SUBJECT", - "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSubjectEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - const std::string subject_peer; - ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(ReturnRef(subject_peer)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_SUBJECT", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSubjectNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_PEER_SUBJECT", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCert) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string expected_cert = ""; - ON_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) - .WillByDefault(ReturnRef(expected_cert)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT", expected_cert); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - std::string expected_cert; - ON_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) - .WillByDefault(ReturnRef(expected_cert)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVStart) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - absl::Time abslStartTime = - TestUtility::parseTime("Dec 18 01:50:34 2018 GMT", "%b %e %H:%M:%S %Y GMT"); - SystemTime startTime = absl::ToChronoTime(abslStartTime); - ON_CALL(*connection_info, validFromPeerCertificate()).WillByDefault(Return(startTime)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_START", "2018-12-18T01:50:34.000Z"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVStartCustom) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - absl::Time abslStartTime = - TestUtility::parseTime("Dec 18 01:50:34 2018 GMT", "%b %e %H:%M:%S %Y GMT"); - SystemTime startTime = absl::ToChronoTime(abslStartTime); - ON_CALL(*connection_info, validFromPeerCertificate()).WillByDefault(Return(startTime)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_START(%b %e %H:%M:%S %Y %Z)", - "Dec 18 01:50:34 2018 UTC"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVStartEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, validFromPeerCertificate()).WillByDefault(Return(absl::nullopt)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_START", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVStartNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_START", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVEnd) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - absl::Time abslStartTime = - TestUtility::parseTime("Dec 17 01:50:34 2020 GMT", "%b %e %H:%M:%S %Y GMT"); - SystemTime startTime = absl::ToChronoTime(abslStartTime); - ON_CALL(*connection_info, expirationPeerCertificate()).WillByDefault(Return(startTime)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_END", "2020-12-17T01:50:34.000Z"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVEndCustom) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - absl::Time abslStartTime = - TestUtility::parseTime("Dec 17 01:50:34 2020 GMT", "%b %e %H:%M:%S %Y GMT"); - SystemTime startTime = absl::ToChronoTime(abslStartTime); - ON_CALL(*connection_info, expirationPeerCertificate()).WillByDefault(Return(startTime)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_END(%b %e %H:%M:%S %Y %Z)", - "Dec 17 01:50:34 2020 UTC"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVEndEmpty) { - NiceMock stream_info; - auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, expirationPeerCertificate()).WillByDefault(Return(absl::nullopt)); - stream_info.downstream_connection_info_provider_->setSslConnection(connection_info); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_END", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVEndNoTls) { - NiceMock stream_info; - stream_info.downstream_connection_info_provider_->setSslConnection(nullptr); - testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_END", EMPTY_STRING); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithStartTime) { - NiceMock stream_info; - absl::Time abslStartTime = - TestUtility::parseTime("Dec 17 01:50:34 2020 GMT", "%b %e %H:%M:%S %Y GMT"); - SystemTime startTime = absl::ToChronoTime(abslStartTime); - EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(startTime)); - testFormatting(stream_info, "START_TIME", "2020-12-17T01:50:34.000Z"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithStartTimeCustom) { - NiceMock stream_info; - absl::Time abslStartTime = - TestUtility::parseTime("Dec 17 01:50:34 2020 GMT", "%b %e %H:%M:%S %Y GMT"); - SystemTime startTime = absl::ToChronoTime(abslStartTime); - EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(startTime)); - testFormatting(stream_info, "START_TIME(%b %e %H:%M:%S %Y %Z)", "Dec 17 01:50:34 2020 UTC"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithUpstreamMetadataVariable) { - NiceMock stream_info; - std::shared_ptr> host( - new NiceMock()); - - auto metadata = std::make_shared( - TestUtility::parseYaml( - R"EOF( - filter_metadata: - namespace: - key: value - nested: - str_key: str_value - "escaped,key": escaped_key_value - bool_key1: true - bool_key2: false - num_key1: 1 - num_key2: 3.14 - null_key: null - list_key: [ list_element ] - struct_key: - deep_key: deep_value - )EOF")); - - // Prove we're testing the expected types. - const auto& nested_struct = - Envoy::Config::Metadata::metadataValue(metadata.get(), "namespace", "nested").struct_value(); - EXPECT_EQ(nested_struct.fields().at("str_key").kind_case(), ProtobufWkt::Value::kStringValue); - EXPECT_EQ(nested_struct.fields().at("bool_key1").kind_case(), ProtobufWkt::Value::kBoolValue); - EXPECT_EQ(nested_struct.fields().at("bool_key2").kind_case(), ProtobufWkt::Value::kBoolValue); - EXPECT_EQ(nested_struct.fields().at("num_key1").kind_case(), ProtobufWkt::Value::kNumberValue); - EXPECT_EQ(nested_struct.fields().at("num_key1").kind_case(), ProtobufWkt::Value::kNumberValue); - EXPECT_EQ(nested_struct.fields().at("null_key").kind_case(), ProtobufWkt::Value::kNullValue); - EXPECT_EQ(nested_struct.fields().at("list_key").kind_case(), ProtobufWkt::Value::kListValue); - EXPECT_EQ(nested_struct.fields().at("struct_key").kind_case(), ProtobufWkt::Value::kStructValue); - - stream_info.upstreamInfo()->setUpstreamHost(host); - ON_CALL(*host, metadata()).WillByDefault(Return(metadata)); - - // Top-level value. - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"key\"])", "value"); - - // Nested string value. - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"str_key\"])", - "str_value"); - - // Boolean values. - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"bool_key1\"])", - "true"); - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"bool_key2\"])", - "false"); - - // Number values. - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"num_key1\"])", "1"); - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"num_key2\"])", - "3.14"); - - // Deeply nested value. - testFormatting(stream_info, - "UPSTREAM_METADATA([\"namespace\", \"nested\", \"struct_key\", \"deep_key\"])", - "deep_value"); - - // Initial metadata lookup fails. - testFormatting(stream_info, "UPSTREAM_METADATA([\"wrong_namespace\", \"key\"])", ""); - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"not_found\"])", ""); - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"not_found\", \"key\"])", ""); - - // Nested metadata lookup fails. - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"not_found\"])", ""); - - // Nested metadata lookup returns non-struct intermediate value. - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"key\", \"invalid\"])", ""); - - // Struct values are not rendered. - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"struct_key\"])", ""); - - // List values are not rendered. - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"list_key\"])", ""); -} - -// Replaces the test of user-defined-headers acting as a Query of Death with -// size checks on user defined headers. -TEST_F(StreamInfoHeaderFormatterTest, ValidateLimitsOnUserDefinedHeaders) { - { - envoy::config::route::v3::RouteConfiguration route; - envoy::config::core::v3::HeaderValueOption* header = - route.mutable_request_headers_to_add()->Add(); - std::string long_string(16385, 'a'); - header->mutable_header()->set_key("header_name"); - header->mutable_header()->set_value(long_string); - header->set_append_action(envoy::config::core::v3::HeaderValueOption::APPEND_IF_EXISTS_OR_ADD); - EXPECT_THROW_WITH_REGEX(TestUtility::validate(route), ProtoValidationException, - "Proto constraint validation failed.*"); - } - { - envoy::config::route::v3::RouteConfiguration route; - for (int i = 0; i < 1001; ++i) { - envoy::config::core::v3::HeaderValueOption* header = - route.mutable_request_headers_to_add()->Add(); - header->mutable_header()->set_key("header_name"); - header->mutable_header()->set_value("value"); - } - EXPECT_THROW_WITH_REGEX(TestUtility::validate(route), ProtoValidationException, - "Proto constraint validation failed.*"); - } -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithUpstreamMetadataVariableMissingHost) { - NiceMock stream_info; - std::shared_ptr> host; - stream_info.upstreamInfo()->setUpstreamHost(host); - - testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"key\"])", ""); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithInvalidUpstreamMetadata) { - NiceMock stream_info; - std::shared_ptr> host; - stream_info.upstreamInfo()->setUpstreamHost(host); - - EXPECT_THROW_WITH_MESSAGE( - testFormatting(stream_info, "UPSTREAM_METADATA(1)", ""), EnvoyException, - "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " - "...]), actual format UPSTREAM_METADATA1"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithRequestMetadata) { - NiceMock stream_info; - envoy::config::core::v3::Metadata metadata; - ProtobufWkt::Struct struct_obj; - - auto& fields_map = *struct_obj.mutable_fields(); - fields_map["foo"] = ValueUtil::stringValue("bar"); - (*metadata.mutable_filter_metadata())["envoy.lb"] = struct_obj; - - EXPECT_CALL(stream_info, dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); - EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); - - testFormatting(stream_info, "DYNAMIC_METADATA([\"envoy.lb\", \"foo\"])", "bar"); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithPerRequestStateVariable) { - Envoy::StreamInfo::FilterStateSharedPtr filter_state( - std::make_shared( - Envoy::StreamInfo::FilterState::LifeSpan::FilterChain)); - filter_state->setData("testing", std::make_unique("test_value"), - StreamInfo::FilterState::StateType::ReadOnly, - StreamInfo::FilterState::LifeSpan::FilterChain); - EXPECT_EQ("test_value", filter_state->getDataReadOnly("testing")->asString()); - - NiceMock stream_info; - ON_CALL(stream_info, filterState()).WillByDefault(ReturnRef(filter_state)); - ON_CALL(Const(stream_info), filterState()).WillByDefault(ReturnRef(*filter_state)); - - testFormatting(stream_info, "PER_REQUEST_STATE(testing)", "test_value"); - testFormatting(stream_info, "PER_REQUEST_STATE(testing2)", ""); - EXPECT_EQ("test_value", filter_state->getDataReadOnly("testing")->asString()); -} - -TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithNonStringPerRequestStateVariable) { - Envoy::StreamInfo::FilterStateSharedPtr filter_state( - std::make_shared( - Envoy::StreamInfo::FilterState::LifeSpan::FilterChain)); - filter_state->setData("testing", std::make_unique(1), - StreamInfo::FilterState::StateType::ReadOnly, - StreamInfo::FilterState::LifeSpan::FilterChain); - EXPECT_EQ(1, filter_state->getDataReadOnly("testing")->access()); - - NiceMock stream_info; - ON_CALL(stream_info, filterState()).WillByDefault(ReturnRef(filter_state)); - ON_CALL(Const(stream_info), filterState()).WillByDefault(ReturnRef(*filter_state)); - - testFormatting(stream_info, "PER_REQUEST_STATE(testing)", ""); -} - -TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnPerRequestStateVariable) { - // No parameters - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("PER_REQUEST_STATE()"), EnvoyException, - "Invalid header configuration. Expected format " - "PER_REQUEST_STATE(), actual format " - "PER_REQUEST_STATE()"); - - // Missing single parens - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("PER_REQUEST_STATE(testing"), EnvoyException, - "Invalid header configuration. Expected format " - "PER_REQUEST_STATE(), actual format " - "PER_REQUEST_STATE(testing"); - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("PER_REQUEST_STATE testing)"), EnvoyException, - "Invalid header configuration. Expected format " - "PER_REQUEST_STATE(), actual format " - "PER_REQUEST_STATE testing)"); -} - -TEST_F(StreamInfoHeaderFormatterTest, UnknownVariable) { testInvalidFormat("INVALID_VARIABLE"); } - -TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { - // Invalid JSON. - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA(abcd)"), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA(abcd), because JSON supplied is not valid. " - "Error(line 1, column 1, token a): syntax error while parsing value - " - "invalid literal; last read: 'a'\n"); - - // No parameters. - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA"), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA"); - - EXPECT_THROW_WITH_MESSAGE( - StreamInfoHeaderFormatter("UPSTREAM_METADATA()"), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format UPSTREAM_METADATA(), " - "because JSON supplied is not valid. Error(line 1, column 1, token ): syntax error while " - "parsing value - unexpected end of input; expected '[', '{', or a literal\n"); - - // One parameter. - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"ns\"])"), - EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA([\"ns\"])"); - - // Missing close paren. - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA("), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA("); - - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA([a,b,c,d]"), - EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA([a,b,c,d]"); - - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"a\",\"b\"]"), - EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA([\"a\",\"b\"]"); - - // Non-string elements. - EXPECT_THROW_WITH_MESSAGE( - StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"a\", 1])"), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA([\"a\", 1]), because JSON field from line 1 accessed with type 'String' " - "does not match actual type 'Integer'."); - - // Invalid string elements. - EXPECT_THROW_WITH_MESSAGE( - StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"a\", \"\\unothex\"])"), EnvoyException, - "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", " - "\"k\", ...]), actual format UPSTREAM_METADATA([\"a\", \"\\unothex\"]), because JSON " - "supplied is not valid. Error(line 1, column 10, token \"\\un): syntax error while parsing " - "value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\un'\n"); - - // Non-array parameters. - EXPECT_THROW_WITH_MESSAGE( - StreamInfoHeaderFormatter("UPSTREAM_METADATA({\"a\":1})"), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA({\"a\":1}), because JSON field from line 1 accessed with type 'Array' " - "does not match actual type 'Object'."); - - // Number overflow. - EXPECT_THROW_WITH_MESSAGE( - StreamInfoHeaderFormatter( - "UPSTREAM_METADATA(-" - "5211111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - "1111111111111111111111111111111111111111111111111)"), - EnvoyException, - "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " - "...]), actual format " - "UPSTREAM_METADATA(-" - "52111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - "1111111111111111111111111111111111111), because JSON supplied is not valid. Error(position: " - "314): number overflow parsing '-" - "52111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - "1111111111111111111111111111111111111'\n"); -} - -TEST(PlainFormatterTest, BasicTest) { PlainHeaderFormatter formatter("test"); } - -TEST(CompoundFormatterTest, BasicTest) { - std::vector formatters; - formatters.push_back(std::make_unique("test1")); - formatters.push_back(std::make_unique("test2")); - - CompoundHeaderFormatter formatter(std::move(formatters)); -} - -TEST(HeaderParserTest, TestParseInternal) { +TEST(HeaderParserTest, TestParse) { struct TestCase { std::string input_; absl::optional expected_output_; @@ -1110,49 +161,6 @@ TEST(HeaderParserTest, TestParseInternal) { "...]), actual format UPSTREAM_METADATA"}}, }; - /* - The following test cases do not make sense after using unified header formatters. - See issue 20389. The test cases are executed only when runtime guard - envoy_reloadable_features_unified_header_formatter is false. - Comments below explain why unified header formatter parser will not fail. - TODO(cpakulski): the following test cases should be removed when - envoy_reloadable_features_unified_header_formatter is deprecated. - */ - static const TestCase obsolete_test_cases[] = { - // Single key is allowed in UPSTREAM_METADATA - {"%UPSTREAM_METADATA(no array)%", - {}, - {"Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA(no array), because JSON supplied is not valid. Error(line 1, " - "column 2, token no): syntax error while parsing value - invalid literal; last read: " - "'no'\n"}}, - // Single key is allowed in UPSTREAM_METADATA - {"%UPSTREAM_METADATA( no array)%", - {}, - {"Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format UPSTREAM_METADATA( " - "no array), because JSON supplied is not valid. Error(line 1, column 3, token no): " - "syntax error while parsing value - invalid literal; last read: ' no'\n"}}, - // [\"unterminated array\" will treated as key. - {"%UPSTREAM_METADATA([\"unterminated array\")%", - {}, - {"Invalid header configuration. Expecting ',', ']', or whitespace after " - "'UPSTREAM_METADATA([\"unterminated array\"', but found ')'"}}, - // [not-a-string] will be treated as key. - {"%UPSTREAM_METADATA([not-a-string])%", - {}, - {"Invalid header configuration. Expecting '\"' or whitespace after 'UPSTREAM_METADATA([', " - "but found 'n'"}}, - // [\"ns\"] will be treated as string. - {"%UPSTREAM_METADATA([\"ns\"])%", - {}, - {"Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " - "...]), actual format UPSTREAM_METADATA([\"ns\"])"}}, - {"%START_TIME(%85n)%", {}, {"Invalid header configuration. Format string contains newline."}}, - - }; - NiceMock stream_info; const std::string requested_server_name = "foo.bar"; stream_info.downstream_connection_info_provider_->setRequestedServerName(requested_server_name); @@ -1203,58 +211,32 @@ TEST(HeaderParserTest, TestParseInternal) { absl::optional rc_details{"via_upstream"}; ON_CALL(stream_info, responseCodeDetails()).WillByDefault(ReturnRef(rc_details)); - // Run all tests twice: once using old parser and again using access log's parser. - for (const bool use_unified_parser : std::vector{false, true}) { - Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.unified_header_formatter", - use_unified_parser); - for (const auto& test_case : test_cases) { - Protobuf::RepeatedPtrField to_add; - envoy::config::core::v3::HeaderValueOption* header = to_add.Add(); - header->mutable_header()->set_key("x-header"); - header->mutable_header()->set_value(test_case.input_); - - if (test_case.expected_exception_) { - EXPECT_FALSE(test_case.expected_output_); - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_header_formatter")) { - EXPECT_THROW_WITH_MESSAGE(HeaderParser::configure(to_add), EnvoyException, - test_case.expected_exception_.value()); - } else { - EXPECT_THROW(HeaderParser::configure(to_add), EnvoyException); - } - continue; - } - - HeaderParserPtr req_header_parser = HeaderParser::configure(to_add); - - Http::TestRequestHeaderMapImpl header_map{{":method", "POST"}}; - req_header_parser->evaluateHeaders(header_map, stream_info); - - std::string descriptor = fmt::format("for test case input: {}", test_case.input_); - - if (!test_case.expected_output_) { - EXPECT_FALSE(header_map.has("x-header")) << descriptor; - continue; - } - - EXPECT_TRUE(header_map.has("x-header")) << descriptor; - EXPECT_EQ(test_case.expected_output_.value(), header_map.get_("x-header")) << descriptor; + for (const auto& test_case : test_cases) { + Protobuf::RepeatedPtrField to_add; + envoy::config::core::v3::HeaderValueOption* header = to_add.Add(); + header->mutable_header()->set_key("x-header"); + header->mutable_header()->set_value(test_case.input_); + + if (test_case.expected_exception_) { + EXPECT_FALSE(test_case.expected_output_); + EXPECT_THROW(HeaderParser::configure(to_add), EnvoyException); + continue; } - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_header_formatter")) { - for (const auto& test_case : obsolete_test_cases) { - Protobuf::RepeatedPtrField to_add; - envoy::config::core::v3::HeaderValueOption* header = to_add.Add(); - header->mutable_header()->set_key("x-header"); - header->mutable_header()->set_value(test_case.input_); - - if (test_case.expected_exception_) { - EXPECT_FALSE(test_case.expected_output_); - EXPECT_THROW_WITH_MESSAGE(HeaderParser::configure(to_add), EnvoyException, - test_case.expected_exception_.value()); - continue; - } - } + HeaderParserPtr req_header_parser = HeaderParser::configure(to_add); + + Http::TestRequestHeaderMapImpl header_map{{":method", "POST"}}; + req_header_parser->evaluateHeaders(header_map, stream_info); + + std::string descriptor = fmt::format("for test case input: {}", test_case.input_); + + if (!test_case.expected_output_) { + EXPECT_FALSE(header_map.has("x-header")) << descriptor; + continue; } + + EXPECT_TRUE(header_map.has("x-header")) << descriptor; + EXPECT_EQ(test_case.expected_output_.value(), header_map.get_("x-header")) << descriptor; } } diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 4742e08e893f..5d5014cbb75c 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -18,7 +18,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/network/dns_resolver:91.6" # A few lines of MacOS code not tested in linux scripts. Tested in MacOS scripts "source/common/protobuf:96.3" "source/common/quic:93.5" -"source/common/router:96.4" +"source/common/router:96.3" "source/common/secret:95.0" "source/common/signal:87.2" # Death tests don't report LCOV "source/common/singleton:95.7" From 3ca5e49ec48d6f4693f1ff0c77051d532f339fdb Mon Sep 17 00:00:00 2001 From: danzh Date: Tue, 16 May 2023 10:54:55 -0400 Subject: [PATCH 248/740] quic: adjust client timeouts for TSAN (#27327) adjust timeouts for tsan Signed-off-by: Dan Zhang --- test/integration/http_integration.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 8436ab90f0ab..38a915872be3 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -378,8 +378,15 @@ void HttpIntegrationTest::initialize() { // Config Google QUIC flow control window. quic_connection_persistent_info->quic_config_.SetInitialStreamFlowControlWindowToSend( Http3::Utility::OptionsLimits::DEFAULT_INITIAL_STREAM_WINDOW_SIZE); - quic_connection_persistent_info_ = std::move(quic_connection_persistent_info); + // Adjust timeouts. + quic::QuicTime::Delta connect_timeout = + quic::QuicTime::Delta::FromSeconds(5 * TSAN_TIMEOUT_FACTOR); + quic_connection_persistent_info->quic_config_.set_max_time_before_crypto_handshake( + connect_timeout); + quic_connection_persistent_info->quic_config_.set_max_idle_time_before_crypto_handshake( + connect_timeout); + quic_connection_persistent_info_ = std::move(quic_connection_persistent_info); #else ASSERT(false, "running a QUIC integration test without compiling QUIC"); #endif From b8d771b8b88a28c99f8fdee98796ecefc8aa5627 Mon Sep 17 00:00:00 2001 From: John Zheng Date: Tue, 16 May 2023 23:06:14 +0800 Subject: [PATCH 249/740] only show grpc_status for grpc requests (#27387) * Fix issue #27201 - only show grpc_status for grpc requests. - add test for changes - Add envoy.reloadable_features.validate_grpc_header_before_log_grpc_status Signed-off-by: John Zheng --- changelogs/current.yaml | 5 + .../formatter/substitution_formatter.cc | 16 +- source/common/runtime/runtime_features.cc | 1 + .../common/access_log/access_log_impl_test.cc | 1 + .../formatter/substitution_formatter_test.cc | 291 ++++++++++++++++++ 5 files changed, 312 insertions(+), 2 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index a18a8932b525..36c196d1ff5e 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -88,6 +88,11 @@ bug_fixes: Fixed the bug that updating :ref:`scope_key_builder ` of SRDS config doesn't work and multiple HCM share the same ``scope_key_builder``. +- area: logging + change: | + Do not display GRPC_STATUS_NUMBER for non gRPC requests. + This behavioral change can be temporarily reverted by setting runtime guard + ``envoy.reloadable_features.validate_grpc_header_before_log_grpc_status`` to false. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index 087d3a9d2f06..731812171306 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -1848,9 +1848,15 @@ GrpcStatusFormatter::GrpcStatusFormatter(const std::string& main_header, : HeaderFormatter(main_header, alternative_header, max_length), format_(format) {} absl::optional GrpcStatusFormatter::format( - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, + const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& info, absl::string_view, AccessLog::AccessLogType) const { + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.validate_grpc_header_before_log_grpc_status")) { + if (!Grpc::Common::isGrpcRequestHeaders(request_headers)) { + return absl::nullopt; + } + } const auto grpc_status = Grpc::Common::getGrpcStatus(response_trailers, response_headers, info, true); if (!grpc_status.has_value()) { @@ -1880,9 +1886,15 @@ absl::optional GrpcStatusFormatter::format( } ProtobufWkt::Value GrpcStatusFormatter::formatValue( - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap& response_headers, + const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& info, absl::string_view, AccessLog::AccessLogType) const { + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.validate_grpc_header_before_log_grpc_status")) { + if (!Grpc::Common::isGrpcRequestHeaders(request_headers)) { + return unspecifiedValue(); + } + } const auto grpc_status = Grpc::Common::getGrpcStatus(response_trailers, response_headers, info, true); if (!grpc_status.has_value()) { diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 721b8d9b0795..03042d34360f 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -77,6 +77,7 @@ RUNTIME_GUARD(envoy_reloadable_features_upstream_wait_for_response_headers_befor RUNTIME_GUARD(envoy_reloadable_features_use_http3_header_normalisation); RUNTIME_GUARD(envoy_reloadable_features_validate_connect); RUNTIME_GUARD(envoy_reloadable_features_validate_detailed_override_host_statuses); +RUNTIME_GUARD(envoy_reloadable_features_validate_grpc_header_before_log_grpc_status); RUNTIME_GUARD(envoy_reloadable_features_validate_upstream_headers); RUNTIME_GUARD(envoy_restart_features_explicit_wildcard_resource); RUNTIME_GUARD(envoy_restart_features_remove_runtime_singleton); diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index dc672a741819..3dcb34ab07c4 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -1216,6 +1216,7 @@ name: accesslog inline_string: "%GRPC_STATUS% %GRPC_STATUS_NUMBER% %GRPC_STATUS()% %GRPC_STATUS(CAMEL_STRING)% %GRPC_STATUS(SNAKE_STRING)% %GRPC_STATUS(NUMBER)%\n" )EOF"; + request_headers_ = {{":method", "GET"}, {":path", "/"}, {"content-type", "application/grpc"}}; InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); { EXPECT_CALL(*file_, write(_)); diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 05053d7b49d3..897a35879de5 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -2677,6 +2677,100 @@ TEST(SubstitutionFormatterTest, StartTimeFormatter) { } TEST(SubstitutionFormatterTest, GrpcStatusFormatterCamelStringTest) { + GrpcStatusFormatter formatter("grpc-status", "", absl::optional(), + GrpcStatusFormatter::Format::CamelString); + NiceMock stream_info; + Http::TestRequestHeaderMapImpl request_header{{"content-type", "application/grpc+proto"}, + {":path", "/"}}; + Http::TestResponseHeaderMapImpl response_header; + Http::TestResponseTrailerMapImpl response_trailer; + std::string body; + + std::vector grpc_statuses{ + "OK", "Canceled", "Unknown", "InvalidArgument", "DeadlineExceeded", + "NotFound", "AlreadyExists", "PermissionDenied", "ResourceExhausted", "FailedPrecondition", + "Aborted", "OutOfRange", "Unimplemented", "Internal", "Unavailable", + "DataLoss", "Unauthenticated"}; + for (size_t i = 0; i < grpc_statuses.size(); ++i) { + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", std::to_string(i)}}; + EXPECT_EQ(grpc_statuses[i], + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue(grpc_statuses[i]))); + } + { + response_trailer = Http::TestResponseTrailerMapImpl{{"not-a-grpc-status", "13"}}; + EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); + } + { + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "-1"}}; + EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("-1"))); + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "42738"}}; + EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("42738"))); + response_trailer.clear(); + } + { + response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "-1"}}; + EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("-1"))); + response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "42738"}}; + EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("42738"))); + response_header.clear(); + } + // Test "not gRPC request" with response_trailer + { + request_header = {{":method", "GET"}, {":path", "/health"}}; + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "0"}}; + EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); + response_trailer.clear(); + request_header = {{":method", "GET"}, {":path", "/"}, {"content-type", "application/grpc"}}; + } + // Test "not gRPC request" with response_header + { + request_header = {{":method", "GET"}, {":path", "/health"}}; + response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "2"}}; + EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); + response_header.clear(); + request_header = {{":method", "GET"}, {":path", "/"}, {"content-type", "application/grpc"}}; + } +} + +TEST(SubstitutionFormatterTest, + GrpcStatusFormatterCamelStringTest_validate_grpc_header_before_log_grpc_status) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({ + {"envoy.reloadable_features.validate_grpc_header_before_log_grpc_status", "false"}, + }); + GrpcStatusFormatter formatter("grpc-status", "", absl::optional(), GrpcStatusFormatter::Format::CamelString); NiceMock stream_info; @@ -2740,6 +2834,112 @@ TEST(SubstitutionFormatterTest, GrpcStatusFormatterCamelStringTest) { } TEST(SubstitutionFormatterTest, GrpcStatusFormatterSnakeStringTest) { + GrpcStatusFormatter formatter("grpc-status", "", absl::optional(), + GrpcStatusFormatter::Format::SnakeString); + NiceMock stream_info; + Http::TestRequestHeaderMapImpl request_header{{"content-type", "application/grpc"}, + {":path", "/"}}; + Http::TestResponseHeaderMapImpl response_header; + Http::TestResponseTrailerMapImpl response_trailer; + std::string body; + + std::vector grpc_statuses{"OK", + "CANCELLED", + "UNKNOWN", + "INVALID_ARGUMENT", + "DEADLINE_EXCEEDED", + "NOT_FOUND", + "ALREADY_EXISTS", + "PERMISSION_DENIED", + "RESOURCE_EXHAUSTED", + "FAILED_PRECONDITION", + "ABORTED", + "OUT_OF_RANGE", + "UNIMPLEMENTED", + "INTERNAL", + "UNAVAILABLE", + "DATA_LOSS", + "UNAUTHENTICATED"}; + for (size_t i = 0; i < grpc_statuses.size(); ++i) { + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", std::to_string(i)}}; + EXPECT_EQ(grpc_statuses[i], + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue(grpc_statuses[i]))); + } + { + response_trailer = Http::TestResponseTrailerMapImpl{{"not-a-grpc-status", "13"}}; + EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); + } + { + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "-1"}}; + EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("-1"))); + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "42738"}}; + EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("42738"))); + response_trailer.clear(); + } + { + response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "-1"}}; + EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("-1"))); + response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "42738"}}; + EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::stringValue("42738"))); + response_header.clear(); + } + // Test "not gRPC request" with response_trailer + { + request_header = {{":method", "GET"}, {":path", "/health"}}; + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "0"}}; + EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); + response_trailer.clear(); + request_header = {{":method", "GET"}, {":path", "/"}, {"content-type", "application/grpc"}}; + } + // Test "not gRPC request" with response_header + { + request_header = {{":method", "GET"}, {":path", "/health"}}; + response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "2"}}; + EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); + response_header.clear(); + request_header = {{":method", "GET"}, {":path", "/"}, {"content-type", "application/grpc"}}; + } +} + +TEST(SubstitutionFormatterTest, + GrpcStatusFormatterSnakeStringTest_validate_grpc_header_before_log_grpc_status) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({ + {"envoy.reloadable_features.validate_grpc_header_before_log_grpc_status", "false"}, + }); + GrpcStatusFormatter formatter("grpc-status", "", absl::optional(), GrpcStatusFormatter::Format::SnakeString); NiceMock stream_info; @@ -2815,6 +3015,97 @@ TEST(SubstitutionFormatterTest, GrpcStatusFormatterSnakeStringTest) { } TEST(SubstitutionFormatterTest, GrpcStatusFormatterNumberTest) { + GrpcStatusFormatter formatter("grpc-status", "", absl::optional(), + GrpcStatusFormatter::Format::Number); + NiceMock stream_info; + Http::TestRequestHeaderMapImpl request_header{{"content-type", "application/grpc"}, + {":path", "/"}}; + Http::TestResponseHeaderMapImpl response_header; + Http::TestResponseTrailerMapImpl response_trailer; + std::string body; + + const int grpcStatuses = static_cast(Grpc::Status::WellKnownGrpcStatus::MaximumKnown) + 1; + + for (size_t i = 0; i < grpcStatuses; ++i) { + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", std::to_string(i)}}; + EXPECT_EQ(std::to_string(i), + formatter.format(request_header, response_header, response_trailer, stream_info, body, + AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(i))); + } + { + response_trailer = Http::TestResponseTrailerMapImpl{{"not-a-grpc-status", "13"}}; + EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); + } + { + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "-1"}}; + EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(-1))); + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "42738"}}; + EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(42738))); + response_trailer.clear(); + } + { + response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "-1"}}; + EXPECT_EQ("-1", formatter.format(request_header, response_header, response_trailer, stream_info, + body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(-1))); + response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "42738"}}; + EXPECT_EQ("42738", formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(42738))); + response_header.clear(); + } + // Test "not gRPC request" with response_trailer + { + request_header = {{":method", "GET"}, {":path", "/health"}}; + response_trailer = Http::TestResponseTrailerMapImpl{{"grpc-status", "0"}}; + EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); + response_trailer.clear(); + request_header = {{":method", "GET"}, {":path", "/"}, {"content-type", "application/grpc"}}; + } + // Test "not gRPC request" with response_header + { + request_header = {{":method", "GET"}, {":path", "/health"}}; + response_header = Http::TestResponseHeaderMapImpl{{"grpc-status", "2"}}; + EXPECT_EQ(absl::nullopt, formatter.format(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(formatter.formatValue(request_header, response_header, response_trailer, + stream_info, body, AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::nullValue())); + response_header.clear(); + request_header = {{":method", "GET"}, {":path", "/"}, {"content-type", "application/grpc"}}; + } +} + +TEST(SubstitutionFormatterTest, + GrpcStatusFormatterNumberTest_validate_grpc_header_before_log_grpc_status) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({ + {"envoy.reloadable_features.validate_grpc_header_before_log_grpc_status", "false"}, + }); + GrpcStatusFormatter formatter("grpc-status", "", absl::optional(), GrpcStatusFormatter::Format::Number); NiceMock stream_info; From dc860e850c577484d31b7b7e3697d6de95cc6d01 Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Tue, 16 May 2023 11:29:40 -0400 Subject: [PATCH 250/740] Cleanup an ext_proc filter API comments. (#27426) Signed-off-by: Yanjun Xiang --- .../extensions/filters/http/ext_proc/v3/ext_proc.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto index a924b7625922..038967f3f055 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto @@ -143,8 +143,9 @@ message ExternalProcessor { // a message on the stream that requires a response, it will reset this timer, // and will stop processing and return an error (subject to the processing mode) // if the timer expires before a matching response is received. There is no - // timeout when the filter is running in asynchronous mode. The - // ``message_timeout`` range is >= 0s and <= 3600s. Default is 200 milliseconds. + // timeout when the filter is running in asynchronous mode. Zero is a valid + // config which means the timer will be triggered immediately. If not + // configured, default is 200 milliseconds. google.protobuf.Duration message_timeout = 7 [(validate.rules).duration = { lte {seconds: 3600} gte {} @@ -169,7 +170,6 @@ message ExternalProcessor { // Specify the upper bound of // :ref:`override_message_timeout ` - // The ``max_message_timeout`` range is >= 0s and <= 3600s. // If not specified, by default it is 0, which will effectively disable the ``override_message_timeout`` API. google.protobuf.Duration max_message_timeout = 10 [(validate.rules).duration = { lte {seconds: 3600} From e45360d435713eaf6eafb8316ddf19dc3f6a11c8 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 16 May 2023 11:32:12 -0400 Subject: [PATCH 251/740] rest: moving rest api fetcher to extensions (#27328) moving extension only code to its extension branch Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- .../filters/network/source/BUILD | 2 +- .../filters/network/source/client_ssl_auth.h | 2 +- source/common/http/BUILD | 15 --------------- .../extensions/config_subscription/rest/BUILD | 18 +++++++++++++++++- .../rest/http_subscription_impl.h | 2 +- .../rest}/rest_api_fetcher.cc | 2 +- .../rest}/rest_api_fetcher.h | 0 source/extensions/filters/http/oauth2/BUILD | 1 - source/extensions/filters/http/oauth2/filter.h | 1 - test/per_file_coverage.sh | 1 + 10 files changed, 22 insertions(+), 22 deletions(-) rename source/{common/http => extensions/config_subscription/rest}/rest_api_fetcher.cc (97%) rename source/{common/http => extensions/config_subscription/rest}/rest_api_fetcher.h (100%) diff --git a/contrib/client_ssl_auth/filters/network/source/BUILD b/contrib/client_ssl_auth/filters/network/source/BUILD index 015cfa4f800f..ba7c4a5b0fc9 100644 --- a/contrib/client_ssl_auth/filters/network/source/BUILD +++ b/contrib/client_ssl_auth/filters/network/source/BUILD @@ -27,11 +27,11 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/http:headers_lib", "//source/common/http:message_lib", - "//source/common/http:rest_api_fetcher_lib", "//source/common/http:utility_lib", "//source/common/json:json_loader_lib", "//source/common/network:cidr_range_lib", "//source/common/network:utility_lib", + "//source/extensions/config_subscription/rest:rest_api_fetcher_lib", "@envoy_api//contrib/envoy/extensions/filters/network/client_ssl_auth/v3:pkg_cc_proto", ], ) diff --git a/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.h b/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.h index 1cda2251eaef..23452b534237 100644 --- a/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.h +++ b/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.h @@ -12,10 +12,10 @@ #include "envoy/thread_local/thread_local.h" #include "envoy/upstream/cluster_manager.h" -#include "source/common/http/rest_api_fetcher.h" #include "source/common/network/cidr_range.h" #include "source/common/network/utility.h" #include "source/common/protobuf/utility.h" +#include "source/extensions/config_subscription/rest/rest_api_fetcher.h" #include "absl/container/node_hash_set.h" #include "contrib/envoy/extensions/filters/network/client_ssl_auth/v3/client_ssl_auth.pb.h" diff --git a/source/common/http/BUILD b/source/common/http/BUILD index 2c04364a7f2b..d22807d0a7b4 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -448,21 +448,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "rest_api_fetcher_lib", - srcs = ["rest_api_fetcher.cc"], - hdrs = ["rest_api_fetcher.h"], - deps = [ - ":message_lib", - ":utility_lib", - "//envoy/event:dispatcher_interface", - "//envoy/runtime:runtime_interface", - "//envoy/upstream:cluster_manager_interface", - "//source/common/common:enum_to_int", - "//source/common/config:utility_lib", - ], -) - envoy_cc_library( name = "user_agent_lib", srcs = ["user_agent.cc"], diff --git a/source/extensions/config_subscription/rest/BUILD b/source/extensions/config_subscription/rest/BUILD index 8ec6b68ee20a..360ed0cdace4 100644 --- a/source/extensions/config_subscription/rest/BUILD +++ b/source/extensions/config_subscription/rest/BUILD @@ -1,6 +1,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_extension", + "envoy_cc_library", "envoy_extension_package", ) @@ -20,6 +21,7 @@ envoy_cc_extension( "//test:__subpackages__", ], deps = [ + ":rest_api_fetcher_lib", "//envoy/config:subscription_interface", "//envoy/event:dispatcher_interface", "//source/common/buffer:buffer_lib", @@ -30,9 +32,23 @@ envoy_cc_extension( "//source/common/config:type_to_endpoint_lib", "//source/common/config:utility_lib", "//source/common/http:headers_lib", - "//source/common/http:rest_api_fetcher_lib", "//source/common/protobuf", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) + +envoy_cc_library( + name = "rest_api_fetcher_lib", + srcs = ["rest_api_fetcher.cc"], + hdrs = ["rest_api_fetcher.h"], + deps = [ + "//envoy/event:dispatcher_interface", + "//envoy/runtime:runtime_interface", + "//envoy/upstream:cluster_manager_interface", + "//source/common/common:enum_to_int", + "//source/common/config:utility_lib", + "//source/common/http:message_lib", + "//source/common/http:utility_lib", + ], +) diff --git a/source/extensions/config_subscription/rest/http_subscription_impl.h b/source/extensions/config_subscription/rest/http_subscription_impl.h index cd42c9e2074e..9275ca616987 100644 --- a/source/extensions/config_subscription/rest/http_subscription_impl.h +++ b/source/extensions/config_subscription/rest/http_subscription_impl.h @@ -8,7 +8,7 @@ #include "source/common/config/api_version.h" #include "source/common/config/type_to_endpoint.h" #include "source/common/config/utility.h" -#include "source/common/http/rest_api_fetcher.h" +#include "source/extensions/config_subscription/rest/rest_api_fetcher.h" namespace Envoy { namespace Config { diff --git a/source/common/http/rest_api_fetcher.cc b/source/extensions/config_subscription/rest/rest_api_fetcher.cc similarity index 97% rename from source/common/http/rest_api_fetcher.cc rename to source/extensions/config_subscription/rest/rest_api_fetcher.cc index cbcb6a99232c..92c06f023d19 100644 --- a/source/common/http/rest_api_fetcher.cc +++ b/source/extensions/config_subscription/rest/rest_api_fetcher.cc @@ -1,4 +1,4 @@ -#include "source/common/http/rest_api_fetcher.h" +#include "source/extensions/config_subscription/rest/rest_api_fetcher.h" #include #include diff --git a/source/common/http/rest_api_fetcher.h b/source/extensions/config_subscription/rest/rest_api_fetcher.h similarity index 100% rename from source/common/http/rest_api_fetcher.h rename to source/extensions/config_subscription/rest/rest_api_fetcher.h diff --git a/source/extensions/filters/http/oauth2/BUILD b/source/extensions/filters/http/oauth2/BUILD index b53009140df5..936579f59266 100644 --- a/source/extensions/filters/http/oauth2/BUILD +++ b/source/extensions/filters/http/oauth2/BUILD @@ -52,7 +52,6 @@ envoy_cc_library( "//source/common/config:datasource_lib", "//source/common/crypto:utility_lib", "//source/common/formatter:substitution_formatter_lib", - "//source/common/http:rest_api_fetcher_lib", "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/source/extensions/filters/http/oauth2/filter.h b/source/extensions/filters/http/oauth2/filter.h index 93e555035ebd..1ecde65b40fd 100644 --- a/source/extensions/filters/http/oauth2/filter.h +++ b/source/extensions/filters/http/oauth2/filter.h @@ -20,7 +20,6 @@ #include "source/common/formatter/substitution_formatter.h" #include "source/common/http/header_map_impl.h" #include "source/common/http/header_utility.h" -#include "source/common/http/rest_api_fetcher.h" #include "source/common/http/utility.h" #include "source/extensions/filters/http/common/pass_through_filter.h" #include "source/extensions/filters/http/oauth2/oauth.h" diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 5d5014cbb75c..93807749cc1b 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -80,6 +80,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/health_checkers/grpc:92.0" "source/extensions/load_balancing_policies:95.5" "source/extensions/load_balancing_policies/subset:94.3" +"source/extensions/config_subscription/rest:94.3" "source/extensions/config_subscription:94.8" "source/extensions/config_subscription/grpc:94.0" ) From 36b3f04e622b7f57a0a8d0b03ddbf0162da6739f Mon Sep 17 00:00:00 2001 From: Christoph Pakulski Date: Tue, 16 May 2023 12:04:27 -0400 Subject: [PATCH 252/740] contrib: enable build with boringssl_fips (#27296) Signed-off-by: Christoph Pakulski --- bazel/foreign_cc/BUILD | 3 +- .../ipp-crypto-skip-dynamic-lib.patch | 46 +++++++++++++++++++ bazel/repositories.bzl | 6 +++ .../private_key_providers/source/BUILD | 3 +- contrib/qat/BUILD | 3 +- contrib/sxg/filters/http/source/BUILD | 3 +- 6 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 bazel/foreign_cc/ipp-crypto-skip-dynamic-lib.patch diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index b5b8302929f7..e450ecfee7b2 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -198,7 +198,8 @@ envoy_cmake( lib_source = "@com_github_google_libsxg//:all", out_static_libs = ["libsxg.a"], tags = ["skip_on_windows"], - deps = ["@boringssl//:ssl"], + # Use boringssl alias to select fips vs non-fips version. + deps = ["//bazel:boringssl"], ) envoy_cmake( diff --git a/bazel/foreign_cc/ipp-crypto-skip-dynamic-lib.patch b/bazel/foreign_cc/ipp-crypto-skip-dynamic-lib.patch new file mode 100644 index 000000000000..acc43511f2a2 --- /dev/null +++ b/bazel/foreign_cc/ipp-crypto-skip-dynamic-lib.patch @@ -0,0 +1,46 @@ +diff --git a/sources/ippcp/crypto_mb/src/CMakeLists.txt b/sources/ippcp/crypto_mb/src/CMakeLists.txt +index f75f448..043a0a2 100644 +--- a/sources/ippcp/crypto_mb/src/CMakeLists.txt ++++ b/sources/ippcp/crypto_mb/src/CMakeLists.txt +@@ -90,41 +90,6 @@ if(CMAKE_C_COMPILER_VERSION VERSION_LESS 20.2.3) + COMPILE_FLAGS "${AVX512_CFLAGS} ${CMAKE_C_FLAGS_SECURITY}") + endif() + +-# Create shared library +-if(DYNAMIC_LIB OR MB_STANDALONE) +- if(WIN32) +- add_library(${MB_DYN_LIB_TARGET} SHARED ${CRYPTO_MB_HEADERS} ${CRYPTO_MB_SOURCES} ${CPU_FEATURES_FILE} ${WIN_RESOURCE_FILE}) +- else() +- add_library(${MB_DYN_LIB_TARGET} SHARED ${CRYPTO_MB_HEADERS} ${CRYPTO_MB_SOURCES} ${CPU_FEATURES_FILE}) +- endif() +- +- set_target_properties(${MB_DYN_LIB_TARGET} PROPERTIES C_VISIBILITY_PRESET hidden +- VISIBILITY_INLINES_HIDDEN ON +- LINK_FLAGS "${LINK_FLAGS_DYNAMIC} ${LINK_FLAG_SECURITY}" +- PUBLIC_HEADER "${PUBLIC_HEADERS}" +- ) +- +- if(UNIX) +- set_target_properties(${MB_DYN_LIB_TARGET} PROPERTIES VERSION ${MBX_INTERFACE_VERSION} +- SOVERSION ${MBX_INTERFACE_VERSION_MAJOR}) +- endif() +- +- target_link_libraries(${MB_DYN_LIB_TARGET} OpenSSL::Crypto) +-endif(DYNAMIC_LIB OR MB_STANDALONE) +- +-# Installation of the shared library +-if (MB_STANDALONE) # standalone crypto_mb's cmake run +- install(TARGETS ${MB_DYN_LIB_TARGET} +- LIBRARY DESTINATION "lib" +- RUNTIME DESTINATION "lib" +- PUBLIC_HEADER DESTINATION "include/crypto_mb") +-elseif (DYNAMIC_LIB) # build from ippcp's cmake +- install(TARGETS ${MB_DYN_LIB_TARGET} +- LIBRARY DESTINATION "lib/intel64" +- RUNTIME DESTINATION "lib/intel64" +- PUBLIC_HEADER DESTINATION "include/crypto_mb") +-endif() +- + # Static library + if(WIN32) + add_library(${MB_STATIC_LIB_TARGET} STATIC ${CRYPTO_MB_HEADERS} ${CRYPTO_MB_SOURCES} ${CPU_FEATURES_FILE} ${WIN_RESOURCE_FILE}) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 4a4e05b7f41a..ec0b65060918 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -473,6 +473,12 @@ def _com_github_unicode_org_icu(): def _com_github_intel_ipp_crypto_crypto_mb(): external_http_archive( name = "com_github_intel_ipp_crypto_crypto_mb", + # Patch removes from CMakeLists.txt instructions to + # to create dynamic *.so library target. Linker fails when linking + # with boringssl_fips library. Envoy uses only static library + # anyways, so created dynamic library would not be used anyways. + patches = ["@envoy//bazel/foreign_cc:ipp-crypto-skip-dynamic-lib.patch"], + patch_args = ["-p1"], build_file_content = BUILD_ALL_CONTENT, ) diff --git a/contrib/cryptomb/private_key_providers/source/BUILD b/contrib/cryptomb/private_key_providers/source/BUILD index 2ea8a4fa6b62..199acb8fe411 100644 --- a/contrib/cryptomb/private_key_providers/source/BUILD +++ b/contrib/cryptomb/private_key_providers/source/BUILD @@ -28,7 +28,8 @@ envoy_cmake( target_compatible_with = envoy_contrib_linux_x86_64_constraints(), visibility = ["//visibility:private"], working_directory = "sources/ippcp/crypto_mb", - deps = ["@boringssl//:ssl"], + # Use boringssl alias to select fips vs non-fips version. + deps = ["//bazel:boringssl"], ) envoy_cc_library( diff --git a/contrib/qat/BUILD b/contrib/qat/BUILD index c47b21a8c2cb..c56f6694b89f 100644 --- a/contrib/qat/BUILD +++ b/contrib/qat/BUILD @@ -32,6 +32,7 @@ configure_make( ], target_compatible_with = envoy_contrib_linux_x86_64_constraints(), visibility = ["//visibility:public"], - deps = ["@boringssl//:ssl"], + # Use boringssl alias to select fips vs non-fips version. + deps = ["//bazel:boringssl"], alwayslink = True, ) diff --git a/contrib/sxg/filters/http/source/BUILD b/contrib/sxg/filters/http/source/BUILD index 34d61bc21a94..a6bb7cec6708 100644 --- a/contrib/sxg/filters/http/source/BUILD +++ b/contrib/sxg/filters/http/source/BUILD @@ -29,8 +29,9 @@ envoy_cc_library( "//source/common/stats:symbol_table_lib", "//source/common/stats:utility_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", - "@boringssl//:ssl", "@envoy_api//contrib/envoy/extensions/filters/http/sxg/v3alpha:pkg_cc_proto", + # use boringssl alias to select fips vs non-fips version. + "//bazel:boringssl", ], ) From 1a5d09bc1a251bfd95716514027c3cebdd148d25 Mon Sep 17 00:00:00 2001 From: code Date: Wed, 17 May 2023 00:05:25 +0800 Subject: [PATCH 253/740] sip_proxy: work around the clang tidy error of mock method (#27397) Signed-off-by: wbpcode --- .../sip_proxy/filters/network/test/mocks.h | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/contrib/sip_proxy/filters/network/test/mocks.h b/contrib/sip_proxy/filters/network/test/mocks.h index 9654f292e8d2..ccbfd2e8ab3e 100644 --- a/contrib/sip_proxy/filters/network/test/mocks.h +++ b/contrib/sip_proxy/filters/network/test/mocks.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "envoy/router/router.h" @@ -209,12 +210,17 @@ class MockTrafficRoutingAssistantHandler : public TrafficRoutingAssistantHandler MOCK_METHOD(void, deleteTrafficRoutingAssistant, (const std::string&, const std::string&, const absl::optional), ()); MOCK_METHOD(void, subscribeTrafficRoutingAssistant, (const std::string&), ()); - MOCK_METHOD(void, complete, - (const TrafficRoutingAssistant::ResponseType&, const std::string&, const absl::any&), - ()); MOCK_METHOD(void, doSubscribe, (const envoy::extensions::filters::network::sip_proxy::v3alpha::CustomizedAffinity), ()); + + // TODO(wbpcode): mock this method will cause the clang-tidy error. To avoid this error and + // updating the source code, we will not mock this method for now. It is OK because this method is + // not used in the unit test. If someone wants to mock this method, please update the source code + // to avoid using the absl::any. + void complete(const TrafficRoutingAssistant::ResponseType&, const std::string&, + const absl::any&) override {} + ~MockTrafficRoutingAssistantHandler() override; }; @@ -243,9 +249,12 @@ class MockTrafficRoutingAssistantHandlerDeep : public TrafficRoutingAssistantHan class MockRequestCallbacks : public TrafficRoutingAssistant::RequestCallbacks { public: - MOCK_METHOD(void, complete, - (const TrafficRoutingAssistant::ResponseType&, const std::string&, const absl::any&), - ()); + // TODO(wbpcode): mock this method will cause the clang-tidy error. To avoid this error and + // updating the source code, we will not mock this method for now. It is OK because this method is + // not used in the unit test. If someone wants to mock this method, please update the source code + // to avoid using the absl::any. + void complete(const TrafficRoutingAssistant::ResponseType&, const std::string&, + const absl::any&) override {} }; } // namespace SipProxy From 732ea6ea7a795a74e7ffefd2b79c8529309ddf54 Mon Sep 17 00:00:00 2001 From: asingh-g <100796504+asingh-g@users.noreply.github.com> Date: Tue, 16 May 2023 13:10:14 -0400 Subject: [PATCH 254/740] Update QUICHE from 5569deaa0 to d687c5fc0 (#27409) https://github.com/google/quiche/compare/5569deaa0..d687c5fc0 ``` $ git log 5569deaa0..d687c5fc0 --date=short --no-merges --format="%ad %al %s" 2023-05-12 fayang Add blocked_by_no_connection_id to logging of quic_bug_12714_33 (failed to flush ACK frame when writer is not blocked). 2023-05-12 quiche-dev Remove use of std::function from CreateContextForMultiPortPath() 2023-05-11 dschinazi Fix warning in EncryptionLevelToProto 2023-05-11 diannahu Tweak BalsaHeadersSequence to prepare for JetstreamSession integration. 2023-05-11 vasilvv Extend WebTransport error code space to 32-bit. 2023-05-10 fayang Let QUIC framer always infer packet header type from framer's version. ``` Signed-off-by: asingh-g --- bazel/external/quiche.BUILD | 1 + bazel/repository_locations.bzl | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index df09b393695c..216669c395c9 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -4501,6 +4501,7 @@ envoy_cc_library( ":quic_platform_base", ":quiche_common_endian_lib", ":quiche_common_print_elements_lib", + ":quiche_web_transport_web_transport_lib", ], ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index be93fc612f36..76868a57b1a4 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1056,12 +1056,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "5569deaa0b1ed0b4d9bbb84bddb5c8dafa3e2720", - sha256 = "34b564bcd917f9331a8c52e0e8ae027a478ee6e4a7f28c7643de9611e70d7e47", + version = "d687c5fc08dc2d05b4e253237173f99eed06488c", + sha256 = "3c704355f9ee4daab15af666b258071d76f1d985c2c725fe65dbe55d643b92cb", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2023-05-09", + release_date = "2023-05-12", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", From 4fe0d764f51516be42cc3a81c2490b61be11fb38 Mon Sep 17 00:00:00 2001 From: DiazAlan <109677874+DiazAlan@users.noreply.github.com> Date: Tue, 16 May 2023 15:27:00 -0400 Subject: [PATCH 255/740] envoy_reloadable_features_delta_xds_subscription_state_tracking_fix deprecation (#27357) Fixes #26901 Signed-off-by: AlanDiaz --- changelogs/current.yaml | 3 + .../xds_mux/delta_subscription_state.cc | 22 +----- source/common/runtime/runtime_features.cc | 1 - .../grpc/new_delta_subscription_state.cc | 21 +----- .../grpc/delta_subscription_state_test.cc | 69 ------------------- 5 files changed, 5 insertions(+), 111 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 36c196d1ff5e..0a932d37a38c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -120,6 +120,9 @@ removed_config_or_runtime: - area: header_formatters change: | removed runtime key ``envoy.reloadable_features.unified_header_formatter`` and legacy code paths. +- area: config + change: | + removed runtime key ``envoy.reloadable_features.delta_xds_subscription_state_tracking_fix`` and legacy code paths. new_features: - area: access_log diff --git a/source/common/config/xds_mux/delta_subscription_state.cc b/source/common/config/xds_mux/delta_subscription_state.cc index a60328b80f02..f7bd686d93a0 100644 --- a/source/common/config/xds_mux/delta_subscription_state.cc +++ b/source/common/config/xds_mux/delta_subscription_state.cc @@ -183,25 +183,6 @@ void DeltaSubscriptionState::handleGoodResponse( } } - if (!Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.delta_xds_subscription_state_tracking_fix")) { - const auto scoped_update = ttl_.scopedTtlUpdate(); - if (requested_resource_state_.contains(Wildcard)) { - for (const auto& resource : message.resources()) { - addResourceStateFromServer(resource); - } - } else { - // We are not subscribed to wildcard, so we only take resources that we explicitly requested - // and ignore the others. - // NOTE: This is not gonna work for xdstp resources with glob resource matching. - for (const auto& resource : message.resources()) { - if (requested_resource_state_.contains(resource.name())) { - addResourceStateFromServer(resource); - } - } - } - } - callbacks().onConfigUpdate(non_heartbeat_resources, message.removed_resources(), message.system_version_info()); @@ -211,8 +192,7 @@ void DeltaSubscriptionState::handleGoodResponse( message.removed_resources()); } - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.delta_xds_subscription_state_tracking_fix")) { + { const auto scoped_update = ttl_.scopedTtlUpdate(); if (requested_resource_state_.contains(Wildcard)) { for (const auto& resource : message.resources()) { diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 03042d34360f..0fe66d8d6868 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -34,7 +34,6 @@ RUNTIME_GUARD(envoy_reloadable_features_append_query_parameters_path_rewriter); RUNTIME_GUARD(envoy_reloadable_features_append_xfh_idempotent); RUNTIME_GUARD(envoy_reloadable_features_conn_pool_delete_when_idle); RUNTIME_GUARD(envoy_reloadable_features_count_unused_mapped_pages_as_free); -RUNTIME_GUARD(envoy_reloadable_features_delta_xds_subscription_state_tracking_fix); RUNTIME_GUARD(envoy_reloadable_features_enable_aws_credentials_file); RUNTIME_GUARD(envoy_reloadable_features_enable_compression_bomb_protection); RUNTIME_GUARD(envoy_reloadable_features_enable_intermediate_ca); diff --git a/source/extensions/config_subscription/grpc/new_delta_subscription_state.cc b/source/extensions/config_subscription/grpc/new_delta_subscription_state.cc index 7798351f873a..291f70506f4b 100644 --- a/source/extensions/config_subscription/grpc/new_delta_subscription_state.cc +++ b/source/extensions/config_subscription/grpc/new_delta_subscription_state.cc @@ -213,24 +213,6 @@ void NewDeltaSubscriptionState::handleGoodResponse( } } - if (!Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.delta_xds_subscription_state_tracking_fix")) { - const auto scoped_update = ttl_.scopedTtlUpdate(); - if (requested_resource_state_.contains(Wildcard)) { - for (const auto& resource : message.resources()) { - addResourceStateFromServer(resource); - } - } else { - // We are not subscribed to wildcard, so we only take resources that we explicitly requested - // and ignore the others. - for (const auto& resource : message.resources()) { - if (requested_resource_state_.contains(resource.name())) { - addResourceStateFromServer(resource); - } - } - } - } - watch_map_.onConfigUpdate(non_heartbeat_resources, message.removed_resources(), message.system_version_info()); @@ -240,8 +222,7 @@ void NewDeltaSubscriptionState::handleGoodResponse( message.removed_resources()); } - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.delta_xds_subscription_state_tracking_fix")) { + { const auto scoped_update = ttl_.scopedTtlUpdate(); if (requested_resource_state_.contains(Wildcard)) { for (const auto& resource : message.resources()) { diff --git a/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc b/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc index 899fb1e55dab..b644f8c6b4ee 100644 --- a/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc +++ b/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc @@ -713,71 +713,6 @@ TEST_P(DeltaSubscriptionStateTest, AckGenerated) { } } -// Verifies that a sequence of good and bad responses from the server all get the appropriate -// ACKs/NACKs from Envoy when the state is updated before the resources are ingested -// (`envoy.reloadable_features.delta_xds_subscription_state_tracking_fix` is false). -// TODO(adisuissa): This test should be removed once the runtime flag -// `envoy.reloadable_features.delta_xds_subscription_state_tracking_fix` is -// removed and the AckGenerated test will cover the correct behavior. -TEST_P(DeltaSubscriptionStateTest, AckGeneratedPreStateFix) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.delta_xds_subscription_state_tracking_fix", "false"}}); - // The xDS server's first response includes items for name1 and 2, but not 3. - { - Protobuf::RepeatedPtrField added_resources = - populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); - EXPECT_CALL(*ttl_timer_, disableTimer()); - UpdateAck ack = deliverDiscoveryResponse(added_resources, {}, "debug1", "nonce1"); - EXPECT_EQ("nonce1", ack.nonce_); - EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); - } - // The next response updates 1 and 2, and adds 3. - { - Protobuf::RepeatedPtrField added_resources = - populateRepeatedResource( - {{"name1", "version1B"}, {"name2", "version2B"}, {"name3", "version3A"}}); - EXPECT_CALL(*ttl_timer_, disableTimer()); - UpdateAck ack = deliverDiscoveryResponse(added_resources, {}, "debug2", "nonce2"); - EXPECT_EQ("nonce2", ack.nonce_); - EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); - } - // The next response tries but fails to update all 3, and so should produce a NACK. - { - Protobuf::RepeatedPtrField added_resources = - populateRepeatedResource( - {{"name1", "version1C"}, {"name2", "version2C"}, {"name3", "version3B"}}); - EXPECT_CALL(*ttl_timer_, disableTimer()); - UpdateAck ack = deliverBadDiscoveryResponse(added_resources, {}, "debug3", "nonce3", "oh no"); - EXPECT_EQ("nonce3", ack.nonce_); - EXPECT_NE(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); - } - // The last response successfully updates all 3. - { - Protobuf::RepeatedPtrField added_resources = - populateRepeatedResource( - {{"name1", "version1D"}, {"name2", "version2D"}, {"name3", "version3C"}}); - EXPECT_CALL(*ttl_timer_, disableTimer()); - UpdateAck ack = deliverDiscoveryResponse(added_resources, {}, "debug4", "nonce4"); - EXPECT_EQ("nonce4", ack.nonce_); - EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); - } - // Bad response error detail is truncated if it's too large. - { - const std::string very_large_error_message(1 << 20, 'A'); - Protobuf::RepeatedPtrField added_resources = - populateRepeatedResource( - {{"name1", "version1D"}, {"name2", "version2D"}, {"name3", "version3D"}}); - EXPECT_CALL(*ttl_timer_, disableTimer()); - UpdateAck ack = deliverBadDiscoveryResponse(added_resources, {}, "debug5", "nonce5", - very_large_error_message); - EXPECT_EQ("nonce5", ack.nonce_); - EXPECT_NE(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); - EXPECT_TRUE(absl::EndsWith(ack.error_detail_.message(), "AAAAAAA...(truncated)")); - EXPECT_LT(ack.error_detail_.message().length(), very_large_error_message.length()); - } -} - // Tests population of the initial_resource_versions map in the first request of a new stream. // Tests that // 1) resources we have a version of are present in the map, @@ -1310,10 +1245,6 @@ TEST_P(DeltaSubscriptionStateTest, NoVersionUpdateOnNack) { Protobuf::RepeatedPtrField added_resources = populateRepeatedResource( {{"name1", "version1B"}, {"name2", "version2B"}, {"name3", "version3A"}}); - if (!Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.delta_xds_subscription_state_tracking_fix")) { - EXPECT_CALL(*ttl_timer_, disableTimer()); - } UpdateAck ack = deliverBadDiscoveryResponse(added_resources, {}, "debug2", "nonce2", "oh no"); EXPECT_EQ("nonce2", ack.nonce_); EXPECT_NE(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); From 84641e16b1ca556f3d7dde0b105cca66a398176a Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 16 May 2023 16:21:04 -0400 Subject: [PATCH 256/740] Expose async client stream info to consumers (#27361) This will allow ext_proc to extract info such as the endpoint used, bytes sent/received, etc. Risk Level: Low Testing: Unit test added. Signed-off-by: Shriram Rajagopalan --- envoy/grpc/BUILD | 1 + envoy/grpc/async_client.h | 6 ++++++ envoy/http/BUILD | 1 + envoy/http/async_client.h | 5 +++++ source/common/grpc/BUILD | 3 +++ source/common/grpc/async_client_impl.cc | 1 - source/common/grpc/async_client_impl.h | 2 ++ source/common/grpc/google_async_client_impl.cc | 6 +++++- source/common/grpc/google_async_client_impl.h | 7 +++++++ source/common/grpc/typed_async_client.h | 1 + source/common/http/BUILD | 1 + source/common/http/async_client_impl.cc | 1 + source/common/http/async_client_impl.h | 1 + test/common/grpc/grpc_client_integration_test.cc | 7 +++++++ test/common/grpc/grpc_client_integration_test_harness.h | 7 +++++-- test/mocks/grpc/BUILD | 1 + test/mocks/grpc/mocks.h | 1 + test/mocks/http/mocks.h | 1 + 18 files changed, 49 insertions(+), 4 deletions(-) diff --git a/envoy/grpc/BUILD b/envoy/grpc/BUILD index abe8126cb18f..62c5f36b1a4b 100644 --- a/envoy/grpc/BUILD +++ b/envoy/grpc/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( "//envoy/buffer:buffer_interface", "//envoy/http:async_client_interface", "//envoy/http:header_map_interface", + "//envoy/stream_info:stream_info_interface", "//envoy/tracing:tracer_interface", "//source/common/common:assert_lib", "//source/common/protobuf", diff --git a/envoy/grpc/async_client.h b/envoy/grpc/async_client.h index 333914a12d70..d286d686e8f5 100644 --- a/envoy/grpc/async_client.h +++ b/envoy/grpc/async_client.h @@ -7,6 +7,7 @@ #include "envoy/grpc/status.h" #include "envoy/http/async_client.h" #include "envoy/http/header_map.h" +#include "envoy/stream_info/stream_info.h" #include "envoy/tracing/tracer.h" #include "source/common/common/assert.h" @@ -64,6 +65,11 @@ class RawAsyncStream { * limits */ virtual bool isAboveWriteBufferHighWatermark() const PURE; + + /** + * @returns the stream info object associated with this stream. + */ + virtual const StreamInfo::StreamInfo& streamInfo() const PURE; }; class RawAsyncRequestCallbacks { diff --git a/envoy/http/BUILD b/envoy/http/BUILD index dbb2e8f80ab3..ea9ef7595241 100644 --- a/envoy/http/BUILD +++ b/envoy/http/BUILD @@ -32,6 +32,7 @@ envoy_cc_library( ":filter_interface", ":message_interface", "//envoy/event:dispatcher_interface", + "//envoy/stream_info:stream_info_interface", "//envoy/tracing:tracer_interface", "//source/common/protobuf", "//source/common/protobuf:utility_lib", diff --git a/envoy/http/async_client.h b/envoy/http/async_client.h index ae292885a1d2..5da55ee23b6f 100644 --- a/envoy/http/async_client.h +++ b/envoy/http/async_client.h @@ -199,6 +199,11 @@ class AsyncClient { * limits */ virtual bool isAboveWriteBufferHighWatermark() const PURE; + + /*** + * @returns the stream info object associated with the stream. + */ + virtual const StreamInfo::StreamInfo& streamInfo() const PURE; }; /*** diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index ba0b8049115f..4c03865913af 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -22,6 +22,7 @@ envoy_cc_library( ":codec_lib", ":context_lib", "//envoy/grpc:async_client_interface", + "//envoy/stream_info:stream_info_interface", "//source/common/buffer:zero_copy_input_stream_lib", "//source/common/http:async_client_lib", ], @@ -36,6 +37,7 @@ envoy_cc_library( ":context_lib", ":typed_async_client_lib", "//envoy/grpc:async_client_interface", + "//envoy/stream_info:stream_info_interface", "//source/common/buffer:zero_copy_input_stream_lib", "//source/common/http:async_client_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", @@ -174,6 +176,7 @@ envoy_cc_library( ":typed_async_client_lib", "//envoy/api:api_interface", "//envoy/grpc:google_grpc_creds_interface", + "//envoy/stream_info:stream_info_interface", "//envoy/thread:thread_interface", "//envoy/thread_local:thread_local_object", "//source/common/common:base64_lib", diff --git a/source/common/grpc/async_client_impl.cc b/source/common/grpc/async_client_impl.cc index f6f087ee38c1..ab940d039850 100644 --- a/source/common/grpc/async_client_impl.cc +++ b/source/common/grpc/async_client_impl.cc @@ -83,7 +83,6 @@ void AsyncStreamImpl::initialize(bool buffer_body_for_retry) { auto& http_async_client = thread_local_cluster->httpAsyncClient(); dispatcher_ = &http_async_client.dispatcher(); stream_ = http_async_client.start(*this, options_.setBufferBodyForRetry(buffer_body_for_retry)); - if (stream_ == nullptr) { callbacks_.onRemoteClose(Status::WellKnownGrpcStatus::Unavailable, EMPTY_STRING); http_reset_ = true; diff --git a/source/common/grpc/async_client_impl.h b/source/common/grpc/async_client_impl.h index d75066e6631b..ffd5c411cf50 100644 --- a/source/common/grpc/async_client_impl.h +++ b/source/common/grpc/async_client_impl.h @@ -5,6 +5,7 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/grpc/async_client.h" +#include "envoy/stream_info/stream_info.h" #include "source/common/common/linked_object.h" #include "source/common/grpc/codec.h" @@ -78,6 +79,7 @@ class AsyncStreamImpl : public RawAsyncStream, } bool hasResetStream() const { return http_reset_; } + const StreamInfo::StreamInfo& streamInfo() const override { return stream_->streamInfo(); } private: void streamError(Status::GrpcStatus grpc_status, const std::string& message); diff --git a/source/common/grpc/google_async_client_impl.cc b/source/common/grpc/google_async_client_impl.cc index 18b95f22aa89..326319585a73 100644 --- a/source/common/grpc/google_async_client_impl.cc +++ b/source/common/grpc/google_async_client_impl.cc @@ -1,11 +1,14 @@ #include "source/common/grpc/google_async_client_impl.h" +#include "envoy/common/time.h" #include "envoy/config/core/v3/grpc_service.pb.h" +#include "envoy/http/protocol.h" #include "envoy/stats/scope.h" #include "source/common/common/base64.h" #include "source/common/common/empty_string.h" #include "source/common/common/lock_guard.h" +#include "source/common/common/utility.h" #include "source/common/config/datasource.h" #include "source/common/grpc/common.h" #include "source/common/grpc/google_grpc_creds_impl.h" @@ -161,7 +164,8 @@ GoogleAsyncStreamImpl::GoogleAsyncStreamImpl(GoogleAsyncClientImpl& parent, const Http::AsyncClient::StreamOptions& options) : parent_(parent), tls_(parent_.tls_), dispatcher_(parent_.dispatcher_), stub_(parent_.stub_), service_full_name_(service_full_name), method_name_(method_name), callbacks_(callbacks), - options_(options) {} + options_(options), unused_stream_info_(Http::Protocol::Http2, dispatcher_.timeSource(), + Network::ConnectionInfoProviderSharedPtr{}) {} GoogleAsyncStreamImpl::~GoogleAsyncStreamImpl() { ENVOY_LOG(debug, "GoogleAsyncStreamImpl destruct"); diff --git a/source/common/grpc/google_async_client_impl.h b/source/common/grpc/google_async_client_impl.h index a790d35fcdb3..9fb4ad022fd8 100644 --- a/source/common/grpc/google_async_client_impl.h +++ b/source/common/grpc/google_async_client_impl.h @@ -9,6 +9,7 @@ #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/grpc/async_client.h" #include "envoy/stats/scope.h" +#include "envoy/stream_info/stream_info.h" #include "envoy/thread/thread.h" #include "envoy/thread_local/thread_local_object.h" #include "envoy/tracing/tracer.h" @@ -20,6 +21,7 @@ #include "source/common/grpc/stat_names.h" #include "source/common/grpc/typed_async_client.h" #include "source/common/router/header_parser.h" +#include "source/common/stream_info/stream_info_impl.h" #include "source/common/tracing/http_tracer_impl.h" #include "absl/container/node_hash_set.h" @@ -232,6 +234,7 @@ class GoogleAsyncStreamImpl : public RawAsyncStream, bool isAboveWriteBufferHighWatermark() const override { return bytes_in_write_pending_queue_ > parent_.perStreamBufferLimitBytes(); } + const StreamInfo::StreamInfo& streamInfo() const override { return unused_stream_info_; } protected: bool callFailed() const { return call_failed_; } @@ -309,6 +312,10 @@ class GoogleAsyncStreamImpl : public RawAsyncStream, // Count of the tags in-flight. This must hit zero before the stream can be // freed. uint32_t inflight_tags_{}; + + // This is unused. + StreamInfo::StreamInfoImpl unused_stream_info_; + // Queue of completed (op, ok) passed from completionThread() to // handleOpCompletion(). std::deque> diff --git a/source/common/grpc/typed_async_client.h b/source/common/grpc/typed_async_client.h index 2fc48e9a23ba..fe48e26501ae 100644 --- a/source/common/grpc/typed_async_client.h +++ b/source/common/grpc/typed_async_client.h @@ -50,6 +50,7 @@ template class AsyncStream /* : public RawAsyncStream */ { } bool operator==(RawAsyncStream* stream) const { return stream_ == stream; } bool operator!=(RawAsyncStream* stream) const { return stream_ != stream; } + const StreamInfo::StreamInfo& streamInfo() const { return stream_->streamInfo(); } private: RawAsyncStream* stream_{}; diff --git a/source/common/http/BUILD b/source/common/http/BUILD index d22807d0a7b4..6177bd4d8a59 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -27,6 +27,7 @@ envoy_cc_library( "//envoy/router:router_ratelimit_interface", "//envoy/router:shadow_writer_interface", "//envoy/ssl:connection_interface", + "//envoy/stream_info:stream_info_interface", "//source/common/common:empty_string", "//source/common/common:linked_object", "//source/common/router:router_lib", diff --git a/source/common/http/async_client_impl.cc b/source/common/http/async_client_impl.cc index dcf06e026fb9..ce3ca75592b9 100644 --- a/source/common/http/async_client_impl.cc +++ b/source/common/http/async_client_impl.cc @@ -166,6 +166,7 @@ void AsyncStreamImpl::sendHeaders(RequestHeaderMap& headers, bool end_stream) { if (send_xff_) { Utility::appendXff(headers, *parent_.config_.local_info_.address()); } + router_.decodeHeaders(headers, end_stream); closeLocal(end_stream); } diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index fc26e6644efb..1bc5536b6212 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -139,6 +139,7 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, void sendTrailers(RequestTrailerMap& trailers) override; void reset() override; bool isAboveWriteBufferHighWatermark() const override { return high_watermark_calls_ > 0; } + const StreamInfo::StreamInfo& streamInfo() const override { return stream_info_; } protected: bool remoteClosed() { return remote_closed_; } diff --git a/test/common/grpc/grpc_client_integration_test.cc b/test/common/grpc/grpc_client_integration_test.cc index 96be74c4871a..3fd32c96d8fc 100644 --- a/test/common/grpc/grpc_client_integration_test.cc +++ b/test/common/grpc/grpc_client_integration_test.cc @@ -52,6 +52,13 @@ TEST_P(GrpcClientIntegrationTest, MultiStream) { auto stream_1 = createStream(empty_metadata_); stream_0->sendRequest(); stream_1->sendRequest(); + // Access stream info to make sure it is present. + if (std::get<1>(GetParam()) == ClientType::EnvoyGrpc) { + envoy::config::core::v3::Metadata m; + (*m.mutable_filter_metadata())["com.foo.bar"] = {}; + EXPECT_THAT(stream_0->grpc_stream_.streamInfo().dynamicMetadata(), ProtoEq(m)); + EXPECT_THAT(stream_1->grpc_stream_.streamInfo().dynamicMetadata(), ProtoEq(m)); + } stream_0->sendServerInitialMetadata(empty_metadata_); stream_0->sendReply(); stream_1->sendServerTrailers(Status::WellKnownGrpcStatus::Unavailable, "", empty_metadata_, true); diff --git a/test/common/grpc/grpc_client_integration_test_harness.h b/test/common/grpc/grpc_client_integration_test_harness.h index dd3bb0831afe..09a13a982b53 100644 --- a/test/common/grpc/grpc_client_integration_test_harness.h +++ b/test/common/grpc/grpc_client_integration_test_harness.h @@ -423,8 +423,11 @@ class GrpcClientIntegrationTest : public GrpcClientIntegrationParamTest { } })); - stream->grpc_stream_ = - grpc_client_->start(*method_descriptor_, *stream, Http::AsyncClient::StreamOptions()); + auto options = Http::AsyncClient::StreamOptions(); + envoy::config::core::v3::Metadata m; + (*m.mutable_filter_metadata())["com.foo.bar"] = {}; + options.setMetadata(m); + stream->grpc_stream_ = grpc_client_->start(*method_descriptor_, *stream, options); EXPECT_NE(stream->grpc_stream_, nullptr); if (!fake_connection_) { diff --git a/test/mocks/grpc/BUILD b/test/mocks/grpc/BUILD index c61c03bf4f90..5ecab4ab40cb 100644 --- a/test/mocks/grpc/BUILD +++ b/test/mocks/grpc/BUILD @@ -15,6 +15,7 @@ envoy_cc_mock( deps = [ "//envoy/grpc:async_client_interface", "//envoy/grpc:async_client_manager_interface", + "//envoy/stream_info:stream_info_interface", "//source/common/grpc:typed_async_client_lib", "//test/mocks/http:http_mocks", "//test/test_common:utility_lib", diff --git a/test/mocks/grpc/mocks.h b/test/mocks/grpc/mocks.h index 5865736582c7..3cff1d2d75aa 100644 --- a/test/mocks/grpc/mocks.h +++ b/test/mocks/grpc/mocks.h @@ -38,6 +38,7 @@ class MockAsyncStream : public RawAsyncStream { MOCK_METHOD(void, closeStream, ()); MOCK_METHOD(void, resetStream, ()); MOCK_METHOD(bool, isAboveWriteBufferHighWatermark, (), (const)); + MOCK_METHOD(const StreamInfo::StreamInfo&, streamInfo, (), (const)); }; template using ResponseTypePtr = std::unique_ptr; diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index fdd09f7f31a8..c6d5f7f3e412 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -554,6 +554,7 @@ class MockAsyncClientStream : public virtual AsyncClient::Stream { MOCK_METHOD(void, setWatermarkCallbacks, (DecoderFilterWatermarkCallbacks & callback), (override)); MOCK_METHOD(void, removeWatermarkCallbacks, (), (override)); + MOCK_METHOD(const StreamInfo::StreamInfo&, streamInfo, (), (const override)); private: absl::optional destructor_callback_; From a170b03435c4b12c0b6e46b6b9800f08dba2bdd5 Mon Sep 17 00:00:00 2001 From: Zhewei Hu Date: Tue, 16 May 2023 17:39:58 -0700 Subject: [PATCH 257/740] [ZK filter] emit fast/slow response metrics for error budget SLI calculation (#26594) Commit Message: [ZK filter] emit fast/slow response metrics for error budget SLI calculation Additional Description: Emit fast/slow response counter metrics for each opcode used for error budget SLI calculation. Risk Level: low Testing: unit tests Docs Changes: This diff will update these two docs ZooKeeper proxy introduction doc and ZooKeeper proxy proto doc. Release Notes: [ZK filter] emit fast/slow response metrics for error budget SLI calculation Platform Specific Features: N/A API Considerations: Did not find a good way to make repeated message unique (LatencyThreshold in this case) with proto validation. Also found that enum cannot be the key of the proto map. Signed-off-by: Zhewei Hu --- .../network/zookeeper_proxy/v3/README.md | 1 + .../zookeeper_proxy/v3/zookeeper_proxy.proto | 63 ++++ .../_include/zookeeper-filter-proxy.yaml | 7 + .../zookeeper_proxy_filter.rst | 131 ++++++-- .../filters/network/zookeeper_proxy/BUILD | 1 + .../filters/network/zookeeper_proxy/config.cc | 20 +- .../filters/network/zookeeper_proxy/filter.cc | 166 ++++++++-- .../filters/network/zookeeper_proxy/filter.h | 129 +++++++- .../network/zookeeper_proxy/config_test.cc | 251 ++++++++++++-- .../network/zookeeper_proxy/filter_test.cc | 312 ++++++++++++++---- 10 files changed, 927 insertions(+), 154 deletions(-) create mode 100644 api/envoy/extensions/filters/network/zookeeper_proxy/v3/README.md diff --git a/api/envoy/extensions/filters/network/zookeeper_proxy/v3/README.md b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/README.md new file mode 100644 index 000000000000..4334e0663aee --- /dev/null +++ b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/README.md @@ -0,0 +1 @@ +Protocol buffer definitions for the ZooKeeper proxy. diff --git a/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto index 4b894051acb2..05422e9c3f8a 100644 --- a/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto +++ b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.filters.network.zookeeper_proxy.v3; +import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; @@ -18,6 +19,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // ZooKeeper Proxy :ref:`configuration overview `. // [#extension: envoy.filters.network.zookeeper_proxy] +// [#next-free-field: 7] message ZooKeeperProxy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.zookeeper_proxy.v1alpha1.ZooKeeperProxy"; @@ -39,4 +41,65 @@ message ZooKeeperProxy { // // if that is set. If it isn't, ZooKeeper's default is also 1Mb. google.protobuf.UInt32Value max_packet_bytes = 3; + + // Whether to emit latency threshold metrics. If not set, defaults to false. + // If false, setting `default_latency_threshold` and `latency_threshold_overrides` will not have effect. + bool enable_latency_threshold_metrics = 4; + + // The default latency threshold to decide the fast/slow responses and emit metrics (used for error budget calculation). + // + // https://sre.google/workbook/implementing-slos/ + // + // If it is not set, the default value is 100 milliseconds. + google.protobuf.Duration default_latency_threshold = 5 + [(validate.rules).duration = {gte {nanos: 1000000}}]; + + // List of latency threshold overrides for opcodes. + // If the threshold override of one opcode is not set, it will fallback to the default latency + // threshold. + // Specifying latency threshold overrides multiple times for one opcode is not allowed. + repeated LatencyThresholdOverride latency_threshold_overrides = 6; +} + +message LatencyThresholdOverride { + enum Opcode { + Connect = 0; + Create = 1; + Delete = 2; + Exists = 3; + GetData = 4; + SetData = 5; + GetAcl = 6; + SetAcl = 7; + GetChildren = 8; + Sync = 9; + Ping = 10; + GetChildren2 = 11; + Check = 12; + Multi = 13; + Create2 = 14; + Reconfig = 15; + CheckWatches = 16; + RemoveWatches = 17; + CreateContainer = 18; + CreateTtl = 19; + Close = 20; + SetAuth = 21; + SetWatches = 22; + GetEphemerals = 23; + GetAllChildrenNumber = 24; + SetWatches2 = 25; + } + + // The ZooKeeper opcodes. Can be found as part of the ZooKeeper source code: + // + // https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/ZooDefs.java + // + Opcode opcode = 1 [(validate.rules).enum = {defined_only: true}]; + + // The latency threshold override of certain opcode. + google.protobuf.Duration threshold = 2 [(validate.rules).duration = { + required: true + gte {nanos: 1000000} + }]; } diff --git a/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml b/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml index 3215fcadad68..4d577e5d7717 100644 --- a/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml +++ b/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml @@ -11,6 +11,13 @@ static_resources: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy stat_prefix: zookeeper + enable_latency_threshold_metrics: true + default_latency_threshold: "0.1s" + latency_threshold_overrides: + - opcode: Create + threshold: "0.15s" + - opcode: Multi + threshold: "0.2s" - name: envoy.filters.network.tcp_proxy typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy diff --git a/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst index 9e7ab55740f5..8f914333364e 100644 --- a/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst @@ -31,8 +31,9 @@ in the configuration snippet below: Statistics ---------- -Every configured ZooKeeper proxy filter has statistics rooted at *.zookeeper.*. The -following counters are available: +Every configured ZooKeeper proxy filter has statistics rooted at *.zookeeper.*. +*_resp_fast* and *_resp_slow* metrics will only be emitted when ``enable_latency_threshold_metrics`` is set to ``true``. +The following counters are available: .. csv-table:: :header: Name, Type, Description @@ -47,27 +48,30 @@ following counters are available: getdata_rq, Counter, Number of getdata requests create_rq, Counter, Number of create requests create2_rq, Counter, Number of create2 requests + createcontainer_rq, Counter, Number of createcontainer requests + createttl_rq, Counter, Number of createttl requests setdata_rq, Counter, Number of setdata requests getchildren_rq, Counter, Number of getchildren requests getchildren2_rq, Counter, Number of getchildren2 requests - remove_rq, Counter, Number of delete requests + delete_rq, Counter, Number of delete requests exists_rq, Counter, Number of stat requests getacl_rq, Counter, Number of getacl requests setacl_rq, Counter, Number of setacl requests sync_rq, Counter, Number of sync requests + check_rq, Counter, Number of check requests multi_rq, Counter, Number of multi transaction requests reconfig_rq, Counter, Number of reconfig requests - close_rq, Counter, Number of close requests setwatches_rq, Counter, Number of setwatches requests setwatches2_rq, Counter, Number of setwatches2 requests checkwatches_rq, Counter, Number of checkwatches requests removewatches_rq, Counter, Number of removewatches requests - check_rq, Counter, Number of check requests + getephemerals_rq, Counter, Number of getephemerals requests + getallchildrennumber_rq, Counter, Number of getallchildrennumber requests + close_rq, Counter, Number of close requests response_bytes, Counter, Number of bytes in decoded response messages connect_resp, Counter, Number of connect responses ping_resp, Counter, Number of ping responses auth_resp, Counter, Number of auth responses - watch_event, Counter, Number of watch events fired by the server getdata_resp, Counter, Number of getdata responses create_resp, Counter, Number of create responses create2_resp, Counter, Number of create2 responses @@ -76,22 +80,77 @@ following counters are available: setdata_resp, Counter, Number of setdata responses getchildren_resp, Counter, Number of getchildren responses getchildren2_resp, Counter, Number of getchildren2 responses - getephemerals_resp, Counter, Number of getephemerals responses - getallchildrennumber_resp, Counter, Number of getallchildrennumber responses - remove_resp, Counter, Number of remove responses + delete_resp, Counter, Number of delete responses exists_resp, Counter, Number of exists responses getacl_resp, Counter, Number of getacl responses setacl_resp, Counter, Number of setacl responses sync_resp, Counter, Number of sync responses + check_resp, Counter, Number of check responses multi_resp, Counter, Number of multi responses reconfig_resp, Counter, Number of reconfig responses - close_resp, Counter, Number of close responses setauth_resp, Counter, Number of setauth responses setwatches_resp, Counter, Number of setwatches responses setwatches2_resp, Counter, Number of setwatches2 responses checkwatches_resp, Counter, Number of checkwatches responses removewatches_resp, Counter, Number of removewatches responses - check_resp, Counter, Number of check responses + getephemerals_resp, Counter, Number of getephemerals responses + getallchildrennumber_resp, Counter, Number of getallchildrennumber responses + close_resp, Counter, Number of close responses + watch_event, Counter, Number of watch events fired by the server + connect_resp_fast, Counter, Number of connect responses faster than the threshold + ping_resp_fast, Counter, Number of ping responses faster than the threshold + auth_resp_fast, Counter, Number of auth responses faster than the threshold + getdata_resp_fast, Counter, Number of getdata responses faster than the threshold + create_resp_fast, Counter, Number of create responses faster than the threshold + create2_resp_fast, Counter, Number of create2 responses faster than the threshold + createcontainer_resp_fast, Counter, Number of createcontainer responses faster than the threshold + createttl_resp_fast, Counter, Number of createttl responses faster than the threshold + setdata_resp_fast, Counter, Number of setdata responses faster than the threshold + getchildren_resp_fast, Counter, Number of getchildren responses faster than the threshold + getchildren2_resp_fast, Counter, Number of getchildren2 responses faster than the threshold + delete_resp_fast, Counter, Number of delete responses faster than the threshold + exists_resp_fast, Counter, Number of exists responses faster than the threshold + getacl_resp_fast, Counter, Number of getacl responses faster than the threshold + setacl_resp_fast, Counter, Number of setacl responses faster than the threshold + sync_resp_fast, Counter, Number of sync responses faster than the threshold + check_resp_fast, Counter, Number of check responses faster than the threshold + multi_resp_fast, Counter, Number of multi responses faster than the threshold + reconfig_resp_fast, Counter, Number of reconfig responses faster than the threshold + setauth_resp_fast, Counter, Number of setauth responses faster than the threshold + setwatches_resp_fast, Counter, Number of setwatches responses faster than the threshold + setwatches2_resp_fast, Counter, Number of setwatches2 responses faster than the threshold + checkwatches_resp_fast, Counter, Number of checkwatches responses faster than the threshold + removewatches_resp_fast, Counter, Number of removewatches responses faster than the threshold + getephemerals_resp_fast, Counter, Number of getephemerals responses faster than the threshold + getallchildrennumber_resp_fast, Counter, Number of getallchildrennumber responses faster than the threshold + close_resp_fast, Counter, Number of close responses faster than the threshold + connect_resp_slow, Counter, Number of connect responses slower than the threshold + ping_resp_slow, Counter, Number of ping responses slower than the threshold + auth_resp_slow, Counter, Number of auth responses slower than the threshold + getdata_resp_slow, Counter, Number of getdata responses slower than the threshold + create_resp_slow, Counter, Number of create responses slower than the threshold + create2_resp_slow, Counter, Number of create2 responses slower than the threshold + createcontainer_resp_slow, Counter, Number of createcontainer responses slower than the threshold + createttl_resp_slow, Counter, Number of createttl responses slower than the threshold + setdata_resp_slow, Counter, Number of setdata responses slower than the threshold + getchildren_resp_slow, Counter, Number of getchildren responses slower than the threshold + getchildren2_resp_slow, Counter, Number of getchildren2 responses slower than the threshold + delete_resp_slow, Counter, Number of delete responses slower than the threshold + exists_resp_slow, Counter, Number of exists responses slower than the threshold + getacl_resp_slow, Counter, Number of getacl responses slower than the threshold + setacl_resp_slow, Counter, Number of setacl responses slower than the threshold + sync_resp_slow, Counter, Number of sync responses slower than the threshold + check_resp_slow, Counter, Number of check responses slower than the threshold + multi_resp_slow, Counter, Number of multi responses slower than the threshold + reconfig_resp_slow, Counter, Number of reconfig responses slower than the threshold + setauth_resp_slow, Counter, Number of setauth responses slower than the threshold + setwatches_resp_slow, Counter, Number of setwatches responses slower than the threshold + setwatches2_resp_slow, Counter, Number of setwatches2 responses slower than the threshold + checkwatches_resp_slow, Counter, Number of checkwatches responses slower than the threshold + removewatches_resp_slow, Counter, Number of removewatches responses slower than the threshold + getephemerals_resp_slow, Counter, Number of getephemerals responses slower than the threshold + getallchildrennumber_resp_slow, Counter, Number of getallchildrennumber responses slower than the threshold + close_resp_slow, Counter, Number of close responses slower than the threshold .. _config_network_filters_zookeeper_proxy_latency_stats: @@ -99,7 +158,7 @@ following counters are available: Per opcode latency statistics ----------------------------- -The filter will gather latency statistics in the *.zookeeper._response_latency* namespace. +The filter will gather latency statistics in the *.zookeeper._resp_latency* namespace. Latency stats are in milliseconds: .. csv-table:: @@ -110,30 +169,30 @@ Latency stats are in milliseconds: ping_response_latency, Histogram, Opcode execution time in milliseconds auth_response_latency, Histogram, Opcode execution time in milliseconds watch_event, Histogram, Opcode execution time in milliseconds - getdata_response_latency, Histogram, Opcode execution time in milliseconds - create_response_latency, Histogram, Opcode execution time in milliseconds - create2_response_latency, Histogram, Opcode execution time in milliseconds - createcontainer_response_latency, Histogram, Opcode execution time in milliseconds - createttl_response_latency, Histogram, Opcode execution time in milliseconds - setdata_response_latency, Histogram, Opcode execution time in milliseconds - getchildren_response_latency, Histogram, Opcode execution time in milliseconds - getchildren2_response_latency, Histogram, Opcode execution time in milliseconds - getephemerals_response_latency, Histogram, Opcode execution time in milliseconds - getallchildrennumber_response_latency, Histogram, Opcode execution time in milliseconds - remove_response_latency, Histogram, Opcode execution time in milliseconds - exists_response_latency, Histogram, Opcode execution time in milliseconds - getacl_response_latency, Histogram, Opcode execution time in milliseconds - setacl_response_latency, Histogram, Opcode execution time in milliseconds - sync_response_latency, Histogram, Opcode execution time in milliseconds - multi_response_latency, Histogram, Opcode execution time in milliseconds - reconfig_response_latency, Histogram, Opcode execution time in milliseconds - close_response_latency, Histogram, Opcode execution time in milliseconds - setauth_response_latency, Histogram, Opcode execution time in milliseconds - setwatches_response_latency, Histogram, Opcode execution time in milliseconds - setwatches2_response_latency, Histogram, Opcode execution time in milliseconds - checkwatches_response_latency, Histogram, Opcode execution time in milliseconds - removewatches_response_latency, Histogram, Opcode execution time in milliseconds - check_response_latency, Histogram, Opcode execution time in milliseconds + getdata_resp_latency, Histogram, Opcode execution time in milliseconds + create_resp_latency, Histogram, Opcode execution time in milliseconds + create2_resp_latency, Histogram, Opcode execution time in milliseconds + createcontainer_resp_latency, Histogram, Opcode execution time in milliseconds + createttl_resp_latency, Histogram, Opcode execution time in milliseconds + setdata_resp_latency, Histogram, Opcode execution time in milliseconds + getchildren_resp_latency, Histogram, Opcode execution time in milliseconds + getchildren2_resp_latency, Histogram, Opcode execution time in milliseconds + getephemerals_resp_latency, Histogram, Opcode execution time in milliseconds + getallchildrennumber_resp_latency, Histogram, Opcode execution time in milliseconds + delete_resp_latency, Histogram, Opcode execution time in milliseconds + exists_resp_latency, Histogram, Opcode execution time in milliseconds + getacl_resp_latency, Histogram, Opcode execution time in milliseconds + setacl_resp_latency, Histogram, Opcode execution time in milliseconds + sync_resp_latency, Histogram, Opcode execution time in milliseconds + multi_resp_latency, Histogram, Opcode execution time in milliseconds + reconfig_resp_latency, Histogram, Opcode execution time in milliseconds + close_resp_latency, Histogram, Opcode execution time in milliseconds + setauth_resp_latency, Histogram, Opcode execution time in milliseconds + setwatches_resp_latency, Histogram, Opcode execution time in milliseconds + setwatches2_resp_latency, Histogram, Opcode execution time in milliseconds + checkwatches_resp_latency, Histogram, Opcode execution time in milliseconds + removewatches_resp_latency, Histogram, Opcode execution time in milliseconds + check_resp_latency, Histogram, Opcode execution time in milliseconds .. _config_network_filters_zookeeper_proxy_dynamic_metadata: diff --git a/source/extensions/filters/network/zookeeper_proxy/BUILD b/source/extensions/filters/network/zookeeper_proxy/BUILD index 245353a623a8..3aa51f72480a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/BUILD +++ b/source/extensions/filters/network/zookeeper_proxy/BUILD @@ -36,6 +36,7 @@ envoy_cc_library( "//source/common/stats:utility_lib", "//source/extensions/filters/network:well_known_names", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/network/zookeeper_proxy/config.cc b/source/extensions/filters/network/zookeeper_proxy/config.cc index 833eb4563b32..691bf9cafef9 100644 --- a/source/extensions/filters/network/zookeeper_proxy/config.cc +++ b/source/extensions/filters/network/zookeeper_proxy/config.cc @@ -27,9 +27,25 @@ Network::FilterFactoryCb ZooKeeperConfigFactory::createFilterFactoryFromProtoTyp const std::string stat_prefix = fmt::format("{}.zookeeper", proto_config.stat_prefix()); const uint32_t max_packet_bytes = PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config, max_packet_bytes, 1024 * 1024); + const bool enable_latency_threshold_metrics = proto_config.enable_latency_threshold_metrics(); + const std::chrono::milliseconds default_latency_threshold( + PROTOBUF_GET_MS_OR_DEFAULT(proto_config, default_latency_threshold, 100)); + const LatencyThresholdOverrideList& latency_threshold_overrides = + proto_config.latency_threshold_overrides(); - ZooKeeperFilterConfigSharedPtr filter_config( - std::make_shared(stat_prefix, max_packet_bytes, context.scope())); + // Check duplicated opcodes in config. + absl::flat_hash_set opcodes; + for (const auto& threshold_override : latency_threshold_overrides) { + auto insertion = opcodes.insert(threshold_override.opcode()); + if (!insertion.second) { + throw EnvoyException(fmt::format("Duplicated opcode find in config: {}", + static_cast(threshold_override.opcode()))); + } + } + + ZooKeeperFilterConfigSharedPtr filter_config(std::make_shared( + stat_prefix, max_packet_bytes, enable_latency_threshold_metrics, default_latency_threshold, + latency_threshold_overrides, context.scope())); auto& time_source = context.mainThreadDispatcher().timeSource(); return [filter_config, &time_source](Network::FilterManager& filter_manager) -> void { diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.cc b/source/extensions/filters/network/zookeeper_proxy/filter.cc index a35700c318e1..665c61ce09e0 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/filter.cc @@ -4,6 +4,8 @@ #include #include "envoy/config/core/v3/base.pb.h" +#include "envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.pb.h" +#include "envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.pb.validate.h" #include "source/common/buffer/buffer_impl.h" #include "source/common/common/assert.h" @@ -18,56 +20,132 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { -ZooKeeperFilterConfig::ZooKeeperFilterConfig(const std::string& stat_prefix, - const uint32_t max_packet_bytes, Stats::Scope& scope) +ZooKeeperFilterConfig::ZooKeeperFilterConfig( + const std::string& stat_prefix, const uint32_t max_packet_bytes, + const bool enable_latency_threshold_metrics, + const std::chrono::milliseconds default_latency_threshold, + const LatencyThresholdOverrideList& latency_threshold_overrides, Stats::Scope& scope) : scope_(scope), max_packet_bytes_(max_packet_bytes), stats_(generateStats(stat_prefix, scope)), stat_name_set_(scope.symbolTable().makeSet("Zookeeper")), stat_prefix_(stat_name_set_->add(stat_prefix)), auth_(stat_name_set_->add("auth")), connect_latency_(stat_name_set_->add("connect_response_latency")), unknown_scheme_rq_(stat_name_set_->add("unknown_scheme_rq")), - unknown_opcode_latency_(stat_name_set_->add("unknown_opcode_latency")) { + unknown_opcode_latency_(stat_name_set_->add("unknown_opcode_latency")), + enable_latency_threshold_metrics_(enable_latency_threshold_metrics), + default_latency_threshold_(default_latency_threshold), + latency_threshold_override_map_(parseLatencyThresholdOverrides(latency_threshold_overrides)) { // https://zookeeper.apache.org/doc/r3.5.4-beta/zookeeperProgrammers.html#sc_BuiltinACLSchemes // lists commons schemes: "world", "auth", "digest", "host", "x509", and // "ip". These are used in filter.cc by appending "_rq". stat_name_set_->rememberBuiltins( {"auth_rq", "digest_rq", "host_rq", "ip_rq", "ping_response_rq", "world_rq", "x509_rq"}); - initOpCode(OpCodes::Ping, stats_.ping_resp_, "ping_response"); - initOpCode(OpCodes::SetAuth, stats_.auth_resp_, "auth_response"); - initOpCode(OpCodes::GetData, stats_.getdata_resp_, "getdata_resp"); - initOpCode(OpCodes::Create, stats_.create_resp_, "create_resp"); - initOpCode(OpCodes::Create2, stats_.create2_resp_, "create2_resp"); - initOpCode(OpCodes::CreateContainer, stats_.createcontainer_resp_, "createcontainer_resp"); - initOpCode(OpCodes::CreateTtl, stats_.createttl_resp_, "createttl_resp"); - initOpCode(OpCodes::SetData, stats_.setdata_resp_, "setdata_resp"); - initOpCode(OpCodes::GetChildren, stats_.getchildren_resp_, "getchildren_resp"); - initOpCode(OpCodes::GetChildren2, stats_.getchildren2_resp_, "getchildren2_resp"); - initOpCode(OpCodes::Delete, stats_.delete_resp_, "delete_resp"); - initOpCode(OpCodes::Exists, stats_.exists_resp_, "exists_resp"); - initOpCode(OpCodes::GetAcl, stats_.getacl_resp_, "getacl_resp"); - initOpCode(OpCodes::SetAcl, stats_.setacl_resp_, "setacl_resp"); - initOpCode(OpCodes::Sync, stats_.sync_resp_, "sync_resp"); - initOpCode(OpCodes::Check, stats_.check_resp_, "check_resp"); - initOpCode(OpCodes::Multi, stats_.multi_resp_, "multi_resp"); - initOpCode(OpCodes::Reconfig, stats_.reconfig_resp_, "reconfig_resp"); - initOpCode(OpCodes::SetWatches, stats_.setwatches_resp_, "setwatches_resp"); - initOpCode(OpCodes::SetWatches2, stats_.setwatches2_resp_, "setwatches2_resp"); - initOpCode(OpCodes::CheckWatches, stats_.checkwatches_resp_, "checkwatches_resp"); - initOpCode(OpCodes::RemoveWatches, stats_.removewatches_resp_, "removewatches_resp"); - initOpCode(OpCodes::GetEphemerals, stats_.getephemerals_resp_, "getephemerals_resp"); + initOpCode(OpCodes::Ping, stats_.ping_resp_, stats_.ping_resp_fast_, stats_.ping_resp_slow_, + "ping_response"); + initOpCode(OpCodes::SetAuth, stats_.auth_resp_, stats_.auth_resp_fast_, stats_.auth_resp_slow_, + "auth_response"); + initOpCode(OpCodes::GetData, stats_.getdata_resp_, stats_.getdata_resp_fast_, + stats_.getdata_resp_slow_, "getdata_resp"); + initOpCode(OpCodes::Create, stats_.create_resp_, stats_.create_resp_fast_, + stats_.create_resp_slow_, "create_resp"); + initOpCode(OpCodes::Create2, stats_.create2_resp_, stats_.create2_resp_fast_, + stats_.create2_resp_slow_, "create2_resp"); + initOpCode(OpCodes::CreateContainer, stats_.createcontainer_resp_, + stats_.createcontainer_resp_fast_, stats_.createcontainer_resp_slow_, + "createcontainer_resp"); + initOpCode(OpCodes::CreateTtl, stats_.createttl_resp_, stats_.createttl_resp_fast_, + stats_.createttl_resp_slow_, "createttl_resp"); + initOpCode(OpCodes::SetData, stats_.setdata_resp_, stats_.setdata_resp_fast_, + stats_.setdata_resp_slow_, "setdata_resp"); + initOpCode(OpCodes::GetChildren, stats_.getchildren_resp_, stats_.getchildren_resp_fast_, + stats_.getchildren_resp_slow_, "getchildren_resp"); + initOpCode(OpCodes::GetChildren2, stats_.getchildren2_resp_, stats_.getchildren2_resp_fast_, + stats_.getchildren2_resp_slow_, "getchildren2_resp"); + initOpCode(OpCodes::Delete, stats_.delete_resp_, stats_.delete_resp_fast_, + stats_.delete_resp_slow_, "delete_resp"); + initOpCode(OpCodes::Exists, stats_.exists_resp_, stats_.exists_resp_fast_, + stats_.exists_resp_slow_, "exists_resp"); + initOpCode(OpCodes::GetAcl, stats_.getacl_resp_, stats_.getacl_resp_fast_, + stats_.getacl_resp_slow_, "getacl_resp"); + initOpCode(OpCodes::SetAcl, stats_.setacl_resp_, stats_.setacl_resp_fast_, + stats_.setacl_resp_slow_, "setacl_resp"); + initOpCode(OpCodes::Sync, stats_.sync_resp_, stats_.sync_resp_fast_, stats_.sync_resp_slow_, + "sync_resp"); + initOpCode(OpCodes::Check, stats_.check_resp_, stats_.check_resp_fast_, stats_.check_resp_slow_, + "check_resp"); + initOpCode(OpCodes::Multi, stats_.multi_resp_, stats_.multi_resp_fast_, stats_.multi_resp_slow_, + "multi_resp"); + initOpCode(OpCodes::Reconfig, stats_.reconfig_resp_, stats_.reconfig_resp_fast_, + stats_.reconfig_resp_slow_, "reconfig_resp"); + initOpCode(OpCodes::SetWatches, stats_.setwatches_resp_, stats_.setwatches_resp_fast_, + stats_.setwatches_resp_slow_, "setwatches_resp"); + initOpCode(OpCodes::SetWatches2, stats_.setwatches2_resp_, stats_.setwatches2_resp_fast_, + stats_.setwatches2_resp_slow_, "setwatches2_resp"); + initOpCode(OpCodes::CheckWatches, stats_.checkwatches_resp_, stats_.checkwatches_resp_fast_, + stats_.checkwatches_resp_slow_, "checkwatches_resp"); + initOpCode(OpCodes::RemoveWatches, stats_.removewatches_resp_, stats_.removewatches_resp_fast_, + stats_.removewatches_resp_slow_, "removewatches_resp"); + initOpCode(OpCodes::GetEphemerals, stats_.getephemerals_resp_, stats_.getephemerals_resp_fast_, + stats_.getephemerals_resp_slow_, "getephemerals_resp"); initOpCode(OpCodes::GetAllChildrenNumber, stats_.getallchildrennumber_resp_, + stats_.getallchildrennumber_resp_fast_, stats_.getallchildrennumber_resp_slow_, "getallchildrennumber_resp"); - initOpCode(OpCodes::Close, stats_.close_resp_, "close_resp"); + initOpCode(OpCodes::Close, stats_.close_resp_, stats_.close_resp_fast_, stats_.close_resp_slow_, + "close_resp"); } -void ZooKeeperFilterConfig::initOpCode(OpCodes opcode, Stats::Counter& counter, - absl::string_view name) { +ErrorBudgetResponseType +ZooKeeperFilterConfig::errorBudgetDecision(const OpCodes opcode, + const std::chrono::milliseconds latency) const { + if (!enable_latency_threshold_metrics_) { + return ErrorBudgetResponseType::None; + } + // Set latency threshold for the current opcode. + std::chrono::milliseconds latency_threshold = default_latency_threshold_; + int32_t opcode_val = enumToSignedInt(opcode); + auto it = latency_threshold_override_map_.find(opcode_val); + if (it != latency_threshold_override_map_.end()) { + latency_threshold = it->second; + } + + // Determine fast/slow response based on the threshold. + if (latency <= latency_threshold) { + return ErrorBudgetResponseType::Fast; + } + + return ErrorBudgetResponseType::Slow; +} + +void ZooKeeperFilterConfig::initOpCode(OpCodes opcode, Stats::Counter& resp_counter, + Stats::Counter& resp_fast_counter, + Stats::Counter& resp_slow_counter, absl::string_view name) { OpCodeInfo& opcode_info = op_code_map_[opcode]; - opcode_info.counter_ = &counter; + opcode_info.resp_counter_ = &resp_counter; + opcode_info.resp_fast_counter_ = &resp_fast_counter; + opcode_info.resp_slow_counter_ = &resp_slow_counter; opcode_info.opname_ = std::string(name); opcode_info.latency_name_ = stat_name_set_->add(absl::StrCat(name, "_latency")); } +int32_t ZooKeeperFilterConfig::getOpCodeIndex(LatencyThresholdOverride_Opcode opcode) { + const OpcodeMap& opcode_map = opcodeMap(); + auto it = opcode_map.find(opcode); + if (it != opcode_map.end()) { + return it->second; + } + throw EnvoyException(fmt::format("Unknown opcode from config: {}", static_cast(opcode))); +} + +LatencyThresholdOverrideMap ZooKeeperFilterConfig::parseLatencyThresholdOverrides( + const LatencyThresholdOverrideList& latency_threshold_overrides) { + LatencyThresholdOverrideMap latency_threshold_override_map; + for (const auto& threshold_override : latency_threshold_overrides) { + latency_threshold_override_map[getOpCodeIndex(threshold_override.opcode())] = + std::chrono::milliseconds(PROTOBUF_GET_MS_REQUIRED(threshold_override, threshold)); + } + return latency_threshold_override_map; +} + ZooKeeperFilter::ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config, TimeSource& time_source) : config_(std::move(config)), decoder_(createDecoder(*this, time_source)) {} @@ -293,6 +371,17 @@ void ZooKeeperFilter::onConnectResponse(const int32_t proto_version, const int32 const std::chrono::milliseconds& latency) { config_->stats_.connect_resp_.inc(); + switch (config_->errorBudgetDecision(OpCodes::Connect, latency)) { + case ErrorBudgetResponseType::Fast: + config_->stats_.connect_resp_fast_.inc(); + break; + case ErrorBudgetResponseType::Slow: + config_->stats_.connect_resp_slow_.inc(); + break; + case ErrorBudgetResponseType::None: + break; + } + Stats::Histogram& histogram = Stats::Utility::histogramFromElements( config_->scope_, {config_->stat_prefix_, config_->connect_latency_}, Stats::Histogram::Unit::Milliseconds); @@ -307,13 +396,24 @@ void ZooKeeperFilter::onConnectResponse(const int32_t proto_version, const int32 void ZooKeeperFilter::onResponse(const OpCodes opcode, const int32_t xid, const int64_t zxid, const int32_t error, const std::chrono::milliseconds& latency) { Stats::StatName opcode_latency = config_->unknown_opcode_latency_; - auto iter = config_->op_code_map_.find(opcode); std::string opname = ""; + auto iter = config_->op_code_map_.find(opcode); if (iter != config_->op_code_map_.end()) { const ZooKeeperFilterConfig::OpCodeInfo& opcode_info = iter->second; - opcode_info.counter_->inc(); - opname = opcode_info.opname_; + opcode_info.resp_counter_->inc(); opcode_latency = opcode_info.latency_name_; + opname = opcode_info.opname_; + + switch (config_->errorBudgetDecision(opcode, latency)) { + case ErrorBudgetResponseType::Fast: + opcode_info.resp_fast_counter_->inc(); + break; + case ErrorBudgetResponseType::Slow: + opcode_info.resp_slow_counter_->inc(); + break; + case ErrorBudgetResponseType::None: + break; + } } Stats::Histogram& histogram = Stats::Utility::histogramFromStatNames( diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.h b/source/extensions/filters/network/zookeeper_proxy/filter.h index 578a32606d7f..85dbb04cb322 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/filter.h @@ -5,6 +5,8 @@ #include #include "envoy/access_log/access_log.h" +#include "envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.pb.h" +#include "envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.pb.validate.h" #include "envoy/network/connection.h" #include "envoy/network/filter.h" #include "envoy/stats/scope.h" @@ -81,7 +83,61 @@ namespace ZooKeeperProxy { COUNTER(checkwatches_resp) \ COUNTER(removewatches_resp) \ COUNTER(check_resp) \ - COUNTER(watch_event) + COUNTER(watch_event) \ + COUNTER(connect_resp_fast) \ + COUNTER(ping_resp_fast) \ + COUNTER(auth_resp_fast) \ + COUNTER(getdata_resp_fast) \ + COUNTER(create_resp_fast) \ + COUNTER(create2_resp_fast) \ + COUNTER(createcontainer_resp_fast) \ + COUNTER(createttl_resp_fast) \ + COUNTER(setdata_resp_fast) \ + COUNTER(getchildren_resp_fast) \ + COUNTER(getchildren2_resp_fast) \ + COUNTER(getephemerals_resp_fast) \ + COUNTER(getallchildrennumber_resp_fast) \ + COUNTER(delete_resp_fast) \ + COUNTER(exists_resp_fast) \ + COUNTER(getacl_resp_fast) \ + COUNTER(setacl_resp_fast) \ + COUNTER(sync_resp_fast) \ + COUNTER(multi_resp_fast) \ + COUNTER(reconfig_resp_fast) \ + COUNTER(close_resp_fast) \ + COUNTER(setauth_resp_fast) \ + COUNTER(setwatches_resp_fast) \ + COUNTER(setwatches2_resp_fast) \ + COUNTER(checkwatches_resp_fast) \ + COUNTER(removewatches_resp_fast) \ + COUNTER(check_resp_fast) \ + COUNTER(connect_resp_slow) \ + COUNTER(ping_resp_slow) \ + COUNTER(auth_resp_slow) \ + COUNTER(getdata_resp_slow) \ + COUNTER(create_resp_slow) \ + COUNTER(create2_resp_slow) \ + COUNTER(createcontainer_resp_slow) \ + COUNTER(createttl_resp_slow) \ + COUNTER(setdata_resp_slow) \ + COUNTER(getchildren_resp_slow) \ + COUNTER(getchildren2_resp_slow) \ + COUNTER(getephemerals_resp_slow) \ + COUNTER(getallchildrennumber_resp_slow) \ + COUNTER(delete_resp_slow) \ + COUNTER(exists_resp_slow) \ + COUNTER(getacl_resp_slow) \ + COUNTER(setacl_resp_slow) \ + COUNTER(sync_resp_slow) \ + COUNTER(multi_resp_slow) \ + COUNTER(reconfig_resp_slow) \ + COUNTER(close_resp_slow) \ + COUNTER(setauth_resp_slow) \ + COUNTER(setwatches_resp_slow) \ + COUNTER(setwatches2_resp_slow) \ + COUNTER(checkwatches_resp_slow) \ + COUNTER(removewatches_resp_slow) \ + COUNTER(check_resp_slow) /** * Struct definition for all ZooKeeper proxy stats. @see stats_macros.h @@ -90,23 +146,40 @@ struct ZooKeeperProxyStats { ALL_ZOOKEEPER_PROXY_STATS(GENERATE_COUNTER_STRUCT) }; +enum class ErrorBudgetResponseType { Fast, Slow, None }; + +using envoy::extensions::filters::network::zookeeper_proxy::v3::LatencyThresholdOverride; +using envoy::extensions::filters::network::zookeeper_proxy::v3::LatencyThresholdOverride_Opcode; +using LatencyThresholdOverrideList = Protobuf::RepeatedPtrField; +using LatencyThresholdOverrideMap = absl::flat_hash_map; +using OpcodeMap = absl::flat_hash_map; + /** * Configuration for the ZooKeeper proxy filter. */ class ZooKeeperFilterConfig { public: - ZooKeeperFilterConfig(const std::string& stat_prefix, uint32_t max_packet_bytes, + ZooKeeperFilterConfig(const std::string& stat_prefix, const uint32_t max_packet_bytes, + const bool enable_latency_threshold_metrics, + const std::chrono::milliseconds default_latency_threshold, + const LatencyThresholdOverrideList& latency_threshold_overrides, Stats::Scope& scope); const ZooKeeperProxyStats& stats() { return stats_; } uint32_t maxPacketBytes() const { return max_packet_bytes_; } - // Captures the counter used to track total op-code usage, as well as the - // StatName under which to collect the latency for that op-code. The - // latency-name will be joined with the stat_prefix_, which varies per filter + // The OpCodeInfo is created as a public member of ZooKeeperFilterConfig. + // Therefore, its lifetime is tied to that of ZooKeeperFilterConfig. + // When the ZooKeeperFilterConfig object is destroyed, the OpCodeInfo will be destroyed as well. + // The the lifetime of scope is tied to the context passed to network filters to access server + // resources. The values of counter elements in OpCodeInfo are used to track total op-code usage, + // as well as the StatName under which to collect the latency, fast/slow responses for that + // op-code. The latency-name will be joined with the stat_prefix_, which varies per filter // instance. struct OpCodeInfo { - Stats::Counter* counter_; + Stats::Counter* resp_counter_; + Stats::Counter* resp_fast_counter_; + Stats::Counter* resp_slow_counter_; std::string opname_; Stats::StatName latency_name_; }; @@ -122,12 +195,54 @@ class ZooKeeperFilterConfig { const Stats::StatName unknown_scheme_rq_; const Stats::StatName unknown_opcode_latency_; + ErrorBudgetResponseType errorBudgetDecision(const OpCodes opcode, + const std::chrono::milliseconds latency) const; + private: - void initOpCode(OpCodes opcode, Stats::Counter& counter, absl::string_view name); + void initOpCode(OpCodes opcode, Stats::Counter& resp_counter, Stats::Counter& resp_fast_counter, + Stats::Counter& resp_slow_counter, absl::string_view name); ZooKeeperProxyStats generateStats(const std::string& prefix, Stats::Scope& scope) { return ZooKeeperProxyStats{ALL_ZOOKEEPER_PROXY_STATS(POOL_COUNTER_PREFIX(scope, prefix))}; } + + static const OpcodeMap& opcodeMap() { + CONSTRUCT_ON_FIRST_USE(OpcodeMap, {{LatencyThresholdOverride::Connect, 0}, + {LatencyThresholdOverride::Create, 1}, + {LatencyThresholdOverride::Delete, 2}, + {LatencyThresholdOverride::Exists, 3}, + {LatencyThresholdOverride::GetData, 4}, + {LatencyThresholdOverride::SetData, 5}, + {LatencyThresholdOverride::GetAcl, 6}, + {LatencyThresholdOverride::SetAcl, 7}, + {LatencyThresholdOverride::GetChildren, 8}, + {LatencyThresholdOverride::Sync, 9}, + {LatencyThresholdOverride::Ping, 11}, + {LatencyThresholdOverride::GetChildren2, 12}, + {LatencyThresholdOverride::Check, 13}, + {LatencyThresholdOverride::Multi, 14}, + {LatencyThresholdOverride::Create2, 15}, + {LatencyThresholdOverride::Reconfig, 16}, + {LatencyThresholdOverride::CheckWatches, 17}, + {LatencyThresholdOverride::RemoveWatches, 18}, + {LatencyThresholdOverride::CreateContainer, 19}, + {LatencyThresholdOverride::CreateTtl, 21}, + {LatencyThresholdOverride::Close, -11}, + {LatencyThresholdOverride::SetAuth, 100}, + {LatencyThresholdOverride::SetWatches, 101}, + {LatencyThresholdOverride::GetEphemerals, 103}, + {LatencyThresholdOverride::GetAllChildrenNumber, 104}, + {LatencyThresholdOverride::SetWatches2, 105}}); + } + + int32_t getOpCodeIndex(LatencyThresholdOverride_Opcode opcode); + LatencyThresholdOverrideMap + parseLatencyThresholdOverrides(const LatencyThresholdOverrideList& latency_threshold_overrides); + + const bool enable_latency_threshold_metrics_; + const std::chrono::milliseconds default_latency_threshold_; + // Key: opcode enum value defined in decoder.h, value: latency threshold override in millisecond. + const LatencyThresholdOverrideMap latency_threshold_override_map_; }; using ZooKeeperFilterConfigSharedPtr = std::shared_ptr; diff --git a/test/extensions/filters/network/zookeeper_proxy/config_test.cc b/test/extensions/filters/network/zookeeper_proxy/config_test.cc index a465c5c47eff..be99a561eca1 100644 --- a/test/extensions/filters/network/zookeeper_proxy/config_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/config_test.cc @@ -2,6 +2,7 @@ #include "envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.pb.validate.h" #include "source/extensions/filters/network/zookeeper_proxy/config.h" +#include "source/extensions/filters/network/zookeeper_proxy/filter.h" #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" @@ -17,47 +18,257 @@ namespace ZooKeeperProxy { using ZooKeeperProxyProtoConfig = envoy::extensions::filters::network::zookeeper_proxy::v3::ZooKeeperProxy; -TEST(ZookeeperFilterConfigTest, ValidateFail) { - testing::NiceMock context; - EXPECT_THROW( - ZooKeeperConfigFactory().createFilterFactoryFromProto(ZooKeeperProxyProtoConfig(), context), - ProtoValidationException); +class ZookeeperFilterConfigTest : public testing::Test { +public: + std::string populateFullConfig(const ProtobufWkt::EnumDescriptor* opcode_descriptor) { + std::string yaml = R"EOF( +stat_prefix: test_prefix +max_packet_bytes: 1048576 +enable_latency_threshold_metrics: true +default_latency_threshold: "0.1s" +latency_threshold_overrides:)EOF"; + + for (int i = 0; i < opcode_descriptor->value_count(); i++) { + const auto* opcode_tuple = opcode_descriptor->value(i); + std::string opcode = opcode_tuple->name(); + int threshold_delta = opcode_tuple->number(); + std::string threshold = fmt::format("0.{}s", 150 + threshold_delta); + yaml += fmt::format(R"EOF( + - opcode: {} + threshold: {})EOF", + opcode, threshold); + }; + + return yaml; + } + + ZooKeeperProxyProtoConfig proto_config_; + testing::NiceMock context_; + ZooKeeperConfigFactory factory_; + Network::MockConnection connection_; +}; + +TEST_F(ZookeeperFilterConfigTest, ValidateFail) { + EXPECT_THROW_WITH_REGEX( + ZooKeeperConfigFactory().createFilterFactoryFromProto(ZooKeeperProxyProtoConfig(), context_), + ProtoValidationException, + "Proto constraint validation failed \\(ZooKeeperProxyValidationError.StatPrefix: value " + "length must be at least 1 characters\\)"); } -TEST(ZookeeperFilterConfigTest, InvalidStatPrefix) { +TEST_F(ZookeeperFilterConfigTest, InvalidStatPrefix) { const std::string yaml = R"EOF( stat_prefix: "" )EOF"; - ZooKeeperProxyProtoConfig proto_config; - EXPECT_THROW(TestUtility::loadFromYamlAndValidate(yaml, proto_config), ProtoValidationException); + EXPECT_THROW_WITH_REGEX( + TestUtility::loadFromYamlAndValidate(yaml, proto_config_), ProtoValidationException, + "Proto constraint validation failed \\(ZooKeeperProxyValidationError.StatPrefix: value " + "length must be at least 1 characters\\)"); } -TEST(ZookeeperFilterConfigTest, InvalidMaxPacketBytes) { +TEST_F(ZookeeperFilterConfigTest, InvalidMaxPacketBytes) { const std::string yaml = R"EOF( stat_prefix: test_prefix max_packet_bytes: -1 )EOF"; - ZooKeeperProxyProtoConfig proto_config; - EXPECT_THROW(TestUtility::loadFromYamlAndValidate(yaml, proto_config), EnvoyException); + EXPECT_THROW_WITH_REGEX( + TestUtility::loadFromYamlAndValidate(yaml, proto_config_), EnvoyException, + "Unable to parse JSON as proto \\(INVALID_ARGUMENT:\\(max_packet_bytes.value\\): invalid " + "value -1 for type TYPE_UINT32\\)"); +} + +TEST_F(ZookeeperFilterConfigTest, UndefinedOpcode) { + const std::string yaml = R"EOF( +stat_prefix: test_prefix +latency_threshold_overrides: + - opcode: Undefined + threshold: + nanos: 150000000 + )EOF"; + + EXPECT_THROW_WITH_MESSAGE( + TestUtility::loadFromYamlAndValidate(yaml, proto_config_), + ProtobufMessage::UnknownProtoFieldException, + "Protobuf message (type envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy " + "reason INVALID_ARGUMENT:(latency_threshold_overrides[0].opcode): invalid value " + "\"Undefined\" for type " + "type.googleapis.com/" + "envoy.extensions.filters.network.zookeeper_proxy.v3.LatencyThresholdOverride.Opcode) has " + "unknown fields"); } -TEST(ZookeeperFilterConfigTest, SimpleConfig) { +TEST_F(ZookeeperFilterConfigTest, NegativeLatencyThreshold) { const std::string yaml = R"EOF( stat_prefix: test_prefix +latency_threshold_overrides: + - opcode: Connect + threshold: + nanos: -150000000 )EOF"; - ZooKeeperProxyProtoConfig proto_config; - TestUtility::loadFromYamlAndValidate(yaml, proto_config); + EXPECT_THROW_WITH_REGEX( + TestUtility::loadFromYamlAndValidate(yaml, proto_config_), EnvoyException, + "Proto constraint validation failed " + "\\(ZooKeeperProxyValidationError.LatencyThresholdOverrides\\[0\\]: embedded message failed " + "validation \\| caused by LatencyThresholdOverrideValidationError.Threshold: value must be " + "greater than or equal to 1ms\\)"); +} + +TEST_F(ZookeeperFilterConfigTest, TooSmallLatencyThreshold) { + const std::string yaml = R"EOF( +stat_prefix: test_prefix +latency_threshold_overrides: + - opcode: Multi + threshold: + nanos: 999999 + )EOF"; + + EXPECT_THROW_WITH_REGEX( + TestUtility::loadFromYamlAndValidate(yaml, proto_config_), EnvoyException, + "Proto constraint validation failed " + "\\(ZooKeeperProxyValidationError.LatencyThresholdOverrides\\[0\\]: embedded message failed " + "validation \\| caused by LatencyThresholdOverrideValidationError.Threshold: value must be " + "greater than or equal to 1ms\\)"); +} + +TEST_F(ZookeeperFilterConfigTest, UnsetLatencyThreshold) { + const std::string yaml = R"EOF( +stat_prefix: test_prefix +latency_threshold_overrides: + - opcode: Create + threshold: + )EOF"; + + EXPECT_THROW_WITH_REGEX( + TestUtility::loadFromYamlAndValidate(yaml, proto_config_), EnvoyException, + "Proto constraint validation failed " + "\\(ZooKeeperProxyValidationError.LatencyThresholdOverrides\\[0\\]: embedded message failed " + "validation \\| caused by LatencyThresholdOverrideValidationError.Threshold: value is " + "required\\)"); +} + +TEST_F(ZookeeperFilterConfigTest, DuplicatedOpcodes) { + const std::string yaml = R"EOF( +stat_prefix: test_prefix +max_packet_bytes: 1048576 +latency_threshold_overrides: + - opcode: Sync + threshold: + nanos: 150000000 + - opcode: Sync + threshold: + nanos: 151000000 + - opcode: Create + threshold: + nanos: 152000000 + - opcode: Create + threshold: + nanos: 153000000 + - opcode: Create + threshold: + nanos: 154000000 + - opcode: Create + threshold: + nanos: 155000000 + )EOF"; + + TestUtility::loadFromYamlAndValidate(yaml, proto_config_); + EXPECT_THROW_WITH_REGEX( + ZooKeeperConfigFactory().createFilterFactoryFromProto(proto_config_, context_), + EnvoyException, "Duplicated opcode find in config"); +} + +TEST_F(ZookeeperFilterConfigTest, SimpleConfig) { + const std::string yaml = R"EOF( +stat_prefix: test_prefix + )EOF"; + + TestUtility::loadFromYamlAndValidate(yaml, proto_config_); + EXPECT_EQ(proto_config_.stat_prefix(), "test_prefix"); + EXPECT_EQ(proto_config_.max_packet_bytes().value(), 0); + EXPECT_EQ(proto_config_.enable_latency_threshold_metrics(), false); + EXPECT_EQ(proto_config_.default_latency_threshold(), + ProtobufWkt::util::TimeUtil::SecondsToDuration(0)); + EXPECT_EQ(proto_config_.latency_threshold_overrides_size(), 0); + + Network::FilterFactoryCb cb = factory_.createFilterFactoryFromProto(proto_config_, context_); + EXPECT_CALL(connection_, addFilter(_)); + cb(connection_); +} + +TEST_F(ZookeeperFilterConfigTest, ConfigWithDefaultLatencyThreshold) { + const std::string yaml = R"EOF( +stat_prefix: test_prefix +default_latency_threshold: "0.15s" + )EOF"; + + TestUtility::loadFromYamlAndValidate(yaml, proto_config_); + EXPECT_EQ(proto_config_.stat_prefix(), "test_prefix"); + EXPECT_EQ(proto_config_.max_packet_bytes().value(), 0); + EXPECT_EQ(proto_config_.enable_latency_threshold_metrics(), false); + EXPECT_EQ(proto_config_.default_latency_threshold(), + ProtobufWkt::util::TimeUtil::MillisecondsToDuration(150)); + EXPECT_EQ(proto_config_.latency_threshold_overrides_size(), 0); + + Network::FilterFactoryCb cb = factory_.createFilterFactoryFromProto(proto_config_, context_); + EXPECT_CALL(connection_, addFilter(_)); + cb(connection_); +} + +TEST_F(ZookeeperFilterConfigTest, ConfigWithConnectLatencyThreshold) { + const std::string yaml = R"EOF( +stat_prefix: test_prefix +latency_threshold_overrides: + - opcode: Connect + threshold: "0.151s" + )EOF"; + + TestUtility::loadFromYamlAndValidate(yaml, proto_config_); + EXPECT_EQ(proto_config_.stat_prefix(), "test_prefix"); + EXPECT_EQ(proto_config_.max_packet_bytes().value(), 0); + EXPECT_EQ(proto_config_.enable_latency_threshold_metrics(), false); + EXPECT_EQ(proto_config_.default_latency_threshold(), + ProtobufWkt::util::TimeUtil::SecondsToDuration(0)); + EXPECT_EQ(proto_config_.latency_threshold_overrides_size(), 1); + LatencyThresholdOverride threshold_override = proto_config_.latency_threshold_overrides().at(0); + EXPECT_EQ(threshold_override.opcode(), LatencyThresholdOverride::Connect); + EXPECT_EQ(threshold_override.threshold(), + ProtobufWkt::util::TimeUtil::MillisecondsToDuration(151)); + + Network::FilterFactoryCb cb = factory_.createFilterFactoryFromProto(proto_config_, context_); + EXPECT_CALL(connection_, addFilter(_)); + cb(connection_); +} + +TEST_F(ZookeeperFilterConfigTest, FullConfig) { + const ProtobufWkt::EnumDescriptor* opcode_descriptor = envoy::extensions::filters::network:: + zookeeper_proxy::v3::LatencyThresholdOverride_Opcode_descriptor(); + std::string yaml = populateFullConfig(opcode_descriptor); + TestUtility::loadFromYamlAndValidate(yaml, proto_config_); + + EXPECT_EQ(proto_config_.stat_prefix(), "test_prefix"); + EXPECT_EQ(proto_config_.max_packet_bytes().value(), 1048576); + EXPECT_EQ(proto_config_.enable_latency_threshold_metrics(), true); + EXPECT_EQ(proto_config_.default_latency_threshold(), + ProtobufWkt::util::TimeUtil::MillisecondsToDuration(100)); + EXPECT_EQ(proto_config_.latency_threshold_overrides_size(), 26); - testing::NiceMock context; - ZooKeeperConfigFactory factory; + for (int i = 0; i < opcode_descriptor->value_count(); i++) { + LatencyThresholdOverride threshold_override = proto_config_.latency_threshold_overrides().at(i); + const auto* opcode_tuple = opcode_descriptor->value(i); + std::string opcode_name = envoy::extensions::filters::network::zookeeper_proxy::v3:: + LatencyThresholdOverride_Opcode_Name(threshold_override.opcode()); + EXPECT_EQ(opcode_name, opcode_tuple->name()); + uint64_t threshold_delta = static_cast(opcode_tuple->number()); + EXPECT_EQ(threshold_override.threshold(), + ProtobufWkt::util::TimeUtil::MillisecondsToDuration(150 + threshold_delta)); + } - Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); - Network::MockConnection connection; - EXPECT_CALL(connection, addFilter(_)); - cb(connection); + Network::FilterFactoryCb cb = factory_.createFilterFactoryFromProto(proto_config_, context_); + EXPECT_CALL(connection_, addFilter(_)); + cb(connection_); } } // namespace ZooKeeperProxy diff --git a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc index 1bae518ba129..707e33ae887d 100644 --- a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc @@ -1,4 +1,5 @@ #include "source/common/buffer/buffer_impl.h" +#include "source/common/protobuf/protobuf.h" #include "source/extensions/filters/network/zookeeper_proxy/decoder.h" #include "source/extensions/filters/network/zookeeper_proxy/filter.h" @@ -27,10 +28,14 @@ MATCHER_P(MapEq, rhs, "") { return protoMapEq(arg, rhs); } class ZooKeeperFilterTest : public testing::Test { public: - ZooKeeperFilterTest() { ENVOY_LOG_MISC(info, "test"); } - - void initialize() { - config_ = std::make_shared(stat_prefix_, 1048576, scope_); + void initialize( + const bool enable_latency_threshold_metrics = true, + const std::chrono::milliseconds default_latency_threshold = std::chrono::milliseconds(100), + const LatencyThresholdOverrideList& latency_threshold_overrides = + LatencyThresholdOverrideList()) { + config_ = std::make_shared( + stat_prefix_, 1048576, enable_latency_threshold_metrics, default_latency_threshold, + latency_threshold_overrides, scope_); filter_ = std::make_unique(config_, time_system_); filter_->initializeReadFilterCallbacks(filter_callbacks_); } @@ -624,14 +629,17 @@ class ZooKeeperFilterTest : public testing::Test { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } - void testResponse(const std::vector& metadata_values, const Stats::Counter& stat, - const uint32_t xid = 1000, const uint64_t zxid = 2000, - const uint32_t response_count = 1) { + void testResponse(const std::vector& metadata_values, + const Stats::Counter& resp_counter, const Stats::Counter& resp_fast_counter, + const Stats::Counter& resp_slow_counter, const uint32_t xid = 1000, + const uint64_t zxid = 2000, const uint32_t response_count = 1) { Buffer::OwnedImpl data = encodeResponseHeader(xid, zxid, 0); expectSetDynamicMetadata(metadata_values); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); - EXPECT_EQ(1UL * response_count, stat.value()); + EXPECT_EQ(1UL * response_count, resp_counter.value()); + EXPECT_EQ(1UL * response_count, resp_fast_counter.value()); + EXPECT_EQ(0UL, resp_slow_counter.value()); EXPECT_EQ(20UL * response_count, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); const auto histogram_name = @@ -653,6 +661,116 @@ class ZooKeeperFilterTest : public testing::Test { Event::SimulatedTimeSystem time_system_; }; +TEST_F(ZooKeeperFilterTest, DisableErrorBudgetCalculation) { + // Disable the error budget metrics emitting. + std::chrono::milliseconds default_latency_threshold(200); + LatencyThresholdOverrideList latency_threshold_overrides; + + initialize(false, default_latency_threshold, latency_threshold_overrides); + + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Connect, std::chrono::milliseconds(50)), + ErrorBudgetResponseType::None); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::SetWatches2, std::chrono::milliseconds(100)), + ErrorBudgetResponseType::None); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Multi, std::chrono::milliseconds(200)), + ErrorBudgetResponseType::None); + + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Ping, std::chrono::milliseconds(201)), + ErrorBudgetResponseType::None); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::GetChildren2, std::chrono::milliseconds(202)), + ErrorBudgetResponseType::None); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Multi, std::chrono::milliseconds(203)), + ErrorBudgetResponseType::None); +} + +TEST_F(ZooKeeperFilterTest, ErrorBudgetDecisionWithDefaultLatencyThresholdConfig) { + // Set default latency threshold as 200 milliseconds. The latency thresholds of all other opcode + // fallback to the default threshold. + std::chrono::milliseconds default_latency_threshold(200); + LatencyThresholdOverrideList latency_threshold_overrides; + + initialize(true, default_latency_threshold, latency_threshold_overrides); + + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Connect, std::chrono::milliseconds(50)), + ErrorBudgetResponseType::Fast); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::SetWatches2, std::chrono::milliseconds(100)), + ErrorBudgetResponseType::Fast); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Multi, std::chrono::milliseconds(200)), + ErrorBudgetResponseType::Fast); + + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Ping, std::chrono::milliseconds(201)), + ErrorBudgetResponseType::Slow); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::GetChildren2, std::chrono::milliseconds(202)), + ErrorBudgetResponseType::Slow); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Multi, std::chrono::milliseconds(203)), + ErrorBudgetResponseType::Slow); +} + +TEST_F(ZooKeeperFilterTest, ErrorBudgetDecisionWithMultiLatencyThresholdConfig) { + // Set default latency threshold as 100 milliseconds. Set latency threshold override for Multi + // opcode as 200 milliseconds. The latency thresholds of all other opcodes fallback to the default + // threshold. + std::chrono::milliseconds default_latency_threshold(100); + LatencyThresholdOverrideList latency_threshold_overrides; + auto* threshold_override = latency_threshold_overrides.Add(); + threshold_override->set_opcode(LatencyThresholdOverride::Multi); + threshold_override->mutable_threshold()->set_nanos(200000000); // 200 milliseconds + + initialize(true, default_latency_threshold, latency_threshold_overrides); + + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Connect, std::chrono::milliseconds(50)), + ErrorBudgetResponseType::Fast); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::SetWatches2, std::chrono::milliseconds(100)), + ErrorBudgetResponseType::Fast); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Multi, std::chrono::milliseconds(200)), + ErrorBudgetResponseType::Fast); + + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Ping, std::chrono::milliseconds(101)), + ErrorBudgetResponseType::Slow); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::GetChildren2, std::chrono::milliseconds(102)), + ErrorBudgetResponseType::Slow); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Multi, std::chrono::milliseconds(201)), + ErrorBudgetResponseType::Slow); +} + +TEST_F(ZooKeeperFilterTest, ErrorBudgetDecisionWithDefaultAndOtherLatencyThresholdConfigs) { + // Set default latency threshold as 150 milliseconds. Set latency threshold for Ping opcode as 50 + // milliseconds. Set latency threshold for Create opcode as 200 milliseconds. The latency + // thresholds of all other opcodes fallback to the default threshold. + std::chrono::milliseconds default_latency_threshold(150); + LatencyThresholdOverrideList latency_threshold_overrides; + auto* threshold_override = latency_threshold_overrides.Add(); + threshold_override->set_opcode(LatencyThresholdOverride::Ping); + threshold_override->mutable_threshold()->set_nanos(50000000); // 50 milliseconds + threshold_override = latency_threshold_overrides.Add(); + threshold_override->set_opcode(LatencyThresholdOverride::Create); + threshold_override->mutable_threshold()->set_nanos(200000000); // 200 milliseconds + + initialize(true, default_latency_threshold, latency_threshold_overrides); + + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Connect, std::chrono::milliseconds(150)), + ErrorBudgetResponseType::Fast); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Ping, std::chrono::milliseconds(50)), + ErrorBudgetResponseType::Fast); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Create, std::chrono::milliseconds(200)), + ErrorBudgetResponseType::Fast); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::SetWatches2, std::chrono::milliseconds(100)), + ErrorBudgetResponseType::Fast); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Multi, std::chrono::milliseconds(150)), + ErrorBudgetResponseType::Fast); + + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Connect, std::chrono::milliseconds(151)), + ErrorBudgetResponseType::Slow); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Ping, std::chrono::milliseconds(51)), + ErrorBudgetResponseType::Slow); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Create, std::chrono::milliseconds(201)), + ErrorBudgetResponseType::Slow); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::GetChildren2, std::chrono::milliseconds(152)), + ErrorBudgetResponseType::Slow); + EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Multi, std::chrono::milliseconds(201)), + ErrorBudgetResponseType::Slow); +} + TEST_F(ZooKeeperFilterTest, Connect) { initialize(); @@ -668,6 +786,8 @@ TEST_F(ZooKeeperFilterTest, Connect) { {{"bytes", "24"}}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); EXPECT_EQ(1UL, config_->stats().connect_resp_.value()); + EXPECT_EQ(1UL, config_->stats().connect_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().connect_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.connect_response_latency")); @@ -689,6 +809,8 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { {{"bytes", "25"}}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); EXPECT_EQ(1UL, config_->stats().connect_resp_.value()); + EXPECT_EQ(1UL, config_->stats().connect_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().connect_resp_slow_.value()); EXPECT_EQ(25UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.connect_response_latency")); @@ -748,7 +870,8 @@ TEST_F(ZooKeeperFilterTest, PingRequest) { testRequest(data, {{{"opname", "ping"}}, {{"bytes", "12"}}}, config_->stats().ping_rq_, 12); testResponse({{{"opname", "ping_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().ping_resp_, enumToSignedInt(XidCodes::PingXid)); + config_->stats().ping_resp_, config_->stats().ping_resp_fast_, + config_->stats().ping_resp_slow_, enumToSignedInt(XidCodes::PingXid)); } TEST_F(ZooKeeperFilterTest, AuthRequest) { @@ -759,7 +882,8 @@ TEST_F(ZooKeeperFilterTest, AuthRequest) { testRequest(data, {{{"opname", "auth"}}, {{"bytes", "36"}}}, store_.counter("test.zookeeper.auth.digest_rq"), 36); testResponse({{{"opname", "auth_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().auth_resp_, enumToSignedInt(XidCodes::AuthXid)); + config_->stats().auth_resp_, config_->stats().auth_resp_fast_, + config_->stats().auth_resp_slow_, enumToSignedInt(XidCodes::AuthXid)); } TEST_F(ZooKeeperFilterTest, GetDataRequest) { @@ -771,7 +895,8 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { {{{"opname", "getdata"}, {"path", "/foo"}, {"watch", "true"}}, {{"bytes", "21"}}}, config_->stats().getdata_rq_, 21); testResponse({{{"opname", "getdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getdata_resp_); + config_->stats().getdata_resp_, config_->stats().getdata_resp_fast_, + config_->stats().getdata_resp_slow_); } TEST_F(ZooKeeperFilterTest, GetDataRequestEmptyPath) { @@ -784,54 +909,67 @@ TEST_F(ZooKeeperFilterTest, GetDataRequestEmptyPath) { testRequest(data, {{{"opname", "getdata"}, {"path", ""}, {"watch", "true"}}, {{"bytes", "17"}}}, config_->stats().getdata_rq_, 17); testResponse({{{"opname", "getdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getdata_resp_); + config_->stats().getdata_resp_, config_->stats().getdata_resp_fast_, + config_->stats().getdata_resp_slow_); } TEST_F(ZooKeeperFilterTest, CreateRequestPersistent) { testCreate(CreateFlags::Persistent); testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_); } TEST_F(ZooKeeperFilterTest, CreateRequestPersistentWithNegativeDataLen) { testCreateWithNegativeDataLen(CreateFlags::Persistent); testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_); } TEST_F(ZooKeeperFilterTest, CreateRequestPersistentSequential) { testCreate(CreateFlags::PersistentSequential); testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_); } -TEST_F(ZooKeeperFilterTest, CreateRequestEphemeral) { testCreate(CreateFlags::Ephemeral); } +TEST_F(ZooKeeperFilterTest, CreateRequestEphemeral) { + testCreate(CreateFlags::Ephemeral); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_); +} TEST_F(ZooKeeperFilterTest, CreateRequestEphemeralSequential) { testCreate(CreateFlags::EphemeralSequential); testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_); } TEST_F(ZooKeeperFilterTest, CreateRequestContainer) { testCreate(CreateFlags::Container, OpCodes::CreateContainer); testResponse( {{{"opname", "createcontainer_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().createcontainer_resp_); + config_->stats().createcontainer_resp_, config_->stats().createcontainer_resp_fast_, + config_->stats().createcontainer_resp_slow_); } TEST_F(ZooKeeperFilterTest, CreateRequestTTL) { testCreate(CreateFlags::PersistentWithTtl, OpCodes::CreateTtl); testResponse( {{{"opname", "createttl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().createttl_resp_); + config_->stats().createttl_resp_, config_->stats().createttl_resp_fast_, + config_->stats().createttl_resp_slow_); } TEST_F(ZooKeeperFilterTest, CreateRequestTTLSequential) { testCreate(CreateFlags::PersistentSequentialWithTtl, OpCodes::CreateTtl); testResponse( {{{"opname", "createttl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().createttl_resp_); + config_->stats().createttl_resp_, config_->stats().createttl_resp_fast_, + config_->stats().createttl_resp_slow_); } TEST_F(ZooKeeperFilterTest, CreateRequest2) { @@ -845,7 +983,8 @@ TEST_F(ZooKeeperFilterTest, CreateRequest2) { {{{"opname", "create2"}, {"path", "/foo"}, {"create_type", "persistent"}}, {{"bytes", "35"}}}, config_->stats().create2_rq_, 35); testResponse({{{"opname", "create2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create2_resp_); + config_->stats().create2_resp_, config_->stats().create2_resp_fast_, + config_->stats().create2_resp_slow_); } TEST_F(ZooKeeperFilterTest, SetRequest) { @@ -856,7 +995,8 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { testRequest(data, {{{"opname", "setdata"}, {"path", "/foo"}}, {{"bytes", "31"}}}, config_->stats().setdata_rq_, 31); testResponse({{{"opname", "setdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setdata_resp_); + config_->stats().setdata_resp_, config_->stats().setdata_resp_fast_, + config_->stats().setdata_resp_slow_); } TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { @@ -870,7 +1010,8 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { config_->stats().getchildren_rq_, 21); testResponse( {{{"opname", "getchildren_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getchildren_resp_); + config_->stats().getchildren_resp_, config_->stats().getchildren_resp_fast_, + config_->stats().getchildren_resp_slow_); } TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { @@ -884,7 +1025,8 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { config_->stats().getchildren2_rq_, 21); testResponse( {{{"opname", "getchildren2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getchildren2_resp_); + config_->stats().getchildren2_resp_, config_->stats().getchildren2_resp_fast_, + config_->stats().getchildren2_resp_slow_); } TEST_F(ZooKeeperFilterTest, DeleteRequest) { @@ -896,7 +1038,8 @@ TEST_F(ZooKeeperFilterTest, DeleteRequest) { {{{"opname", "delete"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "24"}}}, config_->stats().delete_rq_, 24); testResponse({{{"opname", "delete_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().delete_resp_); + config_->stats().delete_resp_, config_->stats().delete_resp_fast_, + config_->stats().delete_resp_slow_); } TEST_F(ZooKeeperFilterTest, ExistsRequest) { @@ -908,7 +1051,8 @@ TEST_F(ZooKeeperFilterTest, ExistsRequest) { {{{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}}, config_->stats().exists_rq_, 21); testResponse({{{"opname", "exists_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().exists_resp_); + config_->stats().exists_resp_, config_->stats().exists_resp_fast_, + config_->stats().exists_resp_slow_); } TEST_F(ZooKeeperFilterTest, GetAclRequest) { @@ -919,7 +1063,8 @@ TEST_F(ZooKeeperFilterTest, GetAclRequest) { testRequest(data, {{{"opname", "getacl"}, {"path", "/foo"}}, {{"bytes", "20"}}}, config_->stats().getacl_rq_, 20); testResponse({{{"opname", "getacl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getacl_resp_); + config_->stats().getacl_resp_, config_->stats().getacl_resp_fast_, + config_->stats().getacl_resp_slow_); } TEST_F(ZooKeeperFilterTest, SetAclRequest) { @@ -931,7 +1076,8 @@ TEST_F(ZooKeeperFilterTest, SetAclRequest) { {{{"opname", "setacl"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "52"}}}, config_->stats().setacl_rq_, 52); testResponse({{{"opname", "setacl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setacl_resp_); + config_->stats().setacl_resp_, config_->stats().setacl_resp_fast_, + config_->stats().setacl_resp_slow_); } TEST_F(ZooKeeperFilterTest, SyncRequest) { @@ -942,7 +1088,8 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { testRequest(data, {{{"opname", "sync"}, {"path", "/foo"}}, {{"bytes", "20"}}}, config_->stats().sync_rq_, 20); testResponse({{{"opname", "sync_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().sync_resp_); + config_->stats().sync_resp_, config_->stats().sync_resp_fast_, + config_->stats().sync_resp_slow_); } TEST_F(ZooKeeperFilterTest, GetEphemeralsRequest) { @@ -954,7 +1101,8 @@ TEST_F(ZooKeeperFilterTest, GetEphemeralsRequest) { config_->stats().getephemerals_rq_, 20); testResponse( {{{"opname", "getephemerals_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getephemerals_resp_); + config_->stats().getephemerals_resp_, config_->stats().getephemerals_resp_fast_, + config_->stats().getephemerals_resp_slow_); } TEST_F(ZooKeeperFilterTest, GetAllChildrenNumberRequest) { @@ -966,7 +1114,9 @@ TEST_F(ZooKeeperFilterTest, GetAllChildrenNumberRequest) { config_->stats().getallchildrennumber_rq_, 20); testResponse({{{"opname", "getallchildrennumber_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getallchildrennumber_resp_); + config_->stats().getallchildrennumber_resp_, + config_->stats().getallchildrennumber_resp_fast_, + config_->stats().getallchildrennumber_resp_slow_); } TEST_F(ZooKeeperFilterTest, CheckRequest) { @@ -982,7 +1132,8 @@ TEST_F(ZooKeeperFilterTest, CheckRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); testResponse({{{"opname", "check_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().check_resp_); + config_->stats().check_resp_, config_->stats().check_resp_fast_, + config_->stats().check_resp_slow_); } TEST_F(ZooKeeperFilterTest, MultiRequest) { @@ -1018,7 +1169,8 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); testResponse({{{"opname", "multi_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().multi_resp_); + config_->stats().multi_resp_, config_->stats().multi_resp_fast_, + config_->stats().multi_resp_slow_); } TEST_F(ZooKeeperFilterTest, ReconfigRequest) { @@ -1029,7 +1181,8 @@ TEST_F(ZooKeeperFilterTest, ReconfigRequest) { testRequest(data, {{{"opname", "reconfig"}}, {{"bytes", "38"}}}, config_->stats().reconfig_rq_, 38); testResponse({{{"opname", "reconfig_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().reconfig_resp_); + config_->stats().reconfig_resp_, config_->stats().reconfig_resp_fast_, + config_->stats().reconfig_resp_slow_); } TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { @@ -1046,7 +1199,8 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { config_->stats().setwatches_rq_, 84); testResponse( {{{"opname", "setwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setwatches_resp_, enumToSignedInt(XidCodes::SetWatchesXid)); + config_->stats().setwatches_resp_, config_->stats().setwatches_resp_fast_, + config_->stats().setwatches_resp_slow_, enumToSignedInt(XidCodes::SetWatchesXid)); } TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { @@ -1062,7 +1216,8 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { config_->stats().setwatches_rq_, 84); testResponse( {{{"opname", "setwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setwatches_resp_); + config_->stats().setwatches_resp_, config_->stats().setwatches_resp_fast_, + config_->stats().setwatches_resp_slow_); } TEST_F(ZooKeeperFilterTest, SetWatches2Request) { @@ -1081,7 +1236,8 @@ TEST_F(ZooKeeperFilterTest, SetWatches2Request) { config_->stats().setwatches2_rq_, 126); testResponse( {{{"opname", "setwatches2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setwatches2_resp_); + config_->stats().setwatches2_resp_, config_->stats().setwatches2_resp_fast_, + config_->stats().setwatches2_resp_slow_); } TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { @@ -1094,7 +1250,8 @@ TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { config_->stats().checkwatches_rq_, 24); testResponse( {{{"opname", "checkwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().checkwatches_resp_); + config_->stats().checkwatches_resp_, config_->stats().checkwatches_resp_fast_, + config_->stats().checkwatches_resp_slow_); } TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { @@ -1107,7 +1264,8 @@ TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { config_->stats().removewatches_rq_, 24); testResponse( {{{"opname", "removewatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().removewatches_resp_); + config_->stats().removewatches_resp_, config_->stats().removewatches_resp_fast_, + config_->stats().removewatches_resp_slow_); } TEST_F(ZooKeeperFilterTest, CloseRequest) { @@ -1117,7 +1275,8 @@ TEST_F(ZooKeeperFilterTest, CloseRequest) { testRequest(data, {{{"opname", "close"}}, {{"bytes", "12"}}}, config_->stats().close_rq_, 12); testResponse({{{"opname", "close_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().close_resp_); + config_->stats().close_resp_, config_->stats().close_resp_fast_, + config_->stats().close_resp_slow_); } TEST_F(ZooKeeperFilterTest, WatchEvent) { @@ -1139,11 +1298,12 @@ TEST_F(ZooKeeperFilterTest, WatchEvent) { TEST_F(ZooKeeperFilterTest, MissingXid) { initialize(); - const auto& stat = config_->stats().getdata_resp_; Buffer::OwnedImpl data = encodeResponseHeader(1000, 2000, 0); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); - EXPECT_EQ(0UL, stat.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(0UL, config_->stats().response_bytes_.value()); EXPECT_EQ(1UL, config_->stats().decoder_error_.value()); } @@ -1178,7 +1338,8 @@ TEST_F(ZooKeeperFilterTest, OneRequestWithMultipleOnDataCalls) { // Response. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_); } // |REQ1|REQ2| @@ -1199,9 +1360,11 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithOneOnDataCall) { // Responses. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1000, 2000); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1000, 2000); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1001, 2001, 2); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1001, 2001, 2); } // |REQ1 -------|REQ2 ---------| @@ -1254,9 +1417,11 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls) { // Responses. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1000, 2000); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1000, 2000); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1001, 2001, 2); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1001, 2001, 2); } // |REQ1 ------|REQ2|REQ3| @@ -1294,11 +1459,14 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls2) { // Responses. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1000, 2000); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1000, 2000); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1001, 2001, 2); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1001, 2001, 2); testResponse({{{"opname", "create_resp"}, {"zxid", "2002"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1002, 2002, 3); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1002, 2002, 3); } // |REQ1|REQ2|REQ3 ------| @@ -1335,11 +1503,14 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls3) { // Responses. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1000, 2000); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1000, 2000); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1001, 2001, 2); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1001, 2001, 2); testResponse({{{"opname", "create_resp"}, {"zxid", "2002"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1002, 2002, 3); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1002, 2002, 3); } // |REQ1|REQ2 ----------|REQ3| @@ -1388,11 +1559,14 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls4) { // Responses. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1000, 2000); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1000, 2000); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1001, 2001, 2); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1001, 2001, 2); testResponse({{{"opname", "create_resp"}, {"zxid", "2002"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, 1002, 2002, 3); + config_->stats().create_resp_, config_->stats().create_resp_fast_, + config_->stats().create_resp_slow_, 1002, 2002, 3); } // |RESP1 ------------| @@ -1413,6 +1587,8 @@ TEST_F(ZooKeeperFilterTest, OneResponseWithMultipleOnWriteCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(0UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(0UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. @@ -1424,6 +1600,8 @@ TEST_F(ZooKeeperFilterTest, OneResponseWithMultipleOnWriteCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(1UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(1UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1448,6 +1626,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithOneOnWriteCall) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(2UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(2UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(48UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1472,6 +1652,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(0UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(0UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. @@ -1487,6 +1669,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(1UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(1UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. @@ -1499,6 +1683,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(2UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(2UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(50UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1525,6 +1711,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls2) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(0UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(0UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. @@ -1539,6 +1727,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls2) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(3UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(3UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(72UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1567,6 +1757,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls3) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(2UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(2UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(48UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. @@ -1578,6 +1770,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls3) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(3UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(3UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(72UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1605,6 +1799,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls4) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(1UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(1UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. @@ -1616,6 +1812,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls4) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(1UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(1UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. @@ -1629,6 +1827,8 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls4) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false)); EXPECT_EQ(3UL, config_->stats().getdata_resp_.value()); + EXPECT_EQ(3UL, config_->stats().getdata_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(72UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } From 2f27d40b5e1b2d4c515b5f2560b91e4cb35c7e79 Mon Sep 17 00:00:00 2001 From: doujiang24 Date: Tue, 16 May 2023 22:20:26 -0700 Subject: [PATCH 258/740] golang filter: register config factory and config parser in a single API (#27351) * golang filter: register config factory and config parser in a single API. Signed-off-by: doujiang24 * add comment. Signed-off-by: doujiang24 --------- Signed-off-by: doujiang24 --- contrib/golang/common/dso/dso.cc | 10 +++--- contrib/golang/common/dso/dso.h | 18 ++++++---- contrib/golang/common/dso/libgolang.h | 8 +++-- contrib/golang/common/dso/test/dso_test.cc | 4 +-- contrib/golang/common/dso/test/mocks.h | 6 ++-- .../common/dso/test/test_data/simple.go | 4 +-- .../filters/http/source/go/pkg/http/config.go | 18 ++++++---- .../http/source/go/pkg/http/filtermanager.go | 33 ++++++++++-------- .../filters/http/source/golang_filter.cc | 34 +++++++++++-------- .../filters/http/source/golang_filter.h | 6 ++-- .../http/test/golang_filter_fuzz_test.cc | 2 +- .../http/test/test_data/basic/config.go | 2 +- .../http/test/test_data/dummy/plugin.go | 2 +- .../http/test/test_data/echo/config.go | 3 +- .../http/test/test_data/passthrough/filter.go | 2 +- .../http/test/test_data/routeconfig/config.go | 3 +- examples/golang/simple/config.go | 3 +- 17 files changed, 93 insertions(+), 65 deletions(-) diff --git a/contrib/golang/common/dso/dso.cc b/contrib/golang/common/dso/dso.cc index ce5c07824ca4..3a7ec19fe5d9 100644 --- a/contrib/golang/common/dso/dso.cc +++ b/contrib/golang/common/dso/dso.cc @@ -58,14 +58,16 @@ HttpFilterDsoImpl::HttpFilterDsoImpl(const std::string dso_name) : HttpFilterDso envoy_go_filter_on_http_destroy_, handler_, dso_name, "envoyGoFilterOnHttpDestroy"); } -GoUint64 HttpFilterDsoImpl::envoyGoFilterNewHttpPluginConfig(GoUint64 p0, GoUint64 p1) { +GoUint64 HttpFilterDsoImpl::envoyGoFilterNewHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2, + GoUint64 p3) { ASSERT(envoy_go_filter_new_http_plugin_config_ != nullptr); - return envoy_go_filter_new_http_plugin_config_(p0, p1); + return envoy_go_filter_new_http_plugin_config_(p0, p1, p2, p3); } -GoUint64 HttpFilterDsoImpl::envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1) { +GoUint64 HttpFilterDsoImpl::envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1, + GoUint64 p2, GoUint64 p3) { ASSERT(envoy_go_filter_merge_http_plugin_config_ != nullptr); - return envoy_go_filter_merge_http_plugin_config_(p0, p1); + return envoy_go_filter_merge_http_plugin_config_(p0, p1, p2, p3); } GoUint64 HttpFilterDsoImpl::envoyGoFilterOnHttpHeader(httpRequest* p0, GoUint64 p1, GoUint64 p2, diff --git a/contrib/golang/common/dso/dso.h b/contrib/golang/common/dso/dso.h index 0531866dba65..36dbedffbc79 100644 --- a/contrib/golang/common/dso/dso.h +++ b/contrib/golang/common/dso/dso.h @@ -30,8 +30,10 @@ class HttpFilterDso : public Dso { HttpFilterDso(const std::string dso_name) : Dso(dso_name){}; virtual ~HttpFilterDso() = default; - virtual GoUint64 envoyGoFilterNewHttpPluginConfig(GoUint64 p0, GoUint64 p1) PURE; - virtual GoUint64 envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1) PURE; + virtual GoUint64 envoyGoFilterNewHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2, + GoUint64 p3) PURE; + virtual GoUint64 envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2, + GoUint64 p3) PURE; virtual GoUint64 envoyGoFilterOnHttpHeader(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) PURE; virtual GoUint64 envoyGoFilterOnHttpData(httpRequest* p0, GoUint64 p1, GoUint64 p2, @@ -44,16 +46,20 @@ class HttpFilterDsoImpl : public HttpFilterDso { HttpFilterDsoImpl(const std::string dso_name); ~HttpFilterDsoImpl() override = default; - GoUint64 envoyGoFilterNewHttpPluginConfig(GoUint64 p0, GoUint64 p1) override; - GoUint64 envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1) override; + GoUint64 envoyGoFilterNewHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2, + GoUint64 p3) override; + GoUint64 envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2, + GoUint64 p3) override; GoUint64 envoyGoFilterOnHttpHeader(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) override; GoUint64 envoyGoFilterOnHttpData(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) override; void envoyGoFilterOnHttpDestroy(httpRequest* p0, int p1) override; private: - GoUint64 (*envoy_go_filter_new_http_plugin_config_)(GoUint64 p0, GoUint64 p1) = {nullptr}; - GoUint64 (*envoy_go_filter_merge_http_plugin_config_)(GoUint64 p0, GoUint64 p1) = {nullptr}; + GoUint64 (*envoy_go_filter_new_http_plugin_config_)(GoUint64 p0, GoUint64 p1, GoUint64 p2, + GoUint64 p3) = {nullptr}; + GoUint64 (*envoy_go_filter_merge_http_plugin_config_)(GoUint64 p0, GoUint64 p1, GoUint64 p2, + GoUint64 p3) = {nullptr}; GoUint64 (*envoy_go_filter_on_http_header_)(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) = {nullptr}; GoUint64 (*envoy_go_filter_on_http_data_)(httpRequest* p0, GoUint64 p1, GoUint64 p2, diff --git a/contrib/golang/common/dso/libgolang.h b/contrib/golang/common/dso/libgolang.h index b568317ffa7a..28e52133f0db 100644 --- a/contrib/golang/common/dso/libgolang.h +++ b/contrib/golang/common/dso/libgolang.h @@ -96,7 +96,9 @@ extern "C" { // go:linkname envoyGoFilterNewHttpPluginConfig // github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoFilterNewHttpPluginConfig extern GoUint64 -envoyGoFilterNewHttpPluginConfig(GoUint64 configPtr, // NOLINT(readability-identifier-naming) +envoyGoFilterNewHttpPluginConfig(GoUint64 namePtr, // NOLINT(readability-identifier-naming) + GoUint64 nameLen, // NOLINT(readability-identifier-naming) + GoUint64 configPtr, // NOLINT(readability-identifier-naming) GoUint64 configLen); // NOLINT(readability-identifier-naming) // go:linkname envoyGoFilterDestroyHttpPluginConfig @@ -106,7 +108,9 @@ extern void envoyGoFilterDestroyHttpPluginConfig(GoUint64 id); // go:linkname envoyGoFilterMergeHttpPluginConfig // github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoFilterMergeHttpPluginConfig extern GoUint64 -envoyGoFilterMergeHttpPluginConfig(GoUint64 parentId, // NOLINT(readability-identifier-naming) +envoyGoFilterMergeHttpPluginConfig(GoUint64 namePtr, // NOLINT(readability-identifier-naming) + GoUint64 nameLen, // NOLINT(readability-identifier-naming) + GoUint64 parentId, // NOLINT(readability-identifier-naming) GoUint64 childId); // NOLINT(readability-identifier-naming) // go:linkname envoyGoFilterOnHttpHeader diff --git a/contrib/golang/common/dso/test/dso_test.cc b/contrib/golang/common/dso/test/dso_test.cc index ea5a4a5b0358..a24d0cdd4d22 100644 --- a/contrib/golang/common/dso/test/dso_test.cc +++ b/contrib/golang/common/dso/test/dso_test.cc @@ -21,7 +21,7 @@ std::string genSoPath(std::string name) { TEST(DsoInstanceTest, SimpleAPI) { auto path = genSoPath("simple.so"); HttpFilterDsoPtr dso(new HttpFilterDsoImpl(path)); - EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0), 100); + EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 0), 100); } TEST(DsoManagerTest, Pub) { @@ -39,7 +39,7 @@ TEST(DsoManagerTest, Pub) { // get after load http filter dso dso = DsoManager::getDsoByID(id); EXPECT_NE(dso, nullptr); - EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0), 100); + EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 0), 100); // second time load http filter dso res = DsoManager::load(id, path); diff --git a/contrib/golang/common/dso/test/mocks.h b/contrib/golang/common/dso/test/mocks.h index 13b44c5adfd3..9894c40ec959 100644 --- a/contrib/golang/common/dso/test/mocks.h +++ b/contrib/golang/common/dso/test/mocks.h @@ -11,8 +11,10 @@ class MockHttpFilterDsoImpl : public HttpFilterDso { MockHttpFilterDsoImpl(); ~MockHttpFilterDsoImpl() override; - MOCK_METHOD(GoUint64, envoyGoFilterNewHttpPluginConfig, (GoUint64 p0, GoUint64 p1)); - MOCK_METHOD(GoUint64, envoyGoFilterMergeHttpPluginConfig, (GoUint64 p0, GoUint64 p1)); + MOCK_METHOD(GoUint64, envoyGoFilterNewHttpPluginConfig, + (GoUint64 p0, GoUint64 p1, GoUint64 p2, GoUint64 p3)); + MOCK_METHOD(GoUint64, envoyGoFilterMergeHttpPluginConfig, + (GoUint64 p0, GoUint64 p1, GoUint64 p2, GoUint64 p3)); MOCK_METHOD(GoUint64, envoyGoFilterOnHttpHeader, (httpRequest * p0, GoUint64 p1, GoUint64 p2, GoUint64 p3)); MOCK_METHOD(GoUint64, envoyGoFilterOnHttpData, diff --git a/contrib/golang/common/dso/test/test_data/simple.go b/contrib/golang/common/dso/test/test_data/simple.go index 3c414e72286c..4b4ca92f1e51 100644 --- a/contrib/golang/common/dso/test/test_data/simple.go +++ b/contrib/golang/common/dso/test/test_data/simple.go @@ -8,7 +8,7 @@ typedef struct { import "C" //export envoyGoFilterNewHttpPluginConfig -func envoyGoFilterNewHttpPluginConfig(configPtr uint64, configLen uint64) uint64 { +func envoyGoFilterNewHttpPluginConfig(namePtr, nameLen, configPtr, configLen uint64) uint64 { return 100 } @@ -17,7 +17,7 @@ func envoyGoFilterDestroyHttpPluginConfig(id uint64) { } //export envoyGoFilterMergeHttpPluginConfig -func envoyGoFilterMergeHttpPluginConfig(parentId uint64, childId uint64) uint64 { +func envoyGoFilterMergeHttpPluginConfig(namePtr, nameLen, parentId, childId uint64) uint64 { return 0 } diff --git a/contrib/golang/filters/http/source/go/pkg/http/config.go b/contrib/golang/filters/http/source/go/pkg/http/config.go index e46e7aaf88c4..703c5bdf97e5 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/config.go +++ b/contrib/golang/filters/http/source/go/pkg/http/config.go @@ -49,14 +49,17 @@ var ( ) //export envoyGoFilterNewHttpPluginConfig -func envoyGoFilterNewHttpPluginConfig(configPtr uint64, configLen uint64) uint64 { +func envoyGoFilterNewHttpPluginConfig(namePtr, nameLen, configPtr, configLen uint64) uint64 { buf := utils.BytesToSlice(configPtr, configLen) var any anypb.Any proto.Unmarshal(buf, &any) configNum := atomic.AddUint64(&configNumGenerator, 1) - if httpFilterConfigParser != nil { - parsedConfig, err := httpFilterConfigParser.Parse(&any) + + name := utils.BytesToString(namePtr, nameLen) + configParser := getHttpFilterConfigParser(name) + if configParser != nil { + parsedConfig, err := configParser.Parse(&any) if err != nil { cAPI.HttpLog(api.Error, fmt.Sprintf("failed to parse golang plugin config: %v", err)) // TODO: we should reject the config in the Envoy side when Go returning 0. @@ -77,8 +80,11 @@ func envoyGoFilterDestroyHttpPluginConfig(id uint64) { } //export envoyGoFilterMergeHttpPluginConfig -func envoyGoFilterMergeHttpPluginConfig(parentId uint64, childId uint64) uint64 { - if httpFilterConfigParser != nil { +func envoyGoFilterMergeHttpPluginConfig(namePtr, nameLen, parentId, childId uint64) uint64 { + name := utils.BytesToString(namePtr, nameLen) + configParser := getHttpFilterConfigParser(name) + + if configParser != nil { parent, ok := configCache.Load(parentId) if !ok { panic(fmt.Sprintf("merge config: get parentId: %d config failed", parentId)) @@ -88,7 +94,7 @@ func envoyGoFilterMergeHttpPluginConfig(parentId uint64, childId uint64) uint64 panic(fmt.Sprintf("merge config: get childId: %d config failed", childId)) } - new := httpFilterConfigParser.Merge(parent, child) + new := configParser.Merge(parent, child) configNum := atomic.AddUint64(&configNumGenerator, 1) configCache.Store(configNum, new) return configNum diff --git a/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go b/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go index 58cc53459d6f..15f8ba5b7c3e 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go +++ b/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go @@ -24,18 +24,21 @@ import ( "github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api" ) -var httpFilterConfigFactory = sync.Map{} +var httpFilterConfigFactoryAndParser = sync.Map{} -func RegisterHttpFilterConfigFactory(name string, f api.StreamFilterConfigFactory) { - httpFilterConfigFactory.Store(name, f) +type filterConfigFactoryAndParser struct { + configFactory api.StreamFilterConfigFactory + configParser api.StreamFilterConfigParser } -// no parser by default -var httpFilterConfigParser api.StreamFilterConfigParser = nil - -// TODO merge it to api.HttpFilterConfigFactory -func RegisterHttpFilterConfigParser(parser api.StreamFilterConfigParser) { - httpFilterConfigParser = parser +// Register config factory and config parser for the specified plugin. +// The "factory" parameter is required, should not be nil, +// and the "parser" parameter is optional, could be nil. +func RegisterHttpFilterConfigFactoryAndParser(name string, factory api.StreamFilterConfigFactory, parser api.StreamFilterConfigParser) { + if factory == nil { + panic("config factory should not be nil") + } + httpFilterConfigFactoryAndParser.Store(name, &filterConfigFactoryAndParser{factory, parser}) } func getOrCreateHttpFilterFactory(name string, configId uint64) api.StreamFilterFactory { @@ -44,15 +47,17 @@ func getOrCreateHttpFilterFactory(name string, configId uint64) api.StreamFilter panic(fmt.Sprintf("get config failed, plugin: %s, configId: %d", name, configId)) } - if v, ok := httpFilterConfigFactory.Load(name); ok { - return (v.(api.StreamFilterConfigFactory))(config) + if v, ok := httpFilterConfigFactoryAndParser.Load(name); ok { + return (v.(*filterConfigFactoryAndParser)).configFactory(config) } // pass through by default return PassThroughFactory(config) } -// streaming and async supported by default -func RegisterStreamingHttpFilterConfigFactory(name string, f api.StreamFilterConfigFactory) { - httpFilterConfigFactory.Store(name, f) +func getHttpFilterConfigParser(name string) api.StreamFilterConfigParser { + if v, ok := httpFilterConfigFactoryAndParser.Load(name); ok { + return (v.(*filterConfigFactoryAndParser)).configParser + } + return nil } diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc index bb210a5f6816..17fe8a3903c2 100644 --- a/contrib/golang/filters/http/source/golang_filter.cc +++ b/contrib/golang/filters/http/source/golang_filter.cc @@ -1039,12 +1039,13 @@ uint64_t FilterConfig::getConfigId() { return config_id_; } - std::string str; - auto res = plugin_config_.SerializeToString(&str); + std::string buf; + auto res = plugin_config_.SerializeToString(&buf); ASSERT(res, "SerializeToString is always successful"); - auto ptr = reinterpret_cast(str.data()); - auto len = str.length(); - config_id_ = dso_lib_->envoyGoFilterNewHttpPluginConfig(ptr, len); + auto buf_ptr = reinterpret_cast(buf.data()); + auto name_ptr = reinterpret_cast(plugin_name_.data()); + config_id_ = dso_lib_->envoyGoFilterNewHttpPluginConfig(name_ptr, plugin_name_.length(), buf_ptr, + buf.length()); ASSERT(config_id_, "config id is always grows"); ENVOY_LOG(debug, "golang filter new plugin config, id: {}", config_id_); @@ -1060,7 +1061,7 @@ FilterConfigPerRoute::FilterConfigPerRoute( for (const auto& it : config.plugins_config()) { auto plugin_name = it.first; auto route_plugin = it.second; - RoutePluginConfigPtr conf(new RoutePluginConfig(route_plugin)); + RoutePluginConfigPtr conf(new RoutePluginConfig(plugin_name, route_plugin)); ENVOY_LOG(debug, "per route golang filter config, type_url: {}", route_plugin.config().type_url()); plugins_config_.insert({plugin_name, std::move(conf)}); @@ -1083,24 +1084,27 @@ uint64_t RoutePluginConfig::getMergedConfigId(uint64_t parent_id, std::string so return merged_config_id_; } + auto name_ptr = reinterpret_cast(plugin_name_.data()); auto dlib = Dso::DsoManager::getDsoByID(so_id); ASSERT(dlib != nullptr, "load at the config parse phase, so it should not be null"); if (config_id_ == 0) { - std::string str; - auto res = plugin_config_.SerializeToString(&str); + std::string buf; + auto res = plugin_config_.SerializeToString(&buf); ASSERT(res, "SerializeToString is always successful"); - auto ptr = reinterpret_cast(str.data()); - auto len = str.length(); - config_id_ = dlib->envoyGoFilterNewHttpPluginConfig(ptr, len); + auto buf_ptr = reinterpret_cast(buf.data()); + config_id_ = dlib->envoyGoFilterNewHttpPluginConfig(name_ptr, plugin_name_.length(), buf_ptr, + buf.length()); ASSERT(config_id_, "config id is always grows"); - ENVOY_LOG(debug, "golang filter new per route plugin config, id: {}", config_id_); + ENVOY_LOG(debug, "golang filter new per route '{}' plugin config, id: {}", plugin_name_, + config_id_); } - merged_config_id_ = dlib->envoyGoFilterMergeHttpPluginConfig(parent_id, config_id_); + merged_config_id_ = dlib->envoyGoFilterMergeHttpPluginConfig(name_ptr, plugin_name_.length(), + parent_id, config_id_); ASSERT(merged_config_id_, "config id is always grows"); - ENVOY_LOG(debug, "golang filter merge plugin config, from {} + {} to {}", parent_id, config_id_, - merged_config_id_); + ENVOY_LOG(debug, "golang filter merge '{}' plugin config, from {} + {} to {}", plugin_name_, + parent_id, config_id_, merged_config_id_); return merged_config_id_; }; diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h index bf456e43ee8f..7fcb94bce04f 100644 --- a/contrib/golang/filters/http/source/golang_filter.h +++ b/contrib/golang/filters/http/source/golang_filter.h @@ -55,8 +55,9 @@ using FilterConfigSharedPtr = std::shared_ptr; class RoutePluginConfig : Logger::Loggable { public: - RoutePluginConfig(const envoy::extensions::filters::http::golang::v3alpha::RouterPlugin& config) - : plugin_config_(config.config()) { + RoutePluginConfig(const std::string plugin_name, + const envoy::extensions::filters::http::golang::v3alpha::RouterPlugin& config) + : plugin_name_(plugin_name), plugin_config_(config.config()) { ENVOY_LOG(debug, "initilizing golang filter route plugin config, type_url: {}", config.config().type_url()); }; @@ -65,6 +66,7 @@ class RoutePluginConfig : Logger::Loggable { uint64_t getMergedConfigId(uint64_t parent_id, std::string so_id); private: + const std::string plugin_name_; const ProtobufWkt::Any plugin_config_; uint64_t config_id_{0}; uint64_t merged_config_id_{0}; diff --git a/contrib/golang/filters/http/test/golang_filter_fuzz_test.cc b/contrib/golang/filters/http/test/golang_filter_fuzz_test.cc index cb845dc96863..202ddf3ba06f 100644 --- a/contrib/golang/filters/http/test/golang_filter_fuzz_test.cc +++ b/contrib/golang/filters/http/test/golang_filter_fuzz_test.cc @@ -49,7 +49,7 @@ DEFINE_PROTO_FUZZER(const envoy::extensions::filters::http::golang::GolangFilter auto dso_lib = std::make_shared(); // hard code the return config_id to 1 since the default 0 is invalid. - ON_CALL(*dso_lib.get(), envoyGoFilterNewHttpPluginConfig(_, _)).WillByDefault(Return(1)); + ON_CALL(*dso_lib.get(), envoyGoFilterNewHttpPluginConfig(_, _, _, _)).WillByDefault(Return(1)); ON_CALL(*dso_lib.get(), envoyGoFilterOnHttpHeader(_, _, _, _)) .WillByDefault(Return(static_cast(GolangStatus::Continue))); ON_CALL(*dso_lib.get(), envoyGoFilterOnHttpData(_, _, _, _)) diff --git a/contrib/golang/filters/http/test/test_data/basic/config.go b/contrib/golang/filters/http/test/test_data/basic/config.go index e14885cbe05e..05e2d7eb8bd1 100644 --- a/contrib/golang/filters/http/test/test_data/basic/config.go +++ b/contrib/golang/filters/http/test/test_data/basic/config.go @@ -8,7 +8,7 @@ import ( const Name = "basic" func init() { - http.RegisterHttpFilterConfigFactory(Name, ConfigFactory) + http.RegisterHttpFilterConfigFactoryAndParser(Name, ConfigFactory, nil) } func ConfigFactory(interface{}) api.StreamFilterFactory { diff --git a/contrib/golang/filters/http/test/test_data/dummy/plugin.go b/contrib/golang/filters/http/test/test_data/dummy/plugin.go index ae9692e2eafa..2fbb9bac7951 100644 --- a/contrib/golang/filters/http/test/test_data/dummy/plugin.go +++ b/contrib/golang/filters/http/test/test_data/dummy/plugin.go @@ -7,7 +7,7 @@ import ( ) func init() { - http.RegisterHttpFilterConfigFactory("", http.PassThroughFactory) + http.RegisterHttpFilterConfigFactoryAndParser("", http.PassThroughFactory, nil) } func main() { diff --git a/contrib/golang/filters/http/test/test_data/echo/config.go b/contrib/golang/filters/http/test/test_data/echo/config.go index 44f457606019..481175951ab5 100644 --- a/contrib/golang/filters/http/test/test_data/echo/config.go +++ b/contrib/golang/filters/http/test/test_data/echo/config.go @@ -10,8 +10,7 @@ import ( const Name = "echo" func init() { - http.RegisterHttpFilterConfigFactory(Name, ConfigFactory) - http.RegisterHttpFilterConfigParser(&parser{}) + http.RegisterHttpFilterConfigFactoryAndParser(Name, ConfigFactory, &parser{}) } type config struct { diff --git a/contrib/golang/filters/http/test/test_data/passthrough/filter.go b/contrib/golang/filters/http/test/test_data/passthrough/filter.go index 9ed60df4dc81..c78dbee0f462 100644 --- a/contrib/golang/filters/http/test/test_data/passthrough/filter.go +++ b/contrib/golang/filters/http/test/test_data/passthrough/filter.go @@ -5,7 +5,7 @@ import ( ) func init() { - http.RegisterHttpFilterConfigFactory("", http.PassThroughFactory) + http.RegisterHttpFilterConfigFactoryAndParser("", http.PassThroughFactory, nil) } func main() { diff --git a/contrib/golang/filters/http/test/test_data/routeconfig/config.go b/contrib/golang/filters/http/test/test_data/routeconfig/config.go index de8cb84ff809..fe7067edb01c 100644 --- a/contrib/golang/filters/http/test/test_data/routeconfig/config.go +++ b/contrib/golang/filters/http/test/test_data/routeconfig/config.go @@ -10,8 +10,7 @@ import ( const Name = "routeconfig" func init() { - http.RegisterHttpFilterConfigFactory(Name, configFactory) - http.RegisterHttpFilterConfigParser(&parser{}) + http.RegisterHttpFilterConfigFactoryAndParser(Name, configFactory, &parser{}) } func configFactory(c interface{}) api.StreamFilterFactory { diff --git a/examples/golang/simple/config.go b/examples/golang/simple/config.go index a8d46b9fb177..588f90745672 100644 --- a/examples/golang/simple/config.go +++ b/examples/golang/simple/config.go @@ -13,8 +13,7 @@ import ( const Name = "simple" func init() { - http.RegisterHttpFilterConfigFactory(Name, ConfigFactory) - http.RegisterHttpFilterConfigParser(&parser{}) + http.RegisterHttpFilterConfigFactoryAndParser(Name, ConfigFactory, &parser{}) } type config struct { From 2a63fe20f7dd77062e8ca04ee19bc605bf9e02d5 Mon Sep 17 00:00:00 2001 From: sh2 Date: Wed, 17 May 2023 19:21:21 +0800 Subject: [PATCH 259/740] fix a typo in internal_listener.rst (#27451) Update internal_listener.rst Signed-off-by: sh2 --- docs/root/configuration/other_features/internal_listener.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index ecc03ce547c7..86f9caac5d6c 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -90,7 +90,7 @@ Decapsulate the CONNECT requests There are some complicated GET-in-CONNECT requests across services or edges. In order to proxy the GET request within Envoy, two layer of :ref:`HTTP connection manager ` -is demanded. The first HHTTP connection manager layer extract the TCP stream from a CONNECT request and redirect the TCP stream to the second +is demanded. The first HTTP connection manager layer extract the TCP stream from a CONNECT request and redirect the TCP stream to the second HTTP connection manager layer to parse the common GET requests. A sample config can be found :repo:`here ` From 2ab9ac36bc6941c2716153fd403e7ee733c29ce9 Mon Sep 17 00:00:00 2001 From: "Dr. Andre Vehreschild" <101638173+vehre-x41@users.noreply.github.com> Date: Wed, 17 May 2023 14:42:33 +0200 Subject: [PATCH 260/740] [fuzz] Reduce valid input gen depth to 20 (#27447) Signed-off-by: Andre Vehreschild --- .../network/common/fuzz/network_readfilter_fuzz_test.cc | 2 +- .../network/common/fuzz/network_writefilter_fuzz_test.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc index 465d98ab9f20..bed56c089f78 100644 --- a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc @@ -46,7 +46,7 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::network::FilterFuzzTestCase factory->createEmptyConfigProto()->GetDescriptor()->full_name())); ProtobufMessage::ValidatedInputGenerator generator( - seed, ProtobufMessage::composeFiltersAnyMap(), 40); + seed, ProtobufMessage::composeFiltersAnyMap(), 20); ProtobufMessage::traverseMessage(generator, *input, true); }}; diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc b/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc index 285160d3e32b..9b321bb8242a 100644 --- a/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc @@ -39,8 +39,8 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::network::FilterFuzzTestCase input->mutable_config()->mutable_typed_config()->set_type_url( absl::StrCat("type.googleapis.com/", factory->createEmptyConfigProto()->GetDescriptor()->full_name())); - ProtobufMessage::ValidatedInputGenerator generator(seed, - ProtobufMessage::composeFiltersAnyMap()); + ProtobufMessage::ValidatedInputGenerator generator( + seed, ProtobufMessage::composeFiltersAnyMap(), 20); ProtobufMessage::traverseMessage(generator, *input, true); }}; try { From 755b7182096896fe07d4eedc522f8dd080acb88d Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Wed, 17 May 2023 09:56:50 -0400 Subject: [PATCH 261/740] sub-formatter: fix wrong handling of UPSTREAM_HEADER_BYTES_SENT (#27291) Signed-off-by: Adi Suissa-Peleg --- source/common/formatter/substitution_formatter.cc | 3 ++- ...terfuzz-testcase-minimized-route_fuzz_test-6556386696495104 | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-6556386696495104 diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index 731812171306..bf97b1df017c 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -1070,7 +1070,8 @@ const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnow [](const std::string&, const absl::optional&) { return std::make_unique( [](const StreamInfo::StreamInfo& stream_info) { - return stream_info.getUpstreamBytesMeter()->headerBytesSent(); + auto bytes_meter = stream_info.getUpstreamBytesMeter(); + return bytes_meter ? bytes_meter->headerBytesSent() : 0; }); }}}, {"DOWNSTREAM_WIRE_BYTES_SENT", diff --git a/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-6556386696495104 b/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-6556386696495104 new file mode 100644 index 000000000000..e3205871b73a --- /dev/null +++ b/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-6556386696495104 @@ -0,0 +1 @@ +config { virtual_hosts { name: " " domains: "*" routes { match { path: "" } route { cluster_header: " " } } } request_headers_to_add { header { key: " " value: "%UPSTREAM_HEADER_BYTES_SENT%" } } } headers { headers { key: "x-forwarded-proto" value: " " } headers { key: ":path" } } \ No newline at end of file From 7854138d4753f4d7a284e271592e0f7e7aeef783 Mon Sep 17 00:00:00 2001 From: code Date: Wed, 17 May 2023 22:21:56 +0800 Subject: [PATCH 262/740] =?UTF-8?q?factory=20context=20refactor:=20refacto?= =?UTF-8?q?r=20the=20transport=20factory=20cont=E2=80=A6=20(take=202)=20(#?= =?UTF-8?q?27354)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "Revert "factory context refactor: refactor the transport factory cont… (#27348)" This reverts commit 52ee63e287cc0606ec8c917a49e4c531aaf14d03. Signed-off-by: wbpcode --- .../source/cryptomb_private_key_provider.cc | 10 +- .../private_key_providers/test/config_test.cc | 4 +- .../test/fake_factory.cc | 4 +- .../network/test/postgres_integration_test.cc | 2 +- .../source/qat_private_key_provider.cc | 7 +- .../private_key_providers/test/config_test.cc | 4 +- .../private_key_providers/test/ops_test.cc | 5 +- .../filters/network/source/utility.h | 1 + .../filters/network/test/conn_manager_test.cc | 3 +- .../filters/network/test/router_test.cc | 3 +- contrib/sxg/filters/http/test/filter_test.cc | 9 +- envoy/server/transport_socket_config.h | 72 ++------- mobile/test/common/integration/test_server.cc | 5 +- .../quic/quic_transport_socket_factory.cc | 8 +- source/common/secret/sds_api.h | 41 +++-- .../upstream/health_discovery_service.cc | 3 +- source/common/upstream/upstream_impl.cc | 16 +- source/extensions/clusters/eds/leds.cc | 2 +- .../listener_manager/listener_impl.cc | 3 +- .../transport_sockets/alts/config.cc | 7 +- .../internal_upstream/config.cc | 2 +- .../proxy_protocol/config.cc | 2 +- .../transport_sockets/tap/config.cc | 17 ++- .../transport_sockets/tcp_stats/config.cc | 2 +- .../transport_sockets/tls/config.cc | 6 +- .../tls/context_config_impl.cc | 5 +- .../tls/context_config_impl.h | 2 +- source/extensions/upstreams/http/config.h | 2 +- source/server/server.h | 6 +- source/server/transport_socket_config_impl.h | 34 ++--- .../grpc_client_integration_test_harness.h | 2 +- test/common/http/http3/conn_pool_test.cc | 5 +- .../quic_transport_socket_factory_test.cc | 2 +- .../common/secret/secret_manager_impl_test.cc | 69 ++++----- test/common/upstream/BUILD | 1 + test/common/upstream/hds_test.cc | 4 +- test/extensions/clusters/eds/leds_test.cc | 2 +- .../filters/http/oauth2/filter_test.cc | 9 +- .../alts/alts_integration_test.cc | 2 +- .../transport_sockets/alts/config_test.cc | 6 +- .../upstream_starttls_integration_test.cc | 2 +- test/extensions/transport_sockets/tls/BUILD | 1 + .../tls/context_impl_test.cc | 35 ++--- .../transport_sockets/tls/ssl_certs_test.h | 2 +- .../transport_sockets/tls/ssl_socket_test.cc | 61 ++++---- .../tls/test_private_key_method_provider.cc | 3 +- test/integration/base_integration_test.cc | 4 +- .../integration/quic_http_integration_test.cc | 4 +- test/integration/ssl_utility.cc | 4 +- test/integration/utility.cc | 4 +- test/integration/xfcc_integration_test.h | 2 +- test/mocks/server/BUILD | 36 +++-- test/mocks/server/instance.cc | 36 +---- test/mocks/server/instance.h | 134 +---------------- test/mocks/server/server_factory_context.cc | 43 ++++++ test/mocks/server/server_factory_context.h | 140 ++++++++++++++++++ .../transport_socket_factory_context.cc | 15 +- .../server/transport_socket_factory_context.h | 24 +-- 58 files changed, 450 insertions(+), 489 deletions(-) create mode 100644 test/mocks/server/server_factory_context.cc create mode 100644 test/mocks/server/server_factory_context.h diff --git a/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc b/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc index e1a7fc96d4ae..f94f37db6323 100644 --- a/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc +++ b/contrib/cryptomb/private_key_providers/source/cryptomb_private_key_provider.cc @@ -516,9 +516,10 @@ CryptoMbPrivateKeyMethodProvider::CryptoMbPrivateKeyMethodProvider( const envoy::extensions::private_key_providers::cryptomb::v3alpha:: CryptoMbPrivateKeyMethodConfig& conf, Server::Configuration::TransportSocketFactoryContext& factory_context, IppCryptoSharedPtr ipp) - : api_(factory_context.api()), - tls_(ThreadLocal::TypedSlot::makeUnique(factory_context.threadLocal())), - stats_(generateCryptoMbStats("cryptomb", factory_context.scope())) { + : api_(factory_context.serverFactoryContext().api()), + tls_(ThreadLocal::TypedSlot::makeUnique( + factory_context.serverFactoryContext().threadLocal())), + stats_(generateCryptoMbStats("cryptomb", factory_context.statsScope())) { if (!ipp->mbxIsCryptoMbApplicable(0)) { throw EnvoyException("Multi-buffer CPU instructions not available."); @@ -527,8 +528,7 @@ CryptoMbPrivateKeyMethodProvider::CryptoMbPrivateKeyMethodProvider( std::chrono::milliseconds poll_delay = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(conf, poll_delay, 200)); - std::string private_key = - Config::DataSource::read(conf.private_key(), false, factory_context.api()); + std::string private_key = Config::DataSource::read(conf.private_key(), false, api_); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); diff --git a/contrib/cryptomb/private_key_providers/test/config_test.cc b/contrib/cryptomb/private_key_providers/test/config_test.cc index 370ce3953692..158936f49037 100644 --- a/contrib/cryptomb/private_key_providers/test/config_test.cc +++ b/contrib/cryptomb/private_key_providers/test/config_test.cc @@ -35,8 +35,8 @@ parsePrivateKeyProviderFromV3Yaml(const std::string& yaml_string) { class CryptoMbConfigTest : public Event::TestUsingSimulatedTime, public testing::Test { public: CryptoMbConfigTest() : api_(Api::createApiForTest(store_, time_system_)) { - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); - ON_CALL(factory_context_, threadLocal()).WillByDefault(ReturnRef(tls_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, threadLocal()).WillByDefault(ReturnRef(tls_)); ON_CALL(factory_context_, sslContextManager()).WillByDefault(ReturnRef(context_manager_)); ON_CALL(context_manager_, privateKeyMethodManager()) .WillByDefault(ReturnRef(private_key_method_manager_)); diff --git a/contrib/cryptomb/private_key_providers/test/fake_factory.cc b/contrib/cryptomb/private_key_providers/test/fake_factory.cc index 7ae205f52f9e..ddd2a5d363e8 100644 --- a/contrib/cryptomb/private_key_providers/test/fake_factory.cc +++ b/contrib/cryptomb/private_key_providers/test/fake_factory.cc @@ -135,8 +135,8 @@ FakeCryptoMbPrivateKeyMethodFactory::createPrivateKeyMethodProviderInstance( std::make_shared(supported_instruction_set_); // We need to get more RSA key params in order to be able to use BoringSSL signing functions. - std::string private_key = - Config::DataSource::read(conf.private_key(), false, private_key_provider_context.api()); + std::string private_key = Config::DataSource::read( + conf.private_key(), false, private_key_provider_context.serverFactoryContext().api()); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); diff --git a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc index 6fb93661e68c..4f1e3c829fb8 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc @@ -313,7 +313,7 @@ class UpstreamSSLBaseIntegrationTest : public PostgresBaseIntegrationTest { TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), downstream_tls_context); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); auto cfg = std::make_unique( downstream_tls_context, mock_factory_ctx); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); diff --git a/contrib/qat/private_key_providers/source/qat_private_key_provider.cc b/contrib/qat/private_key_providers/source/qat_private_key_provider.cc index 6b2d808d254b..863637604157 100644 --- a/contrib/qat/private_key_providers/source/qat_private_key_provider.cc +++ b/contrib/qat/private_key_providers/source/qat_private_key_provider.cc @@ -347,9 +347,9 @@ QatPrivateKeyMethodProvider::QatPrivateKeyMethodProvider( const envoy::extensions::private_key_providers::qat::v3alpha::QatPrivateKeyMethodConfig& conf, Server::Configuration::TransportSocketFactoryContext& factory_context, LibQatCryptoSharedPtr libqat) - : api_(factory_context.api()), libqat_(libqat) { + : api_(factory_context.serverFactoryContext().api()), libqat_(libqat) { - manager_ = factory_context.singletonManager().getTyped( + manager_ = factory_context.serverFactoryContext().singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(qat_manager), [libqat] { return std::make_shared(libqat); }); @@ -358,8 +358,7 @@ QatPrivateKeyMethodProvider::QatPrivateKeyMethodProvider( std::chrono::milliseconds poll_delay = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(conf, poll_delay, 5)); - std::string private_key = - Config::DataSource::read(conf.private_key(), false, factory_context.api()); + std::string private_key = Config::DataSource::read(conf.private_key(), false, api_); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); diff --git a/contrib/qat/private_key_providers/test/config_test.cc b/contrib/qat/private_key_providers/test/config_test.cc index b5527bc97bc8..518c9ba22d6a 100644 --- a/contrib/qat/private_key_providers/test/config_test.cc +++ b/contrib/qat/private_key_providers/test/config_test.cc @@ -48,11 +48,11 @@ class QatConfigTest : public Event::TestUsingSimulatedTime, public testing::Test QatConfigTest() : api_(Api::createApiForTest(store_, time_system_)), libqat_(std::make_shared()), fsm_(libqat_) { - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); ON_CALL(factory_context_, sslContextManager()).WillByDefault(ReturnRef(context_manager_)); ON_CALL(context_manager_, privateKeyMethodManager()) .WillByDefault(ReturnRef(private_key_method_manager_)); - ON_CALL(factory_context_, singletonManager()).WillByDefault(ReturnRef(fsm_)); + ON_CALL(factory_context_.server_context_, singletonManager()).WillByDefault(ReturnRef(fsm_)); } Ssl::PrivateKeyMethodProviderSharedPtr createWithConfig(std::string yaml) { diff --git a/contrib/qat/private_key_providers/test/ops_test.cc b/contrib/qat/private_key_providers/test/ops_test.cc index 0e2f77322107..68922f08bfef 100644 --- a/contrib/qat/private_key_providers/test/ops_test.cc +++ b/contrib/qat/private_key_providers/test/ops_test.cc @@ -59,8 +59,9 @@ class QatProviderTest : public testing::Test { dispatcher_(api_->allocateDispatcher("test_thread")), libqat_(std::make_shared()), fsm_(libqat_) { handle_.setLibqat(libqat_); - ON_CALL(factory_context_, api()).WillByDefault(testing::ReturnRef(*api_)); - ON_CALL(factory_context_, singletonManager()).WillByDefault(testing::ReturnRef(fsm_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, singletonManager()) + .WillByDefault(testing::ReturnRef(fsm_)); } Stats::TestUtil::TestStore store_; diff --git a/contrib/sip_proxy/filters/network/source/utility.h b/contrib/sip_proxy/filters/network/source/utility.h index 1cc4a69872d0..29465c09cab2 100644 --- a/contrib/sip_proxy/filters/network/source/utility.h +++ b/contrib/sip_proxy/filters/network/source/utility.h @@ -156,6 +156,7 @@ class Utility { public: static const std::string& localAddress(Server::Configuration::FactoryContext& context) { return context.getTransportSocketFactoryContext() + .serverFactoryContext() .localInfo() .address() ->ip() diff --git a/contrib/sip_proxy/filters/network/test/conn_manager_test.cc b/contrib/sip_proxy/filters/network/test/conn_manager_test.cc index 6b687031d9e3..8cf88bed959f 100644 --- a/contrib/sip_proxy/filters/network/test/conn_manager_test.cc +++ b/contrib/sip_proxy/filters/network/test/conn_manager_test.cc @@ -80,7 +80,8 @@ class SipConnectionManagerTest : public testing::Test { EXPECT_CALL(context_, getTransportSocketFactoryContext()) .WillRepeatedly(testing::ReturnRef(factory_context_)); - EXPECT_CALL(factory_context_, localInfo()).WillRepeatedly(testing::ReturnRef(local_info_)); + EXPECT_CALL(factory_context_.server_context_, localInfo()) + .WillRepeatedly(testing::ReturnRef(local_info_)); ON_CALL(random_, random()).WillByDefault(Return(42)); filter_ = std::make_unique( *config_, random_, filter_callbacks_.connection_.dispatcher_.timeSource(), context_, diff --git a/contrib/sip_proxy/filters/network/test/router_test.cc b/contrib/sip_proxy/filters/network/test/router_test.cc index f50895c58ead..9964207138da 100644 --- a/contrib/sip_proxy/filters/network/test/router_test.cc +++ b/contrib/sip_proxy/filters/network/test/router_test.cc @@ -105,7 +105,8 @@ class SipRouterTest : public testing::Test { EXPECT_CALL(context_, getTransportSocketFactoryContext()) .WillRepeatedly(testing::ReturnRef(factory_context_)); - EXPECT_CALL(factory_context_, localInfo()).WillRepeatedly(testing::ReturnRef(local_info_)); + EXPECT_CALL(factory_context_.server_context_, localInfo()) + .WillRepeatedly(testing::ReturnRef(local_info_)); transaction_infos_ = std::make_shared(); context_.cluster_manager_.initializeThreadLocalClusters({cluster_name_}); diff --git a/contrib/sxg/filters/http/test/filter_test.cc b/contrib/sxg/filters/http/test/filter_test.cc index 5a874067c10a..92bf1fa92f9e 100644 --- a/contrib/sxg/filters/http/test/filter_test.cc +++ b/contrib/sxg/filters/http/test/filter_test.cc @@ -360,14 +360,13 @@ TEST_F(FilterTest, SdsDynamicGenericSecret) { NiceMock secret_context; NiceMock local_info; Api::ApiPtr api = Api::createApiForTest(); - Stats::IsolatedStoreImpl stats; NiceMock init_manager; Init::TargetHandlePtr init_handle; NiceMock dispatcher; - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api)); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); EXPECT_CALL(secret_context, initManager()).Times(0); EXPECT_CALL(init_manager, add(_)) .WillRepeatedly(Invoke([&init_handle](const Init::Target& target) { diff --git a/envoy/server/transport_socket_config.h b/envoy/server/transport_socket_config.h index 756c27f64365..c39723b573ca 100644 --- a/envoy/server/transport_socket_config.h +++ b/envoy/server/transport_socket_config.h @@ -30,14 +30,23 @@ class TransportSocketFactoryContext { virtual ~TransportSocketFactoryContext() = default; /** - * @return ServerFactoryContext which lifetime is no shorter than the server. + * @return ServerFactoryContext& the server factory context. */ - virtual ServerFactoryContext& getServerFactoryContext() PURE; + virtual ServerFactoryContext& serverFactoryContext() PURE; /** - * @return OptRef the global HTTP admin endpoint for the server. + * @return Upstream::ClusterManager& singleton for use by the entire server. + * TODO(wbpcode): clusterManager() of ServerFactoryContext still be invalid when loading + * static cluster. So we need to provide an cluster manager reference here. + * This could be removed after https://github.com/envoyproxy/envoy/issues/26653 is resolved. */ - virtual OptRef admin() PURE; + virtual Upstream::ClusterManager& clusterManager() PURE; + + /** + * @return ProtobufMessage::ValidationVisitor& validation visitor for cluster configuration + * messages. + */ + virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; /** * @return Ssl::ContextManager& the SSL context manager. @@ -47,7 +56,7 @@ class TransportSocketFactoryContext { /** * @return Stats::Scope& the transport socket's stats scope. */ - virtual Stats::Scope& scope() PURE; + virtual Stats::Scope& statsScope() PURE; /** * Return the instance of secret manager. @@ -55,60 +64,9 @@ class TransportSocketFactoryContext { virtual Secret::SecretManager& secretManager() PURE; /** - * @return the instance of ClusterManager. - */ - virtual Upstream::ClusterManager& clusterManager() PURE; - - /** - * @return information about the local environment the server is running in. - */ - virtual const LocalInfo::LocalInfo& localInfo() const PURE; - - /** - * @return Event::Dispatcher& the main thread's dispatcher. - */ - virtual Event::Dispatcher& mainThreadDispatcher() PURE; - - /** - * @return Server::Options& the command-line options that Envoy was started with. - */ - virtual const Options& options() PURE; - - /** - * @return the server-wide stats store. - */ - virtual Stats::Store& stats() PURE; - - /** - * @return a reference to the instance of an init manager. + * @return the init manager of the particular context. */ virtual Init::Manager& initManager() PURE; - - /** - * @return the server's singleton manager. - */ - virtual Singleton::Manager& singletonManager() PURE; - - /** - * @return the server's TLS slot allocator. - */ - virtual ThreadLocal::SlotAllocator& threadLocal() PURE; - - /** - * @return ProtobufMessage::ValidationVisitor& validation visitor for filter configuration - * messages. - */ - virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; - - /** - * @return reference to the Api object - */ - virtual Api::Api& api() PURE; - - /** - * @return reference to the access log manager object - */ - virtual AccessLog::AccessLogManager& accessLogManager() PURE; }; using TransportSocketFactoryContextPtr = std::unique_ptr; diff --git a/mobile/test/common/integration/test_server.cc b/mobile/test/common/integration/test_server.cc index 771360fc3da0..f47ef32aeb00 100644 --- a/mobile/test/common/integration/test_server.cc +++ b/mobile/test/common/integration/test_server.cc @@ -53,8 +53,9 @@ Network::DownstreamTransportSocketFactoryPtr TestServer::createUpstreamTlsContex TestServer::TestServer() : api_(Api::createApiForTest(stats_store_, time_system_)), version_(Network::Address::IpVersion::v4), upstream_config_(time_system_), port_(0) { - ON_CALL(factory_context_, api()).WillByDefault(testing::ReturnRef(*api_)); - ON_CALL(factory_context_, scope()).WillByDefault(testing::ReturnRef(*stats_store_.rootScope())); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(factory_context_, statsScope()) + .WillByDefault(testing::ReturnRef(*stats_store_.rootScope())); } void TestServer::startTestServer(bool use_quic) { diff --git a/source/common/quic/quic_transport_socket_factory.cc b/source/common/quic/quic_transport_socket_factory.cc index 05d5ddc0c0ca..f7f21ef5db8f 100644 --- a/source/common/quic/quic_transport_socket_factory.cc +++ b/source/common/quic/quic_transport_socket_factory.cc @@ -30,8 +30,8 @@ QuicServerTransportSocketConfigFactory::createTransportSocketFactory( } auto factory = std::make_unique( - PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_transport, enable_early_data, true), context.scope(), - std::move(server_config)); + PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_transport, enable_early_data, true), + context.statsScope(), std::move(server_config)); factory->initialize(); return factory; } @@ -59,9 +59,9 @@ QuicClientTransportSocketConfigFactory::createTransportSocketFactory( QuicClientTransportSocketFactory::QuicClientTransportSocketFactory( Ssl::ClientContextConfigPtr config, Server::Configuration::TransportSocketFactoryContext& factory_context) - : QuicTransportSocketFactoryBase(factory_context.scope(), "client"), + : QuicTransportSocketFactoryBase(factory_context.statsScope(), "client"), fallback_factory_(std::make_unique( - std::move(config), factory_context.sslContextManager(), factory_context.scope())) {} + std::move(config), factory_context.sslContextManager(), factory_context.statsScope())) {} ProtobufTypes::MessagePtr QuicClientTransportSocketConfigFactory::createEmptyConfigProto() { return std::make_unique(); diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index a09d87b6ac52..edc5db1e0ad8 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -137,13 +137,13 @@ class TlsCertificateSdsApi : public SdsApi, public TlsCertificateConfigProvider const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - Config::Utility::checkLocalInfo("TlsCertificateSdsApi", secret_provider_context.localInfo()); + auto& server_context = secret_provider_context.serverFactoryContext(); + Config::Utility::checkLocalInfo("TlsCertificateSdsApi", server_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - secret_provider_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), - destructor_cb, secret_provider_context.mainThreadDispatcher(), - secret_provider_context.api()); + server_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), + destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); } TlsCertificateSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, @@ -223,14 +223,14 @@ class CertificateValidationContextSdsApi : public SdsApi, const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. + auto& server_context = secret_provider_context.serverFactoryContext(); Config::Utility::checkLocalInfo("CertificateValidationContextSdsApi", - secret_provider_context.localInfo()); + server_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - secret_provider_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), - destructor_cb, secret_provider_context.mainThreadDispatcher(), - secret_provider_context.api()); + server_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), + destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); } CertificateValidationContextSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, const std::string& sds_config_name, @@ -318,14 +318,13 @@ class TlsSessionTicketKeysSdsApi : public SdsApi, public TlsSessionTicketKeysCon const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - Config::Utility::checkLocalInfo("TlsSessionTicketKeysSdsApi", - secret_provider_context.localInfo()); + auto& server_context = secret_provider_context.serverFactoryContext(); + Config::Utility::checkLocalInfo("TlsSessionTicketKeysSdsApi", server_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - secret_provider_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), - destructor_cb, secret_provider_context.mainThreadDispatcher(), - secret_provider_context.api()); + server_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), + destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); } TlsSessionTicketKeysSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, @@ -392,13 +391,13 @@ class GenericSecretSdsApi : public SdsApi, public GenericSecretConfigProvider { const std::string& sds_config_name, std::function destructor_cb) { // We need to do this early as we invoke the subscription factory during initialization, which // is too late to throw. - Config::Utility::checkLocalInfo("GenericSecretSdsApi", secret_provider_context.localInfo()); + auto& server_context = secret_provider_context.serverFactoryContext(); + Config::Utility::checkLocalInfo("GenericSecretSdsApi", server_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), - secret_provider_context.mainThreadDispatcher().timeSource(), - secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), - destructor_cb, secret_provider_context.mainThreadDispatcher(), - secret_provider_context.api()); + server_context.mainThreadDispatcher().timeSource(), + secret_provider_context.messageValidationVisitor(), server_context.serverScope().store(), + destructor_cb, server_context.mainThreadDispatcher(), server_context.api()); } GenericSecretSdsApi(const envoy::config::core::v3::ConfigSource& sds_config, diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index 7a8386069184..30f16ac78084 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -516,8 +516,7 @@ ProdClusterInfoFactory::createClusterInfo(const CreateClusterInfoParams& params) Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( params.server_context_, params.ssl_context_manager_, *scope, - params.server_context_.clusterManager(), params.stats_, - params.server_context_.messageValidationVisitor()); + params.server_context_.clusterManager(), params.server_context_.messageValidationVisitor()); // TODO(JimmyCYJ): Support SDS for HDS cluster. Network::UpstreamTransportSocketFactoryPtr socket_factory = diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 1010fbed6fda..5bda8581a491 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -882,11 +882,14 @@ class FactoryContextImpl : public Server::Configuration::CommonFactoryContext { // other contexts taken from TransportSocketFactoryContext. FactoryContextImpl(Stats::Scope& stats_scope, Envoy::Runtime::Loader& runtime, Server::Configuration::TransportSocketFactoryContext& c) - : admin_(c.admin()), server_scope_(*c.stats().rootScope()), stats_scope_(stats_scope), - cluster_manager_(c.clusterManager()), local_info_(c.localInfo()), - dispatcher_(c.mainThreadDispatcher()), runtime_(runtime), - singleton_manager_(c.singletonManager()), tls_(c.threadLocal()), api_(c.api()), - options_(c.options()), message_validation_visitor_(c.messageValidationVisitor()) {} + : admin_(c.serverFactoryContext().admin()), + server_scope_(c.serverFactoryContext().serverScope()), stats_scope_(stats_scope), + cluster_manager_(c.clusterManager()), local_info_(c.serverFactoryContext().localInfo()), + dispatcher_(c.serverFactoryContext().mainThreadDispatcher()), runtime_(runtime), + singleton_manager_(c.serverFactoryContext().singletonManager()), + tls_(c.serverFactoryContext().threadLocal()), api_(c.serverFactoryContext().api()), + options_(c.serverFactoryContext().options()), + message_validation_visitor_(c.messageValidationVisitor()) {} Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } Event::Dispatcher& mainThreadDispatcher() override { return dispatcher_; } @@ -1375,8 +1378,7 @@ ClusterImplBase::ClusterImplBase(const envoy::config::cluster::v3::Cluster& clus transport_factory_context_ = std::make_unique( server_context, cluster_context.sslContextManager(), *stats_scope, - cluster_context.clusterManager(), server_context.serverScope().store(), - cluster_context.messageValidationVisitor()); + cluster_context.clusterManager(), cluster_context.messageValidationVisitor()); transport_factory_context_->setInitManager(init_manager_); auto socket_factory = createTransportSocketFactory(cluster, *transport_factory_context_); diff --git a/source/extensions/clusters/eds/leds.cc b/source/extensions/clusters/eds/leds.cc index 2e2ed0db6941..91c5ad6f32a7 100644 --- a/source/extensions/clusters/eds/leds.cc +++ b/source/extensions/clusters/eds/leds.cc @@ -17,7 +17,7 @@ LedsSubscription::LedsSubscription( Stats::Scope& cluster_stats_scope, const UpdateCb& callback) : Envoy::Config::SubscriptionBase( factory_context.messageValidationVisitor(), leds_config.leds_collection_name()), - local_info_(factory_context.localInfo()), cluster_name_(cluster_name), + local_info_(factory_context.serverFactoryContext().localInfo()), cluster_name_(cluster_name), stats_scope_(cluster_stats_scope.createScope("leds.")), stats_({ALL_LEDS_STATS(POOL_COUNTER(*stats_scope_))}), callback_(callback) { const xds::core::v3::ResourceLocator leds_resource_locator = diff --git a/source/extensions/listener_managers/listener_manager/listener_impl.cc b/source/extensions/listener_managers/listener_manager/listener_impl.cc index 94b29a189e71..47909c0de470 100644 --- a/source/extensions/listener_managers/listener_manager/listener_impl.cc +++ b/source/extensions/listener_managers/listener_manager/listener_impl.cc @@ -354,8 +354,7 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, transport_factory_context_( std::make_shared( parent_.server_.serverFactoryContext(), parent_.server_.sslContextManager(), - listenerScope(), parent_.server_.clusterManager(), parent_.server_.stats(), - validation_visitor_)), + listenerScope(), parent_.server_.clusterManager(), validation_visitor_)), quic_stat_names_(parent_.quicStatNames()), missing_listener_config_stats_({ALL_MISSING_LISTENER_CONFIG_STATS( POOL_COUNTER(listener_factory_context_->listenerScope()))}) { diff --git a/source/extensions/transport_sockets/alts/config.cc b/source/extensions/transport_sockets/alts/config.cc index 7c3d18d27d7a..0af3cb780775 100644 --- a/source/extensions/transport_sockets/alts/config.cc +++ b/source/extensions/transport_sockets/alts/config.cc @@ -97,9 +97,10 @@ TransportSocketFactoryPtr createTransportSocketFactoryHelper( Server::Configuration::TransportSocketFactoryContext& factory_ctxt) { // A reference to this is held in the factory closure to keep the singleton // instance alive. - auto alts_shared_state = factory_ctxt.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(alts_shared_state), - [] { return std::make_shared(); }); + auto alts_shared_state = + factory_ctxt.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(alts_shared_state), + [] { return std::make_shared(); }); auto config = MessageUtil::downcastAndValidate( message, factory_ctxt.messageValidationVisitor()); diff --git a/source/extensions/transport_sockets/internal_upstream/config.cc b/source/extensions/transport_sockets/internal_upstream/config.cc index 2993e34d2815..5517c5c34203 100644 --- a/source/extensions/transport_sockets/internal_upstream/config.cc +++ b/source/extensions/transport_sockets/internal_upstream/config.cc @@ -104,7 +104,7 @@ InternalSocketFactory::InternalSocketFactory( const envoy::extensions::transport_sockets::internal_upstream::v3::InternalUpstreamTransport& config_proto, Network::UpstreamTransportSocketFactoryPtr&& inner_factory) - : PassthroughFactory(std::move(inner_factory)), config_(config_proto, context.scope()) {} + : PassthroughFactory(std::move(inner_factory)), config_(config_proto, context.statsScope()) {} Network::TransportSocketPtr InternalSocketFactory::createTransportSocket(Network::TransportSocketOptionsConstSharedPtr options, diff --git a/source/extensions/transport_sockets/proxy_protocol/config.cc b/source/extensions/transport_sockets/proxy_protocol/config.cc index cf8f1e08b387..bdd57dfdc84f 100644 --- a/source/extensions/transport_sockets/proxy_protocol/config.cc +++ b/source/extensions/transport_sockets/proxy_protocol/config.cc @@ -27,7 +27,7 @@ UpstreamProxyProtocolSocketConfigFactory::createTransportSocketFactory( auto inner_transport_factory = inner_config_factory.createTransportSocketFactory(*inner_factory_config, context); return std::make_unique( - std::move(inner_transport_factory), outer_config.config(), context.scope()); + std::move(inner_transport_factory), outer_config.config(), context.statsScope()); } ProtobufTypes::MessagePtr UpstreamProxyProtocolSocketConfigFactory::createEmptyConfigProto() { diff --git a/source/extensions/transport_sockets/tap/config.cc b/source/extensions/transport_sockets/tap/config.cc index b84b173d2d36..3b565bee92e9 100644 --- a/source/extensions/transport_sockets/tap/config.cc +++ b/source/extensions/transport_sockets/tap/config.cc @@ -44,11 +44,14 @@ UpstreamTapSocketConfigFactory::createTransportSocketFactory( outer_config.transport_socket(), context.messageValidationVisitor(), inner_config_factory); auto inner_transport_factory = inner_config_factory.createTransportSocketFactory(*inner_factory_config, context); + + auto& server_context = context.serverFactoryContext(); return std::make_unique( outer_config, - std::make_unique(context.mainThreadDispatcher().timeSource()), - context.admin(), context.singletonManager(), context.threadLocal(), - context.mainThreadDispatcher(), std::move(inner_transport_factory)); + std::make_unique( + server_context.mainThreadDispatcher().timeSource()), + server_context.admin(), server_context.singletonManager(), server_context.threadLocal(), + server_context.mainThreadDispatcher(), std::move(inner_transport_factory)); } Network::DownstreamTransportSocketFactoryPtr @@ -65,11 +68,13 @@ DownstreamTapSocketConfigFactory::createTransportSocketFactory( outer_config.transport_socket(), context.messageValidationVisitor(), inner_config_factory); auto inner_transport_factory = inner_config_factory.createTransportSocketFactory( *inner_factory_config, context, server_names); + auto& server_context = context.serverFactoryContext(); return std::make_unique( outer_config, - std::make_unique(context.mainThreadDispatcher().timeSource()), - context.admin(), context.singletonManager(), context.threadLocal(), - context.mainThreadDispatcher(), std::move(inner_transport_factory)); + std::make_unique( + server_context.mainThreadDispatcher().timeSource()), + server_context.admin(), server_context.singletonManager(), server_context.threadLocal(), + server_context.mainThreadDispatcher(), std::move(inner_transport_factory)); } ProtobufTypes::MessagePtr TapSocketConfigFactory::createEmptyConfigProto() { diff --git a/source/extensions/transport_sockets/tcp_stats/config.cc b/source/extensions/transport_sockets/tcp_stats/config.cc index aafc940cb01f..0e362f959b1e 100644 --- a/source/extensions/transport_sockets/tcp_stats/config.cc +++ b/source/extensions/transport_sockets/tcp_stats/config.cc @@ -16,7 +16,7 @@ TcpStatsSocketFactory::TcpStatsSocketFactory( Server::Configuration::TransportSocketFactoryContext& context, const envoy::extensions::transport_sockets::tcp_stats::v3::Config& config) { #if defined(__linux__) - config_ = std::make_shared(config, context.scope()); + config_ = std::make_shared(config, context.statsScope()); #else UNREFERENCED_PARAMETER(config); UNREFERENCED_PARAMETER(context); diff --git a/source/extensions/transport_sockets/tls/config.cc b/source/extensions/transport_sockets/tls/config.cc index 75cc7b34a4ba..f2612251584d 100644 --- a/source/extensions/transport_sockets/tls/config.cc +++ b/source/extensions/transport_sockets/tls/config.cc @@ -20,8 +20,8 @@ Network::UpstreamTransportSocketFactoryPtr UpstreamSslSocketFactory::createTrans const envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext&>( message, context.messageValidationVisitor()), context); - return std::make_unique(std::move(client_config), - context.sslContextManager(), context.scope()); + return std::make_unique( + std::move(client_config), context.sslContextManager(), context.statsScope()); } ProtobufTypes::MessagePtr UpstreamSslSocketFactory::createEmptyConfigProto() { @@ -41,7 +41,7 @@ DownstreamSslSocketFactory::createTransportSocketFactory( message, context.messageValidationVisitor()), context); return std::make_unique( - std::move(server_config), context.sslContextManager(), context.scope(), server_names); + std::move(server_config), context.sslContextManager(), context.statsScope(), server_names); } ProtobufTypes::MessagePtr DownstreamSslSocketFactory::createEmptyConfigProto() { diff --git a/source/extensions/transport_sockets/tls/context_config_impl.cc b/source/extensions/transport_sockets/tls/context_config_impl.cc index f6a7e30a4f35..021c1d881fea 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.cc +++ b/source/extensions/transport_sockets/tls/context_config_impl.cc @@ -173,8 +173,9 @@ ContextConfigImpl::ContextConfigImpl( const unsigned default_min_protocol_version, const unsigned default_max_protocol_version, const std::string& default_cipher_suites, const std::string& default_curves, Server::Configuration::TransportSocketFactoryContext& factory_context) - : api_(factory_context.api()), options_(factory_context.options()), - singleton_manager_(factory_context.singletonManager()), + : api_(factory_context.serverFactoryContext().api()), + options_(factory_context.serverFactoryContext().options()), + singleton_manager_(factory_context.serverFactoryContext().singletonManager()), alpn_protocols_(RepeatedPtrUtil::join(config.alpn_protocols(), ",")), cipher_suites_(StringUtil::nonEmptyStringOrDefault( RepeatedPtrUtil::join(config.tls_params().cipher_suites(), ":"), default_cipher_suites)), diff --git a/source/extensions/transport_sockets/tls/context_config_impl.h b/source/extensions/transport_sockets/tls/context_config_impl.h index da3a526d163c..23ca329abe8e 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.h +++ b/source/extensions/transport_sockets/tls/context_config_impl.h @@ -46,7 +46,7 @@ class ContextConfigImpl : public virtual Ssl::ContextConfig { const Network::Address::IpList& tlsKeyLogRemote() const override { return tls_keylog_remote_; }; const std::string& tlsKeyLogPath() const override { return tls_keylog_path_; }; AccessLog::AccessLogManager& accessLogManager() const override { - return factory_context_.accessLogManager(); + return factory_context_.serverFactoryContext().accessLogManager(); } bool isReady() const override { diff --git a/source/extensions/upstreams/http/config.h b/source/extensions/upstreams/http/config.h index 93879e9deb6b..9e1d4c78283b 100644 --- a/source/extensions/upstreams/http/config.h +++ b/source/extensions/upstreams/http/config.h @@ -71,7 +71,7 @@ class ProtocolOptionsConfigFactory : public Server::Configuration::ProtocolOptio const envoy::extensions::upstreams::http::v3::HttpProtocolOptions&>( config, context.messageValidationVisitor()); return std::make_shared(typed_config, - context.getServerFactoryContext()); + context.serverFactoryContext()); } std::string category() const override { return "envoy.upstream_options"; } std::string name() const override { diff --git a/source/server/server.h b/source/server/server.h index d9c9f175a6dd..d5140d0e895a 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -191,6 +191,7 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, ThreadLocal::Instance& threadLocal() override { return server_.threadLocal(); } OptRef admin() override { return server_.admin(); } TimeSource& timeSource() override { return api().timeSource(); } + AccessLog::AccessLogManager& accessLogManager() override { return server_.accessLogManager(); } Api::Api& api() override { return server_.api(); } Grpc::Context& grpcContext() override { return server_.grpcContext(); } Router::Context& routerContext() override { return server_.routerContext(); } @@ -200,12 +201,11 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return server_.bootstrap(); } // Configuration::TransportSocketFactoryContext - ServerFactoryContext& getServerFactoryContext() override { return *this; }; + ServerFactoryContext& serverFactoryContext() override { return *this; } Ssl::ContextManager& sslContextManager() override { return server_.sslContextManager(); } Secret::SecretManager& secretManager() override { return server_.secretManager(); } - Stats::Store& stats() override { return server_.stats(); } + Stats::Scope& statsScope() override { return *server_scope_; } Init::Manager& initManager() override { return server_.initManager(); } - AccessLog::AccessLogManager& accessLogManager() override { return server_.accessLogManager(); } ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { // Server has two message validation visitors, one for static and // other for dynamic configuration. Choose the dynamic validation diff --git a/source/server/transport_socket_config_impl.h b/source/server/transport_socket_config_impl.h index edbb65282e19..87f9aaeac244 100644 --- a/source/server/transport_socket_config_impl.h +++ b/source/server/transport_socket_config_impl.h @@ -14,11 +14,10 @@ class TransportSocketFactoryContextImpl : public TransportSocketFactoryContext { public: TransportSocketFactoryContextImpl(Server::Configuration::ServerFactoryContext& server_context, Ssl::ContextManager& context_manager, Stats::Scope& stats_scope, - Upstream::ClusterManager& cm, Stats::Store& stats, + Upstream::ClusterManager& cm, ProtobufMessage::ValidationVisitor& validation_visitor) : server_context_(server_context), context_manager_(context_manager), - stats_scope_(stats_scope), cluster_manager_(cm), stats_(stats), - validation_visitor_(validation_visitor) {} + stats_scope_(stats_scope), cluster_manager_(cm), validation_visitor_(validation_visitor) {} /** * Pass an init manager to register dynamic secret provider. @@ -27,42 +26,29 @@ class TransportSocketFactoryContextImpl : public TransportSocketFactoryContext { void setInitManager(Init::Manager& init_manager) { init_manager_ = &init_manager; } // TransportSocketFactoryContext - ServerFactoryContext& getServerFactoryContext() override { return server_context_; }; - OptRef admin() override { return server_context_.admin(); } + ServerFactoryContext& serverFactoryContext() override { return server_context_; } + Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } + ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { + return validation_visitor_; + } Ssl::ContextManager& sslContextManager() override { return context_manager_; } - Stats::Scope& scope() override { return stats_scope_; } + Stats::Scope& statsScope() override { return stats_scope_; } Secret::SecretManager& secretManager() override { return clusterManager().clusterManagerFactory().secretManager(); } - Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } - const LocalInfo::LocalInfo& localInfo() const override { return server_context_.localInfo(); } - Event::Dispatcher& mainThreadDispatcher() override { - return server_context_.mainThreadDispatcher(); - } - Stats::Store& stats() override { return stats_; } Init::Manager& initManager() override { ASSERT(init_manager_ != nullptr); return *init_manager_; } - Singleton::Manager& singletonManager() override { return server_context_.singletonManager(); } - ThreadLocal::SlotAllocator& threadLocal() override { return server_context_.threadLocal(); } - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { - return validation_visitor_; - } - Api::Api& api() override { return server_context_.api(); } - const Server::Options& options() override { return server_context_.options(); } - AccessLog::AccessLogManager& accessLogManager() override { - return server_context_.accessLogManager(); - } private: Server::Configuration::ServerFactoryContext& server_context_; Ssl::ContextManager& context_manager_; Stats::Scope& stats_scope_; Upstream::ClusterManager& cluster_manager_; - Stats::Store& stats_; - Init::Manager* init_manager_{}; ProtobufMessage::ValidationVisitor& validation_visitor_; + + Init::Manager* init_manager_{}; }; using TransportSocketFactoryContextImplPtr = std::unique_ptr; diff --git a/test/common/grpc/grpc_client_integration_test_harness.h b/test/common/grpc/grpc_client_integration_test_harness.h index 09a13a982b53..a30067cb190a 100644 --- a/test/common/grpc/grpc_client_integration_test_harness.h +++ b/test/common/grpc/grpc_client_integration_test_harness.h @@ -496,7 +496,7 @@ class GrpcClientIntegrationTest : public GrpcClientIntegrationParamTest { class GrpcSslClientIntegrationTest : public GrpcClientIntegrationTest { public: GrpcSslClientIntegrationTest() { - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); } void TearDown() override { // Reset some state in the superclass before we destruct context_manager_ in our destructor, it diff --git a/test/common/http/http3/conn_pool_test.cc b/test/common/http/http3/conn_pool_test.cc index 69d5e7a85e62..17ed4ac9b2a4 100644 --- a/test/common/http/http3/conn_pool_test.cc +++ b/test/common/http/http3/conn_pool_test.cc @@ -122,6 +122,9 @@ TEST_F(Http3ConnPoolImplTest, FastFailWithoutSecretsLoaded) { } TEST_F(Http3ConnPoolImplTest, FailWithSecretsBecomeEmpty) { + testing::NiceMock mock_store; + ON_CALL(context_, statsScope()).WillByDefault(testing::ReturnRef(*mock_store.rootScope())); + MockQuicClientTransportSocketFactory factory{ std::unique_ptr(new NiceMock), context_}; @@ -144,7 +147,7 @@ TEST_F(Http3ConnPoolImplTest, FailWithSecretsBecomeEmpty) { MockResponseDecoder decoder; ConnPoolCallbacks callbacks; - EXPECT_CALL(context_.store_.counter_, inc()); + EXPECT_CALL(mock_store.counter_, inc()); EXPECT_CALL(callbacks.pool_failure_, ready()); EXPECT_EQ(pool->newStream(decoder, callbacks, {/*can_send_early_data_=*/false, diff --git a/test/common/quic/quic_transport_socket_factory_test.cc b/test/common/quic/quic_transport_socket_factory_test.cc index 0b28fc060978..89aa18174ab1 100644 --- a/test/common/quic/quic_transport_socket_factory_test.cc +++ b/test/common/quic/quic_transport_socket_factory_test.cc @@ -17,7 +17,7 @@ class QuicServerTransportSocketFactoryConfigTest : public Event::TestUsingSimula public: QuicServerTransportSocketFactoryConfigTest() : server_api_(Api::createApiForTest(server_stats_store_, simTime())) { - ON_CALL(context_, api()).WillByDefault(ReturnRef(*server_api_)); + ON_CALL(context_.server_context_, api()).WillByDefault(ReturnRef(*server_api_)); } void verifyQuicServerTransportSocketFactory(std::string yaml, bool expect_early_data) { diff --git a/test/common/secret/secret_manager_impl_test.cc b/test/common/secret/secret_manager_impl_test.cc index daca620733e7..ead2895c984b 100644 --- a/test/common/secret/secret_manager_impl_test.cc +++ b/test/common/secret/secret_manager_impl_test.cc @@ -270,7 +270,6 @@ TEST_F(SecretManagerImplTest, DeduplicateDynamicTlsCertificateSecretProvider) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -278,10 +277,10 @@ TEST_F(SecretManagerImplTest, DeduplicateDynamicTlsCertificateSecretProvider) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); envoy::config::core::v3::ConfigSource config_source; TestUtility::loadFromYaml(R"( @@ -353,7 +352,6 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretUpdateSuccess) { envoy::config::core::v3::ConfigSource config_source; NiceMock local_info; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -361,11 +359,11 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretUpdateSuccess) { .WillOnce(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); - EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api_)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); @@ -403,18 +401,17 @@ TEST_F(SecretManagerImplTest, SdsDynamicGenericSecret) { NiceMock secret_context; NiceMock validation_visitor; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock local_info; Init::TargetHandlePtr init_target_handle; NiceMock init_watcher; - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(*dispatcher_)); EXPECT_CALL(secret_context, messageValidationVisitor()).WillOnce(ReturnRef(validation_visitor)); - EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api_)); EXPECT_CALL(init_manager, add(_)) .WillOnce(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); @@ -452,7 +449,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandler) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -460,10 +456,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandler) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); @@ -722,7 +718,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerWarmingSecrets) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -730,10 +725,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerWarmingSecrets) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); @@ -870,7 +865,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSecrets) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -878,10 +872,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSecrets) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); const std::string tls_certificate = R"EOF( @@ -947,7 +941,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticValidationContext) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -955,10 +948,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticValidationContext) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); const std::string validation_context = R"EOF( @@ -995,7 +988,6 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSessionTicketsContext) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -1003,10 +995,10 @@ TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSessionTicketsContext) { .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); const std::string stek_context = R"EOF( @@ -1076,7 +1068,6 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretPrivateKeyProviderUpdateSuccess) { envoy::config::core::v3::ConfigSource config_source; NiceMock local_info; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock init_watcher; Init::TargetHandlePtr init_target_handle; @@ -1084,11 +1075,11 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretPrivateKeyProviderUpdateSuccess) { .WillOnce(Invoke([&init_target_handle](const Init::Target& target) { init_target_handle = target.createHandle("test"); })); - EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).Times(0); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); - EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api_)); auto secret_provider = secret_manager->findOrCreateTlsCertificateProvider( config_source, "abc.com", secret_context, init_manager); diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 8bebd8275dc3..d52e677e5fec 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -60,6 +60,7 @@ envoy_cc_test( envoy_cc_test( name = "cluster_manager_impl_test", + size = "large", srcs = ["cluster_manager_impl_test.cc"], args = [ # Force creation of c-ares DnsResolverImpl when running test on macOS. diff --git a/test/common/upstream/hds_test.cc b/test/common/upstream/hds_test.cc index 8fdc4eead1d1..e3ee0677e7c1 100644 --- a/test/common/upstream/hds_test.cc +++ b/test/common/upstream/hds_test.cc @@ -583,7 +583,7 @@ TEST_F(HdsTest, TestSocketContext) { params.stats_.createScope(fmt::format("cluster.{}.", params.cluster_.name())); Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( params.server_context_, params.ssl_context_manager_, *scope, - params.server_context_.clusterManager(), params.stats_, + params.server_context_.clusterManager(), params.server_context_.messageValidationVisitor()); // Create a mock socket_factory for the scope of this unit test. @@ -1072,7 +1072,7 @@ TEST_F(HdsTest, TestUpdateSocketContext) { params.stats_.createScope(fmt::format("cluster.{}.", params.cluster_.name())); Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( params.server_context_, params.ssl_context_manager_, *scope, - params.server_context_.clusterManager(), params.stats_, + params.server_context_.clusterManager(), params.server_context_.messageValidationVisitor()); // Create a mock socket_factory for the scope of this unit test. diff --git a/test/extensions/clusters/eds/leds_test.cc b/test/extensions/clusters/eds/leds_test.cc index 8e30b3fbeb11..14a0fc5c0958 100644 --- a/test/extensions/clusters/eds/leds_test.cc +++ b/test/extensions/clusters/eds/leds_test.cc @@ -90,7 +90,7 @@ class LedsTest : public testing::Test { cluster_scope_ = stats_.createScope("cluster.xds_cluster."); Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( server_context_, ssl_context_manager_, *cluster_scope_, server_context_.cluster_manager_, - stats_, validation_visitor_); + validation_visitor_); // Setup LEDS subscription. EXPECT_CALL(server_context_.cluster_manager_.subscription_factory_, diff --git a/test/extensions/filters/http/oauth2/filter_test.cc b/test/extensions/filters/http/oauth2/filter_test.cc index ddf8f16f4fb5..970bbf360e81 100644 --- a/test/extensions/filters/http/oauth2/filter_test.cc +++ b/test/extensions/filters/http/oauth2/filter_test.cc @@ -207,14 +207,13 @@ TEST_F(OAuth2Test, SdsDynamicGenericSecret) { NiceMock secret_context; NiceMock local_info; Api::ApiPtr api = Api::createApiForTest(); - Stats::IsolatedStoreImpl stats; NiceMock init_manager; Init::TargetHandlePtr init_handle; NiceMock dispatcher; - EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); - EXPECT_CALL(secret_context, api()).WillRepeatedly(ReturnRef(*api)); - EXPECT_CALL(secret_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); + EXPECT_CALL(secret_context.server_context_, localInfo()).WillRepeatedly(ReturnRef(local_info)); + EXPECT_CALL(secret_context.server_context_, api()).WillRepeatedly(ReturnRef(*api)); + EXPECT_CALL(secret_context.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); EXPECT_CALL(secret_context, initManager()).Times(0); EXPECT_CALL(init_manager, add(_)) .WillRepeatedly(Invoke([&init_handle](const Init::Target& target) { diff --git a/test/extensions/transport_sockets/alts/alts_integration_test.cc b/test/extensions/transport_sockets/alts/alts_integration_test.cc index 570646820764..4cc0b8f435f7 100644 --- a/test/extensions/transport_sockets/alts/alts_integration_test.cc +++ b/test/extensions/transport_sockets/alts/alts_integration_test.cc @@ -160,7 +160,7 @@ class AltsIntegrationTestBase : public Event::TestUsingSimulatedTime, } }; FakeSingletonManager fsm; - ON_CALL(mock_factory_ctx, singletonManager()).WillByDefault(ReturnRef(fsm)); + ON_CALL(mock_factory_ctx.server_context_, singletonManager()).WillByDefault(ReturnRef(fsm)); UpstreamAltsTransportSocketConfigFactory factory; envoy::extensions::transport_sockets::alts::v3::Alts alts_config; diff --git a/test/extensions/transport_sockets/alts/config_test.cc b/test/extensions/transport_sockets/alts/config_test.cc index 625994ba5404..a848115facf8 100644 --- a/test/extensions/transport_sockets/alts/config_test.cc +++ b/test/extensions/transport_sockets/alts/config_test.cc @@ -19,7 +19,8 @@ namespace { TEST(UpstreamAltsConfigTest, CreateSocketFactory) { NiceMock factory_context; Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; - EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); + EXPECT_CALL(factory_context.server_context_, singletonManager()) + .WillRepeatedly(ReturnRef(singleton_manager)); UpstreamAltsTransportSocketConfigFactory factory; ProtobufTypes::MessagePtr config = factory.createEmptyConfigProto(); @@ -39,7 +40,8 @@ TEST(UpstreamAltsConfigTest, CreateSocketFactory) { TEST(DownstreamAltsConfigTest, CreateSocketFactory) { NiceMock factory_context; Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; - EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); + EXPECT_CALL(factory_context.server_context_, singletonManager()) + .WillRepeatedly(ReturnRef(singleton_manager)); DownstreamAltsTransportSocketConfigFactory factory; ProtobufTypes::MessagePtr config = factory.createEmptyConfigProto(); diff --git a/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc b/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc index 232b95c1acde..fe298d275640 100644 --- a/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc +++ b/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc @@ -269,7 +269,7 @@ void StartTlsIntegrationTest::initialize() { TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), downstream_tls_context); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); auto cfg = std::make_unique( downstream_tls_context, mock_factory_ctx); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); diff --git a/test/extensions/transport_sockets/tls/BUILD b/test/extensions/transport_sockets/tls/BUILD index 375bbb3072e3..098ba510db74 100644 --- a/test/extensions/transport_sockets/tls/BUILD +++ b/test/extensions/transport_sockets/tls/BUILD @@ -13,6 +13,7 @@ envoy_package() envoy_cc_test( name = "ssl_socket_test", + size = "large", srcs = [ "ssl_certs_test.h", "ssl_socket_test.cc", diff --git a/test/extensions/transport_sockets/tls/context_impl_test.cc b/test/extensions/transport_sockets/tls/context_impl_test.cc index 4f082a192725..8e6bed283c68 100644 --- a/test/extensions/transport_sockets/tls/context_impl_test.cc +++ b/test/extensions/transport_sockets/tls/context_impl_test.cc @@ -955,13 +955,12 @@ TEST_F(SslServerContextImplTicketTest, TicketKeySdsNotReady) { NiceMock local_info; NiceMock dispatcher; NiceMock random; - Stats::IsolatedStoreImpl stats; NiceMock cluster_manager; NiceMock init_manager; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); // EXPECT_CALL(factory_context_, random()).WillOnce(ReturnRef(random)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, clusterManager()).WillOnce(ReturnRef(cluster_manager)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); auto* sds_secret_configs = tls_context.mutable_session_ticket_keys_sds_secret_config(); @@ -1408,13 +1407,12 @@ TEST_F(ClientContextConfigImplTest, TlsCertificatesAndSdsConfig) { TEST_F(ClientContextConfigImplTest, SecretNotReady) { envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; NiceMock local_info; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); @@ -1440,13 +1438,12 @@ TEST_F(ClientContextConfigImplTest, ValidationContextNotReady) { client_cert->mutable_private_key()->set_filename(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_key.pem")); NiceMock local_info; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_validation_context_sds_secret_config(); sds_secret_configs->set_name("abc.com"); @@ -1847,13 +1844,12 @@ TEST_F(ServerContextConfigImplTest, MultiSdsConfig) { TEST_F(ServerContextConfigImplTest, SecretNotReady) { envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; NiceMock local_info; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); @@ -1879,13 +1875,12 @@ TEST_F(ServerContextConfigImplTest, ValidationContextNotReady) { server_cert->mutable_private_key()->set_filename(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_key.pem")); NiceMock local_info; - Stats::IsolatedStoreImpl stats; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_validation_context_sds_secret_config(); sds_secret_configs->set_name("abc.com"); diff --git a/test/extensions/transport_sockets/tls/ssl_certs_test.h b/test/extensions/transport_sockets/tls/ssl_certs_test.h index 0fe7d2183b27..fca49c721c7d 100644 --- a/test/extensions/transport_sockets/tls/ssl_certs_test.h +++ b/test/extensions/transport_sockets/tls/ssl_certs_test.h @@ -12,7 +12,7 @@ namespace Envoy { class SslCertsTest : public testing::Test { protected: SslCertsTest() : api_(Api::createApiForTest(store_, time_system_)) { - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); } Event::SimulatedTimeSystem time_system_; diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 3a8d1bd817fc..65b366f01606 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -329,7 +329,7 @@ void testUtil(const TestUtilOptions& options) { NiceMock runtime; testing::NiceMock server_factory_context; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); // For private key method testing. NiceMock context_manager; @@ -370,7 +370,7 @@ void testUtil(const TestUtilOptions& options) { Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_tls_context, client_factory_context); @@ -684,7 +684,7 @@ void testUtilV2(const TestUtilOptionsV2& options) { testing::NiceMock server_factory_context; NiceMock runtime; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; const envoy::config::core::v3::TransportSocket& transport_socket = @@ -708,7 +708,7 @@ void testUtilV2(const TestUtilOptionsV2& options) { Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(options.clientCtxProto(), client_factory_context); @@ -3420,7 +3420,7 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, NiceMock runtime; testing::NiceMock server_factory_context; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext server_tls_context1; TestUtility::loadFromYaml(TestEnvironment::substitute(server_ctx_yaml1), server_tls_context1); @@ -3454,7 +3454,7 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_tls_context, client_factory_context); @@ -3572,7 +3572,7 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml NiceMock runtime; testing::NiceMock server_factory_context; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext server_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(server_ctx_yaml), server_tls_context); @@ -3595,7 +3595,7 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_tls_context, client_factory_context); @@ -4294,7 +4294,7 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya Api::ApiPtr server_api = Api::createApiForTest(server_stats_store, time_system_); testing::NiceMock server_factory_context; - ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + ON_CALL(server_factory_context.server_context_, api()).WillByDefault(ReturnRef(*server_api)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext server_ctx_proto; TestUtility::loadFromYaml(TestEnvironment::substitute(server_ctx_yaml), server_ctx_proto); @@ -4321,7 +4321,7 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system_); testing::NiceMock client_factory_context; - ON_CALL(client_factory_context, api()).WillByDefault(ReturnRef(*client_api)); + ON_CALL(client_factory_context.server_context_, api()).WillByDefault(ReturnRef(*client_api)); auto client_cfg = std::make_unique(client_ctx_proto, client_factory_context); @@ -5683,28 +5683,28 @@ TEST_P(SslSocketTest, OverrideApplicationProtocols) { // Validate that if downstream secrets are not yet downloaded from SDS server, Envoy creates // NotReadySslSocket object to handle downstream connection. TEST_P(SslSocketTest, DownstreamNotReadySslSocket) { - Stats::TestUtil::TestStore stats_store; NiceMock local_info; - testing::NiceMock factory_context; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); - EXPECT_CALL(factory_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context, stats()).WillOnce(ReturnRef(stats_store)); - EXPECT_CALL(factory_context, initManager()).WillRepeatedly(ReturnRef(init_manager)); + + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); sds_secret_configs->mutable_sds_config(); - auto server_cfg = std::make_unique(tls_context, factory_context); + auto server_cfg = std::make_unique(tls_context, factory_context_); EXPECT_TRUE(server_cfg->tlsCertificates().empty()); EXPECT_FALSE(server_cfg->isReady()); ContextManagerImpl manager(time_system_); - ServerSslSocketFactory server_ssl_socket_factory( - std::move(server_cfg), manager, *stats_store.rootScope(), std::vector{}); + ServerSslSocketFactory server_ssl_socket_factory(std::move(server_cfg), manager, + *factory_context_.store_.rootScope(), + std::vector{}); auto transport_socket = server_ssl_socket_factory.createDownstreamTransportSocket(); EXPECT_FALSE(transport_socket->startSecureTransport()); // Noop transport_socket->configureInitialCongestionWindow(200, std::chrono::microseconds(223)); // Noop @@ -5722,28 +5722,26 @@ TEST_P(SslSocketTest, DownstreamNotReadySslSocket) { // Validate that if upstream secrets are not yet downloaded from SDS server, Envoy creates // NotReadySslSocket object to handle upstream connection. TEST_P(SslSocketTest, UpstreamNotReadySslSocket) { - Stats::TestUtil::TestStore stats_store; NiceMock local_info; - testing::NiceMock factory_context; NiceMock init_manager; NiceMock dispatcher; - EXPECT_CALL(factory_context, localInfo()).WillOnce(ReturnRef(local_info)); - EXPECT_CALL(factory_context, stats()).WillOnce(ReturnRef(stats_store)); - EXPECT_CALL(factory_context, initManager()).WillRepeatedly(ReturnRef(init_manager)); - EXPECT_CALL(factory_context, mainThreadDispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(factory_context_.server_context_, localInfo()).WillOnce(ReturnRef(local_info)); + EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(ReturnRef(init_manager)); + EXPECT_CALL(factory_context_.server_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(dispatcher)); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); sds_secret_configs->mutable_sds_config(); - auto client_cfg = std::make_unique(tls_context, factory_context); + auto client_cfg = std::make_unique(tls_context, factory_context_); EXPECT_TRUE(client_cfg->tlsCertificates().empty()); EXPECT_FALSE(client_cfg->isReady()); ContextManagerImpl manager(time_system_); ClientSslSocketFactory client_ssl_socket_factory(std::move(client_cfg), manager, - *stats_store.rootScope()); + *factory_context_.store_.rootScope()); auto transport_socket = client_ssl_socket_factory.createTransportSocket(nullptr, nullptr); EXPECT_EQ(EMPTY_STRING, transport_socket->protocol()); EXPECT_EQ(nullptr, transport_socket->ssl()); @@ -5763,18 +5761,15 @@ TEST_P(SslSocketTest, TestTransportSocketCallback) { ON_CALL(callbacks, ioHandle()).WillByDefault(ReturnRef(io_handle)); // Make SslSocket. - testing::NiceMock factory_context; - Stats::TestUtil::TestStore stats_store; - ON_CALL(factory_context, stats()).WillByDefault(ReturnRef(stats_store)); NiceMock local_info; - ON_CALL(factory_context, localInfo()).WillByDefault(ReturnRef(local_info)); + ON_CALL(factory_context_.server_context_, localInfo()).WillByDefault(ReturnRef(local_info)); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; - auto client_cfg = std::make_unique(tls_context, factory_context); + auto client_cfg = std::make_unique(tls_context, factory_context_); ContextManagerImpl manager(time_system_); ClientSslSocketFactory client_ssl_socket_factory(std::move(client_cfg), manager, - *stats_store.rootScope()); + *factory_context_.store_.rootScope()); Network::TransportSocketPtr transport_socket = client_ssl_socket_factory.createTransportSocket(nullptr, nullptr); diff --git a/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc index 6cae75d1db06..1f01175716ea 100644 --- a/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc +++ b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc @@ -350,7 +350,8 @@ TestPrivateKeyMethodProvider::TestPrivateKeyMethodProvider( } } - std::string private_key = factory_context.api().fileSystem().fileReadToEnd(private_key_path); + std::string private_key = + factory_context.serverFactoryContext().api().fileSystem().fileReadToEnd(private_key_path); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); bssl::UniquePtr pkey(PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, nullptr)); diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 97f9710ddbda..4b45cb7e0f39 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -61,8 +61,8 @@ BaseIntegrationTest::BaseIntegrationTest(const InstanceConstSharedPtrFn& upstrea std::function above_overflow) -> Buffer::Instance* { return new Buffer::WatermarkBuffer(below_low, above_high, above_overflow); })); - ON_CALL(factory_context_, api()).WillByDefault(ReturnRef(*api_)); - ON_CALL(factory_context_, scope()).WillByDefault(ReturnRef(*stats_store_.rootScope())); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(factory_context_, statsScope()).WillByDefault(ReturnRef(*stats_store_.rootScope())); // Allow extension lookup by name in the integration tests. config_helper_.addRuntimeOverride("envoy.reloadable_features.no_extension_lookup_by_name", "false"); diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 5a934afd78f4..3fadbfd2af8b 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -262,8 +262,8 @@ class QuicHttpIntegrationTestBase : public HttpIntegrationTest { // Initialize the transport socket factory using a customized ssl option. ssl_client_option_.setAlpn(true).setSan(san_to_match_).setSni("lyft.com"); NiceMock context; - ON_CALL(context, api()).WillByDefault(testing::ReturnRef(*api_)); - ON_CALL(context, scope()).WillByDefault(testing::ReturnRef(stats_scope_)); + ON_CALL(context.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(context, statsScope()).WillByDefault(testing::ReturnRef(stats_scope_)); ON_CALL(context, sslContextManager()).WillByDefault(testing::ReturnRef(context_manager_)); envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport quic_transport_socket_config; diff --git a/test/integration/ssl_utility.cc b/test/integration/ssl_utility.cc index bd8e0057b067..af93ec2c3a48 100644 --- a/test/integration/ssl_utility.cc +++ b/test/integration/ssl_utility.cc @@ -117,7 +117,7 @@ createClientSslTransportSocketFactory(const ClientSslTransportOptions& options, initializeUpstreamTlsContextConfig(options, tls_context); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx, api()).WillByDefault(ReturnRef(api)); + ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(ReturnRef(api)); auto cfg = std::make_unique( tls_context, mock_factory_ctx); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); @@ -132,7 +132,7 @@ createUpstreamSslContext(ContextManager& context_manager, Api::Api& api, bool us ConfigHelper::initializeTls({}, *tls_context.mutable_common_tls_context()); NiceMock mock_factory_ctx; - ON_CALL(mock_factory_ctx, api()).WillByDefault(ReturnRef(api)); + ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(ReturnRef(api)); auto cfg = std::make_unique( tls_context, mock_factory_ctx); diff --git a/test/integration/utility.cc b/test/integration/utility.cc index db03dded6453..b19dd19bd264 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -135,8 +135,8 @@ IntegrationUtil::createQuicUpstreamTransportSocketFactory(Api::Api& api, Stats:: Ssl::ContextManager& context_manager, const std::string& san_to_match) { NiceMock context; - ON_CALL(context, api()).WillByDefault(testing::ReturnRef(api)); - ON_CALL(context, scope()).WillByDefault(testing::ReturnRef(*store.rootScope())); + ON_CALL(context.server_context_, api()).WillByDefault(testing::ReturnRef(api)); + ON_CALL(context, statsScope()).WillByDefault(testing::ReturnRef(*store.rootScope())); ON_CALL(context, sslContextManager()).WillByDefault(testing::ReturnRef(context_manager)); envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport quic_transport_socket_config; diff --git a/test/integration/xfcc_integration_test.h b/test/integration/xfcc_integration_test.h index def7807d94b2..c787eaa69497 100644 --- a/test/integration/xfcc_integration_test.h +++ b/test/integration/xfcc_integration_test.h @@ -40,7 +40,7 @@ class XfccIntegrationTest : public testing::TestWithParam{admin_})); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); - ON_CALL(*this, timeSource()).WillByDefault(ReturnRef(time_system_)); - ON_CALL(*this, messageValidationContext()).WillByDefault(ReturnRef(validation_context_)); - ON_CALL(*this, messageValidationVisitor()) - .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); - ON_CALL(*this, drainManager()).WillByDefault(ReturnRef(drain_manager_)); - ON_CALL(*this, statsConfig()).WillByDefault(ReturnRef(stats_config_)); - ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); - ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); - ON_CALL(*this, lifecycleNotifier()).WillByDefault(ReturnRef(lifecycle_notifier_)); - ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); -} -MockServerFactoryContext::~MockServerFactoryContext() = default; - -MockStatsConfig::MockStatsConfig() = default; -MockStatsConfig::~MockStatsConfig() = default; - -} // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/instance.h b/test/mocks/server/instance.h index e8d077a1624e..93b5d4fe43d4 100644 --- a/test/mocks/server/instance.h +++ b/test/mocks/server/instance.h @@ -2,44 +2,13 @@ #include "envoy/server/instance.h" -#include "source/common/grpc/context_impl.h" -#include "source/common/http/context_impl.h" -#include "source/common/quic/quic_stat_names.h" -#include "source/common/router/context_impl.h" -#include "source/common/stats/symbol_table.h" -#include "source/extensions/transport_sockets/tls/context_manager_impl.h" +#include "test/mocks/server/server_factory_context.h" +#include "test/mocks/server/transport_socket_factory_context.h" -#include "test/mocks/access_log/mocks.h" -#include "test/mocks/api/mocks.h" -#include "test/mocks/event/mocks.h" -#include "test/mocks/http/mocks.h" -#include "test/mocks/init/mocks.h" -#include "test/mocks/local_info/mocks.h" -#include "test/mocks/network/mocks.h" -#include "test/mocks/protobuf/mocks.h" -#include "test/mocks/runtime/mocks.h" -#include "test/mocks/secret/mocks.h" -#include "test/mocks/stats/mocks.h" -#include "test/mocks/thread_local/mocks.h" -#include "test/mocks/tracing/mocks.h" -#include "test/mocks/upstream/cluster_manager.h" - -#include "admin.h" -#include "drain_manager.h" #include "gmock/gmock.h" -#include "hot_restart.h" -#include "listener_manager.h" -#include "options.h" -#include "overload_manager.h" -#include "server_lifecycle_notifier.h" -#include "transport_socket_factory_context.h" namespace Envoy { namespace Server { -namespace Configuration { -class MockServerFactoryContext; -class MockStatsConfig; -} // namespace Configuration class MockInstance : public Instance { public: @@ -133,104 +102,5 @@ class MockInstance : public Instance { transport_socket_factory_context_; }; -namespace Configuration { -class MockStatsConfig : public virtual StatsConfig { -public: - MockStatsConfig(); - ~MockStatsConfig() override; - - MOCK_METHOD(const std::list&, sinks, (), (const)); - MOCK_METHOD(std::chrono::milliseconds, flushInterval, (), (const)); - MOCK_METHOD(bool, flushOnAdmin, (), (const)); - MOCK_METHOD(const Stats::SinkPredicates*, sinkPredicates, (), (const)); -}; - -class MockServerFactoryContext : public virtual ServerFactoryContext { -public: - MockServerFactoryContext(); - ~MockServerFactoryContext() override; - - MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); - MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); - MOCK_METHOD(Stats::Scope&, scope, ()); - MOCK_METHOD(Stats::Scope&, serverScope, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); - MOCK_METHOD(OptRef, admin, ()); - MOCK_METHOD(TimeSource&, timeSource, ()); - Event::TestTimeSystem& timeSystem() { return time_system_; } - MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); - Grpc::Context& grpcContext() override { return grpc_context_; } - Router::Context& routerContext() override { return router_context_; } - envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return bootstrap_; } - MOCK_METHOD(Server::DrainManager&, drainManager, ()); - MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); - MOCK_METHOD(StatsConfig&, statsConfig, (), ()); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); - - testing::NiceMock cluster_manager_; - testing::NiceMock dispatcher_; - testing::NiceMock drain_manager_; - testing::NiceMock local_info_; - testing::NiceMock runtime_loader_; - testing::NiceMock store_; - testing::NiceMock thread_local_; - testing::NiceMock validation_context_; - testing::NiceMock stats_config_; - testing::NiceMock access_log_manager_; - testing::NiceMock init_manager_; - testing::NiceMock lifecycle_notifier_; - - Singleton::ManagerPtr singleton_manager_; - testing::NiceMock admin_; - Event::GlobalTimeSystem time_system_; - testing::NiceMock api_; - Grpc::ContextImpl grpc_context_; - Router::ContextImpl router_context_; - envoy::config::bootstrap::v3::Bootstrap bootstrap_; - testing::NiceMock options_; -}; - -// Stateless mock ServerFactoryContext for cases where it needs to be used concurrently in different -// threads. Global state in the MockServerFactoryContext causes thread safety issues in this case. -class StatelessMockServerFactoryContext : public virtual ServerFactoryContext { -public: - StatelessMockServerFactoryContext() = default; - ~StatelessMockServerFactoryContext() override = default; - - MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); - MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); - MOCK_METHOD(Stats::Scope&, scope, ()); - MOCK_METHOD(Stats::Scope&, serverScope, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); - MOCK_METHOD(OptRef, admin, ()); - MOCK_METHOD(TimeSource&, timeSource, ()); - MOCK_METHOD(Event::TestTimeSystem&, timeSystem, ()); - MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); - MOCK_METHOD(Grpc::Context&, grpcContext, ()); - MOCK_METHOD(Router::Context&, routerContext, ()); - MOCK_METHOD(envoy::config::bootstrap::v3::Bootstrap&, bootstrap, ()); - MOCK_METHOD(Server::DrainManager&, drainManager, ()); - MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); - MOCK_METHOD(StatsConfig&, statsConfig, (), ()); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); -}; - -} // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/server_factory_context.cc b/test/mocks/server/server_factory_context.cc new file mode 100644 index 000000000000..0acd1b671908 --- /dev/null +++ b/test/mocks/server/server_factory_context.cc @@ -0,0 +1,43 @@ +#include "test/mocks/server/server_factory_context.h" + +namespace Envoy { +namespace Server { +namespace Configuration { + +using ::testing::Return; +using ::testing::ReturnRef; + +MockServerFactoryContext::MockServerFactoryContext() + : singleton_manager_(new Singleton::ManagerImpl(Thread::threadFactoryForTest())), + grpc_context_(store_.symbolTable()), router_context_(store_.symbolTable()) { + ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); + ON_CALL(*this, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); + ON_CALL(*this, drainDecision()).WillByDefault(ReturnRef(drain_manager_)); + ON_CALL(*this, localInfo()).WillByDefault(ReturnRef(local_info_)); + ON_CALL(*this, runtime()).WillByDefault(ReturnRef(runtime_loader_)); + ON_CALL(*this, scope()).WillByDefault(ReturnRef(*store_.rootScope())); + ON_CALL(*this, serverScope()).WillByDefault(ReturnRef(*store_.rootScope())); + ON_CALL(*this, singletonManager()).WillByDefault(ReturnRef(*singleton_manager_)); + ON_CALL(*this, threadLocal()).WillByDefault(ReturnRef(thread_local_)); + ON_CALL(*this, admin()).WillByDefault(Return(OptRef{admin_})); + ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); + ON_CALL(*this, timeSource()).WillByDefault(ReturnRef(time_system_)); + ON_CALL(*this, messageValidationContext()).WillByDefault(ReturnRef(validation_context_)); + ON_CALL(*this, messageValidationVisitor()) + .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); + ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); + ON_CALL(*this, drainManager()).WillByDefault(ReturnRef(drain_manager_)); + ON_CALL(*this, statsConfig()).WillByDefault(ReturnRef(stats_config_)); + ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); + ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); + ON_CALL(*this, lifecycleNotifier()).WillByDefault(ReturnRef(lifecycle_notifier_)); + ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); +} +MockServerFactoryContext::~MockServerFactoryContext() = default; + +MockStatsConfig::MockStatsConfig() = default; +MockStatsConfig::~MockStatsConfig() = default; + +} // namespace Configuration +} // namespace Server +} // namespace Envoy diff --git a/test/mocks/server/server_factory_context.h b/test/mocks/server/server_factory_context.h new file mode 100644 index 000000000000..77a2610680c0 --- /dev/null +++ b/test/mocks/server/server_factory_context.h @@ -0,0 +1,140 @@ +#pragma once + +#include "envoy/server/factory_context.h" + +#include "source/common/grpc/context_impl.h" +#include "source/common/http/context_impl.h" +#include "source/common/quic/quic_stat_names.h" +#include "source/common/router/context_impl.h" +#include "source/common/stats/symbol_table.h" +#include "source/extensions/transport_sockets/tls/context_manager_impl.h" + +#include "test/mocks/access_log/mocks.h" +#include "test/mocks/api/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/secret/mocks.h" +#include "test/mocks/server/admin.h" +#include "test/mocks/server/drain_manager.h" +#include "test/mocks/server/hot_restart.h" +#include "test/mocks/server/listener_manager.h" +#include "test/mocks/server/options.h" +#include "test/mocks/server/overload_manager.h" +#include "test/mocks/server/server_lifecycle_notifier.h" +#include "test/mocks/stats/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Server { +namespace Configuration { + +class MockStatsConfig : public virtual StatsConfig { +public: + MockStatsConfig(); + ~MockStatsConfig() override; + + MOCK_METHOD(const std::list&, sinks, (), (const)); + MOCK_METHOD(std::chrono::milliseconds, flushInterval, (), (const)); + MOCK_METHOD(bool, flushOnAdmin, (), (const)); + MOCK_METHOD(const Stats::SinkPredicates*, sinkPredicates, (), (const)); +}; + +class MockServerFactoryContext : public virtual ServerFactoryContext { +public: + MockServerFactoryContext(); + ~MockServerFactoryContext() override; + + MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); + MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); + MOCK_METHOD(const Server::Options&, options, ()); + MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); + MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); + MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); + MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(Stats::Scope&, serverScope, ()); + MOCK_METHOD(Singleton::Manager&, singletonManager, ()); + MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); + MOCK_METHOD(OptRef, admin, ()); + MOCK_METHOD(TimeSource&, timeSource, ()); + Event::TestTimeSystem& timeSystem() { return time_system_; } + MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); + MOCK_METHOD(Api::Api&, api, ()); + Grpc::Context& grpcContext() override { return grpc_context_; } + Router::Context& routerContext() override { return router_context_; } + envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return bootstrap_; } + MOCK_METHOD(Server::DrainManager&, drainManager, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); + MOCK_METHOD(StatsConfig&, statsConfig, (), ()); + MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); + + testing::NiceMock cluster_manager_; + testing::NiceMock dispatcher_; + testing::NiceMock drain_manager_; + testing::NiceMock local_info_; + testing::NiceMock runtime_loader_; + testing::NiceMock store_; + testing::NiceMock thread_local_; + testing::NiceMock validation_context_; + testing::NiceMock stats_config_; + testing::NiceMock access_log_manager_; + testing::NiceMock init_manager_; + testing::NiceMock lifecycle_notifier_; + + Singleton::ManagerPtr singleton_manager_; + testing::NiceMock admin_; + Event::GlobalTimeSystem time_system_; + testing::NiceMock api_; + Grpc::ContextImpl grpc_context_; + Router::ContextImpl router_context_; + envoy::config::bootstrap::v3::Bootstrap bootstrap_; + testing::NiceMock options_; +}; + +// Stateless mock ServerFactoryContext for cases where it needs to be used concurrently in different +// threads. Global state in the MockServerFactoryContext causes thread safety issues in this case. +class StatelessMockServerFactoryContext : public virtual ServerFactoryContext { +public: + StatelessMockServerFactoryContext() = default; + ~StatelessMockServerFactoryContext() override = default; + + MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); + MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); + MOCK_METHOD(const Server::Options&, options, ()); + MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); + MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); + MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); + MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(Stats::Scope&, serverScope, ()); + MOCK_METHOD(Singleton::Manager&, singletonManager, ()); + MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); + MOCK_METHOD(OptRef, admin, ()); + MOCK_METHOD(TimeSource&, timeSource, ()); + MOCK_METHOD(Event::TestTimeSystem&, timeSystem, ()); + MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); + MOCK_METHOD(Api::Api&, api, ()); + MOCK_METHOD(Grpc::Context&, grpcContext, ()); + MOCK_METHOD(Router::Context&, routerContext, ()); + MOCK_METHOD(envoy::config::bootstrap::v3::Bootstrap&, bootstrap, ()); + MOCK_METHOD(Server::DrainManager&, drainManager, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); + MOCK_METHOD(StatsConfig&, statsConfig, (), ()); + MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); +}; + +} // namespace Configuration +} // namespace Server +} // namespace Envoy diff --git a/test/mocks/server/transport_socket_factory_context.cc b/test/mocks/server/transport_socket_factory_context.cc index 2f45814530f8..a9df701ac278 100644 --- a/test/mocks/server/transport_socket_factory_context.cc +++ b/test/mocks/server/transport_socket_factory_context.cc @@ -1,4 +1,4 @@ -#include "transport_socket_factory_context.h" +#include "test/mocks/server/transport_socket_factory_context.h" #include @@ -14,15 +14,18 @@ using ::testing::ReturnRef; MockTransportSocketFactoryContext::MockTransportSocketFactoryContext() : secret_manager_(std::make_unique(config_tracker_)), singleton_manager_(Thread::threadFactoryForTest()) { + ON_CALL(*this, serverFactoryContext()).WillByDefault(ReturnRef(server_context_)); ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); ON_CALL(*this, messageValidationVisitor()) .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); ON_CALL(*this, sslContextManager()).WillByDefault(ReturnRef(context_manager_)); - ON_CALL(*this, scope()).WillByDefault(ReturnRef(*store_.rootScope())); - ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); - ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); - ON_CALL(*this, singletonManager()).WillByDefault(ReturnRef(singleton_manager_)); + ON_CALL(*this, statsScope()).WillByDefault(ReturnRef(*store_.rootScope())); + + ON_CALL(server_context_, serverScope()).WillByDefault(ReturnRef(*store_.rootScope())); + ON_CALL(server_context_, api()).WillByDefault(ReturnRef(api_)); + ON_CALL(server_context_, options()).WillByDefault(ReturnRef(options_)); + ON_CALL(server_context_, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); + ON_CALL(server_context_, singletonManager()).WillByDefault(ReturnRef(singleton_manager_)); } MockTransportSocketFactoryContext::~MockTransportSocketFactoryContext() = default; diff --git a/test/mocks/server/transport_socket_factory_context.h b/test/mocks/server/transport_socket_factory_context.h index ee23330262d7..035f50e09aa4 100644 --- a/test/mocks/server/transport_socket_factory_context.h +++ b/test/mocks/server/transport_socket_factory_context.h @@ -7,16 +7,17 @@ #include "test/mocks/access_log/mocks.h" #include "test/mocks/api/mocks.h" #include "test/mocks/server/options.h" +#include "test/mocks/server/server_factory_context.h" #include "test/mocks/ssl/mocks.h" #include "test/mocks/stats/mocks.h" #include "test/mocks/upstream/cluster_manager.h" -#include "config_tracker.h" #include "gmock/gmock.h" namespace Envoy { namespace Server { namespace Configuration { + class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { public: MockTransportSocketFactoryContext(); @@ -24,28 +25,19 @@ class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { Secret::SecretManager& secretManager() override { return *(secret_manager_); } - MOCK_METHOD(ServerFactoryContext&, getServerFactoryContext, ()); - MOCK_METHOD(OptRef, admin, ()); - MOCK_METHOD(Ssl::ContextManager&, sslContextManager, ()); - MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(ServerFactoryContext&, serverFactoryContext, ()); MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); - MOCK_METHOD(Envoy::Random::RandomGenerator&, random, ()); - MOCK_METHOD(Stats::Store&, stats, ()); - MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(ThreadLocal::SlotAllocator&, threadLocal, ()); MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, ()); + MOCK_METHOD(Ssl::ContextManager&, sslContextManager, ()); + MOCK_METHOD(Stats::Scope&, statsScope, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + testing::NiceMock server_context_; testing::NiceMock cluster_manager_; testing::NiceMock api_; testing::NiceMock config_tracker_; testing::NiceMock context_manager_; - testing::NiceMock store_; + testing::NiceMock store_; testing::NiceMock options_; std::unique_ptr secret_manager_; testing::NiceMock access_log_manager_; From f4812764c43df5872fd773b25d399f7ff8fa4403 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 17 May 2023 10:46:30 -0400 Subject: [PATCH 263/740] UHV: Request encoder validation (#27299) * Add UHV validation during request encoding Signed-off-by: Yan Avlasov --- source/common/http/codec_client.cc | 39 ++++++++++++++---- source/common/http/http1/codec_impl.cc | 5 +++ source/common/http/http2/codec_impl.cc | 5 ++- .../common/quic/envoy_quic_client_stream.cc | 5 +++ .../common/quic/envoy_quic_server_stream.cc | 10 ++++- test/common/http/codec_client_test.cc | 29 +++++++++++++ test/common/http/http1/codec_impl_test.cc | 6 +++ test/common/http/http2/codec_impl_test.cc | 6 +++ .../compressor/compressor_integration_tests.h | 3 -- ...fault_header_validator_integration_test.cc | 1 + test/integration/filter_integration_test.cc | 6 +-- test/integration/http_integration.cc | 18 ++++---- test/integration/http_integration.h | 3 ++ test/integration/integration_test.cc | 5 +++ test/integration/protocol_integration_test.cc | 41 +++++++++++++++---- 15 files changed, 149 insertions(+), 33 deletions(-) diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index a4dab04c81bb..51ac7cffa591 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -187,14 +187,35 @@ void CodecClient::onData(Buffer::Instance& data) { Status CodecClient::ActiveRequest::encodeHeaders(const RequestHeaderMap& headers, bool end_stream) { #ifdef ENVOY_ENABLE_UHV if (header_validator_) { - auto result = header_validator_->transformRequestHeaders(headers); - if (!result.status.ok()) { - return absl::InvalidArgumentError( - absl::StrCat("header validation failed: ", result.status.details())); + bool failure = false; + std::string failure_details; + auto transformation_result = header_validator_->transformRequestHeaders(headers); + if (!transformation_result.status.ok()) { + ENVOY_CONN_LOG(debug, "Request header transformation failed: {}\n{}", *parent_.connection_, + transformation_result.status.details(), headers); + failure = true; + failure_details = std::string(transformation_result.status.details()); + } else { + // Validate header map after request encoder transformations + const ::Envoy::Http::HeaderValidator::ValidationResult validation_result = + header_validator_->validateRequestHeaders( + transformation_result.new_headers ? *transformation_result.new_headers : headers); + if (!validation_result.ok()) { + ENVOY_CONN_LOG(debug, "Request header validation failed: {}\n{}", *parent_.connection_, + validation_result.details(), + transformation_result.new_headers ? *transformation_result.new_headers + : headers); + failure = true; + failure_details = std::string(validation_result.details()); + } } - if (result.new_headers) { - return RequestEncoderWrapper::encodeHeaders(*result.new_headers, end_stream); + if (failure) { + return absl::InvalidArgumentError( + absl::StrCat("header validation failed: ", failure_details)); } + return RequestEncoderWrapper::encodeHeaders( + transformation_result.new_headers ? *transformation_result.new_headers : headers, + end_stream); } #endif return RequestEncoderWrapper::encodeHeaders(headers, end_stream); @@ -206,14 +227,16 @@ void CodecClient::ActiveRequest::decodeHeaders(ResponseHeaderMapPtr&& headers, b const ::Envoy::Http::HeaderValidator::ValidationResult validation_result = header_validator_->validateResponseHeaders(*headers); bool failure = !validation_result.ok(); + std::string failure_details(validation_result.details()); if (!failure) { const ::Envoy::Http::ClientHeaderValidator::TransformationResult transformation_result = header_validator_->transformResponseHeaders(*headers); failure = !transformation_result.ok(); + failure_details = std::string(validation_result.details()); } if (failure) { - ENVOY_CONN_LOG(debug, "Response header validation failed\n{}", *parent_.connection_, - *headers); + ENVOY_CONN_LOG(debug, "Response header validation failed: {}\n{}", *parent_.connection_, + failure_details, *headers); if ((parent_.codec_->protocol() == Protocol::Http2 && !parent_.host_->cluster() .http2Options() diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index db93c5ee68e7..e43c438f4714 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -442,11 +442,16 @@ void ResponseEncoderImpl::encodeHeaders(const ResponseHeaderMap& headers, bool e static constexpr absl::string_view REQUEST_POSTFIX = " HTTP/1.1\r\n"; Status RequestEncoderImpl::encodeHeaders(const RequestHeaderMap& headers, bool end_stream) { +#ifndef ENVOY_ENABLE_UHV + // Headers are now validated by UHV before encoding by the codec. Two checks below are not needed + // when UHV is enabled. + // // Required headers must be present. This can only happen by some erroneous processing after the // downstream codecs decode. RETURN_IF_ERROR(HeaderUtility::checkRequiredRequestHeaders(headers)); // Verify that a filter hasn't added an invalid header key or value. RETURN_IF_ERROR(HeaderUtility::checkValidRequestHeaders(headers)); +#endif const HeaderEntry* method = headers.Method(); const HeaderEntry* path = headers.Path(); diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 3e4a76d1f3aa..ad7c29b423dc 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -260,12 +260,15 @@ void ConnectionImpl::StreamImpl::encodeHeadersBase(const HeaderMap& headers, boo Status ConnectionImpl::ClientStreamImpl::encodeHeaders(const RequestHeaderMap& headers, bool end_stream) { parent_.updateActiveStreamsOnEncode(*this); +#ifndef ENVOY_ENABLE_UHV + // Headers are now validated by UHV before encoding by the codec. Two checks below are not needed + // when UHV is enabled. + // // Required headers must be present. This can only happen by some erroneous processing after the // downstream codecs decode. RETURN_IF_ERROR(HeaderUtility::checkRequiredRequestHeaders(headers)); // Verify that a filter hasn't added an invalid header key or value. RETURN_IF_ERROR(HeaderUtility::checkValidRequestHeaders(headers)); -#ifndef ENVOY_ENABLE_UHV // Extended CONNECT to H/1 upgrade transformation has moved to UHV // This must exist outside of the scope of isUpgrade as the underlying memory is // needed until encodeHeadersBase has been called. diff --git a/source/common/quic/envoy_quic_client_stream.cc b/source/common/quic/envoy_quic_client_stream.cc index 8d2cf965e37a..e62805e28b2e 100644 --- a/source/common/quic/envoy_quic_client_stream.cc +++ b/source/common/quic/envoy_quic_client_stream.cc @@ -35,11 +35,16 @@ EnvoyQuicClientStream::EnvoyQuicClientStream( Http::Status EnvoyQuicClientStream::encodeHeaders(const Http::RequestHeaderMap& headers, bool end_stream) { ENVOY_STREAM_LOG(debug, "encodeHeaders: (end_stream={}) {}.", *this, end_stream, headers); +#ifndef ENVOY_ENABLE_UHV + // Headers are now validated by UHV before encoding by the codec. Two checks below are not needed + // when UHV is enabled. + // // Required headers must be present. This can only happen by some erroneous processing after the // downstream codecs decode. RETURN_IF_ERROR(Http::HeaderUtility::checkRequiredRequestHeaders(headers)); // Verify that a filter hasn't added an invalid header key or value. RETURN_IF_ERROR(Http::HeaderUtility::checkValidRequestHeaders(headers)); +#endif if (write_side_closed()) { return absl::CancelledError("encodeHeaders is called on write-closed stream."); diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc index 463e2412fa83..123ac025aa14 100644 --- a/source/common/quic/envoy_quic_server_stream.cc +++ b/source/common/quic/envoy_quic_server_stream.cc @@ -236,6 +236,8 @@ void EnvoyQuicServerStream::OnInitialHeadersComplete(bool fin, size_t frame_len, return; } +#ifndef ENVOY_ENABLE_UHV + // These checks are now part of UHV if (Http::HeaderUtility::checkRequiredRequestHeaders(*headers) != Http::okStatus() || Http::HeaderUtility::checkValidRequestHeaders(*headers) != Http::okStatus() || (headers->Protocol() && !spdy_session()->allow_extended_connect())) { @@ -244,13 +246,19 @@ void EnvoyQuicServerStream::OnInitialHeadersComplete(bool fin, size_t frame_len, return; } -#ifndef ENVOY_ENABLE_UHV // Extended CONNECT to H/1 upgrade transformation has moved to UHV if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.use_http3_header_normalisation") && Http::Utility::isH3UpgradeRequest(*headers)) { // Transform Request from H3 to H1 Http::Utility::transformUpgradeRequestFromH3toH1(*headers); } +#else + if (Http::HeaderUtility::checkRequiredRequestHeaders(*headers) != Http::okStatus() || + (headers->Protocol() && !spdy_session()->allow_extended_connect())) { + details_ = Http3ResponseCodeDetailValues::invalid_http_header; + onStreamError(absl::nullopt); + return; + } #endif request_decoder_->decodeHeaders(std::move(headers), /*end_stream=*/fin); diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 2c37efa2fac5..2936e784cb36 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -65,6 +65,8 @@ class CodecClientTest : public Event::TestUsingSimulatedTime, public testing::Te nullptr, host_, dispatcher_); ON_CALL(*connection_, streamInfo()).WillByDefault(ReturnRef(stream_info_)); #ifdef ENVOY_ENABLE_UHV + ON_CALL(*header_validator_, validateRequestHeaders(_)) + .WillByDefault(Return(HeaderValidator::ValidationResult::success())); ON_CALL(*header_validator_, transformRequestHeaders(_)) .WillByDefault( Return(ByMove(ClientHeaderValidator::RequestHeadersTransformationResult::success()))); @@ -310,6 +312,33 @@ TEST_F(CodecClientTest, WatermarkPassthrough) { } #ifdef ENVOY_ENABLE_UHV +TEST_F(CodecClientTest, RequestHeaderValidationFails) { + initialize(); + EXPECT_CALL(*header_validator_, validateRequestHeaders(_)) + .WillOnce(Return(HeaderValidator::ValidationResult{ + HeaderValidator::ValidationResult::Action::Reject, "some error"})); + + ResponseDecoder* inner_decoder; + NiceMock inner_encoder; + EXPECT_CALL(*codec_, newStream(_)) + .WillOnce(Invoke([&](ResponseDecoder& decoder) -> RequestEncoder& { + inner_decoder = &decoder; + return inner_encoder; + })); + + Http::MockResponseDecoder outer_decoder; + Http::RequestEncoder& request_encoder = client_->newStream(outer_decoder); + + TestRequestHeaderMapImpl request_headers{ + {":authority", "host"}, {":path", "/"}, {":method", "GET"}}; + auto status = request_encoder.encodeHeaders(request_headers, true); + EXPECT_THAT(status, StatusHelpers::HasStatus(absl::StatusCode::kInvalidArgument, + testing::HasSubstr("some error"))); + // Router will reset upstream request when encodeHeaders returns failure status + inner_encoder.stream_.callbacks_.front()->onResetStream(StreamResetReason::LocalReset, + "some error"); +} + TEST_F(CodecClientTest, RequestHeaderTransformationFails) { initialize(); EXPECT_CALL(*header_validator_, transformRequestHeaders(_)) diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 04b7b9ce46be..c8e8da3c86ba 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -2959,6 +2959,12 @@ TEST_P(Http1ClientConnectionImplTest, 101ResponseTransferEncodingNotAllowed) { } TEST_P(Http1ClientConnectionImplTest, BadEncodeParams) { +#ifdef ENVOY_ENABLE_UHV + // The check for required headers is done by UHV. When UHV is enabled this test + // is superseded by CodecClientTest.ResponseHeaderValidationFails and + // DownstreamProtocolIntegrationTest.DownstreamRequestWithFaultyFilter tests. + return; +#endif initialize(); NiceMock response_decoder; diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index c08764d1e036..f790f3e7a2b0 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -968,6 +968,12 @@ TEST_P(Http2CodecImplTest, RefusedStreamReset) { TEST_P(Http2CodecImplTest, InvalidHeadersFrameMissing) { initialize(); +#ifdef ENVOY_ENABLE_UHV + // The check for required headers is done by UHV. When UHV is enabled this test + // is superseded by CodecClientTest.ResponseHeaderValidationFails and + // DownstreamProtocolIntegrationTest.DownstreamRequestWithFaultyFilter tests. + return; +#endif const auto status = request_encoder_->encodeHeaders(TestRequestHeaderMapImpl{}, true); driveToCompletion(); diff --git a/test/extensions/filters/http/compressor/compressor_integration_tests.h b/test/extensions/filters/http/compressor/compressor_integration_tests.h index 38d82c04dc58..2022fdc7fb85 100644 --- a/test/extensions/filters/http/compressor/compressor_integration_tests.h +++ b/test/extensions/filters/http/compressor/compressor_integration_tests.h @@ -45,9 +45,6 @@ class CompressorProxyingConnectIntegrationTest : public HttpProtocolIntegrationT public: void initialize() override; Http::TestRequestHeaderMapImpl connect_headers_{{":method", "CONNECT"}, - {":path", "/"}, - {":protocol", "bytestream"}, - {":scheme", "https"}, {":authority", "host:80"}}; IntegrationStreamDecoderPtr response_; }; diff --git a/test/integration/default_header_validator_integration_test.cc b/test/integration/default_header_validator_integration_test.cc index 397c715ab0eb..e821709ce86b 100644 --- a/test/integration/default_header_validator_integration_test.cc +++ b/test/integration/default_header_validator_integration_test.cc @@ -13,6 +13,7 @@ INSTANTIATE_TEST_SUITE_P(Protocols, DownstreamUhvIntegrationTest, // Without the `uhv_translate_backslash_to_slash` override UHV rejects requests with backslash in // the path. TEST_P(DownstreamUhvIntegrationTest, BackslashInUriPathConversionWithUhvOverride) { + disable_client_header_validation_ = true; config_helper_.addRuntimeOverride("envoy.reloadable_features.uhv_translate_backslash_to_slash", "false"); config_helper_.addConfigModifier( diff --git a/test/integration/filter_integration_test.cc b/test/integration/filter_integration_test.cc index d7eab487d5de..9c8deb0f9c6b 100644 --- a/test/integration/filter_integration_test.cc +++ b/test/integration/filter_integration_test.cc @@ -441,7 +441,7 @@ TEST_P(FilterIntegrationTest, DownstreamRequestWithFaultyFilter) { ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); EXPECT_EQ("503", response->headers().getStatusValue()); - EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("missing_required_header")); + EXPECT_THAT(waitForAccessLog(access_log_name_), testing::MatchesRegex(".*required.*header.*")); // Missing path for non-CONNECT response = codec_client_->makeHeaderOnlyRequest( @@ -453,7 +453,7 @@ TEST_P(FilterIntegrationTest, DownstreamRequestWithFaultyFilter) { ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); EXPECT_EQ("503", response->headers().getStatusValue()); - EXPECT_THAT(waitForAccessLog(access_log_name_, 1), HasSubstr("missing_required_header")); + EXPECT_THAT(waitForAccessLog(access_log_name_, 1), testing::MatchesRegex(".*required.*header.*")); } TEST_P(FilterIntegrationTest, FaultyFilterWithConnect) { @@ -478,7 +478,7 @@ TEST_P(FilterIntegrationTest, FaultyFilterWithConnect) { ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); EXPECT_EQ("503", response->headers().getStatusValue()); - EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("missing_required_header")); + EXPECT_THAT(waitForAccessLog(access_log_name_), testing::MatchesRegex(".*required.*header.*")); } // Test hitting the decoder buffer filter with too many request bytes to buffer. Ensure the diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 38a915872be3..7dbb1dd7e1be 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -284,14 +284,16 @@ IntegrationCodecClientPtr HttpIntegrationTest::makeRawHttpConnection( cluster->http2_options_ = http2_options.value(); cluster->http1_settings_.enable_trailers_ = true; - static constexpr absl::string_view empty_header_validator_config = R"EOF( - name: envoy.http.header_validators.envoy_default - typed_config: - "@type": type.googleapis.com/envoy.extensions.http.header_validators.envoy_default.v3.HeaderValidatorConfig -)EOF"; - - cluster->header_validator_factory_ = - IntegrationUtil::makeHeaderValidationFactory(empty_header_validator_config); + if (!disable_client_header_validation_) { + static constexpr absl::string_view empty_header_validator_config = R"EOF( + name: envoy.http.header_validators.envoy_default + typed_config: + "@type": type.googleapis.com/envoy.extensions.http.header_validators.envoy_default.v3.HeaderValidatorConfig + )EOF"; + + cluster->header_validator_factory_ = + IntegrationUtil::makeHeaderValidationFactory(empty_header_validator_config); + } Upstream::HostDescriptionConstSharedPtr host_description{Upstream::makeTestHostDescription( cluster, fmt::format("tcp://{}:80", Network::Test::getLoopbackAddressUrlString(version_)), diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index 8bf48ff28c98..f0d1020e46bd 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -359,6 +359,9 @@ class HttpIntegrationTest : public BaseIntegrationTest { Quic::QuicStatNames quic_stat_names_; std::string san_to_match_{"spiffe://lyft.com/backend-team"}; bool enable_quic_early_data_{true}; + // Set this to true when sending malformed requests to avoid test client codec rejecting it. + // This flag is only valid when UHV build flag is enabled. + bool disable_client_header_validation_{false}; #ifdef ENVOY_ENABLE_QUIC quic::DeterministicConnectionIdGenerator connection_id_generator_{ quic::kQuicDefaultConnectionIdLength}; diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index 8967cabaf55a..cc08069f481a 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -1406,6 +1406,7 @@ TEST_P(IntegrationTest, PipelineInline) { } TEST_P(IntegrationTest, NoHost) { + disable_client_header_validation_ = true; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -2001,6 +2002,7 @@ TEST_P(IntegrationTest, TestFloodUpstreamErrors) { // Make sure flood protection doesn't kick in with many requests sent serially. TEST_P(IntegrationTest, TestManyBadRequests) { + disable_client_header_validation_ = true; config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { @@ -2200,6 +2202,7 @@ TEST_P(IntegrationTest, QuitQuitQuit) { // stream_error_on_invalid_http_message=false: test that HTTP/1.1 connection is left open on invalid // HTTP message (missing :host header) TEST_P(IntegrationTest, ConnectionIsLeftOpenIfHCMStreamErrorIsFalseAndOverrideIsTrue) { + disable_client_header_validation_ = true; config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { @@ -2224,6 +2227,7 @@ TEST_P(IntegrationTest, ConnectionIsLeftOpenIfHCMStreamErrorIsFalseAndOverrideIs // stream_error_on_invalid_http_message=true: test that HTTP/1.1 connection is left open on invalid // HTTP message (missing :host header) TEST_P(IntegrationTest, ConnectionIsLeftOpenIfHCMStreamErrorIsTrueAndOverrideNotSet) { + disable_client_header_validation_ = true; config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { hcm.mutable_stream_error_on_invalid_http_message()->set_value(true); }); @@ -2243,6 +2247,7 @@ TEST_P(IntegrationTest, ConnectionIsLeftOpenIfHCMStreamErrorIsTrueAndOverrideNot // stream_error_on_invalid_http_message=false: test that HTTP/1.1 connection is terminated on // invalid HTTP message (missing :host header) TEST_P(IntegrationTest, ConnectionIsTerminatedIfHCMStreamErrorIsFalseAndOverrideNotSet) { + disable_client_header_validation_ = true; config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index ac73af5f40ed..edf0d14ed38b 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -71,6 +71,10 @@ TEST_P(ProtocolIntegrationTest, ShutdownWithActiveConnPoolConnections) { } TEST_P(ProtocolIntegrationTest, LogicalDns) { +#ifdef ENVOY_ENABLE_UHV + // TODO(#27132): auto_host_rewrite is broken for IPv6 and is failing UHV validation + return; +#endif config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() == 1, ""); auto& cluster = *bootstrap.mutable_static_resources()->mutable_clusters(0); @@ -139,6 +143,7 @@ TEST_P(DownstreamProtocolIntegrationTest, RouterClusterNotFound404) { } TEST_P(DownstreamProtocolIntegrationTest, TestHostWhitespacee) { + disable_client_header_validation_ = true; config_helper_.addConfigModifier(&setDoNotValidateRouteConfig); auto host = config_helper_.createVirtualHost("foo.lyft.com", "/unknown", "unknown_cluster"); host.mutable_routes(0)->mutable_route()->set_cluster_not_found_response_code( @@ -614,7 +619,7 @@ TEST_P(DownstreamProtocolIntegrationTest, DownstreamRequestWithFaultyFilter) { ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); EXPECT_EQ("503", response->headers().getStatusValue()); - EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("missing_required_header")); + EXPECT_THAT(waitForAccessLog(access_log_name_), testing::MatchesRegex(".*required.*header.*")); // Missing path for non-CONNECT response = codec_client_->makeHeaderOnlyRequest( @@ -626,7 +631,7 @@ TEST_P(DownstreamProtocolIntegrationTest, DownstreamRequestWithFaultyFilter) { ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); EXPECT_EQ("503", response->headers().getStatusValue()); - EXPECT_THAT(waitForAccessLog(access_log_name_, 1), HasSubstr("missing_required_header")); + EXPECT_THAT(waitForAccessLog(access_log_name_, 1), testing::MatchesRegex(".*required.*header.*")); } TEST_P(DownstreamProtocolIntegrationTest, FaultyFilterWithConnect) { @@ -657,7 +662,10 @@ TEST_P(DownstreamProtocolIntegrationTest, FaultyFilterWithConnect) { ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); EXPECT_EQ("503", response->headers().getStatusValue()); - EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("missing_required_header")); + // With and without UHV the details string is different: + // non-UHV: filter_removed_required_request_headers{missing_required_header:_host} + // UHV: filter_removed_required_request_headers{header_validation_failed:_uhv.invalid_host} + EXPECT_THAT(waitForAccessLog(access_log_name_), testing::MatchesRegex(".*required.*header.*")); // Wait to process STOP_SENDING on the client for quic. if (downstreamProtocol() == Http::CodecType::HTTP3) { EXPECT_TRUE(response->waitForReset()); @@ -2073,6 +2081,7 @@ TEST_P(DownstreamProtocolIntegrationTest, LargeCookieParsingMany) { } TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLength) { + disable_client_header_validation_ = true; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -2097,6 +2106,7 @@ TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLength) { } TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLengthAllowed) { + disable_client_header_validation_ = true; setDownstreamOverrideStreamErrorOnInvalidHttpMessage(); initialize(); @@ -2126,6 +2136,7 @@ TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLengthAllowed) { } TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengths) { + disable_client_header_validation_ = true; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); auto encoder_decoder = @@ -2148,6 +2159,7 @@ TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengths) { // TODO(PiotrSikora): move this HTTP/2 only variant to http2_integration_test.cc. TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengthsAllowed) { + disable_client_header_validation_ = true; setDownstreamOverrideStreamErrorOnInvalidHttpMessage(); initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -2944,6 +2956,7 @@ TEST_P(DownstreamProtocolIntegrationTest, MaxRequestsPerConnectionReached) { // Make sure that invalid authority headers get blocked at or before the HCM. TEST_P(DownstreamProtocolIntegrationTest, InvalidAuthority) { + disable_client_header_validation_ = true; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -4030,7 +4043,7 @@ TEST_P(DownstreamProtocolIntegrationTest, ValidateUpstreamHeaders) { EXPECT_TRUE(response->complete()); EXPECT_EQ("503", response->headers().getStatusValue()); if (upstreamProtocol() != Http::CodecType::HTTP3) { - EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("invalid_header_value_for:_x-foo")); + EXPECT_THAT(waitForAccessLog(access_log_name_), testing::MatchesRegex(".*invalid.*value.*")); } } @@ -4085,9 +4098,9 @@ TEST_P(ProtocolIntegrationTest, ValidateUpstreamMixedCaseHeaders) { TEST_P(ProtocolIntegrationTest, ValidateUpstreamHeadersWithOverride) { #ifdef ENVOY_ENABLE_UHV - if (upstreamProtocol() != Http::CodecType::HTTP1) { - return; - } + // UHV always validated headers before sending them upstream. This test is not applicable + // when UHV is enabled. + return; #endif if (upstreamProtocol() == Http::CodecType::HTTP3) { testing_upstream_intentionally_ = true; @@ -4404,8 +4417,10 @@ TEST_P(ProtocolIntegrationTest, LocalInterfaceNameForUpstreamConnection) { #endif TEST_P(DownstreamProtocolIntegrationTest, InvalidRequestHeaderName) { + // TODO(yanavlasov): remove runtime override after making disable_client_header_validation_ work + // for non UHV builds config_helper_.addRuntimeOverride("envoy.reloadable_features.validate_upstream_headers", "false"); - + disable_client_header_validation_ = true; initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -4431,8 +4446,10 @@ TEST_P(DownstreamProtocolIntegrationTest, InvalidRequestHeaderName) { } TEST_P(DownstreamProtocolIntegrationTest, InvalidRequestHeaderNameStreamError) { + // TODO(yanavlasov): remove runtime override after making disable_client_header_validation_ work + // for non UHV builds config_helper_.addRuntimeOverride("envoy.reloadable_features.validate_upstream_headers", "false"); - + disable_client_header_validation_ = true; // For H/1 this test is equivalent to InvalidRequestHeaderName if (downstreamProtocol() == Http::CodecType::HTTP1) { return; @@ -4516,6 +4533,7 @@ TEST_P(ProtocolIntegrationTest, InvalidResponseHeaderNameStreamError) { } TEST_P(DownstreamProtocolIntegrationTest, DuplicatedSchemeHeaders) { + disable_client_header_validation_ = true; if (downstreamProtocol() == Http::CodecType::HTTP1) { // send_fully_qualified_url_ is not enabled, so :scheme header isn't used. return; @@ -4536,6 +4554,7 @@ TEST_P(DownstreamProtocolIntegrationTest, DuplicatedSchemeHeaders) { } TEST_P(DownstreamProtocolIntegrationTest, DuplicatedMethodHeaders) { + disable_client_header_validation_ = true; if (downstreamProtocol() == Http::CodecType::HTTP1 && GetParam().http1_implementation == Http1ParserImpl::BalsaParser) { // this test is unreliable in this case. @@ -4561,6 +4580,7 @@ TEST_P(DownstreamProtocolIntegrationTest, DuplicatedMethodHeaders) { } TEST_P(DownstreamProtocolIntegrationTest, MethodHeaderWithWhitespace) { + disable_client_header_validation_ = true; useAccessLog("%RESPONSE_CODE_DETAILS%"); initialize(); @@ -4576,6 +4596,7 @@ TEST_P(DownstreamProtocolIntegrationTest, MethodHeaderWithWhitespace) { } TEST_P(DownstreamProtocolIntegrationTest, EmptyMethodHeader) { + disable_client_header_validation_ = true; useAccessLog("%RESPONSE_CODE_DETAILS%"); initialize(); @@ -4591,6 +4612,7 @@ TEST_P(DownstreamProtocolIntegrationTest, EmptyMethodHeader) { } TEST_P(DownstreamProtocolIntegrationTest, InvalidSchemeHeaderWithWhitespace) { + disable_client_header_validation_ = true; useAccessLog("%RESPONSE_CODE_DETAILS%"); initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -4728,6 +4750,7 @@ TEST_P(DownstreamProtocolIntegrationTest, InvalidTrailerStreamError) { } TEST_P(DownstreamProtocolIntegrationTest, UnknownPseudoHeader) { + disable_client_header_validation_ = true; if (downstreamProtocol() == Http::CodecType::HTTP1) { return; } From cfce3573d6ca915139653b555d7c03b193ebd134 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 17 May 2023 16:58:43 +0100 Subject: [PATCH 264/740] mobile/ci: Fix coverage (#27425) Signed-off-by: Ryan Northey --- .github/workflows/mobile-coverage.yml | 7 +++--- mobile/.bazelrc | 3 +++ test/run_envoy_bazel_coverage.sh | 35 +++++++++++++++------------ 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml index 2a6dd22db74e..c2f106a70a3e 100644 --- a/.github/workflows/mobile-coverage.yml +++ b/.github/workflows/mobile-coverage.yml @@ -32,21 +32,20 @@ jobs: run: ./mobile/tools/should_run_ci.sh - name: 'Run coverage' if: steps.should_run.outputs.run_ci_job == 'true' - continue-on-error: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && BAZEL_BUILD_OPTION_LIST="--config=remote-ci-linux-coverage" \ PATH=/opt/llvm/bin:${PATH} \ - COVERAGE_THRESHOLD=95 \ + COVERAGE_THRESHOLD=76 \ ../test/run_envoy_bazel_coverage.sh //test/common/... - name: 'Package coverage' if: steps.should_run.outputs.run_ci_job == 'true' run: | - cd mobile && tar -czvf coverage.tar.gz generated/coverage + cd mobile && tar -czf coverage.tar.gz generated/coverage - name: 'Upload report' if: steps.should_run.outputs.run_ci_job == 'true' uses: actions/upload-artifact@v3 with: name: coverage.tar.gz - path: coverage.tar.gz + path: mobile/coverage.tar.gz diff --git a/mobile/.bazelrc b/mobile/.bazelrc index df4f01c554d9..5d2960882438 100644 --- a/mobile/.bazelrc +++ b/mobile/.bazelrc @@ -238,6 +238,9 @@ build:remote-ci-linux-coverage --strategy=CoverageReport=local,remote build:remote-ci-linux-coverage --noremote_accept_cached build:remote-ci-linux-coverage --config=remote-ci-common build:remote-ci-linux-coverage --build_runfile_links +build:remote-ci-linux-coverage --legacy_important_outputs=false +build:remote-ci-linux-coverage --test_env=CC_CODE_COVERAGE_SCRIPT=external/envoy/bazel/coverage/collect_cc_coverage.sh + # IPv6 tests fail on CI build:remote-ci-linux-coverage --test_env=ENVOY_IP_TEST_VERSIONS=v4only ############################################################################# diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 0ddb64a7e9b6..9a5a751fb527 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -138,21 +138,24 @@ if [[ "$VALIDATE_COVERAGE" == "true" ]]; then fi fi -# We want to allow per_file_coverage to fail without exiting this script. -set +e -if [[ "$VALIDATE_COVERAGE" == "true" ]] && [[ "${FUZZ_COVERAGE}" == "false" ]]; then - echo "Checking per-extension coverage" - output=$(./test/per_file_coverage.sh) - response=$? - - if [ $response -ne 0 ]; then - echo Per-extension coverage failed: - echo "$output" - COVERAGE_FAILED=1 - echo "##vso[task.setvariable variable=COVERAGE_FAILED]${COVERAGE_FAILED}" - exit 1 - fi - echo Per-extension coverage passed. +if [[ -e ./test/per_file_coverage.sh ]]; then + # We want to allow per_file_coverage to fail without exiting this script. + set +e + if [[ "$VALIDATE_COVERAGE" == "true" ]] && [[ "${FUZZ_COVERAGE}" == "false" ]]; then + echo "Checking per-extension coverage" + output=$(./test/per_file_coverage.sh) + response=$? + + if [ $response -ne 0 ]; then + echo Per-extension coverage failed: + echo "$output" + COVERAGE_FAILED=1 + echo "##vso[task.setvariable variable=COVERAGE_FAILED]${COVERAGE_FAILED}" + exit 1 + fi + echo Per-extension coverage passed. + fi +else + echo "No per-file-coverage file found" fi - echo "HTML coverage report is in ${COVERAGE_DIR}/index.html" From 2202e9fa267f86ffeeba2a01978e540df0a7c1e6 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 17 May 2023 13:13:23 -0400 Subject: [PATCH 265/740] grpc: moving the last of the gRPC mux classes to factory based extension (#27413) Signed-off-by: Alyssa Wilk --- docs/root/api-docs/xds_protocol.rst | 3 + source/common/config/BUILD | 1 - .../config/subscription_factory_impl.cc | 1 - source/common/upstream/BUILD | 2 +- .../common/upstream/cluster_manager_impl.cc | 82 +++++++------------ .../extensions/config_subscription/grpc/BUILD | 2 +- .../grpc_collection_subscription_factory.cc | 2 +- .../grpc/grpc_subscription_factory.cc | 2 +- .../config_subscription/grpc}/xds_mux/BUILD | 7 +- .../grpc}/xds_mux/delta_subscription_state.cc | 2 +- .../grpc}/xds_mux/delta_subscription_state.h | 2 +- .../grpc}/xds_mux/grpc_mux_impl.cc | 56 +++++++++++-- .../grpc}/xds_mux/grpc_mux_impl.h | 13 ++- .../grpc}/xds_mux/sotw_subscription_state.cc | 2 +- .../grpc}/xds_mux/sotw_subscription_state.h | 2 +- .../grpc}/xds_mux/subscription_state.h | 0 source/extensions/extensions_build_config.bzl | 2 + source/extensions/extensions_metadata.yaml | 10 +++ .../filters/network/redis_proxy/BUILD | 1 + source/server/BUILD | 1 - source/server/server.cc | 20 ++--- test/common/config/BUILD | 48 ----------- .../config/grpc_subscription_test_harness.h | 2 +- test/extensions/clusters/eds/BUILD | 2 +- .../extensions/clusters/eds/eds_speed_test.cc | 2 +- .../extensions/config_subscription/grpc/BUILD | 52 +++++++++++- .../grpc/delta_subscription_state_test.cc | 2 +- .../grpc/delta_subscription_test_harness.h | 2 +- .../grpc/new_grpc_mux_impl_test.cc | 2 +- .../grpc}/sotw_subscription_state_test.cc | 2 +- .../grpc}/xds_grpc_mux_impl_test.cc | 2 +- test/per_file_coverage.sh | 4 +- tools/extensions/extensions_schema.yaml | 1 + 33 files changed, 182 insertions(+), 152 deletions(-) rename source/{common/config => extensions/config_subscription/grpc}/xds_mux/BUILD (95%) rename source/{common/config => extensions/config_subscription/grpc}/xds_mux/delta_subscription_state.cc (99%) rename source/{common/config => extensions/config_subscription/grpc}/xds_mux/delta_subscription_state.h (98%) rename source/{common/config => extensions/config_subscription/grpc}/xds_mux/grpc_mux_impl.cc (87%) rename source/{common/config => extensions/config_subscription/grpc}/xds_mux/grpc_mux_impl.h (95%) rename source/{common/config => extensions/config_subscription/grpc}/xds_mux/sotw_subscription_state.cc (98%) rename source/{common/config => extensions/config_subscription/grpc}/xds_mux/sotw_subscription_state.h (97%) rename source/{common/config => extensions/config_subscription/grpc}/xds_mux/subscription_state.h (100%) rename test/{common/config => extensions/config_subscription/grpc}/sotw_subscription_state_test.cc (99%) rename test/{common/config => extensions/config_subscription/grpc}/xds_grpc_mux_impl_test.cc (99%) diff --git a/docs/root/api-docs/xds_protocol.rst b/docs/root/api-docs/xds_protocol.rst index f2e1cf8a594b..bc2a3db82cac 100644 --- a/docs/root/api-docs/xds_protocol.rst +++ b/docs/root/api-docs/xds_protocol.rst @@ -114,6 +114,9 @@ that it is interested in. It then fetches the ``RouteConfiguration`` resources r by the ``Cluster`` resources. In effect, the original ``Listener`` resources are the roots to the client's configuration tree. +.. _extension_envoy.config_mux.delta_grpc_mux_factory: +.. _extension_envoy.config_mux.sotw_grpc_mux_factory: + Variants of the xDS Transport Protocol ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 3b9dc30c03f7..f7a25c71a45e 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -206,7 +206,6 @@ envoy_cc_library( "//envoy/server:instance_interface", "//envoy/upstream:cluster_manager_interface", "//source/common/common:minimal_logger_lib", - "//source/common/config/xds_mux:grpc_mux_lib", "//source/common/http:utility_lib", "//source/common/protobuf", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 20448c94a375..5be1165a79d5 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -6,7 +6,6 @@ #include "source/common/config/custom_config_validators_impl.h" #include "source/common/config/type_to_endpoint.h" #include "source/common/config/utility.h" -#include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/config/xds_resource.h" #include "source/common/http/utility.h" #include "source/common/protobuf/protobuf.h" diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index eed605278f73..23562f7aaa7f 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -102,7 +102,6 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/common:utility_lib", "//source/common/config:custom_config_validators_lib", - "//source/common/config/xds_mux:grpc_mux_lib", "//source/common/config:subscription_factory_lib", "//source/common/config:utility_lib", "//source/common/config:xds_resource_lib", @@ -446,6 +445,7 @@ envoy_cc_library( deps = [ ":load_balancer_lib", ":resource_manager_lib", + ":upstream_http_factory_context_lib", "//envoy/event:timer_interface", "//envoy/local_info:local_info_interface", "//envoy/network:dns_interface", diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index fd24da4c9770..d5a2b85fb0f6 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -28,7 +28,6 @@ #include "source/common/config/custom_config_validators_impl.h" #include "source/common/config/null_grpc_mux_impl.h" #include "source/common/config/utility.h" -#include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/config/xds_resource.h" #include "source/common/grpc/async_client_manager_impl.h" #include "source/common/http/async_client_impl.h" @@ -398,69 +397,46 @@ ClusterManagerImpl::ClusterManagerImpl( if (dyn_resources.ads_config().api_type() == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC) { Config::Utility::checkTransportVersion(dyn_resources.ads_config()); + std::string name; if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { - ads_mux_ = std::make_shared( - Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) - ->createUncachedRawAsyncClient(), - main_thread_dispatcher, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.discovery.v3.AggregatedDiscoveryService." - "DeltaAggregatedResources"), - *stats_.rootScope(), - Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info, - dyn_resources.ads_config().set_node_on_first_message_only(), - std::move(custom_config_validators), std::move(backoff_strategy), - makeOptRefFromPtr(xds_config_tracker_.get())); + name = "envoy.config_mux.delta_grpc_mux_factory"; } else { - auto* factory = Config::Utility::getFactoryByName( - "envoy.config_mux.new_grpc_mux_factory"); - if (!factory) { - throw EnvoyException("envoy.config_mux.new_grpc_mux_factory factory not found"); - } - ads_mux_ = factory->create( - Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) - ->createUncachedRawAsyncClient(), - main_thread_dispatcher, random_, *stats_.rootScope(), dyn_resources.ads_config(), - local_info, std::move(custom_config_validators), std::move(backoff_strategy), - makeOptRefFromPtr(xds_config_tracker_.get()), {}); + name = "envoy.config_mux.new_grpc_mux_factory"; + } + auto* factory = Config::Utility::getFactoryByName(name); + if (!factory) { + throw EnvoyException(fmt::format("{} not found", name)); } + ads_mux_ = factory->create( + Config::Utility::factoryForGrpcApiConfigSource( + *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) + ->createUncachedRawAsyncClient(), + main_thread_dispatcher, random_, *stats_.rootScope(), dyn_resources.ads_config(), + local_info, std::move(custom_config_validators), std::move(backoff_strategy), + makeOptRefFromPtr(xds_config_tracker_.get()), {}); } else { Config::Utility::checkTransportVersion(dyn_resources.ads_config()); const std::string target_xds_authority = Config::Utility::getGrpcControlPlane(dyn_resources.ads_config()).value_or(""); auto xds_delegate_opt_ref = makeOptRefFromPtr(xds_resources_delegate_.get()); - + std::string name; if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { - ads_mux_ = std::make_shared( - Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) - ->createUncachedRawAsyncClient(), - main_thread_dispatcher, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.discovery.v3.AggregatedDiscoveryService." - "StreamAggregatedResources"), - *stats_.rootScope(), - Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info, - bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only(), - std::move(custom_config_validators), std::move(backoff_strategy), - makeOptRefFromPtr(xds_config_tracker_.get()), xds_delegate_opt_ref, - target_xds_authority); + name = "envoy.config_mux.sotw_grpc_mux_factory"; } else { - auto* factory = Config::Utility::getFactoryByName( - "envoy.config_mux.grpc_mux_factory"); - if (!factory) { - throw EnvoyException("envoy.config_mux.grpc_mux_factory factory not found"); - } - ads_mux_ = factory->create( - Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) - ->createUncachedRawAsyncClient(), - main_thread_dispatcher, random_, *stats_.rootScope(), dyn_resources.ads_config(), - local_info, std::move(custom_config_validators), std::move(backoff_strategy), - makeOptRefFromPtr(xds_config_tracker_.get()), xds_delegate_opt_ref); + name = "envoy.config_mux.grpc_mux_factory"; + } + + auto* factory = Config::Utility::getFactoryByName(name); + if (!factory) { + throw EnvoyException(fmt::format("{} not found", name)); } + ads_mux_ = factory->create( + Config::Utility::factoryForGrpcApiConfigSource( + *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) + ->createUncachedRawAsyncClient(), + main_thread_dispatcher, random_, *stats_.rootScope(), dyn_resources.ads_config(), + local_info, std::move(custom_config_validators), std::move(backoff_strategy), + makeOptRefFromPtr(xds_config_tracker_.get()), xds_delegate_opt_ref); } } else { ads_mux_ = std::make_unique(); diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD index ade30d4cb80f..9736956f4938 100644 --- a/source/extensions/config_subscription/grpc/BUILD +++ b/source/extensions/config_subscription/grpc/BUILD @@ -71,7 +71,7 @@ envoy_cc_library( "//source/common/config:custom_config_validators_lib", "//source/common/config:type_to_endpoint_lib", "//source/common/config:xds_resource_lib", - "//source/common/config/xds_mux:grpc_mux_lib", + "//source/extensions/config_subscription/grpc/xds_mux:grpc_mux_lib", ], ) diff --git a/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc index e6b89fc1da05..987524f0c204 100644 --- a/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc +++ b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc @@ -2,10 +2,10 @@ #include "source/common/config/custom_config_validators_impl.h" #include "source/common/config/type_to_endpoint.h" -#include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h" namespace Envoy { namespace Config { diff --git a/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc b/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc index 82d633e1c2c9..c7296942e9e7 100644 --- a/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc +++ b/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc @@ -2,10 +2,10 @@ #include "source/common/config/custom_config_validators_impl.h" #include "source/common/config/type_to_endpoint.h" -#include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h" namespace Envoy { namespace Config { diff --git a/source/common/config/xds_mux/BUILD b/source/extensions/config_subscription/grpc/xds_mux/BUILD similarity index 95% rename from source/common/config/xds_mux/BUILD rename to source/extensions/config_subscription/grpc/xds_mux/BUILD index 0fd4710cb74a..9dd3a7318c7b 100644 --- a/source/common/config/xds_mux/BUILD +++ b/source/extensions/config_subscription/grpc/xds_mux/BUILD @@ -1,12 +1,13 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", "envoy_cc_library", - "envoy_package", + "envoy_extension_package", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_extension_package() envoy_cc_library( name = "delta_subscription_state_lib", @@ -55,7 +56,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_cc_extension( name = "grpc_mux_lib", srcs = ["grpc_mux_impl.cc"], hdrs = ["grpc_mux_impl.h"], diff --git a/source/common/config/xds_mux/delta_subscription_state.cc b/source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.cc similarity index 99% rename from source/common/config/xds_mux/delta_subscription_state.cc rename to source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.cc index f7bd686d93a0..31a640801a14 100644 --- a/source/common/config/xds_mux/delta_subscription_state.cc +++ b/source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.cc @@ -1,4 +1,4 @@ -#include "source/common/config/xds_mux/delta_subscription_state.h" +#include "source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h" #include "envoy/event/dispatcher.h" #include "envoy/service/discovery/v3/discovery.pb.h" diff --git a/source/common/config/xds_mux/delta_subscription_state.h b/source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h similarity index 98% rename from source/common/config/xds_mux/delta_subscription_state.h rename to source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h index 31fcbcdf8547..70b9801d5920 100644 --- a/source/common/config/xds_mux/delta_subscription_state.h +++ b/source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h @@ -5,7 +5,7 @@ #include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "source/common/config/api_version.h" -#include "source/common/config/xds_mux/subscription_state.h" +#include "source/extensions/config_subscription/grpc/xds_mux/subscription_state.h" #include "absl/container/node_hash_map.h" #include "absl/types/optional.h" diff --git a/source/common/config/xds_mux/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc similarity index 87% rename from source/common/config/xds_mux/grpc_mux_impl.cc rename to source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc index af981cd4dbec..a3c5c9383340 100644 --- a/source/common/config/xds_mux/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc @@ -1,4 +1,4 @@ -#include "source/common/config/xds_mux/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h" #include "envoy/service/discovery/v3/discovery.pb.h" @@ -41,8 +41,7 @@ GrpcMuxImpl::GrpcMuxImpl( const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - CustomConfigValidatorsPtr&& config_validators, - JitteredExponentialBackOffStrategyPtr backoff_strategy, + CustomConfigValidatorsPtr&& config_validators, BackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, XdsResourcesDelegateOptRef xds_resources_delegate, const std::string& target_xds_authority) : grpc_stream_(this, std::move(async_client), service_method, dispatcher, scope, @@ -367,7 +366,7 @@ GrpcMuxDelta::GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispat const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node, CustomConfigValidatorsPtr&& config_validators, - JitteredExponentialBackOffStrategyPtr backoff_strategy, + BackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker) : GrpcMuxImpl(std::make_unique(dispatcher), skip_subsequent_node, local_info, std::move(async_client), dispatcher, service_method, scope, @@ -390,7 +389,7 @@ GrpcMuxSotw::GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatch const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node, CustomConfigValidatorsPtr&& config_validators, - JitteredExponentialBackOffStrategyPtr backoff_strategy, + BackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, XdsResourcesDelegateOptRef xds_resources_delegate, const std::string& target_xds_authority) @@ -407,6 +406,53 @@ Config::GrpcMuxWatchPtr NullGrpcMuxImpl::addWatch(const std::string&, throw EnvoyException("ADS must be configured to support an ADS config source"); } +class DeltaGrpcMuxFactory : public MuxFactory { +public: + std::string name() const override { return "envoy.config_mux.delta_grpc_mux_factory"; } + void shutdownAll() override { return GrpcMuxDelta::shutdownAll(); } + std::shared_ptr + create(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + Random::RandomGenerator&, Stats::Scope& scope, + const envoy::config::core::v3::ApiConfigSource& ads_config, + const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, + BackOffStrategyPtr&& backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, + XdsResourcesDelegateOptRef) override { + return std::make_shared( + std::move(async_client), dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v3.AggregatedDiscoveryService." + "DeltaAggregatedResources"), + scope, Envoy::Config::Utility::parseRateLimitSettings(ads_config), local_info, + ads_config.set_node_on_first_message_only(), std::move(config_validators), + std::move(backoff_strategy), xds_config_tracker); + } +}; + +class SotwGrpcMuxFactory : public MuxFactory { +public: + std::string name() const override { return "envoy.config_mux.sotw_grpc_mux_factory"; } + void shutdownAll() override { return GrpcMuxSotw::shutdownAll(); } + std::shared_ptr + create(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + Random::RandomGenerator&, Stats::Scope& scope, + const envoy::config::core::v3::ApiConfigSource& ads_config, + const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, + BackOffStrategyPtr&& backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, + XdsResourcesDelegateOptRef) override { + return std::make_shared( + std::move(async_client), dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v3.AggregatedDiscoveryService." + "StreamAggregatedResources"), + scope, Envoy::Config::Utility::parseRateLimitSettings(ads_config), local_info, + ads_config.set_node_on_first_message_only(), std::move(config_validators), + std::move(backoff_strategy), xds_config_tracker); + } +}; + +REGISTER_FACTORY(DeltaGrpcMuxFactory, MuxFactory); +REGISTER_FACTORY(SotwGrpcMuxFactory, MuxFactory); + } // namespace XdsMux } // namespace Config } // namespace Envoy diff --git a/source/common/config/xds_mux/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h similarity index 95% rename from source/common/config/xds_mux/grpc_mux_impl.h rename to source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h index 03c6a9867141..e5434f9eaaeb 100644 --- a/source/common/config/xds_mux/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h @@ -23,9 +23,9 @@ #include "source/common/config/grpc_stream.h" #include "source/common/config/pausable_ack_queue.h" #include "source/common/config/watch_map.h" -#include "source/common/config/xds_mux/delta_subscription_state.h" -#include "source/common/config/xds_mux/sotw_subscription_state.h" #include "source/common/grpc/common.h" +#include "source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h" +#include "source/extensions/config_subscription/grpc/xds_mux/sotw_subscription_state.h" #include "absl/container/node_hash_map.h" @@ -63,8 +63,7 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - CustomConfigValidatorsPtr&& config_validators, - JitteredExponentialBackOffStrategyPtr backoff_strategy, + CustomConfigValidatorsPtr&& config_validators, BackOffStrategyPtr backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, XdsResourcesDelegateOptRef xds_resources_delegate = absl::nullopt, const std::string& target_xds_authority = ""); @@ -225,8 +224,7 @@ class GrpcMuxDelta : public GrpcMuxImpl( - "envoy.config_mux.new_grpc_mux_factory"); - if (factory) { - factory->shutdownAll(); - } - factory = - Config::Utility::getFactoryByName("envoy.config_mux.grpc_mux_factory"); - if (factory) { - factory->shutdownAll(); + std::vector muxes = { + "envoy.config_mux.new_grpc_mux_factory", "envoy.config_mux.grpc_mux_factory", + "envoy.config_mux.delta_grpc_mux_factory", "envoy.config_mux.sotw_grpc_mux_factory"}; + for (const auto& name : muxes) { + auto* factory = Config::Utility::getFactoryByName(name); + if (factory) { + factory->shutdownAll(); + } } - Config::XdsMux::GrpcMuxSotw::shutdownAll(); - Config::XdsMux::GrpcMuxDelta::shutdownAll(); if (overload_manager_) { overload_manager_->stop(); diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 48adc3bb329e..60f8f952f964 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -31,54 +31,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "sotw_subscription_state_test", - srcs = ["sotw_subscription_state_test.cc"], - external_deps = ["abseil_flat_hash_set"], - deps = [ - "//source/common/config:resource_name_lib", - "//source/common/config/xds_mux:sotw_subscription_state_lib", - "//source/common/stats:isolated_store_lib", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", - ], -) - -envoy_cc_test( - name = "xds_grpc_mux_impl_test", - srcs = ["xds_grpc_mux_impl_test.cc"], - deps = [ - "//source/common/config:protobuf_link_hacks", - "//source/common/config:resource_name_lib", - "//source/common/config/xds_mux:grpc_mux_lib", - "//source/common/protobuf", - "//test/common/stats:stat_test_utility_lib", - "//test/config:v2_link_hacks", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/config:custom_config_validators_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - "//test/test_common:resources_lib", - "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", - "//test/test_common:utility_lib", - "@envoy_api//envoy/api/v2:pkg_cc_proto", - "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "grpc_stream_test", srcs = ["grpc_stream_test.cc"], diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index e6e0d890163f..e1e41448ef87 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -11,9 +11,9 @@ #include "source/common/common/hash.h" #include "source/common/config/api_version.h" -#include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h" #include "test/common/config/subscription_test_harness.h" #include "test/mocks/config/custom_config_validators.h" diff --git a/test/extensions/clusters/eds/BUILD b/test/extensions/clusters/eds/BUILD index f970a51b670b..fe0af3d42039 100644 --- a/test/extensions/clusters/eds/BUILD +++ b/test/extensions/clusters/eds/BUILD @@ -49,9 +49,9 @@ envoy_cc_benchmark_binary( "//envoy/config:xds_resources_delegate_interface", "//source/common/config:protobuf_link_hacks", "//source/common/config:utility_lib", - "//source/common/config/xds_mux:grpc_mux_lib", "//source/extensions/clusters/eds:eds_lib", "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "//source/extensions/config_subscription/grpc/xds_mux:grpc_mux_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/upstream:utility_lib", diff --git a/test/extensions/clusters/eds/eds_speed_test.cc b/test/extensions/clusters/eds/eds_speed_test.cc index 98737b2b79b4..d30b1e1e1129 100644 --- a/test/extensions/clusters/eds/eds_speed_test.cc +++ b/test/extensions/clusters/eds/eds_speed_test.cc @@ -12,11 +12,11 @@ #include "source/common/config/protobuf_link_hacks.h" #include "source/common/config/utility.h" -#include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/singleton/manager_impl.h" #include "source/extensions/clusters/eds/eds.h" #include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h" #include "source/server/transport_socket_config_impl.h" #include "test/benchmark/main.h" diff --git a/test/extensions/config_subscription/grpc/BUILD b/test/extensions/config_subscription/grpc/BUILD index 5ba88b5748df..a6ee6147e4f4 100644 --- a/test/extensions/config_subscription/grpc/BUILD +++ b/test/extensions/config_subscription/grpc/BUILD @@ -14,9 +14,9 @@ envoy_cc_test( srcs = ["new_grpc_mux_impl_test.cc"], deps = [ "//source/common/config:protobuf_link_hacks", - "//source/common/config/xds_mux:grpc_mux_lib", "//source/common/protobuf", "//source/extensions/config_subscription/grpc:new_grpc_mux_lib", + "//source/extensions/config_subscription/grpc/xds_mux:grpc_mux_lib", "//test/common/stats:stat_test_utility_lib", "//test/config:v2_link_hacks", "//test/mocks:common_lib", @@ -91,10 +91,10 @@ envoy_cc_test( name = "delta_subscription_state_test", srcs = ["delta_subscription_state_test.cc"], deps = [ - "//source/common/config/xds_mux:delta_subscription_state_lib", "//source/common/stats:isolated_store_lib", "//source/extensions/config_subscription/grpc:delta_subscription_state_lib", "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "//source/extensions/config_subscription/grpc/xds_mux:delta_subscription_state_lib", "//test/mocks:common_lib", "//test/mocks/config:config_mocks", "//test/mocks/event:event_mocks", @@ -147,3 +147,51 @@ envoy_cc_test_library( "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) + +envoy_cc_test( + name = "sotw_subscription_state_test", + srcs = ["sotw_subscription_state_test.cc"], + external_deps = ["abseil_flat_hash_set"], + deps = [ + "//source/common/config:resource_name_lib", + "//source/common/stats:isolated_store_lib", + "//source/extensions/config_subscription/grpc/xds_mux:sotw_subscription_state_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + ], +) + +envoy_cc_test( + name = "xds_grpc_mux_impl_test", + srcs = ["xds_grpc_mux_impl_test.cc"], + deps = [ + "//source/common/config:protobuf_link_hacks", + "//source/common/config:resource_name_lib", + "//source/common/protobuf", + "//source/extensions/config_subscription/grpc/xds_mux:grpc_mux_lib", + "//test/common/stats:stat_test_utility_lib", + "//test/config:v2_link_hacks", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/config:custom_config_validators_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:resources_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:pkg_cc_proto", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc b/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc index b644f8c6b4ee..7f29e466dbc4 100644 --- a/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc +++ b/test/extensions/config_subscription/grpc/delta_subscription_state_test.cc @@ -5,9 +5,9 @@ #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/config/utility.h" -#include "source/common/config/xds_mux/delta_subscription_state.h" #include "source/common/stats/isolated_store_impl.h" #include "source/extensions/config_subscription/grpc/delta_subscription_state.h" +#include "source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h" #include "test/mocks/config/mocks.h" #include "test/mocks/event/mocks.h" diff --git a/test/extensions/config_subscription/grpc/delta_subscription_test_harness.h b/test/extensions/config_subscription/grpc/delta_subscription_test_harness.h index 956af6412b57..77c4310137aa 100644 --- a/test/extensions/config_subscription/grpc/delta_subscription_test_harness.h +++ b/test/extensions/config_subscription/grpc/delta_subscription_test_harness.h @@ -8,10 +8,10 @@ #include "envoy/config/xds_config_tracker.h" #include "envoy/service/discovery/v3/discovery.pb.h" -#include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/grpc/common.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h" #include "test/common/config/subscription_test_harness.h" #include "test/mocks/common.h" diff --git a/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc index 7fcee773e465..3712709f9f73 100644 --- a/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc @@ -9,9 +9,9 @@ #include "source/common/common/empty_string.h" #include "source/common/config/protobuf_link_hacks.h" #include "source/common/config/utility.h" -#include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h" #include "test/common/stats/stat_test_utility.h" #include "test/config/v2_link_hacks.h" diff --git a/test/common/config/sotw_subscription_state_test.cc b/test/extensions/config_subscription/grpc/sotw_subscription_state_test.cc similarity index 99% rename from test/common/config/sotw_subscription_state_test.cc rename to test/extensions/config_subscription/grpc/sotw_subscription_state_test.cc index 462949e96c3c..8e751ff0965d 100644 --- a/test/common/config/sotw_subscription_state_test.cc +++ b/test/extensions/config_subscription/grpc/sotw_subscription_state_test.cc @@ -3,8 +3,8 @@ #include "source/common/config/resource_name.h" #include "source/common/config/utility.h" -#include "source/common/config/xds_mux/sotw_subscription_state.h" #include "source/common/stats/isolated_store_impl.h" +#include "source/extensions/config_subscription/grpc/xds_mux/sotw_subscription_state.h" #include "test/mocks/config/mocks.h" #include "test/mocks/event/mocks.h" diff --git a/test/common/config/xds_grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc similarity index 99% rename from test/common/config/xds_grpc_mux_impl_test.cc rename to test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc index 3946bad36e76..8530afb7a64f 100644 --- a/test/common/config/xds_grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc @@ -10,8 +10,8 @@ #include "source/common/config/protobuf_link_hacks.h" #include "source/common/config/resource_name.h" #include "source/common/config/utility.h" -#include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/protobuf/protobuf.h" +#include "source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h" #include "test/common/stats/stat_test_utility.h" #include "test/config/v2_link_hacks.h" diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 93807749cc1b..fcc07960f9f5 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -7,7 +7,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/api:82.6" "source/common/api/posix:81.3" "source/common/common/posix:92.7" -"source/common/config:96.1" +"source/common/config:95.6" "source/common/crypto:88.1" "source/common/event:95.1" # Emulated edge events guards don't report LCOV "source/common/filesystem/posix:96.2" # FileReadToEndNotReadable fails in some env; createPath can't test all failure branches. @@ -18,7 +18,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/network/dns_resolver:91.6" # A few lines of MacOS code not tested in linux scripts. Tested in MacOS scripts "source/common/protobuf:96.3" "source/common/quic:93.5" -"source/common/router:96.3" +"source/common/router:96.2" "source/common/secret:95.0" "source/common/signal:87.2" # Death tests don't report LCOV "source/common/singleton:95.7" diff --git a/tools/extensions/extensions_schema.yaml b/tools/extensions/extensions_schema.yaml index e17c33fe4ba9..7558fb7f8068 100644 --- a/tools/extensions/extensions_schema.yaml +++ b/tools/extensions/extensions_schema.yaml @@ -66,6 +66,7 @@ categories: - envoy.compression.decompressor - envoy.config.validators - envoy.config_subscription +- envoy.config_mux - envoy.filters.http - envoy.filters.listener - envoy.filters.network From 77c6e2f5e606edc768c26be540574ffde94f104e Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 17 May 2023 13:22:00 -0400 Subject: [PATCH 266/740] Add more CPU for next batch of flaky integration tests (#27433) Signed-off-by: Yan Avlasov --- .../http/alternate_protocols_cache/BUILD | 3 + .../filters/http/custom_response/BUILD | 3 + test/integration/BUILD | 99 ++++++++++++++++++- 3 files changed, 104 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/http/alternate_protocols_cache/BUILD b/test/extensions/filters/http/alternate_protocols_cache/BUILD index bf1cd383e6d3..f76cee6c60b8 100644 --- a/test/extensions/filters/http/alternate_protocols_cache/BUILD +++ b/test/extensions/filters/http/alternate_protocols_cache/BUILD @@ -32,6 +32,9 @@ envoy_extension_cc_test( "//test/config/integration/certs", ], extension_names = ["envoy.filters.http.alternate_protocols_cache"], + tags = [ + "cpu:3", + ], deps = [ "//source/extensions/filters/http/alternate_protocols_cache:config", "//source/extensions/key_value/file_based:config_lib", diff --git a/test/extensions/filters/http/custom_response/BUILD b/test/extensions/filters/http/custom_response/BUILD index e7f6e01bae70..01c7daba1d5c 100644 --- a/test/extensions/filters/http/custom_response/BUILD +++ b/test/extensions/filters/http/custom_response/BUILD @@ -82,6 +82,9 @@ envoy_extension_cc_test( ], extension_names = ["envoy.filters.http.custom_response"], shard_count = 4, + tags = [ + "cpu:3", + ], deps = [ ":utility_lib", "//source/common/config:well_known_names", diff --git a/test/integration/BUILD b/test/integration/BUILD index fbcbb3b2922b..fbcbfbfaf828 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -254,6 +254,9 @@ envoy_cc_test( name = "cluster_filter_integration_test", size = "large", srcs = ["cluster_filter_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":integration_lib", @@ -270,6 +273,9 @@ envoy_cc_test( name = "custom_cluster_integration_test", size = "large", srcs = ["custom_cluster_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/common/upstream:load_balancer_lib", @@ -426,6 +432,9 @@ envoy_cc_test( name = "http_conn_pool_integration_test", size = "large", srcs = ["http_conn_pool_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", "//test/test_common:test_time_lib", @@ -437,6 +446,9 @@ envoy_cc_test( name = "circuit_breakers_integration_test", size = "large", srcs = ["circuit_breakers_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", "//test/test_common:test_time_lib", @@ -513,6 +525,9 @@ envoy_cc_test( name = "shadow_policy_integration_test", size = "large", srcs = ["shadow_policy_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":integration_lib", @@ -532,6 +547,9 @@ envoy_cc_test( name = "upstream_http_filter_integration_test", size = "large", srcs = ["upstream_http_filter_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":integration_lib", @@ -591,6 +609,9 @@ envoy_cc_test( srcs = [ "http_subset_lb_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/extensions/load_balancing_policies/subset:config", @@ -611,6 +632,9 @@ envoy_cc_test( data = [ "//test/config/integration/certs", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/extensions/load_balancing_policies/subset:config", @@ -627,6 +651,9 @@ envoy_cc_test( srcs = [ "header_casing_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", @@ -653,6 +680,9 @@ envoy_cc_test( "http_timeout_integration_test.cc", "http_timeout_integration_test.h", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "@envoy_api//envoy/extensions/filters/http/router/v3:pkg_cc_proto", @@ -794,6 +824,9 @@ envoy_cc_test( "integration_admin_test.h", ], ), + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", "//envoy/http:header_map_interface", @@ -1251,7 +1284,10 @@ envoy_cc_test( "redirect_integration_test.cc", ], shard_count = 8, - tags = ["nofips"], + tags = [ + "cpu:3", + "nofips", + ], deps = [ ":http_protocol_integration_lib", "//source/common/http:header_map_lib", @@ -1358,6 +1394,9 @@ envoy_cc_test( "//test/config/integration/certs", ], shard_count = 2, + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":integration_lib", @@ -1383,6 +1422,9 @@ envoy_cc_test( size = "large", srcs = ["header_prefix_integration_test.cc"], coverage = False, + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", @@ -1444,6 +1486,9 @@ envoy_cc_test( srcs = envoy_select_admin_functionality([ "rtds_integration_test.cc", ]), + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//test/common/grpc:grpc_client_integration_lib", @@ -1537,6 +1582,7 @@ envoy_cc_test( # TestEnvironment::renameFile() fails on Windows. The renameFile() implementation does not # correctly handle symlinks. tags = [ + "cpu:3", "fails_on_clang_cl", "fails_on_windows", ], @@ -1573,6 +1619,9 @@ envoy_cc_test( srcs = [ "sds_generic_secret_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//envoy/registry", @@ -1694,6 +1743,9 @@ envoy_cc_test( srcs = [ "tcp_conn_pool_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":integration_lib", "//envoy/server:filter_config_interface", @@ -1777,6 +1829,9 @@ envoy_cc_test( ["dynamic_validation_integration_test.cc"], ), data = ["//test/config/integration:server_xds_files"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/common/stats:stats_lib", @@ -1795,6 +1850,9 @@ envoy_cc_test( "//test/config/integration:server_xds_files", "//test/config/integration/certs", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":http_protocol_integration_lib", @@ -1826,6 +1884,9 @@ envoy_cc_test( data = [ "//test/config/integration/certs", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/common/http:header_map_lib", @@ -1993,6 +2054,9 @@ envoy_cc_test( srcs = [ "scoped_rds_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":scoped_rds_lib", @@ -2019,6 +2083,9 @@ envoy_cc_test( srcs = [ "listener_lds_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/common/config:api_version_lib", @@ -2082,6 +2149,9 @@ envoy_cc_test( "local_reply_integration_test.cc", ], shard_count = 2, + tags = [ + "cpu:2", + ], deps = [ ":http_integration_lib", ":http_protocol_integration_lib", @@ -2095,6 +2165,9 @@ envoy_cc_test( srcs = [ "command_formatter_extension_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//test/common/formatter:command_extension_lib", @@ -2106,6 +2179,9 @@ envoy_cc_test( name = "health_check_integration_test", size = "large", srcs = ["health_check_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":integration_lib", @@ -2122,6 +2198,9 @@ envoy_cc_test( srcs = [ "cluster_upstream_extension_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/common/config:api_version_lib", @@ -2221,6 +2300,9 @@ envoy_cc_test( srcs = [ "original_ip_detection_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", "//source/extensions/http/original_ip_detection/custom_header:config", @@ -2233,6 +2315,9 @@ envoy_cc_test( name = "weighted_cluster_integration_test", size = "large", srcs = ["weighted_cluster_integration_test.cc"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":integration_lib", @@ -2254,6 +2339,9 @@ envoy_cc_test( srcs = [ "typed_metadata_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", ":typed_metadata_integration_proto_cc_proto", @@ -2296,6 +2384,9 @@ envoy_cc_test( "xds_delegate_extension_integration_test.cc", ]), external_deps = ["abseil_strings"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":xds_delegate_test_config_proto_cc_proto", @@ -2318,6 +2409,9 @@ envoy_cc_test( size = "large", srcs = ["xds_config_tracker_integration_test.cc"], external_deps = ["abseil_strings"], + tags = [ + "cpu:3", + ], deps = [ ":http_integration_lib", ":xds_config_tracker_test_proto_cc_proto", @@ -2344,6 +2438,9 @@ envoy_cc_test( srcs = [ "default_header_validator_integration_test.cc", ], + tags = [ + "cpu:3", + ], deps = [ ":http_protocol_integration_lib", "//source/common/http:header_map_lib", From ce5c880dd6d96a499742038d9ff56664046cdfc7 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 17 May 2023 14:28:14 -0400 Subject: [PATCH 267/740] Move HTTP/1.0 check into HTTP1 codec (#27415) Signed-off-by: Yan Avlasov --- source/common/http/conn_manager_impl.cc | 23 +--------- source/common/http/http1/codec_impl.cc | 24 +++++++++- source/common/http/http1/codec_impl.h | 1 + test/common/http/conn_manager_impl_test.cc | 28 ------------ test/common/http/http1/codec_impl_test.cc | 51 +++++++++++++++++++++- 5 files changed, 75 insertions(+), 52 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 551c64212041..4696ebf69d72 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1084,32 +1084,11 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt // them as early as possible. const Protocol protocol = connection_manager_.codec_->protocol(); state_.saw_connection_close_ = HeaderUtility::shouldCloseConnection(protocol, *request_headers_); + filter_manager_.streamInfo().protocol(protocol); // We end the decode here to mark that the downstream stream is complete. maybeEndDecode(end_stream); - // Make sure we are getting a codec version we support. - if (protocol == Protocol::Http10) { - // Assume this is HTTP/1.0. This is fine for HTTP/0.9 but this code will also affect any - // requests with non-standard version numbers (0.9, 1.3), basically anything which is not - // HTTP/1.1. - // - // The protocol may have shifted in the HTTP/1.0 case so reset it. - filter_manager_.streamInfo().protocol(protocol); - if (!connection_manager_.config_.http1Settings().accept_http_10_) { - // Send "Upgrade Required" if HTTP/1.0 support is not explicitly configured on. - sendLocalReply(Code::UpgradeRequired, "", nullptr, absl::nullopt, - StreamInfo::ResponseCodeDetails::get().LowVersion); - return; - } - if (!request_headers_->Host() && - !connection_manager_.config_.http1Settings().default_host_for_http_10_.empty()) { - // Add a default host if configured to do so. - request_headers_->setHost( - connection_manager_.config_.http1Settings().default_host_for_http_10_); - } - } - if (!validateHeaders()) { ENVOY_STREAM_LOG(debug, "request headers validation failed\n{}", *this, *request_headers_); return; diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index e43c438f4714..7bef6c5cdc3d 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -840,7 +840,7 @@ Status ConnectionImpl::onHeaderValueImpl(const char* data, size_t length) { StatusOr ConnectionImpl::onHeadersCompleteImpl() { ASSERT(!processing_trailers_); ASSERT(dispatching_); - ENVOY_CONN_LOG(trace, "onHeadersCompleteBase", connection_); + ENVOY_CONN_LOG(trace, "onHeadersCompleteImpl", connection_); RETURN_IF_ERROR(completeCurrentHeader()); if (!parser_->isHttp11()) { @@ -1154,6 +1154,27 @@ Status ServerConnectionImpl::handlePath(RequestHeaderMap& headers, absl::string_ return okStatus(); } +Status ServerConnectionImpl::checkProtocolVersion(RequestHeaderMap& headers) { + if (protocol() == Protocol::Http10) { + // Assume this is HTTP/1.0. This is fine for HTTP/0.9 but this code will also affect any + // requests with non-standard version numbers (0.9, 1.3), basically anything which is not + // HTTP/1.1. + // + // The protocol may have shifted in the HTTP/1.0 case so reset it. + if (!codec_settings_.accept_http_10_) { + // Send "Upgrade Required" if HTTP/1.0 support is not explicitly configured on. + error_code_ = Http::Code::UpgradeRequired; + RETURN_IF_ERROR(sendProtocolError(StreamInfo::ResponseCodeDetails::get().LowVersion)); + return codecProtocolError("Upgrade required for HTTP/1.0 or HTTP/0.9"); + } + if (!headers.Host() && !codec_settings_.default_host_for_http_10_.empty()) { + // Add a default host if configured to do so. + headers.setHost(codec_settings_.default_host_for_http_10_); + } + } + return okStatus(); +} + Envoy::StatusOr ServerConnectionImpl::onHeadersCompleteBase() { // Handle the case where response happens prior to request complete. It's up to upper layer code // to disconnect the connection but we shouldn't fire any more events since it doesn't make @@ -1187,6 +1208,7 @@ Envoy::StatusOr ServerConnectionImpl::onHeadersCompleteBase() { ASSERT(active_request_->request_url_.empty()); headers->setMethod(parser_->methodName()); + RETURN_IF_ERROR(checkProtocolVersion(*headers)); // Make sure the host is valid. auto details = HeaderUtility::requestHeadersValid(*headers); diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index 4343f4f96176..2238f750fe7b 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -557,6 +557,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { Status doFloodProtectionChecks() const; Status checkHeaderNameForUnderscores() override; + Status checkProtocolVersion(RequestHeaderMap& headers); ServerConnectionCallbacks& callbacks_; std::unique_ptr active_request_; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index d33ab4243850..befb0aa44d75 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -3934,34 +3934,6 @@ TEST_F(HttpConnectionManagerImplTest, MaxStreamDurationFiredReturn504IfRequestWa duration_timer->invokeCallback(); } -TEST_F(HttpConnectionManagerImplTest, Http10Rejected) { - setup(false, ""); - EXPECT_CALL(*codec_, protocol()).Times(AnyNumber()).WillRepeatedly(Return(Protocol::Http10)); - EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status { - decoder_ = &conn_manager_->newStream(response_encoder_); - RequestHeaderMapPtr headers{ - new TestRequestHeaderMapImpl{{":authority", "host"}, {":method", "GET"}, {":path", "/"}}}; - decoder_->decodeHeaders(std::move(headers), false); - data.drain(4); - return Http::okStatus(); - })); - - EXPECT_CALL(response_encoder_, encodeHeaders(_, true)) - .WillOnce(Invoke([](const ResponseHeaderMap& headers, bool) -> void { - EXPECT_EQ("426", headers.getStatusValue()); - EXPECT_EQ("close", headers.getConnectionValue()); - })); - // No delay close for HTTP/1.0, even if the request is not complete. - // Note there may be more than one close: the important thing is one does not - // kick off delay. - EXPECT_CALL(filter_callbacks_.connection_, close(_)).Times(AnyNumber()); - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) - .Times(AnyNumber()); - - Buffer::OwnedImpl fake_input("1234"); - conn_manager_->onData(fake_input, false); -} - TEST_F(HttpConnectionManagerImplTest, MaxStreamDurationCallbackNotCalledIfResetStreamValidly) { max_stream_duration_ = std::chrono::milliseconds(5000); setup(false, ""); diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index c8e8da3c86ba..54bfa2561372 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -919,6 +919,7 @@ TEST_P(Http1ServerConnectionImplTest, CodecHasDefaultStreamErrorIfNotSet) { } TEST_P(Http1ServerConnectionImplTest, Http10) { + codec_settings_.accept_http_10_ = true; initialize(); InSequence sequence; @@ -936,7 +937,29 @@ TEST_P(Http1ServerConnectionImplTest, Http10) { EXPECT_EQ(Protocol::Http10, codec_->protocol()); } +TEST_P(Http1ServerConnectionImplTest, Http10HostAdded) { + codec_settings_.accept_http_10_ = true; + codec_settings_.default_host_for_http_10_ = "example.com"; + initialize(); + + InSequence sequence; + + MockRequestDecoder decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + + TestRequestHeaderMapImpl expected_headers{ + {":path", "/"}, {":method", "GET"}, {":authority", "example.com"}}; + EXPECT_CALL(decoder, decodeHeaders_(HeaderMapEqual(&expected_headers), true)); + + Buffer::OwnedImpl buffer("GET / HTTP/1.0\r\n\r\n"); + auto status = codec_->dispatch(buffer); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(0U, buffer.length()); + EXPECT_EQ(Protocol::Http10, codec_->protocol()); +} + TEST_P(Http1ServerConnectionImplTest, Http10AbsoluteNoOp) { + codec_settings_.accept_http_10_ = true; initialize(); TestRequestHeaderMapImpl expected_headers{{":path", "/"}, {":method", "GET"}}; @@ -945,6 +968,7 @@ TEST_P(Http1ServerConnectionImplTest, Http10AbsoluteNoOp) { } TEST_P(Http1ServerConnectionImplTest, Http10Absolute) { + codec_settings_.accept_http_10_ = true; initialize(); TestRequestHeaderMapImpl expected_headers{{":authority", "www.somewhere.com"}, @@ -956,6 +980,7 @@ TEST_P(Http1ServerConnectionImplTest, Http10Absolute) { } TEST_P(Http1ServerConnectionImplTest, Http10MultipleResponses) { + codec_settings_.accept_http_10_ = true; initialize(); MockRequestDecoder decoder; @@ -978,7 +1003,7 @@ TEST_P(Http1ServerConnectionImplTest, Http10MultipleResponses) { ON_CALL(connection_, write(_, _)).WillByDefault(AddBufferToString(&output)); TestResponseHeaderMapImpl headers{{":status", "200"}}; response_encoder->encodeHeaders(headers, true); - EXPECT_EQ("HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n", output); + EXPECT_EQ("HTTP/1.0 200 OK\r\ncontent-length: 0\r\n\r\n", output); EXPECT_EQ(Protocol::Http10, codec_->protocol()); } @@ -995,6 +1020,7 @@ TEST_P(Http1ServerConnectionImplTest, Http10MultipleResponses) { } TEST_P(Http1ServerConnectionImplTest, HttpVersion) { + codec_settings_.accept_http_10_ = true; // SPELLCHECKER(off) HTTPStringTestCase kRequestHTTPStringTestCases[] = { {"", {}, {}}, // HTTP/0.9 has no HTTP-version. @@ -4550,5 +4576,28 @@ TEST_P(Http1ServerConnectionImplTest, MultipleTransferEncoding) { #endif } +TEST_P(Http1ServerConnectionImplTest, Http10Rejected) { + initialize(); + InSequence s; + + StrictMock decoder; + Http::ResponseEncoder* response_encoder = nullptr; + EXPECT_CALL(callbacks_, newStream(_, _)) + .WillOnce(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoder& { + response_encoder = &encoder; + return decoder; + })); + EXPECT_CALL(decoder, sendLocalReply(Http::Code::UpgradeRequired, _, _, _, "low_version")); + + Buffer::OwnedImpl buffer("GET / HTTP/1.0\r\n\r\n"); + + auto status = codec_->dispatch(buffer); + + EXPECT_TRUE(isCodecProtocolError(status)); + EXPECT_EQ("low_version", response_encoder->getStream().responseDetails()); + + EXPECT_THAT(status.message(), StartsWith("Upgrade required")); +} + } // namespace Http } // namespace Envoy From 0f378c2f9bb0d529d1d4b8139c1adf66b74a34a7 Mon Sep 17 00:00:00 2001 From: Kuat Date: Wed, 17 May 2023 12:01:30 -0700 Subject: [PATCH 268/740] deps: update cel-cpp (#27414) * deps: update cel-cpp Signed-off-by: Kuat Yessenov --- bazel/cel-cpp.patch | 146 +++++++++++++++++++++++++++++---- bazel/repository_locations.bzl | 6 +- 2 files changed, 134 insertions(+), 18 deletions(-) diff --git a/bazel/cel-cpp.patch b/bazel/cel-cpp.patch index 677ea33fa5a8..ac7d82733c0f 100644 --- a/bazel/cel-cpp.patch +++ b/bazel/cel-cpp.patch @@ -1,15 +1,131 @@ -# -# Temporary patch to cel-cpp to accomodate switch from ABSL types to STL types. -# -# Patch will be removed once build issues in new cel-cpp are addressed. -# ---- a/eval/public/ast_traverse.cc 2023-01-23 17:38:48.730240356 +0000 -+++ b/eval/public/ast_traverse.cc 2023-01-23 17:39:27.182207871 +0000 -@@ -17,6 +17,7 @@ - #include - - #include "google/api/expr/v1alpha1/syntax.pb.h" -+#include "absl/base/attributes.h" - #include "absl/types/variant.h" - #include "eval/public/ast_visitor.h" - #include "eval/public/source_position.h" +diff --git a/eval/eval/evaluator_stack.h b/eval/eval/evaluator_stack.h +index 1ecab27..9df65d7 100644 +--- a/eval/eval/evaluator_stack.h ++++ b/eval/eval/evaluator_stack.h +@@ -5,6 +5,7 @@ + #include + #include + ++#include "absl/log/log.h" + #include "absl/types/span.h" + #include "eval/eval/attribute_trail.h" + #include "eval/public/cel_value.h" +diff --git a/eval/public/cel_expr_builder_factory.h b/eval/public/cel_expr_builder_factory.h +index 7321e29..0d0d5e6 100644 +--- a/eval/public/cel_expr_builder_factory.h ++++ b/eval/public/cel_expr_builder_factory.h +@@ -1,6 +1,7 @@ + #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_CEL_EXPR_BUILDER_FACTORY_H_ + #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_CEL_EXPR_BUILDER_FACTORY_H_ + ++#include "absl/log/log.h" + #include "google/protobuf/descriptor.h" + #include "eval/public/cel_expression.h" + #include "eval/public/cel_options.h" +diff --git a/eval/public/cel_value.h b/eval/public/cel_value.h +index b2d13f8..73e1909 100644 +--- a/eval/public/cel_value.h ++++ b/eval/public/cel_value.h +@@ -25,7 +25,6 @@ + #include "absl/base/attributes.h" + #include "absl/base/macros.h" + #include "absl/base/optimization.h" +-#include "absl/log/log.h" + #include "absl/status/status.h" + #include "absl/status/statusor.h" + #include "absl/strings/str_cat.h" +@@ -481,8 +480,8 @@ class CelValue { + } + + // Crashes with a null pointer error. +- static void CrashNullPointer(Type type) ABSL_ATTRIBUTE_COLD { +- LOG(FATAL) << "Null pointer supplied for " << TypeName(type); // Crash ok ++ static void CrashNullPointer(Type) ABSL_ATTRIBUTE_COLD { ++ ABSL_ASSERT(false); + } + + // Null pointer checker for pointer-based types. +@@ -493,11 +492,9 @@ class CelValue { + } + + // Crashes with a type mismatch error. +- static void CrashTypeMismatch(Type requested_type, +- Type actual_type) ABSL_ATTRIBUTE_COLD { +- LOG(FATAL) << "Type mismatch" // Crash ok +- << ": expected " << TypeName(requested_type) // Crash ok +- << ", encountered " << TypeName(actual_type); // Crash ok ++ static void CrashTypeMismatch(Type, ++ Type) ABSL_ATTRIBUTE_COLD { ++ ABSL_ASSERT(false); + } + + // Gets value of type specified +diff --git a/eval/public/portable_cel_expr_builder_factory.cc b/eval/public/portable_cel_expr_builder_factory.cc +index 80ac45c..7dceb93 100644 +--- a/eval/public/portable_cel_expr_builder_factory.cc ++++ b/eval/public/portable_cel_expr_builder_factory.cc +@@ -20,6 +20,7 @@ + #include + #include + ++#include "absl/log/log.h" + #include "absl/status/status.h" + #include "eval/compiler/flat_expr_builder.h" + #include "eval/public/cel_options.h" +diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD +index 9187518..5151bb0 100644 +--- a/eval/public/structs/BUILD ++++ b/eval/public/structs/BUILD +@@ -192,7 +192,6 @@ cc_library( + hdrs = ["legacy_type_provider.h"], + deps = [ + ":legacy_type_adapter", +- "//base:type_provider", + "@com_google_absl//absl/types:optional", + ], + ) +diff --git a/eval/public/structs/field_access_impl.cc b/eval/public/structs/field_access_impl.cc +index 788a476..e4b70b3 100644 +--- a/eval/public/structs/field_access_impl.cc ++++ b/eval/public/structs/field_access_impl.cc +@@ -25,6 +25,7 @@ + #include "google/protobuf/arena.h" + #include "google/protobuf/map_field.h" + #include "absl/container/flat_hash_set.h" ++#include "absl/log/log.h" + #include "absl/status/status.h" + #include "absl/status/statusor.h" + #include "absl/strings/str_cat.h" +diff --git a/eval/public/structs/legacy_type_provider.h b/eval/public/structs/legacy_type_provider.h +index b1623fc..d3d88d6 100644 +--- a/eval/public/structs/legacy_type_provider.h ++++ b/eval/public/structs/legacy_type_provider.h +@@ -16,7 +16,6 @@ + #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TYPE_PROVIDER_H_ + + #include "absl/types/optional.h" +-#include "base/type_provider.h" + #include "eval/public/structs/legacy_type_adapter.h" + + namespace google::api::expr::runtime { +@@ -25,8 +24,10 @@ namespace google::api::expr::runtime { + // + // Note: This API is not finalized. Consult the CEL team before introducing new + // implementations. +-class LegacyTypeProvider : public cel::TypeProvider { ++class LegacyTypeProvider { + public: ++ virtual ~LegacyTypeProvider() = default; ++ + // Return LegacyTypeAdapter for the fully qualified type name if available. + // + // nullopt values are interpreted as not present. +@@ -45,7 +46,7 @@ class LegacyTypeProvider : public cel::TypeProvider { + // created ones, the TypeInfoApis returned from this method should be the same + // as the ones used in value creation. + virtual absl::optional ProvideLegacyTypeInfo( +- absl::string_view name) const { ++ absl::string_view) const { + return absl::nullopt; + } + }; diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 76868a57b1a4..f8a3198ab427 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1085,8 +1085,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Common Expression Language (CEL) C++ library", project_desc = "Common Expression Language (CEL) C++ library", project_url = "https://opensource.google/projects/cel", - version = "11f9d662e71f73b28f4104ef2cf0550d96e677ae", - sha256 = "3d00e3fa9def360da18ccd077f261c1dea2f725c046692cc55ecc4d3b15bb390", + version = "da0aba702f44a41ec6d2eb4bbf6a9f01efc2746d", + sha256 = "d62b93fd07c6151749e83855157f3f2778d62c168318f9c40dfcfe1c336c496f", strip_prefix = "cel-cpp-{version}", urls = ["https://github.com/google/cel-cpp/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -1103,7 +1103,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.rbac.matchers.upstream_ip_port", "envoy.formatter.cel", ], - release_date = "2022-09-01", + release_date = "2023-03-08", cpe = "N/A", ), com_github_google_flatbuffers = dict( From e5de065c7523814f0d011b479740eb1bce1a05e4 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 17 May 2023 16:53:48 -0400 Subject: [PATCH 269/740] Increase cluster_manager_impl_test test timeout to avoid TSAN flakes (#27435) Signed-off-by: Yan Avlasov From 1765f150439d603d316ba0356e452f24c9402430 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 17 May 2023 16:55:13 -0400 Subject: [PATCH 270/740] runtime: cleaning up flag which was turned down (#27432) Signed-off-by: Alyssa Wilk --- source/common/http/conn_manager_impl.cc | 8 ++------ source/common/http/http1/codec_impl.cc | 4 +--- source/common/runtime/runtime_features.cc | 2 -- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 4696ebf69d72..80995aa8fd1a 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -259,12 +259,8 @@ void ConnectionManagerImpl::doEndStream(ActiveStream& stream, bool check_for_def bool request_complete = stream.filter_manager_.remoteDecodeComplete(); if (check_for_deferred_close) { - // Don't do delay close for responses which are framed by connection close: - // HTTP/1.0 and below, upgrades, and CONNECT responses. - checkForDeferredClose( - (connection_close && (request_complete || http_10_sans_cl)) || - (stream.state_.is_tunneling_ && - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.no_delay_close_for_upgrades"))); + // Don't do delay close for HTTP/1.0 or if the request is complete. + checkForDeferredClose(connection_close && (request_complete || http_10_sans_cl)); } } diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index 7bef6c5cdc3d..053f397dd831 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -303,9 +303,7 @@ void StreamEncoderImpl::endEncode() { notifyEncodeComplete(); // With CONNECT or TCP tunneling, half-closing the connection is used to signal end stream so // don't delay that signal. - if (connect_request_ || is_tcp_tunneling_ || - (is_response_to_connect_request_ && - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.no_delay_close_for_upgrades"))) { + if (connect_request_ || is_tcp_tunneling_) { connection_.connection().close( Network::ConnectionCloseType::FlushWrite, StreamInfo::LocalCloseReasons::get().CloseForConnectRequestOrTcpTunneling); diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 0fe66d8d6868..7d860806d6ab 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -103,8 +103,6 @@ FALSE_RUNTIME_GUARD(envoy_reloadable_features_runtime_initialized); // TODO(mattklein123): Also unit test this if this sticks and this becomes the default for Apple & // Android. FALSE_RUNTIME_GUARD(envoy_reloadable_features_always_use_v6); -// TODO(alyssawilk) remove in Q2. -FALSE_RUNTIME_GUARD(envoy_reloadable_features_no_delay_close_for_upgrades); // TODO(pradeepcrao) reset this to true after 2 releases (1.27) FALSE_RUNTIME_GUARD(envoy_reloadable_features_enable_include_histograms); From 2e3a270a4cc8b947eb6dee16ade550f69dbe2f9b Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 17 May 2023 22:31:26 +0100 Subject: [PATCH 271/740] mobile/ci: Dont cache results for coverage (#27461) Signed-off-by: Ryan Northey --- mobile/.bazelrc | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/.bazelrc b/mobile/.bazelrc index 5d2960882438..60b815c66132 100644 --- a/mobile/.bazelrc +++ b/mobile/.bazelrc @@ -240,6 +240,7 @@ build:remote-ci-linux-coverage --config=remote-ci-common build:remote-ci-linux-coverage --build_runfile_links build:remote-ci-linux-coverage --legacy_important_outputs=false build:remote-ci-linux-coverage --test_env=CC_CODE_COVERAGE_SCRIPT=external/envoy/bazel/coverage/collect_cc_coverage.sh +build:remote-ci-linux-coverage --nocache_test_results # IPv6 tests fail on CI build:remote-ci-linux-coverage --test_env=ENVOY_IP_TEST_VERSIONS=v4only From 8e8084290dc74c6d8738343a63ad53e5ddb989ec Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 17 May 2023 19:56:11 -0400 Subject: [PATCH 272/740] tools: discouraging exceptions in core c++ files (#27431) Signed-off-by: Alyssa Wilk --- tools/code_format/check_format.py | 4 +- tools/code_format/config.yaml | 130 ++++++++++++++++++++++++++++-- 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index 1d37d54debdf..8484e89752ac 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -329,8 +329,8 @@ def allow_listed_for_raw_try(self, file_path): def deny_listed_for_exceptions(self, file_path): # Returns true when it is a non test header file or the file_path is in DENYLIST or # it is under tools/testdata subdirectory. - - return (file_path.endswith('.h') and not file_path.startswith("./test/") and not file_path in self.config.paths["exception"]["include"]) or file_path in self.config.paths["exception"]["exclude"] \ + return (file_path.endswith('.h') and not file_path.startswith("./test/") and not file_path in self.config.paths["exception"]["include"]) \ + or (file_path.endswith('.cc') and file_path.startswith("./source/") and not file_path.startswith("./source/extensions/") and not file_path in self.config.paths["exception"]["include"]) \ or self.is_in_subdir(file_path, 'tools/testdata') def allow_listed_for_build_urls(self, file_path): diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index fc94f107465b..0aff6d44d5ae 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -81,20 +81,138 @@ paths: - bazel/external/cargo/crates.bzl - bazel/repository_locations.bzl - # Header files that can throw exceptions. These should be limited; the only - # valid situation identified so far is template functions used for config - # processing. exception: include: + # Header files that can throw exceptions. These should be limited; the only + # valid situation identified so far is template functions used for config + # processing. - source/common/config/utility.h - source/common/matcher/map_matcher.h - source/common/matcher/field_matcher.h - source/common/protobuf/visitor_helper.h - source/extensions/common/matcher/trie_matcher.h - # These files should not throw exceptions. Add HTTP/1 when exceptions removed. - exclude: - - source/common/http/http2/codec_impl.h + # legacy core files which throw exceptions. We can add to this list but strongly prefer + # StausOr where possible. + - source/common/upstream/wrsq_scheduler.h + - source/common/upstream/cds_api_helper.cc + - source/common/upstream/thread_aware_lb_impl.cc + - source/common/upstream/load_balancer_impl.cc + - source/common/upstream/health_checker_impl.cc + - source/common/upstream/cluster_manager_impl.cc + - source/common/upstream/od_cds_api_impl.cc + - source/common/upstream/cds_api_impl.cc + - source/common/upstream/outlier_detection_impl.cc + - source/common/upstream/upstream_impl.cc + - source/common/network/connection_impl.cc + - source/common/network/listen_socket_impl.cc + - source/common/network/io_socket_handle_impl.cc + - source/common/network/address_impl.cc + - source/common/network/address_impl.h + - source/common/network/cidr_range.cc + - source/common/network/utility.cc + - source/common/network/dns_resolver/dns_factory_util.cc + - source/common/network/dns_resolver/dns_factory_util.h + - source/common/network/resolver_impl.cc + - source/common/network/utility.h + - source/common/network/lc_trie.h + - source/common/network/socket_impl.cc + - source/common/ssl/tls_certificate_config_impl.cc + - source/common/ssl/certificate_validation_context_config_impl.cc + - source/common/formatter/substitution_formatter.cc + - source/common/formatter/substitution_format_string.cc + - source/common/stats/tag_extractor_impl.cc + - source/common/stats/tag_producer_impl.cc - source/common/http/http2/codec_impl.cc + - source/common/http/filter_chain_helper.cc + - source/common/http/request_id_extension_impl.cc + - source/common/http/utility.cc + - source/common/http/hash_policy.cc + - source/common/http/utility.h + - source/common/http/filter_chain_helper.h + - source/common/http/conn_manager_utility.cc + - source/common/http/match_delegate/config.cc + - source/common/http/http_server_properties_cache_manager_impl.cc + - source/common/protobuf/yaml_utility.cc + - source/common/protobuf/visitor_helper.h + - source/common/protobuf/visitor.cc + - source/common/protobuf/utility.cc + - source/common/protobuf/utility.h + - source/common/protobuf/message_validator_impl.cc + - source/common/access_log/access_log_impl.h + - source/common/access_log/access_log_manager_impl.cc + - source/common/secret/secret_manager_impl.cc + - source/common/secret/sds_api.h + - source/common/secret/sds_api.cc + - source/common/grpc/async_client_manager_impl.cc + - source/common/grpc/common.cc + - source/common/grpc/google_grpc_utils.cc + - source/common/grpc/common.h + - source/common/tcp_proxy/tcp_proxy.cc + - source/common/config/xds_resource.h + - source/common/config/subscription_factory_impl.cc + - source/common/config/xds_resource.cc + - source/common/config/datasource.cc + - source/common/config/xds_mux/delta_subscription_state.cc + - source/common/config/xds_mux/sotw_subscription_state.cc + - source/common/config/xds_mux/grpc_mux_impl.cc + - source/common/config/utility.cc + - source/common/config/watch_map.h + - source/common/config/datasource.h + - source/common/config/null_grpc_mux_impl.h + - source/common/config/utility.h + - source/common/config/custom_config_validators_impl.h + - source/common/config/xds_context_params.cc + - source/common/config/custom_config_validators_impl.cc + - source/common/runtime/runtime_impl.cc + - source/common/quic/quic_transport_socket_factory.cc + - source/common/filter/config_discovery_impl.cc + - source/common/event/timer_impl.h + - source/common/matcher/map_matcher.h + - source/common/matcher/field_matcher.h + - source/common/json/json_internal.cc + - source/common/json/json_internal.h + - source/common/json/json_sanitizer.cc + - source/common/router/header_formatter.cc + - source/common/router/scoped_rds.cc + - source/common/router/rds_impl.cc + - source/common/router/config_impl.cc + - source/common/router/config_impl.h + - source/common/router/scoped_config_impl.cc + - source/common/router/router_ratelimit.cc + - source/common/router/vhds.cc + - source/common/router/config_utility.cc + - source/common/router/scoped_rds.h + - source/common/router/header_parser.cc + - source/common/router/config_utility.h + - source/common/rds/rds_route_config_subscription.cc + - source/common/filesystem/inotify/watcher_impl.cc + - source/common/filesystem/posix/directory_iterator_impl.cc + - source/common/filesystem/posix/filesystem_impl.cc + - source/common/filesystem/kqueue/watcher_impl.cc + - source/common/filesystem/win32/directory_iterator_impl.cc + - source/common/filesystem/win32/filesystem_impl.cc + - source/common/filesystem/win32/watcher_impl.cc + - source/common/common/utility.cc + - source/common/common/regex.cc + - source/common/common/logger.h + - source/common/common/thread.h + - source/common/common/utility.h + - source/common/common/matchers.h + - source/common/common/matchers.cc + - source/exe/stripped_main_base.cc + - source/server/options_impl.cc + - source/server/options_impl.h + - source/server/server.h + - source/server/overload_manager_impl.cc + - source/server/config_validation/cluster_manager.cc + - source/server/config_validation/server.cc + - source/server/admin/html/active_stats.js + - source/server/server.cc + - source/server/configuration_impl.h + - source/server/hot_restarting_base.cc + - source/server/hot_restart_impl.cc + - source/server/ssl_context_manager.cc + - source/server/configuration_impl.cc # Only one C++ file should instantiate grpc_init grpc_init: From f4e2a260f6cb1e6582aa761f82a2f7ad428e01b8 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Wed, 17 May 2023 21:05:13 -0400 Subject: [PATCH 273/740] stats: add 'detailed' buckets-mode that captures all data from circlhist (#27094) Commit Message: The current mechanisms for extracting histograms leave a lot of useful data inaccessible. This adds a new extraction mechanism to provide the full computed detail provided by the histogram library, as well as JSON and textual access to this data through the admin port. Additional Description: This partially addresses #11259 but does not add the new detailed bucket mode to the metrics service gRPC API. This PR sets the stage for admin visualization of histograms in #26472. The visualization could've occurred without this but it was not very rich; it's got more potential for revealing insight if the full bucket data can be rendered. Note that when aggregating histograms over multiple Envoy proxies it may be better to use the configurable fixed bucket sizes so that the histogram data can be combined across Envoy instances by the time-series database (e.g. Prometheus). Risk Level: low: this is new functionality. Testing: //test/... Docs Changes: added doc for new bucket-mode. Release Notes: required Platform Specific Features: n/a Signed-off-by: Joshua Marantz --- changelogs/current.yaml | 3 + docs/root/operations/admin.rst | 66 ++++++++++- envoy/stats/histogram.h | 19 ++++ source/common/stats/histogram_impl.h | 2 - source/common/stats/thread_local_store.cc | 14 +++ source/common/stats/thread_local_store.h | 8 ++ source/common/stats/utility.h | 1 + source/server/admin/stats_handler.cc | 2 +- source/server/admin/stats_render.cc | 113 ++++++++++++++----- source/server/admin/stats_render.h | 14 ++- source/server/admin/utils.cc | 2 + source/server/admin/utils.h | 2 +- test/common/stats/BUILD | 1 + test/common/stats/stat_test_utility.cc | 11 ++ test/common/stats/stat_test_utility.h | 5 + test/common/stats/thread_local_store_test.cc | 6 +- test/mocks/stats/mocks.h | 3 +- test/server/admin/admin_test.cc | 2 +- test/server/admin/stats_render_test.cc | 46 ++++++++ 19 files changed, 282 insertions(+), 38 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 0a932d37a38c..d9e8e8ea1660 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -193,6 +193,9 @@ new_features: `. Enabling this option will write a log to all access loggers when HTTP tunnels (e.g. Websocket and CONNECT) are successfully established. +- area: admin + change: | + Adds a new admin stats html bucket-mode ``detailed`` to generate all recorded buckets and summary percentiles. deprecated: - area: access_log diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index de2295b581a8..d0957b906108 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -475,6 +475,24 @@ modify different aspects of the server: Buckets do not include values from other buckets with smaller upper bounds; the previous bucket's upper bound acts as a lower bound. Compatible with ``usedonly`` and ``filter``. + .. http:get:: /stats?histogram_buckets=detailed + + Shows histograms as both percentile summary data, and raw bucket data. + + Example output + .. code-block:: text + + http.admin.downstream_rq_time: + totals=1,0.25:25, 2,0.25:9 + intervals=1,0.25:2, 2,0.25:3 + summary=P0(1,1) P25(1.0625,1.034) P50(2.0166,1.068) P75(2.058,2.005) P90(2.083,2.06) P95(2.091,2.08) P99(2.09,2.09) P99.5(2.099,2.098) P99.9(2.099,2.099) P100(2.1,2.1) + + Each bucket is shown as `lower_bound,width:count`. In the above example there are two + buckets. `totals` contains the accumulated data-points since the binary was started. + `intervals` shows the new data points since the previous stats flush. + + Compatible with ``usedonly`` and ``filter``. + .. http:get:: /stats?format=html Renders stats using HTML for a web browser, providing form fields to incrementally @@ -647,7 +665,53 @@ modify different aspects of the server: ] } -.. http:get:: /stats?format=prometheus + .. http:get:: /stats?format=json&histogram_buckets=detailed + + Shows histograms as both percentile summary data.. + + Example output: + + .. code-block:: json + + { + "stats": [ + { + "histograms": { + "supported_percentiles": [0, 25, 50, 75, 90, 95, 99, 99.5, 99.9, 100], + "details": [ + { + "name": "http.admin.downstream_rq_time", + "percentiles": [ + { "interval": null, "cumulative": 1 }, + { "interval": null, "cumulative": 1.0351851851851852 }, + { "interval": null "cumulative": 1.0703703703703704 }, + { "interval": null, "cumulative": 2.0136363636363637 }, + { "interval": null "cumulative": 2.0654545454545454 }, + { "interval": null "cumulative": 2.0827272727272725 }, + { "interval": null "cumulative": 2.0965454545454545 }, + { "interval": null, "cumulative": 2.098272727272727 }, + { "interval": null, "cumulative": 2.0996545454545457 }, + { "interval": null "cumulative": 2.1 } + ], + "totals": [ + { "lower_bound": 1, "width": 0.25, "count": 25 }, + { "lower_bound": 2, "width": 0.25, "count": 9 } + ], + "intervals": [ + { "lower_bound": 1, "width": 0.25, "count": 2 }, + { "lower_bound": 2, "width": 0.25, "count": 3 } + ], + }, + ] + } + } + ] + } + + Compatible with ``usedonly`` and ``filter``. + + + .. http:get:: /stats?format=prometheus or alternatively, diff --git a/envoy/stats/histogram.h b/envoy/stats/histogram.h index 6bcb71d77868..e8aa64cb7532 100644 --- a/envoy/stats/histogram.h +++ b/envoy/stats/histogram.h @@ -167,6 +167,25 @@ class ParentHistogram : public Histogram { * Returns the bucket summary representation. */ virtual std::string bucketSummary() const PURE; + + // Holds detailed value and counts for a histogram bucket. + struct Bucket { + double lower_bound_{0}; // Bound of bucket that's closest to zero. + double width_{0}; + uint64_t count_{0}; + }; + + /** + * @return a vector of histogram buckets collected since binary start or reset. + */ + virtual std::vector detailedTotalBuckets() const PURE; + + /** + * @return bucket data collected since the most recent stat sink. Note that + * the number of interval buckets is likely to be much smaller than + * the number of detailed buckets. + */ + virtual std::vector detailedIntervalBuckets() const PURE; }; using ParentHistogramSharedPtr = RefcountPtr; diff --git a/source/common/stats/histogram_impl.h b/source/common/stats/histogram_impl.h index 23a2adaf23c2..da49db9a436f 100644 --- a/source/common/stats/histogram_impl.h +++ b/source/common/stats/histogram_impl.h @@ -49,8 +49,6 @@ class HistogramStatisticsImpl final : public HistogramStatistics, NonCopyable { const histogram_t* histogram_ptr, Histogram::Unit unit = Histogram::Unit::Unspecified, ConstSupportedBuckets& supported_buckets = HistogramSettingsImpl::defaultBuckets()); - static ConstSupportedBuckets& defaultSupportedBuckets(); - void refresh(const histogram_t* new_histogram_ptr); // HistogramStatistics diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index cc5658d71ae2..aee9013d31d0 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -958,6 +958,20 @@ std::string ParentHistogramImpl::bucketSummary() const { } } +std::vector +ParentHistogramImpl::detailedlBucketsHelper(const histogram_t& histogram) { + const uint32_t num_buckets = hist_num_buckets(&histogram); + std::vector buckets(num_buckets); + hist_bucket_t hist_bucket; + for (uint32_t i = 0; i < num_buckets; ++i) { + ParentHistogram::Bucket& bucket = buckets[i]; + hist_bucket_idx_bucket(&histogram, i, &hist_bucket, &bucket.count_); + bucket.lower_bound_ = hist_bucket_to_double(hist_bucket); + bucket.width_ = hist_bucket_to_double_bin_width(hist_bucket); + } + return buckets; +} + void ParentHistogramImpl::addTlsHistogram(const TlsHistogramSharedPtr& hist_ptr) { Thread::LockGuard lock(merge_lock_); tls_histograms_.emplace_back(hist_ptr); diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index a205f6529565..46bd0afa1032 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -105,6 +105,12 @@ class ParentHistogramImpl : public MetricImpl { } std::string quantileSummary() const override; std::string bucketSummary() const override; + std::vector detailedTotalBuckets() const override { + return detailedlBucketsHelper(*cumulative_histogram_); + } + std::vector detailedIntervalBuckets() const override { + return detailedlBucketsHelper(*interval_histogram_); + } // Stats::Metric SymbolTable& symbolTable() override; @@ -121,6 +127,8 @@ class ParentHistogramImpl : public MetricImpl { private: bool usedLockHeld() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(merge_lock_); + static std::vector + detailedlBucketsHelper(const histogram_t& histogram); Histogram::Unit unit_; ThreadLocalStoreImpl& thread_local_store_; diff --git a/source/common/stats/utility.h b/source/common/stats/utility.h index 528514d4ec89..43284f695cec 100644 --- a/source/common/stats/utility.h +++ b/source/common/stats/utility.h @@ -231,6 +231,7 @@ TextReadout& textReadoutFromElements(Scope& scope, const ElementVec& elements, */ TextReadout& textReadoutFromStatNames(Scope& scope, const StatNameVec& elements, StatNameTagVectorOptConstRef tags = absl::nullopt); + } // namespace Utility /** diff --git a/source/server/admin/stats_handler.cc b/source/server/admin/stats_handler.cc index be645ebddd4f..fb8dc1801f93 100644 --- a/source/server/admin/stats_handler.cc +++ b/source/server/admin/stats_handler.cc @@ -186,7 +186,7 @@ Admin::UrlHandler StatsHandler::statsHandler(bool active_mode) { {Admin::ParamDescriptor::Type::Enum, "histogram_buckets", "Histogram bucket display mode", - {"cumulative", "disjoint", "none"}}}; + {"cumulative", "disjoint", "detailed", "none"}}}; Admin::ParamDescriptorVec params; if (!active_mode) { diff --git a/source/server/admin/stats_render.cc b/source/server/admin/stats_render.cc index e75dab1b0a28..7fd55b5b6153 100644 --- a/source/server/admin/stats_render.cc +++ b/source/server/admin/stats_render.cc @@ -12,9 +12,6 @@ constexpr absl::string_view JsonQuoteCloseBrace = "\"}"; } // namespace namespace Envoy { - -using ProtoMap = Protobuf::Map; - namespace Server { StatsTextRender::StatsTextRender(const StatsParams& params) @@ -32,6 +29,11 @@ void StatsTextRender::generate(Buffer::Instance& response, const std::string& na void StatsTextRender::generate(Buffer::Instance& response, const std::string& name, const Stats::ParentHistogram& histogram) { + if (!histogram.used()) { + response.addFragments({name, ": No recorded values\n"}); + return; + } + switch (histogram_buckets_mode_) { case Utility::HistogramBucketsMode::NoBuckets: response.addFragments({name, ": ", histogram.quantileSummary(), "\n"}); @@ -42,11 +44,28 @@ void StatsTextRender::generate(Buffer::Instance& response, const std::string& na case Utility::HistogramBucketsMode::Disjoint: addDisjointBuckets(name, histogram, response); break; + case Utility::HistogramBucketsMode::Detailed: + response.addFragments({name, ":\n totals="}); + addDetail(histogram.detailedTotalBuckets(), response); + response.add("\n intervals="); + addDetail(histogram.detailedIntervalBuckets(), response); + response.addFragments({"\n summary=", histogram.quantileSummary(), "\n"}); + break; } } void StatsTextRender::finalize(Buffer::Instance&) {} +void StatsTextRender::addDetail(const std::vector& buckets, + Buffer::Instance& response) { + absl::string_view delim = ""; + for (const Stats::ParentHistogram::Bucket& bucket : buckets) { + response.addFragments( + {delim, absl::StrCat(bucket.lower_bound_, ",", bucket.width_, ":", bucket.count_)}); + delim = ", "; + } +} + // Computes disjoint buckets as text and adds them to the response buffer. void StatsTextRender::addDisjointBuckets(const std::string& name, const Stats::ParentHistogram& histogram, @@ -126,9 +145,15 @@ void StatsJsonRender::generate(Buffer::Instance& response, const std::string& na void StatsJsonRender::generate(Buffer::Instance&, const std::string& name, const Stats::ParentHistogram& histogram) { switch (histogram_buckets_mode_) { - case Utility::HistogramBucketsMode::NoBuckets: - summarizeBuckets(name, histogram); + case Utility::HistogramBucketsMode::NoBuckets: { + ProtobufWkt::Struct computed_quantile; + ProtoMap& computed_quantile_fields = *computed_quantile.mutable_fields(); + computed_quantile_fields["name"] = ValueUtil::stringValue(name); + populateQuantiles(histogram, "supported_quantiles", + computed_quantile_fields["values"].mutable_list_value()); + *histogram_array_->add_values() = ValueUtil::structValue(computed_quantile); break; + } case Utility::HistogramBucketsMode::Cumulative: { const Stats::HistogramStatistics& interval_statistics = histogram.intervalStatistics(); const std::vector& interval_buckets = interval_statistics.computedBuckets(); @@ -145,6 +170,32 @@ void StatsJsonRender::generate(Buffer::Instance&, const std::string& name, collectBuckets(name, histogram, interval_buckets, cumulative_buckets); break; } + case Utility::HistogramBucketsMode::Detailed: { + std::vector buckets = histogram.detailedTotalBuckets(); + ProtobufWkt::Struct histogram_obj; + ProtoMap& histogram_obj_fields = *histogram_obj.mutable_fields(); + histogram_obj_fields["name"] = ValueUtil::stringValue(histogram.name()); + populateQuantiles(histogram, "supported_percentiles", + histogram_obj_fields["percentiles"].mutable_list_value()); + populateDetail("totals", histogram.detailedTotalBuckets(), histogram_obj_fields); + populateDetail("intervals", histogram.detailedIntervalBuckets(), histogram_obj_fields); + *histogram_array_->add_values() = ValueUtil::structValue(histogram_obj); + break; + } + } +} + +void StatsJsonRender::populateDetail(absl::string_view name, + const std::vector& buckets, + ProtoMap& histogram_obj_fields) { + ProtobufWkt::ListValue* bucket_array = histogram_obj_fields[name].mutable_list_value(); + for (const Stats::ParentHistogram::Bucket& bucket : buckets) { + ProtobufWkt::Struct bucket_json; + ProtoMap& bucket_fields = *bucket_json.mutable_fields(); + bucket_fields["lower_bound"] = ValueUtil::numberValue(bucket.lower_bound_); + bucket_fields["width"] = ValueUtil::numberValue(bucket.width_); + bucket_fields["count"] = ValueUtil::numberValue(bucket.count_); + *bucket_array->add_values() = ValueUtil::structValue(bucket_json); } } @@ -154,11 +205,18 @@ void StatsJsonRender::finalize(Buffer::Instance& response) { if (histogram_array_->values_size() > 0) { ProtoMap& histograms_obj_container_fields = *histograms_obj_container_.mutable_fields(); if (found_used_histogram_) { - ASSERT(histogram_buckets_mode_ == Utility::HistogramBucketsMode::NoBuckets); - ProtoMap& histograms_obj_fields = *histograms_obj_.mutable_fields(); - histograms_obj_fields["computed_quantiles"].set_allocated_list_value( - histogram_array_.release()); - histograms_obj_container_fields["histograms"] = ValueUtil::structValue(histograms_obj_); + if (histogram_buckets_mode_ == Utility::HistogramBucketsMode::NoBuckets) { + ProtoMap& histograms_obj_fields = *histograms_obj_.mutable_fields(); + histograms_obj_fields["computed_quantiles"].set_allocated_list_value( + histogram_array_.release()); + histograms_obj_container_fields["histograms"] = ValueUtil::structValue(histograms_obj_); + } else if (histogram_buckets_mode_ == Utility::HistogramBucketsMode::Detailed) { + ProtoMap& histograms_obj_fields = *histograms_obj_.mutable_fields(); + histograms_obj_fields["details"].set_allocated_list_value(histogram_array_.release()); + histograms_obj_container_fields["histograms"] = ValueUtil::structValue(histograms_obj_); + } else { + IS_ENVOY_BUG("reached unexpected buckets mode with found_used_histogram_ set"); + } } else { ASSERT(histogram_buckets_mode_ != Utility::HistogramBucketsMode::NoBuckets); histograms_obj_container_fields["histograms"].set_allocated_list_value( @@ -177,38 +235,36 @@ void StatsJsonRender::finalize(Buffer::Instance& response) { response.add("]}"); } +void StatsJsonRender::populateVector(absl::string_view name, const std::vector& values, + uint32_t multiplier, ProtoMap& histograms_obj_fields) { + ProtobufWkt::ListValue* array = histograms_obj_fields[name].mutable_list_value(); + + for (double value : values) { + *array->add_values() = ValueUtil::numberValue(value * multiplier); + } +} + // Summarizes the buckets in the specified histogram, collecting JSON objects. // Note, we do not flush this buffer to the network when it grows large, and // if this becomes an issue it should be possible to do, noting that we are // one or two levels nesting below the list of scalar stats due to the Envoy // stats json schema, where histograms are grouped together. -void StatsJsonRender::summarizeBuckets(const std::string& name, - const Stats::ParentHistogram& histogram) { +void StatsJsonRender::populateQuantiles(const Stats::ParentHistogram& histogram, + absl::string_view label, + ProtobufWkt::ListValue* computed_quantile_value_array) { if (!found_used_histogram_) { // It is not possible for the supported quantiles to differ across histograms, so it is ok // to send them once. Stats::HistogramStatisticsImpl empty_statistics; - ProtoMap& histograms_obj_fields = *histograms_obj_.mutable_fields(); - ProtobufWkt::ListValue* supported_quantile_array = - histograms_obj_fields["supported_quantiles"].mutable_list_value(); - - for (double quantile : empty_statistics.supportedQuantiles()) { - *supported_quantile_array->add_values() = ValueUtil::numberValue(quantile * 100); - } - + ProtoMap& fields = *histograms_obj_.mutable_fields(); + populateVector(label, empty_statistics.supportedQuantiles(), 100 /* multiplier */, fields); found_used_histogram_ = true; } - ProtobufWkt::Struct computed_quantile; - ProtoMap& computed_quantile_fields = *computed_quantile.mutable_fields(); - computed_quantile_fields["name"] = ValueUtil::stringValue(name); - - ProtobufWkt::ListValue* computed_quantile_value_array = - computed_quantile_fields["values"].mutable_list_value(); const Stats::HistogramStatistics& interval_statistics = histogram.intervalStatistics(); const std::vector& computed_quantiles = interval_statistics.computedQuantiles(); - const std::vector& cumulative_quantiles = - histogram.cumulativeStatistics().computedQuantiles(); + const Stats::HistogramStatistics& histogram_stats = histogram.cumulativeStatistics(); + const std::vector& cumulative_quantiles = histogram_stats.computedQuantiles(); const size_t min_size = std::min({computed_quantiles.size(), cumulative_quantiles.size(), interval_statistics.supportedQuantiles().size()}); ASSERT(min_size == computed_quantiles.size()); @@ -226,7 +282,6 @@ void StatsJsonRender::summarizeBuckets(const std::string& name, *computed_quantile_value_array->add_values() = ValueUtil::structValue(computed_quantile_value); } - *histogram_array_->add_values() = ValueUtil::structValue(computed_quantile); } // Collects the buckets from the specified histogram, using either the diff --git a/source/server/admin/stats_render.h b/source/server/admin/stats_render.h index 477c3eb6695e..baa747e3f4c1 100644 --- a/source/server/admin/stats_render.h +++ b/source/server/admin/stats_render.h @@ -55,6 +55,9 @@ class StatsTextRender : public StatsRender { void addDisjointBuckets(const std::string& name, const Stats::ParentHistogram& histogram, Buffer::Instance& response); + void addDetail(const std::vector& buckets, + Buffer::Instance& response); + const Utility::HistogramBucketsMode histogram_buckets_mode_; }; @@ -73,18 +76,27 @@ class StatsJsonRender : public StatsRender { void finalize(Buffer::Instance& response) override; private: + using ProtoMap = Protobuf::Map; + // Summarizes the buckets in the specified histogram, collecting JSON objects. // Note, we do not flush this buffer to the network when it grows large, and // if this becomes an issue it should be possible to do, noting that we are // one or two levels nesting below the list of scalar stats due to the Envoy // stats json schema, where histograms are grouped together. - void summarizeBuckets(const std::string& name, const Stats::ParentHistogram& histogram); + void populateQuantiles(const Stats::ParentHistogram& histogram, absl::string_view label, + ProtobufWkt::ListValue* computed_quantile_value_array); // Collects the buckets from the specified histogram. void collectBuckets(const std::string& name, const Stats::ParentHistogram& histogram, const std::vector& interval_buckets, const std::vector& cumulative_buckets); + void populateDetail(absl::string_view name, + const std::vector& buckets, + ProtoMap& histogram_obj_fields); + static void populateVector(absl::string_view name, const std::vector& values, + uint32_t multiplier, ProtoMap& histograms_obj_fields); + ProtobufWkt::Struct histograms_obj_; ProtobufWkt::Struct histograms_obj_container_; std::unique_ptr histogram_array_; diff --git a/source/server/admin/utils.cc b/source/server/admin/utils.cc index 774c1395074c..b348146413c9 100644 --- a/source/server/admin/utils.cc +++ b/source/server/admin/utils.cc @@ -36,6 +36,8 @@ absl::Status histogramBucketsParam(const Http::Utility::QueryParams& params, histogram_buckets_mode = HistogramBucketsMode::Cumulative; } else if (histogram_buckets_query_param.value() == "disjoint") { histogram_buckets_mode = HistogramBucketsMode::Disjoint; + } else if (histogram_buckets_query_param.value() == "detailed") { + histogram_buckets_mode = HistogramBucketsMode::Detailed; } else if (histogram_buckets_query_param.value() != "none") { return absl::InvalidArgumentError( "usage: /stats?histogram_buckets=(cumulative|disjoint|none)\n"); diff --git a/source/server/admin/utils.h b/source/server/admin/utils.h index 6d885f98e331..5deaf651d8a7 100644 --- a/source/server/admin/utils.h +++ b/source/server/admin/utils.h @@ -13,7 +13,7 @@ namespace Envoy { namespace Server { namespace Utility { -enum class HistogramBucketsMode { NoBuckets, Cumulative, Disjoint }; +enum class HistogramBucketsMode { NoBuckets, Cumulative, Disjoint, Detailed }; void populateFallbackResponseHeaders(Http::Code code, Http::ResponseHeaderMap& header_map); diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index 36967c714482..0463996725d8 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -323,6 +323,7 @@ envoy_cc_test( name = "utility_test", srcs = ["utility_test.cc"], deps = [ + ":stat_test_utility_lib", "//source/common/stats:isolated_store_lib", "//source/common/stats:thread_local_store_lib", "//source/common/stats:utility_lib", diff --git a/test/common/stats/stat_test_utility.cc b/test/common/stats/stat_test_utility.cc index b076a12840e4..1454a8a74a70 100644 --- a/test/common/stats/stat_test_utility.cc +++ b/test/common/stats/stat_test_utility.cc @@ -5,6 +5,17 @@ namespace Envoy { namespace Stats { + +bool operator==(const ParentHistogram::Bucket& a, const ParentHistogram::Bucket& b) { + return a.count_ == b.count_ && std::abs(a.lower_bound_ - b.lower_bound_) < 0.001 && + std::abs(a.width_ - b.width_) < 0.001; +} + +std::ostream& operator<<(std::ostream& out, const ParentHistogram::Bucket& bucket) { + return out << "(min_value=" << bucket.lower_bound_ << ", width=" << bucket.width_ + << ", count=" << bucket.count_ << ")"; +} + namespace TestUtil { void forEachSampleStat(int num_clusters, bool include_other_stats, diff --git a/test/common/stats/stat_test_utility.h b/test/common/stats/stat_test_utility.h index 830f70677ad1..9112c039754c 100644 --- a/test/common/stats/stat_test_utility.h +++ b/test/common/stats/stat_test_utility.h @@ -13,6 +13,11 @@ namespace Envoy { namespace Stats { + +// Helper methods to facilitate using testing::ElementsAre with bucket vectors. +bool operator==(const ParentHistogram::Bucket& a, const ParentHistogram::Bucket& b); +std::ostream& operator<<(std::ostream& out, const ParentHistogram::Bucket& bucket); + namespace TestUtil { class TestSymbolTableHelper { diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 1d43ed663eb6..0363a4cd0219 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -37,6 +37,7 @@ using testing::InSequence; using testing::NiceMock; using testing::Ref; using testing::Return; +using testing::UnorderedElementsAre; using testing::UnorderedElementsAreArray; namespace Envoy { @@ -129,6 +130,7 @@ class HistogramWrapper { class HistogramTest : public testing::Test { public: + using Bucket = ParentHistogram::Bucket; using NameHistogramMap = std::map; HistogramTest() @@ -1709,7 +1711,7 @@ TEST_F(HistogramTest, BasicHistogramUsed) { } } -TEST_F(HistogramTest, ParentHistogramBucketSummary) { +TEST_F(HistogramTest, ParentHistogramBucketSummaryAndDetail) { ScopeSharedPtr scope1 = store_->createScope("scope1."); Histogram& histogram = scope_.histogramFromString("histogram", Histogram::Unit::Unspecified); store_->mergeHistograms([]() -> void {}); @@ -1725,6 +1727,8 @@ TEST_F(HistogramTest, ParentHistogramBucketSummary) { "B30000(1,1) B60000(1,1) B300000(1,1) B600000(1,1) B1.8e+06(1,1) " "B3.6e+06(1,1)", parent_histogram->bucketSummary()); + EXPECT_THAT(parent_histogram->detailedTotalBuckets(), UnorderedElementsAre(Bucket{10, 1, 1})); + EXPECT_THAT(parent_histogram->detailedIntervalBuckets(), UnorderedElementsAre(Bucket{10, 1, 1})); } TEST_F(HistogramTest, ForEachHistogram) { diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h index f33559407934..2a145792f50c 100644 --- a/test/mocks/stats/mocks.h +++ b/test/mocks/stats/mocks.h @@ -200,12 +200,13 @@ class MockParentHistogram : public MockMetric { void merge() override {} std::string quantileSummary() const override { return ""; }; std::string bucketSummary() const override { return ""; }; - MOCK_METHOD(bool, used, (), (const)); MOCK_METHOD(Histogram::Unit, unit, (), (const)); MOCK_METHOD(void, recordValue, (uint64_t value)); MOCK_METHOD(const HistogramStatistics&, cumulativeStatistics, (), (const)); MOCK_METHOD(const HistogramStatistics&, intervalStatistics, (), (const)); + MOCK_METHOD(std::vector, detailedTotalBuckets, (), (const)); + MOCK_METHOD(std::vector, detailedIntervalBuckets, (), (const)); // RefcountInterface void incRefCount() override { refcount_helper_.incRefCount(); } diff --git a/test/server/admin/admin_test.cc b/test/server/admin/admin_test.cc index ec9d0f39d1d9..106f7656e8ef 100644 --- a/test/server/admin/admin_test.cc +++ b/test/server/admin/admin_test.cc @@ -172,7 +172,7 @@ TEST_P(AdminInstanceTest, Help) { filter: Regular expression (Google re2) for filtering stats format: Format to use; One of (html, active-html, text, json) type: Stat types to include.; One of (All, Counters, Histograms, Gauges, TextReadouts) - histogram_buckets: Histogram bucket display mode; One of (cumulative, disjoint, none) + histogram_buckets: Histogram bucket display mode; One of (cumulative, disjoint, detailed, none) /stats/prometheus: print server stats in prometheus format usedonly: Only include stats that have been written by system since restart text_readouts: Render text_readouts as new gaugues with value 0 (increases Prometheus data size) diff --git a/test/server/admin/stats_render_test.cc b/test/server/admin/stats_render_test.cc index b2f9ce223153..569995c892be 100644 --- a/test/server/admin/stats_render_test.cc +++ b/test/server/admin/stats_render_test.cc @@ -47,6 +47,19 @@ TEST_F(StatsRenderTest, TextHistogramDisjoint) { EXPECT_EQ(expected, render<>(renderer, "h1", populateHistogram("h1", {200, 300, 300}))); } +TEST_F(StatsRenderTest, TextHistogramDetailed) { + params_.histogram_buckets_mode_ = Utility::HistogramBucketsMode::Detailed; + StatsTextRender renderer(params_); + constexpr absl::string_view expected = + "h1:\n" + " totals=200,10:1, 300,10:2\n" + " intervals=200,10:1, 300,10:2\n" + " summary=P0(200,200) P25(207.5,207.5) P50(302.5,302.5) P75(306.25,306.25) P90(308.5,308.5) " + "P95(309.25,309.25) P99(309.85,309.85) P99.5(309.925,309.925) P99.9(309.985,309.985) " + "P100(310,310)\n"; + EXPECT_EQ(expected, render<>(renderer, "h1", populateHistogram("h1", {200, 300, 300}))); +} + TEST_F(StatsRenderTest, JsonInt) { StatsJsonRender renderer(response_headers_, response_, params_); const std::string expected = R"EOF({"stats":[ {"value":42, "name":"name"}]})EOF"; @@ -297,5 +310,38 @@ TEST_F(StatsRenderTest, JsonHistogramDisjoint) { JsonStringEq(expected)); } +TEST_F(StatsRenderTest, JsonHistogramDetailed) { + params_.histogram_buckets_mode_ = Utility::HistogramBucketsMode::Detailed; + StatsJsonRender renderer(response_headers_, response_, params_); + const std::string expected = R"EOF( +{ + "stats": [{ + "histograms": { + "supported_percentiles": [0, 25, 50, 75, 90, 95, 99, 99.5, 99.9, 100], + "details": [{ + "name": "h1", + "percentiles": [ + {"cumulative": 200, "interval": 200}, + {"cumulative": 207.5, "interval": 207.5}, + {"cumulative": 302.5, "interval": 302.5}, + {"cumulative": 306.25, "interval": 306.25}, + {"cumulative": 308.5, "interval": 308.5}, + {"cumulative": 309.25, "interval": 309.25}, + {"cumulative": 309.85, "interval": 309.85}, + {"cumulative": 309.925, "interval": 309.925}, + {"cumulative": 309.985, "interval": 309.985}, + {"cumulative": 310, "interval": 310} + ], + "totals": [ + {"lower_bound": 200, "width": 10, "count": 1}, + {"lower_bound": 300, "width": 10, "count": 2}], + "intervals":[ + {"lower_bound": 200, "width": 10, "count": 1}, + {"lower_bound": 300, "width": 10, "count": 2}]}]}}]} + )EOF"; + EXPECT_THAT(render<>(renderer, "h1", populateHistogram("h1", {200, 300, 300})), + JsonStringEq(expected)); +} + } // namespace Server } // namespace Envoy From 0bf5813e5264c4b1c1b05d1094fd66c92d146e53 Mon Sep 17 00:00:00 2001 From: StarryNight Date: Thu, 18 May 2023 11:57:36 +0800 Subject: [PATCH 274/740] add go extension trailer del api (#27445) * add go extension trailer del api Signed-off-by: wangkai19 * fix thread race Signed-off-by: wangkai19 --------- Signed-off-by: wangkai19 --- contrib/golang/filters/http/source/cgo.cc | 7 ++ .../filters/http/source/go/pkg/api/api.h | 1 + .../filters/http/source/go/pkg/api/capi.go | 1 + .../http/source/go/pkg/http/capi_impl.go | 5 ++ .../filters/http/source/go/pkg/http/type.go | 4 +- .../filters/http/source/golang_filter.cc | 85 +++++++++++++++++-- .../filters/http/source/golang_filter.h | 1 + .../http/test/golang_integration_test.cc | 8 +- .../http/test/test_data/basic/filter.go | 1 + 9 files changed, 101 insertions(+), 12 deletions(-) diff --git a/contrib/golang/filters/http/source/cgo.cc b/contrib/golang/filters/http/source/cgo.cc index 2529958a58e7..9b263471e55b 100644 --- a/contrib/golang/filters/http/source/cgo.cc +++ b/contrib/golang/filters/http/source/cgo.cc @@ -149,6 +149,13 @@ CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key, void* value, headerAc }); } +CAPIStatus envoyGoFilterHttpRemoveTrailer(void* r, void* key) { + return envoyGoFilterHandlerWrapper(r, [key](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = referGoString(key); + return filter->removeTrailer(key_str); + }); +} + CAPIStatus envoyGoFilterHttpGetStringValue(void* r, int id, void* value) { return envoyGoFilterHandlerWrapper(r, [id, value](std::shared_ptr& filter) -> CAPIStatus { auto value_str = reinterpret_cast(value); diff --git a/contrib/golang/filters/http/source/go/pkg/api/api.h b/contrib/golang/filters/http/source/go/pkg/api/api.h index 3ce0714a1415..76e61903051f 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/api.h +++ b/contrib/golang/filters/http/source/go/pkg/api/api.h @@ -56,6 +56,7 @@ CAPIStatus envoyGoFilterHttpSetBufferHelper(void* r, unsigned long long int buff CAPIStatus envoyGoFilterHttpCopyTrailers(void* r, void* strs, void* buf); CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key, void* value, headerAction action); +CAPIStatus envoyGoFilterHttpRemoveTrailer(void* r, void* key); CAPIStatus envoyGoFilterHttpGetStringValue(void* r, int id, void* value); CAPIStatus envoyGoFilterHttpGetIntegerValue(void* r, int id, void* value); diff --git a/contrib/golang/filters/http/source/go/pkg/api/capi.go b/contrib/golang/filters/http/source/go/pkg/api/capi.go index 2cc1d2a273c5..60b95d49aac8 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/capi.go +++ b/contrib/golang/filters/http/source/go/pkg/api/capi.go @@ -37,6 +37,7 @@ type HttpCAPI interface { HttpCopyTrailers(r unsafe.Pointer, num uint64, bytes uint64) map[string][]string HttpSetTrailer(r unsafe.Pointer, key *string, value *string, add bool) + HttpRemoveTrailer(r unsafe.Pointer, key *string) HttpGetStringValue(r unsafe.Pointer, id int) (string, bool) HttpGetIntegerValue(r unsafe.Pointer, id int) (uint64, bool) diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go index f89dc9f7175d..b559ddaea20b 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go +++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go @@ -210,6 +210,11 @@ func (c *httpCApiImpl) HttpSetTrailer(r unsafe.Pointer, key *string, value *stri handleCApiStatus(res) } +func (c *httpCApiImpl) HttpRemoveTrailer(r unsafe.Pointer, key *string) { + res := C.envoyGoFilterHttpRemoveTrailer(r, unsafe.Pointer(key)) + handleCApiStatus(res) +} + func (c *httpCApiImpl) HttpGetStringValue(r unsafe.Pointer, id int) (string, bool) { var value string // TODO: add a lock to protect filter->req_->strValue field in the Envoy side, from being writing concurrency, diff --git a/contrib/golang/filters/http/source/go/pkg/http/type.go b/contrib/golang/filters/http/source/go/pkg/http/type.go index 0131fdcb677b..411f7db12d7a 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/type.go +++ b/contrib/golang/filters/http/source/go/pkg/http/type.go @@ -226,7 +226,9 @@ func (h *requestOrResponseTrailerMapImpl) Add(key, value string) { } func (h *requestOrResponseTrailerMapImpl) Del(key string) { - panic("unsupported yet") + h.initTrailers() + delete(h.headers, key) + cAPI.HttpRemoveTrailer(unsafe.Pointer(h.request.req), &key) } func (h *requestOrResponseTrailerMapImpl) Range(f func(key, value string) bool) { diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc index 17fe8a3903c2..9e7e2eb5a7ca 100644 --- a/contrib/golang/filters/http/source/golang_filter.cc +++ b/contrib/golang/filters/http/source/golang_filter.cc @@ -840,17 +840,84 @@ CAPIStatus Filter::setTrailer(absl::string_view key, absl::string_view value, he ENVOY_LOG(debug, "invoking cgo api at invalid phase: {}", __func__); return CAPIStatus::CAPIInvalidPhase; } - switch (act) { - case HeaderAdd: - trailers_->addCopy(Http::LowerCaseString(key), value); - break; + if (state.isThreadSafe()) { + switch (act) { + case HeaderAdd: + trailers_->addCopy(Http::LowerCaseString(key), value); + break; - case HeaderSet: - trailers_->setCopy(Http::LowerCaseString(key), value); - break; + case HeaderSet: + trailers_->setCopy(Http::LowerCaseString(key), value); + break; - default: - RELEASE_ASSERT(false, absl::StrCat("unknown header action: ", act)); + default: + RELEASE_ASSERT(false, absl::StrCat("unknown header action: ", act)); + } + } else { + // should deep copy the string_view before post to dipatcher callback. + auto key_str = std::string(key); + auto value_str = std::string(value); + + auto weak_ptr = weak_from_this(); + // dispatch a callback to write trailer in the envoy safe thread, to make the write operation + // safety. otherwise, there might be race between reading in the envoy worker thread and + // writing in the Go thread. + state.getDispatcher().post([this, weak_ptr, key_str, value_str, act] { + if (!weak_ptr.expired() && !hasDestroyed()) { + Thread::LockGuard lock(mutex_); + switch (act) { + case HeaderAdd: + trailers_->addCopy(Http::LowerCaseString(key_str), value_str); + break; + + case HeaderSet: + trailers_->setCopy(Http::LowerCaseString(key_str), value_str); + break; + + default: + RELEASE_ASSERT(false, absl::StrCat("unknown header action: ", act)); + } + } else { + ENVOY_LOG(debug, "golang filter has gone or destroyed in setTrailer"); + } + }); + } + return CAPIStatus::CAPIOK; +} + +CAPIStatus Filter::removeTrailer(absl::string_view key) { + Thread::LockGuard lock(mutex_); + if (has_destroyed_) { + ENVOY_LOG(debug, "golang filter has been destroyed"); + return CAPIStatus::CAPIFilterIsDestroy; + } + auto& state = getProcessorState(); + if (!state.isProcessingInGo()) { + ENVOY_LOG(debug, "golang filter is not processing Go"); + return CAPIStatus::CAPINotInGo; + } + if (trailers_ == nullptr) { + ENVOY_LOG(debug, "invoking cgo api at invalid phase: {}", __func__); + return CAPIStatus::CAPIInvalidPhase; + } + if (state.isThreadSafe()) { + trailers_->remove(Http::LowerCaseString(key)); + } else { + // should deep copy the string_view before post to dipatcher callback. + auto key_str = std::string(key); + + auto weak_ptr = weak_from_this(); + // dispatch a callback to write trailer in the envoy safe thread, to make the write operation + // safety. otherwise, there might be race between reading in the envoy worker thread and writing + // in the Go thread. + state.getDispatcher().post([this, weak_ptr, key_str] { + if (!weak_ptr.expired() && !hasDestroyed()) { + Thread::LockGuard lock(mutex_); + trailers_->remove(Http::LowerCaseString(key_str)); + } else { + ENVOY_LOG(debug, "golang filter has gone or destroyed in removeTrailer"); + } + }); } return CAPIStatus::CAPIOK; } diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h index 7fcb94bce04f..888c8b544c90 100644 --- a/contrib/golang/filters/http/source/golang_filter.h +++ b/contrib/golang/filters/http/source/golang_filter.h @@ -177,6 +177,7 @@ class Filter : public Http::StreamFilter, bufferAction action); CAPIStatus copyTrailers(GoString* go_strs, char* go_buf); CAPIStatus setTrailer(absl::string_view key, absl::string_view value, headerAction act); + CAPIStatus removeTrailer(absl::string_view key); CAPIStatus getStringValue(int id, GoString* value_str); CAPIStatus getIntegerValue(int id, uint64_t* value); CAPIStatus setDynamicMetadata(std::string filter_name, std::string key, absl::string_view buf); diff --git a/contrib/golang/filters/http/test/golang_integration_test.cc b/contrib/golang/filters/http/test/golang_integration_test.cc index 0b3efc96a2f8..0ec933742933 100644 --- a/contrib/golang/filters/http/test/golang_integration_test.cc +++ b/contrib/golang/filters/http/test/golang_integration_test.cc @@ -240,8 +240,8 @@ name: golang codec_client_->sendData(request_encoder, "helloworld", false); codec_client_->sendData(request_encoder, "", false); - Http::TestRequestTrailerMapImpl request_trailers{{"x-test-trailer-0", "foo"}, - {"existed-trailer", "foo"}}; + Http::TestRequestTrailerMapImpl request_trailers{ + {"x-test-trailer-0", "foo"}, {"existed-trailer", "foo"}, {"x-test-trailer-1", "foo"}}; codec_client_->sendTrailers(request_encoder, request_trailers); waitForNextUpstreamRequest(); @@ -292,6 +292,10 @@ name: golang entries = upstream_request_->trailers()->get(Http::LowerCaseString("x-test-trailer-0")); EXPECT_EQ("bar", entries[0]->value().getStringView()); + EXPECT_EQ( + true, + upstream_request_->trailers()->get(Http::LowerCaseString("x-test-trailer-1")).empty()); + Http::TestResponseHeaderMapImpl response_headers{ {":status", "200"}, {"x-test-header-0", "foo"}, diff --git a/contrib/golang/filters/http/test/test_data/basic/filter.go b/contrib/golang/filters/http/test/test_data/basic/filter.go index baaf13f7f0ee..91d6d80ac200 100644 --- a/contrib/golang/filters/http/test/test_data/basic/filter.go +++ b/contrib/golang/filters/http/test/test_data/basic/filter.go @@ -191,6 +191,7 @@ func (f *filter) decodeTrailers(trailers api.RequestTrailerMap) api.StatusType { trailers.Add("existed-trailer", "bar") trailers.Set("x-test-trailer-0", "bar") + trailers.Del("x-test-trailer-1") if f.panic == "decode-trailer" { badcode() From 7d1b1d0040054093575dcda742099be9641e3a64 Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Thu, 18 May 2023 00:50:32 -0400 Subject: [PATCH 275/740] API definition for scrubbing request headers sending from ext_proc filter to ext_proc server (#27333) API definition for scrubbing request headers sending from ext_proc filter to ext_proc server Signed-off-by: Yanjun Xiang --- .../extensions/filters/http/ext_proc/v3/BUILD | 1 + .../filters/http/ext_proc/v3/ext_proc.proto | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/BUILD b/api/envoy/extensions/filters/http/ext_proc/v3/BUILD index aea366ea77e3..e5fde2bafa2d 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/BUILD +++ b/api/envoy/extensions/filters/http/ext_proc/v3/BUILD @@ -8,6 +8,7 @@ api_proto_package( deps = [ "//envoy/config/common/mutation_rules/v3:pkg", "//envoy/config/core/v3:pkg", + "//envoy/type/matcher/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", "@com_github_cncf_udpa//xds/annotations/v3:pkg", ], diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto index 038967f3f055..c84c3790d656 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto @@ -5,6 +5,7 @@ package envoy.extensions.filters.http.ext_proc.v3; import "envoy/config/common/mutation_rules/v3/mutation_rules.proto"; import "envoy/config/core/v3/grpc_service.proto"; import "envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto"; +import "envoy/type/matcher/v3/string.proto"; import "google/protobuf/duration.proto"; @@ -95,7 +96,7 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // messages, and the server must reply with // :ref:`ProcessingResponse `. -// [#next-free-field: 12] +// [#next-free-field: 13] message ExternalProcessor { // Configuration for the gRPC service that the filter will communicate with. // The filter supports both the "Envoy" and "Google" gRPC clients. @@ -180,6 +181,17 @@ message ExternalProcessor { // :ref:`clear_route_cache ` // field is set in an external processor response. bool disable_clear_route_cache = 11; + + // [#not-implemented-hide:] + // Allow headers matching the ``forward_rules`` to be forwarded to the external processing server. + // If not set, all headers are forwarded to the external processing server. + HeaderForwardingRules forward_rules = 12; +} + +// The HeaderForwardingRules structure specifies what headers are +// allowed to be forwarded to the external processing server. +message HeaderForwardingRules { + type.matcher.v3.ListStringMatcher allowed_headers = 1; } // Extra settings that may be added to per-route configuration for a From d9f164ff26802b9217cb6e1f1f266cc804010280 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 18 May 2023 08:05:11 +0100 Subject: [PATCH 276/740] ci/dockerhub: Improve publishing and add test (#27420) Signed-off-by: Ryan Northey --- .azure-pipelines/stage/prechecks.yml | 14 ++++++++++++++ .azure-pipelines/stage/publish.yml | 9 ++++++++- ci/do_ci.sh | 13 +++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/stage/prechecks.yml b/.azure-pipelines/stage/prechecks.yml index 8aeb26c2656e..869b438bc7d5 100644 --- a/.azure-pipelines/stage/prechecks.yml +++ b/.azure-pipelines/stage/prechecks.yml @@ -115,6 +115,20 @@ jobs: timeoutInMinutes: 10 condition: and(failed(), eq(variables['CI_TARGET'], 'docs')) + # Dockerhub readme publishing + - script: | + ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-readme' + displayName: "Dockerhub publishing test" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_RBE: "1" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} + GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + condition: eq(variables['CI_TARGET'], 'docs') + stepsPost: # Format fixes diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index eebe78b2c281..3e97ef8bdbc7 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -260,9 +260,16 @@ jobs: displayName: "Generate docs" - script: | - bazel run //tools/distribution:update_dockerhub_repository + ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-publish' displayName: "Publish Dockerhub description and README" env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_RBE: "1" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} + GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} condition: and(eq(variables['isMain'], 'true'), eq(variables['PostSubmit'], true)) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 378ad44fb123..662dc101406e 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -513,6 +513,19 @@ case $CI_TARGET in "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" "${BUILD_DIR}/build_images" docker ;; + dockerhub-publish) + setup_clang_toolchain + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/distribution:update_dockerhub_repository + ;; + + dockerhub-readme) + setup_clang_toolchain + bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ + //distribution/dockerhub:readme + cat bazel-bin/distribution/dockerhub/readme.md + ;; + fix_proto_format) # proto_format.sh needs to build protobuf. setup_clang_toolchain From 18bbc03c014f4a5f16d0ae23a372fb3c342fb36f Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 18 May 2023 08:36:27 +0100 Subject: [PATCH 277/740] ci: Fix dockerhub publishing credentials Signed-off-by: Ryan Northey --- ci/run_envoy_docker.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index e9c52afec150..2dcda09a6deb 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -116,6 +116,8 @@ docker run --rm \ -e BAZEL_EXTRA_TEST_OPTIONS \ -e BAZEL_FAKE_SCM_REVISION \ -e BAZEL_REMOTE_CACHE \ + -e DOCKERHUB_USERNAME \ + -e DOCKERHUB_PASSWORD \ -e ENVOY_STDLIB \ -e BUILD_REASON \ -e BAZEL_NO_CACHE_TEST_RESULTS \ From 7e50eecb80669d04c74b4125bfee00fc96c7fb85 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 3 May 2023 08:47:05 +0100 Subject: [PATCH 278/740] ci: Workaround dep checker (cves) issue again Signed-off-by: Ryan Northey --- ci/do_ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 662dc101406e..427dfc3a1cf7 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -438,7 +438,8 @@ case $CI_TARGET in bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:check \ --action_env=TODAY_DATE \ -- -v warn \ - -c cves release_dates releases + -c cves release_dates releases || echo "WARNING: Dependency check failed" + # Run dependabot tests echo "Check dependabot ..." bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ From c7d50bf3f32fea67fe5a702d81752500470f3255 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 May 2023 09:03:44 +0100 Subject: [PATCH 279/740] build(deps): bump gsutil from 5.23 to 5.24 in /tools/base (#27471) Bumps [gsutil](https://github.com/GoogleCloudPlatform/gsutil) from 5.23 to 5.24. - [Changelog](https://github.com/GoogleCloudPlatform/gsutil/blob/master/CHANGES.md) - [Commits](https://github.com/GoogleCloudPlatform/gsutil/compare/v5.23...v5.24) --- updated-dependencies: - dependency-name: gsutil dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 03539bbf0380..4025f63386ce 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -748,8 +748,8 @@ googleapis-common-protos==1.59.0 \ --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \ --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f # via google-api-core -gsutil==5.23 \ - --hash=sha256:76cc1d45dcf995f737a16eba69ee57b36fa3d973092f689bdef021d4584c842a +gsutil==5.24 \ + --hash=sha256:1f841645cda40fcc817e9ca84d285cdf541cc015fd38a5862017b085756729a0 # via -r requirements.in httplib2==0.20.4 \ --hash=sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585 \ From 9c9a75a4b556dfd607cca4c41db4d69093600d9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 May 2023 09:18:19 +0100 Subject: [PATCH 280/740] build(deps): bump protobuf from 4.23.0 to 4.23.1 in /examples/grpc-bridge/client (#27469) build(deps): bump protobuf in /examples/grpc-bridge/client Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 4.23.0 to 4.23.1. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v4.23.0...v4.23.1) --- updated-dependencies: - dependency-name: protobuf dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/client/requirements.txt | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index d1e3fa54972a..4f4af0641998 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -112,20 +112,20 @@ idna==3.2 \ --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \ --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3 # via requests -protobuf==4.23.0 \ - --hash=sha256:03eee35b60317112a72d19c54d0bff7bc58ff12fea4cd7b018232bd99758ffdf \ - --hash=sha256:2b94bd6df92d71bd1234a2ffe7ce96ddf6d10cf637a18d6b55ad0a89fbb7fc21 \ - --hash=sha256:36f5370a930cb77c8ad2f4135590c672d0d2c72d4a707c7d0058dce4b4b4a598 \ - --hash=sha256:5f1eba1da2a2f3f7df469fccddef3cc060b8a16cfe3cc65961ad36b4dbcf59c5 \ - --hash=sha256:6c16657d6717a0c62d5d740cb354fbad1b0d8cb811669e06fc1caa0ff4799ddd \ - --hash=sha256:6fe180b56e1169d72ecc4acbd39186339aed20af5384531b8e8979b02bbee159 \ - --hash=sha256:7cb5b9a05ce52c6a782bb97de52679bd3438ff2b7460eff5da348db65650f227 \ - --hash=sha256:9744e934ea5855d12191040ea198eaf704ac78665d365a89d9572e3b627c2688 \ - --hash=sha256:9f5a0fbfcdcc364f3986f9ed9f8bb1328fb84114fd790423ff3d7fdb0f85c2d1 \ - --hash=sha256:baca40d067dddd62141a129f244703160d278648b569e90bb0e3753067644711 \ - --hash=sha256:d5a35ff54e3f62e8fc7be02bb0d2fbc212bba1a5a9cc2748090690093996f07b \ - --hash=sha256:e62fb869762b4ba18666370e2f8a18f17f8ab92dd4467295c6d38be6f8fef60b \ - --hash=sha256:ebde3a023b8e11bfa6c890ef34cd6a8b47d586f26135e86c21344fe433daf2e2 +protobuf==4.23.1 \ + --hash=sha256:2036a3a1e7fc27f973fa0a7888dce712393af644f4695385f117886abc792e39 \ + --hash=sha256:32e78beda26d7a101fecf15d7a4a792278a0d26a31bc327ff05564a9d68ab8ee \ + --hash=sha256:346990f634272caac1f09efbcfbbacb23098b1f606d172534c6fa2d9758bb436 \ + --hash=sha256:3b8905eafe4439076e1f58e9d1fa327025fd2777cf90f14083092ae47f77b0aa \ + --hash=sha256:3ce113b3f3362493bddc9069c2163a38f240a9ed685ff83e7bcb756b05e1deb0 \ + --hash=sha256:410bcc0a5b279f634d3e16082ce221dfef7c3392fac723500e2e64d1806dd2be \ + --hash=sha256:5b9cd6097e6acae48a68cb29b56bc79339be84eca65b486910bb1e7a30e2b7c1 \ + --hash=sha256:65f0ac96ef67d7dd09b19a46aad81a851b6f85f89725577f16de38f2d68ad477 \ + --hash=sha256:91fac0753c3c4951fbb98a93271c43cc7cf3b93cf67747b3e600bb1e5cc14d61 \ + --hash=sha256:95789b569418a3e32a53f43d7763be3d490a831e9c08042539462b6d972c2d7e \ + --hash=sha256:ac50be82491369a9ec3710565777e4da87c6d2e20404e0abb1f3a8f10ffd20f0 \ + --hash=sha256:decf119d54e820f298ee6d89c72d6b289ea240c32c521f00433f9dc420595f38 \ + --hash=sha256:f9510cac91e764e86acd74e2b7f7bc5e6127a7f3fb646d7c8033cfb84fd1176a # via # -r requirements.in # grpcio-tools From 1816b675dbd3e6ae10ac01778f5b9aca6e54f71d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 May 2023 10:09:55 +0100 Subject: [PATCH 281/740] build(deps): bump node from 20.1-bullseye-slim to 20.2-bullseye-slim in /examples/shared/node (#27444) build(deps): bump node in /examples/shared/node Bumps node from 20.1-bullseye-slim to 20.2-bullseye-slim. --- updated-dependencies: - dependency-name: node dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/node/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile index 344e4ac0dd05..7cd02355882d 100644 --- a/examples/shared/node/Dockerfile +++ b/examples/shared/node/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.1-bullseye-slim@sha256:bc5812b018fa74ea7dbe759cb6c0b456ff96a5c2bc8765e132438f6a75cd6946 as node-base +FROM node:20.2-bullseye-slim@sha256:5138ded35380c7e55b7898a5c3666009334aa4af416571060d37347242e1812f as node-base FROM node-base as node-http-auth From 3f06a3b4d2e3d41bcf204c3154e67b1b967dbf98 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 18 May 2023 15:14:19 +0100 Subject: [PATCH 282/740] deps: Bump `rules_apple` -> 2.3.0 (#27477) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index f8a3198ab427..123ef48acc52 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -60,10 +60,10 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Apple Rules for Bazel", project_desc = "Bazel rules for Apple platforms", project_url = "https://github.com/bazelbuild/rules_apple", - version = "2.0.0", - sha256 = "43737f28a578d8d8d7ab7df2fb80225a6b23b9af9655fcdc66ae38eb2abcf2ed", + version = "2.3.0", + sha256 = "a6141240657093fa7ccc7ca1ee5a62408dd9996d1bf47bc2369b8b9faefb2698", urls = ["https://github.com/bazelbuild/rules_apple/releases/download/{version}/rules_apple.{version}.tar.gz"], - release_date = "2022-12-21", + release_date = "2023-05-17", use_category = ["build"], license = "Apache-2.0", license_url = "https://github.com/bazelbuild/rules_apple/blob/{version}/LICENSE", From c6e9cd59fda532c1e2b3c1d74d390d6d85d2e2ff Mon Sep 17 00:00:00 2001 From: Michael Kaufmann Date: Thu, 18 May 2023 16:18:14 +0200 Subject: [PATCH 283/740] build: include "cstdint" for the types uint8_t and uint16_t (#27343) This fixes a build problem on Fedora 38 with clang 16. (there are other build problems in external libraries) Commit Message: Additional Description: Risk Level: Low Testing: Docs Changes: Release Notes: Platform Specific Features: Signed-off-by: Michael Kaufmann --- source/common/common/hex.h | 1 + source/extensions/clusters/redis/crc16.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/common/hex.h b/source/common/common/hex.h index aba722a4fc06..49f78fed6909 100644 --- a/source/common/common/hex.h +++ b/source/common/common/hex.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include diff --git a/source/extensions/clusters/redis/crc16.h b/source/extensions/clusters/redis/crc16.h index 25ba5806dc30..e0b0de8992af 100644 --- a/source/extensions/clusters/redis/crc16.h +++ b/source/extensions/clusters/redis/crc16.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include "absl/strings/ascii.h" From 7a6c72ce9bc6a6722a7cdb1a08b86c77868480cf Mon Sep 17 00:00:00 2001 From: Artur Melo Date: Thu, 18 May 2023 15:34:10 +0100 Subject: [PATCH 284/740] dns_cares: Add debug log entry when resolution response holds no records (#27486) Fixes #27214 Signed-off-by: Restless-ET --- .../extensions/network/dns_resolver/cares/dns_impl.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/extensions/network/dns_resolver/cares/dns_impl.cc b/source/extensions/network/dns_resolver/cares/dns_impl.cc index 9f514933ca86..1c4bd6fa6a51 100644 --- a/source/extensions/network/dns_resolver/cares/dns_impl.cc +++ b/source/extensions/network/dns_resolver/cares/dns_impl.cc @@ -146,11 +146,14 @@ void DnsResolverImpl::AddrInfoPendingResolution::onAresGetAddrInfoCallback( if (status != ARES_SUCCESS) { parent_.chargeGetAddrInfoErrorStats(status, timeouts); - } - if (status != ARES_SUCCESS && !isResponseWithNoRecords(status)) { - ENVOY_LOG_EVENT(debug, "cares_resolution_failure", - "dns resolution for {} failed with c-ares status {}", dns_name_, status); + if (!isResponseWithNoRecords(status)) { + ENVOY_LOG_EVENT(debug, "cares_resolution_failure", + "dns resolution for {} failed with c-ares status {}", dns_name_, status); + } else { + ENVOY_LOG_EVENT(debug, "cares_resolution_no_records", "dns resolution without records for {}", + dns_name_); + } } // We receive ARES_EDESTRUCTION when destructing with pending queries. From e669e5d40f8820262aee293a9e2773688ffbab96 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 18 May 2023 07:45:33 -0700 Subject: [PATCH 285/740] deps: update re2 (#27464) * deps: update re2 Signed-off-by: Kuat Yessenov --- bazel/repository_locations.bzl | 6 +++--- source/common/common/regex.h | 7 ++----- .../path/match/uri_template/uri_template_match.cc | 3 +-- .../rewrite/uri_template/uri_template_rewrite.cc | 4 ++-- .../path/uri_template_lib/uri_template.cc | 4 ++-- .../path/uri_template_lib/uri_template_internal.cc | 8 ++++---- .../path/uri_template_lib/uri_template_internal.h | 5 ----- .../uri_template_lib/uri_template_internal_test.cc | 14 +++++++------- 8 files changed, 21 insertions(+), 30 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 123ef48acc52..6c72be0aaf91 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1137,12 +1137,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "RE2", project_desc = "RE2, a regular expression library", project_url = "https://github.com/google/re2", - version = "2022-12-01", - sha256 = "665b65b6668156db2b46dddd33405cd422bd611352c5052ab3dae6a5fbac5506", + version = "2023-03-01", + sha256 = "7a9a4824958586980926a300b4717202485c4b4115ac031822e29aa4ef207e48", strip_prefix = "re2-{version}", urls = ["https://github.com/google/re2/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2022-11-30", + release_date = "2023-03-01", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/re2/blob/{version}/LICENSE", diff --git a/source/common/common/regex.h b/source/common/common/regex.h index 92b2720bc231..4189df75fc24 100644 --- a/source/common/common/regex.h +++ b/source/common/common/regex.h @@ -28,15 +28,12 @@ class CompiledGoogleReMatcher : public CompiledMatcher { explicit CompiledGoogleReMatcher(const envoy::type::matcher::v3::RegexMatcher& config); // CompiledMatcher - bool match(absl::string_view value) const override { - return re2::RE2::FullMatch(re2::StringPiece(value.data(), value.size()), regex_); - } + bool match(absl::string_view value) const override { return re2::RE2::FullMatch(value, regex_); } // CompiledMatcher std::string replaceAll(absl::string_view value, absl::string_view substitution) const override { std::string result = std::string(value); - re2::RE2::GlobalReplace(&result, regex_, - re2::StringPiece(substitution.data(), substitution.size())); + re2::RE2::GlobalReplace(&result, regex_, substitution); return result; } diff --git a/source/extensions/path/match/uri_template/uri_template_match.cc b/source/extensions/path/match/uri_template/uri_template_match.cc index a156508b9a88..6313c291b4a5 100644 --- a/source/extensions/path/match/uri_template/uri_template_match.cc +++ b/source/extensions/path/match/uri_template/uri_template_match.cc @@ -19,8 +19,7 @@ namespace Match { bool UriTemplateMatcher::match(absl::string_view path) const { RE2 matching_pattern_regex = RE2(convertPathPatternSyntaxToRegex(path_template_).value()); - return RE2::FullMatch(Internal::toStringPiece(Http::PathUtil::removeQueryAndFragment(path)), - matching_pattern_regex); + return RE2::FullMatch(Http::PathUtil::removeQueryAndFragment(path), matching_pattern_regex); } absl::string_view UriTemplateMatcher::uriTemplate() const { return path_template_; } diff --git a/source/extensions/path/rewrite/uri_template/uri_template_rewrite.cc b/source/extensions/path/rewrite/uri_template/uri_template_rewrite.cc index 2e69b72e7b0f..05ab1972ece8 100644 --- a/source/extensions/path/rewrite/uri_template/uri_template_rewrite.cc +++ b/source/extensions/path/rewrite/uri_template/uri_template_rewrite.cc @@ -55,7 +55,7 @@ absl::StatusOr UriTemplateRewriter::rewritePath(absl::string_view p RewriteSegments rewrite_pattern_segments = parseRewritePattern(rewrite_pattern_, regex_pattern_str).value(); - RE2 regex = RE2(Internal::toStringPiece(regex_pattern_str)); + RE2 regex = RE2(regex_pattern_str); if (!regex.ok()) { return absl::InternalError("Regex library failed"); } @@ -63,7 +63,7 @@ absl::StatusOr UriTemplateRewriter::rewritePath(absl::string_view p // First capture is the whole matched regex pattern. int capture_num = regex.NumberOfCapturingGroups() + 1; std::vector captures(capture_num); - if (!regex.Match(Internal::toStringPiece(pattern), /*startpos=*/0, + if (!regex.Match(pattern, /*startpos=*/0, /*endpos=*/pattern.size(), RE2::ANCHOR_BOTH, captures.data(), captures.size())) { return absl::InvalidArgumentError("Pattern not match"); } diff --git a/source/extensions/path/uri_template_lib/uri_template.cc b/source/extensions/path/uri_template_lib/uri_template.cc index af9f3d3e1a59..9eda8154562b 100644 --- a/source/extensions/path/uri_template_lib/uri_template.cc +++ b/source/extensions/path/uri_template_lib/uri_template.cc @@ -43,7 +43,7 @@ absl::StatusOr> parseRewritePattern(absl::string_view // Don't allow contiguous '/' patterns. static const LazyRE2 invalid_regex = {"^.*//.*$"}; - if (RE2::FullMatch(Internal::toStringPiece(path_pattern), *invalid_regex)) { + if (RE2::FullMatch(path_pattern, *invalid_regex)) { return absl::InvalidArgumentError("Invalid rewrite literal"); } @@ -80,7 +80,7 @@ absl::StatusOr> parseRewritePattern(absl::string_view absl::StatusOr parseRewritePattern(absl::string_view pattern, absl::string_view capture_regex) { RewriteSegments parsed_pattern; - RE2 regex = RE2(Internal::toStringPiece(capture_regex)); + RE2 regex = RE2(capture_regex); if (!regex.ok()) { return absl::InternalError(regex.error()); } diff --git a/source/extensions/path/uri_template_lib/uri_template_internal.cc b/source/extensions/path/uri_template_lib/uri_template_internal.cc index 985be5973fa9..0b636fb2f75d 100644 --- a/source/extensions/path/uri_template_lib/uri_template_internal.cc +++ b/source/extensions/path/uri_template_lib/uri_template_internal.cc @@ -122,19 +122,19 @@ bool isValidLiteral(absl::string_view literal) { static const std::string* kValidLiteralRegex = new std::string(absl::StrCat("^[", kLiteral, "]+$")); static const LazyRE2 literal_regex = {kValidLiteralRegex->data()}; - return RE2::FullMatch(toStringPiece(literal), *literal_regex); + return RE2::FullMatch(literal, *literal_regex); } bool isValidRewriteLiteral(absl::string_view literal) { static const std::string* kValidLiteralRegex = new std::string(absl::StrCat("^[", kLiteral, "/]+$")); static const LazyRE2 literal_regex = {kValidLiteralRegex->data()}; - return RE2::FullMatch(toStringPiece(literal), *literal_regex); + return RE2::FullMatch(literal, *literal_regex); } bool isValidVariableName(absl::string_view variable) { static const LazyRE2 variable_regex = {"^[a-zA-Z][a-zA-Z0-9_]*$"}; - return RE2::FullMatch(toStringPiece(variable), *variable_regex); + return RE2::FullMatch(variable, *variable_regex); } absl::StatusOr> parseLiteral(absl::string_view pattern) { @@ -273,7 +273,7 @@ absl::StatusOr parsePathPatternSyntax(absl::string_view path) struct ParsedPathPattern parsed_pattern; static const LazyRE2 printable_regex = {"^/[[:graph:]]*$"}; - if (!RE2::FullMatch(toStringPiece(path), *printable_regex)) { + if (!RE2::FullMatch(path, *printable_regex)) { return absl::InvalidArgumentError("Invalid pattern"); } diff --git a/source/extensions/path/uri_template_lib/uri_template_internal.h b/source/extensions/path/uri_template_lib/uri_template_internal.h index b27bd4062a20..674348e8592b 100644 --- a/source/extensions/path/uri_template_lib/uri_template_internal.h +++ b/source/extensions/path/uri_template_lib/uri_template_internal.h @@ -134,11 +134,6 @@ std::string toRegexPattern(const Variable& pattern); */ std::string toRegexPattern(const struct ParsedPathPattern& pattern); -/** - * Converts string_view to be used in re2::string_piece. - */ -inline re2::StringPiece toStringPiece(absl::string_view text) { return {text.data(), text.size()}; } - /** * Checks end of pattern to ensure glob operator is last. */ diff --git a/test/extensions/path/uri_template_lib/uri_template_internal_test.cc b/test/extensions/path/uri_template_lib/uri_template_internal_test.cc index b7d92e3dd454..dff0e2283ae9 100644 --- a/test/extensions/path/uri_template_lib/uri_template_internal_test.cc +++ b/test/extensions/path/uri_template_lib/uri_template_internal_test.cc @@ -216,24 +216,24 @@ TEST(InternalRegexGen, LiteralEscapes) { TEST(InternalRegexGen, LiteralMatches) { absl::string_view kPattern = "abcABC123/-._~%20!$&'()+,;:@"; - EXPECT_TRUE(RE2::FullMatch(toStringPiece(kPattern), toRegexPattern(kPattern))); + EXPECT_TRUE(RE2::FullMatch(kPattern, toRegexPattern(kPattern))); } TEST(InternalRegexGen, LiteralMatchesInNamedCapture) { - absl::string_view kPattern = "abcABC123/-._~%20!$&'()+,;:@"; + re2::StringPiece kPattern = "abcABC123/-._~%20!$&'()+,;:@"; RE2 regex = RE2(absl::StrCat("(?P", toRegexPattern(kPattern), ")")); ASSERT_EQ(regex.NumberOfCapturingGroups(), 1); // Full matched string + capture groups std::vector captures(2); - ASSERT_TRUE(regex.Match(toStringPiece(kPattern), /*startpos=*/0, /*endpos=*/kPattern.size(), - RE2::ANCHOR_BOTH, captures.data(), captures.size())); + ASSERT_TRUE(regex.Match(kPattern, /*startpos=*/0, /*endpos=*/kPattern.size(), RE2::ANCHOR_BOTH, + captures.data(), captures.size())); // Index 0 would be the full text of the matched string. - EXPECT_EQ(toStringPiece(kPattern), captures[0]); + EXPECT_EQ(kPattern, captures[0]); // Get the pattern matched with the named capture group. - EXPECT_EQ(toStringPiece(kPattern), captures.at(regex.NamedCapturingGroups().at("var"))); + EXPECT_EQ(kPattern, captures.at(regex.NamedCapturingGroups().at("var"))); } TEST(InternalRegexGen, LiteralOnlyMatchesItself) { @@ -434,7 +434,7 @@ TEST_P(GenPatternRegexWithMatch, WithCapture) { /*endpos=*/requestPath().size(), RE2::ANCHOR_BOTH, captures.data(), captures.size())); - EXPECT_EQ(captures[0], toStringPiece(requestPath())); + EXPECT_EQ(captures[0], requestPath()); for (const auto& [name, value] : varValues()) { int capture_index = regex.NamedCapturingGroups().at(name); From 29334c21020d7d823d6851b1d7d9104434d0c1d7 Mon Sep 17 00:00:00 2001 From: Greg Greenway Date: Thu, 18 May 2023 09:03:24 -0700 Subject: [PATCH 286/740] access log: attempt to fix test flake (#27438) CI showed a failure in TEST_F(AccessLogManagerImplTest, ReopenFileOnTimerOnly). Attempting to use more rigid thread-syncronization primitives instead of atomics to avoid races and/or undefined/misunderstood behavior. Signed-off-by: Greg Greenway --- .../access_log/access_log_manager_impl.cc | 25 ++- .../access_log/access_log_manager_impl.h | 4 +- .../access_log_manager_impl_test.cc | 160 +++++------------- test/mocks/filesystem/mocks.cc | 14 +- test/mocks/filesystem/mocks.h | 26 +-- 5 files changed, 82 insertions(+), 147 deletions(-) diff --git a/source/common/access_log/access_log_manager_impl.cc b/source/common/access_log/access_log_manager_impl.cc index 9a9e9b6a23d7..ba3486ca545f 100644 --- a/source/common/access_log/access_log_manager_impl.cc +++ b/source/common/access_log/access_log_manager_impl.cc @@ -72,7 +72,11 @@ Api::IoCallBoolResult AccessLogFileImpl::open() { return result; } -void AccessLogFileImpl::reopen() { reopen_file_ = true; } +void AccessLogFileImpl::reopen() { + Thread::LockGuard lock(write_lock_); + reopen_file_ = true; + flush_event_.notifyOne(); +} AccessLogFileImpl::~AccessLogFileImpl() { { @@ -127,6 +131,11 @@ void AccessLogFileImpl::doWrite(Buffer::Instance& buffer) { void AccessLogFileImpl::flushThreadFunc() { + // Transfer the action from `reopen_file_` to this variable so that `reopen_file_` is only + // accessed while holding the mutex while the actual operation is performed while not holding the + // mutex. + bool do_reopen = false; + while (true) { std::unique_lock flush_lock; @@ -135,6 +144,10 @@ void AccessLogFileImpl::flushThreadFunc() { // flush_event_ can be woken up either by large enough flush_buffer or by timer. // In case it was timer, flush_buffer_ can be empty. + // + // Note: do not stop waiting when only `do_reopen` is true. In this case, we tried to + // reopen and failed. We don't want to retry this in a tight loop, so wait for the next + // event (timer or flush). while (flush_buffer_.length() == 0 && !flush_thread_exit_ && !reopen_file_) { // CondVar::wait() does not throw, so it's safe to pass the mutex rather than the guard. flush_event_.wait(write_lock_); @@ -147,10 +160,14 @@ void AccessLogFileImpl::flushThreadFunc() { flush_lock = std::unique_lock(flush_lock_); about_to_write_buffer_.move(flush_buffer_); ASSERT(flush_buffer_.length() == 0); + + if (reopen_file_) { + do_reopen = true; + reopen_file_ = false; + } } - // if we failed to reopen before, do it next loop. - if (reopen_file_) { + if (do_reopen) { if (file_->isOpen()) { const Api::IoCallBoolResult result = file_->close(); ASSERT(result.return_value_, fmt::format("unable to close file '{}': {}", file_->path(), @@ -160,7 +177,7 @@ void AccessLogFileImpl::flushThreadFunc() { if (!open_result.return_value_) { stats_.reopen_failed_.inc(); } else { - reopen_file_ = false; + do_reopen = false; } } // doWrite no matter file isOpen, if not, we can drain buffer diff --git a/source/common/access_log/access_log_manager_impl.h b/source/common/access_log/access_log_manager_impl.h index 23651b874e1c..e7e150423438 100644 --- a/source/common/access_log/access_log_manager_impl.h +++ b/source/common/access_log/access_log_manager_impl.h @@ -114,8 +114,8 @@ class AccessLogFileImpl : public AccessLogFile { // high performance. It is always local to the process. Thread::ThreadPtr flush_thread_; Thread::CondVar flush_event_; - std::atomic flush_thread_exit_{}; - std::atomic reopen_file_{}; + bool flush_thread_exit_ ABSL_GUARDED_BY(write_lock_){false}; + bool reopen_file_ ABSL_GUARDED_BY(write_lock_){false}; Buffer::OwnedImpl flush_buffer_ ABSL_GUARDED_BY(write_lock_); // This buffer is used by multiple threads. It // gets filled and then flushed either when max diff --git a/test/common/access_log/access_log_manager_impl_test.cc b/test/common/access_log/access_log_manager_impl_test.cc index f72541111172..3bacc57c0bd9 100644 --- a/test/common/access_log/access_log_manager_impl_test.cc +++ b/test/common/access_log/access_log_manager_impl_test.cc @@ -115,12 +115,7 @@ TEST_F(AccessLogManagerImplTest, FlushToLogFilePeriodically) { log_file->write("test"); - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 1) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 1)); waitForCounterEq("filesystem.write_completed", 1); EXPECT_EQ(1UL, store_.counter("filesystem.write_buffered").value()); @@ -144,12 +139,7 @@ TEST_F(AccessLogManagerImplTest, FlushToLogFilePeriodically) { EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); timer->invokeCallback(); - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 2) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 2)); waitForCounterEq("filesystem.write_completed", 2); EXPECT_EQ(0UL, store_.counter("filesystem.write_failed").value()); @@ -179,12 +169,7 @@ TEST_F(AccessLogManagerImplTest, FlushToLogFileOnDemand) { })); log_file->write("prime-it"); uint32_t expected_writes = 1; - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != expected_writes) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, expected_writes)); EXPECT_CALL(*file_, write_(_)) .WillOnce(Invoke([](absl::string_view data) -> Api::IoCallSizeResult { @@ -195,18 +180,13 @@ TEST_F(AccessLogManagerImplTest, FlushToLogFileOnDemand) { log_file->write("test"); { - Thread::LockGuard lock(file_->write_mutex_); + absl::MutexLock lock(&file_->mutex_); EXPECT_EQ(expected_writes, file_->num_writes_); } log_file->flush(); expected_writes++; - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != expected_writes) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, expected_writes)); waitForCounterEq("filesystem.write_completed", 2); EXPECT_EQ(0UL, store_.counter("filesystem.flushed_by_timer").value()); @@ -223,12 +203,7 @@ TEST_F(AccessLogManagerImplTest, FlushToLogFileOnDemand) { timer->invokeCallback(); expected_writes++; - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != expected_writes) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, expected_writes)); EXPECT_CALL(*file_, close_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); } @@ -250,13 +225,7 @@ TEST_F(AccessLogManagerImplTest, FlushCountsIOErrors) { log_file->write("test"); - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 1) { - file_->write_event_.wait(file_->write_mutex_); - } - } - + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 1)); waitForCounterEq("filesystem.write_failed", 1); EXPECT_EQ(0UL, store_.counter("filesystem.write_completed").value()); @@ -282,13 +251,7 @@ TEST_F(AccessLogManagerImplTest, ReopenFile) { log_file->write("before"); timer->invokeCallback(); - - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 1) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 1)); EXPECT_CALL(*file_, close_()) .InSequence(sq) @@ -312,16 +275,12 @@ TEST_F(AccessLogManagerImplTest, ReopenFile) { log_file->write("reopened"); timer->invokeCallback(); - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 2) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 2)); + EXPECT_TRUE(file_->waitForEventCount(file_->num_opens_, 2)); } -// Test that the flush timer will trigger file reopen even if no data is waiting. -TEST_F(AccessLogManagerImplTest, ReopenFileOnTimerOnly) { +// Test that the `reopen()` will trigger file reopen even if no data is waiting. +TEST_F(AccessLogManagerImplTest, ReopenFileNoWrite) { NiceMock* timer = new NiceMock(&dispatcher_); Sequence sq; @@ -340,13 +299,7 @@ TEST_F(AccessLogManagerImplTest, ReopenFileOnTimerOnly) { log_file->write("before"); timer->invokeCallback(); - - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 1) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 1)); EXPECT_CALL(*file_, close_()) .InSequence(sq) @@ -360,14 +313,8 @@ TEST_F(AccessLogManagerImplTest, ReopenFileOnTimerOnly) { .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); log_file->reopen(); - timer->invokeCallback(); - { - Thread::LockGuard lock(file_->open_mutex_); - while (file_->num_opens_ != 2) { - file_->open_event_.wait(file_->open_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_opens_, 2)); } TEST_F(AccessLogManagerImplTest, ReopenRetry) { @@ -390,29 +337,28 @@ TEST_F(AccessLogManagerImplTest, ReopenRetry) { log_file->write("before reopen"); timer->invokeCallback(); - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 1) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 1)); EXPECT_CALL(*file_, close_()) .InSequence(sq) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); EXPECT_CALL(*file_, open_(_)) + .Times(3) .InSequence(sq) - .WillOnce(Return(ByMove(Filesystem::resultFailure(false, 0)))); - - EXPECT_CALL(*file_, open_(_)) - .InSequence(sq) + .WillOnce(Return(ByMove(Filesystem::resultFailure(false, 0)))) + .WillOnce(Return(ByMove(Filesystem::resultFailure(false, 0)))) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); EXPECT_CALL(*file_, write_(_)) + .Times(2) .InSequence(sq) .WillOnce(Invoke([](absl::string_view data) -> Api::IoCallSizeResult { - EXPECT_EQ(0, data.compare("after reopen")); + EXPECT_EQ(data, "retry reopen"); + return Filesystem::resultSuccess(static_cast(data.length())); + })) + .WillOnce(Invoke([](absl::string_view data) -> Api::IoCallSizeResult { + EXPECT_EQ(data, "after reopen"); return Filesystem::resultSuccess(static_cast(data.length())); })); @@ -422,26 +368,24 @@ TEST_F(AccessLogManagerImplTest, ReopenRetry) { log_file->reopen(); - log_file->write("drop data during reopen fail"); - timer->invokeCallback(); + EXPECT_TRUE(file_->waitForEventCount(file_->num_opens_, 2)); - { - Thread::LockGuard lock(file_->open_mutex_); - while (file_->num_opens_ != 3) { - file_->open_event_.wait(file_->open_mutex_); - } - } + // Retry the reopen by calling `reopen()` another time. + // This time is also set to fail. + log_file->reopen(); + EXPECT_TRUE(file_->waitForEventCount(file_->num_opens_, 3)); + + // Retry the reopen by writing more data and running the timer. + log_file->write("retry reopen"); + timer->invokeCallback(); + EXPECT_TRUE(file_->waitForEventCount(file_->num_opens_, 4)); + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 2)); log_file->write("after reopen"); timer->invokeCallback(); - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 2) { - file_->write_event_.wait(file_->write_mutex_); - } - } - waitForCounterEq("filesystem.reopen_failed", 1); + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 3)); + waitForCounterEq("filesystem.reopen_failed", 2); waitForGaugeEq("filesystem.write_total_buffered", 0); } @@ -457,13 +401,7 @@ TEST_F(AccessLogManagerImplTest, BigDataChunkShouldBeFlushedWithoutTimer) { })); log_file->write("a"); - - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 1) { - file_->write_event_.wait(file_->write_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 1)); // First write happens without waiting on thread_flush_. Now make a big string and it should be // flushed even when timer is not enabled @@ -476,13 +414,8 @@ TEST_F(AccessLogManagerImplTest, BigDataChunkShouldBeFlushedWithoutTimer) { std::string big_string(1024 * 64 + 1, 'b'); log_file->write(big_string); + EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 2)); - { - Thread::LockGuard lock(file_->write_mutex_); - while (file_->num_writes_ != 2) { - file_->write_event_.wait(file_->write_mutex_); - } - } EXPECT_CALL(*file_, close_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); } @@ -551,19 +484,8 @@ TEST_F(AccessLogManagerImplTest, ReopenAllFiles) { log->write("this is to force reopen"); log2->write("this is to force reopen"); - { - Thread::LockGuard lock(file_->open_mutex_); - while (file_->num_opens_ != 2) { - file_->open_event_.wait(file_->open_mutex_); - } - } - - { - Thread::LockGuard lock(file2->open_mutex_); - while (file2->num_opens_ != 2) { - file2->open_event_.wait(file2->open_mutex_); - } - } + EXPECT_TRUE(file_->waitForEventCount(file_->num_opens_, 2)); + EXPECT_TRUE(file2->waitForEventCount(file2->num_opens_, 2)); EXPECT_CALL(*file_, close_()) .InSequence(sq) diff --git a/test/mocks/filesystem/mocks.cc b/test/mocks/filesystem/mocks.cc index f6f94c91eef5..bd396f2aae2d 100644 --- a/test/mocks/filesystem/mocks.cc +++ b/test/mocks/filesystem/mocks.cc @@ -6,55 +6,51 @@ namespace Envoy { namespace Filesystem { -MockFile::MockFile() : num_opens_(0), num_writes_(0), is_open_(false) {} +MockFile::MockFile() = default; MockFile::~MockFile() = default; Api::IoCallBoolResult MockFile::open(FlagSet flag) { - Thread::LockGuard lock(open_mutex_); + absl::MutexLock lock(&mutex_); Api::IoCallBoolResult result = open_(flag); is_open_ = result.return_value_; num_opens_++; - open_event_.notifyOne(); return result; } Api::IoCallSizeResult MockFile::write(absl::string_view buffer) { - Thread::LockGuard lock(write_mutex_); + absl::MutexLock lock(&mutex_); if (!is_open_) { return {-1, Api::IoErrorPtr(nullptr, [](Api::IoError*) { PANIC("reached unexpected code"); })}; } Api::IoCallSizeResult result = write_(buffer); num_writes_++; - write_event_.notifyOne(); return result; } Api::IoCallSizeResult MockFile::pread(void* buf, uint64_t count, uint64_t offset) { - Thread::LockGuard lock(pread_mutex_); + absl::MutexLock lock(&mutex_); if (!is_open_) { return {-1, Api::IoErrorPtr(nullptr, [](Api::IoError*) { PANIC("reached unexpected code"); })}; } Api::IoCallSizeResult result = pread_(buf, count, offset); num_preads_++; - pread_event_.notifyOne(); return result; } Api::IoCallSizeResult MockFile::pwrite(const void* buf, uint64_t count, uint64_t offset) { - Thread::LockGuard lock(pwrite_mutex_); + absl::MutexLock lock(&mutex_); if (!is_open_) { return {-1, Api::IoErrorPtr(nullptr, [](Api::IoError*) { PANIC("reached unexpected code"); })}; } Api::IoCallSizeResult result = pwrite_(buf, count, offset); num_pwrites_++; - pwrite_event_.notifyOne(); return result; } diff --git a/test/mocks/filesystem/mocks.h b/test/mocks/filesystem/mocks.h index 2ae9642ed474..7f1d61b578da 100644 --- a/test/mocks/filesystem/mocks.h +++ b/test/mocks/filesystem/mocks.h @@ -36,21 +36,21 @@ class MockFile : public File { MOCK_METHOD(Api::IoCallSizeResult, pread_, (void* buf, uint64_t count, uint64_t offset)); MOCK_METHOD(Api::IoCallSizeResult, pwrite_, (const void* buf, uint64_t count, uint64_t offset)); - size_t num_opens_; - size_t num_writes_; - size_t num_preads_; - size_t num_pwrites_; - Thread::MutexBasicLockable open_mutex_; - Thread::MutexBasicLockable write_mutex_; - Thread::MutexBasicLockable pread_mutex_; - Thread::MutexBasicLockable pwrite_mutex_; - Thread::CondVar open_event_; - Thread::CondVar write_event_; - Thread::CondVar pread_event_; - Thread::CondVar pwrite_event_; + absl::Mutex mutex_; + uint64_t num_opens_{0}; + uint64_t num_writes_{0}; + uint64_t num_preads_{0}; + uint64_t num_pwrites_{0}; + + bool waitForEventCount(uint64_t& event_counter, uint64_t expected) { + auto func = [&]() { return event_counter == expected; }; + bool result = mutex_.LockWhenWithTimeout(absl::Condition(&func), absl::Seconds(15)); + mutex_.Unlock(); + return result; + } private: - bool is_open_; + bool is_open_{false}; }; class MockInstance : public Instance { From 58335ef81559035dab055c23215e313f6ed784f5 Mon Sep 17 00:00:00 2001 From: Raven Black Date: Thu, 18 May 2023 09:17:55 -0700 Subject: [PATCH 287/740] Fix typo - adjustNewConnectionIdForRouting (#27456) Signed-off-by: Raven Black --- source/common/quic/envoy_quic_utils.cc | 2 +- source/common/quic/envoy_quic_utils.h | 2 +- .../envoy_deterministic_connection_id_generator.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/common/quic/envoy_quic_utils.cc b/source/common/quic/envoy_quic_utils.cc index 4a1325060fb1..1b0047a6537b 100644 --- a/source/common/quic/envoy_quic_utils.cc +++ b/source/common/quic/envoy_quic_utils.cc @@ -278,7 +278,7 @@ void configQuicInitialFlowControlWindow(const envoy::config::core::v3::QuicProto static_cast(session_flow_control_window_to_send))); } -void adjustNewConnectionIdForRoutine(quic::QuicConnectionId& new_connection_id, +void adjustNewConnectionIdForRouting(quic::QuicConnectionId& new_connection_id, const quic::QuicConnectionId& old_connection_id) { char* new_connection_id_data = new_connection_id.mutable_data(); const char* old_connection_id_ptr = old_connection_id.data(); diff --git a/source/common/quic/envoy_quic_utils.h b/source/common/quic/envoy_quic_utils.h index 966d7a6a1e93..5e3333b00bc2 100644 --- a/source/common/quic/envoy_quic_utils.h +++ b/source/common/quic/envoy_quic_utils.h @@ -190,7 +190,7 @@ void configQuicInitialFlowControlWindow(const envoy::config::core::v3::QuicProto // Modify new_connection_id according to given old_connection_id to make sure packets with the new // one can be routed to the same listener. -void adjustNewConnectionIdForRoutine(quic::QuicConnectionId& new_connection_id, +void adjustNewConnectionIdForRouting(quic::QuicConnectionId& new_connection_id, const quic::QuicConnectionId& old_connection_id); } // namespace Quic diff --git a/source/extensions/quic/connection_id_generator/envoy_deterministic_connection_id_generator.cc b/source/extensions/quic/connection_id_generator/envoy_deterministic_connection_id_generator.cc index 538979cfb705..92bb75e6756a 100644 --- a/source/extensions/quic/connection_id_generator/envoy_deterministic_connection_id_generator.cc +++ b/source/extensions/quic/connection_id_generator/envoy_deterministic_connection_id_generator.cc @@ -13,7 +13,7 @@ EnvoyDeterministicConnectionIdGenerator::GenerateNextConnectionId( const quic::QuicConnectionId& original) { auto new_cid = DeterministicConnectionIdGenerator::GenerateNextConnectionId(original); if (new_cid.has_value()) { - adjustNewConnectionIdForRoutine(new_cid.value(), original); + adjustNewConnectionIdForRouting(new_cid.value(), original); } return (new_cid.has_value() && new_cid.value() == original) ? absl::nullopt : new_cid; } @@ -23,7 +23,7 @@ EnvoyDeterministicConnectionIdGenerator::MaybeReplaceConnectionId( const quic::QuicConnectionId& original, const quic::ParsedQuicVersion& version) { auto new_cid = DeterministicConnectionIdGenerator::MaybeReplaceConnectionId(original, version); if (new_cid.has_value()) { - adjustNewConnectionIdForRoutine(new_cid.value(), original); + adjustNewConnectionIdForRouting(new_cid.value(), original); } return (new_cid.has_value() && new_cid.value() == original) ? absl::nullopt : new_cid; } From 3696ca57fdd735e0c8a8dc5a08926e14c328206d Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 18 May 2023 17:20:41 +0100 Subject: [PATCH 288/740] ci: Improve `/retest` (#27458) Signed-off-by: Ryan Northey --- .github/workflows/commands.yml | 10 ++++++++-- ci/repokitteh/modules/azure_pipelines.star | 21 +++++++++------------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml index 6d50ad40a10e..590f0d67b489 100644 --- a/.github/workflows/commands.yml +++ b/.github/workflows/commands.yml @@ -1,8 +1,12 @@ name: commands + on: issue_comment: types: [created] +permissions: + contents: read + jobs: retest: if: | @@ -13,8 +17,10 @@ jobs: && github.actor != 'dependabot[bot]' }} name: Retest - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 + permissions: + pull-requests: write steps: - - uses: jpsim/retest@c158dec0a7f67cb85f8367468dc8a9a75308bb7f + - uses: envoyproxy/toolshed/gh-actions/retest@56d5781416445ed530e075b71546dedee94cf054 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci/repokitteh/modules/azure_pipelines.star b/ci/repokitteh/modules/azure_pipelines.star index 9c6cbe9ecc9f..57d6a78f0056 100644 --- a/ci/repokitteh/modules/azure_pipelines.star +++ b/ci/repokitteh/modules/azure_pipelines.star @@ -31,10 +31,12 @@ def _get_azp_link(check_id): return "https://dev.azure.com/{organization}/{project}/_build/results?buildId={buildId}&view=results".format(organization = _azp_organization, project = project, buildId = build_id) def _retry(config, comment_id, command): - msgs = "Retrying Azure Pipelines:\n" + if len(command.parts) > 1 and command.parts[1] == "mobile": + return check_ids, checks = _get_azp_checks() retried_checks = [] + reaction = "confused" for check_id in check_ids: subchecks = [c for c in checks if c["external_id"] == check_id] if len(subchecks) == 0: @@ -50,20 +52,15 @@ def _retry(config, comment_id, command): if check["status"] == "in_progress": has_running = True - if not has_failure: - msgs += "Check {} didn't fail.\n".format(name_with_link) - else: - if has_running: - msgs += "Check {} isn't fully completed, but will still attempt retrying.\n".format(name_with_link) + if has_failure: _, build_id, project = check_id.split("|") _retry_azp(project, build_id, config["token"]) retried_checks.append(name_with_link) - if len(retried_checks) == 0: - react(comment_id, msgs) - else: - react(comment_id, None) - msgs += "Retried failed jobs in: {}".format(", ".join(retried_checks)) - github.issue_create_comment(msgs) + if len(retried_checks) != 0: + reaction = "+1" + + github.issue_create_comment_reaction(comment_id, reaction) + handlers.command(name = "retry-azp", func = _retry) From 2b1a5bc93910cf763f2e70733277e42a02b9f951 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 18 May 2023 17:31:51 +0100 Subject: [PATCH 289/740] deps: Bump `io_hyperscan` -> 5.4.2 (#27482) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 6c72be0aaf91..f5625d19c428 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -459,16 +459,16 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Hyperscan", project_desc = "High-performance regular expression matching library", project_url = "https://hyperscan.io", - version = "5c724f73d1aa5add73c06c97c59930899c713d47", - sha256 = "c0c9074a9f19db2fe70a78d4fcf39709072068a652b474f30f8f3931aa86ef3a", + version = "5.4.2", + sha256 = "32b0f24b3113bbc46b6bfaa05cf7cf45840b6b59333d078cc1f624e4c40b2b99", strip_prefix = "hyperscan-{version}", - urls = ["https://github.com/intel/hyperscan/archive/{version}.tar.gz"], + urls = ["https://github.com/intel/hyperscan/archive/v{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = [ "envoy.matching.input_matchers.hyperscan", "envoy.regex_engines.hyperscan", ], - release_date = "2023-03-23", + release_date = "2023-04-19", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/intel/hyperscan/blob/{version}/LICENSE", From 071ff2ce42dce7704fde5092d4e1aafd4c9e391c Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 18 May 2023 20:27:38 +0100 Subject: [PATCH 290/740] deps: Bump `com_envoyproxy_protoc_gen_validate` -> 1.0.1 (#27480) Signed-off-by: Ryan Northey --- api/bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 156de3232288..f6fd434d84ab 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -17,11 +17,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_desc = "protoc plugin to generate polyglot message validators", project_url = "https://github.com/bufbuild/protoc-gen-validate", use_category = ["api"], - sha256 = "f2c0f71591052f50d25745b2bcef2c3968e70474e70e84a19443d387bc40c31d", - version = "0.10.1", + sha256 = "f1ec013cfdfffa7a17d75b55d41265dad47d24e0e9d86c02311562e15be52da9", + version = "1.0.1", urls = ["https://github.com/bufbuild/protoc-gen-validate/archive/refs/tags/v{version}.zip"], strip_prefix = "protoc-gen-validate-{version}", - release_date = "2023-03-20", + release_date = "2023-05-09", implied_untracked_deps = [ "com_github_iancoleman_strcase", "com_github_lyft_protoc_gen_star", From 80f30f895ff6ab7b0ac412fa143f4a4474388b70 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 18 May 2023 20:54:36 +0100 Subject: [PATCH 291/740] ci: Maybe fix `/retest` permissions (#27489) Signed-off-by: Ryan Northey --- .github/workflows/commands.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml index 590f0d67b489..35da322861f1 100644 --- a/.github/workflows/commands.yml +++ b/.github/workflows/commands.yml @@ -20,6 +20,7 @@ jobs: runs-on: ubuntu-22.04 permissions: pull-requests: write + actions: write steps: - uses: envoyproxy/toolshed/gh-actions/retest@56d5781416445ed530e075b71546dedee94cf054 with: From 79c1ae5dc797b8aa96c0e8ad8b4dcd131948938d Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 18 May 2023 20:55:13 +0100 Subject: [PATCH 292/740] deps: Bump `com_github_nghttp2_nghttp2` -> 1.53.0 (#27479) Fix #27346 Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index f5625d19c428..e87c24f7f4cc 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -445,12 +445,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Nghttp2", project_desc = "Implementation of HTTP/2 and its header compression algorithm HPACK in C", project_url = "https://nghttp2.org", - version = "1.52.0", - sha256 = "9877caa62bd72dde1331da38ce039dadb049817a01c3bdee809da15b754771b8", + version = "1.53.0", + sha256 = "f5f3f18b377d1e7658e4655a32d9a7ce6fef39fa13f600fe920f5f77c60fc34b", strip_prefix = "nghttp2-{version}", urls = ["https://github.com/nghttp2/nghttp2/releases/download/v{version}/nghttp2-{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2023-02-13", + release_date = "2023-05-10", cpe = "cpe:2.3:a:nghttp2:nghttp2:*", license = "MIT", license_url = "https://github.com/nghttp2/nghttp2/blob/v{version}/LICENSE", From 9e15d378f1d43aa3df44f89e029ab40e6fd87f77 Mon Sep 17 00:00:00 2001 From: Robert Femmer <114982872+robertfemmer@users.noreply.github.com> Date: Thu, 18 May 2023 22:15:46 +0200 Subject: [PATCH 293/740] conn_manager_impl_test_base: Use global test symbol table (#27454) Currently, HttpConnectionManagerImplMixin uses a self-allocated symbol table. When used together with other mock classes that use the globally shared symbol table, debug assertions are tripped that require symbol table addresses to be equal. This change would force HttpConnectionManagerImplMixin to use the globally shared test symbol table. Signed-off-by: Robert Femmer --- test/common/http/conn_manager_impl_test_base.cc | 3 ++- test/common/http/conn_manager_impl_test_base.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/common/http/conn_manager_impl_test_base.cc b/test/common/http/conn_manager_impl_test_base.cc index 5d203bc53db0..1c5053246e44 100644 --- a/test/common/http/conn_manager_impl_test_base.cc +++ b/test/common/http/conn_manager_impl_test_base.cc @@ -13,7 +13,8 @@ namespace Envoy { namespace Http { HttpConnectionManagerImplMixin::HttpConnectionManagerImplMixin() - : http_context_(fake_stats_.symbolTable()), access_log_path_("dummy_path"), + : fake_stats_(*symbol_table_), http_context_(fake_stats_.symbolTable()), + access_log_path_("dummy_path"), access_logs_{AccessLog::InstanceSharedPtr{new Extensions::AccessLoggers::File::FileAccessLog( Filesystem::FilePathAndType{Filesystem::DestinationType::File, access_log_path_}, {}, Formatter::SubstitutionFormatUtils::defaultSubstitutionFormatter(), log_manager_)}}, diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index eddca70cb681..99fdc9344118 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -207,6 +207,7 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { std::shared_ptr route_config_{new NiceMock()}; NiceMock scoped_route_config_provider_; Router::MockScopeKeyBuilder scope_key_builder_; + Stats::TestUtil::TestSymbolTable symbol_table_; Stats::IsolatedStoreImpl fake_stats_; Http::ContextImpl http_context_; NiceMock runtime_; From 60643c6e3463d0593b1b19c9778edb8b323d8dca Mon Sep 17 00:00:00 2001 From: Christoph Pakulski Date: Thu, 18 May 2023 18:20:09 -0400 Subject: [PATCH 294/740] nit: update changelog after PR27296 (#27494) Signed-off-by: Christoph Pakulski --- changelogs/current.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index d9e8e8ea1660..a49260da1d22 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -93,6 +93,9 @@ bug_fixes: Do not display GRPC_STATUS_NUMBER for non gRPC requests. This behavioral change can be temporarily reverted by setting runtime guard ``envoy.reloadable_features.validate_grpc_header_before_log_grpc_status`` to false. +- area: boringssl + change: | + Fixed the crash that occurs when contrib is compiled with ``boringssl=fips`` defined. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` From 3f9f8f76b575ecf962d21f59d7e29fdec2eab610 Mon Sep 17 00:00:00 2001 From: Raven Black Date: Thu, 18 May 2023 16:35:31 -0700 Subject: [PATCH 295/740] [file_system_http_cache] fix broken design doc link (#27498) Signed-off-by: Raven Black --- .../file_system_http_cache/v3/file_system_http_cache.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/http/cache/file_system_http_cache/v3/file_system_http_cache.proto b/api/envoy/extensions/http/cache/file_system_http_cache/v3/file_system_http_cache.proto index 91844d3ae880..ebedf30fa26c 100644 --- a/api/envoy/extensions/http/cache/file_system_http_cache/v3/file_system_http_cache.proto +++ b/api/envoy/extensions/http/cache/file_system_http_cache/v3/file_system_http_cache.proto @@ -26,7 +26,7 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // // By default this cache uses a least-recently-used eviction strategy. // -// For implementation details, see `DESIGN.md `_. +// For implementation details, see `DESIGN.md `_. // [#next-free-field: 11] message FileSystemHttpCacheConfig { // Configuration of a manager for how the file system is used asynchronously. From f979b22121d5b2c6f72bf1349680560b29815365 Mon Sep 17 00:00:00 2001 From: Xavid Date: Thu, 18 May 2023 19:36:30 -0400 Subject: [PATCH 296/740] server: support using the ApiListener from a worker thread (#27258) Before this change, the SyntheticReadCallbacks used by the ApiListener hard-coded the main thread dispatcher, resulting in dispatcher assertion failures if the ApiListener was used from a worker thread. This change allows it to use the appropriate dispatcher whether used from the main thread or a worker thread. Signed-off-by: Xavid Pretzer --- envoy/server/api_listener.h | 17 ++-- mobile/library/common/engine.cc | 10 +- mobile/library/common/http/client.cc | 2 +- mobile/library/common/http/client.h | 10 +- mobile/test/common/http/client_test.cc | 12 ++- .../network/http_connection_manager/config.cc | 9 +- .../network/http_connection_manager/config.h | 6 +- source/server/api_listener_impl.cc | 39 ++++---- source/server/api_listener_impl.h | 46 +++++++--- source/server/server.cc | 7 -- .../api_listener_integration_test.cc | 92 ++++++++++++++++--- test/server/api_listener_test.cc | 21 +++-- 12 files changed, 180 insertions(+), 91 deletions(-) diff --git a/envoy/server/api_listener.h b/envoy/server/api_listener.h index b14fb7068abd..35e7f61df605 100644 --- a/envoy/server/api_listener.h +++ b/envoy/server/api_listener.h @@ -21,22 +21,21 @@ class ApiListener { */ virtual absl::string_view name() const PURE; - /** - * Shutdown the ApiListener. This is an interrupt, not a drain. In other words, calling this - * function results in termination of all active streams vs. draining where no new streams are - * allowed, but already existing streams are allowed to finish. - */ - virtual void shutdown() PURE; - /** * @return the Type of the ApiListener. */ virtual Type type() const PURE; /** - * @return valid ref IFF type() == Type::HttpApiListener, otherwise nullopt. + * Create an Http::ApiListener capable of starting synthetic HTTP streams. The returned listener + * must only be deleted in the dispatcher's thread. + * + * While Envoy Mobile only uses this from the main thread, taking a dispatcher as a parameter + * allows other users to use this from worker threads as well. + * + * @return valid pointer IFF type() == Type::HttpApiListener, otherwise nullptr. */ - virtual Http::ApiListenerOptRef http() PURE; + virtual Http::ApiListenerPtr createHttpApiListener(Event::Dispatcher& dispatcher) PURE; }; using ApiListenerPtr = std::unique_ptr; diff --git a/mobile/library/common/engine.cc b/mobile/library/common/engine.cc index d54c6f3b5658..a21c37c80ac5 100644 --- a/mobile/library/common/engine.cc +++ b/mobile/library/common/engine.cc @@ -109,9 +109,10 @@ envoy_status_t Engine::main(std::unique_ptr&& options) { // on-the-fly without risking contention on system with lots of threads. // It also comes with ease of programming. stat_name_set_ = client_scope_->symbolTable().makeSet("pulse"); - auto api_listener = server_->listenerManager().apiListener()->get().http(); - ASSERT(api_listener.has_value()); - http_client_ = std::make_unique(api_listener.value(), *dispatcher_, + auto api_listener = server_->listenerManager().apiListener()->get().createHttpApiListener( + server_->dispatcher()); + ASSERT(api_listener != nullptr); + http_client_ = std::make_unique(std::move(api_listener), *dispatcher_, server_->serverFactoryContext().scope(), server_->api().randomGenerator()); dispatcher_->drain(server_->dispatcher()); @@ -156,6 +157,9 @@ envoy_status_t Engine::terminate() { ASSERT(event_dispatcher_); ASSERT(dispatcher_); + // We must destroy the Http::ApiListener in the main thread. + dispatcher_->post([this]() { http_client_->shutdownApiListener(); }); + // Exit the event loop and finish up in Engine::run(...) if (std::this_thread::get_id() == main_thread_.get_id()) { // TODO(goaway): figure out some way to support this. diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index d2efc7d1e8a6..250e53d437db 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -476,7 +476,7 @@ void Client::startStream(envoy_stream_t new_stream_handle, envoy_http_callbacks // Note: streams created by Envoy Mobile are tagged as is_internally_created. This means that // the Http::ConnectionManager _will not_ sanitize headers when creating a stream. direct_stream->request_decoder_ = - &api_listener_.newStream(*direct_stream->callbacks_, true /* is_internally_created */); + &api_listener_->newStream(*direct_stream->callbacks_, true /* is_internally_created */); streams_.emplace(new_stream_handle, std::move(direct_stream)); ENVOY_LOG(debug, "[S{}] start stream", new_stream_handle); diff --git a/mobile/library/common/http/client.h b/mobile/library/common/http/client.h index 3fb622c3ee0d..5a23f929c7f4 100644 --- a/mobile/library/common/http/client.h +++ b/mobile/library/common/http/client.h @@ -51,9 +51,9 @@ struct HttpClientStats { */ class Client : public Logger::Loggable { public: - Client(ApiListener& api_listener, Event::ProvisionalDispatcher& dispatcher, Stats::Scope& scope, - Random::RandomGenerator& random) - : api_listener_(api_listener), dispatcher_(dispatcher), + Client(ApiListenerPtr&& api_listener, Event::ProvisionalDispatcher& dispatcher, + Stats::Scope& scope, Random::RandomGenerator& random) + : api_listener_(std::move(api_listener)), dispatcher_(dispatcher), stats_( HttpClientStats{ALL_HTTP_CLIENT_STATS(POOL_COUNTER_PREFIX(scope, "http.client."), POOL_HISTOGRAM_PREFIX(scope, "http.client."))}), @@ -129,6 +129,8 @@ class Client : public Logger::Loggable { CONSTRUCT_ON_FIRST_USE(std::string, "client_cancelled_stream"); } + void shutdownApiListener() { api_listener_.reset(); } + private: class DirectStream; friend class ClientTest; @@ -377,7 +379,7 @@ class Client : public Logger::Loggable { void removeStream(envoy_stream_t stream_handle); void setDestinationCluster(RequestHeaderMap& headers); - ApiListener& api_listener_; + ApiListenerPtr api_listener_; Event::ProvisionalDispatcher& dispatcher_; Event::SchedulableCallbackPtr scheduled_callback_; HttpClientStats stats_; diff --git a/mobile/test/common/http/client_test.cc b/mobile/test/common/http/client_test.cc index d64546c8985d..a31c294064c2 100644 --- a/mobile/test/common/http/client_test.cc +++ b/mobile/test/common/http/client_test.cc @@ -131,7 +131,7 @@ class ClientTest : public testing::TestWithParam { // Grab the response encoder in order to dispatch responses on the stream. // Return the request decoder to make sure calls are dispatched to the decoder via the // dispatcher API. - EXPECT_CALL(api_listener_, newStream(_, _)) + EXPECT_CALL(*api_listener_, newStream(_, _)) .WillOnce(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoder& { response_encoder_ = &encoder; return *request_decoder_; @@ -146,7 +146,8 @@ class ClientTest : public testing::TestWithParam { } } - MockApiListener api_listener_; + std::unique_ptr owned_api_listener_ = std::make_unique(); + MockApiListener* api_listener_ = owned_api_listener_.get(); std::unique_ptr> request_decoder_{ std::make_unique>()}; NiceMock stream_info_; @@ -157,7 +158,8 @@ class ClientTest : public testing::TestWithParam { NiceMock random_; Stats::IsolatedStoreImpl stats_store_; bool explicit_flow_control_{GetParam()}; - Client http_client_{api_listener_, dispatcher_, *stats_store_.rootScope(), random_}; + Client http_client_{std::move(owned_api_listener_), dispatcher_, *stats_store_.rootScope(), + random_}; envoy_stream_t stream_ = 1; protected: @@ -423,7 +425,7 @@ TEST_P(ClientTest, MultipleStreams) { // Grab the response encoder in order to dispatch responses on the stream. // Return the request decoder to make sure calls are dispatched to the decoder via the dispatcher // API. - EXPECT_CALL(api_listener_, newStream(_, _)) + EXPECT_CALL(*api_listener_, newStream(_, _)) .WillOnce(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoder& { response_encoder2 = &encoder; return request_decoder2; @@ -684,7 +686,7 @@ TEST_P(ClientTest, NullAccessors) { // Grab the response encoder in order to dispatch responses on the stream. // Return the request decoder to make sure calls are dispatched to the decoder via the dispatcher // API. - EXPECT_CALL(api_listener_, newStream(_, _)) + EXPECT_CALL(*api_listener_, newStream(_, _)) .WillOnce(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoder& { response_encoder_ = &encoder; return *request_decoder_; diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 3fabdb579b3a..7594c5f02247 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -766,12 +766,11 @@ HttpConnectionManagerConfig::getHeaderValidatorStats([[maybe_unused]] Http::Prot } #endif -std::function +std::function HttpConnectionManagerFactory::createHttpConnectionManagerFactoryFromProto( const envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& proto_config, - Server::Configuration::FactoryContext& context, Network::ReadFilterCallbacks& read_callbacks, - bool clear_hop_by_hop_headers) { + Server::Configuration::FactoryContext& context, bool clear_hop_by_hop_headers) { Utility::Singletons singletons = Utility::createSingletons(context); auto filter_config = Utility::createConfig( @@ -783,8 +782,8 @@ HttpConnectionManagerFactory::createHttpConnectionManagerFactoryFromProto( // reference count. // Keep in mind the lambda capture list **doesn't** determine the destruction order, but it's fine // as these captured objects are also global singletons. - return [singletons, filter_config, &context, &read_callbacks, - clear_hop_by_hop_headers]() -> Http::ApiListenerPtr { + return [singletons, filter_config, &context, clear_hop_by_hop_headers]( + Network::ReadFilterCallbacks& read_callbacks) -> Http::ApiListenerPtr { auto conn_manager = std::make_unique( *filter_config, context.drainDecision(), context.api().randomGenerator(), context.httpContext(), context.runtime(), context.localInfo(), context.clusterManager(), diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index da8126fb2aea..09d76f31dcbd 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -366,11 +366,11 @@ class HttpConnectionManagerConfig : Logger::Loggable, */ class HttpConnectionManagerFactory { public: - static std::function createHttpConnectionManagerFactoryFromProto( + static std::function + createHttpConnectionManagerFactoryFromProto( const envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& proto_config, - Server::Configuration::FactoryContext& context, Network::ReadFilterCallbacks& read_callbacks, - bool clear_hop_by_hop_headers); + Server::Configuration::FactoryContext& context, bool clear_hop_by_hop_headers); }; /** diff --git a/source/server/api_listener_impl.cc b/source/server/api_listener_impl.cc index 93aa6e4a5ad7..0ea54f552f7a 100644 --- a/source/server/api_listener_impl.cc +++ b/source/server/api_listener_impl.cc @@ -1,6 +1,7 @@ #include "source/server/api_listener_impl.h" #include "envoy/config/listener/v3/listener.pb.h" +#include "envoy/http/api_listener.h" #include "envoy/stats/scope.h" #include "source/common/http/conn_manager_impl.h" @@ -21,8 +22,7 @@ ApiListenerImplBase::ApiListenerImplBase(const envoy::config::listener::v3::List address_(Network::Address::resolveProtoAddress(config.address())), global_scope_(server.stats().createScope("")), listener_scope_(server.stats().createScope(fmt::format("listener.api.{}.", name_))), - factory_context_(server, config_, *this, *global_scope_, *listener_scope_, isQuic(config)), - read_callbacks_(SyntheticReadCallbacks(*this)) {} + factory_context_(server, config_, *this, *global_scope_, *listener_scope_, isQuic(config)) {} void ApiListenerImplBase::SyntheticReadCallbacks::SyntheticConnection::raiseConnectionEvent( Network::ConnectionEvent event) { @@ -31,6 +31,19 @@ void ApiListenerImplBase::SyntheticReadCallbacks::SyntheticConnection::raiseConn } } +HttpApiListener::ApiListenerWrapper::~ApiListenerWrapper() { + // The Http::ConnectionManagerImpl is a callback target for the read_callback_.connection_. By + // raising connection closure, Http::ConnectionManagerImpl::onEvent is fired. In that case the + // Http::ConnectionManagerImpl will reset any ActiveStreams it has. + read_callbacks_.connection_.raiseConnectionEvent(Network::ConnectionEvent::RemoteClose); +} + +Http::RequestDecoder& +HttpApiListener::ApiListenerWrapper::newStream(Http::ResponseEncoder& response_encoder, + bool is_internally_created) { + return http_connection_manager_->newStream(response_encoder, is_internally_created); +} + HttpApiListener::HttpApiListener(const envoy::config::listener::v3::Listener& config, Server::Instance& server, const std::string& name) : ApiListenerImplBase(config, server, name) { @@ -46,30 +59,20 @@ HttpApiListener::HttpApiListener(const envoy::config::listener::v3::Listener& co http_connection_manager_factory_ = Envoy::Extensions::NetworkFilters::HttpConnectionManager:: HttpConnectionManagerFactory::createHttpConnectionManagerFactoryFromProto( - typed_config.config(), factory_context_, read_callbacks_, false); + typed_config.config(), factory_context_, false); } else { auto typed_config = MessageUtil::anyConvertAndValidate< envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager>( config.api_listener().api_listener(), factory_context_.messageValidationVisitor()); - http_connection_manager_factory_ = Envoy::Extensions::NetworkFilters::HttpConnectionManager:: - HttpConnectionManagerFactory::createHttpConnectionManagerFactoryFromProto( - typed_config, factory_context_, read_callbacks_, true); - } -} - -Http::ApiListenerOptRef HttpApiListener::http() { - if (!http_connection_manager_) { - http_connection_manager_ = http_connection_manager_factory_(); + http_connection_manager_factory_ = + Envoy::Extensions::NetworkFilters::HttpConnectionManager::HttpConnectionManagerFactory:: + createHttpConnectionManagerFactoryFromProto(typed_config, factory_context_, true); } - return Http::ApiListenerOptRef(std::ref(*http_connection_manager_)); } -void HttpApiListener::shutdown() { - // The Http::ConnectionManagerImpl is a callback target for the read_callback_.connection_. By - // raising connection closure, Http::ConnectionManagerImpl::onEvent is fired. In that case the - // Http::ConnectionManagerImpl will reset any ActiveStreams it has. - read_callbacks_.connection_.raiseConnectionEvent(Network::ConnectionEvent::RemoteClose); +Http::ApiListenerPtr HttpApiListener::createHttpApiListener(Event::Dispatcher& dispatcher) { + return std::make_unique(*this, dispatcher); } } // namespace Server diff --git a/source/server/api_listener_impl.h b/source/server/api_listener_impl.h index 2e260666a8f1..1f18ffe1efac 100644 --- a/source/server/api_listener_impl.h +++ b/source/server/api_listener_impl.h @@ -5,6 +5,7 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/listener/v3/listener.pb.h" +#include "envoy/event/dispatcher.h" #include "envoy/network/connection.h" #include "envoy/network/filter.h" #include "envoy/network/socket.h" @@ -57,8 +58,8 @@ class ApiListenerImplBase : public ApiListener, // need this and the SyntheticConnection stub anymore. class SyntheticReadCallbacks : public Network::ReadFilterCallbacks { public: - SyntheticReadCallbacks(ApiListenerImplBase& parent) - : parent_(parent), connection_(SyntheticConnection(*this)) {} + SyntheticReadCallbacks(ApiListenerImplBase& parent, Event::Dispatcher& dispatcher) + : parent_(parent), connection_(SyntheticConnection(*this, dispatcher)) {} // Network::ReadFilterCallbacks void continueReading() override { IS_ENVOY_BUG("Unexpected call to continueReading"); } @@ -80,8 +81,8 @@ class ApiListenerImplBase : public ApiListener, // Network::ReadFilterCallbacks. class SyntheticConnection : public Network::Connection { public: - SyntheticConnection(SyntheticReadCallbacks& parent) - : parent_(parent), + SyntheticConnection(SyntheticReadCallbacks& parent, Event::Dispatcher& dispatcher) + : parent_(parent), dispatcher_(dispatcher), connection_info_provider_(std::make_shared( parent.parent_.address_, parent.parent_.address_)), stream_info_(parent_.parent_.factory_context_.timeSource(), connection_info_provider_), @@ -121,9 +122,7 @@ class ApiListenerImplBase : public ApiListener, } void close(Network::ConnectionCloseType) override {} void close(Network::ConnectionCloseType, absl::string_view) override {} - Event::Dispatcher& dispatcher() override { - return parent_.parent_.factory_context_.mainThreadDispatcher(); - } + Event::Dispatcher& dispatcher() override { return dispatcher_; } uint64_t id() const override { return 12345; } void hashKey(std::vector&) const override {} std::string nextProtocol() const override { return EMPTY_STRING; } @@ -174,6 +173,7 @@ class ApiListenerImplBase : public ApiListener, void dumpState(std::ostream& os, int) const override { os << "SyntheticConnection"; } SyntheticReadCallbacks& parent_; + Event::Dispatcher& dispatcher_; Network::ConnectionInfoSetterSharedPtr connection_info_provider_; StreamInfo::StreamInfoImpl stream_info_; Network::ConnectionSocket::OptionsSharedPtr options_; @@ -190,7 +190,6 @@ class ApiListenerImplBase : public ApiListener, Stats::ScopeSharedPtr global_scope_; Stats::ScopeSharedPtr listener_scope_; FactoryContextImpl factory_context_; - SyntheticReadCallbacks read_callbacks_; }; /** @@ -199,22 +198,39 @@ class ApiListenerImplBase : public ApiListener, */ class HttpApiListener : public ApiListenerImplBase { public: + // Class to wrap an Http::ApiListener and the associated SyntheticReadCallbacks to ensure that + // both objects have the same lifetime. + // + // Public for testing. + class ApiListenerWrapper : public Http::ApiListener { + public: + ApiListenerWrapper(HttpApiListener& parent, Event::Dispatcher& dispatcher) + : read_callbacks_(parent, dispatcher), + http_connection_manager_(parent.http_connection_manager_factory_(read_callbacks_)) {} + ~ApiListenerWrapper() override; + + Http::RequestDecoder& newStream(Http::ResponseEncoder& response_encoder, + bool is_internally_created = false) override; + + SyntheticReadCallbacks& readCallbacks() { return read_callbacks_; } + + private: + SyntheticReadCallbacks read_callbacks_; + Http::ApiListenerPtr http_connection_manager_; + }; + HttpApiListener(const envoy::config::listener::v3::Listener& config, Server::Instance& server, const std::string& name); // ApiListener ApiListener::Type type() const override { return ApiListener::Type::HttpApiListener; } - Http::ApiListenerOptRef http() override; - void shutdown() override; - - Network::ReadFilterCallbacks& readCallbacksForTest() { return read_callbacks_; } + Http::ApiListenerPtr createHttpApiListener(Event::Dispatcher& dispatcher) override; private: // Need to store the factory due to the shared_ptrs that need to be kept alive: date provider, // route config manager, scoped route config manager. - std::function http_connection_manager_factory_; - // Http::ServerConnectionCallbacks is the API surface that this class provides via its handle(). - Http::ApiListenerPtr http_connection_manager_; + std::function + http_connection_manager_factory_; }; } // namespace Server diff --git a/source/server/server.cc b/source/server/server.cc index 42908d7af321..f5c2b062ea8b 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -970,13 +970,6 @@ void InstanceImpl::terminate() { // Shutdown all the workers now that the main dispatch loop is done. if (listener_manager_ != nullptr) { - // Also shutdown the listener manager's ApiListener, if there is one, which runs on the main - // thread. This needs to happen ahead of calling thread_local_.shutdown() below to prevent any - // objects in the ApiListener destructor to reference any objects in thread local storage. - if (listener_manager_->apiListener().has_value()) { - listener_manager_->apiListener()->get().shutdown(); - } - listener_manager_->stopWorkers(); } diff --git a/test/integration/api_listener_integration_test.cc b/test/integration/api_listener_integration_test.cc index e4a206a41a15..e4aa8c4bbbcd 100644 --- a/test/integration/api_listener_integration_test.cc +++ b/test/integration/api_listener_integration_test.cc @@ -86,15 +86,17 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ApiListenerIntegrationTest, TEST_P(ApiListenerIntegrationTest, Basic) { BaseIntegrationTest::initialize(); absl::Notification done; - test_server_->server().dispatcher().post([this, &done]() -> void { + Http::ApiListenerPtr http_api_listener; + test_server_->server().dispatcher().post([this, &done, &http_api_listener]() -> void { ASSERT_TRUE(test_server_->server().listenerManager().apiListener().has_value()); ASSERT_EQ("api_listener", test_server_->server().listenerManager().apiListener()->get().name()); - ASSERT_TRUE(test_server_->server().listenerManager().apiListener()->get().http().has_value()); - auto& http_api_listener = - test_server_->server().listenerManager().apiListener()->get().http()->get(); + http_api_listener = + test_server_->server().listenerManager().apiListener()->get().createHttpApiListener( + test_server_->server().dispatcher()); + ASSERT_TRUE(http_api_listener != nullptr); ON_CALL(stream_encoder_, getStream()).WillByDefault(ReturnRef(stream_encoder_.stream_)); - auto& stream_decoder = http_api_listener.newStream(stream_encoder_); + auto& stream_decoder = http_api_listener->newStream(stream_encoder_); // The AutonomousUpstream responds with 200 OK and a body of 10 bytes. // In the http1 codec the end stream is encoded with encodeData and 0 bytes. @@ -110,6 +112,14 @@ TEST_P(ApiListenerIntegrationTest, Basic) { true); }); ASSERT_TRUE(done.WaitForNotificationWithTimeout(absl::Seconds(1))); + + absl::Notification cleanup_done; + test_server_->server().dispatcher().post([&cleanup_done, &http_api_listener]() { + // Must be deleted in same thread. + http_api_listener.reset(); + cleanup_done.Notify(); + }); + ASSERT_TRUE(cleanup_done.WaitForNotificationWithTimeout(absl::Seconds(1))); } TEST_P(ApiListenerIntegrationTest, DestroyWithActiveStreams) { @@ -120,12 +130,13 @@ TEST_P(ApiListenerIntegrationTest, DestroyWithActiveStreams) { test_server_->server().dispatcher().post([this, &done]() -> void { ASSERT_TRUE(test_server_->server().listenerManager().apiListener().has_value()); ASSERT_EQ("api_listener", test_server_->server().listenerManager().apiListener()->get().name()); - ASSERT_TRUE(test_server_->server().listenerManager().apiListener()->get().http().has_value()); - auto& http_api_listener = - test_server_->server().listenerManager().apiListener()->get().http()->get(); + auto http_api_listener = + test_server_->server().listenerManager().apiListener()->get().createHttpApiListener( + test_server_->server().dispatcher()); + ASSERT_TRUE(http_api_listener != nullptr); ON_CALL(stream_encoder_, getStream()).WillByDefault(ReturnRef(stream_encoder_.stream_)); - auto& stream_decoder = http_api_listener.newStream(stream_encoder_); + auto& stream_decoder = http_api_listener->newStream(stream_encoder_); // Send a headers-only request stream_decoder.decodeHeaders( @@ -134,10 +145,69 @@ TEST_P(ApiListenerIntegrationTest, DestroyWithActiveStreams) { false); done.Notify(); + // http_api_listener is destroyed here with an active stream; thus this test verifies that + // there is no crash when this happens. + http_api_listener.reset(); }); ASSERT_TRUE(done.WaitForNotificationWithTimeout(absl::Seconds(1))); - // The server should shutdown the ApiListener at the right time during server termination such - // that no crashes occur if termination happens when the ApiListener still has ongoing streams. +} + +TEST_P(ApiListenerIntegrationTest, FromWorkerThread) { + BaseIntegrationTest::initialize(); + + absl::Mutex dispatchers_mutex; + std::vector dispatchers; + absl::Notification has_dispatcher; + ThreadLocal::TypedSlotPtr<> slot = + ThreadLocal::TypedSlot<>::makeUnique(test_server_->server().threadLocal()); + slot->set([&dispatchers_mutex, &dispatchers, &has_dispatcher]( + Event::Dispatcher& dispatcher) -> std::shared_ptr { + absl::MutexLock ml(&dispatchers_mutex); + // A string comparison on thread name seems to be the only way to + // distinguish worker threads from the main thread with the slots interface. + if (dispatcher.name() != "main_thread") { + dispatchers.push_back(&dispatcher); + has_dispatcher.Notify(); + } + return nullptr; + }); + ASSERT_TRUE(has_dispatcher.WaitForNotificationWithTimeout(absl::Seconds(1))); + + absl::Notification done; + Http::ApiListenerPtr http_api_listener; + dispatchers[0]->post([this, &done, &http_api_listener, &dispatchers]() -> void { + ASSERT_TRUE(test_server_->server().listenerManager().apiListener().has_value()); + ASSERT_EQ("api_listener", test_server_->server().listenerManager().apiListener()->get().name()); + http_api_listener = + test_server_->server().listenerManager().apiListener()->get().createHttpApiListener( + *dispatchers[0]); + ASSERT_TRUE(http_api_listener != nullptr); + + ON_CALL(stream_encoder_, getStream()).WillByDefault(ReturnRef(stream_encoder_.stream_)); + auto& stream_decoder = http_api_listener->newStream(stream_encoder_); + + // The AutonomousUpstream responds with 200 OK and a body of 10 bytes. + // In the http1 codec the end stream is encoded with encodeData and 0 bytes. + Http::TestResponseHeaderMapImpl expected_response_headers{{":status", "200"}}; + EXPECT_CALL(stream_encoder_, encodeHeaders(_, false)); + EXPECT_CALL(stream_encoder_, encodeData(_, false)); + EXPECT_CALL(stream_encoder_, encodeData(BufferStringEqual(""), true)).WillOnce(Notify(&done)); + + // Send a headers-only request + stream_decoder.decodeHeaders( + Http::RequestHeaderMapPtr(new Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, {":path", "/api"}, {":scheme", "http"}, {":authority", "host"}}), + true); + }); + ASSERT_TRUE(done.WaitForNotificationWithTimeout(absl::Seconds(1))); + + absl::Notification cleanup_done; + dispatchers[0]->post([&cleanup_done, &http_api_listener]() { + // Must be deleted in same thread. + http_api_listener.reset(); + cleanup_done.Notify(); + }); + ASSERT_TRUE(cleanup_done.WaitForNotificationWithTimeout(absl::Seconds(1))); } } // namespace diff --git a/test/server/api_listener_test.cc b/test/server/api_listener_test.cc index 46a2f74a9567..7183f80b8651 100644 --- a/test/server/api_listener_test.cc +++ b/test/server/api_listener_test.cc @@ -57,7 +57,7 @@ name: test_api_listener ASSERT_EQ("test_api_listener", http_api_listener.name()); ASSERT_EQ(ApiListener::Type::HttpApiListener, http_api_listener.type()); - ASSERT_TRUE(http_api_listener.http().has_value()); + ASSERT_NE(http_api_listener.createHttpApiListener(server_.dispatcher()), nullptr); } TEST_F(ApiListenerTest, MobileApiListener) { @@ -92,7 +92,7 @@ name: test_api_listener ASSERT_EQ("test_api_listener", http_api_listener.name()); ASSERT_EQ(ApiListener::Type::HttpApiListener, http_api_listener.type()); - ASSERT_TRUE(http_api_listener.http().has_value()); + ASSERT_NE(http_api_listener.createHttpApiListener(server_.dispatcher()), nullptr); } TEST_F(ApiListenerTest, HttpApiListenerThrowsWithBadConfig) { @@ -163,21 +163,22 @@ name: test_api_listener ASSERT_EQ("test_api_listener", http_api_listener.name()); ASSERT_EQ(ApiListener::Type::HttpApiListener, http_api_listener.type()); - ASSERT_TRUE(http_api_listener.http().has_value()); + auto api_listener = http_api_listener.createHttpApiListener(server_.dispatcher()); + ASSERT_NE(api_listener, nullptr); Network::MockConnectionCallbacks network_connection_callbacks; // TODO(junr03): potentially figure out a way of unit testing this behavior without exposing a // ForTest function. - http_api_listener.readCallbacksForTest().connection().addConnectionCallbacks( - network_connection_callbacks); - EXPECT_FALSE( - http_api_listener.readCallbacksForTest().connection().lastRoundTripTime().has_value()); - http_api_listener.readCallbacksForTest().connection().configureInitialCongestionWindow( - 100, std::chrono::microseconds(123)); + auto& connection = dynamic_cast(api_listener.get()) + ->readCallbacks() + .connection(); + connection.addConnectionCallbacks(network_connection_callbacks); + EXPECT_FALSE(connection.lastRoundTripTime().has_value()); + connection.configureInitialCongestionWindow(100, std::chrono::microseconds(123)); EXPECT_CALL(network_connection_callbacks, onEvent(Network::ConnectionEvent::RemoteClose)); // Shutting down the ApiListener should raise an event on all connection callback targets. - http_api_listener.shutdown(); + api_listener.reset(); } } // namespace Server From 143aafb6e98e2cec5781fe56e7c8905832097105 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 18 May 2023 20:06:11 -0700 Subject: [PATCH 297/740] deps: update abseil (#27463) Commit Message: update abseil to head Additional Description: there's a problem with cel-cpp using an internal macro from abseil. To decouple absl from cel-cpp update, we have to patch cel-cpp temporarily. Risk Level: low Testing: regression --- bazel/cel-cpp.patch | 22 +++++++++++++++++++ bazel/repository_locations.bzl | 6 ++--- .../common/thread_local/thread_local_impl.cc | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/bazel/cel-cpp.patch b/bazel/cel-cpp.patch index ac7d82733c0f..12e2f7342f7b 100644 --- a/bazel/cel-cpp.patch +++ b/bazel/cel-cpp.patch @@ -1,3 +1,25 @@ +diff --git a/base/memory_manager.cc b/base/memory_manager.cc +index 1b7b355..4c7810c 100644 +--- a/base/memory_manager.cc ++++ b/base/memory_manager.cc +@@ -234,7 +234,7 @@ class GlobalMemoryManager final : public MemoryManager { + void* Allocate(size_t size, size_t align) override { + static_cast(size); + static_cast(align); +- ABSL_INTERNAL_UNREACHABLE; ++ ABSL_UNREACHABLE(); + return nullptr; + } + +@@ -242,7 +242,7 @@ class GlobalMemoryManager final : public MemoryManager { + void OwnDestructor(void* pointer, void (*destructor)(void*)) override { + static_cast(pointer); + static_cast(destructor); +- ABSL_INTERNAL_UNREACHABLE; ++ ABSL_UNREACHABLE(); + } + }; + diff --git a/eval/eval/evaluator_stack.h b/eval/eval/evaluator_stack.h index 1ecab27..9df65d7 100644 --- a/eval/eval/evaluator_stack.h diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index e87c24f7f4cc..11220c3d18b9 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -151,12 +151,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Abseil", project_desc = "Open source collection of C++ libraries drawn from the most fundamental pieces of Google’s internal codebase", project_url = "https://abseil.io/", - version = "9bff2a9302a8dbf91712fc215eb2e2cf8ec234e7", - sha256 = "ae959138730b55b3fb968d3c357e740e7ffdeab4648dc3eb28843a1e9fa56b57", + version = "c8b33b0191a2db8364cacf94b267ea8a3f20ad83", + sha256 = "a7803eac00bf68eae1a84ee3b9fcf0c1173e8d9b89b2cee92c7b487ea65be2a9", strip_prefix = "abseil-cpp-{version}", urls = ["https://github.com/abseil/abseil-cpp/archive/{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2022-12-07", + release_date = "2023-05-16", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/abseil/abseil-cpp/blob/{version}/LICENSE", diff --git a/source/common/thread_local/thread_local_impl.cc b/source/common/thread_local/thread_local_impl.cc index a9b0a21ab5aa..88981133cff0 100644 --- a/source/common/thread_local/thread_local_impl.cc +++ b/source/common/thread_local/thread_local_impl.cc @@ -26,7 +26,7 @@ SlotPtr InstanceImpl::allocateSlot() { ASSERT(!shutdown_); if (free_slot_indexes_.empty()) { - SlotPtr slot = std::make_unique(*this, slots_.size()); + SlotPtr slot = std::make_unique(*this, uint32_t(slots_.size())); slots_.push_back(slot.get()); return slot; } From b82371e8b04a9a9bb61f53e3280da60d726333b7 Mon Sep 17 00:00:00 2001 From: David Schinazi Date: Thu, 18 May 2023 20:33:35 -0700 Subject: [PATCH 298/740] Update QUICHE from d687c5fc0 to 4a5e9e692 (#27467) * Update QUICHE from d687c5fc0 to 8cd348bd2 https://github.com/google/quiche/compare/d687c5fc0..8cd348bd2 ``` $ git log d687c5fc0..8cd348bd2 --date=short --no-merges --format="%ad %al %s" 2023-05-17 danzh Rename QuicSpdyStream::AreHeadersValid() to ValidatedRequestHeaders() and retain error details in the stream if it returns false. 2023-05-17 wub Add alpn to interface QuicTestServer::SessionFactory::CreateSession. 2023-05-17 bnc Add test for MAYBE_BODY_BUT_NO_CONTENT_LENGTH warning. 2023-05-17 elburrito Add token expiration time checks to CachedBlindSignAuth to ensure that it only returns fresh tokens. Token expiration time is based on the AT server's signing public key. 2023-05-16 davidben Log encrypted ClientHellos through QuicConnectionDebugVisitor 2023-05-16 vasilvv Remove the header argument from WebTransport OnSessionReady callback 2023-05-16 martinduke Delete useless member variable QuicFramer::last_serialized_client_connection_id_. ``` Signed-off-by: David Schinazi --- bazel/external/quiche.BUILD | 7 +++++-- bazel/repository_locations.bzl | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 216669c395c9..cd3df2b203d3 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -4523,11 +4523,14 @@ envoy_cc_library( name = "quic_core_udp_socket_lib", srcs = select({ "@envoy//bazel:windows_x86_64": [], - "//conditions:default": ["quiche/quic/core/quic_udp_socket_posix.cc"], + "//conditions:default": ["quiche/quic/core/quic_udp_socket.cc"], }), hdrs = select({ "@envoy//bazel:windows_x86_64": [], - "//conditions:default": ["quiche/quic/core/quic_udp_socket.h"], + "//conditions:default": [ + "quiche/quic/core/quic_udp_socket.h", + "quiche/quic/core/quic_udp_socket_posix.inc", + ], }), copts = quiche_copts + select({ # On OSX/iOS, constants from RFC 3542 (e.g. IPV6_RECVPKTINFO) are not usable diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 11220c3d18b9..96730b84e5a3 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1056,12 +1056,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "d687c5fc08dc2d05b4e253237173f99eed06488c", - sha256 = "3c704355f9ee4daab15af666b258071d76f1d985c2c725fe65dbe55d643b92cb", + version = "4a5e9e692040ff0786563c30488a7d29fd58f40d", + sha256 = "59e4712549c8f39fa31b5ded59ab548538384785bcc0854ee59221160f91a16d", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2023-05-12", + release_date = "2023-05-17", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", From 98a37d0e8839f32b506ac9873448af84dfad0c19 Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 19 May 2023 04:35:01 +0100 Subject: [PATCH 299/740] deps: Bump `aspect_bazel_lib` -> 1.32.0 (#27478) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 96730b84e5a3..95e2e28b6b04 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -137,12 +137,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Aspect Bazel helpers", project_desc = "Base Starlark libraries and basic Bazel rules which are useful for constructing rulesets and BUILD files", project_url = "https://github.com/aspect-build/bazel-lib", - version = "1.31.1", - sha256 = "5f3443b1d98a462a8b7330f4742483afc8b2d17c8555dd97ce4146f43e961718", + version = "1.32.0", + sha256 = "f1c181b910f821072f38ee45bb87db6b56275458c7f832c54c54ba6334119eca", strip_prefix = "bazel-lib-{version}", urls = ["https://github.com/aspect-build/bazel-lib/archive/v{version}.tar.gz"], use_category = ["build"], - release_date = "2023-05-05", + release_date = "2023-05-16", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/aspect-build/bazel-lib/blob/v{version}/LICENSE", From 4f3c266bace6c3ddc104fc34d333ceda5d49671c Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 19 May 2023 08:31:18 +0100 Subject: [PATCH 300/740] deps: Bump build-image -> 41c5a05 (#27487) Signed-off-by: Ryan Northey --- .bazelrc | 2 +- .devcontainer/Dockerfile | 2 +- .github/workflows/mobile-android_build.yml | 2 +- .github/workflows/mobile-android_tests.yml | 2 +- .github/workflows/mobile-asan.yml | 2 +- .github/workflows/mobile-cc_tests.yml | 2 +- .github/workflows/mobile-core.yml | 2 +- .github/workflows/mobile-coverage.yml | 2 +- .github/workflows/mobile-docs.yml | 2 +- .github/workflows/mobile-format.yml | 2 +- .github/workflows/mobile-perf.yml | 6 +++--- .github/workflows/mobile-tsan.yml | 2 +- .github/workflows/mobile_release.yml | 2 +- bazel/repository_locations.bzl | 6 +++--- ci/run_envoy_docker.sh | 2 -- examples/shared/build/Dockerfile | 2 +- mobile/third_party/rbe_configs/config/BUILD | 8 ++++---- 17 files changed, 23 insertions(+), 25 deletions(-) diff --git a/.bazelrc b/.bazelrc index 4b8b6c96f71c..fa48c228899a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -304,7 +304,7 @@ build:remote-clang-cl --config=rbe-toolchain-clang-cl # Docker sandbox # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/main/toolchains/rbe_toolchains_config.bzl#L8 -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e323afcb8855..7dd1f7df667d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM gcr.io/envoy-ci/envoy-build:818d28832abf2a7c0cb2bff00435be231729a0bf +FROM gcr.io/envoy-ci/envoy-build:41c5a05d708972d703661b702a63ef5060125c33 ARG USERNAME=vscode ARG USER_UID=501 diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml index 200a58d5b4b5..4e1f8ec328e0 100644 --- a/.github/workflows/mobile-android_build.yml +++ b/.github/workflows/mobile-android_build.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml index d6d53894ba14..f4b44217c0e7 100644 --- a/.github/workflows/mobile-android_tests.yml +++ b/.github/workflows/mobile-android_tests.yml @@ -92,7 +92,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-asan.yml b/.github/workflows/mobile-asan.yml index 72c2ec95ade1..2681176bc068 100644 --- a/.github/workflows/mobile-asan.yml +++ b/.github/workflows/mobile-asan.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 180 container: - image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-cc_tests.yml b/.github/workflows/mobile-cc_tests.yml index f1da9a974d9c..28f6c633ecd8 100644 --- a/.github/workflows/mobile-cc_tests.yml +++ b/.github/workflows/mobile-cc_tests.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 steps: - uses: actions/checkout@v3 - name: Add safe directory diff --git a/.github/workflows/mobile-core.yml b/.github/workflows/mobile-core.yml index edf0a078d8ff..26cd00b8089d 100644 --- a/.github/workflows/mobile-core.yml +++ b/.github/workflows/mobile-core.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 steps: - uses: actions/checkout@v3 - name: Ensure no listener leaks diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml index c2f106a70a3e..e6068b9c2510 100644 --- a/.github/workflows/mobile-coverage.yml +++ b/.github/workflows/mobile-coverage.yml @@ -20,7 +20,7 @@ jobs: run: shell: bash container: - image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/mobile-docs.yml b/.github/workflows/mobile-docs.yml index 292c96c1dd6f..403529803e14 100644 --- a/.github/workflows/mobile-docs.yml +++ b/.github/workflows/mobile-docs.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 20 container: - image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 steps: - uses: actions/checkout@v3 - name: Add safe directory diff --git a/.github/workflows/mobile-format.yml b/.github/workflows/mobile-format.yml index a42b71b5a9ae..97457d9f2015 100644 --- a/.github/workflows/mobile-format.yml +++ b/.github/workflows/mobile-format.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 45 container: - image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 env: CLANG_FORMAT: /opt/llvm/bin/clang-format BUILDIFIER_BIN: /usr/local/bin/buildifier diff --git a/.github/workflows/mobile-perf.yml b/.github/workflows/mobile-perf.yml index 18bb4cacdb85..26084818d469 100644 --- a/.github/workflows/mobile-perf.yml +++ b/.github/workflows/mobile-perf.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 30 container: - image: envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 steps: - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 diff --git a/.github/workflows/mobile-tsan.yml b/.github/workflows/mobile-tsan.yml index 02ea14dfeb7d..8a5898f2b813 100644 --- a/.github/workflows/mobile-tsan.yml +++ b/.github/workflows/mobile-tsan.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile_release.yml b/.github/workflows/mobile_release.yml index 4fe5f89f5fa3..8beed1ab6268 100644 --- a/.github/workflows/mobile_release.yml +++ b/.github/workflows/mobile_release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:mobile-818d28832abf2a7c0cb2bff00435be231729a0bf + image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 95e2e28b6b04..1f7c2b60dc1d 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -91,11 +91,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "envoy-build-tools", project_desc = "Common build tools shared by the Envoy/UDPA ecosystem", project_url = "https://github.com/envoyproxy/envoy-build-tools", - version = "2c557f5c899527b3331c316b2d5ca137fcb046b9", - sha256 = "f7e256ee9ee4d02536fb2124c6b79694633e187fe9b33017273af92e88bbdd6c", + version = "49a27300e7b480955d3a6000eea159ff52998b52", + sha256 = "67fbba8f4329e16f693f9fabaa6e430eddb3f27b80186df884d5b801208be8d9", strip_prefix = "envoy-build-tools-{version}", urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"], - release_date = "2023-05-06", + release_date = "2023-05-16", use_category = ["build"], license = "Apache-2.0", license_url = "https://github.com/envoyproxy/envoy-build-tools/blob/{version}/LICENSE", diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 2dcda09a6deb..373acfa84773 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -60,8 +60,6 @@ else && usermod -a -G pcap envoybuild \ && chown envoybuild:envoygroup /build \ && chown envoybuild /proc/self/fd/2 \ - && rm -rf /usr/bin/cmake \ - && cmake &> /dev/null || echo 'No cmake here!' \ && sudo -EHs -u envoybuild bash -c 'cd /source && $*'") fi diff --git a/examples/shared/build/Dockerfile b/examples/shared/build/Dockerfile index 9e8e5992fbca..8a4355983a6e 100644 --- a/examples/shared/build/Dockerfile +++ b/examples/shared/build/Dockerfile @@ -1,4 +1,4 @@ -FROM envoyproxy/envoy-build-ubuntu:818d28832abf2a7c0cb2bff00435be231729a0bf +FROM envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ diff --git a/mobile/third_party/rbe_configs/config/BUILD b/mobile/third_party/rbe_configs/config/BUILD index b9281332ad58..298016e321ed 100644 --- a/mobile/third_party/rbe_configs/config/BUILD +++ b/mobile/third_party/rbe_configs/config/BUILD @@ -43,8 +43,8 @@ platform( ], exec_properties = { # Please update both the commented tag and the sha256 - # mobile-818d28832abf2a7c0cb2bff00435be231729a0bf - "container-image": "docker://envoyproxy/envoy-build-ubuntu@sha256:20034a0e6afa28e33125760b2dbb069ba9bf0ba717727fbb2d200a4b633297fe", + # mobile-41c5a05d708972d703661b702a63ef5060125c33 + "container-image": "docker://envoyproxy/envoy-build-ubuntu@sha256:ca26ff05bd3f3a09468242faaf38ae48315e57f0a87c102352162f95ac620e6f", "OSFamily": "Linux", "Pool": "linux", }, @@ -60,8 +60,8 @@ platform( ], exec_properties = { # Please update both the commented tag and the sha256 - # mobile-818d28832abf2a7c0cb2bff00435be231729a0bf - "container-image": "docker://envoyproxy/envoy-build-ubuntu@sha256:20034a0e6afa28e33125760b2dbb069ba9bf0ba717727fbb2d200a4b633297fe", + # mobile-41c5a05d708972d703661b702a63ef5060125c33 + "container-image": "docker://envoyproxy/envoy-build-ubuntu@sha256:ca26ff05bd3f3a09468242faaf38ae48315e57f0a87c102352162f95ac620e6f", "OSFamily": "Linux", "Pool": "linux", # Necessary to workaround https://github.com/google/sanitizers/issues/916, otherwise, dangling threads in the From 76f80cab7ae1f1fd21356cc0bbf57120e7a5dca7 Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 19 May 2023 10:05:42 +0100 Subject: [PATCH 301/740] Revert "ci: Workaround dep checker (cves) issue again" (#27505) and fix release date This reverts commit 7e50eecb80669d04c74b4125bfee00fc96c7fb85. Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 2 +- ci/do_ci.sh | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 1f7c2b60dc1d..cb8ff5395b5c 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1061,7 +1061,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2023-05-17", + release_date = "2023-05-18", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 427dfc3a1cf7..662dc101406e 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -438,8 +438,7 @@ case $CI_TARGET in bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:check \ --action_env=TODAY_DATE \ -- -v warn \ - -c cves release_dates releases || echo "WARNING: Dependency check failed" - + -c cves release_dates releases # Run dependabot tests echo "Check dependabot ..." bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ From bf8483c6e02ad4388018027dda1baefdd81cd3c9 Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 19 May 2023 11:21:03 +0100 Subject: [PATCH 302/740] ci: Improve workflow staging/publishing (#27493) Signed-off-by: Ryan Northey --- .azure-pipelines/env.yml | 40 ++++++++++++++++++----- .azure-pipelines/stage/publish.yml | 52 +++++++++++++++++++++++------- .azure-pipelines/stages.yml | 10 ++++-- ci/do_ci.sh | 5 +++ ci/run_envoy_docker.sh | 1 + docs/publish.sh | 36 ++------------------- 6 files changed, 88 insertions(+), 56 deletions(-) diff --git a/.azure-pipelines/env.yml b/.azure-pipelines/env.yml index aa9d16de691a..b90813001600 100644 --- a/.azure-pipelines/env.yml +++ b/.azure-pipelines/env.yml @@ -181,15 +181,36 @@ jobs: set -e PUBLISH_GITHUB_RELEASE=false - PUBLISH_NETLIFY=false - if [[ -n "$POSTSUBMIT" && "$NOSYNC" != true ]]; then - PUBLISH_NETLIFY=true - fi - if [[ -n "$POSTSUBMIT" && "$NOSYNC" != true && "$ISSTABLEBRANCH" == True && "$(state.isdev)" == false ]]; then - PUBLISH_GITHUB_RELEASE=true + PUBLISH_DOCKERHUB=false + PUBLISH_DOCS=false + PUBLISH_DOCS_LATEST=false + PUBLISH_DOCS_RELEASE=false + + if [[ "$ISSTABLEBRANCH" == True && -n "$POSTSUBMIT" && "$NOSYNC" != true ]]; then + # Build docs for publishing either latest or a release build + PUBLISH_DOCS=true + if [[ "$ISMAIN" == True ]]; then + # Update the Dockerhub README + PUBLISH_DOCKERHUB=true + if [[ "$(state.isDev)" == true ]]; then + # Postsubmit on `main` trigger rebuild of latest docs + PUBLISH_DOCS_LATEST=true + fi + elif [[ "$(state.isDev)" == false ]]; then + # A stable release, publish docs to the release and create a Github release + PUBLISH_GITHUB_RELEASE=true + PUBLISH_DOCS_RELEASE=true + else + # Postsubmit for non-main/release, skip publishing docs in this case + PUBLISH_DOCS=false + fi fi + echo "##vso[task.setvariable variable=githubRelease;isoutput=true]${PUBLISH_GITHUB_RELEASE}" - echo "##vso[task.setvariable variable=netlify;isoutput=true]${PUBLISH_NETLIFY}" + echo "##vso[task.setvariable variable=dockerhub;isoutput=true]${PUBLISH_DOCKERHUB}" + echo "##vso[task.setvariable variable=docs;isoutput=true]${PUBLISH_DOCS}" + echo "##vso[task.setvariable variable=docsLatest;isoutput=true]${PUBLISH_DOCS_LATEST}" + echo "##vso[task.setvariable variable=docsRelease;isoutput=true]${PUBLISH_DOCS_RELEASE}" displayName: "Decide what to publish" workingDirectory: $(Build.SourcesDirectory) @@ -210,7 +231,10 @@ jobs: echo "env.outputs['run.checks']: $(run.checks)" echo echo "env.outputs['publish.githubRelease']: $(publish.githubRelease)" - echo "env.outputs['publish.netlify']: $(publish.netlify)" + echo "env.outputs['publish.dockerhub]: $(publish.dockerhub)" + echo "env.outputs['publish.docs]: $(publish.docs)" + echo "env.outputs['publish.docsLatest]: $(publish.docsLatest)" + echo "env.outputs['publish.docsRelease]: $(publish.docsRelease)" displayName: "Print build environment" diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index 3e97ef8bdbc7..444e5db454d0 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -60,14 +60,27 @@ parameters: displayName: "Run Packaging" type: string default: true -- name: publishNetlify - displayName: "Publish Netlify" + +- name: publishDockerhub + displayName: "Publish Dockerhub" type: string - default: true + default: false +- name: publishDocs + displayName: "Publish Docs" + type: string + default: false +- name: publishDocsLatest + displayName: "Publish latest docs" + type: string + default: false +- name: publishDocsRelease + displayName: "Publish release docs" + type: string + default: false - name: publishGithubRelease displayName: "Publish Github release" type: string - default: true + default: false jobs: @@ -233,11 +246,11 @@ jobs: echo "building publishable assets complete" - job: docs - displayName: Publish docs to netlify + displayName: Publish docs dependsOn: ["success"] condition: | and(not(canceled()), - eq(${{ parameters.publishNetlify }}, 'true')) + eq(${{ parameters.publishDocs }}, 'true')) pool: vmImage: "ubuntu-20.04" steps: @@ -261,6 +274,9 @@ jobs: - script: | ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-publish' + condition: | + and(not(canceled()), + eq(${{ parameters.publishDockerhub }}, 'true')) displayName: "Publish Dockerhub description and README" env: ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) @@ -272,7 +288,6 @@ jobs: GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} - condition: and(eq(variables['isMain'], 'true'), eq(variables['PostSubmit'], true)) - script: | ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload' @@ -287,19 +302,32 @@ jobs: GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} condition: eq(variables['isMain'], 'true') + # Trigger Netlify rebuild of latest docs + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs-publish-latest' + condition: | + and(not(canceled()), + eq(${{ parameters.publishDocsLatest }}, 'true')) + displayName: "Publish latest docs" + workingDirectory: $(Build.SourcesDirectory) + env: + NETLIFY_TRIGGER_URL: ${{ parameters.authNetlifyURL }} + + # Publish docs to the website - task: InstallSSHKey@0 + condition: | + and(not(canceled()), + eq(${{ parameters.publishDocsRelease }}, 'true')) inputs: hostName: $(authGithubSSHKeyPublic) sshPublicKey: "${{ parameters.authSSHDocsKeyPublic }}" sshPassphrase: "${{ parameters.authSSHKeyPassphrase }}" sshKeySecureFile: "${{ parameters.authSSHDocsKey }}" - - script: docs/publish.sh - displayName: "Publish to GitHub" + condition: | + and(not(canceled()), + eq(${{ parameters.publishDocsRelease }}, 'true')) + displayName: "Publish release docs" workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - NETLIFY_TRIGGER_URL: ${{ parameters.authNetlifyURL }} - job: github displayName: Publish release tag diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index 05f9d28fa912..6a36bb8da161 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -137,7 +137,10 @@ stages: RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']] RUN_PACKAGING: $[stageDependencies.env.repo.outputs['run.packaging']] PUBLISH_GITHUB_RELEASE: $[stageDependencies.env.repo.outputs['publish.githubRelease']] - PUBLISH_NETLIFY: $[stageDependencies.env.repo.outputs['publish.netlify']] + PUBLISH_DOCKERHUB: $[stageDependencies.env.repo.outputs['publish.dockerhub']] + PUBLISH_DOCS: $[stageDependencies.env.repo.outputs['publish.docs']] + PUBLISH_DOCS_LATEST: $[stageDependencies.env.repo.outputs['publish.docsLatest']] + PUBLISH_DOCS_RELEASE: $[stageDependencies.env.repo.outputs['publish.docsRelease']] jobs: - template: stage/publish.yml parameters: @@ -157,7 +160,10 @@ stages: timeoutDockerPublish: ${{ parameters.timeoutDockerPublish }} runDocker: variables['RUN_DOCKER'] runPackaging: variables['RUN_PACKAGING'] - publishNetlify: variables['PUBLISH_NETLIFY'] + publishDocs: variables['PUBLISH_DOCS'] + publishDocsLatest: variables['PUBLISH_DOCS_LATEST'] + publishDocsRelease: variables['PUBLISH_DOCS_RELEASE'] + publishDockerhub: variables['PUBLISH_DOCKERHUB'] publishGithubRelease: variables['PUBLISH_GITHUB_RELEASE'] - stage: verify diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 662dc101406e..f35425f4de53 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -508,6 +508,11 @@ case $CI_TARGET in "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" /source/generated/docs docs ;; + docs-publish-latest) + BUILD_SHA=$(git rev-parse HEAD) + curl -X POST -d "$BUILD_SHA" "$NETLIFY_TRIGGER_URL" + ;; + docker-upload) setup_clang_toolchain "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" "${BUILD_DIR}/build_images" docker diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 373acfa84773..30cbe0094613 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -133,6 +133,7 @@ docker run --rm \ -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ -e GCS_ARTIFACT_BUCKET \ -e GITHUB_TOKEN \ + -e NETLIFY_TRIGGER_URL \ -e BUILD_SOURCEBRANCHNAME \ -e BAZELISK_BASE_URL \ -e ENVOY_BUILD_ARCH \ diff --git a/docs/publish.sh b/docs/publish.sh index 2a27e1c6266b..dee9cde4303e 100755 --- a/docs/publish.sh +++ b/docs/publish.sh @@ -1,13 +1,6 @@ #!/bin/bash -# This is run on every commit that Azure Pipelines picks up. It assumes that docs have already been built -# via docs/build.sh. The push behavior differs depending on the nature of the commit and non/dev `VERSION.txt`: -# * Main commit (dev): -# pushes docs to https://www.envoyproxy.io/docs/envoy/latest/. -# * Main or release branch commit (non-dev): -# pushes docs to versioned location, e.g. -# https://www.envoyproxy.io/docs/envoy/v1.6.0/. -# * Otherwise: noop. +# This pushes a prebuilt docs version to the website for published releases. set -e @@ -16,30 +9,7 @@ CHECKOUT_DIR=envoy-docs BUILD_SHA=$(git rev-parse HEAD) VERSION="$(cat VERSION.txt)" -MAIN_BRANCH="refs/heads/main" -RELEASE_BRANCH_REGEX="^refs/heads/release/v.*" -DEV_VERSION_REGEX="-dev$" - -if [[ "$VERSION" =~ $DEV_VERSION_REGEX ]]; then - if [[ "$AZP_BRANCH" == "${MAIN_BRANCH}" ]]; then - if [[ -n "$NETLIFY_TRIGGER_URL" ]]; then - echo "Triggering netlify docs build for (${BUILD_SHA})" - curl -X POST -d "$BUILD_SHA" "$NETLIFY_TRIGGER_URL" - fi - else - echo "Ignoring docs push" - fi - exit 0 -else - PUBLISH_DIR="${CHECKOUT_DIR}/docs/envoy/v${VERSION}" -fi - -if [[ "${AZP_BRANCH}" != "${MAIN_BRANCH}" ]] && ! [[ "${AZP_BRANCH}" =~ ${RELEASE_BRANCH_REGEX} ]]; then - # Most likely a tag, do nothing. - echo 'Ignoring non-release branch for docs push.' - exit 0 -fi - +PUBLISH_DIR="${CHECKOUT_DIR}/docs/envoy/v${VERSION}" DOCS_MAIN_BRANCH="main" echo 'cloning' @@ -59,8 +29,6 @@ cd "${CHECKOUT_DIR}" git config user.name "envoy-docs(Azure Pipelines)" git config user.email envoy-docs@users.noreply.github.com -set -x - git add . git commit -m "docs envoy@$BUILD_SHA" git push origin "${DOCS_MAIN_BRANCH}" From 34c49003a4644c0d183c59d2cdd4b145bbc88194 Mon Sep 17 00:00:00 2001 From: code Date: Fri, 19 May 2023 21:26:24 +0800 Subject: [PATCH 303/740] generic proxy: typed metadata support for the generic route (#27503) Signed-off-by: wbpcode --- .../filters/network/source/interface/BUILD | 1 + .../filters/network/source/interface/route.h | 16 ++++-- .../filters/network/source/route.cc | 2 +- .../filters/network/source/route.h | 3 ++ .../filters/network/test/mocks/route.h | 1 + .../filters/network/test/route_test.cc | 49 +++++++++++++++++++ 6 files changed, 67 insertions(+), 5 deletions(-) diff --git a/contrib/generic_proxy/filters/network/source/interface/BUILD b/contrib/generic_proxy/filters/network/source/interface/BUILD index a7144b4fe329..daaa17adde56 100644 --- a/contrib/generic_proxy/filters/network/source/interface/BUILD +++ b/contrib/generic_proxy/filters/network/source/interface/BUILD @@ -68,6 +68,7 @@ envoy_cc_library( "//envoy/event:dispatcher_interface", "//envoy/network:connection_interface", "//envoy/rds:rds_config_interface", + "//envoy/router:router_interface", "//envoy/stream_info:stream_info_interface", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/contrib/generic_proxy/filters/network/source/interface/route.h b/contrib/generic_proxy/filters/network/source/interface/route.h index a417a7a39068..7da77f19f010 100644 --- a/contrib/generic_proxy/filters/network/source/interface/route.h +++ b/contrib/generic_proxy/filters/network/source/interface/route.h @@ -5,6 +5,7 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/typed_metadata.h" #include "envoy/rds/config.h" +#include "envoy/router/router.h" #include "contrib/generic_proxy/filters/network/source/interface/stream.h" @@ -20,9 +21,10 @@ class RouteSpecificFilterConfig { using RouteSpecificFilterConfigConstSharedPtr = std::shared_ptr; /** - * Interface of typed metadata factory. + * Interface of typed metadata factory. Reuse the same interface as the HTTP's router filter because + * part of these abstractions are protocol independent. */ -class RouteTypedMetadataFactory : public Envoy::Config::TypedMetadataFactory {}; +using RouteTypedMetadataFactory = Envoy::Router::HttpRouteTypedMetadataFactory; class RouteEntry { public: @@ -47,10 +49,16 @@ class RouteEntry { } /** - * @return const envoy::config::core::v3::Metadata& return the metadata provided in the config for - * this route. + * @return const envoy::config::core::v3::Metadata& return the metadata provided in the config + * for this route. */ virtual const envoy::config::core::v3::Metadata& metadata() const PURE; + + /** + * @return const Envoy::Config::TypedMetadata& return the typed metadata provided in the config + * for this route. + */ + virtual const Envoy::Config::TypedMetadata& typedMetadata() const PURE; }; using RouteEntryConstSharedPtr = std::shared_ptr; diff --git a/contrib/generic_proxy/filters/network/source/route.cc b/contrib/generic_proxy/filters/network/source/route.cc index bcbedb2d3136..f58998f18925 100644 --- a/contrib/generic_proxy/filters/network/source/route.cc +++ b/contrib/generic_proxy/filters/network/source/route.cc @@ -20,7 +20,7 @@ namespace GenericProxy { RouteEntryImpl::RouteEntryImpl(const ProtoRouteAction& route_action, Envoy::Server::Configuration::ServerFactoryContext& context) : name_(route_action.name()), cluster_name_(route_action.cluster()), - metadata_(route_action.metadata()) { + metadata_(route_action.metadata()), typed_metadata_(metadata_) { for (const auto& proto_filter_config : route_action.per_filter_config()) { auto& factory = Config::Utility::getAndCheckFactoryByName( diff --git a/contrib/generic_proxy/filters/network/source/route.h b/contrib/generic_proxy/filters/network/source/route.h index bea6961e4e22..010a6ec0be17 100644 --- a/contrib/generic_proxy/filters/network/source/route.h +++ b/contrib/generic_proxy/filters/network/source/route.h @@ -44,6 +44,8 @@ class RouteEntryImpl : public RouteEntry { } const envoy::config::core::v3::Metadata& metadata() const override { return metadata_; } + const Envoy::Config::TypedMetadata& typedMetadata() const override { return typed_metadata_; }; + private: static const uint64_t DEFAULT_ROUTE_TIMEOUT_MS = 15000; @@ -51,6 +53,7 @@ class RouteEntryImpl : public RouteEntry { std::string cluster_name_; const envoy::config::core::v3::Metadata metadata_; + const Envoy::Config::TypedMetadataImpl typed_metadata_; absl::flat_hash_map per_filter_configs_; }; diff --git a/contrib/generic_proxy/filters/network/test/mocks/route.h b/contrib/generic_proxy/filters/network/test/mocks/route.h index 428f4285a672..3b5792362e10 100644 --- a/contrib/generic_proxy/filters/network/test/mocks/route.h +++ b/contrib/generic_proxy/filters/network/test/mocks/route.h @@ -17,6 +17,7 @@ class MockRouteEntry : public RouteEntry { MOCK_METHOD(const std::string&, clusterName, (), (const)); MOCK_METHOD(const RouteSpecificFilterConfig*, perFilterConfig, (absl::string_view), (const)); MOCK_METHOD(const envoy::config::core::v3::Metadata&, metadata, (), (const)); + MOCK_METHOD(const Envoy::Config::TypedMetadata&, typedMetadata, (), (const)); MOCK_METHOD(absl::string_view, name, (), (const)); std::string cluster_name_{"fake_cluster_name"}; diff --git a/contrib/generic_proxy/filters/network/test/route_test.cc b/contrib/generic_proxy/filters/network/test/route_test.cc index 7e948fef3191..b1377849c228 100644 --- a/contrib/generic_proxy/filters/network/test/route_test.cc +++ b/contrib/generic_proxy/filters/network/test/route_test.cc @@ -72,6 +72,55 @@ TEST_F(RouteEntryImplTest, RouteMetadata) { Config::Metadata::metadataValue(&route_->metadata(), "mock_filter", "key_0").string_value()); }; +struct Foo : public Envoy::Config::TypedMetadata::Object {}; +struct Baz : public Envoy::Config::TypedMetadata::Object { + Baz(std::string n) : name(n) {} + std::string name; +}; +class BazFactory : public RouteTypedMetadataFactory { +public: + std::string name() const override { return "baz"; } + // Returns nullptr (conversion failure) if d is empty. + std::unique_ptr + parse(const ProtobufWkt::Struct& d) const override { + if (d.fields().find("name") != d.fields().end()) { + return std::make_unique(d.fields().at("name").string_value()); + } + throw EnvoyException("Cannot create a Baz when metadata is empty."); + } + + std::unique_ptr + parse(const ProtobufWkt::Any&) const override { + return nullptr; + } +}; + +/** + * Test the method that get filter metadata from the route entry. + */ +TEST_F(RouteEntryImplTest, RouteTypedMetadata) { + BazFactory baz_factory; + Registry::InjectFactory registered_factory(baz_factory); + + const std::string yaml_config = R"EOF( + cluster: cluster_0 + metadata: + filter_metadata: + foo: + key_0: value_0 + baz: + name: baz_name + )EOF"; + initialize(yaml_config); + + // Only get valid value when type and name are matched. + EXPECT_EQ("baz_name", route_->typedMetadata().get(baz_factory.name())->name); + + EXPECT_EQ(nullptr, route_->typedMetadata().get(baz_factory.name())); + EXPECT_EQ(nullptr, route_->typedMetadata().get("foo")); + EXPECT_EQ(nullptr, route_->typedMetadata().get("foo")); +}; + /** * Test the method that get route level per filter config from the route entry. This test also * verifies that the proto per filter config can be loaded correctly. From 25a76279b4a51cdf7e21b753da2f57968ea7a75d Mon Sep 17 00:00:00 2001 From: Ryan Hamilton Date: Fri, 19 May 2023 06:28:32 -0700 Subject: [PATCH 304/740] mobile: remove the mobile-specific BoringSSL repo (#27491) mobile: remove the mobile-specific boringssl repo Signed-off-by: Ryan Hamilton --- mobile/bazel/envoy_mobile_repositories.bzl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/mobile/bazel/envoy_mobile_repositories.bzl b/mobile/bazel/envoy_mobile_repositories.bzl index 5583cefcb759..3146aa1a7a52 100644 --- a/mobile/bazel/envoy_mobile_repositories.bzl +++ b/mobile/bazel/envoy_mobile_repositories.bzl @@ -38,15 +38,6 @@ def upstream_envoy_overrides(): build_file_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])""", ) - # This should be kept in sync with Envoy itself, we just need to apply this patch - # Remove this once https://boringssl-review.googlesource.com/c/boringssl/+/37804 is in master-with-bazel - http_archive( - name = "boringssl", - sha256 = "579cb415458e9f3642da0a39a72f79fdfe6dc9c1713b3a823f1e276681b9703e", - strip_prefix = "boringssl-648cbaf033401b7fe7acdce02f275b06a88aab5c", - urls = ["https://github.com/google/boringssl/archive/648cbaf033401b7fe7acdce02f275b06a88aab5c.tar.gz"], - ) - def swift_repos(): http_archive( name = "build_bazel_rules_apple", From 1fd0b996f820ef5a85b0889c7937e82696c95f83 Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Fri, 19 May 2023 09:35:52 -0400 Subject: [PATCH 305/740] ext_proc filter crash when sending trailers when request has no body (#27430) * ext_proc filter crash when sending trailers without sending body. Signed-off-by: Yanjun Xiang --- .../filters/http/ext_proc/ext_proc.cc | 21 +++++++++ .../filters/http/ext_proc/ext_proc.h | 5 +-- .../ext_proc/ext_proc_integration_test.cc | 45 +++++++++++++++++++ ..._proc_unit_test_fuzz-5547889199546368.test | 34 ++++++++++++++ 4 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-5547889199546368.test diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index 34c4ce953a9d..ad95d50c368d 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -435,6 +435,16 @@ FilterTrailersStatus Filter::onTrailers(ProcessorState& state, Http::HeaderMap& } if (!body_delivered && state.bodyMode() == ProcessingMode::BUFFERED) { + // If no gRPC stream yet, opens it before sending data. + switch (openStream()) { + case StreamOpenState::Error: + return FilterTrailersStatus::StopIteration; + case StreamOpenState::IgnoreError: + return FilterTrailersStatus::Continue; + case StreamOpenState::Ok: + // Fall through + break; + } // We would like to process the body in a buffered way, but until now the complete // body has not arrived. With the arrival of trailers, we now know that the body // has arrived. @@ -516,6 +526,17 @@ void Filter::sendBodyChunk(ProcessorState& state, const Buffer::Instance& data, stats_.stream_msgs_sent_.inc(); } +void Filter::sendBufferedData(ProcessorState& state, ProcessorState::CallbackState new_state, + bool end_stream) { + if (state.hasBufferedData()) { + sendBodyChunk(state, *state.bufferedData(), new_state, end_stream); + } else { + // If there is no buffered data, sends an empty body. + Buffer::OwnedImpl data(""); + sendBodyChunk(state, data, new_state, end_stream); + } +} + void Filter::sendTrailers(ProcessorState& state, const Http::HeaderMap& trailers) { ProcessingRequest req; auto* trailers_req = state.mutableTrailers(req); diff --git a/source/extensions/filters/http/ext_proc/ext_proc.h b/source/extensions/filters/http/ext_proc/ext_proc.h index 2824cb4d1944..1ff10c8d252e 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.h +++ b/source/extensions/filters/http/ext_proc/ext_proc.h @@ -194,9 +194,8 @@ class Filter : public Logger::Loggable, void onNewTimeout(const ProtobufWkt::Duration& override_message_timeout); void sendBufferedData(ProcessorState& state, ProcessorState::CallbackState new_state, - bool end_stream) { - sendBodyChunk(state, *state.bufferedData(), new_state, end_stream); - } + bool end_stream); + void sendBodyChunk(ProcessorState& state, const Buffer::Instance& data, ProcessorState::CallbackState new_state, bool end_stream); diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc index 1d4565a56470..3c488aaa0e27 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -210,6 +210,27 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, } } + void processRequestTrailersMessage( + FakeUpstream& grpc_upstream, bool first_message, + absl::optional> cb) { + ProcessingRequest request; + if (first_message) { + ASSERT_TRUE(grpc_upstream.waitForHttpConnection(*dispatcher_, processor_connection_)); + ASSERT_TRUE(processor_connection_->waitForNewStream(*dispatcher_, processor_stream_)); + } + ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, request)); + ASSERT_TRUE(request.has_request_trailers()); + if (first_message) { + processor_stream_->startGrpcStream(); + } + ProcessingResponse response; + auto* body = response.mutable_request_trailers(); + const bool sendReply = !cb || (*cb)(request.request_trailers(), *body); + if (sendReply) { + processor_stream_->sendGrpcMessage(response); + } + } + void processResponseHeadersMessage( FakeUpstream& grpc_upstream, bool first_message, absl::optional> cb) { @@ -1969,4 +1990,28 @@ TEST_P(ExtProcIntegrationTest, RequestMessageNewTimeoutOutOfBounds) { newTimeoutWrongConfigTest(override_message_timeout); } +// Set the ext_proc filter in SKIP header, BUFFERED body mode. +// Send a request with headers and trailers. +TEST_P(ExtProcIntegrationTest, SendHeaderAndTrailerInBufferedMode) { + proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::BUFFERED); + proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SKIP); + proto_config_.mutable_processing_mode()->set_request_trailer_mode(ProcessingMode::SEND); + initializeConfig(); + HttpIntegrationTest::initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + auto encoder_decoder = codec_client_->startRequest(headers); + request_encoder_ = &encoder_decoder.first; + IntegrationStreamDecoderPtr response = std::move(encoder_decoder.second); + Http::TestRequestTrailerMapImpl request_trailers{{"request", "trailer"}}; + codec_client_->sendTrailers(*request_encoder_, request_trailers); + processRequestBodyMessage(*grpc_upstreams_[0], true, absl::nullopt); + processRequestTrailersMessage(*grpc_upstreams_[0], false, absl::nullopt); + handleUpstreamRequest(); + processResponseHeadersMessage(*grpc_upstreams_[0], false, absl::nullopt); + verifyDownstreamResponse(*response, 200); +} + } // namespace Envoy diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-5547889199546368.test b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-5547889199546368.test new file mode 100644 index 000000000000..06f1a08753f1 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-5547889199546368.test @@ -0,0 +1,34 @@ +config { + grpc_service { + envoy_grpc { + cluster_name: "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" + retry_policy { + } + } + timeout { + } + } + processing_mode { + request_header_mode: SKIP + request_body_mode: BUFFERED + } + request_attributes: "\022." +} +request { + trailers { + headers { + key: "." + } + } +} +response { + response_body { + response { + header_mutation { + } + } + } + mode_override { + request_header_mode: SKIP + } +} From 01fe4aec8ef235d0dffa2a09231d87ceb0a6a38b Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Fri, 19 May 2023 11:09:25 -0400 Subject: [PATCH 306/740] match_delegate: skip filter on_no_match (#27332) Signed-off-by: tyxia --- .../common/http/match_delegate/config_test.cc | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/common/http/match_delegate/config_test.cc b/test/common/http/match_delegate/config_test.cc index a108b56b84a6..7cbcd104e3bb 100644 --- a/test/common/http/match_delegate/config_test.cc +++ b/test/common/http/match_delegate/config_test.cc @@ -315,6 +315,63 @@ TEST(DelegatingFilterTest, MatchTreeSkipActionDecodingHeaders) { delegating_filter->decodeComplete(); } +// This function creates the matcher tree only with on_no_match set. +template +Matcher::MatchTreeSharedPtr +createMatchTreeWithOnNoMatch(const std::string& name, const std::string& value) { + auto tree = std::make_shared>( + std::make_unique(name), + Matcher::OnMatch{ + []() { return std::make_unique(); }, nullptr}); + + // No action is set on match. i.e., nullptr action factory cb. + tree->addChild( + value, Matcher::OnMatch{[]() { return nullptr; }, nullptr}); + return tree; +} + +TEST(DelegatingFilterTest, MatchTreeOnNoMatchSkipActionDecodingHeaders) { + // The filter is added, but the `on_no_match` action will be invoked since it is not matched on + // request header. The skip action is provided to `on_no_match` so we skip this filter. + std::shared_ptr decoder_filter( + new Envoy::Http::MockStreamDecoderFilter()); + NiceMock callbacks; + + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)); + EXPECT_CALL(*decoder_filter, decodeHeaders(_, _)).Times(0); + EXPECT_CALL(*decoder_filter, decodeData(_, _)).Times(0); + EXPECT_CALL(*decoder_filter, decodeMetadata(_)).Times(0); + EXPECT_CALL(*decoder_filter, decodeTrailers(_)).Times(0); + EXPECT_CALL(*decoder_filter, decodeComplete()).Times(0); + + auto match_tree = + createMatchTreeWithOnNoMatch( + "match-header", "match"); + auto delegating_filter = + std::make_shared(match_tree, decoder_filter, nullptr); + + Envoy::Http::RequestHeaderMapPtr request_headers{ + new Envoy::Http::TestRequestHeaderMapImpl{{":authority", "host"}, + {":path", "/"}, + {":method", "GET"}, + {"match-header", "not_match"}, + {"content-type", "application/grpc"}}}; + Envoy::Http::RequestTrailerMapPtr request_trailers{ + new Envoy::Http::TestRequestTrailerMapImpl{{"test", "test"}}}; + Envoy::Http::MetadataMap metadata_map; + Buffer::OwnedImpl buffer; + + delegating_filter->setDecoderFilterCallbacks(callbacks); + EXPECT_EQ(Envoy::Http::FilterHeadersStatus::Continue, + delegating_filter->decodeHeaders(*request_headers, false)); + EXPECT_EQ(Envoy::Http::FilterDataStatus::Continue, delegating_filter->decodeData(buffer, false)); + EXPECT_EQ(Envoy::Http::FilterMetadataStatus::Continue, + delegating_filter->decodeMetadata(metadata_map)); + EXPECT_EQ(Envoy::Http::FilterTrailersStatus::Continue, + delegating_filter->decodeTrailers(*request_trailers)); + delegating_filter->decodeComplete(); +} + TEST(DelegatingFilterTest, MatchTreeQueryParamsSkipActionDecodingHeaders) { // The filter is added, but since we match on the request header we skip the filter. std::shared_ptr decoder_filter( From 5fba0932745c127db853e3036b678873448fc1ab Mon Sep 17 00:00:00 2001 From: Pradeep Rao <84025829+pradeepcrao@users.noreply.github.com> Date: Fri, 19 May 2023 13:05:01 -0400 Subject: [PATCH 307/740] stats: Skip slowRejects if we have a definitive answer from fastRejects. (#27455) * stats: Skip slowRejects if we have a definitive answer from fastRejects. Signed-off-by: pcrao --- source/common/stats/stats_matcher_impl.cc | 27 +++--- test/common/stats/BUILD | 24 +++++ .../stats/stats_matcher_impl_speed_test.cc | 88 +++++++++++++++++++ test/common/stats/stats_matcher_impl_test.cc | 24 +++++ 4 files changed, 150 insertions(+), 13 deletions(-) create mode 100644 test/common/stats/stats_matcher_impl_speed_test.cc diff --git a/source/common/stats/stats_matcher_impl.cc b/source/common/stats/stats_matcher_impl.cc index a4440f90494c..6e974a0b2c2c 100644 --- a/source/common/stats/stats_matcher_impl.cc +++ b/source/common/stats/stats_matcher_impl.cc @@ -89,21 +89,22 @@ bool StatsMatcherImpl::fastRejectMatch(StatName stat_name) const { } bool StatsMatcherImpl::slowRejects(FastResult fast_result, StatName stat_name) const { - bool match = slowRejectMatch(stat_name); - - if (is_inclusive_ || match || fast_result != FastResult::Matches) { - // is_inclusive_ | match | return - // ---------------+-------+-------- - // T | T | T Default-inclusive and matching an (exclusion) matcher, deny. - // T | F | F Otherwise, allow. - // F | T | F Default-exclusive and matching an (inclusion) matcher, allow. - // F | F | T Otherwise, deny. - // - // This is an XNOR, which can be evaluated by checking for equality. - return match == is_inclusive_; + // Skip slowRejectMatch if we already have a definitive answer from fastRejects. + if (fast_result != FastResult::NoMatch) { + return fast_result == FastResult::Rejects; } - return false; + const bool match = slowRejectMatch(stat_name); + + // is_inclusive_ | match | return + // ---------------+-------+-------- + // T | T | T Default-inclusive and matching an (exclusion) matcher, deny. + // T | F | F Otherwise, allow. + // F | T | F Default-exclusive and matching an (inclusion) matcher, allow. + // F | F | T Otherwise, deny. + // + // This is an XNOR, which can be evaluated by checking for equality. + return match == is_inclusive_; } bool StatsMatcherImpl::slowRejectMatch(StatName stat_name) const { diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index 0463996725d8..6b8dd9689e62 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -231,6 +231,30 @@ envoy_benchmark_test( benchmark_binary = "symbol_table_benchmark", ) +envoy_cc_benchmark_binary( + name = "stats_matcher_impl_benchmark", + srcs = [ + "stats_matcher_impl_speed_test.cc", + ], + external_deps = [ + "abseil_strings", + "benchmark", + ], + deps = [ + "//source/common/memory:stats_lib", + "//source/common/stats:stats_matcher_lib", + "//test/test_common:logging_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto", + "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", + ], +) + +envoy_benchmark_test( + name = "stats_matcher_impl_benchmark_test", + benchmark_binary = "stats_matcher_impl_benchmark", +) + envoy_cc_test( name = "tag_extractor_impl_test", srcs = ["tag_extractor_impl_test.cc"], diff --git a/test/common/stats/stats_matcher_impl_speed_test.cc b/test/common/stats/stats_matcher_impl_speed_test.cc new file mode 100644 index 000000000000..8c8cd6c247ba --- /dev/null +++ b/test/common/stats/stats_matcher_impl_speed_test.cc @@ -0,0 +1,88 @@ +// Note: this should be run with --compilation_mode=opt, and would benefit from a +// quiescent system with disabled cstate power management. + +#include "envoy/config/metrics/v3/stats.pb.h" +#include "envoy/type/matcher/v3/string.pb.h" + +#include "source/common/stats/stats_matcher_impl.h" + +#include "test/test_common/utility.h" + +#include "benchmark/benchmark.h" + +namespace Envoy { +namespace Stats { + +class StatsMatcherPerf { +public: + StatsMatcherPerf() : pool_(symbol_table_) {} + + void initMatcher() { + stats_matcher_impl_ = std::make_unique(stats_config_, symbol_table_); + } + + envoy::type::matcher::v3::StringMatcher* inclusionList() { + return stats_config_.mutable_stats_matcher()->mutable_inclusion_list()->add_patterns(); + } + envoy::type::matcher::v3::StringMatcher* exclusionList() { + return stats_config_.mutable_stats_matcher()->mutable_exclusion_list()->add_patterns(); + } + + SymbolTableImpl symbol_table_; + StatNamePool pool_; + std::unique_ptr stats_matcher_impl_; + +private: + envoy::config::metrics::v3::StatsConfig stats_config_; +}; + +} // namespace Stats +} // namespace Envoy + +// NOLINTNEXTLINE(readability-identifier-naming) +static void BM_Inclusion(benchmark::State& state) { + // Create matcher. + Envoy::Stats::StatsMatcherPerf context; + context.inclusionList()->set_suffix("1"); + context.inclusionList()->set_prefix("2."); + context.initMatcher(); + std::vector stat_names; + stat_names.reserve(10); + for (auto idx = 0; idx < 10; ++idx) { + stat_names.push_back(context.pool_.add(absl::StrCat("2.", idx))); + } + + for (auto _ : state) { // NOLINT + for (auto idx = 0; idx < 1000; ++idx) { + const Envoy::Stats::StatName stat_name = stat_names[idx % 10]; + const Envoy::Stats::StatsMatcher::FastResult fast_result = + context.stats_matcher_impl_->fastRejects(stat_name); + context.stats_matcher_impl_->slowRejects(fast_result, stat_name); + } + } +} +BENCHMARK(BM_Inclusion); + +// NOLINTNEXTLINE(readability-identifier-naming) +static void BM_Exclusion(benchmark::State& state) { + // Create matcher. + Envoy::Stats::StatsMatcherPerf context; + context.exclusionList()->set_suffix("1"); + context.exclusionList()->set_prefix("2."); + context.initMatcher(); + std::vector stat_names; + stat_names.reserve(10); + for (auto idx = 0; idx < 10; ++idx) { + stat_names.push_back(context.pool_.add(absl::StrCat("2.", idx))); + } + + for (auto _ : state) { // NOLINT + for (auto idx = 0; idx < 1000; ++idx) { + const Envoy::Stats::StatName stat_name = stat_names[idx % 10]; + const Envoy::Stats::StatsMatcher::FastResult fast_result = + context.stats_matcher_impl_->fastRejects(stat_name); + context.stats_matcher_impl_->slowRejects(fast_result, stat_name); + } + } +} +BENCHMARK(BM_Exclusion); diff --git a/test/common/stats/stats_matcher_impl_test.cc b/test/common/stats/stats_matcher_impl_test.cc index 821a47c69b1d..75009010e2b2 100644 --- a/test/common/stats/stats_matcher_impl_test.cc +++ b/test/common/stats/stats_matcher_impl_test.cc @@ -330,5 +330,29 @@ TEST_F(StatsMatcherTest, CheckMultipleAssortedExclusionMatchersWithPrefix) { EXPECT_FALSE(stats_matcher_impl_->rejectsAll()); } +TEST_F(StatsMatcherTest, SkipSlowRejectsOnFastReject) { + inclusionList()->set_suffix("xyz"); + initMatcher(); + StatName stat_name = pool_.add("wxyz"); + // Verify that we skip the slow check if fast_result is not NoMatch. + EXPECT_TRUE(stats_matcher_impl_->slowRejects(StatsMatcher::FastResult::Rejects, stat_name)); + + // Verify we don't skip the slow check if fast_result is NoMatch. + EXPECT_FALSE( + stats_matcher_impl_->slowRejects(stats_matcher_impl_->fastRejects(stat_name), stat_name)); +} + +TEST_F(StatsMatcherTest, SkipSlowRejectsOnFastMatches) { + exclusionList()->set_suffix("xyz"); + initMatcher(); + StatName stat_name = pool_.add("wxyz"); + // Verify that we skip the slow check if fast_result is not NoMatch. + EXPECT_FALSE(stats_matcher_impl_->slowRejects(StatsMatcher::FastResult::Matches, stat_name)); + + // Verify we don't skip the slow check if fast_result is NoMatch. + EXPECT_TRUE( + stats_matcher_impl_->slowRejects(stats_matcher_impl_->fastRejects(stat_name), stat_name)); +} + } // namespace Stats } // namespace Envoy From c2ae2211196a48b12d2e36d00c6c2889ae2f434a Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Fri, 19 May 2023 16:09:21 -0400 Subject: [PATCH 308/740] Add metadata to ext_proc and fault filter (#27003) Aids in decorating access logs for requests that are subjected to ext_proc and fault calls. In ext_proc, the struct is copied into the filter state object that contains per grpc exchange stats. In fault case, it is copied into dynamicMetadata under the namespace matching the filter config name. Risk Level: Low Testing: Unit tests Signed-off-by: Shriram Rajagopalan --- .../filters/http/ext_proc/v3/ext_proc.proto | 14 ++++- .../filters/http/fault/v3/fault.proto | 10 +++- changelogs/current.yaml | 12 ++++ envoy/http/filter.h | 5 ++ source/common/http/async_client_impl.h | 1 + source/common/http/filter_manager.h | 1 + source/extensions/filters/http/ext_proc/BUILD | 1 + .../filters/http/ext_proc/ext_proc.cc | 7 ++- .../filters/http/ext_proc/ext_proc.h | 15 ++++- source/extensions/filters/http/fault/BUILD | 2 + .../filters/http/fault/fault_filter.cc | 9 ++- .../filters/http/fault/fault_filter.h | 5 ++ test/extensions/filters/http/ext_proc/BUILD | 1 + .../filters/http/ext_proc/config_test.cc | 2 + .../filters/http/ext_proc/filter_test.cc | 24 +++++++- .../filters/http/fault/config_test.cc | 2 + .../filters/http/fault/fault_filter_test.cc | 55 ++++++++++++++++--- test/mocks/http/mocks.h | 2 + 18 files changed, 149 insertions(+), 19 deletions(-) diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto index c84c3790d656..dce9b24cfd74 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto @@ -8,6 +8,7 @@ import "envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto"; import "envoy/type/matcher/v3/string.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; import "xds/annotations/v3/status.proto"; @@ -95,8 +96,12 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // :ref:`ProcessingRequest ` // messages, and the server must reply with // :ref:`ProcessingResponse `. - -// [#next-free-field: 13] +// +// Stats about each gRPC call are recorded in a :ref:`dynamic filter state +// ` object in a namespace matching the filter +// name. +// +// [#next-free-field: 14] message ExternalProcessor { // Configuration for the gRPC service that the filter will communicate with. // The filter supports both the "Envoy" and "Google" gRPC clients. @@ -186,6 +191,11 @@ message ExternalProcessor { // Allow headers matching the ``forward_rules`` to be forwarded to the external processing server. // If not set, all headers are forwarded to the external processing server. HeaderForwardingRules forward_rules = 12; + + // Additional metadata to be added to the filter state for logging purposes. The metadata + // will be added to StreamInfo's filter state under the namespace corresponding to the + // ext_proc filter name. + google.protobuf.Struct filter_metadata = 13; } // The HeaderForwardingRules structure specifies what headers are diff --git a/api/envoy/extensions/filters/http/fault/v3/fault.proto b/api/envoy/extensions/filters/http/fault/v3/fault.proto index 5d55cfe00198..01f056e7927a 100644 --- a/api/envoy/extensions/filters/http/fault/v3/fault.proto +++ b/api/envoy/extensions/filters/http/fault/v3/fault.proto @@ -6,6 +6,7 @@ import "envoy/config/route/v3/route_components.proto"; import "envoy/extensions/filters/common/fault/v3/fault.proto"; import "envoy/type/v3/percent.proto"; +import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; @@ -55,7 +56,7 @@ message FaultAbort { type.v3.FractionalPercent percentage = 3; } -// [#next-free-field: 16] +// [#next-free-field: 17] message HTTPFault { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.fault.v2.HTTPFault"; @@ -148,4 +149,11 @@ message HTTPFault { // If set to false, dynamic stats storage will be allocated for the downstream cluster name. // Default value is false. bool disable_downstream_cluster_stats = 15; + + // When an abort or delay fault is executed, the metadata struct provided here will be added to the + // request's dynamic metadata under the namespace corresponding to the name of the fault filter. + // This data can be logged as part of Access Logs using the :ref:`command operator + // ` %DYNAMIC_METADATA(NAMESPACE)%, where NAMESPACE is the name of + // the fault filter. + google.protobuf.Struct filter_metadata = 16; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index a49260da1d22..1ff740abec2e 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -65,6 +65,10 @@ minor_behavior_changes: Changed behavior of the fixed heap monitor to count unused mapped pages as free memory. This change can be reverted temporarily by setting the runtime guard ``envoy.reloadable_features.count_unused_mapped_pages_as_free`` to false. +- area: ext_proc + change: | + Filter metadata containing ext proc stats has been moved from ext-proc-logging-info to a namespace corresponding + to the name of the ext_proc filter. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* @@ -199,6 +203,14 @@ new_features: - area: admin change: | Adds a new admin stats html bucket-mode ``detailed`` to generate all recorded buckets and summary percentiles. +- area: fault + change: | + added new field ``envoy.extensions.filters.http.fault.v3.HTTPFault.filter_metadata`` to aid in logging. + Metadata will be stored in StreamInfo dynamic metadata under a namespace corresponding to the name of the fault filter. +- area: ext_proc + change: | + added new field ``filter_metadata downstreamCallbacks() PURE; + + /** + * @return absl::string_view the name of the filter as configured in the filter chain.. + */ + virtual absl::string_view filterConfigName() const PURE; }; class DecoderFilterWatermarkCallbacks { diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index 1bc5536b6212..8a4dc0d67284 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -480,6 +480,7 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, void resetIdleTimer() override {} void setUpstreamOverrideHost(absl::string_view) override {} absl::optional upstreamOverrideHost() const override { return {}; } + absl::string_view filterConfigName() const override { return ""; } // ScopeTrackedObject void dumpState(std::ostream& os, int indent_level) const override { diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index 5c93ef2f9574..87be59dc76a0 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -104,6 +104,7 @@ struct ActiveStreamFilterBase : public virtual StreamFilterCallbacks, Http1StreamEncoderOptionsOptRef http1StreamEncoderOptions() override; OptRef downstreamCallbacks() override; OptRef upstreamCallbacks() override; + absl::string_view filterConfigName() const override { return filter_context_.config_name; } // Functions to set or get iteration state. bool canIterate() { return iteration_state_ == IterationState::Continue; } diff --git a/source/extensions/filters/http/ext_proc/BUILD b/source/extensions/filters/http/ext_proc/BUILD index 2970272c0494..2c2397d6d304 100644 --- a/source/extensions/filters/http/ext_proc/BUILD +++ b/source/extensions/filters/http/ext_proc/BUILD @@ -27,6 +27,7 @@ envoy_cc_library( "//envoy/http:header_map_interface", "//envoy/stats:stats_macros", "//source/common/buffer:buffer_lib", + "//source/common/protobuf", "//source/common/runtime:runtime_features_lib", "//source/extensions/filters/common/mutation_rules:mutation_rules_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index ad95d50c368d..b56f3da39cb1 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -103,12 +103,13 @@ void Filter::setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callb decoding_state_.setDecoderFilterCallbacks(callbacks); const Envoy::StreamInfo::FilterStateSharedPtr& filter_state = callbacks.streamInfo().filterState(); - if (!filter_state->hasData(ExtProcLoggingInfoName)) { - filter_state->setData(ExtProcLoggingInfoName, std::make_shared(), + if (!filter_state->hasData(callbacks.filterConfigName())) { + filter_state->setData(callbacks.filterConfigName(), + std::make_shared(config_->filterMetadata()), Envoy::StreamInfo::FilterState::StateType::Mutable, Envoy::StreamInfo::FilterState::LifeSpan::Request); } - logging_info_ = filter_state->getDataMutable(ExtProcLoggingInfoName); + logging_info_ = filter_state->getDataMutable(callbacks.filterConfigName()); } void Filter::setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks& callbacks) { diff --git a/source/extensions/filters/http/ext_proc/ext_proc.h b/source/extensions/filters/http/ext_proc/ext_proc.h index 1ff10c8d252e..fbdd050f41ad 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.h +++ b/source/extensions/filters/http/ext_proc/ext_proc.h @@ -4,6 +4,7 @@ #include #include +#include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/event/timer.h" #include "envoy/extensions/filters/http/ext_proc/v3/ext_proc.pb.h" @@ -14,6 +15,7 @@ #include "envoy/stats/stats_macros.h" #include "source/common/common/logger.h" +#include "source/common/protobuf/protobuf.h" #include "source/extensions/filters/common/mutation_rules/mutation_rules.h" #include "source/extensions/filters/http/common/pass_through_filter.h" #include "source/extensions/filters/http/ext_proc/client.h" @@ -43,10 +45,11 @@ struct ExtProcFilterStats { ALL_EXT_PROC_FILTER_STATS(GENERATE_COUNTER_STRUCT) }; -inline constexpr absl::string_view ExtProcLoggingInfoName = "ext-proc-logging-info"; - class ExtProcLoggingInfo : public Envoy::StreamInfo::FilterState::Object { public: + explicit ExtProcLoggingInfo(const Envoy::ProtobufWkt::Struct& filter_metadata) + : filter_metadata_(filter_metadata) {} + struct GrpcCall { GrpcCall(const std::chrono::microseconds latency, const Grpc::Status::GrpcStatus status, const ProcessorState::CallbackState callback_state) @@ -62,11 +65,13 @@ class ExtProcLoggingInfo : public Envoy::StreamInfo::FilterState::Object { envoy::config::core::v3::TrafficDirection traffic_direction); const GrpcCalls& grpcCalls(envoy::config::core::v3::TrafficDirection traffic_direction) const; + const Envoy::ProtobufWkt::Struct& filterMetadata() const { return filter_metadata_; } private: GrpcCalls& grpcCalls(envoy::config::core::v3::TrafficDirection traffic_direction); GrpcCalls decoding_processor_grpc_calls_; GrpcCalls encoding_processor_grpc_calls_; + const Envoy::ProtobufWkt::Struct filter_metadata_; }; class FilterConfig { @@ -79,7 +84,8 @@ class FilterConfig { disable_clear_route_cache_(config.disable_clear_route_cache()), message_timeout_(message_timeout), max_message_timeout_ms_(max_message_timeout_ms), stats_(generateStats(stats_prefix, config.stat_prefix(), scope)), - processing_mode_(config.processing_mode()), mutation_checker_(config.mutation_rules()) {} + processing_mode_(config.processing_mode()), mutation_checker_(config.mutation_rules()), + filter_metadata_(config.filter_metadata()) {} bool failureModeAllow() const { return failure_mode_allow_; } @@ -99,6 +105,8 @@ class FilterConfig { bool disableClearRouteCache() const { return disable_clear_route_cache_; } + const Envoy::ProtobufWkt::Struct& filterMetadata() const { return filter_metadata_; } + private: ExtProcFilterStats generateStats(const std::string& prefix, const std::string& filter_stats_prefix, Stats::Scope& scope) { @@ -114,6 +122,7 @@ class FilterConfig { ExtProcFilterStats stats_; const envoy::extensions::filters::http::ext_proc::v3::ProcessingMode processing_mode_; const Filters::Common::MutationRules::Checker mutation_checker_; + const Envoy::ProtobufWkt::Struct filter_metadata_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/http/fault/BUILD b/source/extensions/filters/http/fault/BUILD index fbc3b80420c6..a62c90509656 100644 --- a/source/extensions/filters/http/fault/BUILD +++ b/source/extensions/filters/http/fault/BUILD @@ -32,10 +32,12 @@ envoy_cc_library( "//source/common/http:header_map_lib", "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", + "//source/common/protobuf", "//source/common/protobuf:utility_lib", "//source/common/stats:utility_lib", "//source/extensions/filters/common/fault:fault_config_lib", "//source/extensions/filters/http/common:stream_rate_limiter_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/fault/v3:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", ], diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 6c8e89aebdfe..1ddbf8affced 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -50,7 +50,8 @@ FaultSettings::FaultSettings(const envoy::extensions::filters::http::fault::v3:: response_rate_limit_percent_runtime_( PROTOBUF_GET_STRING_OR_DEFAULT(fault, response_rate_limit_percent_runtime, RuntimeKeys::get().ResponseRateLimitPercentKey)), - disable_downstream_cluster_stats_(fault.disable_downstream_cluster_stats()) { + disable_downstream_cluster_stats_(fault.disable_downstream_cluster_stats()), + filter_metadata_(fault.filter_metadata()) { if (fault.has_abort()) { request_abort_config_ = std::make_unique(fault.abort()); @@ -169,6 +170,9 @@ bool FaultFilter::maybeSetupDelay(const Http::RequestHeaderMap& request_headers) delay_timer_->enableTimer(duration.value(), &decoder_callbacks_->scope()); recordDelaysInjectedStats(); decoder_callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::DelayInjected); + auto& dynamic_metadata = decoder_callbacks_->streamInfo().dynamicMetadata(); + (*dynamic_metadata.mutable_filter_metadata())[decoder_callbacks_->filterConfigName()] = + fault_settings_->filterMetadata(); return true; } return false; @@ -467,6 +471,9 @@ void FaultFilter::abortWithStatus(Http::Code http_status_code, absl::optional grpc_status) { recordAbortsInjectedStats(); decoder_callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::FaultInjected); + auto& dynamic_metadata = decoder_callbacks_->streamInfo().dynamicMetadata(); + (*dynamic_metadata.mutable_filter_metadata())[decoder_callbacks_->filterConfigName()] = + fault_settings_->filterMetadata(); decoder_callbacks_->sendLocalReply(http_status_code, "fault filter abort", nullptr, grpc_status, RcDetails::get().FaultAbort); } diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index 29ab9ba74f96..b377b0348b2b 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -5,6 +5,7 @@ #include #include +#include "envoy/config/core/v3/base.pb.h" #include "envoy/extensions/filters/http/fault/v3/fault.pb.h" #include "envoy/http/filter.h" #include "envoy/http/header_map.h" @@ -16,6 +17,7 @@ #include "source/common/buffer/watermark_buffer.h" #include "source/common/common/token_bucket_impl.h" #include "source/common/http/header_utility.h" +#include "source/common/protobuf/protobuf.h" #include "source/common/stats/symbol_table.h" #include "source/extensions/filters/common/fault/fault_config.h" #include "source/extensions/filters/http/common/stream_rate_limiter.h" @@ -74,6 +76,7 @@ class FaultSettings : public Router::RouteSpecificFilterConfig { return response_rate_limit_percent_runtime_; } bool disableDownstreamClusterStats() const { return disable_downstream_cluster_stats_; } + const Envoy::ProtobufWkt::Struct& filterMetadata() const { return filter_metadata_; } private: class RuntimeKeyValues { @@ -106,6 +109,8 @@ class FaultSettings : public Router::RouteSpecificFilterConfig { const std::string max_active_faults_runtime_; const std::string response_rate_limit_percent_runtime_; const bool disable_downstream_cluster_stats_; + + const Envoy::ProtobufWkt::Struct filter_metadata_; }; /** diff --git a/test/extensions/filters/http/ext_proc/BUILD b/test/extensions/filters/http/ext_proc/BUILD index 247a3979fb76..8a77756a3aab 100644 --- a/test/extensions/filters/http/ext_proc/BUILD +++ b/test/extensions/filters/http/ext_proc/BUILD @@ -38,6 +38,7 @@ envoy_extension_cc_test( "//envoy/network:filter_interface", "//source/common/http:context_lib", "//source/common/network:address_lib", + "//source/common/protobuf", "//source/common/stats:isolated_store_lib", "//source/extensions/filters/http/ext_proc", "//test/common/http:common_lib", diff --git a/test/extensions/filters/http/ext_proc/config_test.cc b/test/extensions/filters/http/ext_proc/config_test.cc index bb87b9fb6545..525c278f3b23 100644 --- a/test/extensions/filters/http/ext_proc/config_test.cc +++ b/test/extensions/filters/http/ext_proc/config_test.cc @@ -30,6 +30,8 @@ TEST(HttpExtProcConfigTest, CorrectConfig) { response_body_mode: buffered request_trailer_mode: skip response_trailer_mode: send + filter_metadata: + hello: "world" )EOF"; ExternalProcessingFilterConfig factory; diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc index aec63f731000..45990a18e3cb 100644 --- a/test/extensions/filters/http/ext_proc/filter_test.cc +++ b/test/extensions/filters/http/ext_proc/filter_test.cc @@ -1,5 +1,6 @@ #include +#include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/http/filter.h" #include "envoy/network/connection.h" @@ -9,6 +10,7 @@ #include "source/common/http/conn_manager_impl.h" #include "source/common/http/context_impl.h" #include "source/common/network/address_impl.h" +#include "source/common/protobuf/protobuf.h" #include "source/common/stats/isolated_store_impl.h" #include "source/extensions/filters/http/ext_proc/ext_proc.h" @@ -64,6 +66,7 @@ using testing::Unused; using namespace std::chrono_literals; static const uint32_t BufferSize = 100000; +static const std::string filter_config_name = "scooby.dooby.doo"; // These tests are all unit tests that directly drive an instance of the // ext_proc filter and verify the behavior using mocks. @@ -91,6 +94,7 @@ class HttpFilterTest : public testing::Test { timers_.push_back(timer); return timer; })); + EXPECT_CALL(decoder_callbacks_, filterConfigName()).WillRepeatedly(Return(filter_config_name)); envoy::extensions::filters::http::ext_proc::v3::ExternalProcessor proto_config{}; if (!yaml.empty()) { @@ -274,7 +278,7 @@ class HttpFilterTest : public testing::Test { stream_info_.filterState() ->getDataReadOnly< Envoy::Extensions::HttpFilters::ExternalProcessing::ExtProcLoggingInfo>( - Envoy::Extensions::HttpFilters::ExternalProcessing::ExtProcLoggingInfoName) + filter_config_name) ->grpcCalls(traffic_direction); int calls_count = std::count_if( grpc_calls.begin(), grpc_calls.end(), @@ -282,6 +286,17 @@ class HttpFilterTest : public testing::Test { EXPECT_EQ(calls_count, expected_calls_count); } + // The metadata configured as part of ext_proc filter should be in the filter state. + void expectMetadataInFilterState(const Envoy::ProtobufWkt::Struct& expected_metadata) { + const Envoy::ProtobufWkt::Struct& loggedMetadata = + stream_info_.filterState() + ->getDataReadOnly< + Envoy::Extensions::HttpFilters::ExternalProcessing::ExtProcLoggingInfo>( + filter_config_name) + ->filterMetadata(); + EXPECT_THAT(loggedMetadata, ProtoEq(expected_metadata)); + } + absl::optional final_expected_grpc_service_; std::unique_ptr client_; ExternalProcessorCallbacks* stream_callbacks_ = nullptr; @@ -310,6 +325,8 @@ TEST_F(HttpFilterTest, SimplestPost) { envoy_grpc: cluster_name: "ext_proc_server" failure_mode_allow: true + filter_metadata: + scooby: "doo" )EOF"); EXPECT_TRUE(config_->failureModeAllow()); @@ -365,6 +382,10 @@ TEST_F(HttpFilterTest, SimplestPost) { EXPECT_EQ(1, config_->stats().streams_closed_.value()); expectGrpcCalls(envoy::config::core::v3::TrafficDirection::INBOUND, Grpc::Status::Ok, 1); expectGrpcCalls(envoy::config::core::v3::TrafficDirection::OUTBOUND, Grpc::Status::Ok, 1); + + Envoy::ProtobufWkt::Struct filter_metadata; + (*filter_metadata.mutable_fields())["scooby"].set_string_value("doo"); + expectMetadataInFilterState(filter_metadata); } // Using the default configuration, test the filter with a processor that @@ -510,6 +531,7 @@ TEST_F(HttpFilterTest, PostAndRespondImmediately) { EXPECT_EQ(1, config_->stats().streams_closed_.value()); expectGrpcCalls(envoy::config::core::v3::TrafficDirection::INBOUND, Grpc::Status::Ok, 1); expectGrpcCalls(envoy::config::core::v3::TrafficDirection::OUTBOUND, Grpc::Status::Ok, 0); + expectMetadataInFilterState(Envoy::ProtobufWkt::Struct()); } // Using the default configuration, test the filter with a processor that diff --git a/test/extensions/filters/http/fault/config_test.cc b/test/extensions/filters/http/fault/config_test.cc index 0dcef3419f74..b445c36b43ae 100644 --- a/test/extensions/filters/http/fault/config_test.cc +++ b/test/extensions/filters/http/fault/config_test.cc @@ -28,6 +28,8 @@ TEST(FaultFilterConfigTest, ValidateFail) { TEST(FaultFilterConfigTest, FaultFilterCorrectJson) { const std::string yaml_string = R"EOF( + filter_metadata: + hello: "world" delay: percentage: numerator: 100 diff --git a/test/extensions/filters/http/fault/fault_filter_test.cc b/test/extensions/filters/http/fault/fault_filter_test.cc index 813734b1329f..2a8cac10b808 100644 --- a/test/extensions/filters/http/fault/fault_filter_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_test.cc @@ -62,6 +62,8 @@ class FaultFilterTest : public testing::Test { numerator: 100 denominator: HUNDRED fixed_delay: 5s + filter_metadata: + hello: "world" )EOF"; const std::string fixed_delay_only_disable_stats_yaml = R"EOF( @@ -92,9 +94,13 @@ class FaultFilterTest : public testing::Test { numerator: 100 denominator: HUNDRED http_status: 503 + filter_metadata: + hello: "world" )EOF"; const std::string header_abort_only_yaml = R"EOF( + filter_metadata: + hello: "world" abort: header_abort: {} percentage: @@ -136,6 +142,8 @@ class FaultFilterTest : public testing::Test { filter_ = std::make_unique(config_); filter_->setDecoderFilterCallbacks(decoder_filter_callbacks_); filter_->setEncoderFilterCallbacks(encoder_filter_callbacks_); + ON_CALL(decoder_filter_callbacks_, filterConfigName) + .WillByDefault(Return("envoy.filters.http.fault")); EXPECT_CALL(decoder_filter_callbacks_.dispatcher_, pushTrackedObject(_)).Times(AnyNumber()); EXPECT_CALL(decoder_filter_callbacks_.dispatcher_, popTrackedObject(_)).Times(AnyNumber()); } @@ -230,12 +238,7 @@ TEST(FaultFilterBadConfigTest, MissingDelayDuration) { } TEST_F(FaultFilterTest, AbortWithHttpStatus) { - envoy::extensions::filters::http::fault::v3::HTTPFault fault; - fault.mutable_abort()->mutable_percentage()->set_numerator(100); - fault.mutable_abort()->mutable_percentage()->set_denominator( - envoy::type::v3::FractionalPercent::HUNDRED); - fault.mutable_abort()->set_http_status(429); - setUpTest(fault); + setUpTest(abort_only_yaml); EXPECT_CALL(runtime_.snapshot_, getInteger("fault.http.max_active_faults", std::numeric_limits::max())) @@ -285,6 +288,12 @@ TEST_F(FaultFilterTest, HeaderAbortWithHttpStatus) { request_headers_.addCopy("x-envoy-fault-abort-request", "429"); + envoy::config::core::v3::Metadata dynamic_metadata; + envoy::config::core::v3::Metadata expected_metadata; + auto& filter_metadata = *expected_metadata.mutable_filter_metadata(); + (*filter_metadata["envoy.filters.http.fault"].mutable_fields())["hello"].set_string_value( + "world"); + EXPECT_CALL(runtime_.snapshot_, getInteger("fault.http.max_active_faults", std::numeric_limits::max())) .WillOnce(Return(std::numeric_limits::max())); @@ -303,6 +312,8 @@ TEST_F(FaultFilterTest, HeaderAbortWithHttpStatus) { EXPECT_CALL(runtime_.snapshot_, getInteger("fault.http.abort.http_status", 429)) .WillOnce(Return(429)); + EXPECT_CALL(decoder_filter_callbacks_.stream_info_, dynamicMetadata()) + .WillOnce(ReturnRef(dynamic_metadata)); Http::TestResponseHeaderMapImpl response_headers{ {":status", "429"}, {"content-length", "18"}, {"content-type", "text/plain"}}; @@ -326,6 +337,7 @@ TEST_F(FaultFilterTest, HeaderAbortWithHttpStatus) { EXPECT_EQ(1UL, config_->stats().aborts_injected_.value()); EXPECT_EQ(0UL, config_->stats().active_faults_.value()); EXPECT_EQ("fault_filter_abort", decoder_filter_callbacks_.details()); + EXPECT_THAT(dynamic_metadata, ProtoEq(expected_metadata)); } TEST_F(FaultFilterTest, AbortWithGrpcStatus) { @@ -501,6 +513,7 @@ TEST_F(FaultFilterTest, FixedDelayZeroDuration) { EXPECT_CALL(runtime_.snapshot_, getInteger("fault.http.abort.http_status", _)).Times(0); EXPECT_CALL(decoder_filter_callbacks_, encodeHeaders_(_, _)).Times(0); EXPECT_CALL(decoder_filter_callbacks_.stream_info_, setResponseFlag(_)).Times(0); + EXPECT_CALL(decoder_filter_callbacks_.stream_info_, dynamicMetadata()).Times(0); EXPECT_CALL(decoder_filter_callbacks_, continueDecoding()).Times(0); // Expect filter to continue execution when delay is 0ms @@ -529,7 +542,7 @@ TEST_F(FaultFilterTest, Overflow) { EXPECT_CALL(runtime_.snapshot_, getInteger("fault.http.max_active_faults", std::numeric_limits::max())) .WillOnce(Return(0)); - + EXPECT_CALL(decoder_filter_callbacks_.stream_info_, dynamicMetadata()).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, true)); EXPECT_EQ(0UL, config_->stats().active_faults_.value()); @@ -598,6 +611,12 @@ TEST_F(FaultFilterTest, FixedDelayDeprecatedPercentAndNonZeroDuration) { TEST_F(FaultFilterTest, ConsecutiveDelayFaults) { setUpTest(fixed_delay_only_yaml); + envoy::config::core::v3::Metadata dynamic_metadata; + envoy::config::core::v3::Metadata expected_metadata; + auto& filter_metadata = *expected_metadata.mutable_filter_metadata(); + (*filter_metadata["envoy.filters.http.fault"].mutable_fields())["hello"].set_string_value( + "world"); + // Set the max number of faults to 1. EXPECT_CALL(runtime_.snapshot_, getInteger("fault.http.max_active_faults", std::numeric_limits::max())) @@ -622,6 +641,9 @@ TEST_F(FaultFilterTest, ConsecutiveDelayFaults) { EXPECT_CALL(decoder_filter_callbacks_.stream_info_, setResponseFlag(StreamInfo::ResponseFlag::DelayInjected)) .Times(2); + EXPECT_CALL(decoder_filter_callbacks_.stream_info_, dynamicMetadata()) + .WillOnce(ReturnRef(dynamic_metadata)) + .WillOnce(ReturnRef(dynamic_metadata)); // Start request 1 with a fault delay. EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, @@ -649,6 +671,7 @@ TEST_F(FaultFilterTest, ConsecutiveDelayFaults) { // Should stop counting as active fault after delay elapsed. EXPECT_EQ(0UL, config_->stats().active_faults_.value()); EXPECT_EQ(0UL, config_->stats().aborts_injected_.value()); + EXPECT_THAT(dynamic_metadata, ProtoEq(expected_metadata)); // Prep up request 2, with setups and expectations same as request 1. setUpTest(fixed_delay_only_yaml); @@ -667,6 +690,7 @@ TEST_F(FaultFilterTest, ConsecutiveDelayFaults) { // Have the fault delay of request 2 kick in, which should be delayed with success. timer_->invokeCallback(); + EXPECT_THAT(dynamic_metadata, ProtoEq(expected_metadata)); filter_->onDestroy(); EXPECT_EQ(2UL, config_->stats().delays_injected_.value()); @@ -775,6 +799,15 @@ TEST_F(FaultFilterTest, DelayForDownstreamClusterDisableTracing) { TEST_F(FaultFilterTest, FixedDelayAndAbortDownstream) { setUpTest(fixed_delay_and_abort_yaml); + envoy::config::core::v3::Metadata dynamic_metadata; + ON_CALL(decoder_filter_callbacks_.stream_info_, dynamicMetadata()) + .WillByDefault(ReturnRef(dynamic_metadata)); + + envoy::config::core::v3::Metadata expected_metadata; + auto& filter_metadata = *expected_metadata.mutable_filter_metadata(); + (*filter_metadata["envoy.filters.http.fault"].mutable_fields())["hello"].set_string_value( + "world"); + EXPECT_CALL(runtime_.snapshot_, getInteger("fault.http.max_active_faults", std::numeric_limits::max())) .WillOnce(Return(std::numeric_limits::max())); @@ -829,6 +862,7 @@ TEST_F(FaultFilterTest, FixedDelayAndAbortDownstream) { EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); EXPECT_EQ(1UL, config_->stats().active_faults_.value()); + EXPECT_THAT(dynamic_metadata, ProtoEq(expected_metadata)); filter_->onDestroy(); EXPECT_EQ(1UL, config_->stats().delays_injected_.value()); @@ -1358,8 +1392,13 @@ TEST_F(FaultFilterRateLimitTest, DelayWithResponseRateLimitEnabled) { } TEST_F(FaultFilterRateLimitTest, ResponseRateLimitEnabled) { - setupRateLimitTest(true); + // Set metadata in fault. Ensure that it does not get reflected in stream info. + envoy::extensions::filters::http::fault::v3::HTTPFault fault; + (*fault.mutable_filter_metadata()->mutable_fields())["hello"].set_string_value("world"); + + setupRateLimitTest(fault); + EXPECT_CALL(decoder_filter_callbacks_.stream_info_, dynamicMetadata()).Times(0); ON_CALL(encoder_filter_callbacks_, encoderBufferLimit()).WillByDefault(Return(1100)); Event::MockTimer* token_timer = new NiceMock(&decoder_filter_callbacks_.dispatcher_); diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index c6d5f7f3e412..d05224a06b04 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -267,6 +267,7 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, MOCK_METHOD(Http1StreamEncoderOptionsOptRef, http1StreamEncoderOptions, ()); MOCK_METHOD(OptRef, downstreamCallbacks, ()); MOCK_METHOD(OptRef, upstreamCallbacks, ()); + MOCK_METHOD(absl::string_view, filterConfigName, (), (const override)); // Http::StreamDecoderFilterCallbacks // NOLINTNEXTLINE(readability-identifier-naming) @@ -358,6 +359,7 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, MOCK_METHOD(Http1StreamEncoderOptionsOptRef, http1StreamEncoderOptions, ()); MOCK_METHOD(OptRef, downstreamCallbacks, ()); MOCK_METHOD(OptRef, upstreamCallbacks, ()); + MOCK_METHOD(absl::string_view, filterConfigName, (), (const override)); // Http::StreamEncoderFilterCallbacks MOCK_METHOD(void, addEncodedData, (Buffer::Instance & data, bool streaming)); From 548412cc44c53570bf0a9290f00419b6f70eb400 Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 22 May 2023 07:55:15 +0100 Subject: [PATCH 309/740] ci: Use bazel.yml for docs publishing (#27514) Signed-off-by: Ryan Northey --- .azure-pipelines/stage/publish.yml | 164 ++++++++++++++--------------- 1 file changed, 78 insertions(+), 86 deletions(-) diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index 444e5db454d0..bd74cb6424c7 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -226,8 +226,83 @@ jobs: set -e rm -rf $(Build.StagingDirectory)/.gnupg +- job: docs + displayName: Publish docs + dependsOn: [] + condition: | + and(not(canceled()), + eq(${{ parameters.publishDocs }}, 'true')) + pool: + vmImage: "ubuntu-20.04" + steps: + - template: ../bazel.yml + parameters: + ciTarget: docs + cacheVersion: $(cacheKeyBazel) + publishEnvoy: false + publishTestResults: false + stepsPost: + + - script: | + ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-publish' + condition: | + and(not(canceled()), + eq(${{ parameters.publishDockerhub }}, 'true')) + displayName: "Publish Dockerhub description and README" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_RBE: "1" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} + GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} + DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} + + # Trigger Netlify rebuild of latest docs + - script: | + ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload' + displayName: "Upload Docs to GCS" + condition: | + and(not(canceled()), + eq(${{ parameters.publishDocsLatest }}, 'true')) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_RBE: "1" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} + GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs-publish-latest' + condition: | + and(not(canceled()), + eq(${{ parameters.publishDocsLatest }}, 'true')) + displayName: "Publish latest docs" + workingDirectory: $(Build.SourcesDirectory) + env: + NETLIFY_TRIGGER_URL: ${{ parameters.authNetlifyURL }} + + # Publish docs to the website + - task: InstallSSHKey@0 + condition: | + and(not(canceled()), + eq(${{ parameters.publishDocsRelease }}, 'true')) + inputs: + hostName: $(authGithubSSHKeyPublic) + sshPublicKey: "${{ parameters.authSSHDocsKeyPublic }}" + sshPassphrase: "${{ parameters.authSSHKeyPassphrase }}" + sshKeySecureFile: "${{ parameters.authSSHDocsKey }}" + - script: docs/publish.sh + condition: | + and(not(canceled()), + eq(${{ parameters.publishDocsRelease }}, 'true')) + displayName: "Publish release docs" + workingDirectory: $(Build.SourcesDirectory) + - job: success - dependsOn: ["docker", "package_x64", "package_arm64"] + dependsOn: ["docker", "package_x64", "package_arm64", "docs"] displayName: Success (linux artefacts) pool: vmImage: "ubuntu-20.04" @@ -239,96 +314,13 @@ jobs: and( in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), in(dependencies.package_x64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), - in(dependencies.package_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) + in(dependencies.package_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), + in(dependencies.docs.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) steps: - checkout: none - bash: | echo "building publishable assets complete" -- job: docs - displayName: Publish docs - dependsOn: ["success"] - condition: | - and(not(canceled()), - eq(${{ parameters.publishDocs }}, 'true')) - pool: - vmImage: "ubuntu-20.04" - steps: - - task: Cache@2 - inputs: - key: 'docs | $(cacheKeyBazelFiles)' - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true - - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_RBE: "true" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" - AZP_BRANCH: $(Build.SourceBranch) - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} - displayName: "Generate docs" - - - script: | - ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-publish' - condition: | - and(not(canceled()), - eq(${{ parameters.publishDockerhub }}, 'true')) - displayName: "Publish Dockerhub description and README" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} - GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} - DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} - DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} - - - script: | - ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload' - displayName: "Upload Docs to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} - GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} - condition: eq(variables['isMain'], 'true') - - # Trigger Netlify rebuild of latest docs - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs-publish-latest' - condition: | - and(not(canceled()), - eq(${{ parameters.publishDocsLatest }}, 'true')) - displayName: "Publish latest docs" - workingDirectory: $(Build.SourcesDirectory) - env: - NETLIFY_TRIGGER_URL: ${{ parameters.authNetlifyURL }} - - # Publish docs to the website - - task: InstallSSHKey@0 - condition: | - and(not(canceled()), - eq(${{ parameters.publishDocsRelease }}, 'true')) - inputs: - hostName: $(authGithubSSHKeyPublic) - sshPublicKey: "${{ parameters.authSSHDocsKeyPublic }}" - sshPassphrase: "${{ parameters.authSSHKeyPassphrase }}" - sshKeySecureFile: "${{ parameters.authSSHDocsKey }}" - - script: docs/publish.sh - condition: | - and(not(canceled()), - eq(${{ parameters.publishDocsRelease }}, 'true')) - displayName: "Publish release docs" - workingDirectory: $(Build.SourcesDirectory) - - job: github displayName: Publish release tag dependsOn: ["docs"] From 6de1595999806897913ac77b45781b251208007a Mon Sep 17 00:00:00 2001 From: Xie Zhihao Date: Mon, 22 May 2023 15:42:50 +0800 Subject: [PATCH 310/740] hyperscan: skip compilation on unused libs (#27534) Signed-off-by: Xie Zhihao --- bazel/foreign_cc/hyperscan.patch | 56 +++++++++++++++++-- .../matching/input_matchers/source/BUILD | 2 + 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/bazel/foreign_cc/hyperscan.patch b/bazel/foreign_cc/hyperscan.patch index 95007a0d3e9b..b8d1179f7103 100644 --- a/bazel/foreign_cc/hyperscan.patch +++ b/bazel/foreign_cc/hyperscan.patch @@ -1,9 +1,20 @@ -# No compilation for binaries +# No compilation for unused binaries and libs. diff --git a/CMakeLists.txt b/CMakeLists.txt -index 8bc6077..2ec9a11 100644 +index 7757916..6241f45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -482,9 +482,9 @@ if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS) +@@ -477,8 +477,8 @@ else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + endif() + +-add_subdirectory(util) +-add_subdirectory(doc/dev-reference) ++# add_subdirectory(util) ++# add_subdirectory(doc/dev-reference) + + if (NOT WIN32) + # PCRE check, we have a fixed requirement for PCRE to use Chimera +@@ -496,9 +496,9 @@ if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS) set(BUILD_CHIMERA TRUE) endif() @@ -15,7 +26,7 @@ index 8bc6077..2ec9a11 100644 endif() if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) add_subdirectory(chimera) -@@ -534,9 +534,9 @@ if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS) +@@ -548,9 +548,9 @@ if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS) set(BUILD_CHIMERA TRUE) endif() @@ -27,7 +38,42 @@ index 8bc6077..2ec9a11 100644 endif() if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) add_subdirectory(chimera) -# Workaround for uninitialized use +@@ -1194,8 +1194,8 @@ if (NOT FAT_RUNTIME) + if (BUILD_STATIC_LIBS) + add_library(hs_exec OBJECT ${hs_exec_SRCS}) + +- add_library(hs_runtime STATIC src/hs_version.c src/hs_valid_platform.c $) +- set_target_properties(hs_runtime PROPERTIES LINKER_LANGUAGE C) ++ # add_library(hs_runtime STATIC src/hs_version.c src/hs_valid_platform.c $) ++ # set_target_properties(hs_runtime PROPERTIES LINKER_LANGUAGE C) + + add_library(hs_compile OBJECT ${hs_compile_SRCS}) + +@@ -1271,10 +1271,10 @@ else (FAT_RUNTIME) + # hs_version.c is added explicitly to avoid some build systems that refuse to + # create a lib without any src (I'm looking at you Xcode) + +- add_library(hs_runtime STATIC src/hs_version.c +- $ +- ${RUNTIME_LIBS}) +- set_target_properties(hs_runtime PROPERTIES LINKER_LANGUAGE C) ++ # add_library(hs_runtime STATIC src/hs_version.c ++ # $ ++ # ${RUNTIME_LIBS}) ++ # set_target_properties(hs_runtime PROPERTIES LINKER_LANGUAGE C) + add_library(hs_compile OBJECT ${hs_compile_SRCS}) + + # we want the static lib for testing +@@ -1342,7 +1342,7 @@ else (FAT_RUNTIME) + endif (NOT FAT_RUNTIME) + + if (NOT BUILD_SHARED_LIBS) +- install(TARGETS hs_runtime DESTINATION ${CMAKE_INSTALL_LIBDIR}) ++ # install(TARGETS hs_runtime DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() + + if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS) +# Workaround for uninitialized use. diff --git a/src/fdr/teddy_runtime_common.h b/src/fdr/teddy_runtime_common.h index b76800e..6e587c2 100644 --- a/src/fdr/teddy_runtime_common.h diff --git a/contrib/hyperscan/matching/input_matchers/source/BUILD b/contrib/hyperscan/matching/input_matchers/source/BUILD index 37f66ae582d2..5342badb28d1 100644 --- a/contrib/hyperscan/matching/input_matchers/source/BUILD +++ b/contrib/hyperscan/matching/input_matchers/source/BUILD @@ -26,10 +26,12 @@ envoy_cmake( "BUILD_AVX512": "on", "BUILD_AVX512VBMI": "on", "BUILD_EXAMPLES": "off", + "CMAKE_BUILD_TYPE": "Release", "CMAKE_INSTALL_LIBDIR": "lib", "FAT_RUNTIME": "on", "RAGEL": "$EXT_BUILD_DEPS/ragel/bin/ragel", }, + default_cache_entries = {}, lib_source = "@io_hyperscan//:all", out_static_libs = ["libhs.a"], tags = ["skip_on_windows"], From 3f1e91e3f96b143263b0ae2fa8cc73fdb551baad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 07:56:46 +0000 Subject: [PATCH 311/740] build(deps): bump envoy-base-utils from 0.4.7 to 0.4.10 in /tools/base (#27537) Bumps [envoy-base-utils](https://github.com/envoyproxy/toolshed) from 0.4.7 to 0.4.10. - [Release notes](https://github.com/envoyproxy/toolshed/releases) - [Commits](https://github.com/envoyproxy/toolshed/commits) --- updated-dependencies: - dependency-name: envoy-base-utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 4025f63386ce..5d6ffba67662 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -25,9 +25,9 @@ aio-api-bazel==0.0.2 \ --hash=sha256:56e36463d236e477b7e282f2d870185a0b978b50e2c3803c1ebf8b8ac4b18f5b \ --hash=sha256:d3f563b7698e874437d80538a89dd4d79bc37de2e850c846330ae456e3f21dcc # via -r requirements.in -aio-api-github==0.2.0 \ - --hash=sha256:09e7425b5e1eb46118ed2442c2d2d830eba6e2b6ae6c01ec1c6329ec900735c8 \ - --hash=sha256:d2d42d8c834baf4740d036fe37ef7dd66abe33f9c51d1ee1daa601185ec98b64 +aio-api-github==0.2.2 \ + --hash=sha256:507971a3a90155066a072d24fc40b3890f69b2f671a42c6325370a48cbbd0e06 \ + --hash=sha256:5e67bb397362ad0ccfd6b6680c94a995fdc8fb8cd795b5074dd29c5f875a9d62 # via # envoy-base-utils # envoy-dependency-check @@ -420,9 +420,9 @@ docutils==0.19 \ # via # envoy-docs-sphinx-runner # sphinx -envoy-base-utils==0.4.7 \ - --hash=sha256:1726809d5edbed6093c0fd779fe189d6ddc0491f2bfddf84b929deb0795292f0 \ - --hash=sha256:af918b0d9db86c1e907d93df7996b036e5bfc212fe7e491daa793dba5795c7e4 +envoy-base-utils==0.4.10 \ + --hash=sha256:1cfccb54a3831c5b5a833efdad67752c1ae0a8708532712842726de42c0c413a \ + --hash=sha256:9567d1317a9ed7e97730a664e3105d8a6d46a4fa9e5cc7256951f5564783d8ae # via # -r requirements.in # envoy-code-check From 70879eaa00d976ee91b87f5a14edafa80e0cdb0b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 09:02:11 +0100 Subject: [PATCH 312/740] build(deps): bump setuptools from 67.7.2 to 67.8.0 in /tools/base (#27536) Bumps [setuptools](https://github.com/pypa/setuptools) from 67.7.2 to 67.8.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/CHANGES.rst) - [Commits](https://github.com/pypa/setuptools/compare/v67.7.2...v67.8.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 5d6ffba67662..ee3068fd0a65 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -1523,9 +1523,9 @@ zstandard==0.21.0 \ # via envoy-base-utils # The following packages are considered to be unsafe in a requirements file: -setuptools==67.7.2 \ - --hash=sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b \ - --hash=sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990 +setuptools==67.8.0 \ + --hash=sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f \ + --hash=sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102 # via # -r requirements.in # sphinxcontrib-jquery From 9db3616fafd7ff593338c2c751daf471b911c0d3 Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 22 May 2023 12:33:00 +0100 Subject: [PATCH 313/740] deps: Bump `prometheus_metrics_model` -> 0.4.0 (#27481) Signed-off-by: Ryan Northey --- api/bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index f6fd434d84ab..45015ca109f1 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -92,11 +92,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Prometheus client model", project_desc = "Data model artifacts for Prometheus", project_url = "https://github.com/prometheus/client_model", - version = "147c58e9608a4f9628b53b6cc863325ca746f63a", - sha256 = "f7da30879dcdfae367fa65af1969945c3148cfbfc462b30b7d36f17134675047", - release_date = "2021-06-07", + version = "0.4.0", + sha256 = "82fc41d9481476a778b120d4553e9e4edf06cc4efd52ee09ba000933d3a2a53d", + release_date = "2023-05-03", strip_prefix = "client_model-{version}", - urls = ["https://github.com/prometheus/client_model/archive/{version}.tar.gz"], + urls = ["https://github.com/prometheus/client_model/archive/v{version}.tar.gz"], use_category = ["api"], license = "Apache-2.0", license_url = "https://github.com/prometheus/client_model/blob/{version}/LICENSE", From cf9950bae4c9e5195c0419d8dab08ef085c0c78b Mon Sep 17 00:00:00 2001 From: Raven Black Date: Mon, 22 May 2023 05:50:11 -0700 Subject: [PATCH 314/740] Fix file_system_buffer flake (#27519) Signed-off-by: Raven Black --- .../filters/http/file_system_buffer/filter.cc | 14 +++++++------- .../filters/http/file_system_buffer/filter.h | 9 +++++++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/source/extensions/filters/http/file_system_buffer/filter.cc b/source/extensions/filters/http/file_system_buffer/filter.cc index c3270085ce27..c88198ad9173 100644 --- a/source/extensions/filters/http/file_system_buffer/filter.cc +++ b/source/extensions/filters/http/file_system_buffer/filter.cc @@ -228,6 +228,7 @@ void FileSystemBufferFilter::setEncoderFilterCallbacks( } void FileSystemBufferFilter::onDestroy() { + *is_destroyed_ = true; if (cancel_in_flight_async_action_) { cancel_in_flight_async_action_(); } @@ -400,11 +401,10 @@ bool FileSystemBufferFilter::maybeStorage(BufferedStreamState& state, // was deleted before the callback, not just do nothing. cancel_in_flight_async_action_ = config_->asyncFileManager().createAnonymousFile( config_->storageBufferPath(), - [this, me = std::weak_ptr(shared_from_this()), - dispatcher = &request_callbacks_->dispatcher(), + [this, is_destroyed = is_destroyed_, dispatcher = &request_callbacks_->dispatcher(), &state](absl::StatusOr file_handle) { - dispatcher->post([this, me = std::move(me), &state, file_handle]() { - if (!me.lock()) { + dispatcher->post([this, is_destroyed, &state, file_handle]() { + if (*is_destroyed) { // If we opened a file but the filter went away in the meantime, close the file // to avoid leaving a dangling file handle. if (file_handle.ok()) { @@ -456,10 +456,10 @@ std::function FileSystemBufferFilter::getOnFileActionComplet } std::function)> FileSystemBufferFilter::getSafeDispatch() { - return [me = std::weak_ptr(shared_from_this()), + return [is_destroyed = is_destroyed_, dispatcher = &request_callbacks_->dispatcher()](std::function callback) { - dispatcher->post([me = std::move(me), callback = std::move(callback)]() { - if (me.lock()) { + dispatcher->post([is_destroyed, callback = std::move(callback)]() { + if (!*is_destroyed) { callback(); } }); diff --git a/source/extensions/filters/http/file_system_buffer/filter.h b/source/extensions/filters/http/file_system_buffer/filter.h index a6a611ed9ede..02dfd1b1e101 100644 --- a/source/extensions/filters/http/file_system_buffer/filter.h +++ b/source/extensions/filters/http/file_system_buffer/filter.h @@ -41,8 +41,7 @@ struct BufferedStreamState { class FileSystemBufferFilter : public Http::StreamFilter, public Http::DownstreamWatermarkCallbacks, - public Logger::Loggable, - public std::enable_shared_from_this { + public Logger::Loggable { public: explicit FileSystemBufferFilter(std::shared_ptr base_config); @@ -67,6 +66,12 @@ class FileSystemBufferFilter : public Http::StreamFilter, void onDestroy() override; private: + // This is captured so that callbacks can alter their behavior (and avoid nullptr) + // if the filter has been destroyed since the callback was queued. + // We use this rather than shared_from_this because onDestroy happens potentially + // significantly earlier than the destructor, and we want to abort callbacks after + // onDestroy. + std::shared_ptr is_destroyed_ = std::make_shared(false); // Merges any per-route config with the default config. Returns false if the config lacked // a file manager. bool initPerRouteConfig(); From 9c6930fe04a46f8ba485c922cecaaaa0bc76b55c Mon Sep 17 00:00:00 2001 From: pianiststickman <34144687+pianiststickman@users.noreply.github.com> Date: Mon, 22 May 2023 10:03:07 -0400 Subject: [PATCH 315/740] socket: user space io handle invokes drain trackers immediately upon write (#27499) * Fix use-after-free in user space IO handle where a drain callback may refer to a freed object (i.e. the connection on the other side of the internal listener, which may already have died). The solution employed herein is to call and clear the drain trackers on write(), since - as far as the handle being written to is concerned - we don't care about the write buffer as soon as it has been written to the peer's IO handle. Signed-off-by: Eugene Chan --- .../network/test/postgres_decoder_test.cc | 1 + envoy/buffer/buffer.h | 12 ++++++++++++ source/common/buffer/buffer_impl.cc | 10 ++++++++++ source/common/buffer/buffer_impl.h | 1 + source/common/buffer/watermark_buffer.cc | 6 ++++++ source/common/buffer/watermark_buffer.h | 1 + .../io_socket/user_space/io_handle_impl.cc | 2 +- test/common/buffer/buffer_fuzz.cc | 4 +++- .../io_socket/user_space/io_handle_impl_test.cc | 17 +++++++++++++++++ test/mocks/buffer/mocks.h | 2 ++ 10 files changed, 54 insertions(+), 2 deletions(-) diff --git a/contrib/postgres_proxy/filters/network/test/postgres_decoder_test.cc b/contrib/postgres_proxy/filters/network/test/postgres_decoder_test.cc index 5700839e915b..988779c1338c 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_decoder_test.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_decoder_test.cc @@ -691,6 +691,7 @@ class FakeBuffer : public Buffer::Instance { MOCK_METHOD(void*, linearize, (uint32_t), (override)); MOCK_METHOD(void, move, (Instance&), (override)); MOCK_METHOD(void, move, (Instance&, uint64_t), (override)); + MOCK_METHOD(void, move, (Instance&, uint64_t, bool), (override)); MOCK_METHOD(Buffer::Reservation, reserveForRead, (), (override)); MOCK_METHOD(Buffer::ReservationSingleSlice, reserveSingleSlice, (uint64_t, bool), (override)); MOCK_METHOD(void, commit, diff --git a/envoy/buffer/buffer.h b/envoy/buffer/buffer.h index b5233746e364..b94fef17fe50 100644 --- a/envoy/buffer/buffer.h +++ b/envoy/buffer/buffer.h @@ -276,6 +276,18 @@ class Instance { */ virtual void move(Instance& rhs, uint64_t length) PURE; + /** + * Move a portion of a buffer into this buffer. If reset_drain_trackers_and_accounting is true, + * then any drain trackers on the source buffer are also called and cleared so that the + * connection originating the source buffer (e.g. an internal listener connection) may be deleted + * without causing a use-after-free. + * @param rhs supplies the buffer to move. + * @param length supplies the amount of data to move. + * @param reset_drain_trackers_and_accounting whether the drain trackers on the source buffers + * should be cleared, so that the source buffer is deletable. + */ + virtual void move(Instance& rhs, uint64_t length, bool reset_drain_trackers_and_accounting) PURE; + /** * Reserve space in the buffer for reading into. The amount of space reserved is determined * based on buffer settings and performance considerations. diff --git a/source/common/buffer/buffer_impl.cc b/source/common/buffer/buffer_impl.cc index 5744c4ef4696..fc62e061303c 100644 --- a/source/common/buffer/buffer_impl.cc +++ b/source/common/buffer/buffer_impl.cc @@ -325,6 +325,10 @@ void OwnedImpl::move(Instance& rhs) { } void OwnedImpl::move(Instance& rhs, uint64_t length) { + move(rhs, length, /*reset_drain_trackers_and_accounting=*/false); +} + +void OwnedImpl::move(Instance& rhs, uint64_t length, bool reset_drain_trackers_and_accounting) { ASSERT(&rhs != this); // See move() above for why we do the static cast. OwnedImpl& other = static_cast(rhs); @@ -340,6 +344,12 @@ void OwnedImpl::move(Instance& rhs, uint64_t length) { other.slices_.front().drain(copy_size); other.length_ -= copy_size; } else { + if (reset_drain_trackers_and_accounting) { + // The other slice is owned by a user-space IO handle and its drain trackers may refer to a + // connection that can die (and be freed) at any time. Call and clear the drain trackers to + // avoid potential use-after-free. + other.slices_.front().callAndClearDrainTrackersAndCharges(); + } coalesceOrAddSlice(std::move(other.slices_.front())); other.slices_.pop_front(); other.length_ -= slice_size; diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 301f78e071d8..058657dc5abd 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -669,6 +669,7 @@ class OwnedImpl : public LibEventInstance { void* linearize(uint32_t size) override; void move(Instance& rhs) override; void move(Instance& rhs, uint64_t length) override; + void move(Instance& rhs, uint64_t length, bool reset_drain_trackers_and_accounting) override; Reservation reserveForRead() override; ReservationSingleSlice reserveSingleSlice(uint64_t length, bool separate_slice = false) override; ssize_t search(const void* data, uint64_t size, size_t start, size_t length) const override; diff --git a/source/common/buffer/watermark_buffer.cc b/source/common/buffer/watermark_buffer.cc index 3292d6c08f14..f7e3297baebc 100644 --- a/source/common/buffer/watermark_buffer.cc +++ b/source/common/buffer/watermark_buffer.cc @@ -67,6 +67,12 @@ void WatermarkBuffer::move(Instance& rhs, uint64_t length) { checkHighAndOverflowWatermarks(); } +void WatermarkBuffer::move(Instance& rhs, uint64_t length, + bool reset_drain_trackers_and_accounting) { + OwnedImpl::move(rhs, length, reset_drain_trackers_and_accounting); + checkHighAndOverflowWatermarks(); +} + SliceDataPtr WatermarkBuffer::extractMutableFrontSlice() { auto result = OwnedImpl::extractMutableFrontSlice(); checkLowWatermark(); diff --git a/source/common/buffer/watermark_buffer.h b/source/common/buffer/watermark_buffer.h index c9efe00b201f..c9de30551a4c 100644 --- a/source/common/buffer/watermark_buffer.h +++ b/source/common/buffer/watermark_buffer.h @@ -38,6 +38,7 @@ class WatermarkBuffer : public OwnedImpl { void drain(uint64_t size) override; void move(Instance& rhs) override; void move(Instance& rhs, uint64_t length) override; + void move(Instance& rhs, uint64_t length, bool reset_drain_trackers_and_accounting) override; SliceDataPtr extractMutableFrontSlice() override; Reservation reserveForRead() override; void postProcess() override { checkLowWatermark(); } diff --git a/source/extensions/io_socket/user_space/io_handle_impl.cc b/source/extensions/io_socket/user_space/io_handle_impl.cc index 959ab9735d81..8795512de3ff 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.cc +++ b/source/extensions/io_socket/user_space/io_handle_impl.cc @@ -43,7 +43,7 @@ uint64_t moveUpTo(Buffer::Instance& dst, Buffer::Instance& src, uint64_t max_len } } uint64_t res = std::min(max_length, src.length()); - dst.move(src, res); + dst.move(src, res, /*reset_drain_trackers_and_accounting=*/true); return res; } } // namespace diff --git a/test/common/buffer/buffer_fuzz.cc b/test/common/buffer/buffer_fuzz.cc index ce9ffc64cbbb..53d9aebe7f44 100644 --- a/test/common/buffer/buffer_fuzz.cc +++ b/test/common/buffer/buffer_fuzz.cc @@ -156,7 +156,9 @@ class StringBuffer : public Buffer::Instance { void move(Buffer::Instance& rhs) override { move(rhs, rhs.length()); } - void move(Buffer::Instance& rhs, uint64_t length) override { + void move(Buffer::Instance& rhs, uint64_t length) override { move(rhs, length, false); } + + void move(Buffer::Instance& rhs, uint64_t length, bool) override { StringBuffer& src = dynamic_cast(rhs); add(src.start(), length); src.start_ += length; diff --git a/test/extensions/io_socket/user_space/io_handle_impl_test.cc b/test/extensions/io_socket/user_space/io_handle_impl_test.cc index 01d441f0f778..de81cb2a4d32 100644 --- a/test/extensions/io_socket/user_space/io_handle_impl_test.cc +++ b/test/extensions/io_socket/user_space/io_handle_impl_test.cc @@ -184,6 +184,23 @@ TEST_F(IoHandleImplTest, ReadContent) { ASSERT_EQ(0, io_handle_->getWriteBuffer()->length()); } +TEST_F(IoHandleImplTest, WriteClearsDrainTrackers) { + Buffer::OwnedImpl buf_to_write("abcdefg"); + { + bool called = false; + // This drain tracker should be called as soon as the write happens; not on read. + buf_to_write.addDrainTracker([&called]() { called = true; }); + io_handle_peer_->write(buf_to_write); + EXPECT_TRUE(called); + } + // Now the drain tracker refers to a stack variable that no longer exists. If the drain tracker + // is called subsequently, this will fail in ASan. + Buffer::OwnedImpl buf; + auto result = io_handle_->read(buf, 10); + ASSERT_TRUE(result.ok()); + ASSERT_EQ(7, result.return_value_); +} + // Test read throttling on watermark buffer. TEST_F(IoHandleImplTest, ReadThrottling) { { diff --git a/test/mocks/buffer/mocks.h b/test/mocks/buffer/mocks.h index 20adb026f6e5..768decce37ce 100644 --- a/test/mocks/buffer/mocks.h +++ b/test/mocks/buffer/mocks.h @@ -24,6 +24,8 @@ template class MockBufferBase : public BaseClass { MOCK_METHOD(void, move, (Buffer::Instance & rhs)); MOCK_METHOD(void, move, (Buffer::Instance & rhs, uint64_t length)); + MOCK_METHOD(void, move, + (Buffer::Instance & rhs, uint64_t length, bool reset_drain_trackers_and_accounting)); MOCK_METHOD(void, drain, (uint64_t size)); void baseMove(Buffer::Instance& rhs) { BaseClass::move(rhs); } From 9d9f6afa56286a13d415ea2f1c582d367d9fc3fb Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 22 May 2023 16:26:02 +0100 Subject: [PATCH 316/740] ci/mobile: Improve `swift_async_await` timeout (#27483) Signed-off-by: Ryan Northey --- .github/workflows/mobile-ios_build.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mobile-ios_build.yml b/.github/workflows/mobile-ios_build.yml index dc824daea5dc..fd3601c1cc9b 100644 --- a/.github/workflows/mobile-ios_build.yml +++ b/.github/workflows/mobile-ios_build.yml @@ -189,7 +189,7 @@ jobs: name: swift_async_await needs: iosbuild runs-on: macos-12 - timeout-minutes: 120 + timeout-minutes: 50 steps: - uses: actions/checkout@v3 with: @@ -226,7 +226,23 @@ jobs: --config=ios \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ //examples/swift/async_await:app &> /tmp/envoy.log & - - run: sed '/\[2\] Uploaded 7 MB of data/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) + - run: | + checklogs () { + sed '/\[2\] Uploaded 7 MB of data/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) + } + export -f checklogs + # TODO(phlax): figure if this needs this long + timeout 5m bash -c checklogs || { + retcode=$? + if [[ "$retcode" != 124 ]]; then + echo "Command failed" >&2 + elif grep -q "Upload failed" /tmp/envoy.log; then + echo "Upload failed" >&2 + else + echo "Upload timed out" >&2 + fi + exit 1 + } if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check upload succeeded' - run: cat /tmp/envoy.log From 00402e2d11faa830811e0fb5f941240dd8e80116 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Mon, 22 May 2023 19:42:51 +0300 Subject: [PATCH 317/740] router: Do not set SNI or SAN due to auto_sni or auto_san if already set (#25800) Signed-off-by: Jarno Rajahalme --- api/envoy/config/core/v3/protocol.proto | 3 ++ changelogs/current.yaml | 4 ++ source/common/router/router.cc | 8 ++- test/common/router/router_test.cc | 66 ++++++++++++++++++------- 4 files changed, 61 insertions(+), 20 deletions(-) diff --git a/api/envoy/config/core/v3/protocol.proto b/api/envoy/config/core/v3/protocol.proto index 95670bb752b4..17fe88311eae 100644 --- a/api/envoy/config/core/v3/protocol.proto +++ b/api/envoy/config/core/v3/protocol.proto @@ -104,12 +104,14 @@ message UpstreamHttpProtocolOptions { // upstream connections based on the downstream HTTP host/authority header or any other arbitrary // header when :ref:`override_auto_sni_header ` // is set, as seen by the :ref:`router filter `. + // Does nothing if a filter before the http router filter sets the corresponding metadata. bool auto_sni = 1; // Automatic validate upstream presented certificate for new upstream connections based on the // downstream HTTP host/authority header or any other arbitrary header when :ref:`override_auto_sni_header ` // is set, as seen by the :ref:`router filter `. // This field is intended to be set with ``auto_sni`` field. + // Does nothing if a filter before the http router filter sets the corresponding metadata. bool auto_san_validation = 2; // An optional alternative to the host/authority header to be used for setting the SNI value. @@ -119,6 +121,7 @@ message UpstreamHttpProtocolOptions { // is not found or the value is empty, host/authority header will be used instead. // This field is intended to be set with ``auto_sni`` and/or ``auto_san_validation`` fields. // If none of these fields are set then setting this would be a no-op. + // Does nothing if a filter before the http router filter sets the corresponding metadata. string override_auto_sni_header = 3 [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 1ff740abec2e..1e8249a02621 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -60,6 +60,10 @@ minor_behavior_changes: the cookie is still accepted, but is planned to be obsoleted in the future. This behavior change can be reverted by setting ``envoy.reloadable_features.stateful_session_encode_ttl_in_cookie`` to ``false``. +- area: router + change: | + Added check for existing metadata before setting metadata due to 'auto_sni', 'auto_san_validation', or + 'override_auto_sni_header' to prevent triggering ENVOY_BUG when an earlier filter has set the metadata. - area: resource_monitors change: | Changed behavior of the fixed heap monitor to count unused mapped pages as diff --git a/source/common/router/router.cc b/source/common/router/router.cc index 9b9de47bcd24..b03ea8b89a5a 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -547,14 +547,18 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, // `host_` returns a string_view so doing this should be safe. absl::string_view sni_value = parsed_authority.host_; - if (should_set_sni && upstream_http_protocol_options.value().auto_sni()) { + if (should_set_sni && upstream_http_protocol_options.value().auto_sni() && + !callbacks_->streamInfo().filterState()->hasDataWithName( + Network::UpstreamServerName::key())) { callbacks_->streamInfo().filterState()->setData( Network::UpstreamServerName::key(), std::make_unique(sni_value), StreamInfo::FilterState::StateType::Mutable); } - if (upstream_http_protocol_options.value().auto_san_validation()) { + if (upstream_http_protocol_options.value().auto_san_validation() && + !callbacks_->streamInfo().filterState()->hasDataWithName( + Network::UpstreamSubjectAltNames::key())) { callbacks_->streamInfo().filterState()->setData( Network::UpstreamSubjectAltNames::key(), std::make_unique( diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 8291e357148e..de30e05444be 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -162,10 +162,15 @@ class RouterTest : public RouterTestBase { router_->onDestroy(); } + // testAutoSniOptions checks that UpstreamServerName is server_name if non-empty, + // and that the first UpstreamSubjectAltNames is alt_server_name if non-empty. + // UpstreamServerName is pre-set to pre_set_sni if non-empty. void testAutoSniOptions( absl::optional dummy_option, - Envoy::Http::TestRequestHeaderMapImpl headers, std::string server_name = "host", - bool should_validate_san = false, std::string alt_server_name = "host") { + Envoy::Http::TestRequestHeaderMapImpl headers, std::string server_name = "", + std::string alt_server_name = "", std::string pre_set_sni = "", + StreamInfo::FilterState::LifeSpan pre_set_life_span = + StreamInfo::FilterState::LifeSpan::FilterChain) { NiceMock stream_info; ON_CALL(*cm_.thread_local_cluster_.cluster_.info_, upstreamHttpProtocolOptions()) .WillByDefault(ReturnRef(dummy_option)); @@ -173,18 +178,26 @@ class RouterTest : public RouterTestBase { .WillByDefault(ReturnRef(stream_info.filterState())); EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) .WillOnce(Return(&cancellable_)); - stream_info.filterState()->setData(Network::UpstreamServerName::key(), - std::make_unique("dummy"), - StreamInfo::FilterState::StateType::Mutable); + + if (!pre_set_sni.empty()) { + // Simulate a network filter setting the server name, e.g. based on SNI seen by the + // tls_inspector by using the LifeSpan::Connection + stream_info.filterState()->setData(Network::UpstreamServerName::key(), + std::make_unique(pre_set_sni), + StreamInfo::FilterState::StateType::Mutable, + pre_set_life_span); + } expectResponseTimerCreate(); HttpTestUtility::addDefaultHeaders(headers); router_->decodeHeaders(headers, true); - EXPECT_EQ(server_name, - stream_info.filterState() - ->getDataReadOnly(Network::UpstreamServerName::key()) - ->value()); - if (should_validate_san) { + if (!server_name.empty()) { + EXPECT_EQ(server_name, stream_info.filterState() + ->getDataReadOnly( + Network::UpstreamServerName::key()) + ->value()); + } + if (!alt_server_name.empty()) { EXPECT_EQ(alt_server_name, stream_info.filterState() ->getDataReadOnly( Network::UpstreamSubjectAltNames::key()) @@ -205,7 +218,24 @@ TEST_F(RouterTest, UpdateServerNameFilterStateWithoutHeaderOverride) { dummy_option.value().set_auto_sni(true); Http::TestRequestHeaderMapImpl headers{}; - testAutoSniOptions(dummy_option, headers); + testAutoSniOptions(dummy_option, headers, "host"); +} + +TEST_F(RouterTest, DontUpdateServerNameFilterStateWhenExists) { + auto dummy_option = absl::make_optional(); + dummy_option.value().set_auto_sni(true); + + Http::TestRequestHeaderMapImpl headers{}; + testAutoSniOptions(dummy_option, headers, "old-host", "", "old-host"); +} + +TEST_F(RouterTest, DontUpdateServerNameFilterStateWhenExistsConnectionLifeSpan) { + auto dummy_option = absl::make_optional(); + dummy_option.value().set_auto_sni(true); + + Http::TestRequestHeaderMapImpl headers{}; + testAutoSniOptions(dummy_option, headers, "old-host", "", "old-host", + StreamInfo::FilterState::LifeSpan::Connection); } TEST_F(RouterTest, UpdateServerNameFilterStateWithHostHeaderOverride) { @@ -214,7 +244,7 @@ TEST_F(RouterTest, UpdateServerNameFilterStateWithHostHeaderOverride) { dummy_option.value().set_override_auto_sni_header(":authority"); Http::TestRequestHeaderMapImpl headers{}; - testAutoSniOptions(dummy_option, headers); + testAutoSniOptions(dummy_option, headers, "host"); } TEST_F(RouterTest, UpdateServerNameFilterStateWithHeaderOverride) { @@ -233,7 +263,7 @@ TEST_F(RouterTest, UpdateServerNameFilterStateWithEmptyValueHeaderOverride) { dummy_option.value().set_override_auto_sni_header("x-host"); Http::TestRequestHeaderMapImpl headers{{"x-host", ""}}; - testAutoSniOptions(dummy_option, headers); + testAutoSniOptions(dummy_option, headers, "host"); } TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithoutHeaderOverride) { @@ -242,7 +272,7 @@ TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithoutHeaderOverride) { dummy_option.value().set_auto_san_validation(true); Http::TestRequestHeaderMapImpl headers{}; - testAutoSniOptions(dummy_option, headers, "host", true); + testAutoSniOptions(dummy_option, headers, "host", "host"); } TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithHostHeaderOverride) { @@ -252,7 +282,7 @@ TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithHostHeaderOverride) { dummy_option.value().set_override_auto_sni_header(":authority"); Http::TestRequestHeaderMapImpl headers{}; - testAutoSniOptions(dummy_option, headers, "host", true); + testAutoSniOptions(dummy_option, headers, "host", "host"); } TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithHeaderOverride) { @@ -263,7 +293,7 @@ TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithHeaderOverride) { const auto server_name = "foo.bar"; Http::TestRequestHeaderMapImpl headers{{"x-host", server_name}}; - testAutoSniOptions(dummy_option, headers, server_name, true, server_name); + testAutoSniOptions(dummy_option, headers, server_name, server_name); } TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithEmptyValueHeaderOverride) { @@ -273,7 +303,7 @@ TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithEmptyValueHeaderOverride) dummy_option.value().set_override_auto_sni_header("x-host"); Http::TestRequestHeaderMapImpl headers{{"x-host", ""}}; - testAutoSniOptions(dummy_option, headers, "host", true); + testAutoSniOptions(dummy_option, headers, "host", "host"); } TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithIpHeaderOverride) { @@ -284,7 +314,7 @@ TEST_F(RouterTest, UpdateSubjectAltNamesFilterStateWithIpHeaderOverride) { const auto server_name = "127.0.0.1"; Http::TestRequestHeaderMapImpl headers{{"x-host", server_name}}; - testAutoSniOptions(dummy_option, headers, "dummy", true, server_name); + testAutoSniOptions(dummy_option, headers, "", server_name); } TEST_F(RouterTest, RouteNotFound) { From fb61801c03c544968606e97303439ebcade240bd Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 22 May 2023 14:15:04 -0400 Subject: [PATCH 318/740] runtime: e2e testing circuit breaker controls (#27544) Signed-off-by: Alyssa Wilk --- .../circuit_breakers_integration_test.cc | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/integration/circuit_breakers_integration_test.cc b/test/integration/circuit_breakers_integration_test.cc index 695b743362ca..b0a7b5409d39 100644 --- a/test/integration/circuit_breakers_integration_test.cc +++ b/test/integration/circuit_breakers_integration_test.cc @@ -66,5 +66,67 @@ TEST_P(CircuitBreakersIntegrationTest, CircuitBreakersWithOutlierDetection) { 0); } +TEST_P(CircuitBreakersIntegrationTest, CircuitBreakerRuntime) { + config_helper_.addRuntimeOverride("circuit_breakers.cluster_0.default.max_requests", "0"); + config_helper_.addRuntimeOverride("circuit_breakers.cluster_0.default.max_retries", "1024"); + + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* static_resources = bootstrap.mutable_static_resources(); + auto* cluster = static_resources->mutable_clusters(0); + + auto* outlier_detection = cluster->mutable_outlier_detection(); + outlier_detection->mutable_consecutive_gateway_failure()->set_value(1); + outlier_detection->mutable_consecutive_5xx()->set_value(1); + outlier_detection->mutable_consecutive_local_origin_failure()->set_value(1); + + outlier_detection->mutable_enforcing_consecutive_gateway_failure()->set_value(100); + outlier_detection->mutable_enforcing_consecutive_5xx()->set_value(100); + outlier_detection->mutable_enforcing_consecutive_local_origin_failure()->set_value(100); + + outlier_detection->set_split_external_local_origin_errors(true); + + outlier_detection->mutable_max_ejection_percent()->set_value(100); + outlier_detection->mutable_interval()->set_nanos(1); + outlier_detection->mutable_base_ejection_time()->set_seconds(3600); + outlier_detection->mutable_max_ejection_time()->set_seconds(3600); + }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeRequestWithBody(default_request_headers_, 1024); + + test_server_->waitForGaugeEq("cluster.cluster_0.upstream_rq_active", 0); + test_server_->waitForGaugeEq("cluster.cluster_0.upstream_rq_pending_active", 0); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_EQ("503", response->headers().getStatusValue()); + test_server_->waitForCounterGe("cluster.cluster_0.upstream_rq_503", 1); + + EXPECT_EQ(test_server_->counter("cluster.cluster_0.upstream_rq_pending_overflow")->value(), 1); + + EXPECT_EQ(test_server_->counter("cluster.cluster_0.outlier_detection.ejections_enforced_total") + ->value(), + 0); +#ifdef ENVOY_ADMIN_FUNCTIONALITY + auto codec_client2 = makeHttpConnection(lookupPort("admin")); + default_request_headers_.setPath("/runtime"); + response = codec_client2->makeHeaderOnlyRequest(default_request_headers_); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("200", response->headers().getStatusValue()); + codec_client2->close(); + + const std::string expected_json1 = R"EOF( + "circuit_breakers.cluster_0.default.max_retries": { +)EOF"; + EXPECT_TRUE(absl::StrContains(response->body(), expected_json1)); + + const std::string expected_json2 = R"EOF("final_value": "1024")EOF"; + EXPECT_TRUE(absl::StrContains(response->body(), expected_json2)); +#endif +} + } // namespace } // namespace Envoy From 29b62b8013587bdeb0849d97a4bd0bd555ec50e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 08:51:36 +0100 Subject: [PATCH 319/740] build(deps): bump requests from 2.25.1 to 2.31.0 in /.github/actions/pr_notifier (#27552) build(deps): bump requests in /.github/actions/pr_notifier Bumps [requests](https://github.com/psf/requests) from 2.25.1 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.25.1...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/pr_notifier/requirements.txt | 85 ++++++++++++++++++-- 1 file changed, 79 insertions(+), 6 deletions(-) diff --git a/.github/actions/pr_notifier/requirements.txt b/.github/actions/pr_notifier/requirements.txt index 1f2fb83fc37f..f241eeaa3fc0 100644 --- a/.github/actions/pr_notifier/requirements.txt +++ b/.github/actions/pr_notifier/requirements.txt @@ -61,9 +61,82 @@ cffi==1.14.5 \ # via # cryptography # pynacl -chardet==4.0.0 \ - --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ - --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 +charset-normalizer==3.1.0 \ + --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ + --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ + --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ + --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ + --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ + --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ + --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ + --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ + --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ + --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ + --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ + --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ + --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ + --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ + --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ + --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ + --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ + --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ + --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ + --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ + --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ + --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ + --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ + --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ + --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ + --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ + --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ + --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ + --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ + --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ + --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ + --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ + --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ + --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ + --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ + --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ + --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ + --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ + --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ + --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ + --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ + --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ + --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ + --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ + --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ + --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ + --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ + --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ + --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ + --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ + --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ + --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ + --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ + --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ + --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ + --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ + --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ + --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ + --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ + --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ + --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ + --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ + --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ + --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ + --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ + --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ + --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ + --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ + --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ + --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ + --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ + --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ + --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ + --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ + --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab # via requests cryptography==39.0.2 \ --hash=sha256:103e8f7155f3ce2ffa0049fe60169878d47a4364b277906386f8de21c9234aa1 \ @@ -130,9 +203,9 @@ pynacl==1.4.0 \ --hash=sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff \ --hash=sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80 # via pygithub -requests==2.25.1 \ - --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ - --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via pygithub six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ From 7dec65ff72c78e3cd3e4b57d5deef214c76c61cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 09:09:04 +0100 Subject: [PATCH 320/740] build(deps): bump grpcio-tools from 1.54.2 to 1.55.0 in /examples/grpc-bridge/client (#27558) build(deps): bump grpcio-tools in /examples/grpc-bridge/client Bumps [grpcio-tools](https://github.com/grpc/grpc) from 1.54.2 to 1.55.0. - [Release notes](https://github.com/grpc/grpc/releases) - [Changelog](https://github.com/grpc/grpc/blob/master/doc/grpc_release_schedule.md) - [Commits](https://github.com/grpc/grpc/compare/v1.54.2...v1.55.0) --- updated-dependencies: - dependency-name: grpcio-tools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/client/requirements.txt | 184 +++++++++---------- 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index 4f4af0641998..4550459a94de 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -12,101 +12,101 @@ charset-normalizer==2.0.6 \ --hash=sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6 \ --hash=sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f # via requests -grpcio==1.54.2 \ - --hash=sha256:0212e2f7fdf7592e4b9d365087da30cb4d71e16a6f213120c89b4f8fb35a3ab3 \ - --hash=sha256:09d4bfd84686cd36fd11fd45a0732c7628308d094b14d28ea74a81db0bce2ed3 \ - --hash=sha256:1e623e0cf99a0ac114f091b3083a1848dbc64b0b99e181473b5a4a68d4f6f821 \ - --hash=sha256:2288d76e4d4aa7ef3fe7a73c1c470b66ea68e7969930e746a8cd8eca6ef2a2ea \ - --hash=sha256:2296356b5c9605b73ed6a52660b538787094dae13786ba53080595d52df13a98 \ - --hash=sha256:2a1e601ee31ef30a9e2c601d0867e236ac54c922d32ed9f727b70dd5d82600d5 \ - --hash=sha256:2be88c081e33f20630ac3343d8ad9f1125f32987968e9c8c75c051c9800896e8 \ - --hash=sha256:33d40954199bddbb6a78f8f6f2b2082660f381cd2583ec860a6c2fa7c8400c08 \ - --hash=sha256:40e1cbf69d6741b40f750f3cccc64326f927ac6145a9914d33879e586002350c \ - --hash=sha256:46a057329938b08e5f0e12ea3d7aed3ecb20a0c34c4a324ef34e00cecdb88a12 \ - --hash=sha256:4864f99aac207e3e45c5e26c6cbb0ad82917869abc2f156283be86c05286485c \ - --hash=sha256:4c44e1a765b31e175c391f22e8fc73b2a2ece0e5e6ff042743d8109b5d2eff9f \ - --hash=sha256:4cb283f630624ebb16c834e5ac3d7880831b07cbe76cb08ab7a271eeaeb8943e \ - --hash=sha256:5008964885e8d23313c8e5ea0d44433be9bfd7e24482574e8cc43c02c02fc796 \ - --hash=sha256:50a9f075eeda5097aa9a182bb3877fe1272875e45370368ac0ee16ab9e22d019 \ - --hash=sha256:51630c92591d6d3fe488a7c706bd30a61594d144bac7dee20c8e1ce78294f474 \ - --hash=sha256:5cc928cfe6c360c1df636cf7991ab96f059666ac7b40b75a769410cc6217df9c \ - --hash=sha256:61f7203e2767800edee7a1e1040aaaf124a35ce0c7fe0883965c6b762defe598 \ - --hash=sha256:66233ccd2a9371158d96e05d082043d47dadb18cbb294dc5accfdafc2e6b02a7 \ - --hash=sha256:70fcac7b94f4c904152809a050164650ac81c08e62c27aa9f156ac518029ebbe \ - --hash=sha256:714242ad0afa63a2e6dabd522ae22e1d76e07060b5af2ddda5474ba4f14c2c94 \ - --hash=sha256:782f4f8662a2157c4190d0f99eaaebc602899e84fb1e562a944e5025929e351c \ - --hash=sha256:7fc2b4edb938c8faa4b3c3ea90ca0dd89b7565a049e8e4e11b77e60e4ed2cc05 \ - --hash=sha256:881d058c5ccbea7cc2c92085a11947b572498a27ef37d3eef4887f499054dca8 \ - --hash=sha256:89dde0ac72a858a44a2feb8e43dc68c0c66f7857a23f806e81e1b7cc7044c9cf \ - --hash=sha256:8cdbcbd687e576d48f7886157c95052825ca9948c0ed2afdc0134305067be88b \ - --hash=sha256:8d6192c37a30a115f4663592861f50e130caed33efc4eec24d92ec881c92d771 \ - --hash=sha256:96a41817d2c763b1d0b32675abeb9179aa2371c72aefdf74b2d2b99a1b92417b \ - --hash=sha256:9bdbb7624d65dc0ed2ed8e954e79ab1724526f09b1efa88dcd9a1815bf28be5f \ - --hash=sha256:9bf88004fe086c786dc56ef8dd6cb49c026833fdd6f42cb853008bce3f907148 \ - --hash=sha256:a08920fa1a97d4b8ee5db2f31195de4a9def1a91bc003544eb3c9e6b8977960a \ - --hash=sha256:a2f5a1f1080ccdc7cbaf1171b2cf384d852496fe81ddedeb882d42b85727f610 \ - --hash=sha256:b04202453941a63b36876a7172b45366dc0cde10d5fd7855c0f4a4e673c0357a \ - --hash=sha256:b38b3de8cff5bc70f8f9c615f51b48eff7313fc9aca354f09f81b73036e7ddfa \ - --hash=sha256:b52d00d1793d290c81ad6a27058f5224a7d5f527867e5b580742e1bd211afeee \ - --hash=sha256:b74ae837368cfffeb3f6b498688a123e6b960951be4dec0e869de77e7fa0439e \ - --hash=sha256:be48496b0e00460717225e7680de57c38be1d8629dc09dadcd1b3389d70d942b \ - --hash=sha256:c0e3155fc5335ec7b3b70f15230234e529ca3607b20a562b6c75fb1b1218874c \ - --hash=sha256:c2392f5b5d84b71d853918687d806c1aa4308109e5ca158a16e16a6be71041eb \ - --hash=sha256:c72956972e4b508dd39fdc7646637a791a9665b478e768ffa5f4fe42123d5de1 \ - --hash=sha256:dc80c9c6b608bf98066a038e0172013a49cfa9a08d53335aefefda2c64fc68f4 \ - --hash=sha256:e416c8baf925b5a1aff31f7f5aecc0060b25d50cce3a5a7255dc5cf2f1d4e5eb \ - --hash=sha256:f8da84bbc61a4e92af54dc96344f328e5822d574f767e9b08e1602bb5ddc254a \ - --hash=sha256:f900ed4ad7a0f1f05d35f955e0943944d5a75f607a836958c6b8ab2a81730ef2 \ - --hash=sha256:fd6c6c29717724acf9fc1847c4515d57e4dc12762452457b9cb37461f30a81bb +grpcio==1.55.0 \ + --hash=sha256:054b7164b25712ec71339e139875a66708a2ab09be36ac75e73b2d337ab2dc1b \ + --hash=sha256:0d3d5c644d523dee82ffcc44ad50cd66e3bf66e7fa60ad3cdb1eb868228e4ab0 \ + --hash=sha256:1041cad23f00943d8889ad15427d87bbdacbbe2df5cec951c314f2f3967d4691 \ + --hash=sha256:10af4774da9c0665a1bf519333694ac40d72d83cb514534b99db0a5e3d5c3593 \ + --hash=sha256:1173a05117798aca4834d3edd504e6adc25ae9967df0f44b91a612884fb2707a \ + --hash=sha256:157f5615c7b5d0968727472f6394dee01555ef4246d2f2cfb6555be857936d74 \ + --hash=sha256:1982c99c7091d1b7e3e78b1173097f705feef233e253a27e99746b11815ac897 \ + --hash=sha256:2663741acc117370fd53336267cfb24c965e9d3ea1e4933a3e4411712d3091fb \ + --hash=sha256:29ab0e879b1585be41cfbb02faed67913700ced8015da4763f1f0bdd7dfb4ab7 \ + --hash=sha256:2d25d7fcb528a40578b3d0428d401745fd5c0eeeda81f35ce2f40a10d79afd19 \ + --hash=sha256:322d4ebc37cbc8d8596b1da6055e3e81e8cfd36816ab4b285c1163c3042e6067 \ + --hash=sha256:3ab9bf80c19c91847f45ff32af94c85d282545a62db39d797838244d57831d78 \ + --hash=sha256:4370d2cca37301bcc69453d3dd3c1576d06d6b3e337bfec55b3aab2fe106b25c \ + --hash=sha256:48f6088d898e1e987d761d58dc4cd724e7457a7a86d11561fa95c3b826d025dc \ + --hash=sha256:51b7a27a129f743d68394f94029f88ef3da090fc13776b9dfa3c79c5f4b30525 \ + --hash=sha256:56631cc0bdf86d15ea1599b9697ace65e6b52c6b136d3666bf7769d3d6d087a8 \ + --hash=sha256:60efab181c32e029e0960f238508396dd001ba2064168f8148e6356db093967c \ + --hash=sha256:67c4fda71f92225c5e74fa15bffa6be022c07111f674fe1f234c1ef4c1bb7927 \ + --hash=sha256:6b8dbb151b116825c10f01e5b7b75e14edd0e60736a65311d0d98a4cd0489303 \ + --hash=sha256:70de2b73cf22241173cb21d308786ba4ea443e4c88441a2ce445829aa638dda8 \ + --hash=sha256:74780f570c76feb8e62a8c019b495fea435b60218682fce513ff2c71262c346c \ + --hash=sha256:7b38e028a7bbc97a9ae5e418712452f298618b9d0493390770bf2de785251ae7 \ + --hash=sha256:7b8665da31b5bd701b338a581de7b9631d50b4b7ee67125c2d1dc2228cc119d8 \ + --hash=sha256:7c00263d792a244bef67a8d3b357ccbcdae6341c5961dbee494d8f967f9aee69 \ + --hash=sha256:7c32f87bec58a8a0d4f4d5387bd61a383bd32b2caffb2de3cd579e47490b7e19 \ + --hash=sha256:89107071b5f14af6bbb855183d338a0fa94136bbeb3989c9773c6184e51a95e9 \ + --hash=sha256:8a910fa9b95a286f4bc1879dcf8d5ccb95b5e33bb63323fc4414d157f23afef1 \ + --hash=sha256:8b440ccc434c1ad5874465bfae40c0a27f562ae5f7c5b468b6689bc55e8bf1c1 \ + --hash=sha256:8bd4f4932ef63ed32a725065aebb8585e4118a523d923db896e85c09429a36e6 \ + --hash=sha256:9a11b1dd4b1572e85fba5911309c15980a1ff77c555fad0ecdbe3711ef741908 \ + --hash=sha256:a202dcf0c512292fd7a2154e4044c70400212eaa726685ebf8af105e25693c5a \ + --hash=sha256:a82283d6e0403d3e2e7eebb99cb0d2783e20b6791c8c94bd8d4a4233b58b1ea0 \ + --hash=sha256:ab784204d9923368e0e5877d7795584b9606a51b128ee199ad8b5888d0c66592 \ + --hash=sha256:b1e2b705d524e780998218cf429d30b6ffc54cb6e54812c9597bc5df12dbcb5b \ + --hash=sha256:b2a3b837d5837b9069783026b57aa0ff12e34d3218fdeda3f9c06d3950266d8e \ + --hash=sha256:ba32a8e9bc3eecc6bab6824b905f04c3fdc31659c3e6e06841b774e7cb4410af \ + --hash=sha256:c33dbeecc14f1a413e8af8ae1208cb383b063fa2ff2e1f309b4d3d7739b0927e \ + --hash=sha256:c97cfae0b7a17dc1a0a3e4333f4f46daa114d85f950a67f39cc141b5425182e4 \ + --hash=sha256:ce82d06cdfb8a9292fb857f00bee11a2430e4ac2742e07b46c1a3072d683256a \ + --hash=sha256:d0209fb3cb55c5288a1dec72dcaae2c1b501edceca10d22c0f0baa5e60e2b22c \ + --hash=sha256:d396ec4d520b58f43142958cff071e5ad1c50ac87d29d086a9c6a990a09ea536 \ + --hash=sha256:dad999423b33ad5409e986587593b6062a8260b74ae8fc8162ce231c6b7a929e \ + --hash=sha256:dd15027a171ff93c97f9c704fa120bc5d0691dc7e71ae450e2ecade1a2799b53 \ + --hash=sha256:ee0de9cb6813704969e53743e0969fd95225ff24bd686c89ed12a18147f6566c \ + --hash=sha256:fe78365c64b2c7470d31c4941e10c6654042bcbb53897b9b1e2c96d6d0da9ef9 # via # -r requirements.in # grpcio-tools -grpcio-tools==1.54.2 \ - --hash=sha256:0239b929eb8b3b30b2397eef3b9abb245087754d77c3721e3be43c44796de87d \ - --hash=sha256:0ab1b323905d449298523db5d34fa5bf5fffd645bd872b25598e2f8a01f0ea39 \ - --hash=sha256:0de05c7698c655e9a240dc34ae91d6017b93143ac89e5b20046d7ca3bd09c27c \ - --hash=sha256:0f952c8a5c47e9204fe8959f7e9add149e660f6579d67cf65024c32736d34caf \ - --hash=sha256:10dd41862f579d185c60f629b5ee89103e216f63b576079d258d974d980bad87 \ - --hash=sha256:11939c9a8a39bd4815c7e88cb2fee48e1948775b59dbb06de8fcae5991e84f9e \ - --hash=sha256:129de5579f95d6a55dde185f188b4cbe19d1e2f1471425431d9930c31d300d70 \ - --hash=sha256:1b8ee3099c51ce987fa8a08e6b93fc342b10228415dd96b5c0caa0387f636a6f \ - --hash=sha256:21b1467e31e44429d2a78b50135c9cdbd4b8f6d3b5cd548bc98985d3bdc352d0 \ - --hash=sha256:21b9d2dee80f3f77e4097252e7f0db89772335a7300b72ab3d2e5c280872b1db \ - --hash=sha256:27671c68c7e0e3c5ff9967f5500799f65a04e7b153b8ce10243c87c43199039d \ - --hash=sha256:2b96f5f17d3156058be247fd25b062b4768138665694c00b056659618b8fb418 \ - --hash=sha256:30a49b8b168aced2a4ff40959e6c4383ad6cfd7a20839a47a215e9837eb722dc \ - --hash=sha256:3237149beec39e897fd62cef4aa1e1cd9422d7a95661d24bd0a79200b167e730 \ - --hash=sha256:37393ef90674964175923afe3859fc5a208e1ece565f642b4f76a8c0224a0993 \ - --hash=sha256:39fd530cfdf58dc05125775cc233b05554d553d27478f14ae5fd8a6306f0cb28 \ - --hash=sha256:3bb9ec4aea0f2b3006fb002fa59e5c10f92b48fc374619fbffd14d2b0e388c3e \ - --hash=sha256:49c2846dcc4803476e839d8bd4db8845e928f19130e0ea86121f2d1f43d2b452 \ - --hash=sha256:4abfc1892380abe6cef381eab86f9350cbd703bfe5d834095aa66fd91c886b6d \ - --hash=sha256:4f285f8ef3de422717a36bd372239ae778b8cc112ce780ca3c7fe266dadc49fb \ - --hash=sha256:503ef1351c62fb1d6747eaf74932b609d8fdd4345b3591ef910adef8fa9969d0 \ - --hash=sha256:5ef30c2dbc63c1e0a462423ca4f95001814d26ef4fe66208e53fcf220ea3b717 \ - --hash=sha256:6037f123905dc0141f7c8383ca616ef0195e79cd3b4d82faaee789d4045e891b \ - --hash=sha256:72d15de4c4b6a764a76c4ae69d99c35f7a0751223688c3f7e62dfa95eb4f61be \ - --hash=sha256:7b24fbab9e7598518ce4549e066df00aab79c2bf9bedcdde23fb5ef6a3cf532f \ - --hash=sha256:7baa210c20f71a242d9ae0e02734628f6948e8bee3bf538647894af427d28800 \ - --hash=sha256:7d7e6e8d62967b3f037f952620cb7381cc39a4bd31790c75fcfba56cc975d70b \ - --hash=sha256:7f4624ef2e76a3a5313c4e61a81be38bcc16b59a68a85d30758b84cd2102b161 \ - --hash=sha256:8742122782953d2fd038f0a199f047a24e941cc9718b1aac90876dbdb7167739 \ - --hash=sha256:8e4531267736d88fde1022b36dd42ed8163e3575bcbd12bfed96662872aa93fe \ - --hash=sha256:8e4c5a48f7b2e8798ce381498ee7b9a83c65b87ae66ee5022387394e5eb51771 \ - --hash=sha256:9acf443dcf6f68fbea3b7fb519e1716e014db1a561939f5aecc4abda74e4015d \ - --hash=sha256:a0b7049814442f918b522d66b1d015286afbeb9e6d141af54bbfafe31710a3c8 \ - --hash=sha256:a3ce0b98fb581c471424d2cda45120f57658ed97677c6fec4d6decf5d7c1b976 \ - --hash=sha256:b80585e06c4f0082327eb5c9ad96fbdb2b0e7c14971ea5099fe78c22f4608451 \ - --hash=sha256:b82ca472db9c914c44e39a41e9e8bd3ed724523dd7aff5ce37592b8d16920ed9 \ - --hash=sha256:c4128c01cd6f5ea8f7c2db405dbfd8582cd967d36e6fa0952565436633b0e591 \ - --hash=sha256:d512de051342a576bb89777476d13c5266d9334cf4badb6468aed9dc8f5bdec1 \ - --hash=sha256:df079479fb1b9e488334312e35ebbf30cbf5ecad6c56599f1a961800b33ab7c1 \ - --hash=sha256:e11c2c2aee53f340992e8e4d6a59172cbbbd0193f1351de98c4f810a5041d5ca \ - --hash=sha256:e3d0e5188ff8dbaddac2ee44731d36f09c4eccd3eac7328e547862c44f75cacd \ - --hash=sha256:e543f457935ba7b763b121f1bf893974393b4d30065042f947f85a8d81081b80 \ - --hash=sha256:e5c7292dd899ad8fa09a2be96719648cee37b17909fe8c12007e3bff58ebee61 \ - --hash=sha256:f39d8e8806b8857fb473ca6a9c7bd800b0673dfdb7283ff569af0345a222f32c \ - --hash=sha256:f6787d07fdab31a32c433c1ba34883dea6559d8a3fbe08fb93d834ca34136b71 +grpcio-tools==1.55.0 \ + --hash=sha256:053bbdfb74f76511db47e1e18a1962432468ae9f356cc00f15d1f1353eaf32a1 \ + --hash=sha256:07c23ed940e046c9dd471bc870eb5db4d93e518f90011cf9aebf8bfda6cd68a5 \ + --hash=sha256:0dead7fb37bfe7c7eb8294143015645297f4affa683783b8bbf2cd4d7f7036d4 \ + --hash=sha256:2ba87592f2cd689e127cd4fce76ec23b19562e230fa41ea089af8b15120aea78 \ + --hash=sha256:2eacb0b1e8e5cfd0b40e12e62bd5adebbbae8c73cdf6e04fad9ddd37e32d98a4 \ + --hash=sha256:350303ef3a2b25ed1b90e42764923e40b664d9f10840f7a0f06117c4dc414aff \ + --hash=sha256:36745762689df18f83273a9a004848897793f63a10a30acd18acb2d170c663a9 \ + --hash=sha256:3724e48c3db499b2d212c5a89d7cc4b49ccd476dc26bf8a9b855d59b6cc00796 \ + --hash=sha256:41005002cbfa0ad39972486bde8116b2a042804119e5b998086a4dc26e625d6a \ + --hash=sha256:416a8b61ed4223715755b4519858419e1f4653d64572a28029f2ac63e677e3d2 \ + --hash=sha256:4580df5a9867f7bcbb828a5485c030ca232c1578e615caf751333c7a7980d838 \ + --hash=sha256:4a41130c97775bb0dfaf87e34b492f2eca448d02d213410005544c534f3f7c26 \ + --hash=sha256:4a6db1494955d2a5531575b5fcdc08094ea4a331a94b9cdf864d78e801c5fa23 \ + --hash=sha256:4dea66623548b52429fb03495f2c76f4c993bf9a56267c6b3d0fb62573dd52c2 \ + --hash=sha256:51a1ccab6f67edd1a3768a75ac495907fe0cd6d6617af2f9f2033400b5858a11 \ + --hash=sha256:52e34e9b6496f4c1e3289ada7bc41d759e4a8ec5f2679e187067cab8532ffbf4 \ + --hash=sha256:632364ffbd4fb0338cb03c590a2ddc258d9cd59bff0bf4199c02e3e581f802d7 \ + --hash=sha256:734ede84d613b044f72e7d9c190bd2388ebb83e85bcd3aa75afa9f30c096dbc7 \ + --hash=sha256:73ef9e0e0ee8ab055a621e7b42e5fb32753b0b6607900887dba6d55df5947be8 \ + --hash=sha256:7f084cd619cf66d8620a99f8586018f19b918ffb2ddb92d3e5943a06038bead8 \ + --hash=sha256:87152893c7c3bef58a6a9b548db290aa318cc314c700ae7d7f2970aa567f875e \ + --hash=sha256:87dbc98528f88faa3f8f56a47d41dc6fda382928abbdb5537b5444eb8bb1ac1b \ + --hash=sha256:89f6ed47415a22568bbf4a62336bfde7cafb53492a5a9f33a22243411b00f443 \ + --hash=sha256:8e59fd4a58688117cb5128d7785909d45a6e5f8212efeb65b6fd74bb9b8b9512 \ + --hash=sha256:9395c4fdee6b22137e878ebd461444854a3cd9c6c260c43f4a4c4a4023700129 \ + --hash=sha256:946266cbd639847548c9f97e38da0682746c2eadea790ceb4320b1f85387bd6d \ + --hash=sha256:95428be2db12412ff23f0969386fc51d2aa6de38a57cc54c57363352f1d7a832 \ + --hash=sha256:98ff3129ff7134a95f4d2857663625771f6838ac44b7799c34259b7ea87ebe5c \ + --hash=sha256:9933a1f18f780c42214b126ef27e273b54c9c28de3fae5b1887b413ceb374c4c \ + --hash=sha256:a61567f27661ab9327dc060615dc22d2bde80c56731f1e856008f1fd8ee83311 \ + --hash=sha256:ab64f9d6f5e3636ae6298e2d795225daa83aacb057105943728ed50a8a582237 \ + --hash=sha256:b00a67a1230968c1a0424915922d17983d824ed45e8db06f9f17be6d5571faee \ + --hash=sha256:b131b2bbf25198d9e508dfa588cb215580629b514e293d5609eeee98c8941dbc \ + --hash=sha256:b197de69ca0431b718ffa47b32a733703fa5503da49f49dd315c866842b6cfbd \ + --hash=sha256:b674de79571357c5381bc5fa12e3b89fefef74c164ab9077ed22158c3529aa8e \ + --hash=sha256:bc23034b1959d6cda27347b2207fee0fb0fb0aff242da228a6b7c1a18fce4116 \ + --hash=sha256:bcf5e1858137cbe13ef10a7931a7edc745c77f8b39f032f52072443f0dd681e1 \ + --hash=sha256:c3c7b7eb89f963b87922ecc0c0ab2485fff05997ada66dffd53597b507a83bc8 \ + --hash=sha256:c7a18bd5f994b7911d3e70e0abb05bea9f1b084a1725d404a8e231bf9727613b \ + --hash=sha256:cfc82c11ce51de6ed5836fbafbc188d9eac0737abc116978f151c40271783817 \ + --hash=sha256:d796f5d7cea260ef2afed12d13ec34b13e09dd74d7f292d7428c506fa8c17a74 \ + --hash=sha256:e76f35e5e65600a75c3547855e8c9ab935c55c473f5409e7746cca8f1f7c8f4a \ + --hash=sha256:f87d99aa3826aa20c3b89493984cf278f4c9d20b3418534a46239c804fee506c \ + --hash=sha256:f900bce944b5777effecb9078e5fd3a224e42b1ca33c7546c3d043f9ef9eb4e8 \ + --hash=sha256:fbbe2bee4af93c03ba064d40199dbf38067d2aa6ae98dfa0687a08ee980ebfd5 # via -r requirements.in idna==3.2 \ --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \ From 575f6967087cb181b5368bfd1549952b758b02cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 09:09:47 +0100 Subject: [PATCH 321/740] build(deps): bump postgres from `8d45935` to `b115fe7` in /examples/shared/postgres (#27559) build(deps): bump postgres in /examples/shared/postgres Bumps postgres from `8d45935` to `b115fe7`. --- updated-dependencies: - dependency-name: postgres dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile index a4ab85b4bc82..84b2d96a722c 100644 --- a/examples/shared/postgres/Dockerfile +++ b/examples/shared/postgres/Dockerfile @@ -1,3 +1,3 @@ -FROM postgres:latest@sha256:8d45935fb783e72c871072e9eb72ee8c817a9eaf25c405b0e62526b14191368d +FROM postgres:latest@sha256:b115fe7743919fff0b96f19405ed350017ef9333b3760fc0cc885713eb07f565 COPY docker-healthcheck.sh /usr/local/bin/ HEALTHCHECK CMD ["docker-healthcheck.sh"] From 599f675495c054ad5c2e983e268e368c9871ba19 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 09:11:36 +0100 Subject: [PATCH 322/740] build(deps): bump redis from `ea30bef` to `47e4e43` in /examples/redis (#27560) Bumps redis from `ea30bef` to `47e4e43`. --- updated-dependencies: - dependency-name: redis dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/redis/Dockerfile-redis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/redis/Dockerfile-redis b/examples/redis/Dockerfile-redis index f5847517ea98..e05ddfb56f9f 100644 --- a/examples/redis/Dockerfile-redis +++ b/examples/redis/Dockerfile-redis @@ -1 +1 @@ -FROM redis@sha256:ea30bef6a1424d032295b90db20a869fc8db76331091543b7a80175cede7d887 +FROM redis@sha256:47e4e4395af10811cfd893e3e2dcfdf13cf45b0d4feeeafc92842d6902656ede From fe92962b45df3c4346382adf1439713a869ea1bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 09:15:45 +0100 Subject: [PATCH 323/740] build(deps): bump nginx from `480868e` to `a2f6ffd` in /examples/local_ratelimit (#27561) build(deps): bump nginx in /examples/local_ratelimit Bumps nginx from `480868e` to `a2f6ffd`. --- updated-dependencies: - dependency-name: nginx dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/local_ratelimit/Dockerfile-nginx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/local_ratelimit/Dockerfile-nginx b/examples/local_ratelimit/Dockerfile-nginx index 46b5fa67f3a1..d48af7252182 100644 --- a/examples/local_ratelimit/Dockerfile-nginx +++ b/examples/local_ratelimit/Dockerfile-nginx @@ -1 +1 @@ -FROM nginx@sha256:480868e8c8c797794257e2abd88d0f9a8809b2fe956cbfbc05dcc0bca1f7cd43 +FROM nginx@sha256:a2f6ffddd272625464710d6189f1bf5d02e068392facae344762a512209e74d5 From 6550e3c4361a28a1eac5055db3e5db329026171c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 09:16:46 +0100 Subject: [PATCH 324/740] build(deps): bump debian from `f4da3f9` to `7606bef` in /examples/shared/websocket (#27563) build(deps): bump debian in /examples/shared/websocket Bumps debian from `f4da3f9` to `7606bef`. --- updated-dependencies: - dependency-name: debian dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/websocket/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/websocket/Dockerfile b/examples/shared/websocket/Dockerfile index e070c5148ea0..3e047c58941f 100644 --- a/examples/shared/websocket/Dockerfile +++ b/examples/shared/websocket/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim@sha256:f4da3f9b18fc242b739807a0fb3e77747f644f2fb3f67f4403fafce2286b431a as websocket-base +FROM debian:bullseye-slim@sha256:7606bef5684b393434f06a50a3d1a09808fee5a0240d37da5d181b1b121e7637 as websocket-base ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ From 49dccb3acd48a5efee3f74c5551d4fdc7bb92b63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 09:21:07 +0100 Subject: [PATCH 325/740] build(deps): bump python from `551c952` to `6c0b067` in /examples/shared/python (#27564) build(deps): bump python in /examples/shared/python Bumps python from `551c952` to `6c0b067`. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/python/Dockerfile b/examples/shared/python/Dockerfile index cc94f641fbff..8a87913da3aa 100644 --- a/examples/shared/python/Dockerfile +++ b/examples/shared/python/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.3-slim-bullseye@sha256:551c9529e77896518ac5693d7e98ee5e12051d625de450ac2a68da1eae15ec87 as python-base +FROM python:3.11.3-slim-bullseye@sha256:6c0b067294c78ee6538cc9d6163fd1653797d0f4fd567e55703a5365c7757138 as python-base RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache ARG PYTHON_REQUIREMENTS_FILE=aiohttp/requirements.txt From ff1546da85f89bf7d0bd75467a5deafda58c799e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 09:21:37 +0100 Subject: [PATCH 326/740] build(deps): bump node from `5138ded` to `95a950e` in /examples/shared/node (#27565) build(deps): bump node in /examples/shared/node Bumps node from `5138ded` to `95a950e`. --- updated-dependencies: - dependency-name: node dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/node/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile index 7cd02355882d..9ef656782642 100644 --- a/examples/shared/node/Dockerfile +++ b/examples/shared/node/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.2-bullseye-slim@sha256:5138ded35380c7e55b7898a5c3666009334aa4af416571060d37347242e1812f as node-base +FROM node:20.2-bullseye-slim@sha256:95a950ec61796f4c00f6b208cb51000b8bd127ee53b0c1c52f2539a5ab66f8ef as node-base FROM node-base as node-http-auth From c0dce839eda044eaccf219896809a0d8bbd1bb53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 10:19:09 +0100 Subject: [PATCH 327/740] build(deps): bump requests from 2.30.0 to 2.31.0 in /mobile/docs (#27554) Bumps [requests](https://github.com/psf/requests) from 2.30.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.30.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mobile/docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/docs/requirements.txt b/mobile/docs/requirements.txt index 4118d6bee302..a5feef288926 100644 --- a/mobile/docs/requirements.txt +++ b/mobile/docs/requirements.txt @@ -158,9 +158,9 @@ pyparsing==3.0.9 \ pytz==2023.3 \ --hash=sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588 \ --hash=sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb -requests==2.30.0 \ - --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ - --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 six==1.16.0 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 From 04c42f8ce23880aaed99956caf00853aa7b8199e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 11:20:12 +0100 Subject: [PATCH 328/740] build(deps): bump requests from 2.30.0 to 2.31.0 in /tools/base (#27566) Bumps [requests](https://github.com/psf/requests) from 2.30.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.30.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index ee3068fd0a65..69409c5bac86 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -1147,9 +1147,9 @@ pyyaml==6.0 \ # aio-core # envoy-base-utils # yamllint -requests==2.30.0 \ - --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ - --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via # google-api-core # google-auth From 72c2ec85ecb1e0dc5e71badd8a921304cfcda2ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 11:20:51 +0100 Subject: [PATCH 329/740] build(deps): bump debian from `f4da3f9` to `7606bef` in /examples/shared/golang (#27562) build(deps): bump debian in /examples/shared/golang Bumps debian from `f4da3f9` to `7606bef`. --- updated-dependencies: - dependency-name: debian dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/golang/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile index feaafc04b9b4..c439916f78ea 100644 --- a/examples/shared/golang/Dockerfile +++ b/examples/shared/golang/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim@sha256:f4da3f9b18fc242b739807a0fb3e77747f644f2fb3f67f4403fafce2286b431a as os-base +FROM debian:bullseye-slim@sha256:7606bef5684b393434f06a50a3d1a09808fee5a0240d37da5d181b1b121e7637 as os-base RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache From a23fa16e0dd7085cb5acd899fe1bc5ece182a399 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 11:22:19 +0100 Subject: [PATCH 330/740] build(deps): bump requests from 2.30.0 to 2.31.0 in /examples/grpc-bridge/client (#27555) build(deps): bump requests in /examples/grpc-bridge/client Bumps [requests](https://github.com/psf/requests) from 2.30.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.30.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/client/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index 4550459a94de..41ca3dbc8d03 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -129,9 +129,9 @@ protobuf==4.23.1 \ # via # -r requirements.in # grpcio-tools -requests==2.30.0 \ - --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ - --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via -r requirements.in urllib3==1.26.7 \ --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \ From b201b68e8cff31b9e85c1bb47cf84d24a9d2ab25 Mon Sep 17 00:00:00 2001 From: "Dr. Andre Vehreschild" <101638173+vehre-x41@users.noreply.github.com> Date: Tue, 23 May 2023 15:21:45 +0200 Subject: [PATCH 331/740] [fuzz] Only add filters that are allowlisted (#27373) Signed-off-by: Andre Vehreschild Co-authored-by: Ryan Northey --- source/docs/network_filter_fuzzing.md | 58 ++++++------------- source/extensions/all_extensions.bzl | 12 +--- .../filters/network/common/fuzz/BUILD | 37 +++++++++--- .../common/fuzz/uber_per_readfilter.cc | 27 ++------- .../common/fuzz/uber_per_writefilter.cc | 20 ++----- tools/base/requirements.in | 2 +- tools/base/requirements.txt | 6 +- 7 files changed, 64 insertions(+), 98 deletions(-) diff --git a/source/docs/network_filter_fuzzing.md b/source/docs/network_filter_fuzzing.md index f23354423bcd..590b4a94cd87 100644 --- a/source/docs/network_filter_fuzzing.md +++ b/source/docs/network_filter_fuzzing.md @@ -1,55 +1,31 @@ # Generic network-level filter fuzzers overview -Network filters need to be fuzzed. Filters come in two flavors, each with their own fuzzer. Read filters should be added into the [Generic ReadFilter Fuzzer](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc). Write Filters should added into the [Generic WriteFilter Fuzzer](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc). Some filters are both raed and write filters: They should be added into both fuzzers. +Network filters need to be fuzzed. Filters come in two flavors, each with their own fuzzer. Read filters should be added into the [Generic ReadFilter Fuzzer](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc). Write Filters should added into the [Generic WriteFilter Fuzzer](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc). Some filters are both read and write filters: They should be added into both fuzzers. Before adding the new filter into the fuzzers, please make sure the filter is designed to accept untrusted inputs, or ready to be hardened to accept untrusted inputs. # Add a new ReadFilter into Generic Readfilter Fuzzer -## Step1. Make sure the filter can be linked into the fuzzer -There are two ways to link it into the fuzzer. -* [Recommended] In the file [extensions_build_config.bzl](https://github.com/envoyproxy/envoy/blob/main/source/extensions/extensions_build_config.bzl), the name of the filter should have a prefix `envoy.filters.network`. If it has such a prefix, the filter will be automatically linked into Generic ReadFilter Fuzzer. -* [Not recommended]If for some reasons the filter's name doesn't have such a prefix, the config of the filter must be added into the `deps` field of `network_readfilter_fuzz_test` module in the file [BUILD](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/BUILD). -### Step2. Add the filter name into supported_filter_names -In [uber_per_readfilter.cc](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc), add the filter name into the vector `supported_filter_names` in method `UberFilterFuzzer::filterNames()`. +Only one step is needed to add a new filter to the fuzzer: +* In the file [BUILD](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/BUILD) the name of the filter has to be added to the `READFILTER_FUZZ_FILTERS` list. The fuzz test will figure the available filters from the factories. ``` -const std::vector supported_filter_names = { -... -NetworkFilterNames::get().ExtAuthorization, NetworkFilterNames::get().TheNewFilterCreatedByYou, -... -}; +READFILTER_FUZZ_FILTERS = [ + "envoy.filters.network.client_ssl_auth", + "envoy.filters.network.ext_authz", + "envoy.filters.network.envoy_mobile_http_connection_manager", + # A dedicated http_connection_manager fuzzer can be found in + # test/common/http/conn_manager_impl_fuzz_test.cc + "envoy.filters.network.http_connection_manager", + "envoy.filters.network.local_ratelimit", + "envoy.filters.network.rbac", + # TODO(asraa): Remove when fuzzer sets up connections for TcpProxy properly. + # "envoy.filters.network.tcp_proxy", + "the_new_filter_created_by_you", // <---Add the filter name here +] ``` # Add a new WriteFilter into Generic Writefilter Fuzzer -## Step 1. Make sure the filter can be linked into the fuzzer -For WriteFilter, the config of the filter must be added into the `deps` field of `network_writefilter_fuzz_test` module in the file [BUILD](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/BUILD). -``` -envoy_cc_fuzz_test( - name = "network_writefilter_fuzz_test", - srcs = ["network_writefilter_fuzz_test.cc"], - corpus = "network_writefilter_corpus", - # All Envoy network filters must be linked to the test in order for the fuzzer to pick - # these up via the NamedNetworkFilterConfigFactory. - deps = [ - ":uber_writefilter_lib", - "//source/common/config:utility_lib", - "//source/extensions/filters/network/mongo_proxy:config", - "//contrib/mysql_proxy/filters/network/source:config", - "//source/extensions/filters/network/zookeeper_proxy:config", - "//source/extensions/filters/network/the_new_filter_created_by_you:config", // <---Add the filter config module here - "//test/config:utility_lib", - ], -) -``` -## Step 2. Add the filter name into supported_filter_names -In [uber_per_writefilter.cc](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc), add the filter name into the vector `supported_filter_names` in method `UberWriteFilterFuzzer::filterNames()`. -``` -const std::vector supported_filter_names = { - ... - NetworkFilterNames::get().ExtAuthorization, NetworkFilterNames::get().TheNewFilterCreatedByYou, - ... - }; -``` +For WriteFilter, the name of the filter must be added into the `WRITEFILTER_FUZZ_FILTERS` list of the file [BUILD](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/BUILD). # Add test cases into corpus Good test cases can provide good examples for fuzzers to find more paths in the code, increase the coverage and help find bugs more efficiently. diff --git a/source/extensions/all_extensions.bzl b/source/extensions/all_extensions.bzl index c2c39bc3d6d4..1a18c2cf44e1 100644 --- a/source/extensions/all_extensions.bzl +++ b/source/extensions/all_extensions.bzl @@ -50,14 +50,8 @@ def envoy_all_http_filters(): return {_selected_extension_target(v): True for k, v in all_extensions.items() if (k.startswith(_http_filter_prefix) or k.startswith(_upstream_http_filter_prefix))}.keys() -# All network-layer filters are extensions with names that have the following prefix. -_network_filter_prefix = "envoy.filters.network" - -# All thrift filters are extensions with names that have the following prefix. -_thrift_filter_prefix = "envoy.filters.thrift" - -# Return all network-layer filter extensions to be compiled into network-layer filter generic fuzzer. -def envoy_all_network_filters(): +# Return 'selected' network-layer filter extensions to be compiled into network-layer filter generic fuzzer. +def envoy_filters_from_selected(selected): all_extensions = dicts.add(_required_extensions, EXTENSIONS) - return [_selected_extension_target(v) for k, v in all_extensions.items() if (k.startswith(_network_filter_prefix) or k.startswith(_thrift_filter_prefix))] + return [_selected_extension_target(v) for k, v in all_extensions.items() if (k in selected)] diff --git a/test/extensions/filters/network/common/fuzz/BUILD b/test/extensions/filters/network/common/fuzz/BUILD index 7197944a15b2..6778bee09fd0 100644 --- a/test/extensions/filters/network/common/fuzz/BUILD +++ b/test/extensions/filters/network/common/fuzz/BUILD @@ -7,7 +7,7 @@ load( ) load( "//source/extensions:all_extensions.bzl", - "envoy_all_network_filters", + "envoy_filters_from_selected", ) licenses(["notice"]) # Apache 2 @@ -36,6 +36,22 @@ envoy_proto_library( ], ) +# The filters to be fuzzed by the network_readfilter_fuzz_test. Prefer to add only stable filters, +# that do not have a dedicated fuzzer to them. +READFILTER_FUZZ_FILTERS = [ + "envoy.filters.network.client_ssl_auth", + "envoy.filters.network.ext_authz", + "envoy.filters.network.envoy_mobile_http_connection_manager", + # A dedicated http_connection_manager fuzzer can be found in + # test/common/http/conn_manager_impl_fuzz_test.cc + "envoy.filters.network.http_connection_manager", + "envoy.filters.network.local_ratelimit", + "envoy.filters.network.ratelimit", + "envoy.filters.network.rbac", + # TODO(asraa): Remove when fuzzer sets up connections for TcpProxy properly. + # "envoy.filters.network.tcp_proxy", +] + envoy_cc_test_library( name = "vig_anymap_ext_lib", srcs = ["validated_input_generator_any_map_extensions.cc"], @@ -85,17 +101,26 @@ envoy_cc_fuzz_test( srcs = ["network_readfilter_fuzz_test.cc"], corpus = "network_readfilter_corpus", dictionaries = ["network_readfilter_fuzz_test.dict"], - # All Envoy network filters must be linked to the test in order for the fuzzer to pick - # these up via the NamedNetworkFilterConfigFactory. deps = [ ":uber_readfilter_lib", ":vig_anymap_ext_lib", "//source/common/config:utility_lib", "//test/config:utility_lib", "//test/test_common:test_runtime_lib", - ] + envoy_all_network_filters(), + ] + envoy_filters_from_selected(READFILTER_FUZZ_FILTERS), ) +# The filters to be fuzzed by the network_writefilter_fuzz_test. Prefer to add only stable filters, +# that do not have a dedicated fuzzer to them. +WRITEFILTER_FUZZ_FILTERS = [ + "envoy.filters.network.zookeeper_proxy", + "envoy.filters.network.kafka_broker", + "envoy.filters.network.mongo_proxy", + "envoy.filters.network.mysql_proxy", + # TODO(jianwendong) Add "NetworkFilterNames::get().Postgres" after it + # supports untrusted data. +] + envoy_cc_test_library( name = "uber_writefilter_lib", srcs = [ @@ -123,8 +148,6 @@ envoy_cc_fuzz_test( ":uber_writefilter_lib", ":vig_anymap_ext_lib", "//source/common/config:utility_lib", - "//source/extensions/filters/network/mongo_proxy:config", - "//source/extensions/filters/network/zookeeper_proxy:config", "//test/config:utility_lib", - ], + ] + envoy_filters_from_selected(WRITEFILTER_FUZZ_FILTERS), ) diff --git a/test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc b/test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc index 0b52349bda91..f697e8964060 100644 --- a/test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc +++ b/test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc @@ -21,28 +21,11 @@ std::vector UberFilterFuzzer::filterNames() { // traffic. static std::vector filter_names; if (filter_names.empty()) { - const auto factories = Registry::FactoryRegistry< - Server::Configuration::NamedNetworkFilterConfigFactory>::factories(); - const std::vector supported_filter_names = { - NetworkFilterNames::get().ClientSslAuth, NetworkFilterNames::get().ExtAuthorization, - NetworkFilterNames::get().EnvoyMobileHttpConnectionManager, - // A dedicated http_connection_manager fuzzer can be found in - // test/common/http/conn_manager_impl_fuzz_test.cc - NetworkFilterNames::get().HttpConnectionManager, NetworkFilterNames::get().LocalRateLimit, - NetworkFilterNames::get().RateLimit, NetworkFilterNames::get().Rbac, - // TODO(asraa): Remove when fuzzer sets up connections for TcpProxy properly. - // NetworkFilterNames::get().TcpProxy, - }; - // Check whether each filter is loaded into Envoy. - // Some customers build Envoy without some filters. When they run fuzzing, the use of a filter - // that does not exist will cause fatal errors. - for (auto& filter_name : supported_filter_names) { - if (factories.contains(filter_name)) { - filter_names.push_back(filter_name); - } else { - ENVOY_LOG_MISC(debug, "Filter name not found in the factory: {}", filter_name); - } - } + // Only use the names of the filters that are compiled into envoy. The build system takes care + // about reducing these to the allowed set. + // See test/extensions/filters/network/common/fuzz/BUILD for more information. + filter_names = Registry::FactoryRegistry< + Server::Configuration::NamedNetworkFilterConfigFactory>::registeredNames(); } return filter_names; } diff --git a/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc b/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc index 24babee1697d..696935566fb4 100644 --- a/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc +++ b/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc @@ -10,21 +10,11 @@ std::vector UberWriteFilterFuzzer::filterNames() { // Will extend to cover other network filters one by one. static std::vector filter_names; if (filter_names.empty()) { - const auto factories = Registry::FactoryRegistry< - Server::Configuration::NamedNetworkFilterConfigFactory>::factories(); - const std::vector supported_filter_names = { - NetworkFilterNames::get().ZooKeeperProxy, NetworkFilterNames::get().KafkaBroker, - NetworkFilterNames::get().MongoProxy, NetworkFilterNames::get().MySQLProxy - // TODO(jianwendong) Add "NetworkFilterNames::get().Postgres" after it supports untrusted - // data. - }; - for (auto& filter_name : supported_filter_names) { - if (factories.contains(filter_name)) { - filter_names.push_back(filter_name); - } else { - ENVOY_LOG_MISC(debug, "Filter name not found in the factory: {}", filter_name); - } - } + // Only use the names of the filters that are compiled into envoy. The build system takes care + // about reducing these to the allowed set. + // See test/extensions/filters/network/common/fuzz/BUILD for more information. + filter_names = Registry::FactoryRegistry< + Server::Configuration::NamedNetworkFilterConfigFactory>::registeredNames(); } return filter_names; } diff --git a/tools/base/requirements.in b/tools/base/requirements.in index b10adff378ce..68c563b40516 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -6,7 +6,7 @@ colorama coloredlogs dependatool>=0.2.2 envoy.base.utils>=0.4.7 -envoy.code.check>=0.4.1 +envoy.code.check>=0.5.0 envoy.dependency.check>=0.1.7 envoy.distribution.release>=0.0.9 envoy.distribution.repo>=0.0.8 diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 69409c5bac86..8ad6e2ebff2d 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -434,9 +434,9 @@ envoy-base-utils==0.4.10 \ # envoy-docs-sphinx-runner # envoy-github-release # envoy-gpg-sign -envoy-code-check==0.4.1 \ - --hash=sha256:3b4d3c9f8f80d47edd0bedd4df2bc7f8a90fcb37cf4142f4882703f4141a1b01 \ - --hash=sha256:a3806deee1565d80488e3c7758e54b9a19025fbe4e91378969f6be342f083eda +envoy-code-check==0.5.0 \ + --hash=sha256:326ad3f188705fe7d0652bc707f7bc5eac190ff5c68d21f2fc5ebebc3b100e45 \ + --hash=sha256:46314932f38cab7caf02ff0da1d7a961f94b4ceedb11e19d0552c36481a041e6 # via -r requirements.in envoy-dependency-check==0.1.8 \ --hash=sha256:ac9820e446bb44e05121e5c93c210f40ca37076580b0d082da2c63e7784c338a \ From fc5361a35bcfeeae8603e5d3957fefddf9d68a4d Mon Sep 17 00:00:00 2001 From: danzh Date: Tue, 23 May 2023 09:45:26 -0400 Subject: [PATCH 332/740] test: fix flaky test DownstreamProtocolIntegrationTest.HandleDownstreamSocketFail (#27546) The test manually introduces EBADF error on the connection between Envoy and the test client. However, the IoHandleMatcher doesn't distinguish UDP socket from TCP. If the upstream HTTP/3 connection gets the same port number as downstream TCP connection, the EBADF error will be applied to the wrong connection. Enforce IoHandleMatcher to match socket type. Risk Level: low, test only Testing: existing tests Docs Changes: N/A Release Notes: N/A Partially Fix #27490 Signed-off-by: Dan Zhang --- .../buffer_accounting_integration_test.cc | 3 ++- test/integration/filters/test_socket_interface.h | 10 +++++++++- test/integration/http2_flood_integration_test.cc | 4 +++- test/integration/multiplexed_integration_test.cc | 8 +++++++- test/integration/protocol_integration_test.cc | 8 ++++++-- test/integration/shadow_policy_integration_test.cc | 3 ++- test/integration/socket_interface_swap.cc | 3 ++- test/integration/socket_interface_swap.h | 12 ++++++++---- 8 files changed, 39 insertions(+), 12 deletions(-) diff --git a/test/integration/buffer_accounting_integration_test.cc b/test/integration/buffer_accounting_integration_test.cc index 98034bee1639..f9fe4ff3b8f6 100644 --- a/test/integration/buffer_accounting_integration_test.cc +++ b/test/integration/buffer_accounting_integration_test.cc @@ -120,7 +120,8 @@ class Http2BufferWatermarksTest } Http2BufferWatermarksTest() - : HttpIntegrationTest( + : SocketInterfaceSwap(Network::Socket::Type::Stream), + HttpIntegrationTest( std::get<0>(GetParam()).downstream_protocol, std::get<0>(GetParam()).version, ConfigHelper::httpProxyConfig( /*downstream_is_quic=*/std::get<0>(GetParam()).downstream_protocol == diff --git a/test/integration/filters/test_socket_interface.h b/test/integration/filters/test_socket_interface.h index af28e319ce04..6ecdf2122667 100644 --- a/test/integration/filters/test_socket_interface.h +++ b/test/integration/filters/test_socket_interface.h @@ -30,7 +30,12 @@ class TestIoSocketHandle : public Test::IoSocketHandlePlatformImpl { TestIoSocketHandle(WriteOverrideProc write_override_proc, os_fd_t fd = INVALID_SOCKET, bool socket_v6only = false, absl::optional domain = absl::nullopt) : Test::IoSocketHandlePlatformImpl(fd, socket_v6only, domain), - write_override_(write_override_proc) {} + write_override_(write_override_proc) { + int type; + socklen_t length = sizeof(int); + EXPECT_EQ(0, getOption(SOL_SOCKET, SO_TYPE, &type, &length).return_value_); + socket_type_ = type == SOCK_STREAM ? Socket::Type::Stream : Socket::Type::Datagram; + } void initializeFileEvent(Event::Dispatcher& dispatcher, Event::FileReadyCb cb, Event::FileTriggerType trigger, uint32_t events) override { @@ -59,6 +64,8 @@ class TestIoSocketHandle : public Test::IoSocketHandlePlatformImpl { return Test::IoSocketHandlePlatformImpl::peerAddress(); } + Socket::Type getSocketType() const { return socket_type_; } + private: IoHandlePtr accept(struct sockaddr* addr, socklen_t* addrlen) override; Api::IoCallUint64Result writev(const Buffer::RawSlice* slices, uint64_t num_slice) override; @@ -72,6 +79,7 @@ class TestIoSocketHandle : public Test::IoSocketHandlePlatformImpl { const WriteOverrideProc write_override_; absl::Mutex mutex_; Event::Dispatcher* dispatcher_ ABSL_GUARDED_BY(mutex_) = nullptr; + Socket::Type socket_type_; }; /** diff --git a/test/integration/http2_flood_integration_test.cc b/test/integration/http2_flood_integration_test.cc index c7f0d2fb148f..6a08c69b12a8 100644 --- a/test/integration/http2_flood_integration_test.cc +++ b/test/integration/http2_flood_integration_test.cc @@ -56,7 +56,9 @@ class Http2FloodMitigationTest public testing::TestWithParam>, public Http2RawFrameIntegrationTest { public: - Http2FloodMitigationTest() : Http2RawFrameIntegrationTest(std::get<0>(GetParam())) { + Http2FloodMitigationTest() + : SocketInterfaceSwap(Network::Socket::Type::Stream), + Http2RawFrameIntegrationTest(std::get<0>(GetParam())) { // This test tracks the number of buffers created, and the tag extraction check uses some // buffers, so disable it in this test. skip_tag_extraction_rule_check_ = true; diff --git a/test/integration/multiplexed_integration_test.cc b/test/integration/multiplexed_integration_test.cc index 1a78f0273f18..190e57065d1c 100644 --- a/test/integration/multiplexed_integration_test.cc +++ b/test/integration/multiplexed_integration_test.cc @@ -2406,7 +2406,13 @@ TEST_P(MultiplexedIntegrationTest, ConnectionPoolPerDownstream) { // Ordering of inheritance is important here, SocketInterfaceSwap must be // destroyed after HttpProtocolIntegrationTest. class SocketSwappableMultiplexedIntegrationTest : public SocketInterfaceSwap, - public HttpProtocolIntegrationTest {}; + public HttpProtocolIntegrationTest { +public: + SocketSwappableMultiplexedIntegrationTest() + : SocketInterfaceSwap(GetParam().downstream_protocol == Http::CodecType::HTTP3 + ? Network::Socket::Type::Datagram + : Network::Socket::Type::Stream) {} +}; INSTANTIATE_TEST_SUITE_P(IpVersions, SocketSwappableMultiplexedIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index edf0d14ed38b..2b1bb3f77b63 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -4286,7 +4286,9 @@ TEST_P(DownstreamProtocolIntegrationTest, HandleDownstreamSocketFail) { NoUdpGso reject_gso_; TestThreadsafeSingletonInjector os_calls{&reject_gso_}; ASSERT(!Api::OsSysCallsSingleton::get().supportsUdpGso()); - SocketInterfaceSwap socket_swap; + SocketInterfaceSwap socket_swap(downstreamProtocol() == Http::CodecType::HTTP3 + ? Network::Socket::Type::Datagram + : Network::Socket::Type::Stream); initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -4320,7 +4322,9 @@ TEST_P(DownstreamProtocolIntegrationTest, HandleDownstreamSocketFail) { } TEST_P(ProtocolIntegrationTest, HandleUpstreamSocketFail) { - SocketInterfaceSwap socket_swap; + SocketInterfaceSwap socket_swap(upstreamProtocol() == Http::CodecType::HTTP3 + ? Network::Socket::Type::Datagram + : Network::Socket::Type::Stream); useAccessLog("%RESPONSE_CODE_DETAILS%"); initialize(); diff --git a/test/integration/shadow_policy_integration_test.cc b/test/integration/shadow_policy_integration_test.cc index 190bff3858dc..4f4c1b8855d7 100644 --- a/test/integration/shadow_policy_integration_test.cc +++ b/test/integration/shadow_policy_integration_test.cc @@ -19,7 +19,8 @@ class ShadowPolicyIntegrationTest public SocketInterfaceSwap { public: ShadowPolicyIntegrationTest() - : HttpIntegrationTest(Http::CodecType::HTTP2, std::get<0>(GetParam())) { + : HttpIntegrationTest(Http::CodecType::HTTP2, std::get<0>(GetParam())), + SocketInterfaceSwap(Network::Socket::Type::Stream) { scoped_runtime_.mergeValues( {{"envoy.reloadable_features.streaming_shadow", streaming_shadow_ ? "true" : "false"}}); setUpstreamProtocol(Http::CodecType::HTTP2); diff --git a/test/integration/socket_interface_swap.cc b/test/integration/socket_interface_swap.cc index f2b716127d48..608a90363440 100644 --- a/test/integration/socket_interface_swap.cc +++ b/test/integration/socket_interface_swap.cc @@ -4,7 +4,8 @@ namespace Envoy { void preserveIoError(Api::IoError*) {} -SocketInterfaceSwap::SocketInterfaceSwap() { +SocketInterfaceSwap::SocketInterfaceSwap(Network::Socket::Type socket_type) + : write_matcher_(std::make_shared(socket_type)) { Envoy::Network::SocketInterfaceSingleton::clear(); test_socket_interface_loader_ = std::make_unique( std::make_unique( diff --git a/test/integration/socket_interface_swap.h b/test/integration/socket_interface_swap.h index 676b8989453e..f5f3b168ef68 100644 --- a/test/integration/socket_interface_swap.h +++ b/test/integration/socket_interface_swap.h @@ -14,10 +14,13 @@ class SocketInterfaceSwap { // Object of this class hold the state determining the IoHandle which // should return the supplied return from the `writev` or `sendmsg` calls. struct IoHandleMatcher { + explicit IoHandleMatcher(Network::Socket::Type type) : socket_type_(type) {} + Network::IoSocketError* returnOverride(Envoy::Network::TestIoSocketHandle* io_handle) { absl::MutexLock lock(&mutex_); - if (error_ && (io_handle->localAddress()->ip()->port() == src_port_ || - (dst_port_ && io_handle->peerAddress()->ip()->port() == dst_port_))) { + if (socket_type_ == io_handle->getSocketType() && error_ && + (io_handle->localAddress()->ip()->port() == src_port_ || + (dst_port_ && io_handle->peerAddress()->ip()->port() == dst_port_))) { ASSERT(matched_iohandle_ == nullptr || matched_iohandle_ == io_handle, "Matched multiple io_handles, expected at most one to match."); matched_iohandle_ = io_handle; @@ -59,9 +62,10 @@ class SocketInterfaceSwap { uint32_t dst_port_ ABSL_GUARDED_BY(mutex_) = 0; Network::IoSocketError* error_ ABSL_GUARDED_BY(mutex_) = nullptr; Network::TestIoSocketHandle* matched_iohandle_{}; + Network::Socket::Type socket_type_; }; - SocketInterfaceSwap(); + explicit SocketInterfaceSwap(Network::Socket::Type socket_type); ~SocketInterfaceSwap() { test_socket_interface_loader_.reset(); @@ -70,7 +74,7 @@ class SocketInterfaceSwap { Envoy::Network::SocketInterface* const previous_socket_interface_{ Envoy::Network::SocketInterfaceSingleton::getExisting()}; - std::shared_ptr write_matcher_{std::make_shared()}; + std::shared_ptr write_matcher_; std::unique_ptr test_socket_interface_loader_; }; From 0b7a589a51a396372fb1e234ca24acfc74e8e97b Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Tue, 23 May 2023 10:19:42 -0400 Subject: [PATCH 333/740] Adding allow_mode_override API in ext_proc filter config. (#27542) * Adding allow_mode_override API in ext_proc filter config. Signed-off-by: Yanjun Xiang --- .../extensions/filters/http/ext_proc/v3/ext_proc.proto | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto index dce9b24cfd74..02fa12d7455c 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto @@ -101,7 +101,7 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // ` object in a namespace matching the filter // name. // -// [#next-free-field: 14] +// [#next-free-field: 15] message ExternalProcessor { // Configuration for the gRPC service that the filter will communicate with. // The filter supports both the "Envoy" and "Google" gRPC clients. @@ -196,6 +196,14 @@ message ExternalProcessor { // will be added to StreamInfo's filter state under the namespace corresponding to the // ext_proc filter name. google.protobuf.Struct filter_metadata = 13; + + // [#not-implemented-hide:] + // If ``allow_mode_override`` is set to true, the filter config :ref:`processing_mode + // ` + // can be overridden by the response message from the external processing server + // :ref:`mode_override `. + // If not set, ``mode_override`` API in the response message will be ignored. + bool allow_mode_override = 14; } // The HeaderForwardingRules structure specifies what headers are From 6ea3cf4625d550d74b2a506e28162e95489b6ea0 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 23 May 2023 12:59:09 -0400 Subject: [PATCH 334/740] dns: changing cache keys to include port to avoid issues (#27407) adding port to DNS cache key so that you can have a DFP filter which points to both secure and insecure clusters. Previously, this would result in failed connection attempts as the created and inserted hosts would share a key (hostname no port) but have incompatible addresses (addresses with ports) Risk Level: medium Testing: unit, integration Release Notes: inline Fixes #27331 Signed-off-by: Alyssa Wilk --- .github/workflows/mobile-coverage.yml | 2 +- changelogs/current.yaml | 5 + .../kotlin/integration/MixedSchemeTest.kt | 101 ++++++++ source/common/http/header_utility.cc | 13 + source/common/http/header_utility.h | 5 + source/common/runtime/runtime_features.cc | 1 + .../clusters/dynamic_forward_proxy/cluster.cc | 33 ++- .../common/dynamic_forward_proxy/dns_cache.h | 20 ++ .../dynamic_forward_proxy/dns_cache_impl.cc | 6 +- .../network/sni_dynamic_forward_proxy/BUILD | 1 + .../sni_dynamic_forward_proxy/proxy_filter.cc | 7 + test/common/http/header_utility_test.cc | 20 ++ .../dynamic_forward_proxy/cluster_test.cc | 88 ++++--- .../dns_cache_impl_test.cc | 238 ++++++++++-------- .../proxy_filter_integration_test.cc | 42 +++- 15 files changed, 425 insertions(+), 157 deletions(-) create mode 100644 mobile/test/kotlin/integration/MixedSchemeTest.kt diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml index e6068b9c2510..cdb1cf2a9853 100644 --- a/.github/workflows/mobile-coverage.yml +++ b/.github/workflows/mobile-coverage.yml @@ -38,7 +38,7 @@ jobs: cd mobile && BAZEL_BUILD_OPTION_LIST="--config=remote-ci-linux-coverage" \ PATH=/opt/llvm/bin:${PATH} \ COVERAGE_THRESHOLD=76 \ - ../test/run_envoy_bazel_coverage.sh //test/common/... + ../test/run_envoy_bazel_coverage.sh //test/common/... //test/cc/... - name: 'Package coverage' if: steps.should_run.outputs.run_ci_job == 'true' run: | diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 1e8249a02621..c73629831412 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -33,6 +33,11 @@ minor_behavior_changes: behavior can be reverted by setting runtime flag ``envoy.reloadable_features.http1_allow_codec_error_response_after_1xx_headers`` to false. +- area: dns + change: | + Changing the DNS cache to use host:port as the cache key rather than host. This allows a + downstream DFP filter to serve both secure and insecure clusters. This behavioral change + can be reverted by setting runtime flag ``envoy.reloadable_features.dfp_mixed_scheme`` to false. - area: uhv change: | Preserve case of %-encoded triplets in the default header validator. This behavior can be reverted by setting runtime flag diff --git a/mobile/test/kotlin/integration/MixedSchemeTest.kt b/mobile/test/kotlin/integration/MixedSchemeTest.kt new file mode 100644 index 000000000000..ec054681e1c4 --- /dev/null +++ b/mobile/test/kotlin/integration/MixedSchemeTest.kt @@ -0,0 +1,101 @@ +package test.kotlin.integration + +import io.envoyproxy.envoymobile.LogLevel +import io.envoyproxy.envoymobile.Standard +import io.envoyproxy.envoymobile.EngineBuilder +import io.envoyproxy.envoymobile.RequestHeadersBuilder +import io.envoyproxy.envoymobile.RequestMethod +import io.envoyproxy.envoymobile.engine.JniLibrary +import java.nio.ByteBuffer +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.fail +import org.junit.Test + +private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" + +class ReceiveDataTest { + + init { + JniLibrary.loadTestLibrary() + } + + @Test + fun `response headers and response data call onResponseHeaders and onResponseData`() { + + val engine = EngineBuilder(Standard()) + .addLogLevel(LogLevel.TRACE) + .build() + val client = engine.streamClient() + + // httpi + val headersExpectation2 = CountDownLatch(1) + val dataExpectation2 = CountDownLatch(1) + + var status2: Int? = null + + val requestHeaders2 = RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "http", + authority = "example.com", + path = "/" + ) + .build() + + client.newStreamPrototype() + .setOnResponseHeaders { responseHeaders, _, _ -> + status2 = responseHeaders.httpStatus + headersExpectation2.countDown() + } + .setOnResponseData { data, _, _ -> + dataExpectation2.countDown() + } + .setOnError { _, _ -> + fail("Unexpected error") + } + .start() + .sendHeaders(requestHeaders2, true) + + headersExpectation2.await(10, TimeUnit.SECONDS) + dataExpectation2.await(10, TimeUnit.SECONDS) + + assertThat(headersExpectation2.count).isEqualTo(0) + assertThat(dataExpectation2.count).isEqualTo(0) + + // https + val requestHeaders = RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/" + ) + .build() + + val headersExpectation = CountDownLatch(1) + val dataExpectation = CountDownLatch(1) + + var status: Int? = null + client.newStreamPrototype() + .setOnResponseHeaders { responseHeaders, _, _ -> + status = responseHeaders.httpStatus + headersExpectation.countDown() + } + .setOnResponseData { data, _, _ -> + dataExpectation.countDown() + } + .setOnError { _, _ -> fail("Unexpected error") } + .start() + .sendHeaders(requestHeaders, true) + + headersExpectation.await(10, TimeUnit.SECONDS) + dataExpectation.await(10, TimeUnit.SECONDS) + engine.terminate() + + assertThat(headersExpectation.count).isEqualTo(0) + assertThat(dataExpectation.count).isEqualTo(0) + + assertThat(status).isEqualTo(200) + assertThat(status2).isEqualTo(200) + } +} diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc index 23b87471061f..71f1942eb788 100644 --- a/source/common/http/header_utility.cc +++ b/source/common/http/header_utility.cc @@ -301,6 +301,19 @@ void HeaderUtility::stripTrailingHostDot(RequestHeaderMap& headers) { } } +bool HeaderUtility::hostHasPort(absl::string_view original_host) { + const absl::string_view::size_type port_start = getPortStart(original_host); + const absl::string_view port_str = original_host.substr(port_start + 1); + if (port_start == absl::string_view::npos) { + return false; + } + uint32_t port = 0; + if (!absl::SimpleAtoi(port_str, &port)) { + return false; + } + return true; +} + absl::optional HeaderUtility::stripPortFromHost(RequestHeaderMap& headers, absl::optional listener_port) { const absl::string_view original_host = headers.getHostValue(); diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h index 8ce7b2724c8f..bd63fe4c0bbe 100644 --- a/source/common/http/header_utility.h +++ b/source/common/http/header_utility.h @@ -215,6 +215,11 @@ class HeaderUtility { */ static void stripTrailingHostDot(RequestHeaderMap& headers); + /** + * @return bool true if the provided host has a port, false otherwise. + */ + static bool hostHasPort(absl::string_view host); + /** * @brief Remove the port part from host/authority header if it is equal to provided port. * @return absl::optional containing the port, if removed, else absl::nullopt. diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 7d860806d6ab..360998402fa3 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -34,6 +34,7 @@ RUNTIME_GUARD(envoy_reloadable_features_append_query_parameters_path_rewriter); RUNTIME_GUARD(envoy_reloadable_features_append_xfh_idempotent); RUNTIME_GUARD(envoy_reloadable_features_conn_pool_delete_when_idle); RUNTIME_GUARD(envoy_reloadable_features_count_unused_mapped_pages_as_free); +RUNTIME_GUARD(envoy_reloadable_features_dfp_mixed_scheme); RUNTIME_GUARD(envoy_reloadable_features_enable_aws_credentials_file); RUNTIME_GUARD(envoy_reloadable_features_enable_compression_bomb_protection); RUNTIME_GUARD(envoy_reloadable_features_enable_intermediate_ca); diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc index 3d400213b7c4..ace4989c478c 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -6,6 +6,7 @@ #include "envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.pb.h" #include "envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.pb.validate.h" #include "envoy/router/string_accessor.h" +#include "envoy/stream_info/uint32_accessor.h" #include "source/common/http/utility.h" #include "source/common/network/transport_socket_options_impl.h" @@ -322,15 +323,39 @@ Cluster::LoadBalancer::chooseHost(Upstream::LoadBalancerContext* context) { .getDataReadOnly("envoy.upstream.dynamic_host"); } - absl::string_view host; + absl::string_view raw_host; if (dynamic_host_filter_state) { - host = dynamic_host_filter_state->asString(); + raw_host = dynamic_host_filter_state->asString(); } else if (context->downstreamHeaders()) { - host = context->downstreamHeaders()->getHostValue(); + raw_host = context->downstreamHeaders()->getHostValue(); } else if (context->downstreamConnection()) { - host = context->downstreamConnection()->requestedServerName(); + raw_host = context->downstreamConnection()->requestedServerName(); } + // For host lookup, we need to make sure to match the host of any DNS cache + // insert. Two code points currently do DNS cache insert: the http DFP filter, + // which inserts for HTTP traffic, and sets port based on the cluster's + // security level, and the SNI DFP network filter which sets port based on + // stream metadata, or configuration (which is then added as stream metadata). + const bool is_secure = cluster_.info() + ->transportSocketMatcher() + .resolve(nullptr) + .factory_.implementsSecureTransport(); + uint32_t port = is_secure ? 443 : 80; + if (context->downstreamConnection()) { + const StreamInfo::UInt32Accessor* dynamic_port_filter_state = + context->downstreamConnection() + ->streamInfo() + .filterState() + .getDataReadOnly("envoy.upstream.dynamic_port"); + if (dynamic_port_filter_state != nullptr && dynamic_port_filter_state->value() > 0 && + dynamic_port_filter_state->value() <= 65535) { + port = dynamic_port_filter_state->value(); + } + } + + std::string host = Common::DynamicForwardProxy::DnsHostInfo::normalizeHostForDfp(raw_host, port); + if (host.empty()) { return nullptr; } diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache.h b/source/extensions/common/dynamic_forward_proxy/dns_cache.h index d48d24c9f5c6..89670576681a 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache.h @@ -7,6 +7,9 @@ #include "envoy/thread_local/thread_local.h" #include "envoy/upstream/resource_manager.h" +#include "source/common/http/header_utility.h" +#include "source/common/runtime/runtime_features.h" + namespace Envoy { namespace Extensions { namespace Common { @@ -17,6 +20,23 @@ namespace DynamicForwardProxy { */ class DnsHostInfo { public: + // DFP hosts are created after a DNS lookup for a domain name (e.g. foo.com) but are created + // with an IP address and port to resolve to. To prevent bugs where we insert say foo.com + // (default port 80) then later look up foo.com (default port 443) and get IP addresses with + // port 80, we "fully qualify" hostnames with the port. + // + // This normalizes hostnames, respecting the port if it exists, and adding the default port + // if there is no port. + static std::string normalizeHostForDfp(absl::string_view host, uint16_t default_port) { + if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.dfp_mixed_scheme")) { + return std::string(host); + } + if (Http::HeaderUtility::hostHasPort(host)) { + return std::string(host); + } + return absl::StrCat(host, ":", default_port); + } + virtual ~DnsHostInfo() = default; /** diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index ccded48194f3..56c99b48b4ec 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -83,8 +83,10 @@ DnsCacheStats DnsCacheImpl::generateDnsCacheStats(Stats::Scope& scope) { } DnsCacheImpl::LoadDnsCacheEntryResult -DnsCacheImpl::loadDnsCacheEntry(absl::string_view host, uint16_t default_port, bool is_proxy_lookup, - LoadDnsCacheEntryCallbacks& callbacks) { +DnsCacheImpl::loadDnsCacheEntry(absl::string_view raw_host, uint16_t default_port, + bool is_proxy_lookup, LoadDnsCacheEntryCallbacks& callbacks) { + std::string host = DnsHostInfo::normalizeHostForDfp(raw_host, default_port); + ENVOY_LOG(debug, "thread local lookup for host '{}' {}", host, is_proxy_lookup ? "proxy mode " : ""); ThreadLocalHostInfo& tls_host_info = *tls_slot_; diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD b/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD index 424c985cb9d9..1f523fd0e360 100644 --- a/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD @@ -20,6 +20,7 @@ envoy_cc_library( "//envoy/stream_info:uint32_accessor_interface", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", + "//source/common/stream_info:uint32_accessor_lib", "//source/common/tcp_proxy", "//source/extensions/common/dynamic_forward_proxy:dns_cache_interface", "@envoy_api//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3:pkg_cc_proto", diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc index d4790943f9ae..7481f6e767d6 100644 --- a/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc @@ -7,6 +7,7 @@ #include "envoy/upstream/thread_local_cluster.h" #include "source/common/common/assert.h" +#include "source/common/stream_info/uint32_accessor_impl.h" #include "source/common/tcp_proxy/tcp_proxy.h" namespace Envoy { @@ -67,6 +68,12 @@ Network::FilterStatus ProxyFilter::onNewConnection() { port = dynamic_port_filter_state->value(); } else { port = config_->port(); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.dfp_mixed_scheme")) { + read_callbacks_->connection().streamInfo().filterState()->setData( + "envoy.upstream.dynamic_port", std::make_shared(port), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + } } auto result = config_->cache().loadDnsCacheEntry(host, port, false, *this); diff --git a/test/common/http/header_utility_test.cc b/test/common/http/header_utility_test.cc index b67c1da9e286..f8ad125c256f 100644 --- a/test/common/http/header_utility_test.cc +++ b/test/common/http/header_utility_test.cc @@ -56,6 +56,26 @@ TEST_F(HeaderUtilityTest, HasHost) { } } +TEST_F(HeaderUtilityTest, HostHasPort) { + const std::vector> host_headers{ + {"localhost", false}, // w/o port part + {"localhost:443", true}, // name w/ port + {"", false}, // empty + {":443", true}, // just port + {"192.168.1.1", false}, // ipv4 + {"192.168.1.1:443", true}, // ipv4 w/ port + {"[fc00::1]:443", true}, // ipv6 w/ port + {"[fc00::1]", false}, // ipv6 + {":", false}, // malformed string #1 + {"]:", false}, // malformed string #2 + {":abc", false}, // malformed string #3 + }; + + for (const auto& host_pair : host_headers) { + EXPECT_EQ(host_pair.second, HeaderUtility::hostHasPort(host_pair.first)); + } +} + // Port's part from host header get removed TEST_F(HeaderUtilityTest, RemovePortsFromHost) { const std::vector> host_headers{ diff --git a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc index 55495f4175f3..472a301e0860 100644 --- a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc +++ b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc @@ -225,69 +225,93 @@ TEST_F(ClusterTest, CreateSubClusterConfig) { EXPECT_EQ(false, sub_cluster_pair.second.has_value()); } +TEST_F(ClusterTest, InvalidSubClusterConfig) { + const std::string bad_sub_cluster_yaml_config = R"EOF( +name: name +connect_timeout: 0.25s +cluster_type: + name: dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + sub_clusters_config: + max_sub_clusters: 1024 + lb_policy: CLUSTER_PROVIDED +)EOF"; + + EXPECT_THROW(initialize(bad_sub_cluster_yaml_config, false), EnvoyException); +} + // Basic flow of the cluster including adding hosts and removing them. TEST_F(ClusterTest, BasicFlow) { initialize(default_yaml_config_, false); - makeTestHost("host1", "1.2.3.4"); + makeTestHost("host1:0", "1.2.3.4"); InSequence s; + EXPECT_EQ(nullptr, lb_->chooseHost(setHostAndReturnContext(""))); + // Verify no host LB cases. EXPECT_EQ(nullptr, lb_->chooseHost(setHostAndReturnContext("foo"))); EXPECT_EQ(nullptr, lb_->peekAnotherHost(setHostAndReturnContext("foo"))); // LB will immediately resolve host1. EXPECT_CALL(*this, onMemberUpdateCb(SizeIs(1), SizeIs(0))); - update_callbacks_->onDnsHostAddOrUpdate("host1", host_map_["host1"]); + update_callbacks_->onDnsHostAddOrUpdate("host1:0", host_map_["host1:0"]); EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ("1.2.3.4:0", cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->address()->asString()); - EXPECT_CALL(*host_map_["host1"], touch()); - EXPECT_EQ("1.2.3.4:0", lb_->chooseHost(setHostAndReturnContext("host1"))->address()->asString()); + EXPECT_CALL(*host_map_["host1:0"], touch()); + EXPECT_EQ("1.2.3.4:0", + lb_->chooseHost(setHostAndReturnContext("host1:0"))->address()->asString()); // After changing the address, LB will immediately resolve the new address with a refresh. - updateTestHostAddress("host1", "2.3.4.5"); - update_callbacks_->onDnsHostAddOrUpdate("host1", host_map_["host1"]); + updateTestHostAddress("host1:0", "2.3.4.5"); + update_callbacks_->onDnsHostAddOrUpdate("host1:0", host_map_["host1:0"]); EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ("2.3.4.5:0", cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->address()->asString()); - EXPECT_CALL(*host_map_["host1"], touch()); - EXPECT_EQ("2.3.4.5:0", lb_->chooseHost(setHostAndReturnContext("host1"))->address()->asString()); + EXPECT_CALL(*host_map_["host1:0"], touch()); + EXPECT_EQ("2.3.4.5:0", + lb_->chooseHost(setHostAndReturnContext("host1:0"))->address()->asString()); // Remove the host, LB will immediately fail to find the host in the map. EXPECT_CALL(*this, onMemberUpdateCb(SizeIs(0), SizeIs(1))); - update_callbacks_->onDnsHostRemove("host1"); + update_callbacks_->onDnsHostRemove("host1:0"); EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); - EXPECT_EQ(nullptr, lb_->chooseHost(setHostAndReturnContext("host1"))); + EXPECT_EQ(nullptr, lb_->chooseHost(setHostAndReturnContext("host1:0"))); } // Outlier detection TEST_F(ClusterTest, OutlierDetection) { initialize(default_yaml_config_, false); - makeTestHost("host1", "1.2.3.4"); - makeTestHost("host2", "5.6.7.8"); + makeTestHost("host1:0", "1.2.3.4"); + makeTestHost("host2:0", "5.6.7.8"); InSequence s; EXPECT_CALL(*this, onMemberUpdateCb(SizeIs(1), SizeIs(0))); - update_callbacks_->onDnsHostAddOrUpdate("host1", host_map_["host1"]); - EXPECT_CALL(*host_map_["host1"], touch()); - EXPECT_EQ("1.2.3.4:0", lb_->chooseHost(setHostAndReturnContext("host1"))->address()->asString()); + update_callbacks_->onDnsHostAddOrUpdate("host1:0", host_map_["host1:0"]); + EXPECT_CALL(*host_map_["host1:0"], touch()); + EXPECT_EQ("1.2.3.4:0", + lb_->chooseHost(setHostAndReturnContext("host1:0"))->address()->asString()); EXPECT_CALL(*this, onMemberUpdateCb(SizeIs(1), SizeIs(0))); - update_callbacks_->onDnsHostAddOrUpdate("host2", host_map_["host2"]); - EXPECT_CALL(*host_map_["host2"], touch()); - EXPECT_EQ("5.6.7.8:0", lb_->chooseHost(setHostAndReturnContext("host2"))->address()->asString()); + update_callbacks_->onDnsHostAddOrUpdate("host2:0", host_map_["host2:0"]); + EXPECT_CALL(*host_map_["host2:0"], touch()); + EXPECT_EQ("5.6.7.8:0", + lb_->chooseHost(setHostAndReturnContext("host2:0"))->address()->asString()); // Fail outlier check for host1 - setOutlierFailed("host1"); - EXPECT_EQ(nullptr, lb_->chooseHost(setHostAndReturnContext("host1"))); - // "host2" should not be affected - EXPECT_CALL(*host_map_["host2"], touch()); - EXPECT_EQ("5.6.7.8:0", lb_->chooseHost(setHostAndReturnContext("host2"))->address()->asString()); + setOutlierFailed("host1:0"); + EXPECT_EQ(nullptr, lb_->chooseHost(setHostAndReturnContext("host1:0"))); + // "host2:0" should not be affected + EXPECT_CALL(*host_map_["host2:0"], touch()); + EXPECT_EQ("5.6.7.8:0", + lb_->chooseHost(setHostAndReturnContext("host2:0"))->address()->asString()); // Clear outlier check failure for host1, it should be available again - clearOutlierFailed("host1"); - EXPECT_CALL(*host_map_["host1"], touch()); - EXPECT_EQ("1.2.3.4:0", lb_->chooseHost(setHostAndReturnContext("host1"))->address()->asString()); + clearOutlierFailed("host1:0"); + EXPECT_CALL(*host_map_["host1:0"], touch()); + EXPECT_EQ("1.2.3.4:0", + lb_->chooseHost(setHostAndReturnContext("host1:0"))->address()->asString()); } // Various invalid LB context permutations in case the cluster is used outside of HTTP. @@ -300,19 +324,19 @@ TEST_F(ClusterTest, InvalidLbContext) { TEST_F(ClusterTest, FilterStateHostOverride) { initialize(default_yaml_config_, false); - makeTestHost("host1", "1.2.3.4"); + makeTestHost("host1:0", "1.2.3.4"); EXPECT_CALL(*this, onMemberUpdateCb(SizeIs(1), SizeIs(0))); - update_callbacks_->onDnsHostAddOrUpdate("host1", host_map_["host1"]); - EXPECT_CALL(*host_map_["host1"], touch()); + update_callbacks_->onDnsHostAddOrUpdate("host1:0", host_map_["host1:0"]); + EXPECT_CALL(*host_map_["host1:0"], touch()); EXPECT_EQ("1.2.3.4:0", - lb_->chooseHost(setFilterStateHostAndReturnContext("host1"))->address()->asString()); + lb_->chooseHost(setFilterStateHostAndReturnContext("host1:0"))->address()->asString()); } // Verify cluster attaches to a populated cache. TEST_F(ClusterTest, PopulatedCache) { - makeTestHost("host1", "1.2.3.4"); - makeTestHost("host2", "1.2.3.5"); + makeTestHost("host1:0", "1.2.3.4"); + makeTestHost("host2:0", "1.2.3.5"); initialize(default_yaml_config_, false); EXPECT_EQ(2UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); } diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index 5fba329071a9..b33e2ed770b4 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -126,18 +126,18 @@ void verifyCaresDnsConfigAndUnpack( TEST_F(DnsCacheImplTest, PreresolveSuccess) { Network::DnsResolver::ResolveCb resolve_cb; - std::string hostname = "bar.baz.com"; - EXPECT_CALL(*resolver_, resolve(hostname, _, _)) + std::string hostname = "bar.baz.com:443"; + EXPECT_CALL(*resolver_, resolve("bar.baz.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - EXPECT_CALL( - update_callbacks_, - onDnsHostAddOrUpdate("bar.baz.com", DnsHostInfoEquals("10.0.0.1:443", "bar.baz.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("bar.baz.com", + onDnsHostAddOrUpdate("bar.baz.com:443", + DnsHostInfoEquals("10.0.0.1:443", "bar.baz.com", false))); + EXPECT_CALL(update_callbacks_, + onDnsResolutionComplete("bar.baz.com:443", DnsHostInfoEquals("10.0.0.1:443", "bar.baz.com", false), Network::DnsResolver::ResolutionStatus::Success)); - initialize({"bar.baz.com"} /* preresolve_hostnames */); + initialize({"bar.baz.com:443"} /* preresolve_hostnames */); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({"10.0.0.1"})); @@ -145,7 +145,7 @@ TEST_F(DnsCacheImplTest, PreresolveSuccess) { 1 /* added */, 0 /* removed */, 1 /* num hosts */); MockLoadDnsCacheEntryCallbacks callbacks; - auto result = dns_cache_->loadDnsCacheEntry("bar.baz.com", 80, false, callbacks); + auto result = dns_cache_->loadDnsCacheEntry("bar.baz.com", 443, false, callbacks); EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::InCache, result.status_); EXPECT_EQ(result.handle_, nullptr); EXPECT_NE(absl::nullopt, result.host_info_); @@ -178,12 +178,14 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { 1 /* added */, 0 /* removed */, 1 /* num hosts */); EXPECT_CALL(*timeout_timer, disableTimer()); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -204,7 +206,8 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { // Address does not change. EXPECT_CALL(*timeout_timer, disableTimer()); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -224,10 +227,12 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { // Address does change. EXPECT_CALL(*timeout_timer, disableTimer()); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); - EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -295,11 +300,11 @@ TEST_F(DnsCacheImplTest, Ipv4Address) { EXPECT_CALL(*timeout_timer, disableTimer()); EXPECT_CALL( update_callbacks_, - onDnsHostAddOrUpdate("127.0.0.1", DnsHostInfoEquals("127.0.0.1:80", "127.0.0.1", true))); + onDnsHostAddOrUpdate("127.0.0.1:80", DnsHostInfoEquals("127.0.0.1:80", "127.0.0.1", true))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("127.0.0.1:80", "127.0.0.1", true))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("127.0.0.1", + onDnsResolutionComplete("127.0.0.1:80", DnsHostInfoEquals("127.0.0.1:80", "127.0.0.1", true), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); @@ -358,10 +363,10 @@ TEST_F(DnsCacheImplTest, Ipv6Address) { EXPECT_CALL(*timeout_timer, disableTimer()); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("[::1]", DnsHostInfoEquals("[::1]:80", "::1", true))); + onDnsHostAddOrUpdate("[::1]:80", DnsHostInfoEquals("[::1]:80", "::1", true))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("[::1]:80", "::1", true))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("[::1]", DnsHostInfoEquals("[::1]:80", "::1", true), + onDnsResolutionComplete("[::1]:80", DnsHostInfoEquals("[::1]:80", "::1", true), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -418,12 +423,14 @@ TEST_F(DnsCacheImplTest, TTL) { 1 /* added */, 0 /* removed */, 1 /* num hosts */); EXPECT_CALL(*timeout_timer, disableTimer()); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(6000), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -443,7 +450,8 @@ TEST_F(DnsCacheImplTest, TTL) { EXPECT_CALL(*timeout_timer, disableTimer()); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(6000), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -454,7 +462,7 @@ TEST_F(DnsCacheImplTest, TTL) { // Re-resolve with ~1m passed. This is not realistic as we would have re-resolved many times // during this period but it's good enough for the test. simTime().advanceTimeWait(std::chrono::seconds(60000)); - EXPECT_CALL(update_callbacks_, onDnsHostRemove("foo.com")); + EXPECT_CALL(update_callbacks_, onDnsHostRemove("foo.com:80")); resolve_timer->invokeCallback(); checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 1 /* removed */, 0 /* num hosts */); @@ -492,12 +500,14 @@ TEST_F(DnsCacheImplTest, TTLWithMinRefreshRate) { EXPECT_EQ(absl::nullopt, result.host_info_); EXPECT_CALL(*timeout_timer, disableTimer()); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(45000), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -525,12 +535,14 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { EXPECT_EQ(absl::nullopt, result.host_info_); EXPECT_CALL(*timeout_timer, disableTimer()); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -544,7 +556,8 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { resolve_timer->invokeCallback(); EXPECT_CALL(*timeout_timer, disableTimer()); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -552,7 +565,7 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { // Re-resolve with ~30s passed. TTL should expire. simTime().advanceTimeWait(std::chrono::milliseconds(30001)); - EXPECT_CALL(update_callbacks_, onDnsHostRemove("foo.com")); + EXPECT_CALL(update_callbacks_, onDnsHostRemove("foo.com:80")); resolve_timer->invokeCallback(); } @@ -584,11 +597,11 @@ TEST_F(DnsCacheImplTest, InlineResolve) { EXPECT_CALL(*timeout_timer, disableTimer()); EXPECT_CALL( update_callbacks_, - onDnsHostAddOrUpdate("localhost", DnsHostInfoEquals("127.0.0.1:80", "localhost", false))); + onDnsHostAddOrUpdate("localhost:80", DnsHostInfoEquals("127.0.0.1:80", "localhost", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("127.0.0.1:80", "localhost", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("localhost", + onDnsResolutionComplete("localhost:80", DnsHostInfoEquals("127.0.0.1:80", "localhost", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); @@ -619,7 +632,7 @@ TEST_F(DnsCacheImplTest, ResolveTimeout) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoAddressIsNull())); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoAddressIsNull(), + onDnsResolutionComplete("foo.com:80", DnsHostInfoAddressIsNull(), Network::DnsResolver::ResolutionStatus::Failure)); // The resolve timeout will be the default TTL as there was no specific TTL // overriding. @@ -654,7 +667,7 @@ TEST_F(DnsCacheImplTest, ResolveFailure) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoAddressIsNull())); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoAddressIsNull(), + onDnsResolutionComplete("foo.com:80", DnsHostInfoAddressIsNull(), Network::DnsResolver::ResolutionStatus::Failure)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(configured_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Failure, TestUtility::makeDnsResponse({})); @@ -706,7 +719,7 @@ TEST_F(DnsCacheImplTest, ResolveFailureWithFailureRefreshRate) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoAddressIsNull())); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoAddressIsNull(), + onDnsResolutionComplete("foo.com:80", DnsHostInfoAddressIsNull(), Network::DnsResolver::ResolutionStatus::Failure)); ON_CALL(context_.api_.random_, random()).WillByDefault(Return(8000)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(1000), _)); @@ -756,7 +769,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithEmptyResult) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoAddressIsNull())); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoAddressIsNull(), + onDnsResolutionComplete("foo.com:80", DnsHostInfoAddressIsNull(), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(configured_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({})); @@ -797,10 +810,12 @@ TEST_F(DnsCacheImplTest, CancelResolve) { EXPECT_EQ(absl::nullopt, result.host_info_); result.handle_.reset(); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); - EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({"10.0.0.1"})); @@ -827,14 +842,16 @@ TEST_F(DnsCacheImplTest, MultipleResolveSameHost) { EXPECT_NE(result2.handle_, nullptr); EXPECT_EQ(absl::nullopt, result2.host_info_); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks2, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks1, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({"10.0.0.1"})); @@ -865,22 +882,26 @@ TEST_F(DnsCacheImplTest, MultipleResolveDifferentHost) { EXPECT_EQ(absl::nullopt, result2.host_info_); EXPECT_EQ(dns_cache_->getHost("bar.com"), absl::nullopt); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("bar.com", DnsHostInfoEquals("10.0.0.1:443", "bar.com", false))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("bar.com:443", DnsHostInfoEquals("10.0.0.1:443", "bar.com", false))); EXPECT_CALL(callbacks2, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:443", "bar.com", false))); - EXPECT_CALL(update_callbacks_, onDnsResolutionComplete( - "bar.com", DnsHostInfoEquals("10.0.0.1:443", "bar.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + EXPECT_CALL(update_callbacks_, + onDnsResolutionComplete("bar.com:443", + DnsHostInfoEquals("10.0.0.1:443", "bar.com", false), + Network::DnsResolver::ResolutionStatus::Success)); resolve_cb2(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({"10.0.0.1"})); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); EXPECT_CALL(callbacks1, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); resolve_cb1(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({"10.0.0.2"})); @@ -889,14 +910,14 @@ TEST_F(DnsCacheImplTest, MultipleResolveDifferentHost) { dns_cache_->iterateHostMap( [&](absl::string_view host, const DnsHostInfoSharedPtr& info) { hosts.emplace(host, info); }); EXPECT_EQ(2, hosts.size()); - EXPECT_THAT(hosts["bar.com"], DnsHostInfoEquals("10.0.0.1:443", "bar.com", false)); - EXPECT_THAT(hosts["foo.com"], DnsHostInfoEquals("10.0.0.2:80", "foo.com", false)); + EXPECT_THAT(hosts["bar.com:443"], DnsHostInfoEquals("10.0.0.1:443", "bar.com", false)); + EXPECT_THAT(hosts["foo.com:80"], DnsHostInfoEquals("10.0.0.2:80", "foo.com", false)); - EXPECT_TRUE(dns_cache_->getHost("bar.com").has_value()); - EXPECT_THAT(dns_cache_->getHost("bar.com").value(), + EXPECT_TRUE(dns_cache_->getHost("bar.com:443").has_value()); + EXPECT_THAT(dns_cache_->getHost("bar.com:443").value(), DnsHostInfoEquals("10.0.0.1:443", "bar.com", false)); - EXPECT_TRUE(dns_cache_->getHost("foo.com").has_value()); - EXPECT_THAT(dns_cache_->getHost("foo.com").value(), + EXPECT_TRUE(dns_cache_->getHost("foo.com:80").has_value()); + EXPECT_THAT(dns_cache_->getHost("foo.com:80").value(), DnsHostInfoEquals("10.0.0.2:80", "foo.com", false)); EXPECT_EQ(dns_cache_->getHost("baz.com"), absl::nullopt); } @@ -915,12 +936,14 @@ TEST_F(DnsCacheImplTest, CacheHit) { EXPECT_NE(result.handle_, nullptr); EXPECT_EQ(absl::nullopt, result.host_info_); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({"10.0.0.1"})); @@ -946,44 +969,24 @@ TEST_F(DnsCacheImplTest, CacheHitWithDifferentDefaultPort) { EXPECT_NE(result.handle_, nullptr); EXPECT_EQ(absl::nullopt, result.host_info_); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({"10.0.0.1"})); - // Using the same DNS cache, resolve the same host but different default port 443, - // Envoy returns LoadDnsCacheEntryStatus::InCache since the DNS cache - // key is the host string, which does not include the port number. - // This means if HTTP and HTTPS traffic are destined to DFP clusters using same DNS - // cache table, it will cause traffic forwarding issues. - result = dns_cache_->loadDnsCacheEntry("foo.com", 443, false, callbacks); - EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::InCache, result.status_); - - // The way to resolve this is to config a different dns_cache_config name for HTTP and - // HTTPS DFP clusters. Then they will not conflict. - envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config_https; - // Set the dns_cache_config name to "bar" to make the HTTPS cache entries - // stored and looked up in a separate DNS cache table. - config_https.set_name("bar"); - config_https.set_dns_lookup_family(envoy::config::cluster::v3::Cluster::V4_ONLY); - std::unique_ptr dns_cache_https; - dns_cache_https = std::make_unique(context_, config_https); - MockUpdateCallbacks update_callbacks_https; - DnsCache::AddUpdateCallbacksHandlePtr update_callbacks_handle_https; - update_callbacks_handle_https = dns_cache_https->addUpdateCallbacks(update_callbacks_https); - MockLoadDnsCacheEntryCallbacks callbacks_https; - Network::DnsResolver::ResolveCb resolve_cb_https; + // Using the same DNS cache, resolve the same host but different default port 443 + // This currently will re-resolve as we don't have an optimization to look up + // cache entries with different ports. EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) - .WillOnce(DoAll(SaveArg<2>(&resolve_cb_https), Return(&resolver_->active_query_))); - // So, now to resolve the same host and port 443 in DNS cache table "bar", it doesn't find - // an existing entry and returns LoadDnsCacheEntryStatus::Loading as expected. - auto result_https = dns_cache_https->loadDnsCacheEntry("foo.com", 80, false, callbacks_https); - EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result_https.status_); + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + result = dns_cache_->loadDnsCacheEntry("foo.com", 443, false, callbacks); } // Make sure we destroy active queries if the cache goes away. @@ -1022,7 +1025,7 @@ TEST_F(DnsCacheImplTest, InvalidPort) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoAddressIsNull())); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com:abc", DnsHostInfoAddressIsNull(), + onDnsResolutionComplete("foo.com:abc:80", DnsHostInfoAddressIsNull(), Network::DnsResolver::ResolutionStatus::Success)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({})); } @@ -1381,13 +1384,15 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { EXPECT_CALL(*timeout_timer, disableTimer()); // Make sure the store gets the first insert. - EXPECT_CALL(*store, addOrUpdate("foo.com", "10.0.0.1:80|6|0", kNoTtl)); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); + EXPECT_CALL(*store, addOrUpdate("foo.com:80", "10.0.0.1:80|6|0", kNoTtl)); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(6000), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -1407,9 +1412,10 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { // Address does not change. EXPECT_CALL(*timeout_timer, disableTimer()); - EXPECT_CALL(*store, addOrUpdate("foo.com", "10.0.0.1:80|6|0", kNoTtl)); + EXPECT_CALL(*store, addOrUpdate("foo.com:80", "10.0.0.1:80|6|0", kNoTtl)); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -1429,11 +1435,13 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { EXPECT_CALL(*timeout_timer, disableTimer()); // Make sure the store gets the updated addresses - EXPECT_CALL(*store, addOrUpdate("foo.com", "10.0.0.2:80|6|0\n10.0.0.1:80|6|0", kNoTtl)); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); + EXPECT_CALL(*store, addOrUpdate("foo.com:80", "10.0.0.2:80|6|0\n10.0.0.1:80|6|0", kNoTtl)); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -1452,9 +1460,10 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { // Address does not change. EXPECT_CALL(*timeout_timer, disableTimer()); - EXPECT_CALL(*store, addOrUpdate("foo.com", "10.0.0.2:80|40|0\n10.0.0.1:80|40|0", kNoTtl)); + EXPECT_CALL(*store, addOrUpdate("foo.com:80", "10.0.0.2:80|40|0\n10.0.0.1:80|40|0", kNoTtl)); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), + onDnsResolutionComplete("foo.com:80", + DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), Network::DnsResolver::ResolutionStatus::Success)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(40000), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, @@ -1480,13 +1489,13 @@ TEST_F(DnsCacheImplTest, CacheLoad) { store = ret.get(); // Make sure there's an attempt to load from the key value store. EXPECT_CALL(*store, iterate).WillOnce(Invoke([&](KeyValueStore::ConstIterateCb fn) { - fn("foo.com", "10.0.0.2:80|40|0"); - fn("bar.com", "1.1.1.1:1|20|1\n2.2.2.2:2|30|2"); + fn("foo.com:80", "10.0.0.2:80|40|0"); + fn("bar.com:80", "1.1.1.1:1|20|1\n2.2.2.2:2|30|2"); // No port. EXPECT_LOG_CONTAINS("warning", "Unable to parse cache line '1.1.1.1|20|1'", - fn("eep.com", "1.1.1.1|20|1")); + fn("eep.com:80", "1.1.1.1|20|1")); // Won't be loaded because of prior error. - fn("eep.com", "1.1.1.1|20|1:1"); + fn("eep.com:80", "1.1.1.1|20|1:1"); })); return ret; @@ -1532,6 +1541,19 @@ TEST(DnsCacheManagerImplTest, TestLifetime) { EXPECT_TRUE(cache_manager->getCache(config1) != nullptr); } +TEST(NoramlizeHost, NormalizeHost) { + EXPECT_EQ("localhost:80", DnsHostInfo::normalizeHostForDfp("localhost:80", 80)); + EXPECT_EQ("localhost:80", DnsHostInfo::normalizeHostForDfp("localhost:80", 443)); + EXPECT_EQ("localhost:443", DnsHostInfo::normalizeHostForDfp("localhost:443", 80)); + EXPECT_EQ("localhost:443", DnsHostInfo::normalizeHostForDfp("localhost:443", 443)); + EXPECT_EQ("localhost:123", DnsHostInfo::normalizeHostForDfp("localhost:123", 80)); + EXPECT_EQ("localhost:80", DnsHostInfo::normalizeHostForDfp("localhost", 80)); + EXPECT_EQ("localhost:443", DnsHostInfo::normalizeHostForDfp("localhost", 443)); + + EXPECT_EQ("[fc00::1]:443", DnsHostInfo::normalizeHostForDfp("[fc00::1]:443", 80)); + EXPECT_EQ("[fc00::1]:80", DnsHostInfo::normalizeHostForDfp("[fc00::1]", 80)); +} + } // namespace } // namespace DynamicForwardProxy } // namespace Common diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 38ee775b6174..8a4885eedf14 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -222,10 +222,12 @@ name: envoy.clusters.dynamic_forward_proxy ASSERT_FALSE(response->headers().get(Http::LowerCaseString("dns_start")).empty()); ASSERT_FALSE(response->headers().get(Http::LowerCaseString("dns_end")).empty()); - const Extensions::TransportSockets::Tls::SslHandshakerImpl* ssl_socket = - dynamic_cast( - fake_upstream_connection_->connection().ssl().get()); - EXPECT_STREQ("localhost", SSL_get_servername(ssl_socket->ssl(), TLSEXT_NAMETYPE_host_name)); + if (upstream_tls_) { + const Extensions::TransportSockets::Tls::SslHandshakerImpl* ssl_socket = + dynamic_cast( + fake_upstream_connection_->connection().ssl().get()); + EXPECT_STREQ("localhost", SSL_get_servername(ssl_socket->ssl(), TLSEXT_NAMETYPE_host_name)); + } } void requestWithUnknownDomainTest(const std::string& typed_dns_resolver_config = "") { @@ -256,7 +258,7 @@ name: envoy.clusters.dynamic_forward_proxy }; int64_t getHeaderValue(const Http::ResponseHeaderMap& headers, absl::string_view name) { - EXPECT_FALSE(headers.get(Http::LowerCaseString(name)).empty()); + EXPECT_FALSE(headers.get(Http::LowerCaseString(name)).empty()) << "Mising " << name; int64_t val; if (!headers.get(Http::LowerCaseString(name)).empty() && absl::SimpleAtoi(headers.get(Http::LowerCaseString(name))[0]->value().getStringView(), @@ -268,11 +270,11 @@ int64_t getHeaderValue(const Http::ResponseHeaderMap& headers, absl::string_view void ProxyFilterIntegrationTest::testConnectionTiming(IntegrationStreamDecoderPtr& response, bool cached_dns, int64_t original_usec) { + int64_t handshake_end; int64_t dns_start = getHeaderValue(response->headers(), "dns_start"); int64_t dns_end = getHeaderValue(response->headers(), "dns_end"); int64_t connect_start = getHeaderValue(response->headers(), "upstream_connect_start"); int64_t connect_end = getHeaderValue(response->headers(), "upstream_connect_complete"); - int64_t handshake_end = getHeaderValue(response->headers(), "upstream_handshake_complete"); int64_t request_send_end = getHeaderValue(response->headers(), "request_send_end"); int64_t response_begin = getHeaderValue(response->headers(), "response_begin"); Event::DispatcherImpl dispatcher("foo", *api_, timeSystem()); @@ -284,11 +286,14 @@ void ProxyFilterIntegrationTest::testConnectionTiming(IntegrationStreamDecoderPt } else { ASSERT_LE(dns_end, connect_start); } + if (upstream_tls_) { + handshake_end = getHeaderValue(response->headers(), "upstream_handshake_complete"); + ASSERT_LE(connect_end, handshake_end); + ASSERT_LE(handshake_end, request_send_end); + ASSERT_LT(handshake_end, timeSystem().monotonicTime().time_since_epoch().count()); + } ASSERT_LE(connect_start, connect_end); - ASSERT_LE(connect_end, handshake_end); - ASSERT_LE(handshake_end, request_send_end); ASSERT_LE(request_send_end, response_begin); - ASSERT_LT(handshake_end, timeSystem().monotonicTime().time_since_epoch().count()); } class ProxyFilterWithSimtimeIntegrationTest : public Event::TestUsingSimulatedTime, @@ -305,6 +310,23 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ProxyFilterIntegrationTest, // should hit the TLS cache. TEST_P(ProxyFilterIntegrationTest, RequestWithBody) { requestWithBodyTest(); } +TEST_P(ProxyFilterIntegrationTest, MultiPortTest) { + upstream_tls_ = false; + requestWithBodyTest(); + + // Create a second upstream, and send a request there. + // The second upstream is autonomous where the first was not so we'll only get a 200-ok if we hit + // the new port. + // this regression tests https://github.com/envoyproxy/envoy/issues/27331 + autonomous_upstream_ = true; + createUpstream(Network::Test::getCanonicalLoopbackAddress(version_), upstreamConfig()); + default_request_headers_.setHost( + fmt::format("localhost:{}", fake_upstreams_[1]->localAddress()->ip()->port())); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + // Do a sanity check using the getaddrinfo() resolver. TEST_P(ProxyFilterIntegrationTest, RequestWithBodyGetAddrInfoResolver) { // getaddrinfo() does not reliably return v6 addresses depending on the environment. For now @@ -448,7 +470,7 @@ TEST_P(ProxyFilterIntegrationTest, DNSCacheHostOverflow) { // Verify that the filter works without TLS. TEST_P(ProxyFilterIntegrationTest, UpstreamCleartext) { - upstream_tls_ = true; + upstream_tls_ = false; initializeWithArgs(); codec_client_ = makeHttpConnection(lookupPort("http")); const Http::TestRequestHeaderMapImpl request_headers{ From c99607b614ba9dcb73204c06b190bc2d43389a1d Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 23 May 2023 13:06:09 -0400 Subject: [PATCH 335/740] mobile: removing unused test filters (#27550) Signed-off-by: Alyssa Wilk --- mobile/envoy_build_config/BUILD | 2 - mobile/envoy_build_config/test_extensions.cc | 4 -- .../filters/http/test_accessor/BUILD | 44 -------------- .../filters/http/test_accessor/config.cc | 28 --------- .../filters/http/test_accessor/config.h | 36 ------------ .../filters/http/test_accessor/filter.cc | 31 ---------- .../filters/http/test_accessor/filter.h | 46 --------------- .../filters/http/test_accessor/filter.proto | 10 ---- .../extensions/filters/http/test_read/BUILD | 42 -------------- .../filters/http/test_read/config.cc | 24 -------- .../filters/http/test_read/config.h | 33 ----------- .../filters/http/test_read/filter.cc | 57 ------------------- .../filters/http/test_read/filter.h | 35 ------------ .../filters/http/test_read/filter.proto | 6 -- 14 files changed, 398 deletions(-) delete mode 100644 mobile/library/common/extensions/filters/http/test_accessor/BUILD delete mode 100644 mobile/library/common/extensions/filters/http/test_accessor/config.cc delete mode 100644 mobile/library/common/extensions/filters/http/test_accessor/config.h delete mode 100644 mobile/library/common/extensions/filters/http/test_accessor/filter.cc delete mode 100644 mobile/library/common/extensions/filters/http/test_accessor/filter.h delete mode 100644 mobile/library/common/extensions/filters/http/test_accessor/filter.proto delete mode 100644 mobile/library/common/extensions/filters/http/test_read/BUILD delete mode 100644 mobile/library/common/extensions/filters/http/test_read/config.cc delete mode 100644 mobile/library/common/extensions/filters/http/test_read/config.h delete mode 100644 mobile/library/common/extensions/filters/http/test_read/filter.cc delete mode 100644 mobile/library/common/extensions/filters/http/test_read/filter.h delete mode 100644 mobile/library/common/extensions/filters/http/test_read/filter.proto diff --git a/mobile/envoy_build_config/BUILD b/mobile/envoy_build_config/BUILD index 3b3c27bddcf1..5f520f25a89f 100644 --- a/mobile/envoy_build_config/BUILD +++ b/mobile/envoy_build_config/BUILD @@ -117,11 +117,9 @@ envoy_cc_library( "@envoy//source/extensions/filters/http/buffer:config", "@envoy_mobile//library/common/extensions/filters/http/assertion:config", "@envoy_mobile//library/common/extensions/filters/http/route_cache_reset:config", - "@envoy_mobile//library/common/extensions/filters/http/test_accessor:config", "@envoy_mobile//library/common/extensions/filters/http/test_event_tracker:config", "@envoy_mobile//library/common/extensions/filters/http/test_kv_store:config", "@envoy_mobile//library/common/extensions/filters/http/test_logger:config", - "@envoy_mobile//library/common/extensions/filters/http/test_read:config", "@envoy_mobile//library/common/extensions/filters/http/test_remote_response:config", ], alwayslink = 1, diff --git a/mobile/envoy_build_config/test_extensions.cc b/mobile/envoy_build_config/test_extensions.cc index d0bb9c56087b..f97634722058 100644 --- a/mobile/envoy_build_config/test_extensions.cc +++ b/mobile/envoy_build_config/test_extensions.cc @@ -4,20 +4,16 @@ #include "external/envoy_build_config/test_extensions.h" #include "library/common/extensions/filters/http/assertion/config.h" #include "library/common/extensions/filters/http/route_cache_reset/config.h" -#include "library/common/extensions/filters/http/test_accessor/config.h" #include "library/common/extensions/filters/http/test_event_tracker/config.h" #include "library/common/extensions/filters/http/test_kv_store/config.h" #include "library/common/extensions/filters/http/test_logger/config.h" -#include "library/common/extensions/filters/http/test_read/config.h" void register_test_extensions() { Envoy::Extensions::HttpFilters::Assertion::forceRegisterAssertionFilterFactory(); Envoy::Extensions::HttpFilters::BufferFilter::forceRegisterBufferFilterFactory(); Envoy::Extensions::HttpFilters::RouteCacheReset::forceRegisterRouteCacheResetFilterFactory(); - Envoy::Extensions::HttpFilters::TestAccessor::forceRegisterTestAccessorFilterFactory(); Envoy::Extensions::HttpFilters::TestEventTracker::forceRegisterTestEventTrackerFilterFactory(); Envoy::Extensions::HttpFilters::TestKeyValueStore::forceRegisterTestKeyValueStoreFilterFactory(); Envoy::Extensions::HttpFilters::TestLogger::forceRegisterFactory(); - Envoy::HttpFilters::TestRead::forceRegisterTestReadFilterFactory(); Envoy::Upstream::forceRegisterStaticClusterFactory(); } diff --git a/mobile/library/common/extensions/filters/http/test_accessor/BUILD b/mobile/library/common/extensions/filters/http/test_accessor/BUILD deleted file mode 100644 index 977df4ab6783..000000000000 --- a/mobile/library/common/extensions/filters/http/test_accessor/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -load( - "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", - "envoy_proto_library", -) - -licenses(["notice"]) # Apache 2 - -envoy_extension_package() - -envoy_proto_library( - name = "filter", - srcs = ["filter.proto"], - deps = [ - "@envoy_api//envoy/config/common/matcher/v3:pkg", - ], -) - -envoy_cc_extension( - name = "test_accessor_filter_lib", - srcs = ["filter.cc"], - hdrs = ["filter.h"], - repository = "@envoy", - deps = [ - "filter_cc_proto", - "//library/common/api:c_types", - "//library/common/api:external_api_lib", - "//library/common/data:utility_lib", - "@envoy//source/common/common:assert_lib", - "@envoy//source/extensions/filters/http/common:pass_through_filter_lib", - ], -) - -envoy_cc_extension( - name = "config", - srcs = ["config.cc"], - hdrs = ["config.h"], - repository = "@envoy", - deps = [ - ":test_accessor_filter_lib", - "@envoy//source/extensions/filters/http/common:factory_base_lib", - ], -) diff --git a/mobile/library/common/extensions/filters/http/test_accessor/config.cc b/mobile/library/common/extensions/filters/http/test_accessor/config.cc deleted file mode 100644 index 44deda4b9561..000000000000 --- a/mobile/library/common/extensions/filters/http/test_accessor/config.cc +++ /dev/null @@ -1,28 +0,0 @@ -#include "library/common/extensions/filters/http/test_accessor/config.h" - -#include "library/common/extensions/filters/http/test_accessor/filter.h" - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace TestAccessor { - -Http::FilterFactoryCb TestAccessorFilterFactory::createFilterFactoryFromProtoTyped( - const envoymobile::extensions::filters::http::test_accessor::TestAccessor& proto_config, - const std::string&, Server::Configuration::FactoryContext&) { - - auto config = std::make_shared(proto_config); - return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addStreamFilter(std::make_shared(config)); - }; -} - -/** - * Static registration for the TestAccessor filter. @see NamedHttpFilterConfigFactory. - */ -REGISTER_FACTORY(TestAccessorFilterFactory, Server::Configuration::NamedHttpFilterConfigFactory); - -} // namespace TestAccessor -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_accessor/config.h b/mobile/library/common/extensions/filters/http/test_accessor/config.h deleted file mode 100644 index 44bd26007b76..000000000000 --- a/mobile/library/common/extensions/filters/http/test_accessor/config.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once - -#include - -#include "source/extensions/filters/http/common/factory_base.h" - -#include "library/common/extensions/filters/http/test_accessor/filter.h" -#include "library/common/extensions/filters/http/test_accessor/filter.pb.h" -#include "library/common/extensions/filters/http/test_accessor/filter.pb.validate.h" - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace TestAccessor { - -/** - * Config registration for the TestAccessor filter. @see NamedHttpFilterConfigFactory. - */ -class TestAccessorFilterFactory - : public Common::FactoryBase< - envoymobile::extensions::filters::http::test_accessor::TestAccessor> { -public: - TestAccessorFilterFactory() : FactoryBase("test_accessor") {} - -private: - ::Envoy::Http::FilterFactoryCb createFilterFactoryFromProtoTyped( - const envoymobile::extensions::filters::http::test_accessor::TestAccessor& config, - const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; -}; - -DECLARE_FACTORY(TestAccessorFilterFactory); - -} // namespace TestAccessor -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_accessor/filter.cc b/mobile/library/common/extensions/filters/http/test_accessor/filter.cc deleted file mode 100644 index e35f152dd004..000000000000 --- a/mobile/library/common/extensions/filters/http/test_accessor/filter.cc +++ /dev/null @@ -1,31 +0,0 @@ -#include "library/common/extensions/filters/http/test_accessor/filter.h" - -#include "envoy/server/filter_config.h" - -#include "source/common/common/assert.h" - -#include "library/common/data/utility.h" - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace TestAccessor { - -TestAccessorFilterConfig::TestAccessorFilterConfig( - const envoymobile::extensions::filters::http::test_accessor::TestAccessor& proto_config) - : accessor_(static_cast( - Api::External::retrieveApi(proto_config.accessor_name()))), - expected_string_(proto_config.expected_string()) {} - -Http::FilterHeadersStatus TestAccessorFilter::decodeHeaders(Http::RequestHeaderMap&, bool) { - RELEASE_ASSERT(config_->expectedString() == - Data::Utility::copyToString( - config_->accessor()->get_string(config_->accessor()->context)), - "accessed string is not equal to expected string"); - return Http::FilterHeadersStatus::Continue; -} - -} // namespace TestAccessor -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_accessor/filter.h b/mobile/library/common/extensions/filters/http/test_accessor/filter.h deleted file mode 100644 index d5a7355c19e3..000000000000 --- a/mobile/library/common/extensions/filters/http/test_accessor/filter.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include "envoy/http/filter.h" - -#include "source/extensions/filters/http/common/pass_through_filter.h" - -#include "library/common/api/c_types.h" -#include "library/common/api/external.h" -#include "library/common/extensions/filters/http/test_accessor/filter.pb.h" - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace TestAccessor { - -class TestAccessorFilterConfig { -public: - TestAccessorFilterConfig( - const envoymobile::extensions::filters::http::test_accessor::TestAccessor& proto_config); - - const envoy_string_accessor* accessor() const { return accessor_; } - const std::string& expectedString() const { return expected_string_; } - -private: - const envoy_string_accessor* accessor_; - const std::string expected_string_; -}; - -using TestAccessorFilterConfigSharedPtr = std::shared_ptr; - -class TestAccessorFilter final : public ::Envoy::Http::PassThroughFilter { -public: - TestAccessorFilter(TestAccessorFilterConfigSharedPtr config) : config_(config) {} - - // StreamDecoderFilter - ::Envoy::Http::FilterHeadersStatus decodeHeaders(::Envoy::Http::RequestHeaderMap& headers, - bool end_stream) override; - -private: - const TestAccessorFilterConfigSharedPtr config_; -}; - -} // namespace TestAccessor -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_accessor/filter.proto b/mobile/library/common/extensions/filters/http/test_accessor/filter.proto deleted file mode 100644 index 938da1b89d95..000000000000 --- a/mobile/library/common/extensions/filters/http/test_accessor/filter.proto +++ /dev/null @@ -1,10 +0,0 @@ -syntax = "proto3"; - -package envoymobile.extensions.filters.http.test_accessor; - -import "validate/validate.proto"; - -message TestAccessor { - string accessor_name = 1 [(validate.rules).string.min_len = 1]; - string expected_string = 2 [(validate.rules).string.min_len = 1]; -} diff --git a/mobile/library/common/extensions/filters/http/test_read/BUILD b/mobile/library/common/extensions/filters/http/test_read/BUILD deleted file mode 100644 index 54324cca68ac..000000000000 --- a/mobile/library/common/extensions/filters/http/test_read/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -load( - "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", - "envoy_proto_library", -) - -licenses(["notice"]) # Apache 2 - -envoy_extension_package() - -envoy_proto_library( - name = "filter", - srcs = ["filter.proto"], - deps = [ - "@envoy_api//envoy/config/common/matcher/v3:pkg", - ], -) - -envoy_cc_extension( - name = "test_read_filter_lib", - srcs = ["filter.cc"], - hdrs = ["filter.h"], - repository = "@envoy", - deps = [ - "filter_cc_proto", - "@envoy//source/common/http:utility_lib", - "@envoy//source/common/stream_info:stream_info_lib", - "@envoy//source/extensions/filters/http/common:pass_through_filter_lib", - ], -) - -envoy_cc_extension( - name = "config", - srcs = ["config.cc"], - hdrs = ["config.h"], - repository = "@envoy", - deps = [ - ":test_read_filter_lib", - "@envoy//source/extensions/filters/http/common:factory_base_lib", - ], -) diff --git a/mobile/library/common/extensions/filters/http/test_read/config.cc b/mobile/library/common/extensions/filters/http/test_read/config.cc deleted file mode 100644 index 3c5ce3c402f8..000000000000 --- a/mobile/library/common/extensions/filters/http/test_read/config.cc +++ /dev/null @@ -1,24 +0,0 @@ -#include "library/common/extensions/filters/http/test_read/config.h" - -#include "library/common/extensions/filters/http/test_read/filter.h" - -namespace Envoy { -namespace HttpFilters { -namespace TestRead { - -Http::FilterFactoryCb TestReadFilterFactory::createFilterFactoryFromProtoTyped( - const envoymobile::test::integration::filters::http::test_read::TestRead& /*config*/, - const std::string&, Server::Configuration::FactoryContext& /*context*/) { - return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addStreamDecoderFilter(std::make_shared()); - }; -} - -/** - * Static registration for the TestRead filter. @see NamedHttpFilterConfigFactory. - */ -REGISTER_FACTORY(TestReadFilterFactory, Server::Configuration::NamedHttpFilterConfigFactory); - -} // namespace TestRead -} // namespace HttpFilters -} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_read/config.h b/mobile/library/common/extensions/filters/http/test_read/config.h deleted file mode 100644 index 24e22d706cb1..000000000000 --- a/mobile/library/common/extensions/filters/http/test_read/config.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include - -#include "source/extensions/filters/http/common/factory_base.h" - -#include "library/common/extensions/filters/http/test_read/filter.pb.h" -#include "library/common/extensions/filters/http/test_read/filter.pb.validate.h" - -namespace Envoy { -namespace HttpFilters { -namespace TestRead { - -/** - * Config registration for the TestRead filter. @see NamedHttpFilterConfigFactory. - */ -class TestReadFilterFactory - : public Envoy::Extensions::HttpFilters::Common::FactoryBase< - envoymobile::test::integration::filters::http::test_read::TestRead> { -public: - TestReadFilterFactory() : FactoryBase("test_read") {} - -private: - ::Envoy::Http::FilterFactoryCb createFilterFactoryFromProtoTyped( - const envoymobile::test::integration::filters::http::test_read::TestRead& config, - const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; -}; - -DECLARE_FACTORY(TestReadFilterFactory); - -} // namespace TestRead -} // namespace HttpFilters -} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.cc b/mobile/library/common/extensions/filters/http/test_read/filter.cc deleted file mode 100644 index fb7c42752727..000000000000 --- a/mobile/library/common/extensions/filters/http/test_read/filter.cc +++ /dev/null @@ -1,57 +0,0 @@ -#include "library/common/extensions/filters/http/test_read/filter.h" - -#include "envoy/server/filter_config.h" - -namespace Envoy { -namespace HttpFilters { -namespace TestRead { - -Http::FilterHeadersStatus TestReadFilter::decodeHeaders(Http::RequestHeaderMap& request_headers, - bool) { - // sample path is /failed?error=0x10000 - Http::Utility::QueryParams query_parameters = - Http::Utility::parseQueryString(request_headers.Path()->value().getStringView()); - auto error = query_parameters.find("error"); - uint64_t response_flag; - if (error != query_parameters.end() && absl::SimpleAtoi(error->second, &response_flag)) { - // set response error code - StreamInfo::StreamInfo& stream_info = decoder_callbacks_->streamInfo(); - stream_info.setResponseFlag(TestReadFilter::mapErrorToResponseFlag(response_flag)); - - // check if we want a quic server error: sample path is /failed?quic=1&error=0x10000 - if (query_parameters.find("quic") != query_parameters.end()) { - stream_info.setUpstreamInfo(std::make_shared()); - stream_info.upstreamInfo()->setUpstreamProtocol(Http::Protocol::Http3); - } - - // trigger the error and stop iteration to other filters - decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, "test_read filter threw: ", nullptr, - absl::nullopt, ""); - return Http::FilterHeadersStatus::StopIteration; - } - - // continue to other filters since the provided query string is invalid for error simulation - return Http::FilterHeadersStatus::Continue; -} - -StreamInfo::ResponseFlag TestReadFilter::mapErrorToResponseFlag(uint64_t errorCode) { - switch (errorCode) { - case 0x4000000: - return StreamInfo::DnsResolutionFailed; - case 0x40: - return StreamInfo::UpstreamConnectionTermination; - case 0x20: - return StreamInfo::UpstreamConnectionFailure; - case 0x10: - return StreamInfo::UpstreamRemoteReset; - case 0x10000: - return StreamInfo::StreamIdleTimeout; - default: - // Any other error that we aren't interested in. I picked a random error. - return StreamInfo::RateLimitServiceError; - } -} - -} // namespace TestRead -} // namespace HttpFilters -} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.h b/mobile/library/common/extensions/filters/http/test_read/filter.h deleted file mode 100644 index dc5c2e73cea2..000000000000 --- a/mobile/library/common/extensions/filters/http/test_read/filter.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include "envoy/http/filter.h" - -#include "source/common/common/logger.h" -#include "source/common/http/utility.h" -#include "source/common/stream_info/stream_info_impl.h" -#include "source/extensions/filters/http/common/pass_through_filter.h" - -#include "library/common/extensions/filters/http/test_read/filter.pb.h" - -namespace Envoy { -namespace HttpFilters { -namespace TestRead { - -/** - * This is a test-only filter to return specified error code based on a request url query string. - * It either simulates the requested error if the url query matches the error patterns - * or does nothing - */ -class TestReadFilter final : public Http::PassThroughFilter, - public Logger::Loggable { -public: - Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& request_headers, bool) override; - -private: - /* A mapping of the envoymobile errors we care about for testing - * From https://github.com/envoyproxy/envoy/blob/main/envoy/stream_info/stream_info.h - */ - StreamInfo::ResponseFlag mapErrorToResponseFlag(uint64_t errorCode); -}; - -} // namespace TestRead -} // namespace HttpFilters -} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.proto b/mobile/library/common/extensions/filters/http/test_read/filter.proto deleted file mode 100644 index de164df7d450..000000000000 --- a/mobile/library/common/extensions/filters/http/test_read/filter.proto +++ /dev/null @@ -1,6 +0,0 @@ -syntax = "proto3"; - -package envoymobile.test.integration.filters.http.test_read; - -message TestRead { -} From 17e5ff881d11463b8af817e918d46b8b65fce4d6 Mon Sep 17 00:00:00 2001 From: Alex Xu Date: Wed, 24 May 2023 01:21:46 +0800 Subject: [PATCH 336/740] ci: fix the flaky of TcpProxyOdcdsIntegrationTest.ShutdownAllConnectionsOnClusterLookupTimeout (#27578) Signed-off-by: He Jie Xu --- test/integration/tcp_proxy_odcds_integration_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/tcp_proxy_odcds_integration_test.cc b/test/integration/tcp_proxy_odcds_integration_test.cc index e4081b5bf10c..7eab2092ef6a 100644 --- a/test/integration/tcp_proxy_odcds_integration_test.cc +++ b/test/integration/tcp_proxy_odcds_integration_test.cc @@ -303,8 +303,8 @@ TEST_P(TcpProxyOdcdsIntegrationTest, ShutdownAllConnectionsOnClusterLookupTimeou IntegrationTcpClientPtr tcp_client_2 = makeTcpConnection(lookupPort("tcp_proxy")); test_server_->waitForCounterEq("tcp.tcpproxy_stats.on_demand_cluster_attempt", 2); - tcp_client_1->waitForHalfClose(); - tcp_client_2->waitForHalfClose(); + tcp_client_1->waitForHalfClose(true); + tcp_client_2->waitForHalfClose(true); assertOnDemandCounters(0, 0, 2); tcp_client_1->close(); tcp_client_2->close(); From 7c4bbe1785ad41a868dec0b66f4ea06e802bce95 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 23 May 2023 10:40:29 -0700 Subject: [PATCH 337/740] bazel: remove tags from repository definition (#27583) I don't think these do anything here except for produce this warning: ``` DEBUG: Rule 'com_github_bufbuild_buf' indicated that a canonical reproducible form can be obtained by dropping arguments ["tags"] DEBUG: Repository com_github_bufbuild_buf instantiated at: /Users/ksmiley/dev/envoy4/WORKSPACE:9:23: in /Users/ksmiley/dev/envoy4/bazel/api_repositories.bzl:4:21: in envoy_api_dependencies /private/var/tmp/_bazel_ksmiley/81424bb29de8eeef22a825a179047d5f/external/envoy_api/bazel/repositories.bzl:47:26: in api_dependencies /private/var/tmp/_bazel_ksmiley/81424bb29de8eeef22a825a179047d5f/external/envoy_api/bazel/repositories.bzl:9:23: in external_http_archive /private/var/tmp/_bazel_ksmiley/81424bb29de8eeef22a825a179047d5f/external/envoy_api/bazel/envoy_http_archive.bzl:16:17: in envoy_http_archive ``` Signed-off-by: Keith Smiley --- api/bazel/repositories.bzl | 1 - api/bazel/repository_locations.bzl | 1 - 2 files changed, 2 deletions(-) diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index b23156e21f12..b49db8c59a0c 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -47,7 +47,6 @@ def api_dependencies(): external_http_archive( name = "com_github_bufbuild_buf", build_file_content = BUF_BUILD_CONTENT, - tags = ["manual"], ) PROMETHEUSMETRICS_BUILD_CONTENT = """ diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 45015ca109f1..c259e3633f93 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -137,7 +137,6 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/bufbuild/buf/releases/download/v{version}/buf-Linux-x86_64.tar.gz"], release_date = "2023-02-09", use_category = ["api"], - tags = ["manual"], license = "Apache-2.0", license_url = "https://github.com/bufbuild/buf/blob/v{version}/LICENSE", ), From 67e4108c3a78c75cfc2237f876a316115429e1de Mon Sep 17 00:00:00 2001 From: Chris Schoener Date: Tue, 23 May 2023 15:48:47 -0400 Subject: [PATCH 338/740] Mobile: Replace direct response APIs with test server in Swift (#27429) This is a followup to #25582 We migrate the swift tests to use a mock test server. And then delete the direct response engine builder functionality entirely. Signed-off-by: caschoener --- mobile/bazel/apple.bzl | 3 +- mobile/library/cc/engine_builder.cc | 131 ------------------ mobile/library/cc/engine_builder.h | 5 - .../library/objective-c/EnvoyConfiguration.h | 3 - .../library/objective-c/EnvoyConfiguration.mm | 7 - mobile/library/swift/BUILD | 1 - mobile/library/swift/EngineBuilder.swift | 15 -- mobile/library/swift/TestEngineBuilder.swift | 19 --- mobile/test/common/integration/test_server.cc | 31 +++-- mobile/test/common/integration/test_server.h | 17 ++- .../integration/test_server_interface.cc | 23 ++- .../integration/test_server_interface.h | 9 +- mobile/test/common/jni/test_jni_interface.cc | 19 +-- .../engine/testing/QuicTestServerTest.java | 4 +- .../envoymobile/engine/testing/TestJni.java | 26 +--- .../test/kotlin/integration/SendDataTest.kt | 2 +- mobile/test/objective-c/BUILD | 16 ++- mobile/test/objective-c/EnvoyTestServer.h | 17 +++ mobile/test/objective-c/EnvoyTestServer.mm | 25 ++++ mobile/test/swift/integration/BUILD | 80 +---------- ...eContainsHeadersMatchIntegrationTest.swift | 60 -------- ...onseExactHeadersMatchIntegrationTest.swift | 59 -------- ...esponseExactPathMatchIntegrationTest.swift | 53 ------- ...esponseFilterMutationIntegrationTest.swift | 93 ------------- ...nsePrefixHeadersMatchIntegrationTest.swift | 59 -------- ...nseSuffixHeadersMatchIntegrationTest.swift | 59 -------- ...est.swift => EndToEndNetworkingTest.swift} | 25 ++-- 27 files changed, 154 insertions(+), 707 deletions(-) delete mode 100644 mobile/library/swift/TestEngineBuilder.swift create mode 100644 mobile/test/objective-c/EnvoyTestServer.h create mode 100644 mobile/test/objective-c/EnvoyTestServer.mm delete mode 100644 mobile/test/swift/integration/DirectResponseContainsHeadersMatchIntegrationTest.swift delete mode 100644 mobile/test/swift/integration/DirectResponseExactHeadersMatchIntegrationTest.swift delete mode 100644 mobile/test/swift/integration/DirectResponseExactPathMatchIntegrationTest.swift delete mode 100644 mobile/test/swift/integration/DirectResponseFilterMutationIntegrationTest.swift delete mode 100644 mobile/test/swift/integration/DirectResponsePrefixHeadersMatchIntegrationTest.swift delete mode 100644 mobile/test/swift/integration/DirectResponseSuffixHeadersMatchIntegrationTest.swift rename mobile/test/swift/integration/{DirectResponsePrefixPathMatchIntegrationTest.swift => EndToEndNetworkingTest.swift} (70%) diff --git a/mobile/bazel/apple.bzl b/mobile/bazel/apple.bzl index 209c8f31cedf..33945b329e7b 100644 --- a/mobile/bazel/apple.bzl +++ b/mobile/bazel/apple.bzl @@ -3,7 +3,7 @@ load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_defines") load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") -def envoy_objc_library(name, hdrs = [], visibility = [], data = [], deps = [], module_name = None, sdk_frameworks = [], srcs = []): +def envoy_objc_library(name, hdrs = [], visibility = [], data = [], deps = [], module_name = None, sdk_frameworks = [], srcs = [], testonly = False): native.objc_library( name = name, srcs = srcs, @@ -15,6 +15,7 @@ def envoy_objc_library(name, hdrs = [], visibility = [], data = [], deps = [], m visibility = visibility, data = data, deps = deps, + testonly = testonly, ) # Macros providing a way to easily/consistently define Swift/ObjC unit test targets. diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc index 767132fb2d4f..bc4bce48509c 100644 --- a/mobile/library/cc/engine_builder.cc +++ b/mobile/library/cc/engine_builder.cc @@ -286,12 +286,6 @@ EngineBuilder& EngineBuilder::setRuntimeGuard(std::string guard, bool value) { return *this; } -EngineBuilder& -EngineBuilder::addDirectResponse(DirectResponseTesting::DirectResponse direct_response) { - direct_responses_.push_back(direct_response); - return *this; -} - EngineBuilder& EngineBuilder::setOverrideConfigForTests(std::string config) { config_override_for_tests_ = std::move(config); return *this; @@ -319,46 +313,6 @@ std::unique_ptr EngineBuilder::generate remote_service->set_name("remote_service"); remote_service->add_domains("127.0.0.1"); - for (auto& direct_response_in : direct_responses_) { - auto* direct_response_route = remote_service->add_routes(); - auto* direct_response = direct_response_route->mutable_direct_response(); - direct_response->set_status(direct_response_in.status); - direct_response->mutable_body()->set_inline_string(direct_response_in.body); - auto* direct_response_route_match = direct_response_route->mutable_match(); - auto matcher = direct_response_in.matcher; - if (!matcher.fullPath.empty()) { - direct_response_route_match->set_path(matcher.fullPath); - } else if (!matcher.pathPrefix.empty()) { - direct_response_route_match->set_prefix(matcher.pathPrefix); - } - - for (auto& header : matcher.headers) { - auto* direct_response_headers = direct_response_route_match->add_headers(); - direct_response_headers->set_name(header.name); - switch (header.mode) { - case DirectResponseTesting::MatchMode::Contains: - direct_response_headers->set_contains_match(header.value); - break; - case DirectResponseTesting::MatchMode::Exact: - direct_response_headers->set_exact_match(header.value); - break; - case DirectResponseTesting::MatchMode::Prefix: - direct_response_headers->set_prefix_match(header.value); - break; - case DirectResponseTesting::MatchMode::Suffix: - direct_response_headers->set_suffix_match(header.value); - break; - } - } - - for (auto& header_in : direct_response_in.headers) { - auto* resp_header = direct_response_route->add_response_headers_to_add(); - auto* header = resp_header->mutable_header(); - header->set_key(header_in.first); - header->set_value(header_in.second); - } - } - auto* route = remote_service->add_routes(); route->mutable_match()->set_prefix("/"); route->mutable_direct_response()->set_status(404); @@ -370,38 +324,6 @@ std::unique_ptr EngineBuilder::generate api_service->set_include_attempt_count_in_response(true); api_service->add_domains("*"); - for (auto& direct_response_in : direct_responses_) { - auto* this_route = api_service->add_routes(); - auto* mutable_route = this_route->mutable_route(); - mutable_route->set_cluster("fake_remote"); - auto* direct_response_route_match = this_route->mutable_match(); - auto matcher = direct_response_in.matcher; - if (!matcher.fullPath.empty()) { - direct_response_route_match->set_path(matcher.fullPath); - } else if (!matcher.pathPrefix.empty()) { - direct_response_route_match->set_prefix(matcher.pathPrefix); - } - - for (auto& header : matcher.headers) { - auto* direct_response_headers = direct_response_route_match->add_headers(); - direct_response_headers->set_name(header.name); - switch (header.mode) { - case DirectResponseTesting::MatchMode::Contains: - direct_response_headers->set_contains_match(header.value); - break; - case DirectResponseTesting::MatchMode::Exact: - direct_response_headers->set_exact_match(header.value); - break; - case DirectResponseTesting::MatchMode::Prefix: - direct_response_headers->set_prefix_match(header.value); - break; - case DirectResponseTesting::MatchMode::Suffix: - direct_response_headers->set_suffix_match(header.value); - break; - } - } - } - route = api_service->add_routes(); route->mutable_match()->set_prefix("/"); route->add_request_headers_to_remove("x-forwarded-proto"); @@ -540,14 +462,6 @@ std::unique_ptr EngineBuilder::generate tag_filter->mutable_typed_config()->PackFrom(tag_config); } - if (!direct_responses_.empty()) { - auto* cache_reset_filter = hcm->add_http_filters(); - cache_reset_filter->set_name("envoy.filters.http.route_cache_reset"); - cache_reset_filter->mutable_typed_config()->set_type_url( - "type.googleapis.com/" - "envoymobile.extensions.filters.http.route_cache_reset.RouteCacheReset"); - } - // Set up the always-present filters envoymobile::extensions::filters::http::network_configuration::NetworkConfiguration network_config; @@ -617,35 +531,6 @@ std::unique_ptr EngineBuilder::generate auto* static_resources = bootstrap->mutable_static_resources(); - if (!direct_responses_.empty()) { - auto* fake_remote_listener = static_resources->add_listeners(); - fake_remote_listener->set_name("fake_remote_listener"); - auto* base_address = fake_remote_listener->mutable_address(); - base_address->mutable_socket_address()->set_address("127.0.0.1"); - base_address->mutable_socket_address()->set_port_value(10101); - auto* filter = fake_remote_listener->add_filter_chains()->add_filters(); - envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager - fake_remote_listener_config; - filter->set_name("envoy.filters.network.http_connection_manager"); - fake_remote_listener_config.set_stat_prefix("remote_hcm"); - auto* route_config = fake_remote_listener_config.mutable_route_config(); - route_config->set_name("remote_route"); - auto* virtual_host = route_config->add_virtual_hosts(); - virtual_host->add_domains("*"); - virtual_host->set_name("remote_service"); - auto* route = virtual_host->add_routes(); - route->mutable_match()->set_prefix("/"); - route->mutable_direct_response()->set_status(404); - route->mutable_direct_response()->mutable_body()->set_inline_string("not found"); - route->add_request_headers_to_remove("x-forwarded-proto"); - route->add_request_headers_to_remove("x-envoy-mobile-cluster"); - auto* router_filter = fake_remote_listener_config.add_http_filters(); - envoy::extensions::filters::http::router::v3::Router router_config; - router_filter->set_name("envoy.router"); - router_filter->mutable_typed_config()->PackFrom(router_config); - filter->mutable_typed_config()->PackFrom(fake_remote_listener_config); - } - // Finally create the base listener, and point it at the HCM. auto* base_listener = static_resources->add_listeners(); base_listener->set_name("base_api_listener"); @@ -696,22 +581,6 @@ std::unique_ptr EngineBuilder::generate base_tls_socket.set_name("envoy.transport_sockets.http_11_proxy"); base_tls_socket.mutable_typed_config()->PackFrom(ssl_proxy_socket); - if (!direct_responses_.empty()) { - // fake remote cluster - auto* fake_remote_cluster = static_resources->add_clusters(); - fake_remote_cluster->set_name("fake_remote"); - fake_remote_cluster->set_type(envoy::config::cluster::v3::Cluster::LOGICAL_DNS); - fake_remote_cluster->mutable_connect_timeout()->set_seconds(30); - fake_remote_cluster->mutable_load_assignment()->set_cluster_name("fake_remote"); - auto* address = fake_remote_cluster->mutable_load_assignment() - ->add_endpoints() - ->add_lb_endpoints() - ->mutable_endpoint() - ->mutable_address(); - address->mutable_socket_address()->set_address("127.0.0.1"); - address->mutable_socket_address()->set_port_value(10101); - } - envoy::extensions::upstreams::http::v3::HttpProtocolOptions h2_protocol_options; h2_protocol_options.mutable_explicit_http_config()->mutable_http2_protocol_options(); if (!stats_domain_.empty()) { diff --git a/mobile/library/cc/engine_builder.h b/mobile/library/cc/engine_builder.h index 63dcc36374ac..4eedcaac8296 100644 --- a/mobile/library/cc/engine_builder.h +++ b/mobile/library/cc/engine_builder.h @@ -104,10 +104,6 @@ class EngineBuilder { EngineBuilder& setRuntimeGuard(std::string guard, bool value); - // Add a direct response. For testing purposes only. - // TODO(jpsim): Move this out of the main engine builder API - EngineBuilder& addDirectResponse(DirectResponseTesting::DirectResponse direct_response); - // These functions don't affect the Bootstrap configuration but instead perform registrations. EngineBuilder& addKeyValueStore(std::string name, KeyValueStoreSharedPtr key_value_store); EngineBuilder& addStringAccessor(std::string name, StringAccessorSharedPtr accessor); @@ -189,7 +185,6 @@ class EngineBuilder { std::vector native_filter_chain_; std::vector dns_preresolve_hostnames_; - std::vector direct_responses_; std::vector> runtime_guards_; absl::flat_hash_map string_accessors_; diff --git a/mobile/library/objective-c/EnvoyConfiguration.h b/mobile/library/objective-c/EnvoyConfiguration.h index 313f77ab1ba5..7c2afa305c42 100644 --- a/mobile/library/objective-c/EnvoyConfiguration.h +++ b/mobile/library/objective-c/EnvoyConfiguration.h @@ -39,7 +39,6 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic, strong) NSString *appVersion; @property (nonatomic, strong) NSString *appId; @property (nonatomic, strong) NSDictionary *runtimeGuards; -@property (nonatomic, strong) NSArray *typedDirectResponses; @property (nonatomic, strong) NSArray *nativeFilterChain; @property (nonatomic, strong) NSArray *httpPlatformFilterFactories; @property (nonatomic, strong) NSDictionary *stringAccessors; @@ -94,8 +93,6 @@ NS_ASSUME_NONNULL_BEGIN appId:(NSString *)appId runtimeGuards: (NSDictionary *)runtimeGuards - typedDirectResponses: - (NSArray *)typedDirectResponses nativeFilterChain: (NSArray *)nativeFilterChain platformFilterChain: diff --git a/mobile/library/objective-c/EnvoyConfiguration.mm b/mobile/library/objective-c/EnvoyConfiguration.mm index aa76f9135d75..b75d01bb85a2 100644 --- a/mobile/library/objective-c/EnvoyConfiguration.mm +++ b/mobile/library/objective-c/EnvoyConfiguration.mm @@ -97,8 +97,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled appId:(NSString *)appId runtimeGuards: (NSDictionary *)runtimeGuards - typedDirectResponses: - (NSArray *)typedDirectResponses nativeFilterChain: (NSArray *)nativeFilterChain platformFilterChain: @@ -160,7 +158,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled self.appVersion = appVersion; self.appId = appId; self.runtimeGuards = runtimeGuards; - self.typedDirectResponses = typedDirectResponses; self.nativeFilterChain = nativeFilterChain; self.httpPlatformFilterFactories = httpPlatformFilterFactories; self.stringAccessors = stringAccessors; @@ -211,10 +208,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled builder.setRuntimeGuard([key toCXXString], value); } - for (EMODirectResponse *directResponse in self.typedDirectResponses) { - builder.addDirectResponse([directResponse toCXX]); - } - builder.addConnectTimeoutSeconds(self.connectTimeoutSeconds); builder.addDnsFailureRefreshSeconds(self.dnsFailureRefreshSecondsBase, diff --git a/mobile/library/swift/BUILD b/mobile/library/swift/BUILD index 9ddeb5d61ca0..58b660e7b607 100644 --- a/mobile/library/swift/BUILD +++ b/mobile/library/swift/BUILD @@ -44,7 +44,6 @@ swift_library( "StreamClientImpl.swift", "StreamIntel.swift", "StreamPrototype.swift", - "TestEngineBuilder.swift", "Trailers.swift", "extensions/UserDefaults+KeyValueStore.swift", "filters/AsyncRequestFilter.swift", diff --git a/mobile/library/swift/EngineBuilder.swift b/mobile/library/swift/EngineBuilder.swift index 2e187639be2a..806548792066 100644 --- a/mobile/library/swift/EngineBuilder.swift +++ b/mobile/library/swift/EngineBuilder.swift @@ -59,7 +59,6 @@ open class EngineBuilder: NSObject { private var stringAccessors: [String: EnvoyStringAccessor] = [:] private var keyValueStores: [String: EnvoyKeyValueStore] = [:] private var runtimeGuards: [String: Bool] = [:] - private var directResponses: [DirectResponse] = [] private var statsSinks: [String] = [] private var rtdsLayerName: String? private var rtdsTimeoutSeconds: UInt32 = 0 @@ -693,15 +692,6 @@ open class EngineBuilder: NSObject { return self } - /// Add a direct response to be used when configuring the engine. - /// This function is internal so it is not publicly exposed to production builders, - /// but is available for use by the `TestEngineBuilder`. - /// - /// - parameter directResponse: The response configuration to add. - func addDirectResponseInternal(_ directResponse: DirectResponse) { - self.directResponses.append(directResponse) - } - func makeConfig() -> EnvoyConfiguration { EnvoyConfiguration( adminInterfaceEnabled: self.adminInterfaceEnabled, @@ -733,7 +723,6 @@ open class EngineBuilder: NSObject { appVersion: self.appVersion, appId: self.appId, runtimeGuards: self.runtimeGuards.mapValues({ "\($0)" }), - typedDirectResponses: self.directResponses.map({ $0.toObjC() }), nativeFilterChain: self.nativeFilterChain, platformFilterChain: self.platformFilterChain, stringAccessors: self.stringAccessors, @@ -816,10 +805,6 @@ private extension EngineBuilder { cxxBuilder.setRuntimeGuard(runtimeGuard.toCXX(), value) } - for directResponse in self.directResponses { - cxxBuilder.addDirectResponse(directResponse.toCXX()) - } - for filter in self.nativeFilterChain.reversed() { cxxBuilder.addNativeFilter(filter.name.toCXX(), filter.typedConfig.toCXX()) } diff --git a/mobile/library/swift/TestEngineBuilder.swift b/mobile/library/swift/TestEngineBuilder.swift deleted file mode 100644 index e654764b1b10..000000000000 --- a/mobile/library/swift/TestEngineBuilder.swift +++ /dev/null @@ -1,19 +0,0 @@ -import Foundation - -/// Builder that can be used to construct instances of the Envoy engine -/// which have the added functionality of returning "direct" responses to -/// requests over a local connection. This can be particularly useful for mocking/testing. -public final class TestEngineBuilder: EngineBuilder { - /// Adds a direct response configuration which will be used when starting the engine. - /// Doing so will cause Envoy to clear its route cache for each stream in order to allow - /// filters to mutate headers (which can subsequently affect routing). - /// - /// - parameter response: The response configuration to add. - /// - /// - returns: This builder. - @discardableResult - public func addDirectResponse(_ response: DirectResponse) -> Self { - self.addDirectResponseInternal(response) - return self - } -} diff --git a/mobile/test/common/integration/test_server.cc b/mobile/test/common/integration/test_server.cc index f47ef32aeb00..de0847c1821a 100644 --- a/mobile/test/common/integration/test_server.cc +++ b/mobile/test/common/integration/test_server.cc @@ -4,6 +4,7 @@ #include "test/integration/server.h" #include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" namespace Envoy { @@ -58,29 +59,35 @@ TestServer::TestServer() .WillByDefault(testing::ReturnRef(*stats_store_.rootScope())); } -void TestServer::startTestServer(bool use_quic) { +void TestServer::startTestServer(TestServerType test_server_type) { ASSERT(!upstream_); // pre-setup: see https://github.com/envoyproxy/envoy/blob/main/test/test_runner.cc Logger::Context logging_state(spdlog::level::level_enum::err, "[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v", lock, false, false); // end pre-setup + Network::DownstreamTransportSocketFactoryPtr factory; - if (use_quic) { + switch (test_server_type) { + case TestServerType::HTTP3: upstream_config_.upstream_protocol_ = Http::CodecType::HTTP3; upstream_config_.udp_fake_upstream_ = FakeUpstreamConfig::UdpConfig(); - } else { + factory = createQuicUpstreamTlsContext(factory_context_); + break; + case TestServerType::HTTP2_WITH_TLS: upstream_config_.upstream_protocol_ = Http::CodecType::HTTP2; + factory = createUpstreamTlsContext(factory_context_); + break; + case TestServerType::HTTP1_WITHOUT_TLS: + upstream_config_.upstream_protocol_ = Http::CodecType::HTTP1; + factory = Network::Test::createRawBufferDownstreamSocketFactory(); + break; } - Network::DownstreamTransportSocketFactoryPtr factory = - use_quic ? createQuicUpstreamTlsContext(factory_context_) - : createUpstreamTlsContext(factory_context_); - upstream_ = std::make_unique(std::move(factory), port_, version_, upstream_config_, true); // Legacy behavior for cronet tests. - if (use_quic) { + if (test_server_type == TestServerType::HTTP3) { upstream_->setResponseHeaders( std::make_unique(Http::TestResponseHeaderMapImpl( {{":status", "200"}, @@ -102,4 +109,12 @@ int TestServer::getServerPort() { ASSERT(upstream_); return upstream_->localAddress()->ip()->port(); } + +void TestServer::setHeadersAndData(absl::string_view header_key, absl::string_view header_value, + absl::string_view response_body) { + upstream_->setResponseHeaders( + std::make_unique(Http::TestResponseHeaderMapImpl( + {{std::string(header_key), std::string(header_value)}, {":status", "200"}}))); + upstream_->setResponseBody(std::string(response_body)); +} } // namespace Envoy diff --git a/mobile/test/common/integration/test_server.h b/mobile/test/common/integration/test_server.h index cc90122b89b0..ff5411c7ad39 100644 --- a/mobile/test/common/integration/test_server.h +++ b/mobile/test/common/integration/test_server.h @@ -10,6 +10,13 @@ #include "test/mocks/server/transport_socket_factory_context.h" namespace Envoy { + +enum class TestServerType { + HTTP1_WITHOUT_TLS, + HTTP2_WITH_TLS, + HTTP3, +}; + class TestServer { private: testing::NiceMock factory_context_; @@ -37,8 +44,9 @@ class TestServer { /** * Starts the server. Can only have one server active per JVM. This is blocking until the port can * start accepting requests. + * test_server_type: selects between HTTP3, HTTP2, or HTTP1 without TLS */ - void startTestServer(bool use_quic); + void startTestServer(TestServerType test_server_type); /** * Shutdowns the server. Can be restarted later. This is blocking until the server has freed all @@ -50,6 +58,13 @@ class TestServer { * Returns the port that got attributed. Can only be called once the server has been started. */ int getServerPort(); + + /** + * Sets headers and data for the test server to return on all future requests. + * Can only be called once the server has been started. + */ + void setHeadersAndData(absl::string_view header_key, absl::string_view header_value, + absl::string_view response_body); }; } // namespace Envoy diff --git a/mobile/test/common/integration/test_server_interface.cc b/mobile/test/common/integration/test_server_interface.cc index fc70e258c4dc..f4e18a7bb53a 100644 --- a/mobile/test/common/integration/test_server_interface.cc +++ b/mobile/test/common/integration/test_server_interface.cc @@ -9,27 +9,36 @@ static std::weak_ptr weak_test_server_; static std::shared_ptr test_server() { return weak_test_server_.lock(); } -void start_server(bool use_quic) { +void start_server(Envoy::TestServerType test_server_type) { Envoy::ExtensionRegistry::registerFactories(); strong_test_server_ = std::make_shared(); weak_test_server_ = strong_test_server_; - if (auto e = test_server()) { - e->startTestServer(use_quic); + if (auto server = test_server()) { + server->startTestServer(test_server_type); } } void shutdown_server() { // Reset the primary handle to the test_server, // but retain it long enough to synchronously shutdown. - auto e = strong_test_server_; + auto server = strong_test_server_; strong_test_server_.reset(); - e->shutdownTestServer(); + server->shutdownTestServer(); } int get_server_port() { - if (auto e = test_server()) { - return e->getServerPort(); + if (auto server = test_server()) { + return server->getServerPort(); } return -1; // failure } + +void set_headers_and_data(absl::string_view header_key, absl::string_view header_value, + absl::string_view response_body) { + if (auto server = test_server()) { + // start_server() must be called before headers and data can be added. + ASSERT(server); + server->setHeadersAndData(header_key, header_value, response_body); + } +} diff --git a/mobile/test/common/integration/test_server_interface.h b/mobile/test/common/integration/test_server_interface.h index 1bc33286270e..0b51d2a6e55f 100644 --- a/mobile/test/common/integration/test_server_interface.h +++ b/mobile/test/common/integration/test_server_interface.h @@ -12,7 +12,7 @@ extern "C" { // functions * Starts the server. Can only have one server active per JVM. This is blocking until the port can * start accepting requests. */ -void start_server(bool use_quic); +void start_server(Envoy::TestServerType test_server_type); /** * Shutdowns the server. Can be restarted later. This is blocking until the server has freed all @@ -25,6 +25,13 @@ void shutdown_server(); */ int get_server_port(); +/** + * Set response data for server to return for any URL. Can only be called once the server has been + * started. + */ +void set_headers_and_data(absl::string_view header_key, absl::string_view header_value, + absl::string_view response_body); + #ifdef __cplusplus } // functions #endif diff --git a/mobile/test/common/jni/test_jni_interface.cc b/mobile/test/common/jni/test_jni_interface.cc index 6b549d77e4ed..6cc0e31909cb 100644 --- a/mobile/test/common/jni/test_jni_interface.cc +++ b/mobile/test/common/jni/test_jni_interface.cc @@ -10,10 +10,10 @@ // Quic Test ServerJniLibrary extern "C" JNIEXPORT void JNICALL -Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartQuicTestServer(JNIEnv* env, - jclass clazz) { +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartHttp3TestServer(JNIEnv* env, + jclass clazz) { jni_log("[QTS]", "starting server"); - start_server(true); + start_server(Envoy::TestServerType::HTTP3); } extern "C" JNIEXPORT jint JNICALL @@ -24,17 +24,10 @@ Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeGetServerPort(JNIEnv } extern "C" JNIEXPORT void JNICALL -Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeShutdownQuicTestServer(JNIEnv* env, - jclass clazz) { - jni_log("[QTS]", "shutting down server"); - shutdown_server(); -} - -extern "C" JNIEXPORT void JNICALL -Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartTestServer(JNIEnv* env, - jclass clazz) { +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartHttp2TestServer(JNIEnv* env, + jclass clazz) { jni_log("[QTS]", "starting server"); - start_server(false); + start_server(Envoy::TestServerType::HTTP2_WITH_TLS); } extern "C" JNIEXPORT void JNICALL diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/QuicTestServerTest.java b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/QuicTestServerTest.java index f885d97b9244..089314ac05dc 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/QuicTestServerTest.java +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/QuicTestServerTest.java @@ -111,7 +111,7 @@ public static void loadJniLibrary() { @Before public void setUpEngine() throws Exception { - TestJni.startQuicTestServer(); + TestJni.startHttp3TestServer(); CountDownLatch latch = new CountDownLatch(1); engine = new AndroidEngineBuilder(appContext, new Custom(String.format(CONFIG, TestJni.getServerPort()))) @@ -127,7 +127,7 @@ public void setUpEngine() throws Exception { @After public void shutdownEngine() { engine.terminate(); - TestJni.shutdownQuicTestServer(); + TestJni.shutdownTestServer(); } @Test diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/TestJni.java b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/TestJni.java index d64f4d8c0492..177fc68d2155 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/TestJni.java +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/TestJni.java @@ -14,21 +14,21 @@ public final class TestJni { /* * Starts the server. Throws an {@link IllegalStateException} if already started. */ - public static void startQuicTestServer() { + public static void startHttp3TestServer() { if (!sServerRunning.compareAndSet(false, true)) { - throw new IllegalStateException("Quic server is already running"); + throw new IllegalStateException("Http3 server is already running"); } - nativeStartQuicTestServer(); + nativeStartHttp3TestServer(); } /* * Starts the server. Throws an {@link IllegalStateException} if already started. */ - public static void startTestServer() { + public static void startHttp2TestServer() { if (!sServerRunning.compareAndSet(false, true)) { throw new IllegalStateException("Server is already running"); } - nativeStartTestServer(); + nativeStartHttp2TestServer(); } /** @@ -41,16 +41,6 @@ public static void shutdownTestServer() { nativeShutdownTestServer(); } - /** - * Shutdowns the server. No-op if the server is already shutdown. - */ - public static void shutdownQuicTestServer() { - if (!sServerRunning.compareAndSet(true, false)) { - return; - } - nativeShutdownQuicTestServer(); - } - public static String getServerURL() { return "https://" + getServerHost() + ":" + getServerPort(); } @@ -71,11 +61,9 @@ public static String createYaml(EnvoyConfiguration envoyConfiguration) { return nativeCreateYaml(envoyConfiguration.createBootstrap()); } - private static native void nativeStartQuicTestServer(); - - private static native void nativeShutdownQuicTestServer(); + private static native void nativeStartHttp3TestServer(); - private static native void nativeStartTestServer(); + private static native void nativeStartHttp2TestServer(); private static native void nativeShutdownTestServer(); diff --git a/mobile/test/kotlin/integration/SendDataTest.kt b/mobile/test/kotlin/integration/SendDataTest.kt index 391553ff2ca0..e2917c1fc882 100644 --- a/mobile/test/kotlin/integration/SendDataTest.kt +++ b/mobile/test/kotlin/integration/SendDataTest.kt @@ -27,7 +27,7 @@ class SendDataTest { @Test fun `successful sending data`() { - TestJni.startTestServer(); + TestJni.startHttp2TestServer(); val port = TestJni.getServerPort(); val expectation = CountDownLatch(1) diff --git a/mobile/test/objective-c/BUILD b/mobile/test/objective-c/BUILD index 53e993aa3780..a67acb0ae0ef 100644 --- a/mobile/test/objective-c/BUILD +++ b/mobile/test/objective-c/BUILD @@ -1,4 +1,4 @@ -load("@envoy_mobile//bazel:apple.bzl", "envoy_mobile_objc_test") +load("@envoy_mobile//bazel:apple.bzl", "envoy_mobile_objc_test", "envoy_objc_library") licenses(["notice"]) # Apache 2 @@ -28,3 +28,17 @@ envoy_mobile_objc_test( "//library/objective-c:envoy_objc_bridge_lib", ], ) + +envoy_objc_library( + name = "envoy_test_server", + testonly = True, + srcs = [ + "EnvoyTestServer.mm", + ], + hdrs = ["EnvoyTestServer.h"], + module_name = "EnvoyTestServer", + visibility = ["//visibility:public"], + deps = [ + "//test/common/integration:test_server_interface_lib", + ], +) diff --git a/mobile/test/objective-c/EnvoyTestServer.h b/mobile/test/objective-c/EnvoyTestServer.h new file mode 100644 index 000000000000..651d53f4d245 --- /dev/null +++ b/mobile/test/objective-c/EnvoyTestServer.h @@ -0,0 +1,17 @@ +#import + +// Interface for starting and managing a test server. Calls into to test_server.cc +@interface EnvoyTestServer : NSObject + +// Get the port of the upstream server. ++ (NSInteger)getEnvoyPort; +// Starts a server with HTTP1 and no TLS. ++ (void)startHttp1PlaintextServer; +// Shut down and clean up server. ++ (void)shutdownTestServer; +// Add response data to the upstream. ++ (void)setHeadersAndData:(NSString *)header_key + header_value:(NSString *)header_value + response_body:(NSString *)response_body; + +@end diff --git a/mobile/test/objective-c/EnvoyTestServer.mm b/mobile/test/objective-c/EnvoyTestServer.mm new file mode 100644 index 000000000000..808863fb67cd --- /dev/null +++ b/mobile/test/objective-c/EnvoyTestServer.mm @@ -0,0 +1,25 @@ +#import "test/objective-c/EnvoyTestServer.h" +#import "test/common/integration/test_server_interface.h" + +@implementation EnvoyTestServer + ++ (NSInteger)getEnvoyPort { + return get_server_port(); +} + ++ (void)startHttp1PlaintextServer { + start_server(Envoy::TestServerType::HTTP1_WITHOUT_TLS); +} + ++ (void)shutdownTestServer { + shutdown_server(); +} + ++ (void)setHeadersAndData:(NSString *)header_key + header_value:(NSString *)header_value + response_body:(NSString *)response_body { + set_headers_and_data([header_key UTF8String], [header_value UTF8String], + [response_body UTF8String]); +} + +@end diff --git a/mobile/test/swift/integration/BUILD b/mobile/test/swift/integration/BUILD index 08be85a67018..86be3c4319e3 100644 --- a/mobile/test/swift/integration/BUILD +++ b/mobile/test/swift/integration/BUILD @@ -20,54 +20,9 @@ envoy_mobile_swift_test( ) envoy_mobile_swift_test( - name = "direct_response_contains_headers_match_test", + name = "end_to_end_networking_test", srcs = [ - "DirectResponseContainsHeadersMatchIntegrationTest.swift", - ], - tags = [ - "no-remote-exec", - ], - visibility = ["//visibility:public"], - deps = [ - ":test_extensions", - "//library/objective-c:envoy_engine_objc_lib", - ], -) - -envoy_mobile_swift_test( - name = "direct_response_exact_match_test", - srcs = [ - "DirectResponseExactPathMatchIntegrationTest.swift", - ], - tags = [ - "no-remote-exec", - ], - visibility = ["//visibility:public"], - deps = [ - ":test_extensions", - "//library/objective-c:envoy_engine_objc_lib", - ], -) - -envoy_mobile_swift_test( - name = "direct_response_filter_mutation", - srcs = [ - "DirectResponseFilterMutationIntegrationTest.swift", - ], - tags = [ - "no-remote-exec", - ], - visibility = ["//visibility:public"], - deps = [ - ":test_extensions", - "//library/objective-c:envoy_engine_objc_lib", - ], -) - -envoy_mobile_swift_test( - name = "direct_response_prefix_headers_match", - srcs = [ - "DirectResponsePrefixHeadersMatchIntegrationTest.swift", + "EndToEndNetworkingTest.swift", ], tags = [ "no-remote-exec", @@ -76,6 +31,7 @@ envoy_mobile_swift_test( deps = [ ":test_extensions", "//library/objective-c:envoy_engine_objc_lib", + "//test/objective-c:envoy_test_server", ], ) @@ -94,36 +50,6 @@ envoy_mobile_swift_test( ], ) -envoy_mobile_swift_test( - name = "direct_response_prefix_path_match", - srcs = [ - "DirectResponsePrefixPathMatchIntegrationTest.swift", - ], - tags = [ - "no-remote-exec", - ], - visibility = ["//visibility:public"], - deps = [ - ":test_extensions", - "//library/objective-c:envoy_engine_objc_lib", - ], -) - -envoy_mobile_swift_test( - name = "direct_response_suffix_headers_match", - srcs = [ - "DirectResponseSuffixHeadersMatchIntegrationTest.swift", - ], - tags = [ - "no-remote-exec", - ], - visibility = ["//visibility:public"], - deps = [ - ":test_extensions", - "//library/objective-c:envoy_engine_objc_lib", - ], -) - envoy_mobile_swift_test( name = "engine_api_test", srcs = [ diff --git a/mobile/test/swift/integration/DirectResponseContainsHeadersMatchIntegrationTest.swift b/mobile/test/swift/integration/DirectResponseContainsHeadersMatchIntegrationTest.swift deleted file mode 100644 index 98d6323ba49d..000000000000 --- a/mobile/test/swift/integration/DirectResponseContainsHeadersMatchIntegrationTest.swift +++ /dev/null @@ -1,60 +0,0 @@ -import Envoy -import TestExtensions -import XCTest - -final class DirectResponseContainsHeadersMatchIntegrationTest: XCTestCase { - override static func setUp() { - super.setUp() - register_test_extensions() - } - - func testDirectResponseWithContainsHeadersMatch() { - let headersExpectation = self.expectation(description: "Response headers received") - let dataExpectation = self.expectation(description: "Response data received") - - let requestHeaders = RequestHeadersBuilder( - method: .get, authority: "127.0.0.1", path: "/v1/abc" - ) - .add(name: "x-foo", value: "123") - .add(name: "x-foo", value: "456") - .build() - - let engine = TestEngineBuilder() - .addDirectResponse( - .init( - matcher: RouteMatcher( - fullPath: "/v1/abc", headers: [ - .init(name: "x-foo", value: "123", mode: .contains), - ] - ), - status: 200, body: "hello world", headers: ["x-response-foo": "aaa"] - ) - ) - .build() - - var responseBuffer = Data() - engine - .streamClient() - .newStreamPrototype() - .setOnResponseHeaders { headers, endStream, _ in - XCTAssertEqual(200, headers.httpStatus) - XCTAssertEqual(["aaa"], headers.value(forName: "x-response-foo")) - XCTAssertFalse(endStream) - headersExpectation.fulfill() - } - .setOnResponseData { data, endStream, _ in - responseBuffer.append(contentsOf: data) - if endStream { - XCTAssertEqual("hello world", String(data: responseBuffer, encoding: .utf8)) - dataExpectation.fulfill() - } - } - .start() - .sendHeaders(requestHeaders, endStream: true) - - let expectations = [headersExpectation, dataExpectation] - XCTAssertEqual(.completed, XCTWaiter().wait(for: expectations, timeout: 10, enforceOrder: true)) - - engine.terminate() - } -} diff --git a/mobile/test/swift/integration/DirectResponseExactHeadersMatchIntegrationTest.swift b/mobile/test/swift/integration/DirectResponseExactHeadersMatchIntegrationTest.swift deleted file mode 100644 index 186328d12803..000000000000 --- a/mobile/test/swift/integration/DirectResponseExactHeadersMatchIntegrationTest.swift +++ /dev/null @@ -1,59 +0,0 @@ -import Envoy -import TestExtensions -import XCTest - -final class DirectResponseExactHeadersMatchIntegrationTest: XCTestCase { - override static func setUp() { - super.setUp() - register_test_extensions() - } - - func testDirectResponseWithExactHeadersMatch() { - let headersExpectation = self.expectation(description: "Response headers received") - let dataExpectation = self.expectation(description: "Response data received") - - let requestHeaders = RequestHeadersBuilder( - method: .get, authority: "127.0.0.1", path: "/v1/abc" - ) - .add(name: "x-foo", value: "123") - .build() - - let engine = TestEngineBuilder() - .addDirectResponse( - .init( - matcher: RouteMatcher( - fullPath: "/v1/abc", headers: [ - .init(name: "x-foo", value: "123", mode: .exact), - ] - ), - status: 200, body: "hello world", headers: ["x-response-foo": "aaa"] - ) - ) - .build() - - var responseBuffer = Data() - engine - .streamClient() - .newStreamPrototype() - .setOnResponseHeaders { headers, endStream, _ in - XCTAssertEqual(200, headers.httpStatus) - XCTAssertEqual(["aaa"], headers.value(forName: "x-response-foo")) - XCTAssertFalse(endStream) - headersExpectation.fulfill() - } - .setOnResponseData { data, endStream, _ in - responseBuffer.append(contentsOf: data) - if endStream { - XCTAssertEqual("hello world", String(data: responseBuffer, encoding: .utf8)) - dataExpectation.fulfill() - } - } - .start() - .sendHeaders(requestHeaders, endStream: true) - - let expectations = [headersExpectation, dataExpectation] - XCTAssertEqual(.completed, XCTWaiter().wait(for: expectations, timeout: 10, enforceOrder: true)) - - engine.terminate() - } -} diff --git a/mobile/test/swift/integration/DirectResponseExactPathMatchIntegrationTest.swift b/mobile/test/swift/integration/DirectResponseExactPathMatchIntegrationTest.swift deleted file mode 100644 index 863890e51e98..000000000000 --- a/mobile/test/swift/integration/DirectResponseExactPathMatchIntegrationTest.swift +++ /dev/null @@ -1,53 +0,0 @@ -import Envoy -import TestExtensions -import XCTest - -final class DirectResponseExactPathMatchIntegrationTest: XCTestCase { - override static func setUp() { - super.setUp() - register_test_extensions() - } - - func testDirectResponseWithExactPathMatch() { - let headersExpectation = self.expectation(description: "Response headers received") - let dataExpectation = self.expectation(description: "Response data received") - - let requestHeaders = RequestHeadersBuilder( - method: .get, authority: "127.0.0.1", path: "/v1/abc" - ).build() - - let engine = TestEngineBuilder() - .addDirectResponse( - .init( - matcher: RouteMatcher(fullPath: "/v1/abc"), - status: 200, body: "hello world", headers: ["x-response-foo": "aaa"] - ) - ) - .build() - - var responseBuffer = Data() - engine - .streamClient() - .newStreamPrototype() - .setOnResponseHeaders { headers, endStream, _ in - XCTAssertEqual(200, headers.httpStatus) - XCTAssertEqual(["aaa"], headers.value(forName: "x-response-foo")) - XCTAssertFalse(endStream) - headersExpectation.fulfill() - } - .setOnResponseData { data, endStream, _ in - responseBuffer.append(contentsOf: data) - if endStream { - XCTAssertEqual("hello world", String(data: responseBuffer, encoding: .utf8)) - dataExpectation.fulfill() - } - } - .start() - .sendHeaders(requestHeaders, endStream: true) - - let expectations = [headersExpectation, dataExpectation] - XCTAssertEqual(.completed, XCTWaiter().wait(for: expectations, timeout: 10, enforceOrder: true)) - - engine.terminate() - } -} diff --git a/mobile/test/swift/integration/DirectResponseFilterMutationIntegrationTest.swift b/mobile/test/swift/integration/DirectResponseFilterMutationIntegrationTest.swift deleted file mode 100644 index b772717b1235..000000000000 --- a/mobile/test/swift/integration/DirectResponseFilterMutationIntegrationTest.swift +++ /dev/null @@ -1,93 +0,0 @@ -import Envoy -import TestExtensions -import XCTest - -private final class MockHeaderMutationFilter: RequestFilter { - private let headersToAdd: [String: String] - - init(headersToAdd: [String: String]) { - self.headersToAdd = headersToAdd - } - - func onRequestHeaders(_ headers: RequestHeaders, endStream: Bool, streamIntel: StreamIntel) - -> FilterHeadersStatus - { - let builder = headers.toRequestHeadersBuilder() - for (name, value) in self.headersToAdd { - builder.add(name: name, value: value) - } - return .continue(headers: builder.build()) - } - - func onRequestData(_ body: Data, endStream: Bool, streamIntel: StreamIntel) - -> FilterDataStatus - { - return .continue(data: body) - } - - func onRequestTrailers(_ trailers: RequestTrailers, streamIntel: StreamIntel) - -> FilterTrailersStatus - { - return .continue(trailers: trailers) - } -} - -final class DirectResponseFilterMutationIntegrationTest: XCTestCase { - override static func setUp() { - super.setUp() - register_test_extensions() - } - - func testDirectResponseThatOnlyMatchesWhenUsingHeadersAddedByFilter() { - let headersExpectation = self.expectation(description: "Response headers received") - let dataExpectation = self.expectation(description: "Response data received") - - let requestHeaders = RequestHeadersBuilder( - method: .get, authority: "127.0.0.1", path: "/v1/abc" - ) - .build() - - // This test validates that Envoy is able to properly route direct responses when a filter - // mutates the outbound request in a way that makes it match one of the direct response - // configurations (whereas if the filter was not present in the chain, the request would not - // match any configurations). This behavior is provided by the C++ `RouteCacheResetFilter`. - let engine = TestEngineBuilder() - .addPlatformFilter { MockHeaderMutationFilter(headersToAdd: ["x-foo": "123"]) } - .addDirectResponse( - .init( - matcher: RouteMatcher( - fullPath: "/v1/abc", headers: [ - .init(name: "x-foo", value: "123", mode: .exact), - ] - ), - status: 200, body: "hello world", headers: ["x-response-foo": "aaa"] - ) - ) - .build() - - var responseBuffer = Data() - engine - .streamClient() - .newStreamPrototype() - .setOnResponseHeaders { headers, endStream, _ in - XCTAssertEqual(200, headers.httpStatus) - XCTAssertEqual(["aaa"], headers.value(forName: "x-response-foo")) - XCTAssertFalse(endStream) - headersExpectation.fulfill() - } - .setOnResponseData { data, endStream, _ in - responseBuffer.append(contentsOf: data) - if endStream { - XCTAssertEqual("hello world", String(data: responseBuffer, encoding: .utf8)) - dataExpectation.fulfill() - } - } - .start() - .sendHeaders(requestHeaders, endStream: true) - - let expectations = [headersExpectation, dataExpectation] - XCTAssertEqual(.completed, XCTWaiter().wait(for: expectations, timeout: 10, enforceOrder: true)) - - engine.terminate() - } -} diff --git a/mobile/test/swift/integration/DirectResponsePrefixHeadersMatchIntegrationTest.swift b/mobile/test/swift/integration/DirectResponsePrefixHeadersMatchIntegrationTest.swift deleted file mode 100644 index c99754d06d36..000000000000 --- a/mobile/test/swift/integration/DirectResponsePrefixHeadersMatchIntegrationTest.swift +++ /dev/null @@ -1,59 +0,0 @@ -import Envoy -import TestExtensions -import XCTest - -final class DirectResponsePrefixHeadersMatchIntegrationTest: XCTestCase { - override static func setUp() { - super.setUp() - register_test_extensions() - } - - func testDirectResponseWithPrefixHeadersMatch() { - let headersExpectation = self.expectation(description: "Response headers received") - let dataExpectation = self.expectation(description: "Response data received") - - let requestHeaders = RequestHeadersBuilder( - method: .get, authority: "127.0.0.1", path: "/v1/abc" - ) - .add(name: "x-foo", value: "123456") - .build() - - let engine = TestEngineBuilder() - .addDirectResponse( - .init( - matcher: RouteMatcher( - fullPath: "/v1/abc", headers: [ - .init(name: "x-foo", value: "123", mode: .prefix), - ] - ), - status: 200, body: "hello world", headers: ["x-response-foo": "aaa"] - ) - ) - .build() - - var responseBuffer = Data() - engine - .streamClient() - .newStreamPrototype() - .setOnResponseHeaders { headers, endStream, _ in - XCTAssertEqual(200, headers.httpStatus) - XCTAssertEqual(["aaa"], headers.value(forName: "x-response-foo")) - XCTAssertFalse(endStream) - headersExpectation.fulfill() - } - .setOnResponseData { data, endStream, _ in - responseBuffer.append(contentsOf: data) - if endStream { - XCTAssertEqual("hello world", String(data: responseBuffer, encoding: .utf8)) - dataExpectation.fulfill() - } - } - .start() - .sendHeaders(requestHeaders, endStream: true) - - let expectations = [headersExpectation, dataExpectation] - XCTAssertEqual(.completed, XCTWaiter().wait(for: expectations, timeout: 10, enforceOrder: true)) - - engine.terminate() - } -} diff --git a/mobile/test/swift/integration/DirectResponseSuffixHeadersMatchIntegrationTest.swift b/mobile/test/swift/integration/DirectResponseSuffixHeadersMatchIntegrationTest.swift deleted file mode 100644 index 9413a1d36327..000000000000 --- a/mobile/test/swift/integration/DirectResponseSuffixHeadersMatchIntegrationTest.swift +++ /dev/null @@ -1,59 +0,0 @@ -import Envoy -import TestExtensions -import XCTest - -final class DirectResponseSuffixHeadersMatchIntegrationTest: XCTestCase { - override static func setUp() { - super.setUp() - register_test_extensions() - } - - func testDirectResponseWithSuffixHeadersMatch() { - let headersExpectation = self.expectation(description: "Response headers received") - let dataExpectation = self.expectation(description: "Response data received") - - let requestHeaders = RequestHeadersBuilder( - method: .get, authority: "127.0.0.1", path: "/v1/abc" - ) - .add(name: "x-foo", value: "123456") - .build() - - let engine = TestEngineBuilder() - .addDirectResponse( - .init( - matcher: RouteMatcher( - fullPath: "/v1/abc", headers: [ - .init(name: "x-foo", value: "456", mode: .suffix), - ] - ), - status: 200, body: "hello world", headers: ["x-response-foo": "aaa"] - ) - ) - .build() - - var responseBuffer = Data() - engine - .streamClient() - .newStreamPrototype() - .setOnResponseHeaders { headers, endStream, _ in - XCTAssertEqual(200, headers.httpStatus) - XCTAssertEqual(["aaa"], headers.value(forName: "x-response-foo")) - XCTAssertFalse(endStream) - headersExpectation.fulfill() - } - .setOnResponseData { data, endStream, _ in - responseBuffer.append(contentsOf: data) - if endStream { - XCTAssertEqual("hello world", String(data: responseBuffer, encoding: .utf8)) - dataExpectation.fulfill() - } - } - .start() - .sendHeaders(requestHeaders, endStream: true) - - let expectations = [headersExpectation, dataExpectation] - XCTAssertEqual(.completed, XCTWaiter().wait(for: expectations, timeout: 10, enforceOrder: true)) - - engine.terminate() - } -} diff --git a/mobile/test/swift/integration/DirectResponsePrefixPathMatchIntegrationTest.swift b/mobile/test/swift/integration/EndToEndNetworkingTest.swift similarity index 70% rename from mobile/test/swift/integration/DirectResponsePrefixPathMatchIntegrationTest.swift rename to mobile/test/swift/integration/EndToEndNetworkingTest.swift index cd68b94cec37..140f10c75d17 100644 --- a/mobile/test/swift/integration/DirectResponsePrefixPathMatchIntegrationTest.swift +++ b/mobile/test/swift/integration/EndToEndNetworkingTest.swift @@ -1,28 +1,28 @@ import Envoy +import EnvoyTestServer +import Foundation import TestExtensions import XCTest -final class DirectResponsePrefixPathMatchIntegrationTest: XCTestCase { +final class EndToEndNetworkingTest: XCTestCase { override static func setUp() { super.setUp() register_test_extensions() } - func testDirectResponseWithPrefixMatch() { + func testNetworkRequestReturnsHeadersAndData() { + EnvoyTestServer.startHttp1PlaintextServer() + EnvoyTestServer.setHeadersAndData( + "x-response-foo", header_value: "aaa", response_body: "hello world") let headersExpectation = self.expectation(description: "Response headers received") let dataExpectation = self.expectation(description: "Response data received") - + let port = String(EnvoyTestServer.getEnvoyPort()) let requestHeaders = RequestHeadersBuilder( - method: .get, authority: "127.0.0.1", path: "/v1/foo/bar?param=1" - ).build() + method: .get, scheme: "http", authority: "localhost:" + port, path: "/" + ) + .build() - let engine = TestEngineBuilder() - .addDirectResponse( - .init( - matcher: RouteMatcher(pathPrefix: "/v1/foo"), - status: 200, body: "hello world", headers: ["x-response-foo": "aaa"] - ) - ) + let engine = EngineBuilder() .build() var responseBuffer = Data() @@ -49,5 +49,6 @@ final class DirectResponsePrefixPathMatchIntegrationTest: XCTestCase { XCTAssertEqual(.completed, XCTWaiter().wait(for: expectations, timeout: 10, enforceOrder: true)) engine.terminate() + EnvoyTestServer.shutdownTestServer() } } From ab3cc1fcc488436086407dd2127509e5b88d289d Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 23 May 2023 20:56:04 +0100 Subject: [PATCH 339/740] mobile/ci: Improve mobile jobs (#27404) Signed-off-by: Ryan Northey --- .github/workflows/env.yml | 64 ++++++++++++++ .github/workflows/mobile-android_build.yml | 81 ++++++------------ .github/workflows/mobile-android_tests.yml | 37 +++------ .github/workflows/mobile-asan.yml | 16 ++-- .github/workflows/mobile-cc_tests.yml | 10 ++- .../workflows/mobile-compile_time_options.yml | 37 +++------ .github/workflows/mobile-core.yml | 4 +- .github/workflows/mobile-coverage.yml | 18 ++-- .github/workflows/mobile-docs.yml | 4 +- .github/workflows/mobile-format.yml | 50 ++++------- .github/workflows/mobile-ios_build.yml | 83 +++++-------------- .github/workflows/mobile-ios_tests.yml | 27 +++--- .github/workflows/mobile-perf.yml | 12 +-- .../workflows/mobile-release_validation.yml | 19 ++--- .github/workflows/mobile-tsan.yml | 16 ++-- mobile/tools/should_run_ci.sh | 61 -------------- mobile/tools/what_to_run.sh | 52 ++++++++++++ 17 files changed, 260 insertions(+), 331 deletions(-) create mode 100644 .github/workflows/env.yml delete mode 100755 mobile/tools/should_run_ci.sh create mode 100755 mobile/tools/what_to_run.sh diff --git a/.github/workflows/env.yml b/.github/workflows/env.yml new file mode 100644 index 000000000000..a0caaa67cfb6 --- /dev/null +++ b/.github/workflows/env.yml @@ -0,0 +1,64 @@ +name: Environment + +on: + workflow_call: + outputs: + mobile_android_build: + value: ${{ jobs.mobile.outputs.mobile_android_build }} + mobile_android_build_all: + value: ${{ jobs.mobile.outputs.mobile_android_build_all }} + mobile_android_tests: + value: ${{ jobs.mobile.outputs.mobile_android_tests }} + mobile_asan: + value: ${{ jobs.mobile.outputs.mobile_asan }} + mobile_cc_tests: + value: ${{ jobs.mobile.outputs.mobile_cc_tests }} + mobile_compile_time_options: + value: ${{ jobs.mobile.outputs.mobile_compile_time_options }} + mobile_coverage: + value: ${{ jobs.mobile.outputs.mobile_coverage }} + mobile_formatting: + value: ${{ jobs.mobile.outputs.mobile_formatting }} + mobile_ios_build: + value: ${{ jobs.mobile.outputs.mobile_ios_build }} + mobile_ios_build_all: + value: ${{ jobs.mobile.outputs.mobile_ios_build_all }} + mobile_ios_tests: + value: ${{ jobs.mobile.outputs.mobile_ios_tests }} + mobile_release_validation: + value: ${{ jobs.mobile.outputs.mobile_release_validation }} + mobile_tsan: + value: ${{ jobs.mobile.outputs.mobile_tsan }} + +concurrency: + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }}-env + cancel-in-progress: true + +jobs: + mobile: + if: github.repository == 'envoyproxy/envoy' + runs-on: ubuntu-20.04 + outputs: + mobile_android_build: ${{ steps.should_run.outputs.mobile_android_build }} + mobile_android_build_all: ${{ steps.should_run.outputs.mobile_android_build_all }} + mobile_android_tests: ${{ steps.should_run.outputs.mobile_android_tests }} + mobile_asan: ${{ steps.should_run.outputs.mobile_asan }} + mobile_cc_tests: ${{ steps.should_run.outputs.mobile_cc_tests }} + mobile_compile_time_options: ${{ steps.should_run.outputs.mobile_compile_time_options }} + mobile_coverage: ${{ steps.should_run.outputs.mobile_coverage }} + mobile_formatting: ${{ steps.should_run.outputs.mobile_formatting }} + mobile_ios_build: ${{ steps.should_run.outputs.mobile_ios_build }} + mobile_ios_build_all: ${{ steps.should_run.outputs.mobile_ios_build_all }} + mobile_ios_tests: ${{ steps.should_run.outputs.mobile_ios_tests }} + mobile_release_validation: ${{ steps.should_run.outputs.mobile_release_validation }} + mobile_tsan: ${{ steps.should_run.outputs.mobile_tsan }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Add safe directory + run: git config --global --add safe.directory /__w/envoy/envoy + - id: should_run + name: 'Check what to run' + run: | + ./mobile/tools/what_to_run.sh diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml index 4e1f8ec328e0..8f5c71a7d978 100644 --- a/.github/workflows/mobile-android_build.yml +++ b/.github/workflows/mobile-android_build.yml @@ -7,12 +7,18 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + androidbuild: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_android_build == 'true' }} + needs: env name: android_build runs-on: ubuntu-20.04 timeout-minutes: 90 @@ -23,15 +29,9 @@ jobs: CXX: /opt/llvm/bin/clang++ steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Build envoy.aar distributable' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -40,31 +40,26 @@ jobs: --fat_apk_cpu=x86_64 \ --linkopt=-fuse-ld=lld \ //:android_dist + javahelloworld: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_android_build_all == 'true' }} + needs: + - env + - androidbuild name: java_helloworld - needs: androidbuild runs-on: macos-12 timeout-minutes: 50 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 - if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - run: cd mobile && ./ci/mac_ci_setup.sh --android - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Start emulator' with: timeout_minutes: 10 @@ -74,7 +69,6 @@ jobs: # cd mobile && ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/java/hello_world:hello_envoy # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - name: 'Start java app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -85,7 +79,6 @@ jobs: adb install -r --no-incremental bazel-bin/examples/java/hello_world/hello_envoy.apk adb shell am start -n io.envoyproxy.envoymobile.helloenvoy/.MainActivity - name: 'Check connectivity' - if: steps.should_run.outputs.run_ci_job == 'true' run: | timeout 30 adb logcat -e "received headers with status 301" -m 1 || { echo "Failed checking for headers in adb logcat" >&2 @@ -95,30 +88,24 @@ jobs: exit 1 } kotlinhelloworld: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_android_build == 'true' }} + needs: + - env + - androidbuild name: kotlin_helloworld - needs: androidbuild runs-on: macos-12 timeout-minutes: 50 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 - if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - name: 'Install dependencies' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh --android - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Start emulator' with: timeout_minutes: 10 @@ -128,7 +115,6 @@ jobs: # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - name: 'Start kotlin app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -139,7 +125,6 @@ jobs: adb install -r --no-incremental bazel-bin/examples/kotlin/hello_world/hello_envoy_kt.apk adb shell am start -n io.envoyproxy.envoymobile.helloenvoykotlin/.MainActivity - name: 'Check connectivity' - if: steps.should_run.outputs.run_ci_job == 'true' run: | timeout 30 adb logcat -e "received headers with status 200" -m 1 || { echo "Failed checking for headers in adb logcat" >&2 @@ -150,30 +135,24 @@ jobs: } kotlinbaselineapp: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_android_build_all == 'true' }} + needs: + - env + - androidbuild name: kotlin_baseline_app - needs: androidbuild runs-on: macos-12 timeout-minutes: 50 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 - if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - name: 'Install dependencies' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh --android - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Start emulator' with: timeout_minutes: 10 @@ -183,7 +162,6 @@ jobs: # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - name: 'Start kotlin app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -194,7 +172,6 @@ jobs: adb install -r --no-incremental bazel-bin/test/kotlin/apps/baseline/hello_envoy_kt.apk adb shell am start -n io.envoyproxy.envoymobile.helloenvoybaselinetest/.MainActivity - name: 'Check connectivity' - if: steps.should_run.outputs.run_ci_job == 'true' run: | timeout 30 adb logcat -e "received headers with status 301" -m 1 || { echo "Failed checking for headers in adb logcat" >&2 @@ -204,30 +181,24 @@ jobs: exit 1 } kotlinexperimentalapp: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_android_build_all == 'true' }} + needs: + - env + - androidbuild name: kotlin_experimental_app - needs: androidbuild runs-on: macos-12 timeout-minutes: 50 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 - if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - name: 'Install dependencies' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh --android - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Start emulator' with: timeout_minutes: 10 @@ -237,7 +208,6 @@ jobs: # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - name: 'Start kotlin app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -250,7 +220,6 @@ jobs: adb install -r --no-incremental bazel-bin/test/kotlin/apps/experimental/hello_envoy_kt.apk adb shell am start -n io.envoyproxy.envoymobile.helloenvoyexperimentaltest/.MainActivity - name: 'Check connectivity' - if: steps.should_run.outputs.run_ci_job == 'true' run: | timeout 30 adb logcat -e "received headers with status 200" -m 1 || { echo "Failed checking for headers in adb logcat" >&2 diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml index f4b44217c0e7..9baf8b9922a7 100644 --- a/.github/workflows/mobile-android_tests.yml +++ b/.github/workflows/mobile-android_tests.yml @@ -7,12 +7,17 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + uses: ./.github/workflows/env.yml + secrets: inherit + kotlintestsmac: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_android_tests == 'true' }} + needs: env # revert to //test/kotlin/... once fixed # https://github.com/envoyproxy/envoy-mobile/issues/1932 name: kotlin_tests_mac @@ -20,13 +25,7 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Java setup' - if: steps.should_run.outputs.run_ci_job == 'true' uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 with: java-version: '8' @@ -34,10 +33,8 @@ jobs: architecture: x64 distribution: zulu - name: 'Install dependencies' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh - name: 'Run Kotlin library tests' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -48,19 +45,14 @@ jobs: --define=signal_trace=disabled \ //test/kotlin/io/... javatestsmac: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_android_tests == 'true' }} + needs: env name: java_tests_mac runs-on: macos-12 timeout-minutes: 120 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Java setup' - if: steps.should_run.outputs.run_ci_job == 'true' uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 with: java-version: '8' @@ -68,10 +60,8 @@ jobs: architecture: x64 distribution: zulu - name: 'Install dependencies' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh - name: 'Run Java library tests' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -85,7 +75,8 @@ jobs: --define=system-helper=android \ //test/java/... kotlintestslinux: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_android_tests == 'true' }} + needs: env # Only kotlin tests are executed since with linux: # https://github.com/envoyproxy/envoy-mobile/issues/1418. name: kotlin_tests_linux @@ -98,15 +89,9 @@ jobs: CXX: /opt/llvm/bin/clang++ steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Run Kotlin library integration tests' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/mobile-asan.yml b/.github/workflows/mobile-asan.yml index 2681176bc068..e5a9378c262e 100644 --- a/.github/workflows/mobile-asan.yml +++ b/.github/workflows/mobile-asan.yml @@ -7,12 +7,18 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + asan: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_asan == 'true' }} + needs: env name: asan runs-on: ubuntu-20.04 timeout-minutes: 180 @@ -23,17 +29,11 @@ jobs: CXX: /opt/llvm/bin/clang++ steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Run tests' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: steps.should_run.outputs.run_ci_job == 'true' run: | cd mobile && ./bazelw test --test_output=all \ --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ diff --git a/.github/workflows/mobile-cc_tests.yml b/.github/workflows/mobile-cc_tests.yml index 28f6c633ecd8..c824fc8f6ece 100644 --- a/.github/workflows/mobile-cc_tests.yml +++ b/.github/workflows/mobile-cc_tests.yml @@ -7,12 +7,18 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + cctests: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_cc_tests == 'true' }} + needs: env name: cc_tests runs-on: ubuntu-20.04 timeout-minutes: 120 diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index b9ee5e056dda..fc8ff2073781 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -7,12 +7,18 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + cc_test: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} + needs: env name: cc_test runs-on: ubuntu-20.04 timeout-minutes: 120 @@ -20,15 +26,9 @@ jobs: image: envoyproxy/envoy-build-ubuntu:b0ff77ae3f25b0bf595f9b8bba46b489723ab446 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Build C++ library' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Envoy Mobile build verifies that the build configuration where HTTP/3 is enabled and @@ -45,22 +45,16 @@ jobs: --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ //test/cc/... swift_build: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} + needs: env name: swift_build runs-on: macos-12 timeout-minutes: 120 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Build Swift library' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -79,29 +73,22 @@ jobs: --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ //library/swift:ios_framework kotlin_build: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} + needs: env name: kotlin_build runs-on: macos-12 timeout-minutes: 120 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 - if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - name: 'Install dependencies' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh --android - name: 'Build Kotlin library' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/mobile-core.yml b/.github/workflows/mobile-core.yml index 26cd00b8089d..7c111e04a88b 100644 --- a/.github/workflows/mobile-core.yml +++ b/.github/workflows/mobile-core.yml @@ -7,12 +7,12 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: unittests: - if: github.repository == 'envoyproxy/envoy' + if: ${{ github.repository == 'envoyproxy/envoy' }} name: unit_tests runs-on: ubuntu-20.04 timeout-minutes: 120 diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml index cdb1cf2a9853..8ff027449a9e 100644 --- a/.github/workflows/mobile-coverage.yml +++ b/.github/workflows/mobile-coverage.yml @@ -7,12 +7,18 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + coverage: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_coverage == 'true' }} + needs: env name: coverage runs-on: ubuntu-20.04 timeout-minutes: 120 @@ -23,15 +29,9 @@ jobs: image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Run coverage' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -40,11 +40,9 @@ jobs: COVERAGE_THRESHOLD=76 \ ../test/run_envoy_bazel_coverage.sh //test/common/... //test/cc/... - name: 'Package coverage' - if: steps.should_run.outputs.run_ci_job == 'true' run: | cd mobile && tar -czf coverage.tar.gz generated/coverage - name: 'Upload report' - if: steps.should_run.outputs.run_ci_job == 'true' uses: actions/upload-artifact@v3 with: name: coverage.tar.gz diff --git a/.github/workflows/mobile-docs.yml b/.github/workflows/mobile-docs.yml index 403529803e14..093b5f29740c 100644 --- a/.github/workflows/mobile-docs.yml +++ b/.github/workflows/mobile-docs.yml @@ -7,12 +7,12 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: docs: - if: github.repository == 'envoyproxy/envoy' + if: ${{ github.repository == 'envoyproxy/envoy' }} runs-on: ubuntu-20.04 timeout-minutes: 20 container: diff --git a/.github/workflows/mobile-format.yml b/.github/workflows/mobile-format.yml index 97457d9f2015..ac48d4308063 100644 --- a/.github/workflows/mobile-format.yml +++ b/.github/workflows/mobile-format.yml @@ -7,12 +7,18 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + formatall: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_formatting == 'true' }} + needs: env name: format_all runs-on: ubuntu-20.04 timeout-minutes: 45 @@ -25,36 +31,25 @@ jobs: ENVOY_BAZEL_PREFIX: "@envoy" steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Run formatters' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./tools/check_format.sh precommit: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_formatting == 'true' }} + needs: env name: precommit runs-on: macos-12 timeout-minutes: 45 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Install precommit' - if: steps.should_run.outputs.run_ci_job == 'true' run: brew install pre-commit - name: 'Run precommit' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && find mobile/* | pre-commit run --files swiftlint: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_formatting == 'true' }} + needs: env name: swift_lint runs-on: ubuntu-latest timeout-minutes: 5 @@ -66,46 +61,34 @@ jobs: run: swiftlint lint --strict working-directory: mobile drstring: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_formatting == 'true' }} + needs: env name: drstring runs-on: macos-12 timeout-minutes: 10 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Run DrString' - if: steps.should_run.outputs.run_ci_job == 'true' env: DEVELOPER_DIR: /Applications/Xcode_14.1.app run: cd mobile && ./bazelw run @DrString//:drstring check kotlinlint: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_formatting == 'true' }} + needs: env name: kotlin_lint runs-on: macos-12 timeout-minutes: 45 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 - if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - run: cd mobile && ./ci/mac_ci_setup.sh - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Run Kotlin Lint (Detekt)' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -114,5 +97,4 @@ jobs: //library/kotlin/io/envoyproxy/envoymobile:envoy_lib_lint \ //examples/kotlin/hello_world:hello_envoy_kt_lint - name: 'Run Kotlin Formatter (ktlint)' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./bazelw build kotlin_format diff --git a/.github/workflows/mobile-ios_build.yml b/.github/workflows/mobile-ios_build.yml index fd3601c1cc9b..a1d117ff13cd 100644 --- a/.github/workflows/mobile-ios_build.yml +++ b/.github/workflows/mobile-ios_build.yml @@ -7,27 +7,26 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + iosbuild: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_ios_build == 'true' }} + needs: env name: ios_build runs-on: macos-12 timeout-minutes: 120 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Build Envoy.framework distributable' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -37,23 +36,16 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ //library/swift:ios_framework swifthelloworld: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_ios_build == 'true' }} name: swift_helloworld needs: iosbuild runs-on: macos-12 timeout-minutes: 50 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Build app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -62,7 +54,6 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ //examples/swift/hello_world:app - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Start simulator' with: timeout_minutes: 5 @@ -70,7 +61,6 @@ jobs: command: ./mobile/ci/start_ios_simulator.sh # Run the app in the background and redirect logs. - name: 'Run app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -79,29 +69,23 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ //examples/swift/hello_world:app &> /tmp/envoy.log & - run: sed '/received headers with status 200/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check connectivity' - run: cat /tmp/envoy.log if: ${{ failure() || cancelled() }} name: 'Log app run' swiftbaselineapp: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} + needs: + - env + - iosbuild name: swift_baseline_app - needs: iosbuild runs-on: macos-12 timeout-minutes: 50 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Build app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -110,7 +94,6 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ //test/swift/apps/baseline:app - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Start simulator' with: timeout_minutes: 5 @@ -118,7 +101,6 @@ jobs: command: ./mobile/ci/start_ios_simulator.sh # Run the app in the background and redirect logs. - name: 'Run app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -127,29 +109,22 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ //test/swift/apps/baseline:app &> /tmp/envoy.log & - run: sed '/received headers with status 301/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check connectivity' - run: cat /tmp/envoy.log if: ${{ failure() || cancelled() }} name: 'Log app run' swiftexperimentalapp: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} + needs: env name: swift_experimental_app needs: iosbuild runs-on: macos-12 timeout-minutes: 50 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Build app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -160,7 +135,6 @@ jobs: --define envoy_mobile_listener=enabled \ //test/swift/apps/experimental:app - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Start simulator' with: timeout_minutes: 5 @@ -168,7 +142,6 @@ jobs: command: ./mobile/ci/start_ios_simulator.sh # Run the app in the background and redirect logs. - name: 'Run app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -179,29 +152,22 @@ jobs: --define envoy_mobile_listener=enabled \ //test/swift/apps/experimental:app &> /tmp/envoy.log & - run: sed '/received headers with status 200/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check connectivity' - run: cat /tmp/envoy.log if: ${{ failure() || cancelled() }} name: 'Log app run' swiftasyncawait: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} + needs: env name: swift_async_await needs: iosbuild runs-on: macos-12 timeout-minutes: 50 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Build app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -210,7 +176,6 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ //examples/swift/async_await:app - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Start simulator' with: timeout_minutes: 5 @@ -218,7 +183,6 @@ jobs: command: ./mobile/ci/start_ios_simulator.sh # Run the app in the background and redirect logs. - name: 'Run app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -249,23 +213,17 @@ jobs: if: ${{ failure() || cancelled() }} name: 'Log app run' objchelloworld: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} + needs: env name: objc_helloworld needs: iosbuild runs-on: macos-12 timeout-minutes: 50 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Build app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -274,7 +232,6 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ //examples/objective-c/hello_world:app - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Start simulator' with: timeout_minutes: 5 @@ -282,7 +239,6 @@ jobs: command: ./mobile/ci/start_ios_simulator.sh # Run the app in the background and redirect logs. - name: 'Run app' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -291,7 +247,6 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ //examples/objective-c/hello_world:app &> /tmp/envoy.log & - run: sed '/received headers with status 301/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check connectivity' - run: cat /tmp/envoy.log if: ${{ failure() || cancelled() }} diff --git a/.github/workflows/mobile-ios_tests.yml b/.github/workflows/mobile-ios_tests.yml index c2b141027d9f..e6f25e960d7e 100644 --- a/.github/workflows/mobile-ios_tests.yml +++ b/.github/workflows/mobile-ios_tests.yml @@ -7,27 +7,26 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + swifttests: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_ios_tests == 'true' }} + needs: env name: swift_tests runs-on: macos-12 timeout-minutes: 120 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Install dependencies' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh - name: 'Run swift library tests' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # runs with admin enabled due to regression testing admin interface @@ -43,22 +42,16 @@ jobs: --define=admin_functionality=enabled \ //test/swift/... objctests: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_ios_tests == 'true' }} + needs: env name: c_and_objc_tests runs-on: macos-12 timeout-minutes: 120 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Install dependencies' - if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh - name: 'Run Objective-C library tests' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/mobile-perf.yml b/.github/workflows/mobile-perf.yml index 26084818d469..b1c142bf2c1b 100644 --- a/.github/workflows/mobile-perf.yml +++ b/.github/workflows/mobile-perf.yml @@ -7,12 +7,12 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: sizecurrent: - if: github.repository == 'envoyproxy/envoy' + if: ${{ github.repository == 'envoyproxy/envoy' }} name: size_current runs-on: ubuntu-20.04 timeout-minutes: 120 @@ -39,7 +39,7 @@ jobs: name: sizecurrent path: mobile/bazel-bin/test/performance/test_binary_size sizemain: - if: github.repository == 'envoyproxy/envoy' + if: ${{ github.repository == 'envoyproxy/envoy' }} name: size_main runs-on: ubuntu-20.04 timeout-minutes: 90 @@ -68,9 +68,11 @@ jobs: name: sizemain path: mobile/bazel-bin/test/performance/test_binary_size sizecompare: - if: github.repository == 'envoyproxy/envoy' + if: ${{ github.repository == 'envoyproxy/envoy' }} + needs: + - sizecurrent + - sizemain name: size_compare - needs: [sizecurrent, sizemain] runs-on: ubuntu-20.04 timeout-minutes: 30 container: diff --git a/.github/workflows/mobile-release_validation.yml b/.github/workflows/mobile-release_validation.yml index f5f03d84420a..34cf2878aeea 100644 --- a/.github/workflows/mobile-release_validation.yml +++ b/.github/workflows/mobile-release_validation.yml @@ -7,27 +7,26 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + validate_swiftpm_example: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_release_validation == 'true' }} + needs: env name: validate_swiftpm_example runs-on: macos-12 timeout-minutes: 120 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Build xcframework' - if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -37,9 +36,7 @@ jobs: //:ios_xcframework # Ignore errors: Bad CRC when unzipping large files: https://bbs.archlinux.org/viewtopic.php?id=153011 - run: unzip mobile/bazel-bin/library/swift/Envoy.xcframework.zip -d mobile/examples/swift/swiftpm/Packages || true - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Unzip xcframework' - run: xcodebuild -project mobile/examples/swift/swiftpm/EnvoySwiftPMExample.xcodeproj -scheme EnvoySwiftPMExample -destination platform="iOS Simulator,name=iPhone 14 Pro Max,OS=16.1" - if: steps.should_run.outputs.run_ci_job == 'true' name: 'Build app' # TODO(jpsim): Run app and inspect logs to validate diff --git a/.github/workflows/mobile-tsan.yml b/.github/workflows/mobile-tsan.yml index 8a5898f2b813..918613f15146 100644 --- a/.github/workflows/mobile-tsan.yml +++ b/.github/workflows/mobile-tsan.yml @@ -7,12 +7,18 @@ on: pull_request: concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + tsan: - if: github.repository == 'envoyproxy/envoy' + if: ${{ needs.env.outputs.mobile_tsan == 'true' }} + needs: env name: tsan runs-on: ubuntu-20.04 timeout-minutes: 90 @@ -23,17 +29,11 @@ jobs: CXX: /opt/llvm/bin/clang++ steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: should_run - name: 'Check whether to run' - run: ./mobile/tools/should_run_ci.sh - name: 'Run tests' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: steps.should_run.outputs.run_ci_job == 'true' run: | cd mobile && ./bazelw test \ --test_output=all \ diff --git a/mobile/tools/should_run_ci.sh b/mobile/tools/should_run_ci.sh deleted file mode 100755 index 1ee2622db98f..000000000000 --- a/mobile/tools/should_run_ci.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -######################################################################## -# should_run_ci.sh -# -# Checks current branch and changed PR paths to determine if mobile CI -# jobs should be run. -######################################################################## - -job="$GITHUB_JOB" -branch_name="$GITHUB_REF_NAME" - -function success() { - echo "Running $job because there are $1 changes on $branch_name" - echo "run_ci_job=true" >> "$GITHUB_OUTPUT" -} - -function failure() { - echo "Skipping $job because there are no mobile changes on $branch_name" - echo "run_ci_job=false" >> "$GITHUB_OUTPUT" -} - -if [[ $branch_name == "main" ]]; then - # Skip some example apps on `main` to reduce CI load - case "$job" in - swiftbaselineapp|swiftexperimentalapp|swiftasyncawait|objchelloworld|javahelloworld|kotlinbaselineapp|kotlinexperimentalapp) - echo "Skipping $job because current branch is main" - echo "run_ci_job=false" >> "$GITHUB_OUTPUT" - exit 0 - ;; - esac - - # Run all other mobile CI jobs on `main` - echo "Running $job because current branch is main" - echo "run_ci_job=true" >> "$GITHUB_OUTPUT" - exit 0 -fi - -if [[ $GITHUB_BASE_REF == release/* ]]; then - # Skip mobile CI jobs on PRs targeting release branches - echo "Skipping $job because the PR is targeting a release branch" - echo "run_ci_job=false" >> "$GITHUB_OUTPUT" - exit 0 -fi - -base_commit="$(git merge-base origin/main HEAD)" -changed_files="$(git diff "$base_commit" --name-only)" - -if grep -q "^mobile/" <<< "$changed_files"; then - success "mobile" -elif grep -q "^bazel/repository_locations\.bzl" <<< "$changed_files"; then - success "bazel/repository_locations.bzl" -elif grep -q "^\.bazelrc" <<< "$changed_files"; then - success ".bazelrc" -elif grep -q "^\.github/workflows/mobile-*" <<< "$changed_files"; then - success "GitHub Workflows" -else - failure -fi diff --git a/mobile/tools/what_to_run.sh b/mobile/tools/what_to_run.sh new file mode 100755 index 000000000000..d67ca8e9c9b4 --- /dev/null +++ b/mobile/tools/what_to_run.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -euo pipefail + +BRANCH_NAME="$GITHUB_REF_NAME" +BASE_COMMIT="$(git merge-base origin/main HEAD)" +CHANGED_FILES="$(git diff "${BASE_COMMIT}" --name-only)" +CHANGE_MATCH='^mobile/|^bazel/repository_locations\.bzl|^\.bazelrc|^\.github/workflows/mobile-*|^\.github/workflows/env.yml' + +# The logic in this file is roughly: +# +# pull_request + changed files = run all mobile CI +# +# main = run some mobile CI +# +# all other commits = run minimal mobile CI (these jobs have no conditions) +# +# Branches are not currently tested, altho that should be restricted by the workflow + + +run_default_ci () { + { + echo "mobile_android_build=true" + echo "mobile_android_tests=true" + echo "mobile_asan=true" + echo "mobile_cc_tests=true" + echo "mobile_compile_time_options=true" + echo "mobile_coverage=true" + echo "mobile_formatting=true" + echo "mobile_ios_build=true" + echo "mobile_ios_tests=true" + echo "mobile_release_validation=true" + echo "mobile_tsan=true" + } >> "$GITHUB_OUTPUT" +} + +run_ci_for_changed_files () { + run_default_ci + { + echo "mobile_android_build_all=true" + echo "mobile_ios_build_all=true" + } >> "$GITHUB_OUTPUT" +} + +if [[ "$BRANCH_NAME" == "main" ]]; then + run_default_ci + exit 0 +fi + +if grep -qE "$CHANGE_MATCH" <<< "$CHANGED_FILES"; then + run_ci_for_changed_files +fi From 9f1aa4330060331bd074be2ee20516252f909d1a Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 23 May 2023 16:58:21 -0400 Subject: [PATCH 340/740] test: moving ssl utility to use parsed protos (#27585) Signed-off-by: Alyssa Wilk --- test/integration/BUILD | 6 ++-- test/integration/ssl_utility.cc | 53 +++++++++++---------------------- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/test/integration/BUILD b/test/integration/BUILD index fbcbfbfaf828..cb6be550d383 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1101,12 +1101,14 @@ envoy_cc_test_library( name = "utility_lib", srcs = [ "server.cc", + "ssl_utility.cc", "utility.cc", - ] + envoy_select_enable_yaml(["ssl_utility.cc"]), + ], hdrs = [ "server.h", + "ssl_utility.h", "utility.h", - ] + envoy_select_enable_yaml(["ssl_utility.h"]), + ], data = ["//test/common/runtime:filesystem_test_data"], deps = [ ":server_stats_interface", diff --git a/test/integration/ssl_utility.cc b/test/integration/ssl_utility.cc index af93ec2c3a48..be1c9d8a2009 100644 --- a/test/integration/ssl_utility.cc +++ b/test/integration/ssl_utility.cc @@ -25,47 +25,30 @@ namespace Ssl { void initializeUpstreamTlsContextConfig( const ClientSslTransportOptions& options, envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext& tls_context) { - std::string yaml_plain = R"EOF( - common_tls_context: - validation_context: - trusted_ca: - filename: "{{ test_rundir }}/test/config/integration/certs/cacert.pem" -)EOF"; + std::string rundir = TestEnvironment::runfilesDirectory(); + tls_context.mutable_common_tls_context() + ->mutable_validation_context() + ->mutable_trusted_ca() + ->set_filename(rundir + "/test/config/integration/certs/cacert.pem"); + auto* certs = tls_context.mutable_common_tls_context()->add_tls_certificates(); + std::string chain; + std::string key; if (options.client_ecdsa_cert_) { - yaml_plain += R"EOF( - tls_certificates: - certificate_chain: - filename: "{{ test_rundir }}/test/config/integration/certs/client_ecdsacert.pem" - private_key: - filename: "{{ test_rundir }}/test/config/integration/certs/client_ecdsakey.pem" -)EOF"; + chain = rundir + "/test/config/integration/certs/client_ecdsacert.pem"; + key = rundir + "/test/config/integration/certs/client_ecdsakey.pem"; } else if (options.use_expired_spiffe_cert_) { - yaml_plain += R"EOF( - tls_certificates: - certificate_chain: - filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_cert.pem" - private_key: - filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_key.pem" -)EOF"; + chain = rundir + "/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_cert.pem"; + key = rundir + "/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_key.pem"; } else if (options.client_with_intermediate_cert_) { - yaml_plain += R"EOF( - tls_certificates: - certificate_chain: - filename: "{{ test_rundir }}/test/config/integration/certs/client2_chain.pem" - private_key: - filename: "{{ test_rundir }}/test/config/integration/certs/client2key.pem" -)EOF"; + chain = rundir + "/test/config/integration/certs/client2_chain.pem"; + key = rundir + "/test/config/integration/certs/client2key.pem"; } else { - yaml_plain += R"EOF( - tls_certificates: - certificate_chain: - filename: "{{ test_rundir }}/test/config/integration/certs/clientcert.pem" - private_key: - filename: "{{ test_rundir }}/test/config/integration/certs/clientkey.pem" -)EOF"; + chain = rundir + "/test/config/integration/certs/clientcert.pem"; + key = rundir + "/test/config/integration/certs/clientkey.pem"; } + certs->mutable_certificate_chain()->set_filename(chain); + certs->mutable_private_key()->set_filename(key); - TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), tls_context); auto* common_context = tls_context.mutable_common_tls_context(); if (options.alpn_) { From ce9c61fe760fafa6d3033fc6bc45a013d37a8b5e Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 23 May 2023 20:16:27 -0400 Subject: [PATCH 341/740] Ext proc logging: track bytes transferred and upstream info in filter state (#27521) ext proc: track bytes transferred and upstream info in filter state Risk Level: Low Testing: Unit tests Signed-off-by: Shriram Rajagopalan --- changelogs/current.yaml | 2 +- envoy/http/filter.h | 2 +- .../extensions/filters/http/ext_proc/client.h | 1 + .../filters/http/ext_proc/client_impl.h | 2 ++ .../filters/http/ext_proc/ext_proc.cc | 10 +++++++ .../filters/http/ext_proc/ext_proc.h | 25 ++++++++++++++++ .../filters/http/ext_proc/filter_test.cc | 30 +++++++++++++++---- .../filters/http/ext_proc/mock_server.h | 1 + .../filters/http/ext_proc/ordering_test.cc | 3 ++ .../unit_test_fuzz/ext_proc_unit_test_fuzz.cc | 3 ++ .../http/ext_proc/unit_test_fuzz/mocks.h | 1 + 11 files changed, 72 insertions(+), 8 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index c73629831412..a39d3c460410 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -219,7 +219,7 @@ new_features: - area: ext_proc change: | added new field ``filter_metadata downstreamCallbacks() PURE; /** - * @return absl::string_view the name of the filter as configured in the filter chain.. + * @return absl::string_view the name of the filter as configured in the filter chain. */ virtual absl::string_view filterConfigName() const PURE; }; diff --git a/source/extensions/filters/http/ext_proc/client.h b/source/extensions/filters/http/ext_proc/client.h index 95c50f9ec5b1..1cf1bb605308 100644 --- a/source/extensions/filters/http/ext_proc/client.h +++ b/source/extensions/filters/http/ext_proc/client.h @@ -20,6 +20,7 @@ class ExternalProcessorStream { bool end_stream) PURE; // Idempotent close. Return true if it actually closed. virtual bool close() PURE; + virtual const StreamInfo::StreamInfo& streamInfo() const PURE; }; using ExternalProcessorStreamPtr = std::unique_ptr; diff --git a/source/extensions/filters/http/ext_proc/client_impl.h b/source/extensions/filters/http/ext_proc/client_impl.h index 498139e3a3e5..4f808b02b67f 100644 --- a/source/extensions/filters/http/ext_proc/client_impl.h +++ b/source/extensions/filters/http/ext_proc/client_impl.h @@ -7,6 +7,7 @@ #include "envoy/grpc/async_client_manager.h" #include "envoy/service/ext_proc/v3/external_processor.pb.h" #include "envoy/stats/scope.h" +#include "envoy/stream_info/stream_info.h" #include "source/common/grpc/typed_async_client.h" #include "source/extensions/filters/http/ext_proc/client.h" @@ -54,6 +55,7 @@ class ExternalProcessorStreamImpl : public ExternalProcessorStream, void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&& metadata) override; void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&& metadata) override; void onRemoteClose(Grpc::Status::GrpcStatus status, const std::string& message) override; + const StreamInfo::StreamInfo& streamInfo() const override { return stream_.streamInfo(); } private: ExternalProcessorCallbacks& callbacks_; diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index b56f3da39cb1..fd45f89921c8 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -127,12 +127,22 @@ Filter::StreamOpenState Filter::openStream() { // Stream failed while starting and either onGrpcError or onGrpcClose was already called return sent_immediate_response_ ? StreamOpenState::Error : StreamOpenState::IgnoreError; } + // For custom access logging purposes. Applicable only for Envoy gRPC as Google gRPC does not + // have a proper implementation of streamInfo. + if (grpc_service_.has_envoy_grpc()) { + logging_info_->setClusterInfo(stream_->streamInfo().upstreamClusterInfo()); + logging_info_->setUpstreamHost(stream_->streamInfo().upstreamInfo()->upstreamHost()); + } } return StreamOpenState::Ok; } void Filter::closeStream() { if (stream_) { + if (grpc_service_.has_envoy_grpc()) { + logging_info_->setBytesSent(stream_->streamInfo().bytesSent()); + logging_info_->setBytesReceived(stream_->streamInfo().bytesReceived()); + } ENVOY_LOG(debug, "Calling close on stream"); if (stream_->close()) { stats_.streams_closed_.inc(); diff --git a/source/extensions/filters/http/ext_proc/ext_proc.h b/source/extensions/filters/http/ext_proc/ext_proc.h index fbdd050f41ad..f3a380547a8b 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.h +++ b/source/extensions/filters/http/ext_proc/ext_proc.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -13,6 +14,9 @@ #include "envoy/service/ext_proc/v3/external_processor.pb.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" +#include "envoy/stream_info/stream_info.h" +#include "envoy/upstream/host_description.h" +#include "envoy/upstream/upstream.h" #include "source/common/common/logger.h" #include "source/common/protobuf/protobuf.h" @@ -63,7 +67,23 @@ class ExtProcLoggingInfo : public Envoy::StreamInfo::FilterState::Object { void recordGrpcCall(std::chrono::microseconds latency, Grpc::Status::GrpcStatus call_status, ProcessorState::CallbackState callback_state, envoy::config::core::v3::TrafficDirection traffic_direction); + void setBytesSent(uint64_t bytes_sent) { bytes_sent_ = bytes_sent; } + void setBytesReceived(uint64_t bytes_received) { bytes_received_ = bytes_received; } + void setClusterInfo(absl::optional cluster_info) { + if (cluster_info) { + cluster_info_ = cluster_info.value(); + } + } + void setUpstreamHost(absl::optional upstream_host) { + if (upstream_host) { + upstream_host_ = upstream_host.value(); + } + } + uint64_t bytesSent() const { return bytes_sent_; } + uint64_t bytesReceived() const { return bytes_received_; } + Upstream::ClusterInfoConstSharedPtr clusterInfo() const { return cluster_info_; } + Upstream::HostDescriptionConstSharedPtr upstreamHost() const { return upstream_host_; } const GrpcCalls& grpcCalls(envoy::config::core::v3::TrafficDirection traffic_direction) const; const Envoy::ProtobufWkt::Struct& filterMetadata() const { return filter_metadata_; } @@ -72,6 +92,11 @@ class ExtProcLoggingInfo : public Envoy::StreamInfo::FilterState::Object { GrpcCalls decoding_processor_grpc_calls_; GrpcCalls encoding_processor_grpc_calls_; const Envoy::ProtobufWkt::Struct filter_metadata_; + // The following stats are populated for ext_proc filters using Envoy gRPC only. + // The bytes sent and received are for the entire stream. + uint64_t bytes_sent_{0}, bytes_received_{0}; + Upstream::ClusterInfoConstSharedPtr cluster_info_; + Upstream::HostDescriptionConstSharedPtr upstream_host_; }; class FilterConfig { diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc index 45990a18e3cb..009458d53c6d 100644 --- a/test/extensions/filters/http/ext_proc/filter_test.cc +++ b/test/extensions/filters/http/ext_proc/filter_test.cc @@ -81,6 +81,17 @@ class HttpFilterTest : public testing::Test { EXPECT_CALL(decoder_callbacks_, dispatcher()).WillRepeatedly(ReturnRef(dispatcher_)); EXPECT_CALL(decoder_callbacks_, route()).WillRepeatedly(Return(route_)); EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(stream_info_)); + + EXPECT_CALL(async_client_stream_info_, bytesSent()).WillRepeatedly(Return(100)); + EXPECT_CALL(async_client_stream_info_, bytesReceived()).WillRepeatedly(Return(200)); + EXPECT_CALL(async_client_stream_info_, upstreamClusterInfo()); + EXPECT_CALL(testing::Const(async_client_stream_info_), upstreamInfo()); + // Get pointer to MockUpstreamInfo. + std::shared_ptr mock_upstream_info = + std::dynamic_pointer_cast( + async_client_stream_info_.upstreamInfo()); + EXPECT_CALL(testing::Const(*mock_upstream_info), upstreamHost()); + EXPECT_CALL(dispatcher_, createTimer_(_)) .Times(AnyNumber()) .WillRepeatedly(Invoke([this](Unused) { @@ -138,6 +149,9 @@ class HttpFilterTest : public testing::Test { auto stream = std::make_unique(); // We never send with the "close" flag set EXPECT_CALL(*stream, send(_, false)).WillRepeatedly(Invoke(this, &HttpFilterTest::doSend)); + + EXPECT_CALL(*stream, streamInfo()).WillRepeatedly(ReturnRef(async_client_stream_info_)); + // close is idempotent and only called once per filter EXPECT_CALL(*stream, close()).WillOnce(Invoke(this, &HttpFilterTest::doSendClose)); return stream; @@ -287,14 +301,17 @@ class HttpFilterTest : public testing::Test { } // The metadata configured as part of ext_proc filter should be in the filter state. - void expectMetadataInFilterState(const Envoy::ProtobufWkt::Struct& expected_metadata) { - const Envoy::ProtobufWkt::Struct& loggedMetadata = + // In addition, bytes sent/received should also be stored. + void expectFilterState(const Envoy::ProtobufWkt::Struct& expected_metadata) { + const auto* filterState = stream_info_.filterState() ->getDataReadOnly< Envoy::Extensions::HttpFilters::ExternalProcessing::ExtProcLoggingInfo>( - filter_config_name) - ->filterMetadata(); + filter_config_name); + const Envoy::ProtobufWkt::Struct& loggedMetadata = filterState->filterMetadata(); EXPECT_THAT(loggedMetadata, ProtoEq(expected_metadata)); + EXPECT_EQ(filterState->bytesSent(), 100); + EXPECT_EQ(filterState->bytesReceived(), 200); } absl::optional final_expected_grpc_service_; @@ -310,6 +327,7 @@ class HttpFilterTest : public testing::Test { testing::NiceMock encoder_callbacks_; Router::RouteConstSharedPtr route_; testing::NiceMock stream_info_; + testing::NiceMock async_client_stream_info_; Http::TestRequestHeaderMapImpl request_headers_; Http::TestResponseHeaderMapImpl response_headers_; Http::TestRequestTrailerMapImpl request_trailers_; @@ -385,7 +403,7 @@ TEST_F(HttpFilterTest, SimplestPost) { Envoy::ProtobufWkt::Struct filter_metadata; (*filter_metadata.mutable_fields())["scooby"].set_string_value("doo"); - expectMetadataInFilterState(filter_metadata); + expectFilterState(filter_metadata); } // Using the default configuration, test the filter with a processor that @@ -531,7 +549,7 @@ TEST_F(HttpFilterTest, PostAndRespondImmediately) { EXPECT_EQ(1, config_->stats().streams_closed_.value()); expectGrpcCalls(envoy::config::core::v3::TrafficDirection::INBOUND, Grpc::Status::Ok, 1); expectGrpcCalls(envoy::config::core::v3::TrafficDirection::OUTBOUND, Grpc::Status::Ok, 0); - expectMetadataInFilterState(Envoy::ProtobufWkt::Struct()); + expectFilterState(Envoy::ProtobufWkt::Struct()); } // Using the default configuration, test the filter with a processor that diff --git a/test/extensions/filters/http/ext_proc/mock_server.h b/test/extensions/filters/http/ext_proc/mock_server.h index f9d931603a79..f655e977a2e4 100644 --- a/test/extensions/filters/http/ext_proc/mock_server.h +++ b/test/extensions/filters/http/ext_proc/mock_server.h @@ -24,6 +24,7 @@ class MockStream : public ExternalProcessorStream { ~MockStream() override; MOCK_METHOD(void, send, (envoy::service::ext_proc::v3::ProcessingRequest&&, bool)); MOCK_METHOD(bool, close, ()); + MOCK_METHOD(const StreamInfo::StreamInfo&, streamInfo, (), (const override)); }; } // namespace ExternalProcessing diff --git a/test/extensions/filters/http/ext_proc/ordering_test.cc b/test/extensions/filters/http/ext_proc/ordering_test.cc index 1d0d4c52b7cb..2f9575c6265a 100644 --- a/test/extensions/filters/http/ext_proc/ordering_test.cc +++ b/test/extensions/filters/http/ext_proc/ordering_test.cc @@ -86,6 +86,7 @@ class OrderingTest : public testing::Test { stream_callbacks_ = &callbacks; auto stream = std::make_unique(); EXPECT_CALL(*stream, send(_, _)).WillRepeatedly(Invoke(this, &OrderingTest::doSend)); + EXPECT_CALL(*stream, streamInfo()).WillRepeatedly(ReturnRef(async_client_stream_info_)); EXPECT_CALL(*stream, close()); return stream; } @@ -206,6 +207,7 @@ class OrderingTest : public testing::Test { NiceMock decoder_callbacks_; NiceMock encoder_callbacks_; testing::NiceMock stream_info_; + testing::NiceMock async_client_stream_info_; Http::TestRequestHeaderMapImpl request_headers_; Http::TestResponseHeaderMapImpl response_headers_; Http::TestRequestTrailerMapImpl request_trailers_; @@ -221,6 +223,7 @@ class FastFailOrderingTest : public OrderingTest { const StreamInfo::StreamInfo&) override { auto stream = std::make_unique(); EXPECT_CALL(*stream, close()); + EXPECT_CALL(*stream, streamInfo()).WillRepeatedly(ReturnRef(async_client_stream_info_)); callbacks.onGrpcError(Grpc::Status::Internal); return stream; } diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc index 89d6c55c9069..477a04bb2841 100644 --- a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc @@ -33,6 +33,7 @@ class FuzzerMocks { NiceMock connection_; NiceMock request_trailers_; NiceMock response_trailers_; + testing::NiceMock async_client_stream_info_; }; DEFINE_PROTO_FUZZER( @@ -85,6 +86,8 @@ DEFINE_PROTO_FUZZER( input.response()); filter->onReceiveMessage(std::move(response)); })); + EXPECT_CALL(*stream, streamInfo()) + .WillRepeatedly(ReturnRef(mocks.async_client_stream_info_)); ON_CALL(*stream, close()).WillByDefault(Return(false)); return stream; })); diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/mocks.h b/test/extensions/filters/http/ext_proc/unit_test_fuzz/mocks.h index 9ae229e83a15..cbfafb5f2473 100644 --- a/test/extensions/filters/http/ext_proc/unit_test_fuzz/mocks.h +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/mocks.h @@ -20,6 +20,7 @@ class MockStream : public ExternalProcessing::ExternalProcessorStream { MOCK_METHOD(void, send, (envoy::service::ext_proc::v3::ProcessingRequest && request, bool end_stream)); MOCK_METHOD(bool, close, ()); + MOCK_METHOD(const StreamInfo::StreamInfo&, streamInfo, (), (const override)); }; class MockClient : public ExternalProcessing::ExternalProcessorClient { From af9b04edc8349ee34fa21ab98b0965f8816f718d Mon Sep 17 00:00:00 2001 From: code Date: Wed, 24 May 2023 08:35:48 +0800 Subject: [PATCH 342/740] upstream: the cluster provoded load balancing policy could be configured as extension (#27512) * upstream: the cluster provoded load balancing policy could be configured as extension Signed-off-by: wbpcode * address comments Signed-off-by: wbpcode --------- Signed-off-by: wbpcode --- CODEOWNERS | 1 + api/BUILD | 1 + .../cluster_provided/v3/BUILD | 9 ++ .../v3/cluster_provided.proto | 24 ++++ api/versioning/BUILD | 1 + changelogs/current.yaml | 5 + .../common/upstream/cluster_manager_impl.cc | 58 +++++----- source/extensions/extensions_build_config.bzl | 5 +- source/extensions/extensions_metadata.yaml | 7 ++ .../cluster_provided/BUILD | 19 ++++ .../cluster_provided/config.cc | 24 ++++ .../cluster_provided/config.h | 31 ++++++ .../cluster_provided/BUILD | 40 +++++++ .../cluster_provided/config_test.cc | 41 +++++++ .../cluster_provided/integration_test.cc | 103 ++++++++++++++++++ 15 files changed, 342 insertions(+), 27 deletions(-) create mode 100644 api/envoy/extensions/load_balancing_policies/cluster_provided/v3/BUILD create mode 100644 api/envoy/extensions/load_balancing_policies/cluster_provided/v3/cluster_provided.proto create mode 100644 source/extensions/load_balancing_policies/cluster_provided/BUILD create mode 100644 source/extensions/load_balancing_policies/cluster_provided/config.cc create mode 100644 source/extensions/load_balancing_policies/cluster_provided/config.h create mode 100644 test/extensions/load_balancing_policies/cluster_provided/BUILD create mode 100644 test/extensions/load_balancing_policies/cluster_provided/config_test.cc create mode 100644 test/extensions/load_balancing_policies/cluster_provided/integration_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index adc46f60230d..efa25078addf 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -298,6 +298,7 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/load_balancing_policies/ring_hash @wbpcode @UNOWNED /*/extensions/load_balancing_policies/maglev @wbpcode @UNOWNED /*/extensions/load_balancing_policies/subset @wbpcode @zuercher +/*/extensions/load_balancing_policies/cluster_provided @wbpcode @zuercher # Early header mutation /*/extensions/http/early_header_mutation/header_mutation @wbpcode @UNOWNED # Network matching extensions diff --git a/api/BUILD b/api/BUILD index 898f318b5795..cc3a5068531d 100644 --- a/api/BUILD +++ b/api/BUILD @@ -252,6 +252,7 @@ proto_library( "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", "//envoy/extensions/key_value/file_based/v3:pkg", + "//envoy/extensions/load_balancing_policies/cluster_provided/v3:pkg", "//envoy/extensions/load_balancing_policies/common/v3:pkg", "//envoy/extensions/load_balancing_policies/least_request/v3:pkg", "//envoy/extensions/load_balancing_policies/maglev/v3:pkg", diff --git a/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/BUILD b/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/BUILD new file mode 100644 index 000000000000..ee92fb652582 --- /dev/null +++ b/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/cluster_provided.proto b/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/cluster_provided.proto new file mode 100644 index 000000000000..bf12abc6e899 --- /dev/null +++ b/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/cluster_provided.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package envoy.extensions.load_balancing_policies.cluster_provided.v3; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.load_balancing_policies.cluster_provided.v3"; +option java_outer_classname = "ClusterProvidedProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/cluster_provided/v3;cluster_providedv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Cluster Provided Load Balancing Policy] +// [#extension: envoy.load_balancing_policies.cluster_provided] + +// The cluster provided load balancing policy allows cluster to specify its own load balancing. +// If this extension is configured, the target cluster must provide load balancer when the cluster +// is created. +// +// ..note:: +// Cluster provided load balancing policy could not be used as sub-policy of other hierarchical +// load balancing policies, such as subset load balancing policy. +message ClusterProvided { +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index ad7aa0d75b47..4f4a25d60e82 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -191,6 +191,7 @@ proto_library( "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", "//envoy/extensions/key_value/file_based/v3:pkg", "//envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3:pkg", + "//envoy/extensions/load_balancing_policies/cluster_provided/v3:pkg", "//envoy/extensions/load_balancing_policies/common/v3:pkg", "//envoy/extensions/load_balancing_policies/least_request/v3:pkg", "//envoy/extensions/load_balancing_policies/maglev/v3:pkg", diff --git a/changelogs/current.yaml b/changelogs/current.yaml index a39d3c460410..e0205dcb8888 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -212,6 +212,11 @@ new_features: - area: admin change: | Adds a new admin stats html bucket-mode ``detailed`` to generate all recorded buckets and summary percentiles. +- area: upstream + change: | + Added :ref:`cluster provided extension + ` + to suppport the :ref:`load balancer policy `. - area: fault change: | added new field ``envoy.extensions.filters.http.fault.v3.HTTPFault.filter_metadata`` to aid in logging. diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index d5a2b85fb0f6..a273a397b791 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -835,24 +835,35 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust auto& lb = new_cluster_pair_or_error->second; Cluster& cluster_reference = *new_cluster; + const auto cluster_info = cluster_reference.info(); + if (!added_via_api) { - if (cluster_map.find(new_cluster->info()->name()) != cluster_map.end()) { + if (cluster_map.find(cluster_info->name()) != cluster_map.end()) { throw EnvoyException( - fmt::format("cluster manager: duplicate cluster '{}'", new_cluster->info()->name())); + fmt::format("cluster manager: duplicate cluster '{}'", cluster_info->name())); } } - if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided && lb == nullptr) { + // Check if the cluster provided load balancing policy is used. We need handle it as special + // case. + bool cluster_provided_lb = cluster_info->lbType() == LoadBalancerType::ClusterProvided; + if (cluster_info->lbType() == LoadBalancerType::LoadBalancingPolicyConfig) { + TypedLoadBalancerFactory* typed_lb_factory = cluster_info->loadBalancerFactory(); + RELEASE_ASSERT(typed_lb_factory != nullptr, "ClusterInfo should contain a valid factory"); + cluster_provided_lb = + typed_lb_factory->name() == "envoy.load_balancing_policies.cluster_provided"; + } + + if (cluster_provided_lb && lb == nullptr) { throw EnvoyException(fmt::format("cluster manager: cluster provided LB specified but cluster " "'{}' did not provide one. Check cluster documentation.", - new_cluster->info()->name())); + cluster_info->name())); } - - if (cluster_reference.info()->lbType() != LoadBalancerType::ClusterProvided && lb != nullptr) { + if (!cluster_provided_lb && lb != nullptr) { throw EnvoyException( fmt::format("cluster manager: cluster provided LB not specified but cluster " "'{}' provided one. Check cluster documentation.", - new_cluster->info()->name())); + cluster_info->name())); } if (new_cluster->healthChecker() != nullptr) { @@ -876,7 +887,7 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust }); } ClusterDataPtr result; - auto cluster_entry_it = cluster_map.find(cluster_reference.info()->name()); + auto cluster_entry_it = cluster_map.find(cluster_info->name()); if (cluster_entry_it != cluster_map.end()) { result = std::exchange(cluster_entry_it->second, std::make_unique(cluster, cluster_hash, version_info, @@ -885,7 +896,7 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust } else { bool inserted = false; std::tie(cluster_entry_it, inserted) = cluster_map.emplace( - cluster_reference.info()->name(), + cluster_info->name(), std::make_unique(cluster, cluster_hash, version_info, added_via_api, std::move(new_cluster), time_source_)); ASSERT(inserted); @@ -893,30 +904,27 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust // If an LB is thread aware, create it here. The LB is not initialized until cluster pre-init // finishes. For RingHash/Maglev don't create the LB here if subset balancing is enabled, // because the thread_aware_lb_ field takes precedence over the subset lb). - if (cluster_reference.info()->lbType() == LoadBalancerType::RingHash) { - if (!cluster_reference.info()->lbSubsetInfo().isEnabled()) { + if (cluster_info->lbType() == LoadBalancerType::RingHash) { + if (!cluster_info->lbSubsetInfo().isEnabled()) { auto& factory = Config::Utility::getAndCheckFactoryByName( "envoy.load_balancing_policies.ring_hash"); - cluster_entry_it->second->thread_aware_lb_ = - factory.create(*cluster_reference.info(), cluster_reference.prioritySet(), runtime_, - random_, time_source_); + cluster_entry_it->second->thread_aware_lb_ = factory.create( + *cluster_info, cluster_reference.prioritySet(), runtime_, random_, time_source_); } - } else if (cluster_reference.info()->lbType() == LoadBalancerType::Maglev) { - if (!cluster_reference.info()->lbSubsetInfo().isEnabled()) { + } else if (cluster_info->lbType() == LoadBalancerType::Maglev) { + if (!cluster_info->lbSubsetInfo().isEnabled()) { auto& factory = Config::Utility::getAndCheckFactoryByName( "envoy.load_balancing_policies.maglev"); - cluster_entry_it->second->thread_aware_lb_ = - factory.create(*cluster_reference.info(), cluster_reference.prioritySet(), runtime_, - random_, time_source_); + cluster_entry_it->second->thread_aware_lb_ = factory.create( + *cluster_info, cluster_reference.prioritySet(), runtime_, random_, time_source_); } - } else if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided) { + } else if (cluster_provided_lb) { cluster_entry_it->second->thread_aware_lb_ = std::move(lb); - } else if (cluster_reference.info()->lbType() == LoadBalancerType::LoadBalancingPolicyConfig) { - TypedLoadBalancerFactory* typed_lb_factory = cluster_reference.info()->loadBalancerFactory(); + } else if (cluster_info->lbType() == LoadBalancerType::LoadBalancingPolicyConfig) { + TypedLoadBalancerFactory* typed_lb_factory = cluster_info->loadBalancerFactory(); RELEASE_ASSERT(typed_lb_factory != nullptr, "ClusterInfo should contain a valid factory"); - cluster_entry_it->second->thread_aware_lb_ = - typed_lb_factory->create(*cluster_reference.info(), cluster_reference.prioritySet(), - runtime_, random_, time_source_); + cluster_entry_it->second->thread_aware_lb_ = typed_lb_factory->create( + *cluster_info, cluster_reference.prioritySet(), runtime_, random_, time_source_); } updateClusterCounts(); diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 9024d14b66c0..e0370c799dc2 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -426,8 +426,9 @@ EXTENSIONS = { "envoy.load_balancing_policies.random": "//source/extensions/load_balancing_policies/random:config", "envoy.load_balancing_policies.round_robin": "//source/extensions/load_balancing_policies/round_robin:config", "envoy.load_balancing_policies.maglev": "//source/extensions/load_balancing_policies/maglev:config", - "envoy.load_balancing_policies.ring_hash": "//source/extensions/load_balancing_policies/ring_hash:config", - "envoy.load_balancing_policies.subset": "//source/extensions/load_balancing_policies/subset:config", + "envoy.load_balancing_policies.ring_hash": "//source/extensions/load_balancing_policies/ring_hash:config", + "envoy.load_balancing_policies.subset": "//source/extensions/load_balancing_policies/subset:config", + "envoy.load_balancing_policies.cluster_provided": "//source/extensions/load_balancing_policies/cluster_provided:config", # # HTTP Early Header Mutation diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 4513925d8d31..c1e704971d5a 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -1490,6 +1490,13 @@ envoy.load_balancing_policies.subset: - envoy.load_balancing_policies security_posture: robust_to_untrusted_downstream_and_upstream status: stable +envoy.load_balancing_policies.cluster_provided: + categories: + - envoy.load_balancing_policies + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable + type_urls: + - envoy.extensions.load_balancing_policies.cluster_provided.v3.ClusterProvided envoy.http.early_header_mutation.header_mutation: categories: - envoy.http.early_header_mutation diff --git a/source/extensions/load_balancing_policies/cluster_provided/BUILD b/source/extensions/load_balancing_policies/cluster_provided/BUILD new file mode 100644 index 000000000000..2cd5fcce2124 --- /dev/null +++ b/source/extensions/load_balancing_policies/cluster_provided/BUILD @@ -0,0 +1,19 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//source/common/upstream:load_balancer_factory_base_lib", + "@envoy_api//envoy/extensions/load_balancing_policies/cluster_provided/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/load_balancing_policies/cluster_provided/config.cc b/source/extensions/load_balancing_policies/cluster_provided/config.cc new file mode 100644 index 000000000000..33245f9b7907 --- /dev/null +++ b/source/extensions/load_balancing_policies/cluster_provided/config.cc @@ -0,0 +1,24 @@ +#include "source/extensions/load_balancing_policies/cluster_provided/config.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace ClusterProvided { + +Upstream::ThreadAwareLoadBalancerPtr Factory::create(const Upstream::ClusterInfo&, + const Upstream::PrioritySet&, Runtime::Loader&, + Random::RandomGenerator&, TimeSource&) { + // Cluster provided load balancer has empty implementation. Because it is a special case to + // tell the cluster to use the load balancer provided by the cluster. + return nullptr; +}; + +/** + * Static registration for the Factory. @see RegisterFactory. + */ +REGISTER_FACTORY(Factory, Upstream::TypedLoadBalancerFactory); + +} // namespace ClusterProvided +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/load_balancing_policies/cluster_provided/config.h b/source/extensions/load_balancing_policies/cluster_provided/config.h new file mode 100644 index 000000000000..ef1d73da6969 --- /dev/null +++ b/source/extensions/load_balancing_policies/cluster_provided/config.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include "envoy/extensions/load_balancing_policies/cluster_provided/v3/cluster_provided.pb.h" +#include "envoy/upstream/load_balancer.h" + +#include "source/common/upstream/load_balancer_factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace ClusterProvided { + +class Factory + : public Upstream::TypedLoadBalancerFactoryBase< + envoy::extensions::load_balancing_policies::cluster_provided::v3::ClusterProvided> { +public: + Factory() : TypedLoadBalancerFactoryBase("envoy.load_balancing_policies.cluster_provided") {} + + Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& priority_set, + Runtime::Loader& runtime, + Random::RandomGenerator& random, + TimeSource& time_source) override; +}; + +} // namespace ClusterProvided +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/load_balancing_policies/cluster_provided/BUILD b/test/extensions/load_balancing_policies/cluster_provided/BUILD new file mode 100644 index 000000000000..af0b1d96c2af --- /dev/null +++ b/test/extensions/load_balancing_policies/cluster_provided/BUILD @@ -0,0 +1,40 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.load_balancing_policies.cluster_provided"], + deps = [ + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//test/mocks/server:factory_context_mocks", + "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:priority_set_mocks", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "integration_test", + size = "large", + srcs = ["integration_test.cc"], + extension_names = ["envoy.load_balancing_policies.cluster_provided"], + deps = [ + "//source/common/protobuf", + "//source/extensions/clusters/original_dst:original_dst_cluster_lib", + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/load_balancing_policies/cluster_provided/config_test.cc b/test/extensions/load_balancing_policies/cluster_provided/config_test.cc new file mode 100644 index 000000000000..e0966208e7aa --- /dev/null +++ b/test/extensions/load_balancing_policies/cluster_provided/config_test.cc @@ -0,0 +1,41 @@ +#include "envoy/config/core/v3/extension.pb.h" + +#include "source/extensions/load_balancing_policies/cluster_provided/config.h" + +#include "test/mocks/server/factory_context.h" +#include "test/mocks/upstream/cluster_info.h" +#include "test/mocks/upstream/priority_set.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace ClusterProvided { +namespace { + +TEST(ClusterProvidedConfigTest, ClusterProvidedConfigTest) { + NiceMock context; + NiceMock cluster_info; + NiceMock main_thread_priority_set; + NiceMock thread_local_priority_set; + + envoy::config::core::v3::TypedExtensionConfig config; + config.set_name("envoy.load_balancing_policies.cluster_provided"); + envoy::extensions::load_balancing_policies::cluster_provided::v3::ClusterProvided config_msg; + config.mutable_typed_config()->PackFrom(config_msg); + + auto& factory = Config::Utility::getAndCheckFactory(config); + EXPECT_EQ("envoy.load_balancing_policies.cluster_provided", factory.name()); + + auto thread_aware_lb = + factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_, + context.api_.random_, context.time_system_); + EXPECT_EQ(nullptr, thread_aware_lb); +} + +} // namespace +} // namespace ClusterProvided +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/load_balancing_policies/cluster_provided/integration_test.cc b/test/extensions/load_balancing_policies/cluster_provided/integration_test.cc new file mode 100644 index 000000000000..c190be34bc8d --- /dev/null +++ b/test/extensions/load_balancing_policies/cluster_provided/integration_test.cc @@ -0,0 +1,103 @@ +#include +#include + +#include "envoy/config/endpoint/v3/endpoint_components.pb.h" + +#include "source/common/common/base64.h" +#include "source/common/http/utility.h" +#include "source/common/protobuf/protobuf.h" +#include "source/extensions/load_balancing_policies/cluster_provided/config.h" + +#include "test/integration/http_integration.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace ClusterProvided { +namespace { + +class ClusterProvidedIntegrationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + ClusterProvidedIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) { + // Create 3 different upstream server. + setUpstreamCount(3); + + // Update endpoints of default cluster `cluster_0` to `original_dst` cluster. + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + + std::string cluster_yaml = R"EOF( + name: cluster_0 + connect_timeout: 1.250s + type: ORIGINAL_DST + lb_policy: CLUSTER_PROVIDED + original_dst_lb_config: + use_http_header: true + )EOF"; + + TestUtility::loadFromYaml(cluster_yaml, *cluster_0); + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.cluster_provided + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.cluster_provided.v3.ClusterProvided + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, ClusterProvidedIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +// Test the case where the cluster provided load balancer is configured by the load balancing +// policy API and it works as expected. +TEST_P(ClusterProvidedIntegrationTest, NormalLoadBalancing) { + initialize(); + + for (uint64_t i = 0; i < 4; i++) { + for (size_t upstream_index = 0; upstream_index < fake_upstreams_.size(); upstream_index++) { + codec_client_ = makeHttpConnection(lookupPort("http")); + + const auto& upstream_target_address = + fake_upstreams_[upstream_index]->localAddress()->asString(); + + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "example.com"}, + {"x-envoy-original-dst-host", upstream_target_address}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + auto upstream = waitForNextUpstreamRequest({0, 1, 2}); + EXPECT_EQ(upstream.value(), upstream_index); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + cleanupUpstreamAndDownstream(); + } + } +} + +} // namespace +} // namespace ClusterProvided +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy From 1cb5be99507b698c50c87ae332ca3fe95cbb12ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E6=B3=BD=E8=BD=A9?= Date: Wed, 24 May 2023 10:51:41 +0800 Subject: [PATCH 343/740] chore(dlb): remove unused isOpen call (#27539) Signed-off-by: spacewander --- .../connection_balance/dlb/source/connection_balancer_impl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc b/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc index 6d8cec6c3c04..1e5fd46d5bb9 100644 --- a/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc +++ b/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc @@ -236,9 +236,9 @@ void DlbBalancedConnectionHandlerImpl::setDlbEvent() { dlb_event_->setEnabled(Event::FileReadyType::Read); } -void DlbBalancedConnectionHandlerImpl::post(Network::ConnectionSocketPtr&& socket) { +void DlbBalancedConnectionHandlerImpl::post( + [[maybe_unused]] Network::ConnectionSocketPtr&& socket) { #ifdef DLB_DISABLED - socket->isOpen(); throw EnvoyException("X86_64 architecture is required for Dlb."); #else // The pointer will be casted to unique_ptr in onDlbEvents(), no need to consider free. From cbadae62b8b12e0c1e47a374b10ade88a6a0bba5 Mon Sep 17 00:00:00 2001 From: Peter Leng Date: Tue, 23 May 2023 19:52:18 -0700 Subject: [PATCH 344/740] Route level config support for AWS request signing filter (#27092) Signed-off-by: yleng --- .../v3/aws_request_signing.proto | 9 +++ ...t-signing-filter-route-level-override.yaml | 66 +++++++++++++++++++ .../_include/aws-request-signing-filter.yaml | 52 +++++++++++++++ .../aws_request_signing_filter.rst | 30 +++++---- source/extensions/extensions_metadata.yaml | 1 + .../aws_request_signing_filter.cc | 38 +++++++---- .../aws_request_signing_filter.h | 6 +- .../http/aws_request_signing/config.cc | 24 ++++++- .../filters/http/aws_request_signing/config.h | 23 +++++-- .../aws_request_signing_filter_test.cc | 19 ++++++ .../http/aws_request_signing/config_test.cc | 30 ++++++++- 11 files changed, 260 insertions(+), 38 deletions(-) create mode 100644 docs/root/configuration/http/http_filters/_include/aws-request-signing-filter-route-level-override.yaml create mode 100644 docs/root/configuration/http/http_filters/_include/aws-request-signing-filter.yaml diff --git a/api/envoy/extensions/filters/http/aws_request_signing/v3/aws_request_signing.proto b/api/envoy/extensions/filters/http/aws_request_signing/v3/aws_request_signing.proto index ba26a3882407..df4633ce63ee 100644 --- a/api/envoy/extensions/filters/http/aws_request_signing/v3/aws_request_signing.proto +++ b/api/envoy/extensions/filters/http/aws_request_signing/v3/aws_request_signing.proto @@ -64,3 +64,12 @@ message AwsRequestSigning { // When applied, all headers that start with "x-envoy" and headers "foo" and "bar" will not be signed. repeated type.matcher.v3.StringMatcher match_excluded_headers = 5; } + +message AwsRequestSigningPerRoute { + // Override the global configuration of the filter with this new config. + // This overrides the entire message of AwsRequestSigning and not at field level. + AwsRequestSigning aws_request_signing = 1; + + // The human readable prefix to use when emitting stats. + string stat_prefix = 2 [(validate.rules).string = {min_len: 1}]; +} diff --git a/docs/root/configuration/http/http_filters/_include/aws-request-signing-filter-route-level-override.yaml b/docs/root/configuration/http/http_filters/_include/aws-request-signing-filter-route-level-override.yaml new file mode 100644 index 000000000000..987d99134053 --- /dev/null +++ b/docs/root/configuration/http/http_filters/_include/aws-request-signing-filter-route-level-override.yaml @@ -0,0 +1,66 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: app + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: versioned-cluster + typed_per_filter_config: + envoy.filters.http.aws_request_signing: + "@type": type.googleapis.com/envoy.extensions.filters.http.aws_request_signing.v3.AwsRequestSigningPerRoute + aws_request_signing: + service_name: s3 + region: us-west-1 + use_unsigned_payload: true + host_rewrite: new-host + match_excluded_headers: + - prefix: x-envoy + - prefix: x-forwarded + - exact: x-amzn-trace-id + stat_prefix: some-prefix + http_filters: + - name: envoy.filters.http.aws_request_signing + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.aws_request_signing.v3.AwsRequestSigning + service_name: s3 + region: us-west-2 + use_unsigned_payload: true + host_rewrite: default-host + match_excluded_headers: + - prefix: x-envoy + - prefix: x-forwarded + - exact: x-amzn-trace-id + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + clusters: + - name: versioned-cluster + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: versioned-cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8080 diff --git a/docs/root/configuration/http/http_filters/_include/aws-request-signing-filter.yaml b/docs/root/configuration/http/http_filters/_include/aws-request-signing-filter.yaml new file mode 100644 index 000000000000..16dde43446f0 --- /dev/null +++ b/docs/root/configuration/http/http_filters/_include/aws-request-signing-filter.yaml @@ -0,0 +1,52 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: app + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: versioned-cluster + http_filters: + - name: envoy.filters.http.aws_request_signing + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.aws_request_signing.v3.AwsRequestSigning + service_name: s3 + region: us-west-2 + use_unsigned_payload: true + match_excluded_headers: + - prefix: x-envoy + - prefix: x-forwarded + - exact: x-amzn-trace-id + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + clusters: + - name: versioned-cluster + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: versioned-cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8080 diff --git a/docs/root/configuration/http/http_filters/aws_request_signing_filter.rst b/docs/root/configuration/http/http_filters/aws_request_signing_filter.rst index 2f33a1e4f6ef..886baee1d27a 100644 --- a/docs/root/configuration/http/http_filters/aws_request_signing_filter.rst +++ b/docs/root/configuration/http/http_filters/aws_request_signing_filter.rst @@ -34,19 +34,23 @@ Example configuration Example filter configuration: -.. code-block:: yaml - - name: envoy.filters.http.aws_request_signing - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.aws_request_signing.v3.AwsRequestSigning - service_name: s3 - region: us-west-2 - use_unsigned_payload: true - match_excluded_headers: - - prefix: x-envoy - - prefix: x-forwarded - - exact: x-amzn-trace-id - +.. literalinclude:: _include/aws-request-signing-filter.yaml + :language: yaml + :lines: 25-35 + :lineno-start: 25 + :linenos: + :caption: :download:`aws-request-signing-filter.yaml <_include/aws-request-signing-filter.yaml>` + +Note that this filter also supports per route configuration: + +.. literalinclude:: _include/aws-request-signing-filter-route-level-override.yaml + :language: yaml + :lines: 20-37 + :lineno-start: 20 + :linenos: + :caption: :download:`aws-request-signing-filter-route-level-override.yaml <_include/aws-request-signing-filter-route-level-override.yaml>` + +Above shows an example of route-level config overriding the config on the virtual-host level. .. include:: _include/aws_credentials.rst diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index c1e704971d5a..c0113843a8dd 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -208,6 +208,7 @@ envoy.filters.http.aws_request_signing: status: alpha type_urls: - envoy.extensions.filters.http.aws_request_signing.v3.AwsRequestSigning + - envoy.extensions.filters.http.aws_request_signing.v3.AwsRequestSigningPerRoute envoy.filters.http.bandwidth_limit: categories: - envoy.filters.http diff --git a/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.cc b/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.cc index 94b1c6add1a6..beefccd8c800 100644 --- a/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.cc +++ b/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.cc @@ -4,6 +4,7 @@ #include "source/common/common/hex.h" #include "source/common/crypto/utility.h" +#include "source/common/http/utility.h" namespace Envoy { namespace Extensions { @@ -31,8 +32,10 @@ FilterStats Filter::generateStats(const std::string& prefix, Stats::Scope& scope } Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, bool end_stream) { - const auto& host_rewrite = config_->hostRewrite(); - const bool use_unsigned_payload = config_->useUnsignedPayload(); + auto& config = getConfig(); + + const auto& host_rewrite = config.hostRewrite(); + const bool use_unsigned_payload = config.useUnsignedPayload(); if (!host_rewrite.empty()) { headers.setHost(host_rewrite); @@ -47,22 +50,24 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, ENVOY_LOG(debug, "aws request signing from decodeHeaders use_unsigned_payload: {}", use_unsigned_payload); if (use_unsigned_payload) { - config_->signer().signUnsignedPayload(headers); + config.signer().signUnsignedPayload(headers); } else { - config_->signer().signEmptyPayload(headers); + config.signer().signEmptyPayload(headers); } - config_->stats().signing_added_.inc(); + config.stats().signing_added_.inc(); } catch (const EnvoyException& e) { // TODO: sign should not throw to avoid exceptions in the request path ENVOY_LOG(debug, "signing failed: {}", e.what()); - config_->stats().signing_failed_.inc(); + config.stats().signing_failed_.inc(); } return Http::FilterHeadersStatus::Continue; } Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_stream) { - if (config_->useUnsignedPayload()) { + auto& config = getConfig(); + + if (config.useUnsignedPayload()) { return Http::FilterDataStatus::Continue; } @@ -80,19 +85,28 @@ Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_strea try { ENVOY_LOG(debug, "aws request signing from decodeData"); ASSERT(request_headers_ != nullptr); - config_->signer().sign(*request_headers_, hash); - config_->stats().signing_added_.inc(); - config_->stats().payload_signing_added_.inc(); + config.signer().sign(*request_headers_, hash); + config.stats().signing_added_.inc(); + config.stats().payload_signing_added_.inc(); } catch (const EnvoyException& e) { // TODO: sign should not throw to avoid exceptions in the request path ENVOY_LOG(debug, "signing failed: {}", e.what()); - config_->stats().signing_failed_.inc(); - config_->stats().payload_signing_failed_.inc(); + config.stats().signing_failed_.inc(); + config.stats().payload_signing_failed_.inc(); } return Http::FilterDataStatus::Continue; } +FilterConfig& Filter::getConfig() const { + auto* config = const_cast( + Http::Utility::resolveMostSpecificPerFilterConfig(decoder_callbacks_)); + if (config) { + return *config; + } + return *config_; +} + } // namespace AwsRequestSigningFilter } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.h b/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.h index 2ae73277d995..c0a70106b619 100644 --- a/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.h +++ b/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.h @@ -34,9 +34,9 @@ struct FilterStats { /** * Abstract filter configuration. */ -class FilterConfig { +class FilterConfig : public Router::RouteSpecificFilterConfig { public: - virtual ~FilterConfig() = default; + ~FilterConfig() override = default; /** * @return the config's signer. @@ -95,6 +95,8 @@ class Filter : public Http::PassThroughDecoderFilter, Logger::Loggable config_; Http::RequestHeaderMap* request_headers_{}; }; diff --git a/source/extensions/filters/http/aws_request_signing/config.cc b/source/extensions/filters/http/aws_request_signing/config.cc index 6743c9473c8b..c277f8e600c6 100644 --- a/source/extensions/filters/http/aws_request_signing/config.cc +++ b/source/extensions/filters/http/aws_request_signing/config.cc @@ -15,8 +15,8 @@ namespace HttpFilters { namespace AwsRequestSigningFilter { Http::FilterFactoryCb AwsRequestSigningFilterFactory::createFilterFactoryFromProtoTyped( - const envoy::extensions::filters::http::aws_request_signing::v3::AwsRequestSigning& config, - const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + const AwsRequestSigningProtoConfig& config, const std::string& stats_prefix, + Server::Configuration::FactoryContext& context) { auto credentials_provider = std::make_shared( @@ -35,6 +35,26 @@ Http::FilterFactoryCb AwsRequestSigningFilterFactory::createFilterFactoryFromPro }; } +Router::RouteSpecificFilterConfigConstSharedPtr +AwsRequestSigningFilterFactory::createRouteSpecificFilterConfigTyped( + const AwsRequestSigningProtoPerRouteConfig& per_route_config, + Server::Configuration::ServerFactoryContext& context, ProtobufMessage::ValidationVisitor&) { + auto credentials_provider = + std::make_shared( + context.api(), Extensions::Common::Aws::Utility::fetchMetadata); + const auto matcher_config = Extensions::Common::Aws::AwsSigV4HeaderExclusionVector( + per_route_config.aws_request_signing().match_excluded_headers().begin(), + per_route_config.aws_request_signing().match_excluded_headers().end()); + auto signer = std::make_unique( + per_route_config.aws_request_signing().service_name(), + per_route_config.aws_request_signing().region(), credentials_provider, + context.mainThreadDispatcher().timeSource(), matcher_config); + return std::make_shared( + std::move(signer), per_route_config.stat_prefix(), context.scope(), + per_route_config.aws_request_signing().host_rewrite(), + per_route_config.aws_request_signing().use_unsigned_payload()); +} + /** * Static registration for the AWS request signing filter. @see RegisterFactory. */ diff --git a/source/extensions/filters/http/aws_request_signing/config.h b/source/extensions/filters/http/aws_request_signing/config.h index 55214c6afdbb..e066323ee9e5 100644 --- a/source/extensions/filters/http/aws_request_signing/config.h +++ b/source/extensions/filters/http/aws_request_signing/config.h @@ -10,20 +10,31 @@ namespace Extensions { namespace HttpFilters { namespace AwsRequestSigningFilter { +using AwsRequestSigningProtoConfig = + envoy::extensions::filters::http::aws_request_signing::v3::AwsRequestSigning; + +using AwsRequestSigningProtoPerRouteConfig = + envoy::extensions::filters::http::aws_request_signing::v3::AwsRequestSigningPerRoute; + /** * Config registration for the AWS request signing filter. */ class AwsRequestSigningFilterFactory - : public Common::FactoryBase< - envoy::extensions::filters::http::aws_request_signing::v3::AwsRequestSigning> { + : public Common::FactoryBase { public: AwsRequestSigningFilterFactory() : FactoryBase("envoy.filters.http.aws_request_signing") {} private: - Http::FilterFactoryCb createFilterFactoryFromProtoTyped( - const envoy::extensions::filters::http::aws_request_signing::v3::AwsRequestSigning& - proto_config, - const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; + Http::FilterFactoryCb + createFilterFactoryFromProtoTyped(const AwsRequestSigningProtoConfig& proto_config, + const std::string& stats_prefix, + Server::Configuration::FactoryContext& context) override; + + Router::RouteSpecificFilterConfigConstSharedPtr + createRouteSpecificFilterConfigTyped(const AwsRequestSigningProtoPerRouteConfig& per_route_config, + Server::Configuration::ServerFactoryContext& context, + ProtobufMessage::ValidationVisitor&) override; }; } // namespace AwsRequestSigningFilter diff --git a/test/extensions/filters/http/aws_request_signing/aws_request_signing_filter_test.cc b/test/extensions/filters/http/aws_request_signing/aws_request_signing_filter_test.cc index 27aa33350919..e6b6fcbc5bad 100644 --- a/test/extensions/filters/http/aws_request_signing/aws_request_signing_filter_test.cc +++ b/test/extensions/filters/http/aws_request_signing/aws_request_signing_filter_test.cc @@ -198,6 +198,25 @@ TEST_F(AwsRequestSigningFilterTest, FilterConfigImplGetters) { EXPECT_EQ(true, config.useUnsignedPayload()); } +// Verify filter functionality when a host rewrite happens on route-level config. +TEST_F(AwsRequestSigningFilterTest, PerRouteConfigSignWithHostRewrite) { + setup(); + filter_config_->host_rewrite_ = "original-host"; + + Stats::IsolatedStoreImpl stats; + auto signer = std::make_unique(); + EXPECT_CALL(*(signer), signEmptyPayload(An(), An())); + + FilterConfigImpl per_route_config(std::move(signer), "prefix", *stats.rootScope(), + "overridden-host", false); + ON_CALL(*decoder_callbacks_.route_, mostSpecificPerFilterConfig(_)) + .WillByDefault(Return(&per_route_config)); + + Http::TestRequestHeaderMapImpl headers; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); + EXPECT_EQ("overridden-host", headers.getHostValue()); +} + } // namespace } // namespace AwsRequestSigningFilter } // namespace HttpFilters diff --git a/test/extensions/filters/http/aws_request_signing/config_test.cc b/test/extensions/filters/http/aws_request_signing/config_test.cc index 27c9577a4b49..95e8773f5f96 100644 --- a/test/extensions/filters/http/aws_request_signing/config_test.cc +++ b/test/extensions/filters/http/aws_request_signing/config_test.cc @@ -1,6 +1,7 @@ #include "envoy/extensions/filters/http/aws_request_signing/v3/aws_request_signing.pb.h" #include "envoy/extensions/filters/http/aws_request_signing/v3/aws_request_signing.pb.validate.h" +#include "source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.h" #include "source/extensions/filters/http/aws_request_signing/config.h" #include "test/mocks/server/factory_context.h" @@ -14,13 +15,11 @@ namespace Extensions { namespace HttpFilters { namespace AwsRequestSigningFilter { -using AwsRequestSigningProtoConfig = - envoy::extensions::filters::http::aws_request_signing::v3::AwsRequestSigning; - TEST(AwsRequestSigningFilterConfigTest, SimpleConfig) { const std::string yaml = R"EOF( service_name: s3 region: us-west-2 +host_rewrite: new-host match_excluded_headers: - prefix: x-envoy - exact: foo @@ -33,6 +32,7 @@ region: us-west-2 AwsRequestSigningProtoConfig expected_config; expected_config.set_service_name("s3"); expected_config.set_region("us-west-2"); + expected_config.set_host_rewrite("new-host"); expected_config.add_match_excluded_headers()->set_prefix("x-envoy"); expected_config.add_match_excluded_headers()->set_exact("foo"); expected_config.add_match_excluded_headers()->set_exact("bar"); @@ -51,6 +51,30 @@ region: us-west-2 cb(filter_callbacks); } +TEST(AwsRequestSigningFilterConfigTest, RouteSpecificFilterConfig) { + const std::string yaml = R"EOF( +aws_request_signing: + service_name: s3 + region: us-west-2 + host_rewrite: new-host + match_excluded_headers: + - prefix: x-envoy + - exact: foo + - exact: bar +stat_prefix: foo_prefix + )EOF"; + + AwsRequestSigningProtoPerRouteConfig proto_config; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + + testing::NiceMock context; + AwsRequestSigningFilterFactory factory; + + const auto route_config = factory.createRouteSpecificFilterConfig( + proto_config, context, ProtobufMessage::getNullValidationVisitor()); + ASSERT_NE(route_config, nullptr); +} + } // namespace AwsRequestSigningFilter } // namespace HttpFilters } // namespace Extensions From ef090cf51bb71ed1df45f231e332e736184cac1f Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 24 May 2023 02:32:09 -0400 Subject: [PATCH 345/740] Revert "mobile: removing unused test filters (#27550)" (#27588) This reverts commit c99607b614ba9dcb73204c06b190bc2d43389a1d. --- mobile/envoy_build_config/BUILD | 2 + mobile/envoy_build_config/test_extensions.cc | 4 ++ .../filters/http/test_accessor/BUILD | 44 ++++++++++++++ .../filters/http/test_accessor/config.cc | 28 +++++++++ .../filters/http/test_accessor/config.h | 36 ++++++++++++ .../filters/http/test_accessor/filter.cc | 31 ++++++++++ .../filters/http/test_accessor/filter.h | 46 +++++++++++++++ .../filters/http/test_accessor/filter.proto | 10 ++++ .../extensions/filters/http/test_read/BUILD | 42 ++++++++++++++ .../filters/http/test_read/config.cc | 24 ++++++++ .../filters/http/test_read/config.h | 33 +++++++++++ .../filters/http/test_read/filter.cc | 57 +++++++++++++++++++ .../filters/http/test_read/filter.h | 35 ++++++++++++ .../filters/http/test_read/filter.proto | 6 ++ 14 files changed, 398 insertions(+) create mode 100644 mobile/library/common/extensions/filters/http/test_accessor/BUILD create mode 100644 mobile/library/common/extensions/filters/http/test_accessor/config.cc create mode 100644 mobile/library/common/extensions/filters/http/test_accessor/config.h create mode 100644 mobile/library/common/extensions/filters/http/test_accessor/filter.cc create mode 100644 mobile/library/common/extensions/filters/http/test_accessor/filter.h create mode 100644 mobile/library/common/extensions/filters/http/test_accessor/filter.proto create mode 100644 mobile/library/common/extensions/filters/http/test_read/BUILD create mode 100644 mobile/library/common/extensions/filters/http/test_read/config.cc create mode 100644 mobile/library/common/extensions/filters/http/test_read/config.h create mode 100644 mobile/library/common/extensions/filters/http/test_read/filter.cc create mode 100644 mobile/library/common/extensions/filters/http/test_read/filter.h create mode 100644 mobile/library/common/extensions/filters/http/test_read/filter.proto diff --git a/mobile/envoy_build_config/BUILD b/mobile/envoy_build_config/BUILD index 5f520f25a89f..3b3c27bddcf1 100644 --- a/mobile/envoy_build_config/BUILD +++ b/mobile/envoy_build_config/BUILD @@ -117,9 +117,11 @@ envoy_cc_library( "@envoy//source/extensions/filters/http/buffer:config", "@envoy_mobile//library/common/extensions/filters/http/assertion:config", "@envoy_mobile//library/common/extensions/filters/http/route_cache_reset:config", + "@envoy_mobile//library/common/extensions/filters/http/test_accessor:config", "@envoy_mobile//library/common/extensions/filters/http/test_event_tracker:config", "@envoy_mobile//library/common/extensions/filters/http/test_kv_store:config", "@envoy_mobile//library/common/extensions/filters/http/test_logger:config", + "@envoy_mobile//library/common/extensions/filters/http/test_read:config", "@envoy_mobile//library/common/extensions/filters/http/test_remote_response:config", ], alwayslink = 1, diff --git a/mobile/envoy_build_config/test_extensions.cc b/mobile/envoy_build_config/test_extensions.cc index f97634722058..d0bb9c56087b 100644 --- a/mobile/envoy_build_config/test_extensions.cc +++ b/mobile/envoy_build_config/test_extensions.cc @@ -4,16 +4,20 @@ #include "external/envoy_build_config/test_extensions.h" #include "library/common/extensions/filters/http/assertion/config.h" #include "library/common/extensions/filters/http/route_cache_reset/config.h" +#include "library/common/extensions/filters/http/test_accessor/config.h" #include "library/common/extensions/filters/http/test_event_tracker/config.h" #include "library/common/extensions/filters/http/test_kv_store/config.h" #include "library/common/extensions/filters/http/test_logger/config.h" +#include "library/common/extensions/filters/http/test_read/config.h" void register_test_extensions() { Envoy::Extensions::HttpFilters::Assertion::forceRegisterAssertionFilterFactory(); Envoy::Extensions::HttpFilters::BufferFilter::forceRegisterBufferFilterFactory(); Envoy::Extensions::HttpFilters::RouteCacheReset::forceRegisterRouteCacheResetFilterFactory(); + Envoy::Extensions::HttpFilters::TestAccessor::forceRegisterTestAccessorFilterFactory(); Envoy::Extensions::HttpFilters::TestEventTracker::forceRegisterTestEventTrackerFilterFactory(); Envoy::Extensions::HttpFilters::TestKeyValueStore::forceRegisterTestKeyValueStoreFilterFactory(); Envoy::Extensions::HttpFilters::TestLogger::forceRegisterFactory(); + Envoy::HttpFilters::TestRead::forceRegisterTestReadFilterFactory(); Envoy::Upstream::forceRegisterStaticClusterFactory(); } diff --git a/mobile/library/common/extensions/filters/http/test_accessor/BUILD b/mobile/library/common/extensions/filters/http/test_accessor/BUILD new file mode 100644 index 000000000000..977df4ab6783 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_accessor/BUILD @@ -0,0 +1,44 @@ +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", + "envoy_proto_library", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_proto_library( + name = "filter", + srcs = ["filter.proto"], + deps = [ + "@envoy_api//envoy/config/common/matcher/v3:pkg", + ], +) + +envoy_cc_extension( + name = "test_accessor_filter_lib", + srcs = ["filter.cc"], + hdrs = ["filter.h"], + repository = "@envoy", + deps = [ + "filter_cc_proto", + "//library/common/api:c_types", + "//library/common/api:external_api_lib", + "//library/common/data:utility_lib", + "@envoy//source/common/common:assert_lib", + "@envoy//source/extensions/filters/http/common:pass_through_filter_lib", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + repository = "@envoy", + deps = [ + ":test_accessor_filter_lib", + "@envoy//source/extensions/filters/http/common:factory_base_lib", + ], +) diff --git a/mobile/library/common/extensions/filters/http/test_accessor/config.cc b/mobile/library/common/extensions/filters/http/test_accessor/config.cc new file mode 100644 index 000000000000..44deda4b9561 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_accessor/config.cc @@ -0,0 +1,28 @@ +#include "library/common/extensions/filters/http/test_accessor/config.h" + +#include "library/common/extensions/filters/http/test_accessor/filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TestAccessor { + +Http::FilterFactoryCb TestAccessorFilterFactory::createFilterFactoryFromProtoTyped( + const envoymobile::extensions::filters::http::test_accessor::TestAccessor& proto_config, + const std::string&, Server::Configuration::FactoryContext&) { + + auto config = std::make_shared(proto_config); + return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(std::make_shared(config)); + }; +} + +/** + * Static registration for the TestAccessor filter. @see NamedHttpFilterConfigFactory. + */ +REGISTER_FACTORY(TestAccessorFilterFactory, Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace TestAccessor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_accessor/config.h b/mobile/library/common/extensions/filters/http/test_accessor/config.h new file mode 100644 index 000000000000..44bd26007b76 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_accessor/config.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "source/extensions/filters/http/common/factory_base.h" + +#include "library/common/extensions/filters/http/test_accessor/filter.h" +#include "library/common/extensions/filters/http/test_accessor/filter.pb.h" +#include "library/common/extensions/filters/http/test_accessor/filter.pb.validate.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TestAccessor { + +/** + * Config registration for the TestAccessor filter. @see NamedHttpFilterConfigFactory. + */ +class TestAccessorFilterFactory + : public Common::FactoryBase< + envoymobile::extensions::filters::http::test_accessor::TestAccessor> { +public: + TestAccessorFilterFactory() : FactoryBase("test_accessor") {} + +private: + ::Envoy::Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoymobile::extensions::filters::http::test_accessor::TestAccessor& config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +DECLARE_FACTORY(TestAccessorFilterFactory); + +} // namespace TestAccessor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_accessor/filter.cc b/mobile/library/common/extensions/filters/http/test_accessor/filter.cc new file mode 100644 index 000000000000..e35f152dd004 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_accessor/filter.cc @@ -0,0 +1,31 @@ +#include "library/common/extensions/filters/http/test_accessor/filter.h" + +#include "envoy/server/filter_config.h" + +#include "source/common/common/assert.h" + +#include "library/common/data/utility.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TestAccessor { + +TestAccessorFilterConfig::TestAccessorFilterConfig( + const envoymobile::extensions::filters::http::test_accessor::TestAccessor& proto_config) + : accessor_(static_cast( + Api::External::retrieveApi(proto_config.accessor_name()))), + expected_string_(proto_config.expected_string()) {} + +Http::FilterHeadersStatus TestAccessorFilter::decodeHeaders(Http::RequestHeaderMap&, bool) { + RELEASE_ASSERT(config_->expectedString() == + Data::Utility::copyToString( + config_->accessor()->get_string(config_->accessor()->context)), + "accessed string is not equal to expected string"); + return Http::FilterHeadersStatus::Continue; +} + +} // namespace TestAccessor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_accessor/filter.h b/mobile/library/common/extensions/filters/http/test_accessor/filter.h new file mode 100644 index 000000000000..d5a7355c19e3 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_accessor/filter.h @@ -0,0 +1,46 @@ +#pragma once + +#include "envoy/http/filter.h" + +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "library/common/api/c_types.h" +#include "library/common/api/external.h" +#include "library/common/extensions/filters/http/test_accessor/filter.pb.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TestAccessor { + +class TestAccessorFilterConfig { +public: + TestAccessorFilterConfig( + const envoymobile::extensions::filters::http::test_accessor::TestAccessor& proto_config); + + const envoy_string_accessor* accessor() const { return accessor_; } + const std::string& expectedString() const { return expected_string_; } + +private: + const envoy_string_accessor* accessor_; + const std::string expected_string_; +}; + +using TestAccessorFilterConfigSharedPtr = std::shared_ptr; + +class TestAccessorFilter final : public ::Envoy::Http::PassThroughFilter { +public: + TestAccessorFilter(TestAccessorFilterConfigSharedPtr config) : config_(config) {} + + // StreamDecoderFilter + ::Envoy::Http::FilterHeadersStatus decodeHeaders(::Envoy::Http::RequestHeaderMap& headers, + bool end_stream) override; + +private: + const TestAccessorFilterConfigSharedPtr config_; +}; + +} // namespace TestAccessor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_accessor/filter.proto b/mobile/library/common/extensions/filters/http/test_accessor/filter.proto new file mode 100644 index 000000000000..938da1b89d95 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_accessor/filter.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package envoymobile.extensions.filters.http.test_accessor; + +import "validate/validate.proto"; + +message TestAccessor { + string accessor_name = 1 [(validate.rules).string.min_len = 1]; + string expected_string = 2 [(validate.rules).string.min_len = 1]; +} diff --git a/mobile/library/common/extensions/filters/http/test_read/BUILD b/mobile/library/common/extensions/filters/http/test_read/BUILD new file mode 100644 index 000000000000..54324cca68ac --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_read/BUILD @@ -0,0 +1,42 @@ +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", + "envoy_proto_library", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_proto_library( + name = "filter", + srcs = ["filter.proto"], + deps = [ + "@envoy_api//envoy/config/common/matcher/v3:pkg", + ], +) + +envoy_cc_extension( + name = "test_read_filter_lib", + srcs = ["filter.cc"], + hdrs = ["filter.h"], + repository = "@envoy", + deps = [ + "filter_cc_proto", + "@envoy//source/common/http:utility_lib", + "@envoy//source/common/stream_info:stream_info_lib", + "@envoy//source/extensions/filters/http/common:pass_through_filter_lib", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + repository = "@envoy", + deps = [ + ":test_read_filter_lib", + "@envoy//source/extensions/filters/http/common:factory_base_lib", + ], +) diff --git a/mobile/library/common/extensions/filters/http/test_read/config.cc b/mobile/library/common/extensions/filters/http/test_read/config.cc new file mode 100644 index 000000000000..3c5ce3c402f8 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_read/config.cc @@ -0,0 +1,24 @@ +#include "library/common/extensions/filters/http/test_read/config.h" + +#include "library/common/extensions/filters/http/test_read/filter.h" + +namespace Envoy { +namespace HttpFilters { +namespace TestRead { + +Http::FilterFactoryCb TestReadFilterFactory::createFilterFactoryFromProtoTyped( + const envoymobile::test::integration::filters::http::test_read::TestRead& /*config*/, + const std::string&, Server::Configuration::FactoryContext& /*context*/) { + return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamDecoderFilter(std::make_shared()); + }; +} + +/** + * Static registration for the TestRead filter. @see NamedHttpFilterConfigFactory. + */ +REGISTER_FACTORY(TestReadFilterFactory, Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace TestRead +} // namespace HttpFilters +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_read/config.h b/mobile/library/common/extensions/filters/http/test_read/config.h new file mode 100644 index 000000000000..24e22d706cb1 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_read/config.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include "source/extensions/filters/http/common/factory_base.h" + +#include "library/common/extensions/filters/http/test_read/filter.pb.h" +#include "library/common/extensions/filters/http/test_read/filter.pb.validate.h" + +namespace Envoy { +namespace HttpFilters { +namespace TestRead { + +/** + * Config registration for the TestRead filter. @see NamedHttpFilterConfigFactory. + */ +class TestReadFilterFactory + : public Envoy::Extensions::HttpFilters::Common::FactoryBase< + envoymobile::test::integration::filters::http::test_read::TestRead> { +public: + TestReadFilterFactory() : FactoryBase("test_read") {} + +private: + ::Envoy::Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoymobile::test::integration::filters::http::test_read::TestRead& config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +DECLARE_FACTORY(TestReadFilterFactory); + +} // namespace TestRead +} // namespace HttpFilters +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.cc b/mobile/library/common/extensions/filters/http/test_read/filter.cc new file mode 100644 index 000000000000..fb7c42752727 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_read/filter.cc @@ -0,0 +1,57 @@ +#include "library/common/extensions/filters/http/test_read/filter.h" + +#include "envoy/server/filter_config.h" + +namespace Envoy { +namespace HttpFilters { +namespace TestRead { + +Http::FilterHeadersStatus TestReadFilter::decodeHeaders(Http::RequestHeaderMap& request_headers, + bool) { + // sample path is /failed?error=0x10000 + Http::Utility::QueryParams query_parameters = + Http::Utility::parseQueryString(request_headers.Path()->value().getStringView()); + auto error = query_parameters.find("error"); + uint64_t response_flag; + if (error != query_parameters.end() && absl::SimpleAtoi(error->second, &response_flag)) { + // set response error code + StreamInfo::StreamInfo& stream_info = decoder_callbacks_->streamInfo(); + stream_info.setResponseFlag(TestReadFilter::mapErrorToResponseFlag(response_flag)); + + // check if we want a quic server error: sample path is /failed?quic=1&error=0x10000 + if (query_parameters.find("quic") != query_parameters.end()) { + stream_info.setUpstreamInfo(std::make_shared()); + stream_info.upstreamInfo()->setUpstreamProtocol(Http::Protocol::Http3); + } + + // trigger the error and stop iteration to other filters + decoder_callbacks_->sendLocalReply(Http::Code::BadRequest, "test_read filter threw: ", nullptr, + absl::nullopt, ""); + return Http::FilterHeadersStatus::StopIteration; + } + + // continue to other filters since the provided query string is invalid for error simulation + return Http::FilterHeadersStatus::Continue; +} + +StreamInfo::ResponseFlag TestReadFilter::mapErrorToResponseFlag(uint64_t errorCode) { + switch (errorCode) { + case 0x4000000: + return StreamInfo::DnsResolutionFailed; + case 0x40: + return StreamInfo::UpstreamConnectionTermination; + case 0x20: + return StreamInfo::UpstreamConnectionFailure; + case 0x10: + return StreamInfo::UpstreamRemoteReset; + case 0x10000: + return StreamInfo::StreamIdleTimeout; + default: + // Any other error that we aren't interested in. I picked a random error. + return StreamInfo::RateLimitServiceError; + } +} + +} // namespace TestRead +} // namespace HttpFilters +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.h b/mobile/library/common/extensions/filters/http/test_read/filter.h new file mode 100644 index 000000000000..dc5c2e73cea2 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_read/filter.h @@ -0,0 +1,35 @@ +#pragma once + +#include "envoy/http/filter.h" + +#include "source/common/common/logger.h" +#include "source/common/http/utility.h" +#include "source/common/stream_info/stream_info_impl.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "library/common/extensions/filters/http/test_read/filter.pb.h" + +namespace Envoy { +namespace HttpFilters { +namespace TestRead { + +/** + * This is a test-only filter to return specified error code based on a request url query string. + * It either simulates the requested error if the url query matches the error patterns + * or does nothing + */ +class TestReadFilter final : public Http::PassThroughFilter, + public Logger::Loggable { +public: + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& request_headers, bool) override; + +private: + /* A mapping of the envoymobile errors we care about for testing + * From https://github.com/envoyproxy/envoy/blob/main/envoy/stream_info/stream_info.h + */ + StreamInfo::ResponseFlag mapErrorToResponseFlag(uint64_t errorCode); +}; + +} // namespace TestRead +} // namespace HttpFilters +} // namespace Envoy diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.proto b/mobile/library/common/extensions/filters/http/test_read/filter.proto new file mode 100644 index 000000000000..de164df7d450 --- /dev/null +++ b/mobile/library/common/extensions/filters/http/test_read/filter.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +package envoymobile.test.integration.filters.http.test_read; + +message TestRead { +} From 2123fa90e709229b56a7edecc2bcf875870bbcc6 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 24 May 2023 08:27:59 +0100 Subject: [PATCH 346/740] mobile/ci: Fix workflow dependencies (#27592) Signed-off-by: Ryan Northey --- .github/workflows/mobile-ios_build.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/mobile-ios_build.yml b/.github/workflows/mobile-ios_build.yml index a1d117ff13cd..8f946ac1484f 100644 --- a/.github/workflows/mobile-ios_build.yml +++ b/.github/workflows/mobile-ios_build.yml @@ -38,7 +38,9 @@ jobs: swifthelloworld: if: ${{ needs.env.outputs.mobile_ios_build == 'true' }} name: swift_helloworld - needs: iosbuild + needs: + - env + - iosbuild runs-on: macos-12 timeout-minutes: 50 steps: @@ -115,9 +117,10 @@ jobs: name: 'Log app run' swiftexperimentalapp: if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} - needs: env + needs: + - env + - iosbuild name: swift_experimental_app - needs: iosbuild runs-on: macos-12 timeout-minutes: 50 steps: @@ -158,9 +161,10 @@ jobs: name: 'Log app run' swiftasyncawait: if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} - needs: env + needs: + - env + - iosbuild name: swift_async_await - needs: iosbuild runs-on: macos-12 timeout-minutes: 50 steps: @@ -214,9 +218,10 @@ jobs: name: 'Log app run' objchelloworld: if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} - needs: env + needs: + - env + - iosbuild name: objc_helloworld - needs: iosbuild runs-on: macos-12 timeout-minutes: 50 steps: From dd290cd048a9f6d7851f3a43b4df8f9202a6e6f0 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 24 May 2023 08:57:29 +0100 Subject: [PATCH 347/740] ci: Improve release publishing (#27526) Signed-off-by: Ryan Northey --- .azure-pipelines/env.yml | 15 ++++-- .azure-pipelines/stage/publish.yml | 75 ++++++++++++++++++------------ ci/do_ci.sh | 46 +++++++++--------- distribution/BUILD | 48 +++++++++++++++++++ tools/base/requirements.in | 2 +- 5 files changed, 128 insertions(+), 58 deletions(-) diff --git a/.azure-pipelines/env.yml b/.azure-pipelines/env.yml index b90813001600..01a9a78d081f 100644 --- a/.azure-pipelines/env.yml +++ b/.azure-pipelines/env.yml @@ -122,6 +122,8 @@ jobs: - bash: | set -e VERSION_DEV="$(cat VERSION.txt | cut -d- -f2)" + VERSION_PATCH="$(cat VERSION.txt | cut -d- -f1 | rev | cut -d. -f1 | rev)" + echo "##vso[task.setvariable variable=versionPatch;isoutput=true]$VERSION_PATCH" if [[ $VERSION_DEV == "dev" ]]; then echo "##vso[task.setvariable variable=isDev;isoutput=true]true" else @@ -180,7 +182,7 @@ jobs: # TODO(phlax): move this to a script to ensure proper linting etc set -e - PUBLISH_GITHUB_RELEASE=false + PUBLISH_GITHUB_RELEASE=$(run.packaging) PUBLISH_DOCKERHUB=false PUBLISH_DOCS=false PUBLISH_DOCS_LATEST=false @@ -189,6 +191,7 @@ jobs: if [[ "$ISSTABLEBRANCH" == True && -n "$POSTSUBMIT" && "$NOSYNC" != true ]]; then # Build docs for publishing either latest or a release build PUBLISH_DOCS=true + # main if [[ "$ISMAIN" == True ]]; then # Update the Dockerhub README PUBLISH_DOCKERHUB=true @@ -196,9 +199,13 @@ jobs: # Postsubmit on `main` trigger rebuild of latest docs PUBLISH_DOCS_LATEST=true fi + # Not main, and not -dev elif [[ "$(state.isDev)" == false ]]; then - # A stable release, publish docs to the release and create a Github release - PUBLISH_GITHUB_RELEASE=true + if [[ "$(state.versionPatch)" -eq 0 ]]; then + # A just-forked branch + PUBLISH_GITHUB_RELEASE=false + fi + # A stable release, publish docs to the release PUBLISH_DOCS_RELEASE=true else # Postsubmit for non-main/release, skip publishing docs in this case @@ -226,9 +233,11 @@ jobs: echo "env.outputs['changed.requirements']: $(changed.requirements)" echo echo "env.outputs['state.isDev']: $(state.isDev)" + echo "env.outputs['state.versionPatch']: $(state.versionPatch)" echo echo "env.outputs['run.build']: $(run.build)" echo "env.outputs['run.checks']: $(run.checks)" + echo "env.outputs['run.packaging']: $(run.packaging)" echo echo "env.outputs['publish.githubRelease']: $(publish.githubRelease)" echo "env.outputs['publish.dockerhub]: $(publish.dockerhub)" diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index bd74cb6424c7..248b7400b7ea 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -301,8 +301,43 @@ jobs: displayName: "Publish release docs" workingDirectory: $(Build.SourcesDirectory) +- job: signed_release + displayName: Signed binaries + dependsOn: + - package_x64 + - package_arm64 + condition: | + and(succeeded(), + eq(${{ parameters.runPackaging }}, 'true')) + timeoutInMinutes: 120 + pool: + vmImage: "ubuntu-20.04" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release" + itemPattern: "bazel.release/**/bin/*" + targetPath: $(Build.StagingDirectory) + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.distribution" + itemPattern: "bazel.distribution/**/packages.*.tar.gz" + targetPath: $(Build.StagingDirectory) + - template: ../bazel.yml + parameters: + ciTarget: release.signed + stepsPre: + - template: ../gpg.yml + parameters: + authGPGPassphrase: ${{ parameters.authGPGPassphrase }} + authGPGPath: ${{ parameters.authGPGPath }} + authGPGKey: ${{ parameters.authGPGKey }} + pathGPGConfiguredHome: /build/.gnupg + pathGPGHome: $(Build.StagingDirectory)/.gnupg - job: success - dependsOn: ["docker", "package_x64", "package_arm64", "docs"] + dependsOn: ["docker", "docs", "signed_release"] displayName: Success (linux artefacts) pool: vmImage: "ubuntu-20.04" @@ -313,9 +348,8 @@ jobs: condition: | and( in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), - in(dependencies.package_x64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), - in(dependencies.package_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), - in(dependencies.docs.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) + in(dependencies.docs.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), + in(dependencies.signed_release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) steps: - checkout: none - bash: | @@ -323,39 +357,22 @@ jobs: - job: github displayName: Publish release tag - dependsOn: ["docs"] + dependsOn: ["success"] condition: | and(not(canceled()), eq(${{ parameters.publishGithubRelease }}, 'true')) pool: vmImage: "ubuntu-20.04" steps: - - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "bazel.release" - itemPattern: "bazel.release/bin/*" - targetPath: $(Build.StagingDirectory) - task: DownloadBuildArtifacts@0 inputs: buildType: current - artifactName: "bazel.release.arm64" - itemPattern: "bazel.release.arm64/bin/*" + artifactName: "release.signed" + itemPattern: "release.signed/release.signed.tar.zst" targetPath: $(Build.StagingDirectory) - - template: ../gpg.yml + - template: ../bazel.yml parameters: - authGPGPassphrase: ${{ parameters.authGPGPassphrase }} - authGPGPath: ${{ parameters.authGPGPath }} - authGPGKey: ${{ parameters.authGPGKey }} - pathGPGConfiguredHome: /build/.gnupg - pathGPGHome: $(Build.StagingDirectory)/.gnupg - - script: ./ci/run_envoy_docker.sh './ci/do_ci.sh publish' - displayName: "Publish release to Github" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - GITHUB_TOKEN: ${{ parameters.authGithub }} - - bash: | - set -e - rm -rf $(Build.StagingDirectory)/.gnupg + ciTarget: publish + cacheVersion: $(cacheKeyBazel) + publishEnvoy: false + publishTestResults: false diff --git a/ci/do_ci.sh b/ci/do_ci.sh index f35425f4de53..4d516321753c 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -585,32 +585,18 @@ case $CI_TARGET in ;; publish) - # If we are on a non-main release branch but the patch version is 0 - then this branch - # has just been created, and the tag/release was cut from `main` - there is no need to - # create the tag/release from here - version="$(cat VERSION.txt)" - patch_version="$(echo "$version" | rev | cut -d. -f1)" - if [[ "$AZP_BRANCH" == "main" || "$AZP_BRANCH" == "refs/heads/main" ]]; then - if [[ "$patch_version" -eq 0 ]]; then - # It can take some time to get here in CI so the branch may have changed - create the release - # from the current commit (as this only happens on non-PRs we are safe from merges) - BUILD_SHA="$(git rev-parse HEAD)" - bazel run "${BAZEL_BUILD_OPTIONS[@]}" @envoy_repo//:publish -- --publish-commitish="$BUILD_SHA" - # TODO(phlax): move this to pytooling - mkdir -p linux/amd64 linux/arm64 publish - # linux/amd64 - tar xf /build/bazel.release/release.tar.zst -C ./linux/amd64 - cp -a linux/amd64/envoy "publish/envoy-${version}-linux-x86_64" - cp -a linux/amd64/envoy-contrib "publish/envoy-contrib-${version}-linux-x86_64" - # linux/arm64 - tar xf /build/bazel.release.arm64/release.tar.zst -C ./linux/arm64 - cp -a linux/arm64/envoy "publish/envoy-${version}-linux-aarch_64" - cp -a linux/arm64/envoy-contrib "publish/envoy-contrib-${version}-linux-aarch_64" - "${ENVOY_SRCDIR}/ci/publish_github_assets.sh" "v${version}" "${PWD}/publish" - exit 0 - fi + setup_clang_toolchain + BUILD_SHA="$(git rev-parse HEAD)" + VERSION_DEV="$(cut -d- -f2 < VERSION.txt)" + PUBLISH_ARGS=( + --publish-commitish="$BUILD_SHA" + --publish-assets=/build/release.signed/release.signed.tar.zst) + if [[ "$VERSION_DEV" == "dev" ]]; then + PUBLISH_ARGS+=(--dry-run) fi - echo "Not creating a tag/release for ${version} from ${AZP_BRANCH}" + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + @envoy_repo//:publish \ + -- "${PUBLISH_ARGS[@]}" ;; release) @@ -666,6 +652,16 @@ case $CI_TARGET in bazel_envoy_binary_build release ;; + release.signed) + echo "Signing binary packages..." + setup_clang_toolchain + # The default config expects these files + mkdir -p distribution/custom + cp -a /build/bazel.*/*64 distribution/custom/ + bazel build "${BAZEL_BUILD_OPTIONS[@]}" //distribution:signed + cp -a bazel-bin/distribution/release.signed.tar.zst "${BUILD_DIR}/envoy/" + ;; + sizeopt) setup_clang_toolchain echo "Testing ${TEST_TARGETS[*]}" diff --git a/distribution/BUILD b/distribution/BUILD index ad70af90bfe8..611db69eaef7 100644 --- a/distribution/BUILD +++ b/distribution/BUILD @@ -10,6 +10,8 @@ MAINTAINER = "Envoy maintainers " exports_files(glob([ "custom/envoy*", + "custom/arm64/**/*", + "custom/x64/**/*", ])) # Configurable binary flags @@ -73,3 +75,49 @@ sh_binary( "//tools/distribution:verify", ], ) + +label_flag( + name = "x64-packages", + build_setting_default = "//distribution:custom/x64/packages.x64.tar.gz", +) + +label_flag( + name = "arm64-packages", + build_setting_default = "//distribution:custom/arm64/packages.arm64.tar.gz", +) + +label_flag( + name = "x64-release", + build_setting_default = "//distribution:custom/x64/bin/release.tar.zst", +) + +label_flag( + name = "arm64-release", + build_setting_default = "//distribution:custom/arm64/bin/release.tar.zst", +) + +genrule( + name = "signed", + outs = ["release.signed.tar.zst"], + cmd = """ + # Sign the packages + $(location //tools/distribution:sign) \ + "deb.x64:$(location :x64-packages)" \ + "deb.arm64:$(location :arm64-packages)" \ + "x64:$(location :x64-release)" \ + "arm64:$(location :arm64-release)" \ + -m x64/envoy:bin/envoy-x86_64 \ + -m x64/envoy-contrib:bin/envoy-contrib-x86_64 \ + -m arm64/envoy:bin/envoy-aarch_64 \ + -m arm64/envoy-contrib:bin/envoy-contrib-aarch_64 \ + --out $@ + """, + exec_tools = [ + ":arm64-packages", + ":x64-packages", + ":arm64-release", + ":x64-release", + "//tools/distribution:sign", + ], + tags = ["no-remote"], +) diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 68c563b40516..653124e4adcf 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -5,7 +5,7 @@ cffi>=1.15.0 colorama coloredlogs dependatool>=0.2.2 -envoy.base.utils>=0.4.7 +envoy.base.utils>=0.4.10 envoy.code.check>=0.5.0 envoy.dependency.check>=0.1.7 envoy.distribution.release>=0.0.9 From aa5ab8d093996d9c141c9f5f68abecbc79e7183a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 May 2023 09:06:59 +0100 Subject: [PATCH 348/740] build(deps): bump envoy-base-utils from 0.4.10 to 0.4.11 in /tools/base (#27596) Bumps [envoy-base-utils](https://github.com/envoyproxy/toolshed) from 0.4.10 to 0.4.11. - [Release notes](https://github.com/envoyproxy/toolshed/releases) - [Commits](https://github.com/envoyproxy/toolshed/commits) --- updated-dependencies: - dependency-name: envoy-base-utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 8ad6e2ebff2d..05590e525ebc 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -25,9 +25,9 @@ aio-api-bazel==0.0.2 \ --hash=sha256:56e36463d236e477b7e282f2d870185a0b978b50e2c3803c1ebf8b8ac4b18f5b \ --hash=sha256:d3f563b7698e874437d80538a89dd4d79bc37de2e850c846330ae456e3f21dcc # via -r requirements.in -aio-api-github==0.2.2 \ - --hash=sha256:507971a3a90155066a072d24fc40b3890f69b2f671a42c6325370a48cbbd0e06 \ - --hash=sha256:5e67bb397362ad0ccfd6b6680c94a995fdc8fb8cd795b5074dd29c5f875a9d62 +aio-api-github==0.2.4 \ + --hash=sha256:ccbc7c6c61b25994e87474d78c48549e9fbc98c2cc04314b50b80ba1f40fd521 \ + --hash=sha256:eccfccd1503f50384de3f6526bd780ca02107cb440a666b2c1ab978d99c7db5e # via # envoy-base-utils # envoy-dependency-check @@ -420,9 +420,9 @@ docutils==0.19 \ # via # envoy-docs-sphinx-runner # sphinx -envoy-base-utils==0.4.10 \ - --hash=sha256:1cfccb54a3831c5b5a833efdad67752c1ae0a8708532712842726de42c0c413a \ - --hash=sha256:9567d1317a9ed7e97730a664e3105d8a6d46a4fa9e5cc7256951f5564783d8ae +envoy-base-utils==0.4.11 \ + --hash=sha256:5ced696c470b4c3090e6fc3f74e7e33f5fe217e775b1fc1fb56dfc756b781fbe \ + --hash=sha256:97c79177bb89360b7772e58fb20c671c67bf0b6cdee74c0b9f8a80433f0370cc # via # -r requirements.in # envoy-code-check From 52c6d78723ab53f3338bbf23554801d4019e3dc1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 May 2023 09:07:36 +0100 Subject: [PATCH 349/740] build(deps): bump orjson from 3.8.12 to 3.8.13 in /tools/base (#27595) Bumps [orjson](https://github.com/ijl/orjson) from 3.8.12 to 3.8.13. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.8.12...3.8.13) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 94 ++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 05590e525ebc..33733690523d 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -926,53 +926,53 @@ oauth2client==4.1.3 \ # via # gcs-oauth2-boto-plugin # google-apitools -orjson==3.8.12 \ - --hash=sha256:062e67108c218fdb9475edd5272b1629c05b56c66416fa915de5656adde30e73 \ - --hash=sha256:06e528f9a84fbb4000fd0eee573b5db543ee70ae586fdbc53e740b0ac981701c \ - --hash=sha256:0ba645c92801417933fa74448622ba614a275ea82df05e888095c7742d913bb4 \ - --hash=sha256:135f29cf936283a0cd1b8bce86540ca181108f2a4d4483eedad6b8026865d2a9 \ - --hash=sha256:29706dd8189835bcf1781faed286e99ae54fd6165437d364dfdbf0276bf39b19 \ - --hash=sha256:2ad149ed76dce2bbdfbadd61c35959305e77141badf364a158beb4ef3d88ec37 \ - --hash=sha256:355055e0977c43b0e5325b9312b7208c696fe20cd54eed1d6fc80b0a4d6721f5 \ - --hash=sha256:397670665f94cf5cff779054781d80395084ba97191d82f7b3a86f0a20e6102b \ - --hash=sha256:3fa58ca064c640fa9d823f98fbbc8e71940ecb78cea3ac2507da1cbf49d60b51 \ - --hash=sha256:44f7bb4c995652106276442de1147c9993716d1e2d79b7fd435afa154ff236b9 \ - --hash=sha256:4fd240e736ce52cd757d74142d9933fd35a3184396be887c435f0574e0388654 \ - --hash=sha256:62f999798f2fa55e567d483864ebfc30120fb055c2696a255979439323a5b15c \ - --hash=sha256:6cae2ff288a80e81ce30313e735c5436495ab58cf8d4fbe84900e616d0ee7a78 \ - --hash=sha256:6d1acf52d3a4b9384af09a5c2658c3a7a472a4d62a0ad1fe2c8fab8ef460c9b4 \ - --hash=sha256:6f1b01f641f5e87168b819ac1cbd81aa6278e7572c326f3d27e92dea442a2c0d \ - --hash=sha256:6f568205519bb0197ca91915c5da6058cfbb59993e557b02dfc3b2718b34770a \ - --hash=sha256:710c40c214b753392e46f9275fd795e9630dd737a5ab4ac6e4ee1a02fe83cc0d \ - --hash=sha256:77710774faed337ac4ad919dadc5f3b655b0cd40518e5386e6f1f116de9c6c25 \ - --hash=sha256:7d50d9b1ae409ea15534365fec0ce8a5a5f7dc94aa790aacfb8cfec87ab51aa4 \ - --hash=sha256:7d63f524048825e05950db3b6998c756d5377a5e8c469b2e3bdb9f3217523d74 \ - --hash=sha256:7e405d54c84c30d9b1c918c290bcf4ef484a45c69d5583a95db81ffffba40b44 \ - --hash=sha256:7e549468867991f6f9cfbd9c5bbc977330173bd8f6ceb79973bbd4634e13e1b9 \ - --hash=sha256:82d65e478a21f98107b4eb8390104746bb3024c27084b57edab7d427385f1f70 \ - --hash=sha256:834b50df79f1fe89bbaced3a1c1d8c8c92cc99e84cdcd374d8da4974b3560d2a \ - --hash=sha256:83e8c740a718fa6d511a82e463adc7ab17631c6eea81a716b723e127a9c51d57 \ - --hash=sha256:8682f752c19f6a7d9fc727fd98588b4c8b0dce791b5794bb814c7379ccd64a79 \ - --hash=sha256:8d153b228b6e24f8bccf732a51e01e8e938eef59efed9030c5c257778fbe0804 \ - --hash=sha256:8f00038bf5d07439d13c0c2c5cd6ad48eb86df7dbd7a484013ce6a113c421b14 \ - --hash=sha256:96fb1eb82b578eb6c0e53e3cf950839fe98ea210626f87c8204bd4fc2cc6ba02 \ - --hash=sha256:9a6c1594d5a9ff56e5babc4a87ac372af38d37adef9e06744e9f158431e33f43 \ - --hash=sha256:9f0f042cf002a474a6aea006dd9f8d7a5497e35e5fb190ec78eb4d232ec19955 \ - --hash=sha256:a72b50719bdd6bb0acfca3d4d1c841aa4b191f3ff37268e7aba04e5d6be44ccd \ - --hash=sha256:aff761de5ed5543a0a51e9f703668624749aa2239de5d7d37d9c9693daeaf5dc \ - --hash=sha256:becbd5af6d035a7ec2ee3239d4700929d52d8517806b97dd04efcc37289403f7 \ - --hash=sha256:c6390ce0bce24c107fc275736aa8a4f768ef7eb5df935d7dca0cc99815eb5d99 \ - --hash=sha256:c84046e890e13a119404a83f2e09e622509ed4692846ff94c4ca03654fbc7fb5 \ - --hash=sha256:cd6fbd1413559572e81b5ac64c45388147c3ba85cc3df2eaa11002945e0dbd1f \ - --hash=sha256:d937503e4dfba5edc8d5e0426d3cc97ed55716e93212b2e12a198664487b9965 \ - --hash=sha256:dc27a8ec13f28e92dc1ea89bf1232d77e7d3ebfd5c1ccf4f3729a70561cb63bd \ - --hash=sha256:de3d096dde3e46d01841abc1982b906694ab3c92f338d37a2e6184739dc8a958 \ - --hash=sha256:eb16e0195febd24b44f4db1ab3be85ecf6038f92fd511370cebc004b3d422294 \ - --hash=sha256:ebb03e4c7648f7bb299872002a6120082da018f41ba7a9ebf4ceae8d765443d2 \ - --hash=sha256:ec4f0130d9a27cb400423e09e0f9e46480e9e977f05fdcf663a7a2c68735513e \ - --hash=sha256:efb3a10030462a22c731682434df5c137a67632a8339f821cd501920b169007e \ - --hash=sha256:f480ae7b84369b1860d8867f0baf8d885fede400fda390ce088bfa8edf97ffdc \ - --hash=sha256:f4e22b0aa70c963ac01fcd620de15be21a5027711b0e5d4b96debcdeea43e3ae +orjson==3.8.13 \ + --hash=sha256:0055168bc38c9caf7211e66e7c06d7f127d2c1dd1cd1d806c58f3a81d6074a6c \ + --hash=sha256:05bfef2719d68b44ab38061f9cd2b3a58d9994f7230734ba6d3c16db97c5e94a \ + --hash=sha256:0a3bc7e12f69f7bcefe522c4e4dac33a9b3b450aae0b3170ab61fbce0a6e1b37 \ + --hash=sha256:0ca2aced3fa6ce6d440a2a2e55bb7618fd24fce146068523472f349598e992ee \ + --hash=sha256:0d2e9a8ea45db847864868f7a566bece7d425c06627e5dbdd5fc8399a9c3330b \ + --hash=sha256:11a457fafdd207f361986750a5229fc36911fc9fdfc274d078fdf1654845ef45 \ + --hash=sha256:1234110f782af5e81893b5419b9374ca2559dbd976cbd515e6c3afc292cdfb6a \ + --hash=sha256:1316c60c0f55440e765b0211e94d171ab2c11d00fe8dcf0ac70c9bd1d9818e6b \ + --hash=sha256:14e54713703d5436a7be54ff50d780b6b09358f1a0be6107a3ea4f3537a4f6d8 \ + --hash=sha256:156bd6325a4f4a0c88556b7d774e3e18713c8134b6f807571a3eec14dfcafff6 \ + --hash=sha256:17788155c50f47d9fd037e12ac82a57381c157ea4de50e8946df8519da0f7f02 \ + --hash=sha256:246e22d167ede9ebf09685587187bde9e2440a515bd5eab2e97f029b9de57677 \ + --hash=sha256:24f923cf8d7e2e9a975f4507f93e93c262f26b4a1a4f72e4d6e75eda45de8f40 \ + --hash=sha256:2e362090bdd4261608eceefd8ed127cd2bfc48643601f9c0cf5d162ca6a7c4cd \ + --hash=sha256:305ffd227857cede7318c056020d1a3f3295e8adf8e7f2cbd78c26c530a0f234 \ + --hash=sha256:386e60a09585b2b5db84879ebad6d49427ae5a9677f86a90bff9cbbec42b03be \ + --hash=sha256:41585f90cfe24d0ae7d5bc96968617b8bcebb618e19db5b0bbadce6bc82f3455 \ + --hash=sha256:47cb98386a7ff79d0ace6a7c9d5c49ca2b4ea42e4339c565f5efe7757790dd04 \ + --hash=sha256:4b98fbca0ea0f5e56b3c1d050b78460ca9708419780ec218cef1eca424db2ee5 \ + --hash=sha256:50cfa3449157c4a4ad017a041dbb5fe37091800220fd5e651c0e5fff63bdac61 \ + --hash=sha256:5492a1d9eea5a1cb33ae6d225091c69dc79f16d952885625c00070388489d412 \ + --hash=sha256:59d79e5de4a1de246517b4c92dcf6a7ef1fb12e3ce4bbfc6c0f99d1d905405fd \ + --hash=sha256:59d81f5b9e280ac3ced615e726bfba722785cc5f7fc3aa1e0ea304c5a4114e94 \ + --hash=sha256:5e56ca7bd82b25f40955184df21c977369debe51c4b83fc3113b6427726312f3 \ + --hash=sha256:637d55ba6b48b698973d7e647b9de6bb2b424c445f51c86df4e976e672300b21 \ + --hash=sha256:6b323bb4af76c16636ac1fec403331208f978ae8a2c6bcab904ee1683c05ad7a \ + --hash=sha256:6dcccda35f11f12ebb36db0ebdca9854327530e1fffe02331cde78177851ae7f \ + --hash=sha256:6fe2981bd0f6959d821253604e9ba2c5ffa03c6202d11f0e3c190e5712b6835b \ + --hash=sha256:8075487b7b2e7cc2c44d8ee7950845b6854cd08a04df80b36055cc0236c28edd \ + --hash=sha256:8d2ce41c5992dbe9962ef75db1e70ed33636959f2f4b929f9d8cbb2e30472a08 \ + --hash=sha256:97d8444cf48f8fe2718fd3b99484906c29a909dc3a8177e8751170a9a28bcf33 \ + --hash=sha256:9b5b005841394e563f1ca3314a6884101a1b1f1dd30c569b4a0335e1ebf49fbf \ + --hash=sha256:a4a182e7a58114a81d52d67bdc034eb83571690158c4b8d3f1bf5c5f772f77b1 \ + --hash=sha256:b05ef096362c8a96fdcd85392c68156c9b680271aea350b490c2d0f3ef1b6b6a \ + --hash=sha256:b2325d8471867c99c432c96861d72d8b7336293860ebb17c9d70e1d377cc2b32 \ + --hash=sha256:bf934a036dafe63c3b1d630efaf996b85554e7ab03754019a18cc0fe2bdcc3a9 \ + --hash=sha256:c97be6a6ff4d546579f08f1d67aad92715313a06b214e3f2df9bb9f1b45765c2 \ + --hash=sha256:ce737ddf9d5f960996b63c12dbcc82ae2315c45f19165b2fe14a5b33ab8705bb \ + --hash=sha256:cf79f51a7ca59ac322a1e65430142ab1cb9c9a845e893e0e3958deaefe1c9873 \ + --hash=sha256:d30b8b9fe1ff56fb6ff64d2c2e227d49819b58ae8dac51089f393e31b39a4080 \ + --hash=sha256:e62f14f3eabccdd2108e3d5884fb66197255accc42b9ffa7f04d9dbf7336b479 \ + --hash=sha256:e792d286ad175d36f6b77b7ba77f1654a13f705a7ccfef7819e9b6d49277120d \ + --hash=sha256:ea6899624661d2258a71bde33266c3c08c8d9596865acf0ac19a9552c08fa1a6 \ + --hash=sha256:f084ce58b3fd496429deb3435aa7295ab57e349a33cdb99b3cb5f0a66a610a84 \ + --hash=sha256:f8aa77df01c60b7d8b0ff5501d6b8583a4acb06c4373c59bf769025ff8b8b4cb \ + --hash=sha256:fc42b2006abaa4fb72c9193931a9236dd85ce0483cc74079c315ce8529568ca1 # via # -r requirements.in # envoy-base-utils From 2bfacbdc2782375cd259444439c85a80c2bc458a Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 24 May 2023 11:52:57 +0100 Subject: [PATCH 350/740] ci: Fix latest docs publishing (#27608) Signed-off-by: Ryan Northey --- .azure-pipelines/bazel.yml | 5 +++++ .azure-pipelines/stage/publish.yml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index 79a43fa80107..7a3c701ac8ac 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -58,6 +58,9 @@ parameters: type: stepList default: [] +- name: env + type: object + default: {} steps: - checkout: self @@ -179,6 +182,8 @@ steps: BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)" ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: BAZEL_REMOTE_INSTANCE_BRANCH: "$(Build.SourceBranchName)" + ${{ each var in parameters.env }}: + ${{ var.key }}: ${{ var.value }} displayName: "Run CI script ${{ parameters.ciTarget }}" - bash: | diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index 248b7400b7ea..d7c79e7cabf4 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -241,6 +241,8 @@ jobs: cacheVersion: $(cacheKeyBazel) publishEnvoy: false publishTestResults: false + env: + AZP_BRANCH: $(Build.SourceBranch) stepsPost: - script: | From 6e6f515295152a260afb5902c291aa965ee428ef Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 24 May 2023 13:53:07 +0100 Subject: [PATCH 351/740] Revert "test: moving ssl utility to use parsed protos (#27585)" (#27606) This reverts commit 9f1aa4330060331bd074be2ee20516252f909d1a. Signed-off-by: Ryan Northey --- test/integration/BUILD | 6 ++-- test/integration/ssl_utility.cc | 53 ++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/test/integration/BUILD b/test/integration/BUILD index cb6be550d383..fbcbfbfaf828 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1101,14 +1101,12 @@ envoy_cc_test_library( name = "utility_lib", srcs = [ "server.cc", - "ssl_utility.cc", "utility.cc", - ], + ] + envoy_select_enable_yaml(["ssl_utility.cc"]), hdrs = [ "server.h", - "ssl_utility.h", "utility.h", - ], + ] + envoy_select_enable_yaml(["ssl_utility.h"]), data = ["//test/common/runtime:filesystem_test_data"], deps = [ ":server_stats_interface", diff --git a/test/integration/ssl_utility.cc b/test/integration/ssl_utility.cc index be1c9d8a2009..af93ec2c3a48 100644 --- a/test/integration/ssl_utility.cc +++ b/test/integration/ssl_utility.cc @@ -25,30 +25,47 @@ namespace Ssl { void initializeUpstreamTlsContextConfig( const ClientSslTransportOptions& options, envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext& tls_context) { - std::string rundir = TestEnvironment::runfilesDirectory(); - tls_context.mutable_common_tls_context() - ->mutable_validation_context() - ->mutable_trusted_ca() - ->set_filename(rundir + "/test/config/integration/certs/cacert.pem"); - auto* certs = tls_context.mutable_common_tls_context()->add_tls_certificates(); - std::string chain; - std::string key; + std::string yaml_plain = R"EOF( + common_tls_context: + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/config/integration/certs/cacert.pem" +)EOF"; if (options.client_ecdsa_cert_) { - chain = rundir + "/test/config/integration/certs/client_ecdsacert.pem"; - key = rundir + "/test/config/integration/certs/client_ecdsakey.pem"; + yaml_plain += R"EOF( + tls_certificates: + certificate_chain: + filename: "{{ test_rundir }}/test/config/integration/certs/client_ecdsacert.pem" + private_key: + filename: "{{ test_rundir }}/test/config/integration/certs/client_ecdsakey.pem" +)EOF"; } else if (options.use_expired_spiffe_cert_) { - chain = rundir + "/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_cert.pem"; - key = rundir + "/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_key.pem"; + yaml_plain += R"EOF( + tls_certificates: + certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_cert.pem" + private_key: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_key.pem" +)EOF"; } else if (options.client_with_intermediate_cert_) { - chain = rundir + "/test/config/integration/certs/client2_chain.pem"; - key = rundir + "/test/config/integration/certs/client2key.pem"; + yaml_plain += R"EOF( + tls_certificates: + certificate_chain: + filename: "{{ test_rundir }}/test/config/integration/certs/client2_chain.pem" + private_key: + filename: "{{ test_rundir }}/test/config/integration/certs/client2key.pem" +)EOF"; } else { - chain = rundir + "/test/config/integration/certs/clientcert.pem"; - key = rundir + "/test/config/integration/certs/clientkey.pem"; + yaml_plain += R"EOF( + tls_certificates: + certificate_chain: + filename: "{{ test_rundir }}/test/config/integration/certs/clientcert.pem" + private_key: + filename: "{{ test_rundir }}/test/config/integration/certs/clientkey.pem" +)EOF"; } - certs->mutable_certificate_chain()->set_filename(chain); - certs->mutable_private_key()->set_filename(key); + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), tls_context); auto* common_context = tls_context.mutable_common_tls_context(); if (options.alpn_) { From 932e09c1d826f62a34eead67111a035d95ff243e Mon Sep 17 00:00:00 2001 From: botengyao Date: Wed, 24 May 2023 09:13:38 -0400 Subject: [PATCH 352/740] formatter: add two tests NaN and Infinity (#27589) #26464 removed JsonOrDie function but added ENVOY_BUG in the format call path. The pre-release version of Protobuf includes this change: Expect fail when serialize inf and nan for Value.number_value in json format. fixes #11259 protocolbuffers/protobuf@ca1cb1b If we bump the Protobuf version, there will be an exception & crash using NaN/inf/-inf in protobuf.struct number_value when performing the serialization, while there is no check for this behavior right now. Added 2 tests to catch this behavior change in the future. Signed-off-by: Boteng Yao --- .../formatter/substitution_formatter_test.cc | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 897a35879de5..c5211b117cc7 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -2248,6 +2248,36 @@ TEST(SubstitutionFormatterTest, DynamicMetadataFormatter) { stream_info, body, AccessLog::AccessLogType::NotSet), ProtoEq(ValueUtil::stringValue("test_value"))); } + + // Add the NaN test for number value. This behavior will be changed in the future Protobuf + // dependency upgrade, and the Json serialization will fail if the number value is NaN or + // Infinity. We need to change the `getJsonStringFromMessage` description, re-evaluate the use + // of ENVOY_BUG in the MetaDataFormatter::format method, and modify the following two tests. + { + ProtobufWkt::Value val; + val.set_number_value(std::numeric_limits::quiet_NaN()); + ProtobufWkt::Struct struct_obj; + (*struct_obj.mutable_fields())["nan_val"] = val; + (*metadata.mutable_filter_metadata())["com.test"] = struct_obj; + + DynamicMetadataFormatter formatter("com.test", {"nan_val"}, absl::optional()); + EXPECT_EQ("\"NaN\"", formatter.format(request_headers, response_headers, response_trailers, + stream_info, body, AccessLog::AccessLogType::NotSet)); + } + + // Add the Infinity test for number value. + { + ProtobufWkt::Value val; + val.set_number_value(std::numeric_limits::infinity()); + ProtobufWkt::Struct struct_obj; + (*struct_obj.mutable_fields())["inf_val"] = val; + (*metadata.mutable_filter_metadata())["com.test"] = struct_obj; + + DynamicMetadataFormatter formatter("com.test", {"inf_val"}, absl::optional()); + EXPECT_EQ("\"Infinity\"", + formatter.format(request_headers, response_headers, response_trailers, stream_info, + body, AccessLog::AccessLogType::NotSet)); + } } TEST(SubstitutionFormatterTest, FilterStateFormatter) { From 0297335b2be793b203990f102d9c12c23198cc8a Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 24 May 2023 15:13:38 +0100 Subject: [PATCH 353/740] ci/fuzz: Update to use `--extensions_fuzzed_count` flag for check (#27610) Signed-off-by: Ryan Northey --- source/docs/network_filter_fuzzing.md | 2 +- .../filters/network/common/fuzz/BUILD | 20 ++++--------------- .../filters/network/common/fuzz/config.bzl | 15 ++++++++++++++ tools/base/requirements.in | 2 +- tools/base/requirements.txt | 6 +++--- tools/code/BUILD | 5 +++++ 6 files changed, 29 insertions(+), 21 deletions(-) create mode 100644 test/extensions/filters/network/common/fuzz/config.bzl diff --git a/source/docs/network_filter_fuzzing.md b/source/docs/network_filter_fuzzing.md index 590b4a94cd87..aa8ffd28e333 100644 --- a/source/docs/network_filter_fuzzing.md +++ b/source/docs/network_filter_fuzzing.md @@ -7,7 +7,7 @@ Before adding the new filter into the fuzzers, please make sure the filter is de # Add a new ReadFilter into Generic Readfilter Fuzzer Only one step is needed to add a new filter to the fuzzer: -* In the file [BUILD](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/BUILD) the name of the filter has to be added to the `READFILTER_FUZZ_FILTERS` list. The fuzz test will figure the available filters from the factories. +* In the file [config.bzl](https://github.com/envoyproxy/envoy/blob/main/test/extensions/filters/network/common/fuzz/config.bzl) the name of the filter has to be added to the `READFILTER_FUZZ_FILTERS` list. The fuzz test will figure the available filters from the factories. ``` READFILTER_FUZZ_FILTERS = [ "envoy.filters.network.client_ssl_auth", diff --git a/test/extensions/filters/network/common/fuzz/BUILD b/test/extensions/filters/network/common/fuzz/BUILD index 6778bee09fd0..edddb378e109 100644 --- a/test/extensions/filters/network/common/fuzz/BUILD +++ b/test/extensions/filters/network/common/fuzz/BUILD @@ -9,6 +9,10 @@ load( "//source/extensions:all_extensions.bzl", "envoy_filters_from_selected", ) +load( + ":config.bzl", + "READFILTER_FUZZ_FILTERS", +) licenses(["notice"]) # Apache 2 @@ -36,22 +40,6 @@ envoy_proto_library( ], ) -# The filters to be fuzzed by the network_readfilter_fuzz_test. Prefer to add only stable filters, -# that do not have a dedicated fuzzer to them. -READFILTER_FUZZ_FILTERS = [ - "envoy.filters.network.client_ssl_auth", - "envoy.filters.network.ext_authz", - "envoy.filters.network.envoy_mobile_http_connection_manager", - # A dedicated http_connection_manager fuzzer can be found in - # test/common/http/conn_manager_impl_fuzz_test.cc - "envoy.filters.network.http_connection_manager", - "envoy.filters.network.local_ratelimit", - "envoy.filters.network.ratelimit", - "envoy.filters.network.rbac", - # TODO(asraa): Remove when fuzzer sets up connections for TcpProxy properly. - # "envoy.filters.network.tcp_proxy", -] - envoy_cc_test_library( name = "vig_anymap_ext_lib", srcs = ["validated_input_generator_any_map_extensions.cc"], diff --git a/test/extensions/filters/network/common/fuzz/config.bzl b/test/extensions/filters/network/common/fuzz/config.bzl new file mode 100644 index 000000000000..c275f3dfed79 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/config.bzl @@ -0,0 +1,15 @@ +# The filters to be fuzzed by the network_readfilter_fuzz_test. Prefer to add only stable filters, +# that do not have a dedicated fuzzer to them. +READFILTER_FUZZ_FILTERS = [ + "envoy.filters.network.client_ssl_auth", + "envoy.filters.network.ext_authz", + "envoy.filters.network.envoy_mobile_http_connection_manager", + # A dedicated http_connection_manager fuzzer can be found in + # test/common/http/conn_manager_impl_fuzz_test.cc + "envoy.filters.network.http_connection_manager", + "envoy.filters.network.local_ratelimit", + "envoy.filters.network.ratelimit", + "envoy.filters.network.rbac", + # TODO(asraa): Remove when fuzzer sets up connections for TcpProxy properly. + # "envoy.filters.network.tcp_proxy", +] diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 653124e4adcf..361ce48bf0bd 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -6,7 +6,7 @@ colorama coloredlogs dependatool>=0.2.2 envoy.base.utils>=0.4.10 -envoy.code.check>=0.5.0 +envoy.code.check>=0.5.1 envoy.dependency.check>=0.1.7 envoy.distribution.release>=0.0.9 envoy.distribution.repo>=0.0.8 diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 33733690523d..34743052ec71 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -434,9 +434,9 @@ envoy-base-utils==0.4.11 \ # envoy-docs-sphinx-runner # envoy-github-release # envoy-gpg-sign -envoy-code-check==0.5.0 \ - --hash=sha256:326ad3f188705fe7d0652bc707f7bc5eac190ff5c68d21f2fc5ebebc3b100e45 \ - --hash=sha256:46314932f38cab7caf02ff0da1d7a961f94b4ceedb11e19d0552c36481a041e6 +envoy-code-check==0.5.1 \ + --hash=sha256:379162ed8d83c9cb045917269b58f6e59812523359d8b0772b35078ab06dc8c1 \ + --hash=sha256:4b2b11b4fff634f1268b8dfb754c1d99300b8e58f5bc4276c96a8eb30cb9cea6 # via -r requirements.in envoy-dependency-check==0.1.8 \ --hash=sha256:ac9820e446bb44e05121e5c93c210f40ca37076580b0d082da2c63e7784c338a \ diff --git a/tools/code/BUILD b/tools/code/BUILD index 61393c4ac239..1af762c87d1d 100644 --- a/tools/code/BUILD +++ b/tools/code/BUILD @@ -2,6 +2,10 @@ load("//bazel:envoy_build_system.bzl", "envoy_package") load("//tools/base:envoy_python.bzl", "envoy_entry_point") load("@envoy_repo//:path.bzl", "PATH") load("@aspect_bazel_lib//lib:jq.bzl", "jq") +load( + "//test/extensions/filters/network/common/fuzz:config.bzl", + "READFILTER_FUZZ_FILTERS", +) licenses(["notice"]) # Apache 2 @@ -22,6 +26,7 @@ envoy_entry_point( name = "check", args = [ "--extensions_build_config=$(location :extensions_build_config)", + "--extensions_fuzzed_count=%s" % len(READFILTER_FUZZ_FILTERS), "--path=%s" % PATH, "-b shellcheck:$(location @com_github_aignas_rules_shellcheck//:shellcheck)", "-b gofmt:$(location @go_sdk//:bin/gofmt)", From d04c3a406f5d5a896fa35e86a4689e4daca21a0f Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 24 May 2023 10:14:34 -0400 Subject: [PATCH 354/740] http: allowing for mixed case HTTP/1.1 scheme (#27581) Signed-off-by: Alyssa Wilk --- changelogs/current.yaml | 5 ++++ source/common/http/http1/codec_impl.cc | 9 +++++-- source/common/runtime/runtime_features.cc | 1 + test/integration/integration_test.cc | 30 +++++++++++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index e0205dcb8888..7f7af0bd7f0f 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -25,6 +25,11 @@ minor_behavior_changes: - area: custom response change: | The filter now traverses matchers from most specific to least specific per filter config till a match is found for the response. +- area: http1 + change: | + Allowing mixed case schemes in absolute urls (e.g. HtTp://www.google.com). Mixed case schemes will be normalized to + the lower cased equivalents before being forwarded upstream. This behavior can be reverted by setting runtime flag + ``envoy.reloadable_features.allow_absolute_url_with_mixed_scheme`` to false. - area: http1 change: | The HTTP1 server-side codec no longer considers encoding 1xx headers as diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index 053f397dd831..00a439c00241 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -1132,8 +1132,13 @@ Status ServerConnectionImpl::handlePath(RequestHeaderMap& headers, absl::string_ // Add the scheme and validate to ensure no https:// // requests are accepted over unencrypted connections by front-line Envoys. if (!is_connect) { - headers.setScheme(absolute_url.scheme()); - if (!HeaderUtility::schemeIsValid(absolute_url.scheme())) { + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.allow_absolute_url_with_mixed_scheme")) { + headers.setScheme(absl::AsciiStrToLower(absolute_url.scheme())); + } else { + headers.setScheme(absolute_url.scheme()); + } + if (!HeaderUtility::schemeIsValid(headers.getSchemeValue())) { RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().InvalidScheme)); return codecProtocolError("http/1.1 protocol error: invalid scheme"); } diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 360998402fa3..14e980901d74 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -29,6 +29,7 @@ // If issues are found that require a runtime feature to be disabled, it should be reported // ASAP by filing a bug on github. Overriding non-buggy code is strongly discouraged to avoid the // problem of the bugs being found after the old code path has been removed. +RUNTIME_GUARD(envoy_reloadable_features_allow_absolute_url_with_mixed_scheme); RUNTIME_GUARD(envoy_reloadable_features_allow_compact_maglev); RUNTIME_GUARD(envoy_reloadable_features_append_query_parameters_path_rewriter); RUNTIME_GUARD(envoy_reloadable_features_append_xfh_idempotent); diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index cc08069f481a..7c30ed707159 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -1538,6 +1538,36 @@ TEST_P(IntegrationTest, AbsolutePathWithPort) { EXPECT_THAT(response, StartsWith("HTTP/1.1 301")); } +TEST_P(IntegrationTest, AbsolutePathWithMixedScheme) { + // Configure www.namewithport.com:1234 to send a redirect, and ensure the redirect is + // encountered via absolute URL with a port. + auto host = config_helper_.createVirtualHost("www.namewithport.com:1234", "/"); + host.set_require_tls(envoy::config::route::v3::VirtualHost::ALL); + config_helper_.addVirtualHost(host); + initialize(); + std::string response; + sendRawHttpAndWaitForResponse( + lookupPort("http"), "GET hTtp://www.namewithport.com:1234 HTTP/1.1\r\nHost: host\r\n\r\n", + &response, true); + EXPECT_THAT(response, StartsWith("HTTP/1.1 301")); +} + +TEST_P(IntegrationTest, AbsolutePathWithMixedSchemeLegacy) { + config_helper_.addRuntimeOverride( + "envoy.reloadable_features.allow_absolute_url_with_mixed_scheme", "false"); + + // Mixed scheme requests will be rejected + auto host = config_helper_.createVirtualHost("www.namewithport.com:1234", "/"); + host.set_require_tls(envoy::config::route::v3::VirtualHost::ALL); + config_helper_.addVirtualHost(host); + initialize(); + std::string response; + sendRawHttpAndWaitForResponse( + lookupPort("http"), "GET hTtp://www.namewithport.com:1234 HTTP/1.1\r\nHost: host\r\n\r\n", + &response, true); + EXPECT_THAT(response, StartsWith("HTTP/1.1 400")); +} + TEST_P(IntegrationTest, AbsolutePathWithoutPort) { // Add a restrictive default match, to avoid the request hitting the * / catchall. config_helper_.setDefaultHostAndRoute("foo.com", "/found"); From d9275856a07a7ec312230b9c2c31260229642e50 Mon Sep 17 00:00:00 2001 From: "Dr. Andre Vehreschild" <101638173+vehre-x41@users.noreply.github.com> Date: Wed, 24 May 2023 16:22:08 +0200 Subject: [PATCH 355/740] [fuzz] Use absl::Cleanup to ensure leaveMessage is called (#27609) Signed-off-by: Andre Vehreschild --- test/fuzz/BUILD | 1 + test/fuzz/mutable_visitor.cc | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/fuzz/BUILD b/test/fuzz/BUILD index 129b42d04bdb..df66c8862a78 100644 --- a/test/fuzz/BUILD +++ b/test/fuzz/BUILD @@ -89,6 +89,7 @@ envoy_cc_test_library( "//source/common/protobuf:visitor_lib", "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto", "@com_github_cncf_udpa//xds/type/v3:pkg_cc_proto", + "@com_google_absl//absl/cleanup", ], ) diff --git a/test/fuzz/mutable_visitor.cc b/test/fuzz/mutable_visitor.cc index 4ee7f29b29e8..b011d2caff42 100644 --- a/test/fuzz/mutable_visitor.cc +++ b/test/fuzz/mutable_visitor.cc @@ -5,6 +5,7 @@ #include "source/common/protobuf/utility.h" #include "source/common/protobuf/visitor_helper.h" +#include "absl/cleanup/cleanup.h" #include "udpa/type/v1/typed_struct.pb.h" #include "xds/type/v3/typed_struct.pb.h" @@ -17,6 +18,9 @@ void traverseMessageWorkerExt(ProtoVisitor& visitor, Protobuf::Message& message, bool was_any_or_top_level, bool recurse_into_any, absl::string_view const& field_name) { visitor.onEnterMessage(message, parents, was_any_or_top_level, field_name); + absl::Cleanup message_leaver = [&visitor, &parents, &message, was_any_or_top_level, field_name] { + visitor.onLeaveMessage(message, parents, was_any_or_top_level, field_name); + }; // If told to recurse into Any messages, do that here and skip the rest of the function. if (recurse_into_any) { @@ -71,7 +75,6 @@ void traverseMessageWorkerExt(ProtoVisitor& visitor, Protobuf::Message& message, } } } - visitor.onLeaveMessage(message, parents, was_any_or_top_level, field_name); } } // namespace From fdd36d3893a3282ec06060c2e4523fbd5808e6b1 Mon Sep 17 00:00:00 2001 From: sky Date: Wed, 24 May 2023 22:39:45 +0800 Subject: [PATCH 356/740] Optimize the supportsIpTransparent function to avoid getOriginalDst failure (#27538) Signed-off-by: wangfakang --- envoy/api/os_sys_calls.h | 4 +- envoy/network/socket.h | 120 ++++++++++++++++++ source/common/api/BUILD | 1 + source/common/api/posix/os_sys_calls_impl.cc | 36 +++--- source/common/api/posix/os_sys_calls_impl.h | 2 +- source/common/api/win32/os_sys_calls_impl.cc | 2 +- source/common/api/win32/os_sys_calls_impl.h | 2 +- source/common/network/socket_option_impl.h | 119 ----------------- source/common/network/utility.cc | 2 +- .../filters/udp/udp_proxy/udp_proxy_filter.h | 4 +- test/common/network/utility_test.cc | 8 +- .../udp/udp_proxy/udp_proxy_filter_test.cc | 8 +- test/mocks/api/mocks.h | 2 +- 13 files changed, 156 insertions(+), 154 deletions(-) diff --git a/envoy/api/os_sys_calls.h b/envoy/api/os_sys_calls.h index 44e67c3ef4d9..03509b4eef08 100644 --- a/envoy/api/os_sys_calls.h +++ b/envoy/api/os_sys_calls.h @@ -121,9 +121,9 @@ class OsSysCalls { virtual bool supportsUdpGso() const PURE; /** - * return true if the OS support both IP_TRANSPARENT and IPV6_TRANSPARENT options + * return true if the OS support IP_TRANSPARENT or IPV6_TRANSPARENT options by the ip version. */ - virtual bool supportsIpTransparent() const PURE; + virtual bool supportsIpTransparent(Network::Address::IpVersion version) const PURE; /** * return true if the OS supports multi-path TCP diff --git a/envoy/network/socket.h b/envoy/network/socket.h index 8756457cbdc2..0f2cde06ff19 100644 --- a/envoy/network/socket.h +++ b/envoy/network/socket.h @@ -42,6 +42,126 @@ struct SocketOptionName { #define ENVOY_MAKE_SOCKET_OPTION_NAME(level, option) \ Network::SocketOptionName(level, option, #level "/" #option) +// Moved from source/common/network/socket_option_impl.h to avoid dependency loops. +#ifdef IP_TRANSPARENT +#define ENVOY_SOCKET_IP_TRANSPARENT ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_TRANSPARENT) +#else +#define ENVOY_SOCKET_IP_TRANSPARENT Network::SocketOptionName() +#endif + +#ifdef IPV6_TRANSPARENT +#define ENVOY_SOCKET_IPV6_TRANSPARENT ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_TRANSPARENT) +#else +#define ENVOY_SOCKET_IPV6_TRANSPARENT Network::SocketOptionName() +#endif + +#ifdef IP_FREEBIND +#define ENVOY_SOCKET_IP_FREEBIND ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_FREEBIND) +#else +#define ENVOY_SOCKET_IP_FREEBIND Network::SocketOptionName() +#endif + +#ifdef IPV6_FREEBIND +#define ENVOY_SOCKET_IPV6_FREEBIND ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_FREEBIND) +#else +#define ENVOY_SOCKET_IPV6_FREEBIND Network::SocketOptionName() +#endif + +#ifdef SO_KEEPALIVE +#define ENVOY_SOCKET_SO_KEEPALIVE ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_KEEPALIVE) +#else +#define ENVOY_SOCKET_SO_KEEPALIVE Network::SocketOptionName() +#endif + +#ifdef SO_MARK +#define ENVOY_SOCKET_SO_MARK ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_MARK) +#else +#define ENVOY_SOCKET_SO_MARK Network::SocketOptionName() +#endif + +#ifdef SO_NOSIGPIPE +#define ENVOY_SOCKET_SO_NOSIGPIPE ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_NOSIGPIPE) +#else +#define ENVOY_SOCKET_SO_NOSIGPIPE Network::SocketOptionName() +#endif + +#ifdef SO_REUSEPORT +#define ENVOY_SOCKET_SO_REUSEPORT ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_REUSEPORT) +#else +#define ENVOY_SOCKET_SO_REUSEPORT Network::SocketOptionName() +#endif + +#ifdef SO_ORIGINAL_DST +#define ENVOY_SOCKET_SO_ORIGINAL_DST ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_IP, SO_ORIGINAL_DST) +#else +#define ENVOY_SOCKET_SO_ORIGINAL_DST Network::SocketOptionName() +#endif + +#ifdef IP6T_SO_ORIGINAL_DST +#define ENVOY_SOCKET_IP6T_SO_ORIGINAL_DST \ + ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_IPV6, IP6T_SO_ORIGINAL_DST) +#else +#define ENVOY_SOCKET_IP6T_SO_ORIGINAL_DST Network::SocketOptionName() +#endif + +#ifdef UDP_GRO +#define ENVOY_SOCKET_UDP_GRO ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_UDP, UDP_GRO) +#else +#define ENVOY_SOCKET_UDP_GRO Network::SocketOptionName() +#endif + +#ifdef TCP_KEEPCNT +#define ENVOY_SOCKET_TCP_KEEPCNT ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPCNT) +#else +#define ENVOY_SOCKET_TCP_KEEPCNT Network::SocketOptionName() +#endif + +#ifdef TCP_KEEPIDLE +#define ENVOY_SOCKET_TCP_KEEPIDLE ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPIDLE) +#elif TCP_KEEPALIVE // macOS uses a different name from Linux for just this option. +#define ENVOY_SOCKET_TCP_KEEPIDLE ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPALIVE) +#else +#define ENVOY_SOCKET_TCP_KEEPIDLE Network::SocketOptionName() +#endif + +#ifdef TCP_KEEPINTVL +#define ENVOY_SOCKET_TCP_KEEPINTVL ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPINTVL) +#else +#define ENVOY_SOCKET_TCP_KEEPINTVL Network::SocketOptionName() +#endif + +#ifdef TCP_FASTOPEN +#define ENVOY_SOCKET_TCP_FASTOPEN ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_FASTOPEN) +#else +#define ENVOY_SOCKET_TCP_FASTOPEN Network::SocketOptionName() +#endif + +// Linux uses IP_PKTINFO for both sending source address and receiving destination +// address. +// FreeBSD uses IP_RECVDSTADDR for receiving destination address and IP_SENDSRCADDR for sending +// source address. And these two have same value for convenience purpose. +#ifdef IP_RECVDSTADDR +#ifdef IP_SENDSRCADDR +static_assert(IP_RECVDSTADDR == IP_SENDSRCADDR); +#endif +#define ENVOY_IP_PKTINFO IP_RECVDSTADDR +#elif IP_PKTINFO +#define ENVOY_IP_PKTINFO IP_PKTINFO +#endif + +#define ENVOY_SELF_IP_ADDR ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, ENVOY_IP_PKTINFO) + +// Both Linux and FreeBSD use IPV6_RECVPKTINFO for both sending source address and +// receiving destination address. +#define ENVOY_SELF_IPV6_ADDR ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_RECVPKTINFO) + +#ifdef SO_ATTACH_REUSEPORT_CBPF +#define ENVOY_ATTACH_REUSEPORT_CBPF \ + ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF) +#else +#define ENVOY_ATTACH_REUSEPORT_CBPF Network::SocketOptionName() +#endif + /** * Interfaces for providing a socket's various addresses. This is split into a getters interface * and a getters + setters interface. This is so that only the getters portion can be overridden diff --git a/source/common/api/BUILD b/source/common/api/BUILD index 0ee05a74470d..3230052364e2 100644 --- a/source/common/api/BUILD +++ b/source/common/api/BUILD @@ -57,6 +57,7 @@ envoy_cc_library( }), deps = [ "//envoy/api:os_sys_calls_interface", + "//envoy/network:socket_interface", "//source/common/network:address_lib", "//source/common/singleton:threadsafe_singleton", "//third_party/android:android_lib", diff --git a/source/common/api/posix/os_sys_calls_impl.cc b/source/common/api/posix/os_sys_calls_impl.cc index e6f35422384c..7df7e6384e90 100644 --- a/source/common/api/posix/os_sys_calls_impl.cc +++ b/source/common/api/posix/os_sys_calls_impl.cc @@ -5,6 +5,8 @@ #include #include +#include "envoy/network/socket.h" + #include "source/common/api/os_sys_calls_impl.h" #include "source/common/network/address_impl.h" @@ -138,8 +140,9 @@ bool OsSysCallsImpl::supportsUdpGso() const { #endif } -bool OsSysCallsImpl::supportsIpTransparent() const { -#if !defined(__linux__) || !defined(IPV6_TRANSPARENT) +bool OsSysCallsImpl::supportsIpTransparent(Network::Address::IpVersion ip_version) const { +#if !defined(__linux__) + UNREFERENCED_PARAMETER(ip_version); return false; #else // The linux kernel supports IP_TRANSPARENT by following patch(starting from v2.6.28) : @@ -149,32 +152,27 @@ bool OsSysCallsImpl::supportsIpTransparent() const { // https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/net/ipv6/ipv6_sockglue.c?id=6c46862280c5f55eda7750391bc65cd7e08c7535 // // So, almost recent linux kernel supports both IP_TRANSPARENT and IPV6_TRANSPARENT options. + // But there are also has ipv4 only or ipv6 only scenarios. // // And these socket options need CAP_NET_ADMIN capability to be applied. // The CAP_NET_ADMIN capability should be applied by root user before call this function. - static const bool is_supported = [] { - // Check ipv4 case - int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, IPPROTO_UDP); + + static constexpr auto transparent_supported = [](int family) { + auto opt_tp = family == AF_INET ? ENVOY_SOCKET_IP_TRANSPARENT : ENVOY_SOCKET_IPV6_TRANSPARENT; + int fd = ::socket(family, SOCK_DGRAM | SOCK_NONBLOCK, IPPROTO_UDP); if (fd < 0) { return false; } int val = 1; - bool result = (0 == ::setsockopt(fd, IPPROTO_IP, IP_TRANSPARENT, &val, sizeof(val))); - ::close(fd); - if (!result) { - return false; - } - // Check ipv6 case - fd = ::socket(AF_INET6, SOCK_DGRAM | SOCK_NONBLOCK, IPPROTO_UDP); - if (fd < 0) { - return false; - } - val = 1; - result = (0 == ::setsockopt(fd, IPPROTO_IPV6, IPV6_TRANSPARENT, &val, sizeof(val))); + bool result = (0 == ::setsockopt(fd, opt_tp.level(), opt_tp.option(), &val, sizeof(val))); ::close(fd); return result; - }(); - return is_supported; + }; + // Check ipv4 case + static const bool ipv4_is_supported = transparent_supported(AF_INET); + // Check ipv6 case + static const bool ipv6_is_supported = transparent_supported(AF_INET6); + return ip_version == Network::Address::IpVersion::v4 ? ipv4_is_supported : ipv6_is_supported; #endif } diff --git a/source/common/api/posix/os_sys_calls_impl.h b/source/common/api/posix/os_sys_calls_impl.h index 846fa4b2e115..e3e241be5739 100644 --- a/source/common/api/posix/os_sys_calls_impl.h +++ b/source/common/api/posix/os_sys_calls_impl.h @@ -31,7 +31,7 @@ class OsSysCallsImpl : public OsSysCalls { bool supportsMmsg() const override; bool supportsUdpGro() const override; bool supportsUdpGso() const override; - bool supportsIpTransparent() const override; + bool supportsIpTransparent(Network::Address::IpVersion version) const override; bool supportsMptcp() const override; SysCallIntResult close(os_fd_t fd) override; SysCallIntResult ftruncate(int fd, off_t length) override; diff --git a/source/common/api/win32/os_sys_calls_impl.cc b/source/common/api/win32/os_sys_calls_impl.cc index 30229e3aa3ca..3e1e7780ca49 100644 --- a/source/common/api/win32/os_sys_calls_impl.cc +++ b/source/common/api/win32/os_sys_calls_impl.cc @@ -212,7 +212,7 @@ bool OsSysCallsImpl::supportsUdpGso() const { return false; } -bool OsSysCallsImpl::supportsIpTransparent() const { +bool OsSysCallsImpl::supportsIpTransparent(Network::Address::IpVersion) const { // Windows doesn't support it. return false; } diff --git a/source/common/api/win32/os_sys_calls_impl.h b/source/common/api/win32/os_sys_calls_impl.h index 4a19d8b91bdf..a686b16ca10d 100644 --- a/source/common/api/win32/os_sys_calls_impl.h +++ b/source/common/api/win32/os_sys_calls_impl.h @@ -32,7 +32,7 @@ class OsSysCallsImpl : public OsSysCalls { bool supportsMmsg() const override; bool supportsUdpGro() const override; bool supportsUdpGso() const override; - bool supportsIpTransparent() const override; + bool supportsIpTransparent(Network::Address::IpVersion version) const override; bool supportsMptcp() const override; SysCallIntResult close(os_fd_t fd) override; SysCallIntResult ftruncate(int fd, off_t length) override; diff --git a/source/common/network/socket_option_impl.h b/source/common/network/socket_option_impl.h index 9e2c03ccac33..cea24762ae7f 100644 --- a/source/common/network/socket_option_impl.h +++ b/source/common/network/socket_option_impl.h @@ -11,125 +11,6 @@ namespace Envoy { namespace Network { -#ifdef IP_TRANSPARENT -#define ENVOY_SOCKET_IP_TRANSPARENT ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_TRANSPARENT) -#else -#define ENVOY_SOCKET_IP_TRANSPARENT Network::SocketOptionName() -#endif - -#ifdef IPV6_TRANSPARENT -#define ENVOY_SOCKET_IPV6_TRANSPARENT ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_TRANSPARENT) -#else -#define ENVOY_SOCKET_IPV6_TRANSPARENT Network::SocketOptionName() -#endif - -#ifdef IP_FREEBIND -#define ENVOY_SOCKET_IP_FREEBIND ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_FREEBIND) -#else -#define ENVOY_SOCKET_IP_FREEBIND Network::SocketOptionName() -#endif - -#ifdef IPV6_FREEBIND -#define ENVOY_SOCKET_IPV6_FREEBIND ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_FREEBIND) -#else -#define ENVOY_SOCKET_IPV6_FREEBIND Network::SocketOptionName() -#endif - -#ifdef SO_KEEPALIVE -#define ENVOY_SOCKET_SO_KEEPALIVE ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_KEEPALIVE) -#else -#define ENVOY_SOCKET_SO_KEEPALIVE Network::SocketOptionName() -#endif - -#ifdef SO_MARK -#define ENVOY_SOCKET_SO_MARK ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_MARK) -#else -#define ENVOY_SOCKET_SO_MARK Network::SocketOptionName() -#endif - -#ifdef SO_NOSIGPIPE -#define ENVOY_SOCKET_SO_NOSIGPIPE ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_NOSIGPIPE) -#else -#define ENVOY_SOCKET_SO_NOSIGPIPE Network::SocketOptionName() -#endif - -#ifdef SO_REUSEPORT -#define ENVOY_SOCKET_SO_REUSEPORT ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_REUSEPORT) -#else -#define ENVOY_SOCKET_SO_REUSEPORT Network::SocketOptionName() -#endif - -#ifdef SO_ORIGINAL_DST -#define ENVOY_SOCKET_SO_ORIGINAL_DST ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_IP, SO_ORIGINAL_DST) -#else -#define ENVOY_SOCKET_SO_ORIGINAL_DST Network::SocketOptionName() -#endif - -#ifdef IP6T_SO_ORIGINAL_DST -#define ENVOY_SOCKET_IP6T_SO_ORIGINAL_DST \ - ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_IPV6, IP6T_SO_ORIGINAL_DST) -#else -#define ENVOY_SOCKET_IP6T_SO_ORIGINAL_DST Network::SocketOptionName() -#endif - -#ifdef UDP_GRO -#define ENVOY_SOCKET_UDP_GRO ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_UDP, UDP_GRO) -#else -#define ENVOY_SOCKET_UDP_GRO Network::SocketOptionName() -#endif - -#ifdef TCP_KEEPCNT -#define ENVOY_SOCKET_TCP_KEEPCNT ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPCNT) -#else -#define ENVOY_SOCKET_TCP_KEEPCNT Network::SocketOptionName() -#endif - -#ifdef TCP_KEEPIDLE -#define ENVOY_SOCKET_TCP_KEEPIDLE ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPIDLE) -#elif TCP_KEEPALIVE // macOS uses a different name from Linux for just this option. -#define ENVOY_SOCKET_TCP_KEEPIDLE ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPALIVE) -#else -#define ENVOY_SOCKET_TCP_KEEPIDLE Network::SocketOptionName() -#endif - -#ifdef TCP_KEEPINTVL -#define ENVOY_SOCKET_TCP_KEEPINTVL ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPINTVL) -#else -#define ENVOY_SOCKET_TCP_KEEPINTVL Network::SocketOptionName() -#endif - -#ifdef TCP_FASTOPEN -#define ENVOY_SOCKET_TCP_FASTOPEN ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_FASTOPEN) -#else -#define ENVOY_SOCKET_TCP_FASTOPEN Network::SocketOptionName() -#endif - -// Linux uses IP_PKTINFO for both sending source address and receiving destination -// address. -// FreeBSD uses IP_RECVDSTADDR for receiving destination address and IP_SENDSRCADDR for sending -// source address. And these two have same value for convenience purpose. -#ifdef IP_RECVDSTADDR -#ifdef IP_SENDSRCADDR -static_assert(IP_RECVDSTADDR == IP_SENDSRCADDR); -#endif -#define ENVOY_IP_PKTINFO IP_RECVDSTADDR -#elif IP_PKTINFO -#define ENVOY_IP_PKTINFO IP_PKTINFO -#endif - -#define ENVOY_SELF_IP_ADDR ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, ENVOY_IP_PKTINFO) - -// Both Linux and FreeBSD use IPV6_RECVPKTINFO for both sending source address and -// receiving destination address. -#define ENVOY_SELF_IPV6_ADDR ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_RECVPKTINFO) - -#ifdef SO_ATTACH_REUSEPORT_CBPF -#define ENVOY_ATTACH_REUSEPORT_CBPF \ - ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF) -#else -#define ENVOY_ATTACH_REUSEPORT_CBPF Network::SocketOptionName() -#endif - class SocketOptionImpl : public Socket::Option, Logger::Loggable { public: SocketOptionImpl(envoy::config::core::v3::SocketOption::SocketState in_state, diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index 08349765fb39..8e244f0a520e 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -392,7 +392,7 @@ Address::InstanceConstSharedPtr Utility::getOriginalDst(Socket& sock) { sock.getSocketOption(opt_dst.level(), opt_dst.option(), &orig_addr, &addr_len).return_value_; if (status != 0) { - if (Api::OsSysCallsSingleton::get().supportsIpTransparent()) { + if (Api::OsSysCallsSingleton::get().supportsIpTransparent(*ipVersion)) { socklen_t flag_len = sizeof(int); int is_tp; status = diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h index 925dc8453e17..a31b66c25696 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h @@ -82,7 +82,9 @@ class UdpProxyFilterConfig { // Default prefer_gro to true for upstream client traffic. upstream_socket_config_(config.upstream_socket_config(), true), random_(context.api().randomGenerator()) { - if (use_original_src_ip_ && !Api::OsSysCallsSingleton::get().supportsIpTransparent()) { + if (use_original_src_ip_ && + !Api::OsSysCallsSingleton::get().supportsIpTransparent( + context.getServerFactoryContext().options().localAddressIpVersion())) { ExceptionUtil::throwEnvoyException( "The platform does not support either IP_TRANSPARENT or IPV6_TRANSPARENT. Or the envoy " "is not running with the CAP_NET_ADMIN capability."); diff --git a/test/common/network/utility_test.cc b/test/common/network/utility_test.cc index 74d3ccbd9c07..03e7d8c8354b 100644 --- a/test/common/network/utility_test.cc +++ b/test/common/network/utility_test.cc @@ -354,7 +354,7 @@ TEST(NetworkUtility, GetOriginalDst) { // Transparent socket gets original dst from local address while connection tracking disabled EXPECT_CALL(socket, getSocketOption(Eq(SOL_IP), Eq(SO_ORIGINAL_DST), _, _)) .WillOnce(Return(Api::SysCallIntResult{-1, 0})); - EXPECT_CALL(os_sys_calls, supportsIpTransparent()).WillOnce(Return(true)); + EXPECT_CALL(os_sys_calls, supportsIpTransparent(Address::IpVersion::v4)).WillOnce(Return(true)); EXPECT_CALL(socket, getSocketOption(Eq(SOL_IP), Eq(IP_TRANSPARENT), _, _)) .WillOnce(DoAll(SetArg2Int(1), Return(Api::SysCallIntResult{0, 0}))); EXPECT_CALL(os_sys_calls, getsockname(_, _, _)) @@ -364,7 +364,7 @@ TEST(NetworkUtility, GetOriginalDst) { // Non-transparent socket fails to get original dst while connection tracking disabled EXPECT_CALL(socket, getSocketOption(Eq(SOL_IP), Eq(SO_ORIGINAL_DST), _, _)) .WillOnce(Return(Api::SysCallIntResult{-1, 0})); - EXPECT_CALL(os_sys_calls, supportsIpTransparent()).WillOnce(Return(true)); + EXPECT_CALL(os_sys_calls, supportsIpTransparent(Address::IpVersion::v4)).WillOnce(Return(true)); EXPECT_CALL(socket, getSocketOption(Eq(SOL_IP), Eq(IP_TRANSPARENT), _, _)) .WillOnce(DoAll(SetArg2Int(0), Return(Api::SysCallIntResult{0, 0}))); EXPECT_EQ(nullptr, Utility::getOriginalDst(socket)); @@ -383,7 +383,7 @@ TEST(NetworkUtility, GetOriginalDst) { // Transparent socket gets original dst from local address while connection tracking disabled EXPECT_CALL(socket, getSocketOption(Eq(SOL_IPV6), Eq(IP6T_SO_ORIGINAL_DST), _, _)) .WillOnce(Return(Api::SysCallIntResult{-1, 0})); - EXPECT_CALL(os_sys_calls, supportsIpTransparent()).WillOnce(Return(true)); + EXPECT_CALL(os_sys_calls, supportsIpTransparent(Address::IpVersion::v6)).WillOnce(Return(true)); EXPECT_CALL(socket, getSocketOption(Eq(SOL_IPV6), Eq(IPV6_TRANSPARENT), _, _)) .WillOnce(DoAll(SetArg2Int(1), Return(Api::SysCallIntResult{0, 0}))); EXPECT_CALL(os_sys_calls, getsockname(_, _, _)) @@ -393,7 +393,7 @@ TEST(NetworkUtility, GetOriginalDst) { // Non-transparent socket fails to get original dst while connection tracking disabled EXPECT_CALL(socket, getSocketOption(Eq(SOL_IPV6), Eq(IP6T_SO_ORIGINAL_DST), _, _)) .WillOnce(Return(Api::SysCallIntResult{-1, 0})); - EXPECT_CALL(os_sys_calls, supportsIpTransparent()).WillOnce(Return(true)); + EXPECT_CALL(os_sys_calls, supportsIpTransparent(Address::IpVersion::v6)).WillOnce(Return(true)); EXPECT_CALL(socket, getSocketOption(Eq(SOL_IPV6), Eq(IPV6_TRANSPARENT), _, _)) .WillOnce(DoAll(SetArg2Int(0), Return(Api::SysCallIntResult{0, 0}))); EXPECT_EQ(nullptr, Utility::getOriginalDst(socket)); diff --git a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc index 78014ad92105..22876780059d 100644 --- a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc +++ b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc @@ -200,7 +200,7 @@ class UdpProxyFilterTest : public UdpProxyFilterBase { ON_CALL(*factory_context_.access_log_manager_.file_, write(_)) .WillByDefault( Invoke([&](absl::string_view data) { output_.push_back(std::string(data)); })); - ON_CALL(os_sys_calls_, supportsIpTransparent()).WillByDefault(Return(true)); + ON_CALL(os_sys_calls_, supportsIpTransparent(_)).WillByDefault(Return(true)); EXPECT_CALL(os_sys_calls_, supportsUdpGro()).Times(AtLeast(0)).WillRepeatedly(Return(true)); EXPECT_CALL(callbacks_, udpListener()).Times(AtLeast(0)); EXPECT_CALL(*factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, address()) @@ -917,7 +917,7 @@ TEST_F(UdpProxyFilterTest, SocketOptionForUseOriginalSrcIp) { // The option is not supported on this platform. Just skip the test. GTEST_SKIP(); } - EXPECT_CALL(os_sys_calls_, supportsIpTransparent()); + EXPECT_CALL(os_sys_calls_, supportsIpTransparent(_)); InSequence s; @@ -1142,7 +1142,7 @@ TEST_F(UdpProxyFilterIpv6Test, SocketOptionForUseOriginalSrcIpInCaseOfIpv6) { // The option is not supported on this platform. Just skip the test. GTEST_SKIP(); } - EXPECT_CALL(os_sys_calls_, supportsIpTransparent()); + EXPECT_CALL(os_sys_calls_, supportsIpTransparent(_)); InSequence s; @@ -1199,7 +1199,7 @@ stat_prefix: foo // Make sure exit when use the use_original_src_ip but platform does not support ip // transparent option. TEST_F(UdpProxyFilterTest, ExitIpTransparentNoPlatformSupport) { - EXPECT_CALL(os_sys_calls_, supportsIpTransparent()).WillOnce(Return(false)); + EXPECT_CALL(os_sys_calls_, supportsIpTransparent(_)).WillOnce(Return(false)); InSequence s; diff --git a/test/mocks/api/mocks.h b/test/mocks/api/mocks.h index 0d66fb84f3ed..1496bef7aa19 100644 --- a/test/mocks/api/mocks.h +++ b/test/mocks/api/mocks.h @@ -124,7 +124,7 @@ class MockOsSysCalls : public OsSysCallsImpl { MOCK_METHOD(SysCallBoolResult, socketTcpInfo, (os_fd_t sockfd, EnvoyTcpInfo* tcp_info)); MOCK_METHOD(bool, supportsMmsg, (), (const)); MOCK_METHOD(bool, supportsUdpGro, (), (const)); - MOCK_METHOD(bool, supportsIpTransparent, (), (const)); + MOCK_METHOD(bool, supportsIpTransparent, (Network::Address::IpVersion version), (const)); MOCK_METHOD(bool, supportsMptcp, (), (const)); MOCK_METHOD(bool, supportsGetifaddrs, (), (const)); MOCK_METHOD(SysCallIntResult, getifaddrs, (InterfaceAddressVector & interfaces)); From 332852df3ca261b954c47c699a785850663a0a58 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 24 May 2023 10:43:26 -0400 Subject: [PATCH 357/740] test: allowing bootstrap config in integration tests (#27436) Signed-off-by: Alyssa Wilk --- .../kv_store_xds_delegate_integration_test.cc | 3 +- mobile/library/cc/engine_builder.cc | 13 +---- mobile/library/cc/engine_builder.h | 15 +----- mobile/test/cc/integration/lifetimes_test.cc | 5 +- .../test/cc/integration/send_headers_test.cc | 5 +- .../base_client_integration_test.cc | 14 +++--- .../base_client_integration_test.h | 5 +- .../common/integration/test_engine_builder.cc | 11 ++-- .../common/integration/test_engine_builder.h | 21 ++++++-- test/config/utility.cc | 18 +++++-- test/config/utility.h | 6 ++- .../http_grpc_access_log_integration_test.cc | 3 +- .../internal_listener_integration_test.cc | 9 ++-- .../http/tap/tap_filter_integration_test.cc | 3 +- test/integration/base_integration_test.cc | 30 ++++++++--- test/integration/base_integration_test.h | 4 ++ .../listener_lds_integration_test.cc | 3 +- test/integration/multi_envoy_test.cc | 3 +- .../integration/quic_http_integration_test.cc | 16 ++---- .../tcp_tunneling_integration_test.cc | 3 +- test/integration/xds_integration_test.cc | 50 ++++--------------- 21 files changed, 112 insertions(+), 128 deletions(-) diff --git a/contrib/config/test/kv_store_xds_delegate_integration_test.cc b/contrib/config/test/kv_store_xds_delegate_integration_test.cc index 049d9e727454..18a8836293b6 100644 --- a/contrib/config/test/kv_store_xds_delegate_integration_test.cc +++ b/contrib/config/test/kv_store_xds_delegate_integration_test.cc @@ -248,8 +248,7 @@ class KeyValueStoreXdsDelegateIntegrationTest on_server_init_function_ = nullptr; // Set up a new Envoy, using the previous Envoy's configuration, and create the test server. - ConfigHelper helper(version_, *api_, - MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper helper(version_, config_helper_.bootstrap()); // Close the connection between Envoy and the xDS FakeUpstreams. closeConnection(sds_connection_); diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc index bc4bce48509c..aed3ca25acf6 100644 --- a/mobile/library/cc/engine_builder.cc +++ b/mobile/library/cc/engine_builder.cc @@ -286,11 +286,6 @@ EngineBuilder& EngineBuilder::setRuntimeGuard(std::string guard, bool value) { return *this; } -EngineBuilder& EngineBuilder::setOverrideConfigForTests(std::string config) { - config_override_for_tests_ = std::move(config); - return *this; -} - std::unique_ptr EngineBuilder::generateBootstrap() const { // The yaml utilities have non-relevant thread asserts. Thread::SkipAsserts skip; @@ -885,11 +880,6 @@ EngineSharedPtr EngineBuilder::build() { envoy_event_tracker null_tracker{}; - std::unique_ptr bootstrap; - if (config_override_for_tests_.empty()) { - bootstrap = generateBootstrap(); - } - envoy_engine_t envoy_engine = init_engine(callbacks_->asEnvoyEngineCallbacks(), null_logger, null_tracker); @@ -911,10 +901,9 @@ EngineSharedPtr EngineBuilder::build() { if (auto cast_engine = reinterpret_cast(envoy_engine)) { auto options = std::make_unique(); + std::unique_ptr bootstrap = generateBootstrap(); if (bootstrap) { options->setConfigProto(std::move(bootstrap)); - } else { - options->setConfigYaml(config_override_for_tests_); } options->setLogLevel(options->parseAndValidateLogLevel(logLevelToString(log_level_).c_str())); options->setConcurrency(1); diff --git a/mobile/library/cc/engine_builder.h b/mobile/library/cc/engine_builder.h index 4eedcaac8296..84b8ce489983 100644 --- a/mobile/library/cc/engine_builder.h +++ b/mobile/library/cc/engine_builder.h @@ -37,6 +37,7 @@ struct NodeLocality { class EngineBuilder { public: EngineBuilder(); + virtual ~EngineBuilder() {} EngineBuilder& addLogLevel(LogLevel log_level); EngineBuilder& setOnEngineRunning(std::function closure); @@ -109,21 +110,10 @@ class EngineBuilder { EngineBuilder& addStringAccessor(std::string name, StringAccessorSharedPtr accessor); // This is separated from build() for the sake of testability - std::unique_ptr generateBootstrap() const; + virtual std::unique_ptr generateBootstrap() const; EngineSharedPtr build(); -protected: - // Overrides the EngineBuilder's constructed configuration with the YAML config supplied in the - // `config` parameter. Any prior or subsequent calls to the EngineBuilder's APIs are ignored in - // favor of using the YAML string to construct the Envoy Engine's Bootsrap configuration. - // - // This method is protected so that production code does not rely on it. - // - // Tests that need access to this method should go through the TestEngineBuilder class, which - // provides access to this method. - EngineBuilder& setOverrideConfigForTests(std::string config); - private: struct NativeFilterConfig { NativeFilterConfig(std::string name, std::string typed_config) @@ -149,7 +139,6 @@ class EngineBuilder { std::string app_version_ = "unspecified"; std::string app_id_ = "unspecified"; std::string device_os_ = "unspecified"; - std::string config_override_for_tests_ = ""; int stream_idle_timeout_seconds_ = 15; int per_try_idle_timeout_seconds_ = 15; bool gzip_decompression_filter_ = true; diff --git a/mobile/test/cc/integration/lifetimes_test.cc b/mobile/test/cc/integration/lifetimes_test.cc index 61a0fed765e5..b5fc701c0825 100644 --- a/mobile/test/cc/integration/lifetimes_test.cc +++ b/mobile/test/cc/integration/lifetimes_test.cc @@ -1,4 +1,5 @@ #include "test/common/integration/test_engine_builder.h" +#include "test/test_common/utility.h" #include "absl/synchronization/notification.h" #include "gtest/gtest.h" @@ -90,7 +91,9 @@ void sendRequest(Platform::EngineSharedPtr engine, Status& status, void sendRequestEndToEnd() { TestEngineBuilder engine_builder; - Platform::EngineSharedPtr engine = engine_builder.createEngine(CONFIG); + auto bootstrap = std::make_unique(); + TestUtility::loadFromYaml(CONFIG, *bootstrap); + Platform::EngineSharedPtr engine = engine_builder.createEngine(std::move(bootstrap)); Status status; absl::Notification stream_complete; diff --git a/mobile/test/cc/integration/send_headers_test.cc b/mobile/test/cc/integration/send_headers_test.cc index 3dd1494baa25..b17ec5f9f650 100644 --- a/mobile/test/cc/integration/send_headers_test.cc +++ b/mobile/test/cc/integration/send_headers_test.cc @@ -1,4 +1,5 @@ #include "test/common/integration/test_engine_builder.h" +#include "test/test_common/utility.h" #include "absl/synchronization/notification.h" #include "gtest/gtest.h" @@ -62,7 +63,9 @@ struct Status { TEST(SendHeadersTest, CanSendHeaders) { TestEngineBuilder engine_builder; - Platform::EngineSharedPtr engine = engine_builder.createEngine(CONFIG); + auto bootstrap = std::make_unique(); + TestUtility::loadFromYaml(CONFIG, *bootstrap); + Platform::EngineSharedPtr engine = engine_builder.createEngine(std::move(bootstrap)); Status status; absl::Notification stream_complete; diff --git a/mobile/test/common/integration/base_client_integration_test.cc b/mobile/test/common/integration/base_client_integration_test.cc index dd40c81b12d5..5830b8d82972 100644 --- a/mobile/test/common/integration/base_client_integration_test.cc +++ b/mobile/test/common/integration/base_client_integration_test.cc @@ -79,15 +79,16 @@ Platform::LogLevel getPlatformLogLevelFromOptions() { // Use the Envoy mobile default config as much as possible in this test. // There are some config modifiers below which do result in deltas. // Note: This function is only used to build the Engine if `override_builder_config_` is true. -std::string defaultConfig() { +envoy::config::bootstrap::v3::Bootstrap defaultConfig() { Platform::EngineBuilder builder; std::unique_ptr bootstrap = builder.generateBootstrap(); - return MessageUtil::getYamlStringFromMessage(*bootstrap); + envoy::config::bootstrap::v3::Bootstrap to_return = *bootstrap; + return to_return; } -BaseClientIntegrationTest::BaseClientIntegrationTest(Network::Address::IpVersion ip_version, - const std::string& bootstrap_config) - : BaseIntegrationTest(ip_version, bootstrap_config) { +BaseClientIntegrationTest::BaseClientIntegrationTest(Network::Address::IpVersion ip_version) + : BaseIntegrationTest(BaseIntegrationTest::defaultAddressFunction(ip_version), ip_version, + defaultConfig()) { skip_tag_extraction_rule_check_ = true; full_dispatcher_ = api_->allocateDispatcher("fake_envoy_mobile"); use_lds_ = false; @@ -204,7 +205,8 @@ void BaseClientIntegrationTest::createEnvoy() { finalizeConfigWithPorts(config_helper_, ports, use_lds_); ASSERT_FALSE(config_helper_.bootstrap().has_admin()) << "Bootstrap config should not have `admin` configured in Envoy Mobile"; - builder_.setOverrideConfig(MessageUtil::getYamlStringFromMessage(config_helper_.bootstrap())); + builder_.setOverrideConfig( + std::make_unique(config_helper_.bootstrap())); } else { ENVOY_LOG_MISC(warn, "Using builder config and ignoring config modifiers"); } diff --git a/mobile/test/common/integration/base_client_integration_test.h b/mobile/test/common/integration/base_client_integration_test.h index 31143e94c64f..b52030bfb663 100644 --- a/mobile/test/common/integration/base_client_integration_test.h +++ b/mobile/test/common/integration/base_client_integration_test.h @@ -29,7 +29,7 @@ Http::ResponseHeaderMapPtr toResponseHeaders(envoy_headers headers); // Creates a default bootstrap config from the EngineBuilder. // Only used to build the Engine if `override_builder_config_` is set to true. -std::string defaultConfig(); +envoy::config::bootstrap::v3::Bootstrap defaultConfig(); // A base class for Envoy Mobile client integration tests which interact with Envoy through the // Http::Client class. @@ -38,8 +38,7 @@ std::string defaultConfig(); // into a test lib. class BaseClientIntegrationTest : public BaseIntegrationTest { public: - BaseClientIntegrationTest(Network::Address::IpVersion ip_version, - const std::string& bootstrap_config = defaultConfig()); + BaseClientIntegrationTest(Network::Address::IpVersion ip_version); virtual ~BaseClientIntegrationTest() = default; // Note: This class does not inherit from testing::Test and so this TearDown() method // does not override testing::Test::TearDown(). As a result, it will not be called diff --git a/mobile/test/common/integration/test_engine_builder.cc b/mobile/test/common/integration/test_engine_builder.cc index 2292457fceaf..c81c77166eb2 100644 --- a/mobile/test/common/integration/test_engine_builder.cc +++ b/mobile/test/common/integration/test_engine_builder.cc @@ -4,18 +4,15 @@ namespace Envoy { -Platform::EngineSharedPtr TestEngineBuilder::createEngine(std::string config) { +Platform::EngineSharedPtr +TestEngineBuilder::createEngine(std::unique_ptr config) { absl::Notification engine_running; - Platform::EngineSharedPtr engine = setOverrideConfigForTests(std::move(config)) - .addLogLevel(Platform::LogLevel::debug) + setOverrideConfig(std::move(config)); + Platform::EngineSharedPtr engine = addLogLevel(Platform::LogLevel::debug) .setOnEngineRunning([&]() { engine_running.Notify(); }) .build(); engine_running.WaitForNotification(); return engine; } -void TestEngineBuilder::setOverrideConfig(std::string config) { - setOverrideConfigForTests(std::move(config)); -} - } // namespace Envoy diff --git a/mobile/test/common/integration/test_engine_builder.h b/mobile/test/common/integration/test_engine_builder.h index 3b0f67ce8d57..1c969046249d 100644 --- a/mobile/test/common/integration/test_engine_builder.h +++ b/mobile/test/common/integration/test_engine_builder.h @@ -6,18 +6,33 @@ namespace Envoy { // A wrapper class around EngineBuilder, specifically for mobile tests. // -// Mobile tests often supply their own YAML configuration for convenience, instead of using the +// Mobile tests often supply their own configuration for convenience, instead of using the // EngineBuilder APIs. This wrapper class builds an Envoy Mobile Engine with the ability to // access setOverrideConfigForTests(), which is a protected method inside EngineBuilder. class TestEngineBuilder : public Platform::EngineBuilder { public: + virtual ~TestEngineBuilder() {} + // Creates an Envoy Engine from the provided config and waits until the engine is running before // returning the Engine as a shared_ptr. - Platform::EngineSharedPtr createEngine(std::string config); + Platform::EngineSharedPtr + createEngine(std::unique_ptr config); // Overrides the EngineBuilder's config with the provided string. Calls to the EngineBuilder's // bootstrap modifying APIs do not take effect after this function is called. - void setOverrideConfig(std::string config); + void setOverrideConfig(std::unique_ptr&& config) { + override_bootstrap_ = std::move(config); + } + + std::unique_ptr generateBootstrap() const override { + if (override_bootstrap_ != nullptr) { + return std::make_unique(*override_bootstrap_); + } + return Platform::EngineBuilder::generateBootstrap(); + } + +private: + std::unique_ptr override_bootstrap_; }; } // namespace Envoy diff --git a/test/config/utility.cc b/test/config/utility.cc index e2621357519f..ab1d97d89575 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -30,6 +30,13 @@ #include "gtest/gtest.h" namespace Envoy { +namespace { +envoy::config::bootstrap::v3::Bootstrap& +basicBootstrap(envoy::config::bootstrap::v3::Bootstrap& bootstrap, const std::string& config) { + TestUtility::loadFromYaml(config, bootstrap); + return bootstrap; +} +} // namespace std::string ConfigHelper::baseConfigNoListeners() { return fmt::format(R"EOF( @@ -721,11 +728,14 @@ envoy::config::endpoint::v3::Endpoint ConfigHelper::buildEndpoint(const std::str return endpoint; } -ConfigHelper::ConfigHelper(const Network::Address::IpVersion version, Api::Api& api, - const std::string& config) { +ConfigHelper::ConfigHelper(const Network::Address::IpVersion version, Api::Api&, + const std::string& config) + : ConfigHelper(version, basicBootstrap(bootstrap_, config)) {} + +ConfigHelper::ConfigHelper(const Network::Address::IpVersion version, + const envoy::config::bootstrap::v3::Bootstrap& bootstrap) { RELEASE_ASSERT(!finalized_, ""); - std::string filename = TestEnvironment::writeStringToFileForTest("basic_config.yaml", config); - TestUtility::loadFromFile(filename, bootstrap_, api); + bootstrap_ = bootstrap; // Fix up all the socket addresses with the correct version. auto* admin = bootstrap_.mutable_admin(); diff --git a/test/config/utility.h b/test/config/utility.h index dce790b7463f..61c7ee881a75 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -139,12 +139,16 @@ class ConfigHelper { absl::optional max_verify_depth_{absl::nullopt}; }; + // Sets up config with the provided bootstrap. + ConfigHelper(const Network::Address::IpVersion version, + const envoy::config::bootstrap::v3::Bootstrap& bootstrap); + // Set up basic config, using the specified IpVersion for all connections: listeners, upstream, // and admin connections. // // By default, this runs with an L7 proxy config, but config can be set to TCP_PROXY_CONFIG // to test L4 proxying. - ConfigHelper(const Network::Address::IpVersion version, Api::Api& api, + ConfigHelper(const Network::Address::IpVersion version, Api::Api&, const std::string& config = httpProxyConfig(false, false)); static void diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 07e6d7a5191a..3cd94ec56378 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -331,8 +331,7 @@ TEST_P(AccessLogIntegrationTest, GrpcLoggerSurvivesAfterReloadConfig) { test_server_->waitForCounterEq("access_logs.grpc_access_log.logs_written", 2); // Create a new config with HTTP/1.0 proxying. The goal is to trigger a listener update. - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { diff --git a/test/extensions/bootstrap/internal_listener/internal_listener_integration_test.cc b/test/extensions/bootstrap/internal_listener/internal_listener_integration_test.cc index 72e54fdc495c..b1a36c24b443 100644 --- a/test/extensions/bootstrap/internal_listener/internal_listener_integration_test.cc +++ b/test/extensions/bootstrap/internal_listener/internal_listener_integration_test.cc @@ -45,8 +45,7 @@ TEST_P(InternalListenerIntegrationTest, BasicConfigUpdate) { initialize(); EXPECT_EQ(1, test_server_->counter("listener_manager.lds.update_success")->value()); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -66,8 +65,7 @@ TEST_P(InternalListenerIntegrationTest, InplaceUpdate) { initialize(); EXPECT_EQ(1, test_server_->counter("listener_manager.lds.update_success")->value()); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -90,8 +88,7 @@ TEST_P(InternalListenerIntegrationTest, DeleteListener) { initialize(); EXPECT_EQ(1, test_server_->counter("listener_manager.lds.update_success")->value()); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { bootstrap.mutable_static_resources()->mutable_listeners()->RemoveLast(); diff --git a/test/extensions/filters/http/tap/tap_filter_integration_test.cc b/test/extensions/filters/http/tap/tap_filter_integration_test.cc index af716d2917da..4fb6ae65c707 100644 --- a/test/extensions/filters/http/tap/tap_filter_integration_test.cc +++ b/test/extensions/filters/http/tap/tap_filter_integration_test.cc @@ -428,8 +428,7 @@ config_id: test_config_id startAdminRequest(admin_request_yaml); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.prependFilter(admin_filter_config_); new_config_helper.renameListener("foo"); new_config_helper.setLds("1"); diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 4b45cb7e0f39..a90bf2210a43 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -29,6 +29,12 @@ #include "gtest/gtest.h" namespace Envoy { +envoy::config::bootstrap::v3::Bootstrap configToBootstrap(const std::string& config) { + envoy::config::bootstrap::v3::Bootstrap bootstrap; + TestUtility::loadFromYaml(config, bootstrap); + return bootstrap; +} + using ::testing::_; using ::testing::AssertionFailure; using ::testing::AssertionResult; @@ -40,13 +46,13 @@ using ::testing::ReturnRef; BaseIntegrationTest::BaseIntegrationTest(const InstanceConstSharedPtrFn& upstream_address_fn, Network::Address::IpVersion version, - const std::string& config) + const envoy::config::bootstrap::v3::Bootstrap& bootstrap) : api_(Api::createApiForTest(stats_store_, time_system_)), mock_buffer_factory_(new NiceMock), dispatcher_(api_->allocateDispatcher("test_thread", Buffer::WatermarkFactoryPtr{mock_buffer_factory_})), version_(version), upstream_address_fn_(upstream_address_fn), - config_helper_(version, *api_, config), + config_helper_(version, bootstrap), default_log_level_(TestEnvironment::getOptions().logLevel()) { Envoy::Server::validateProtoDescriptors(); // This is a hack, but there are situations where we disconnect fake upstream connections and @@ -73,14 +79,22 @@ BaseIntegrationTest::BaseIntegrationTest(const InstanceConstSharedPtrFn& upstrea #endif } +BaseIntegrationTest::BaseIntegrationTest(const InstanceConstSharedPtrFn& upstream_address_fn, + Network::Address::IpVersion version, + const std::string& config) + : BaseIntegrationTest(upstream_address_fn, version, configToBootstrap(config)) {} + +const BaseIntegrationTest::InstanceConstSharedPtrFn +BaseIntegrationTest::defaultAddressFunction(Network::Address::IpVersion version) { + return [version](int) { + return Network::Utility::parseInternetAddress(Network::Test::getLoopbackAddressString(version), + 0); + }; +} + BaseIntegrationTest::BaseIntegrationTest(Network::Address::IpVersion version, const std::string& config) - : BaseIntegrationTest( - [version](int) { - return Network::Utility::parseInternetAddress( - Network::Test::getLoopbackAddressString(version), 0); - }, - version, config) {} + : BaseIntegrationTest(defaultAddressFunction(version), version, config) {} Network::ClientConnectionPtr BaseIntegrationTest::makeClientConnection(uint32_t port) { return makeClientConnectionWithOptions(port, nullptr); diff --git a/test/integration/base_integration_test.h b/test/integration/base_integration_test.h index ae18ad704c5e..fddd40fbff4e 100644 --- a/test/integration/base_integration_test.h +++ b/test/integration/base_integration_test.h @@ -66,7 +66,11 @@ struct ApiFilesystemConfig { class BaseIntegrationTest : protected Logger::Loggable { public: using InstanceConstSharedPtrFn = std::function; + static const InstanceConstSharedPtrFn defaultAddressFunction(Network::Address::IpVersion version); + BaseIntegrationTest(const InstanceConstSharedPtrFn& upstream_address_fn, + Network::Address::IpVersion version, + const envoy::config::bootstrap::v3::Bootstrap& bootstrap); // Creates a test fixture with an upstream bound to INADDR_ANY on an unspecified port using the // provided IP |version|. BaseIntegrationTest(Network::Address::IpVersion version, diff --git a/test/integration/listener_lds_integration_test.cc b/test/integration/listener_lds_integration_test.cc index cb34ae5d9dee..d72316ad2fcc 100644 --- a/test/integration/listener_lds_integration_test.cc +++ b/test/integration/listener_lds_integration_test.cc @@ -1718,8 +1718,7 @@ TEST_P(RebalancerTest, BindToPortUpdate) { concurrency_ = 2; initialize(); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { diff --git a/test/integration/multi_envoy_test.cc b/test/integration/multi_envoy_test.cc index d050705f6fe5..3d37290cc474 100644 --- a/test/integration/multi_envoy_test.cc +++ b/test/integration/multi_envoy_test.cc @@ -28,8 +28,7 @@ void MultiEnvoyTest::createL1Envoy() { zero.push_back(0); } } - ConfigHelper l1_helper(version_, *api_, - MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper l1_helper(version_, config_helper_.bootstrap()); l1_helper.setPorts(zero, true); // Zero out ports set by config_helper_'s finalize(); const std::string bootstrap_path = finalizeConfigWithPorts(l1_helper, ports, use_lds_); diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 3fadbfd2af8b..cb506c233f37 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -1489,8 +1489,7 @@ TEST_P(QuicInplaceLdsIntegrationTest, ReloadConfigUpdateNonDefaultFilterChain) { makeHttpConnection(makeClientConnectionWithHost(lookupPort("http"), "lyft.com")); // Remove filter_chain_1. - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -1532,8 +1531,7 @@ TEST_P(QuicInplaceLdsIntegrationTest, ReloadConfigUpdateDefaultFilterChain) { makeHttpConnection(makeClientConnectionWithHost(lookupPort("http"), "www.lyft.com")); // Remove filter_chain_1. - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -1552,8 +1550,7 @@ TEST_P(QuicInplaceLdsIntegrationTest, ReloadConfigUpdateDefaultFilterChain) { makeRequestAndWaitForResponse(*codec_client_0); // Modify the default filter chain. - ConfigHelper new_config_helper1( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(new_config_helper.bootstrap())); + ConfigHelper new_config_helper1(version_, new_config_helper.bootstrap()); new_config_helper1.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto default_filter_chain = @@ -1576,9 +1573,7 @@ TEST_P(QuicInplaceLdsIntegrationTest, ReloadConfigUpdateDefaultFilterChain) { makeRequestAndWaitForResponse(*codec_client_1); // Remove the default filter chain. - ConfigHelper new_config_helper2( - version_, *api_, - MessageUtil::getJsonStringFromMessageOrError(new_config_helper1.bootstrap())); + ConfigHelper new_config_helper2(version_, config_helper_.bootstrap()); new_config_helper2.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -1607,8 +1602,7 @@ TEST_P(QuicInplaceLdsIntegrationTest, EnableAndDisableEarlyData) { codec_client_0->close(); // Modify 1st transport socket factory to disable early data. - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addQuicDownstreamTransportSocketConfig(/*enable_early_data=*/false); new_config_helper.setLds("1"); diff --git a/test/integration/tcp_tunneling_integration_test.cc b/test/integration/tcp_tunneling_integration_test.cc index 742e34eb17d5..2b2220997f72 100644 --- a/test/integration/tcp_tunneling_integration_test.cc +++ b/test/integration/tcp_tunneling_integration_test.cc @@ -955,8 +955,7 @@ TEST_P(TcpTunnelingIntegrationTest, HeaderEvaluatorConfigUpdate) { ASSERT_TRUE(tcp_client_->write("hello", false)); ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, 5)); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* header = diff --git a/test/integration/xds_integration_test.cc b/test/integration/xds_integration_test.cc index 41ee5759b0ef..c244d1313694 100644 --- a/test/integration/xds_integration_test.cc +++ b/test/integration/xds_integration_test.cc @@ -235,8 +235,7 @@ TEST_P(LdsInplaceUpdateTcpProxyIntegrationTest, ReloadConfigDeletingFilterChain) FakeRawConnectionPtr fake_upstream_connection_1; ASSERT_TRUE(fake_upstreams_[1]->waitForRawConnection(fake_upstream_connection_1)); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -295,8 +294,7 @@ TEST_P(LdsInplaceUpdateTcpProxyIntegrationTest, ReloadConfigAddingFilterChain) { FakeRawConnectionPtr fake_upstream_connection_0; ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection_0)); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -520,8 +518,7 @@ TEST_P(LdsInplaceUpdateHttpIntegrationTest, ReloadConfigDeletingFilterChain) { auto codec_client_0 = createHttpCodec("alpn0"); auto codec_client_default = createHttpCodec("alpndefault"); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -559,8 +556,7 @@ TEST_P(LdsInplaceUpdateHttpIntegrationTest, ReloadConfigAddingFilterChain) { Cleanup cleanup0([c0 = codec_client_0.get()]() { c0->close(); }); test_server_->waitForGaugeGe("http.hcm0.downstream_cx_active", 1); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -604,8 +600,7 @@ TEST_P(LdsInplaceUpdateHttpIntegrationTest, ReloadConfigUpdatingDefaultFilterCha auto codec_client_default = createHttpCodec("alpndefault"); Cleanup cleanup0([c_default = codec_client_default.get()]() { c_default->close(); }); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto default_filter_chain = @@ -632,8 +627,7 @@ TEST_P(LdsInplaceUpdateHttpIntegrationTest, OverlappingFilterChainServesNewConne auto codec_client_0 = createHttpCodec("alpn0"); Cleanup cleanup([c0 = codec_client_0.get()]() { c0->close(); }); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -680,8 +674,7 @@ TEST_P(LdsIntegrationTest, ReloadConfig) { EXPECT_TRUE(response.find("HTTP/1.1 426 Upgrade Required\r\n") == 0); // Create a new config with HTTP/1.0 proxying. - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { @@ -712,8 +705,7 @@ TEST_P(LdsIntegrationTest, NewListenerWithBadPostListenSocketOption) { // Reserve a port that we can then use on the integration listener with reuse_port. auto addr_socket = Network::Test::bindFreeLoopbackPort(version_, Network::Socket::Type::Stream, true); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* new_listener = bootstrap.mutable_static_resources()->add_listeners(); @@ -764,8 +756,7 @@ TEST_P(LdsStsIntegrationTest, DISABLED_TcpListenerRemoveFilterChainCalledAfterLi FakeRawConnectionPtr fake_upstream_connection_0; ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection_0)); - ConfigHelper new_config_helper( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); new_config_helper.addConfigModifier( [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); @@ -775,28 +766,7 @@ TEST_P(LdsStsIntegrationTest, DISABLED_TcpListenerRemoveFilterChainCalledAfterLi // 2. Remove the tcp listener immediately. This listener update should stack in the same poller // cycle so that this listener update has the same time stamp as the first update. - ConfigHelper new_config_helper1( - version_, *api_, MessageUtil::getJsonStringFromMessageOrError(config_helper_.bootstrap())); - new_config_helper1.addConfigModifier( - [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - bootstrap.mutable_static_resources()->mutable_listeners(0)->Swap( - bootstrap.mutable_static_resources()->mutable_listeners(1)); - bootstrap.mutable_static_resources()->mutable_listeners()->RemoveLast(); - }); - new_config_helper1.setLds("2"); - - std::string observed_data_0; - ASSERT_TRUE(fake_upstream_connection_0->waitForData(5, &observed_data_0)); - EXPECT_EQ("hello", observed_data_0); - - ASSERT_TRUE(fake_upstream_connection_0->write("world")); - while (response_0.find("world") == std::string::npos) { - ASSERT_TRUE(client_conn_0->run(Event::Dispatcher::RunType::NonBlock)); - } - // Wait for the filter chain removal start. - test_server_->waitForGaugeEq("listener_manager.total_filter_chains_draining", 1); - - client_conn_0->close(); + ConfigHelper new_config_helper1(version_, config_helper_.bootstrap()); while (!client_conn_0->closed()) { dispatcher_->run(Event::Dispatcher::RunType::NonBlock); } From 19231ba2b311a115ba85391a18dc73370e1f28d3 Mon Sep 17 00:00:00 2001 From: DiazAlan <109677874+DiazAlan@users.noreply.github.com> Date: Wed, 24 May 2023 13:05:33 -0400 Subject: [PATCH 358/740] Add "Internal" flag to Metric::Flags to allow the ForEach loop to ignore or skip metrics with that flag set (#27293) Creates a special flag "hidden" to use in Metrics. This will allow the ForEach Loops for different metrics to check for this flag and skip flags that have this set. Currently only Gauge has this implemented to help close this PR 23921. Signed-off-by: AlanDiaz --- envoy/stats/stats.h | 9 +++ source/common/stats/allocator_impl.cc | 14 +++++ source/common/stats/histogram_impl.h | 2 + source/common/stats/null_counter.h | 1 + source/common/stats/null_gauge.h | 1 + source/common/stats/null_text_readout.h | 1 + source/common/stats/thread_local_store.cc | 12 +++- source/common/stats/thread_local_store.h | 2 + test/common/stats/allocator_impl_test.cc | 17 +++++ test/common/stats/thread_local_store_test.cc | 66 ++++++++++++++++++++ test/integration/server.h | 1 + test/mocks/stats/mocks.cc | 4 ++ test/mocks/stats/mocks.h | 9 +++ 13 files changed, 137 insertions(+), 2 deletions(-) diff --git a/envoy/stats/stats.h b/envoy/stats/stats.h index 9db2c9dbcac9..cdc1ba896c9b 100644 --- a/envoy/stats/stats.h +++ b/envoy/stats/stats.h @@ -92,6 +92,11 @@ class Metric : public RefcountInterface { */ virtual bool used() const PURE; + /** + * Indicates whether this metric is hidden. + */ + virtual bool hidden() const PURE; + /** * Flags: * Used: used by all stats types to figure out whether they have been used. @@ -101,6 +106,7 @@ class Metric : public RefcountInterface { static constexpr uint8_t Used = 0x01; static constexpr uint8_t LogicAccumulate = 0x02; static constexpr uint8_t NeverImport = 0x04; + static constexpr uint8_t Hidden = 0x08; }; virtual SymbolTable& symbolTable() PURE; virtual const SymbolTable& constSymbolTable() const PURE; @@ -129,10 +135,13 @@ using CounterSharedPtr = RefcountPtr; */ class Gauge : public Metric { public: + // TODO(diazalan): Rename ImportMode to more generic name enum class ImportMode { Uninitialized, // Gauge was discovered during hot-restart transfer. NeverImport, // On hot-restart, each process starts with gauge at 0. Accumulate, // Transfers gauge state on hot-restart. + // TODO(Diazalan): Add functionality for Hidden to be ignored by admin/stats-sink + HiddenAccumulate, // Will be transferred on hot-restart and ignored by admin/stats-sink }; ~Gauge() override = default; diff --git a/source/common/stats/allocator_impl.cc b/source/common/stats/allocator_impl.cc index f69166acdcd2..a8c9745461c2 100644 --- a/source/common/stats/allocator_impl.cc +++ b/source/common/stats/allocator_impl.cc @@ -85,6 +85,7 @@ template class StatsSharedImpl : public MetricImpl // Metric SymbolTable& symbolTable() final { return alloc_.symbolTable(); } bool used() const override { return flags_ & Metric::Flags::Used; } + bool hidden() const override { return flags_ & Metric::Flags::Hidden; } // RefcountInterface void incRefCount() override { ++ref_count_; } @@ -183,6 +184,10 @@ class GaugeImpl : public StatsSharedImpl { // an alternate scope. See // https://github.com/envoyproxy/envoy/issues/7227. break; + case ImportMode::HiddenAccumulate: + flags_ |= Flags::Hidden; + flags_ |= Flags::LogicAccumulate; + break; } } @@ -210,15 +215,19 @@ class GaugeImpl : public StatsSharedImpl { } uint64_t value() const override { return child_value_ + parent_value_; } + // TODO(diazalan): Rename importMode and to more generic name ImportMode importMode() const override { if (flags_ & Flags::NeverImport) { return ImportMode::NeverImport; + } else if ((flags_ & Flags::Hidden) && (flags_ & Flags::LogicAccumulate)) { + return ImportMode::HiddenAccumulate; } else if (flags_ & Flags::LogicAccumulate) { return ImportMode::Accumulate; } return ImportMode::Uninitialized; } + // TODO(diazalan): Rename mergeImportMode and to more generic name void mergeImportMode(ImportMode import_mode) override { ImportMode current = importMode(); if (current == import_mode) { @@ -243,6 +252,11 @@ class GaugeImpl : public StatsSharedImpl { flags_ &= ~Flags::Used; flags_ |= Flags::NeverImport; break; + case ImportMode::HiddenAccumulate: + ASSERT(current == ImportMode::Uninitialized); + flags_ |= Flags::Hidden; + flags_ |= Flags::LogicAccumulate; + break; } } diff --git a/source/common/stats/histogram_impl.h b/source/common/stats/histogram_impl.h index da49db9a436f..32a47980735b 100644 --- a/source/common/stats/histogram_impl.h +++ b/source/common/stats/histogram_impl.h @@ -109,6 +109,7 @@ class HistogramImpl : public HistogramImplHelper { void recordValue(uint64_t value) override { parent_.deliverHistogramToSinks(*this, value); } bool used() const override { return true; } + bool hidden() const override { return false; } SymbolTable& symbolTable() final { return parent_.symbolTable(); } private: @@ -129,6 +130,7 @@ class NullHistogramImpl : public HistogramImplHelper { ~NullHistogramImpl() override { MetricImpl::clear(symbol_table_); } bool used() const override { return false; } + bool hidden() const override { return false; } SymbolTable& symbolTable() override { return symbol_table_; } Unit unit() const override { return Unit::Null; }; diff --git a/source/common/stats/null_counter.h b/source/common/stats/null_counter.h index 074349125a3f..ca576310f0d7 100644 --- a/source/common/stats/null_counter.h +++ b/source/common/stats/null_counter.h @@ -31,6 +31,7 @@ class NullCounterImpl : public MetricImpl { // Metric bool used() const override { return false; } + bool hidden() const override { return false; } SymbolTable& symbolTable() override { return symbol_table_; } // RefcountInterface diff --git a/source/common/stats/null_gauge.h b/source/common/stats/null_gauge.h index 2c12f923aeda..5af09aa5999e 100644 --- a/source/common/stats/null_gauge.h +++ b/source/common/stats/null_gauge.h @@ -35,6 +35,7 @@ class NullGaugeImpl : public MetricImpl { // Metric bool used() const override { return false; } + bool hidden() const override { return false; } SymbolTable& symbolTable() override { return symbol_table_; } // RefcountInterface diff --git a/source/common/stats/null_text_readout.h b/source/common/stats/null_text_readout.h index 0cde15ab5611..3073fa8182b4 100644 --- a/source/common/stats/null_text_readout.h +++ b/source/common/stats/null_text_readout.h @@ -28,6 +28,7 @@ class NullTextReadoutImpl : public MetricImpl { // Metric bool used() const override { return false; } + bool hidden() const override { return false; } SymbolTable& symbolTable() override { return symbol_table_; } // RefcountInterface diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index aee9013d31d0..387cf8ea89d2 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -585,7 +585,8 @@ void ThreadLocalStoreImpl::deliverHistogramToSinks(const Histogram& histogram, u Gauge& ThreadLocalStoreImpl::ScopeImpl::gaugeFromStatNameWithTags( const StatName& name, StatNameTagVectorOptConstRef stat_name_tags, Gauge::ImportMode import_mode) { - if (parent_.rejectsAll()) { + // If a gauge is "hidden" it should not be rejected as these are used for deferred stats. + if (parent_.rejectsAll() && import_mode != Gauge::ImportMode::HiddenAccumulate) { return parent_.null_gauge_; } @@ -594,7 +595,12 @@ Gauge& ThreadLocalStoreImpl::ScopeImpl::gaugeFromStatNameWithTags( TagUtility::TagStatNameJoiner joiner(prefix_.statName(), name, stat_name_tags, symbolTable()); StatName final_stat_name = joiner.nameWithTags(); - StatsMatcher::FastResult fast_reject_result = parent_.fastRejects(final_stat_name); + StatsMatcher::FastResult fast_reject_result; + if (import_mode != Gauge::ImportMode::HiddenAccumulate) { + fast_reject_result = parent_.fastRejects(final_stat_name); + } else { + fast_reject_result = StatsMatcher::FastResult::Matches; + } if (fast_reject_result == StatsMatcher::FastResult::Rejects) { return parent_.null_gauge_; } @@ -906,6 +912,8 @@ bool ParentHistogramImpl::used() const { return merged_; } +bool ParentHistogramImpl::hidden() const { return false; } + void ParentHistogramImpl::merge() { Thread::ReleasableLockGuard lock(merge_lock_); if (merged_ || usedLockHeld()) { diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index 46bd0afa1032..081aee90fbe4 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -60,6 +60,7 @@ class ThreadLocalHistogramImpl : public HistogramImplHelper { // Stats::Metric SymbolTable& symbolTable() final { return symbol_table_; } bool used() const override { return used_; } + bool hidden() const override { return false; } private: Histogram::Unit unit_; @@ -115,6 +116,7 @@ class ParentHistogramImpl : public MetricImpl { // Stats::Metric SymbolTable& symbolTable() override; bool used() const override; + bool hidden() const override; // RefcountInterface void incRefCount() override; diff --git a/test/common/stats/allocator_impl_test.cc b/test/common/stats/allocator_impl_test.cc index bbb0da55f25e..eea975dcb3ea 100644 --- a/test/common/stats/allocator_impl_test.cc +++ b/test/common/stats/allocator_impl_test.cc @@ -139,6 +139,23 @@ TEST_F(AllocatorImplTest, RefCountDecAllocRaceSynchronized) { EXPECT_FALSE(alloc_.isMutexLockedForTest()); } +TEST_F(AllocatorImplTest, HiddenGauge) { + GaugeSharedPtr hidden_gauge = + alloc_.makeGauge(makeStat("hidden"), StatName(), {}, Gauge::ImportMode::HiddenAccumulate); + EXPECT_EQ(hidden_gauge->importMode(), Gauge::ImportMode::HiddenAccumulate); + EXPECT_TRUE(hidden_gauge->hidden()); + + GaugeSharedPtr non_hidden_gauge = + alloc_.makeGauge(makeStat("non_hidden"), StatName(), {}, Gauge::ImportMode::Accumulate); + EXPECT_NE(non_hidden_gauge->importMode(), Gauge::ImportMode::HiddenAccumulate); + EXPECT_FALSE(non_hidden_gauge->hidden()); + + GaugeSharedPtr never_import_hidden_gauge = alloc_.makeGauge( + makeStat("never_import_hidden"), StatName(), {}, Gauge::ImportMode::NeverImport); + EXPECT_NE(never_import_hidden_gauge->importMode(), Gauge::ImportMode::HiddenAccumulate); + EXPECT_FALSE(never_import_hidden_gauge->hidden()); +} + TEST_F(AllocatorImplTest, ForEachCounter) { StatNameHashSet stat_names; std::vector counters; diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 0363a4cd0219..9cd079074338 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -1149,6 +1149,72 @@ TEST_F(StatsMatcherTLSTest, RejectPrefixNoDot) { EXPECT_MEMORY_LE(mem_consumed, 3500000); } +TEST_F(StatsMatcherTLSTest, DoNotRejectHiddenPrefixExclusion) { + envoy::config::metrics::v3::StatsConfig stats_config_; + stats_config_.mutable_stats_matcher()->mutable_exclusion_list()->add_patterns()->set_prefix( + "cluster."); + store_->setStatsMatcher(std::make_unique(stats_config_, symbol_table_)); + + Gauge& accumulate_gauge = + scope_.gaugeFromString("cluster.accumulate_gauge", Gauge::ImportMode::Accumulate); + EXPECT_EQ(accumulate_gauge.name(), ""); + Gauge& hidden_gauge = + scope_.gaugeFromString("cluster.hidden_gauge", Gauge::ImportMode::HiddenAccumulate); + EXPECT_EQ(hidden_gauge.name(), "cluster.hidden_gauge"); +} + +TEST_F(StatsMatcherTLSTest, DoNotRejectHiddenPrefixInclusive) { + envoy::config::metrics::v3::StatsConfig stats_config_; + stats_config_.mutable_stats_matcher()->mutable_inclusion_list()->add_patterns()->set_prefix( + "cluster."); + store_->setStatsMatcher(std::make_unique(stats_config_, symbol_table_)); + + Gauge& accumulate_gauge = + scope_.gaugeFromString("accumulate_gauge", Gauge::ImportMode::Accumulate); + EXPECT_EQ(accumulate_gauge.name(), ""); + Gauge& hidden_gauge = scope_.gaugeFromString("hidden_gauge", Gauge::ImportMode::HiddenAccumulate); + EXPECT_EQ(hidden_gauge.name(), "hidden_gauge"); +} + +TEST_F(StatsMatcherTLSTest, DoNotRejectHiddenExclusionRegex) { + envoy::config::metrics::v3::StatsConfig stats_config_; + stats_config_.mutable_stats_matcher()->mutable_exclusion_list()->add_patterns()->MergeFrom( + TestUtility::createRegexMatcher(".*")); + store_->setStatsMatcher(std::make_unique(stats_config_, symbol_table_)); + + Gauge& accumulate_gauge = + scope_.gaugeFromString("accumulate_gauge", Gauge::ImportMode::Accumulate); + EXPECT_EQ(accumulate_gauge.name(), ""); + Gauge& hidden_gauge = scope_.gaugeFromString("hidden_gauge", Gauge::ImportMode::HiddenAccumulate); + EXPECT_EQ(hidden_gauge.name(), "hidden_gauge"); +} + +TEST_F(StatsMatcherTLSTest, DoNotRejectHiddenInclusionRegex) { + envoy::config::metrics::v3::StatsConfig stats_config_; + // Create inclusion list to only accept names that have at least one capital letter. + stats_config_.mutable_stats_matcher()->mutable_inclusion_list()->add_patterns()->MergeFrom( + TestUtility::createRegexMatcher(".*[A-Z].*")); + store_->setStatsMatcher(std::make_unique(stats_config_, symbol_table_)); + + Gauge& accumulate_gauge = + scope_.gaugeFromString("accumulate_gauge", Gauge::ImportMode::Accumulate); + EXPECT_EQ(accumulate_gauge.name(), ""); + Gauge& hidden_gauge = scope_.gaugeFromString("hidden_gauge", Gauge::ImportMode::HiddenAccumulate); + EXPECT_EQ(hidden_gauge.name(), "hidden_gauge"); +} + +TEST_F(StatsMatcherTLSTest, DoNotRejectAllHidden) { + envoy::config::metrics::v3::StatsConfig stats_config_; + stats_config_.mutable_stats_matcher()->set_reject_all(true); + store_->setStatsMatcher(std::make_unique(stats_config_, symbol_table_)); + + Gauge& accumulate_gauge = + scope_.gaugeFromString("accumulate_gauge", Gauge::ImportMode::Accumulate); + EXPECT_EQ(accumulate_gauge.name(), ""); + Gauge& hidden_gauge = scope_.gaugeFromString("hidden_gauge", Gauge::ImportMode::HiddenAccumulate); + EXPECT_EQ(hidden_gauge.name(), "hidden_gauge"); +} + // Tests the logic for caching the stats-matcher results, and in particular the // private impl method checkAndRememberRejection(). That method behaves // differently depending on whether TLS is enabled or not, so we parameterize diff --git a/test/integration/server.h b/test/integration/server.h index 657396ccf4eb..78b8e7c8b091 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -193,6 +193,7 @@ class NotifyingCounter : public Stats::Counter { uint32_t use_count() const override { return counter_->use_count(); } StatName tagExtractedStatName() const override { return counter_->tagExtractedStatName(); } bool used() const override { return counter_->used(); } + bool hidden() const override { return counter_->hidden(); } SymbolTable& symbolTable() override { return counter_->symbolTable(); } const SymbolTable& constSymbolTable() const override { return counter_->constSymbolTable(); } diff --git a/test/mocks/stats/mocks.cc b/test/mocks/stats/mocks.cc index b1239195c284..d0108548aaa8 100644 --- a/test/mocks/stats/mocks.cc +++ b/test/mocks/stats/mocks.cc @@ -19,6 +19,7 @@ namespace Stats { MockCounter::MockCounter() { ON_CALL(*this, used()).WillByDefault(ReturnPointee(&used_)); + ON_CALL(*this, hidden()).WillByDefault(ReturnPointee(&hidden_)); ON_CALL(*this, value()).WillByDefault(ReturnPointee(&value_)); ON_CALL(*this, latch()).WillByDefault(ReturnPointee(&latch_)); } @@ -26,6 +27,7 @@ MockCounter::~MockCounter() = default; MockGauge::MockGauge() : used_(false), value_(0), import_mode_(ImportMode::Accumulate) { ON_CALL(*this, used()).WillByDefault(ReturnPointee(&used_)); + ON_CALL(*this, hidden()).WillByDefault(ReturnPointee(&hidden_)); ON_CALL(*this, value()).WillByDefault(ReturnPointee(&value_)); ON_CALL(*this, importMode()).WillByDefault(ReturnPointee(&import_mode_)); } @@ -33,6 +35,7 @@ MockGauge::~MockGauge() = default; MockTextReadout::MockTextReadout() { ON_CALL(*this, used()).WillByDefault(ReturnPointee(&used_)); + ON_CALL(*this, hidden()).WillByDefault(ReturnPointee(&hidden_)); ON_CALL(*this, value()).WillByDefault(ReturnPointee(&value_)); } MockTextReadout::~MockTextReadout() = default; @@ -49,6 +52,7 @@ MockHistogram::~MockHistogram() = default; MockParentHistogram::MockParentHistogram() { ON_CALL(*this, used()).WillByDefault(ReturnPointee(&used_)); + ON_CALL(*this, hidden()).WillByDefault(ReturnPointee(&hidden_)); ON_CALL(*this, unit()).WillByDefault(ReturnPointee(&unit_)); ON_CALL(*this, recordValue(_)).WillByDefault(Invoke([this](uint64_t value) { if (store_ != nullptr) { diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h index 2a145792f50c..008caa3d9f9f 100644 --- a/test/mocks/stats/mocks.h +++ b/test/mocks/stats/mocks.h @@ -142,9 +142,11 @@ class MockCounter : public MockStatWithRefcount { MOCK_METHOD(uint64_t, latch, ()); MOCK_METHOD(void, reset, ()); MOCK_METHOD(bool, used, (), (const)); + MOCK_METHOD(bool, hidden, (), (const)); MOCK_METHOD(uint64_t, value, (), (const)); bool used_; + bool hidden_; uint64_t value_; uint64_t latch_; }; @@ -162,11 +164,13 @@ class MockGauge : public MockStatWithRefcount { MOCK_METHOD(void, sub, (uint64_t amount)); MOCK_METHOD(void, mergeImportMode, (ImportMode)); MOCK_METHOD(bool, used, (), (const)); + MOCK_METHOD(bool, hidden, (), (const)); MOCK_METHOD(uint64_t, value, (), (const)); MOCK_METHOD(absl::optional, cachedShouldImport, (), (const)); MOCK_METHOD(ImportMode, importMode, (), (const)); bool used_; + bool hidden_; uint64_t value_; ImportMode import_mode_; }; @@ -177,6 +181,7 @@ class MockHistogram : public MockMetric { ~MockHistogram() override; MOCK_METHOD(bool, used, (), (const)); + MOCK_METHOD(bool, hidden, (), (const)); MOCK_METHOD(Histogram::Unit, unit, (), (const)); MOCK_METHOD(void, recordValue, (uint64_t value)); @@ -201,6 +206,7 @@ class MockParentHistogram : public MockMetric { std::string quantileSummary() const override { return ""; }; std::string bucketSummary() const override { return ""; }; MOCK_METHOD(bool, used, (), (const)); + MOCK_METHOD(bool, hidden, (), (const)); MOCK_METHOD(Histogram::Unit, unit, (), (const)); MOCK_METHOD(void, recordValue, (uint64_t value)); MOCK_METHOD(const HistogramStatistics&, cumulativeStatistics, (), (const)); @@ -214,6 +220,7 @@ class MockParentHistogram : public MockMetric { uint32_t use_count() const override { return refcount_helper_.use_count(); } bool used_; + bool hidden_; Unit unit_{Histogram::Unit::Unspecified}; Store* store_{}; std::shared_ptr histogram_stats_ = @@ -230,9 +237,11 @@ class MockTextReadout : public MockMetric { MOCK_METHOD(void, set, (absl::string_view value), (override)); MOCK_METHOD(bool, used, (), (const, override)); + MOCK_METHOD(bool, hidden, (), (const)); MOCK_METHOD(std::string, value, (), (const, override)); bool used_; + bool hidden_; std::string value_; }; From 5b270c2f2a14ea4eac609bf855edcb8c051c2a39 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 24 May 2023 21:52:39 +0100 Subject: [PATCH 359/740] ci/fuzz: Ensure fuzz check is included in code check (#27615) Signed-off-by: Ryan Northey --- .../filters/network/common/fuzz/config.bzl | 6 +++++- tools/base/requirements.in | 2 +- tools/base/requirements.txt | 6 +++--- tools/code/BUILD | 15 +++++++++++++-- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/test/extensions/filters/network/common/fuzz/config.bzl b/test/extensions/filters/network/common/fuzz/config.bzl index c275f3dfed79..44682ce1ad72 100644 --- a/test/extensions/filters/network/common/fuzz/config.bzl +++ b/test/extensions/filters/network/common/fuzz/config.bzl @@ -10,6 +10,10 @@ READFILTER_FUZZ_FILTERS = [ "envoy.filters.network.local_ratelimit", "envoy.filters.network.ratelimit", "envoy.filters.network.rbac", +] + +# These are marked as robust to downstream, but not currently fuzzed +READFILTER_NOFUZZ_FILTERS = [ # TODO(asraa): Remove when fuzzer sets up connections for TcpProxy properly. - # "envoy.filters.network.tcp_proxy", + "envoy.filters.network.tcp_proxy", ] diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 361ce48bf0bd..69fd8d5d441f 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -6,7 +6,7 @@ colorama coloredlogs dependatool>=0.2.2 envoy.base.utils>=0.4.10 -envoy.code.check>=0.5.1 +envoy.code.check>=0.5.4 envoy.dependency.check>=0.1.7 envoy.distribution.release>=0.0.9 envoy.distribution.repo>=0.0.8 diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 34743052ec71..ab3a456306a0 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -434,9 +434,9 @@ envoy-base-utils==0.4.11 \ # envoy-docs-sphinx-runner # envoy-github-release # envoy-gpg-sign -envoy-code-check==0.5.1 \ - --hash=sha256:379162ed8d83c9cb045917269b58f6e59812523359d8b0772b35078ab06dc8c1 \ - --hash=sha256:4b2b11b4fff634f1268b8dfb754c1d99300b8e58f5bc4276c96a8eb30cb9cea6 +envoy-code-check==0.5.4 \ + --hash=sha256:b3c338a0e607960ea75eb8298e786548d317655ac4c89d89b259395684eaf134 \ + --hash=sha256:ec919ea1e5523c5ad669f6601bb58c8da77bc1891c8846950add3b563c629ac5 # via -r requirements.in envoy-dependency-check==0.1.8 \ --hash=sha256:ac9820e446bb44e05121e5c93c210f40ca37076580b0d082da2c63e7784c338a \ diff --git a/tools/code/BUILD b/tools/code/BUILD index 1af762c87d1d..1b800c6a6dee 100644 --- a/tools/code/BUILD +++ b/tools/code/BUILD @@ -5,12 +5,18 @@ load("@aspect_bazel_lib//lib:jq.bzl", "jq") load( "//test/extensions/filters/network/common/fuzz:config.bzl", "READFILTER_FUZZ_FILTERS", + "READFILTER_NOFUZZ_FILTERS", ) licenses(["notice"]) # Apache 2 envoy_package() +FUZZ_FILTER_COUNT = ( + len(READFILTER_FUZZ_FILTERS) + + len(READFILTER_NOFUZZ_FILTERS) +) + jq( name = "extensions_build_config", srcs = [ @@ -26,7 +32,7 @@ envoy_entry_point( name = "check", args = [ "--extensions_build_config=$(location :extensions_build_config)", - "--extensions_fuzzed_count=%s" % len(READFILTER_FUZZ_FILTERS), + "--extensions_fuzzed_count=%s" % FUZZ_FILTER_COUNT, "--path=%s" % PATH, "-b shellcheck:$(location @com_github_aignas_rules_shellcheck//:shellcheck)", "-b gofmt:$(location @go_sdk//:bin/gofmt)", @@ -45,14 +51,19 @@ genrule( cmd = """ $(location :check) \ --fix \ + -l warn \ -v warn \ -x mobile/dist/envoy-pom.xml \ --extensions_build_config=$(location :extensions_build_config) \ + --extensions_fuzzed_count=%s \ --path=%s \ -b shellcheck:$(location @com_github_aignas_rules_shellcheck//:shellcheck) \ -b gofmt:$(location @go_sdk//:bin/gofmt) \ > $@ 2>&1 || : - """ % PATH, + """ % ( + FUZZ_FILTER_COUNT, + PATH, + ), tags = ["no-remote-exec"], tools = [ ":check", From eac62b6462c3b4a0e91dffb012ca81ec3df3e92b Mon Sep 17 00:00:00 2001 From: Loong Dai Date: Thu, 25 May 2023 15:05:28 +0800 Subject: [PATCH 360/740] dlb: support virtual DLB devices (#27338) Signed-off-by: Loong --- bazel/foreign_cc/dlb.patch | 22 ---------------- bazel/repositories.bzl | 2 -- bazel/repository_locations.bzl | 8 +++--- .../dlb/source/connection_balancer_impl.cc | 25 +++++++++---------- .../dlb/source/connection_balancer_impl.h | 5 ++-- 5 files changed, 19 insertions(+), 43 deletions(-) delete mode 100644 bazel/foreign_cc/dlb.patch diff --git a/bazel/foreign_cc/dlb.patch b/bazel/foreign_cc/dlb.patch deleted file mode 100644 index ca7649f6a0ee..000000000000 --- a/bazel/foreign_cc/dlb.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/dlb/libdlb/dlb_ops.h b/dlb/libdlb/dlb_ops.h -index 3f384f8..bd621be 100644 ---- a/dlb/libdlb/dlb_ops.h -+++ b/dlb/libdlb/dlb_ops.h -@@ -33,13 +33,13 @@ static void dlb_movntdq(struct dlb_enqueue_qe *qe4, uint64_t *pp_addr) - __v2di srcData1 = (__v2di){_qe[2], _qe[3]}; - __v2di srcData2 = (__v2di){_qe[4], _qe[5]}; - __v2di srcData3 = (__v2di){_qe[6], _qe[7]}; -- __builtin_ia32_movntdq((__v2di *) pp_addr, (__v2di) srcData0); -+ __builtin_nontemporal_store((__v2di) srcData0, (__v2di *) pp_addr); - _mm_sfence(); -- __builtin_ia32_movntdq((__v2di *) pp_addr, (__v2di) srcData1); -+ __builtin_nontemporal_store((__v2di) srcData1, (__v2di *) pp_addr); - _mm_sfence(); -- __builtin_ia32_movntdq((__v2di *) pp_addr, (__v2di) srcData2); -+ __builtin_nontemporal_store((__v2di) srcData2, (__v2di *) pp_addr); - _mm_sfence(); -- __builtin_ia32_movntdq((__v2di *) pp_addr, (__v2di) srcData3); -+ __builtin_nontemporal_store((__v2di) srcData3, (__v2di *) pp_addr); - /* movntdq requires an sfence between writes to the PP MMIO address */ - _mm_sfence(); - } diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index ec0b65060918..c77baa9b132c 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1260,8 +1260,6 @@ filegroup( visibility = ["@envoy//contrib/network/connection_balance/dlb/source:__pkg__"], ) """, - patch_args = ["-p1"], - patches = ["@envoy//bazel/foreign_cc:dlb.patch"], ) def _rules_fuzzing(): diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index cb8ff5395b5c..387ae12c3aaa 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1350,12 +1350,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Intel Dlb", project_desc = "Dlb", project_url = "https://networkbuilders.intel.com/solutionslibrary/queue-management-and-load-balancing-on-intel-architecture", - version = "7.8.0", - sha256 = "9070388d9a22055ca37c7e1ec6a4b55c6bfd829b1a99b860056af268b68c98b6", - urls = ["https://downloadmirror.intel.com/744007/dlb_linux_src_release_7.8.0_2022_09_20.txz"], + version = "8.0.0", + sha256 = "075533229bb2bd2f945ec8089a707205f3f8e8d87a8030e9208603d997236171", + urls = ["https://downloadmirror.intel.com/763709/dlb_linux_src_release8.0.0.txz"], use_category = ["dataplane_ext"], extensions = ["envoy.network.connection_balance.dlb"], - release_date = "2022-03-30", + release_date = "2022-12-15", cpe = "N/A", ), rules_license = dict( diff --git a/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc b/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc index 1e5fd46d5bb9..03da81765524 100644 --- a/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc +++ b/contrib/network/connection_balance/dlb/source/connection_balancer_impl.cc @@ -27,9 +27,9 @@ DlbConnectionBalanceFactory::createConnectionBalancerFromProto( fmt::format("unexpected dlb config: {}", typed_config.DebugString())); } - const int num = context.options().concurrency(); + const uint32_t worker_num = context.options().concurrency(); - if (num > 32) { + if (worker_num > 32) { ExceptionUtil::throwEnvoyException( "Dlb connection balanncer only supports up to 32 worker threads, " "please decrease the number of threads by `--concurrency`"); @@ -67,21 +67,20 @@ DlbConnectionBalanceFactory::createConnectionBalancerFromProto( } ENVOY_LOG(debug, - "dlb available resources: {}, domains: {}, LDB queues: {}, LDB ports: {}, SN slots: {} " - "{}, ES entries: {}, Contig ES entries: {}, LDB credits: {}, Contig LDB cred: {}, LDB " - "credit pls: {}", + "dlb available resources: domains: {}, LDB queues: {}, LDB ports: {}, " + "ES entries: {}, Contig ES entries: {}, LDB credits: {}, Config LDB credits: {}, LDB " + "credit pools: {}", rsrcs.num_sched_domains, rsrcs.num_ldb_queues, rsrcs.num_ldb_ports, - rsrcs.num_sn_slots[0], rsrcs.num_sn_slots[1], rsrcs.num_ldb_event_state_entries, - rsrcs.max_contiguous_ldb_event_state_entries, rsrcs.num_ldb_credits, - rsrcs.max_contiguous_ldb_credits, rsrcs.num_ldb_credit_pools); + rsrcs.num_ldb_event_state_entries, rsrcs.max_contiguous_ldb_event_state_entries, + rsrcs.num_ldb_credits, rsrcs.max_contiguous_ldb_credits, rsrcs.num_ldb_credit_pools); - uint num_seq_numbers; - if (dlb_get_ldb_sequence_number_allocation(dlb, 0, &num_seq_numbers)) { + if (rsrcs.num_ldb_ports < 2 * worker_num) { ExceptionUtil::throwEnvoyException( - fmt::format("dlb_get_ldb_sequence_number_allocation {}", errorDetails(errno))); + fmt::format("no available dlb port resources, request: {}, available: {}", 2 * worker_num, + rsrcs.num_ldb_ports)); } - domain_id = createSchedDomain(dlb, rsrcs, cap); + domain_id = createSchedDomain(dlb, rsrcs, cap, 2 * worker_num); if (domain_id == -1) { ExceptionUtil::throwEnvoyException( fmt::format("dlb_create_sched_domain_ {}", errorDetails(errno))); @@ -127,7 +126,7 @@ DlbConnectionBalanceFactory::createConnectionBalancerFromProto( ExceptionUtil::throwEnvoyException(fmt::format("tx create_ldb_queue {}", errorDetails(errno))); } - for (int i = 0; i < num; i++) { + for (uint i = 0; i < worker_num; i++) { int tx_port_id = createLdbPort(domain, cap, ldb_pool_id, dir_pool_id); if (tx_port_id == -1) { ExceptionUtil::throwEnvoyException( diff --git a/contrib/network/connection_balance/dlb/source/connection_balancer_impl.h b/contrib/network/connection_balance/dlb/source/connection_balancer_impl.h index adeb440b65f9..e046e9cc44e0 100644 --- a/contrib/network/connection_balance/dlb/source/connection_balancer_impl.h +++ b/contrib/network/connection_balance/dlb/source/connection_balancer_impl.h @@ -124,12 +124,13 @@ class DlbConnectionBalanceFactory : public Envoy::Network::ConnectionBalanceFact return dlb_create_ldb_port(domain, &args); } - static int createSchedDomain(dlb_hdl_t dlb, dlb_resources_t rsrcs, dlb_dev_cap_t cap) { + static int createSchedDomain(dlb_hdl_t dlb, dlb_resources_t rsrcs, dlb_dev_cap_t cap, + uint32_t num_ldb_ports) { int p_rsrsc = 100; dlb_create_sched_domain_t args; args.num_ldb_queues = 1; - args.num_ldb_ports = 64; + args.num_ldb_ports = num_ldb_ports; args.num_dir_ports = 0; args.num_ldb_event_state_entries = 2 * args.num_ldb_ports * cq_depth; if (!cap.combined_credits) { From 0953f28aef26021b29654ce3d94d324924dcdea3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 08:16:03 +0100 Subject: [PATCH 361/740] build(deps): bump redis from `47e4e43` to `f972469` in /examples/redis (#27597) Bumps redis from `47e4e43` to `f972469`. --- updated-dependencies: - dependency-name: redis dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/redis/Dockerfile-redis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/redis/Dockerfile-redis b/examples/redis/Dockerfile-redis index e05ddfb56f9f..aa02190f6263 100644 --- a/examples/redis/Dockerfile-redis +++ b/examples/redis/Dockerfile-redis @@ -1 +1 @@ -FROM redis@sha256:47e4e4395af10811cfd893e3e2dcfdf13cf45b0d4feeeafc92842d6902656ede +FROM redis@sha256:f9724694a0b97288d2255ff2b69642dfba7f34c8e41aaf0a59d33d10d8a42687 From df984fde92a34554d45596484a0e7093369076eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 08:21:14 +0100 Subject: [PATCH 362/740] build(deps): bump postgres from `b115fe7` to `31c9342` in /examples/shared/postgres (#27598) build(deps): bump postgres in /examples/shared/postgres Bumps postgres from `b115fe7` to `31c9342`. --- updated-dependencies: - dependency-name: postgres dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile index 84b2d96a722c..523758fea8e5 100644 --- a/examples/shared/postgres/Dockerfile +++ b/examples/shared/postgres/Dockerfile @@ -1,3 +1,3 @@ -FROM postgres:latest@sha256:b115fe7743919fff0b96f19405ed350017ef9333b3760fc0cc885713eb07f565 +FROM postgres:latest@sha256:31c9342603866f29206a06b77c8fed48b3c3f70d710a4be4e8216b134f92d0df COPY docker-healthcheck.sh /usr/local/bin/ HEALTHCHECK CMD ["docker-healthcheck.sh"] From 268bbf8701f27258882ded520d748fb90d2c9e45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 08:23:07 +0100 Subject: [PATCH 363/740] build(deps): bump python from `6c0b067` to `eaee5f7` in /examples/shared/python (#27602) build(deps): bump python in /examples/shared/python Bumps python from `6c0b067` to `eaee5f7`. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/python/Dockerfile b/examples/shared/python/Dockerfile index 8a87913da3aa..d64811d72d2d 100644 --- a/examples/shared/python/Dockerfile +++ b/examples/shared/python/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.3-slim-bullseye@sha256:6c0b067294c78ee6538cc9d6163fd1653797d0f4fd567e55703a5365c7757138 as python-base +FROM python:3.11.3-slim-bullseye@sha256:eaee5f73efa9ae962d2077756292bc4878c04fcbc13dc168bb00cc365f35647e as python-base RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache ARG PYTHON_REQUIREMENTS_FILE=aiohttp/requirements.txt From 0d59221ba32ddefd89903a6344db7ff06f59c36c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 08:30:12 +0100 Subject: [PATCH 364/740] build(deps): bump node from `95a950e` to `dc19067` in /examples/shared/node (#27604) build(deps): bump node in /examples/shared/node Bumps node from `95a950e` to `dc19067`. --- updated-dependencies: - dependency-name: node dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/node/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile index 9ef656782642..eba7faacef82 100644 --- a/examples/shared/node/Dockerfile +++ b/examples/shared/node/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.2-bullseye-slim@sha256:95a950ec61796f4c00f6b208cb51000b8bd127ee53b0c1c52f2539a5ab66f8ef as node-base +FROM node:20.2-bullseye-slim@sha256:dc1906714d1993d291e1e7b5f236291236b0a0b6dfacdb164e4a9ea44d09c52e as node-base FROM node-base as node-http-auth From ffa5f802745b463792dd9ae2d584288764eed451 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 08:34:42 +0100 Subject: [PATCH 365/740] build(deps): bump golang from `2dc5c56` to `918857f` in /examples/shared/golang (#27605) build(deps): bump golang in /examples/shared/golang Bumps golang from `2dc5c56` to `918857f`. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/golang/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile index c439916f78ea..17bff24cf5a4 100644 --- a/examples/shared/golang/Dockerfile +++ b/examples/shared/golang/Dockerfile @@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache -FROM golang:1.20.4-bullseye@sha256:2dc5c568c8a314583090e887e8d96d313e081e2c5333d0a7b935906baf77cee9 as golang-base +FROM golang:1.20.4-bullseye@sha256:918857f4064db0fff49799ce5e7c4d43e394f452111cd89cca9af539c18a76a8 as golang-base FROM golang-base as golang-control-plane-builder From 9005e80b33787d850bb4498382aaa5b7aa28f21b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 08:44:00 +0100 Subject: [PATCH 366/740] build(deps): bump otel/opentelemetry-collector from `e5bd89e` to `5c3a03d` in /examples/opentelemetry (#27603) build(deps): bump otel/opentelemetry-collector Bumps otel/opentelemetry-collector from `e5bd89e` to `5c3a03d`. --- updated-dependencies: - dependency-name: otel/opentelemetry-collector dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/opentelemetry/Dockerfile-opentelemetry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry index 4d05281c3000..177882fbdf92 100644 --- a/examples/opentelemetry/Dockerfile-opentelemetry +++ b/examples/opentelemetry/Dockerfile-opentelemetry @@ -1,7 +1,7 @@ FROM alpine:3.18@sha256:02bb6f428431fbc2809c5d1b41eab5a68350194fb508869a33cb1af4444c9b11 as otelc_curl RUN apk --update add curl -FROM otel/opentelemetry-collector:latest@sha256:e5bd89e5ec7cd5bb3ff01fbd811e3730be75d8725405ae0c0d282cc69b9597e0 +FROM otel/opentelemetry-collector:latest@sha256:5c3a03dac40b1f37069c63a2ee75d7bd8c7ba81fc42831085053b1c70e528349 COPY --from=otelc_curl / / From 9231baba699a58b04db10a6fb804cf06fb9f04e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 09:08:34 +0100 Subject: [PATCH 367/740] build(deps): bump actions/setup-python from 4.6.0 to 4.6.1 (#27627) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.6.0 to 4.6.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/57ded4d7d5e986d7296eab16560982c6dd7c923b...bd6b4b6205c4dbad673328db7b31b7fab9e241c0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/check-deps.yml | 2 +- .github/workflows/pr_notifier.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-deps.yml b/.github/workflows/check-deps.yml index 8754308add19..1744e3fa332e 100644 --- a/.github/workflows/check-deps.yml +++ b/.github/workflows/check-deps.yml @@ -21,7 +21,7 @@ jobs: with: ref: ${{ github.head_ref }} - name: Set up Python (3.10) - uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b + uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 with: python-version: "3.10" diff --git a/.github/workflows/pr_notifier.yml b/.github/workflows/pr_notifier.yml index dab80b2579f2..782ad94075da 100644 --- a/.github/workflows/pr_notifier.yml +++ b/.github/workflows/pr_notifier.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python 3.8 - uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b + uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 with: python-version: '3.8' architecture: 'x64' From c80665747981281ea0d0897f718f3542ec45cfd5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 09:28:16 +0100 Subject: [PATCH 368/740] build(deps): bump github/codeql-action from 2.3.3 to 2.3.4 (#27625) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.3.3 to 2.3.4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/29b1f65c5e92e24fe6b6647da1eaabe529cec70f...f0e3dfb30302f8a0881bb509b044e0de4f6ef589) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 78225d327520..660eec1f12a5 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -33,7 +33,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@29b1f65c5e92e24fe6b6647da1eaabe529cec70f + uses: github/codeql-action/init@f0e3dfb30302f8a0881bb509b044e0de4f6ef589 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -64,4 +64,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@29b1f65c5e92e24fe6b6647da1eaabe529cec70f + uses: github/codeql-action/analyze@f0e3dfb30302f8a0881bb509b044e0de4f6ef589 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index 183c47767366..fcb710edb6e4 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -45,7 +45,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@29b1f65c5e92e24fe6b6647da1eaabe529cec70f + uses: github/codeql-action/init@f0e3dfb30302f8a0881bb509b044e0de4f6ef589 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -78,4 +78,4 @@ jobs: - name: Perform CodeQL Analysis if: env.BUILD_TARGETS != '' - uses: github/codeql-action/analyze@29b1f65c5e92e24fe6b6647da1eaabe529cec70f + uses: github/codeql-action/analyze@f0e3dfb30302f8a0881bb509b044e0de4f6ef589 From acc3baed11a610db97bf8aab6df34934ace469ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 09:53:52 +0100 Subject: [PATCH 369/740] build(deps): bump mysql from `a43f6e7` to `d6164ff` in /examples/mysql (#27628) Bumps mysql from `a43f6e7` to `d6164ff`. --- updated-dependencies: - dependency-name: mysql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/mysql/Dockerfile-mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mysql/Dockerfile-mysql b/examples/mysql/Dockerfile-mysql index eb0333f639fa..de8c80df6a3e 100644 --- a/examples/mysql/Dockerfile-mysql +++ b/examples/mysql/Dockerfile-mysql @@ -1 +1 @@ -FROM mysql:8.0.33@sha256:a43f6e7e7f3a5e5b90f857fbed4e3103ece771b19f0f75880f767cf66bbb6577 +FROM mysql:8.0.33@sha256:d6164ff4855b9b3f2c7748c6ec564ccff841f79a7023db0f9293143481a44b6e From d4e8e0bff6fd76e76bf4a4073b2cdfe9fa579dcc Mon Sep 17 00:00:00 2001 From: StarryNight Date: Thu, 25 May 2023 19:39:26 +0800 Subject: [PATCH 370/740] add go extension filter state set api (#27513) * add go extension filter state set api Signed-off-by: wangkai19 * fix review comments Signed-off-by: wangkai19 * fix format Signed-off-by: wangkai19 * optimize string copy Signed-off-by: wangkai19 --------- Signed-off-by: wangkai19 --- contrib/golang/filters/http/source/cgo.cc | 13 ++++++ .../filters/http/source/go/pkg/api/api.h | 3 ++ .../filters/http/source/go/pkg/api/capi.go | 2 + .../filters/http/source/go/pkg/api/filter.go | 30 ++++++++++++++ .../http/source/go/pkg/http/capi_impl.go | 5 +++ .../filters/http/source/go/pkg/http/filter.go | 14 +++++++ .../filters/http/source/golang_filter.cc | 41 +++++++++++++++++++ .../filters/http/source/golang_filter.h | 11 +++++ .../http/test/golang_integration_test.cc | 2 + .../http/test/test_data/basic/filter.go | 3 ++ 10 files changed, 124 insertions(+) diff --git a/contrib/golang/filters/http/source/cgo.cc b/contrib/golang/filters/http/source/cgo.cc index 9b263471e55b..56742609932a 100644 --- a/contrib/golang/filters/http/source/cgo.cc +++ b/contrib/golang/filters/http/source/cgo.cc @@ -200,6 +200,19 @@ CAPIStatus envoyGoFilterHttpSendPanicReply(void* r, void* details) { }); } +CAPIStatus envoyGoFilterHttpSetStringFilterState(void* r, void* key, void* value, int state_type, + int life_span, int stream_sharing) { + return envoyGoFilterHandlerWrapper(r, + [key, value, state_type, life_span, stream_sharing]( + std::shared_ptr& filter) -> CAPIStatus { + auto key_str = referGoString(key); + auto value_str = referGoString(value); + return filter->setStringFilterState(key_str, value_str, + state_type, life_span, + stream_sharing); + }); +} + #ifdef __cplusplus } #endif diff --git a/contrib/golang/filters/http/source/go/pkg/api/api.h b/contrib/golang/filters/http/source/go/pkg/api/api.h index 76e61903051f..3673e458d5c8 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/api.h +++ b/contrib/golang/filters/http/source/go/pkg/api/api.h @@ -68,6 +68,9 @@ void envoyGoFilterHttpLog(uint32_t level, void* message); void envoyGoFilterHttpFinalize(void* r, int reason); +CAPIStatus envoyGoFilterHttpSetStringFilterState(void* r, void* key, void* value, int state_type, + int life_span, int stream_sharing); + #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/golang/filters/http/source/go/pkg/api/capi.go b/contrib/golang/filters/http/source/go/pkg/api/capi.go index 60b95d49aac8..764cf3845431 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/capi.go +++ b/contrib/golang/filters/http/source/go/pkg/api/capi.go @@ -48,4 +48,6 @@ type HttpCAPI interface { HttpLog(level LogType, message string) HttpFinalize(r unsafe.Pointer, reason int) + + HttpSetStringFilterState(r unsafe.Pointer, key string, value string, stateType StateType, lifeSpan LifeSpan, streamSharing StreamSharing) } diff --git a/contrib/golang/filters/http/source/go/pkg/api/filter.go b/contrib/golang/filters/http/source/go/pkg/api/filter.go index 8c2e12093bc4..1979fc2935f0 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/filter.go +++ b/contrib/golang/filters/http/source/go/pkg/api/filter.go @@ -77,6 +77,8 @@ type StreamInfo interface { UpstreamHostAddress() (string, bool) // UpstreamClusterName return the upstream host cluster. UpstreamClusterName() (string, bool) + // FilterState return the filter state interface. + FilterState() FilterState } type StreamFilterCallbacks interface { @@ -102,3 +104,31 @@ type DynamicMetadata interface { // TODO: Get(filterName string) map[string]interface{} Set(filterName string, key string, value interface{}) } + +type StateType int + +const ( + StateTypeReadOnly StateType = 0 + StateTypeMutable StateType = 1 +) + +type LifeSpan int + +const ( + LifeSpanFilterChain LifeSpan = 0 + LifeSpanRequest LifeSpan = 1 + LifeSpanConnection LifeSpan = 2 + LifeSpanTopSpan LifeSpan = 3 +) + +type StreamSharing int + +const ( + None StreamSharing = 0 + SharedWithUpstreamConnection StreamSharing = 1 + SharedWithUpstreamConnectionOnce StreamSharing = 2 +) + +type FilterState interface { + SetString(key, value string, stateType StateType, lifeSpan LifeSpan, streamSharing StreamSharing) +} diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go index b559ddaea20b..f043b5671224 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go +++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go @@ -265,3 +265,8 @@ var cAPI api.HttpCAPI = &httpCApiImpl{} func SetHttpCAPI(api api.HttpCAPI) { cAPI = api } + +func (c *httpCApiImpl) HttpSetStringFilterState(r unsafe.Pointer, key string, value string, stateType api.StateType, lifeSpan api.LifeSpan, streamSharing api.StreamSharing) { + res := C.envoyGoFilterHttpSetStringFilterState(r, unsafe.Pointer(&key), unsafe.Pointer(&value), C.int(stateType), C.int(lifeSpan), C.int(streamSharing)) + handleCApiStatus(res) +} diff --git a/contrib/golang/filters/http/source/go/pkg/http/filter.go b/contrib/golang/filters/http/source/go/pkg/http/filter.go index 039b041b03ba..881b2825b1b7 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/filter.go +++ b/contrib/golang/filters/http/source/go/pkg/http/filter.go @@ -196,3 +196,17 @@ func (s *streamInfo) UpstreamHostAddress() (string, bool) { func (s *streamInfo) UpstreamClusterName() (string, bool) { return cAPI.HttpGetStringValue(unsafe.Pointer(s.request.req), ValueUpstreamClusterName) } + +type filterState struct { + request *httpRequest +} + +func (s *streamInfo) FilterState() api.FilterState { + return &filterState{ + request: s.request, + } +} + +func (f *filterState) SetString(key, value string, stateType api.StateType, lifeSpan api.LifeSpan, streamSharing api.StreamSharing) { + cAPI.HttpSetStringFilterState(unsafe.Pointer(f.request.req), key, value, stateType, lifeSpan, streamSharing) +} diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc index 9e7e2eb5a7ca..8dfd65f14ca6 100644 --- a/contrib/golang/filters/http/source/golang_filter.cc +++ b/contrib/golang/filters/http/source/golang_filter.cc @@ -1065,6 +1065,47 @@ void Filter::setDynamicMetadataInternal(ProcessorState& state, std::string filte state.streamInfo().setDynamicMetadata(filter_name, value); } +CAPIStatus Filter::setStringFilterState(absl::string_view key, absl::string_view value, + int state_type, int life_span, int stream_sharing) { + // lock until this function return since it may running in a Go thread. + Thread::LockGuard lock(mutex_); + if (has_destroyed_) { + ENVOY_LOG(debug, "golang filter has been destroyed"); + return CAPIStatus::CAPIFilterIsDestroy; + } + + auto& state = getProcessorState(); + if (!state.isProcessingInGo()) { + ENVOY_LOG(debug, "golang filter is not processing Go"); + return CAPIStatus::CAPINotInGo; + } + + if (state.isThreadSafe()) { + state.streamInfo().filterState()->setData( + key, std::make_shared(value), + static_cast(state_type), + static_cast(life_span), + static_cast(stream_sharing)); + } else { + auto key_str = std::string(key); + auto filter_state = std::make_shared(value); + auto weak_ptr = weak_from_this(); + state.getDispatcher().post( + [this, &state, weak_ptr, key_str, filter_state, state_type, life_span, stream_sharing] { + if (!weak_ptr.expired() && !hasDestroyed()) { + Thread::LockGuard lock(mutex_); + state.streamInfo().filterState()->setData( + key_str, filter_state, static_cast(state_type), + static_cast(life_span), + static_cast(stream_sharing)); + } else { + ENVOY_LOG(info, "golang filter has gone or destroyed in setStringFilterState"); + } + }); + } + return CAPIStatus::CAPIOK; +} + /* ConfigId */ uint64_t Filter::getMergedConfigId(ProcessorState& state) { diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h index 888c8b544c90..af1423103c15 100644 --- a/contrib/golang/filters/http/source/golang_filter.h +++ b/contrib/golang/filters/http/source/golang_filter.h @@ -181,6 +181,8 @@ class Filter : public Http::StreamFilter, CAPIStatus getStringValue(int id, GoString* value_str); CAPIStatus getIntegerValue(int id, uint64_t* value); CAPIStatus setDynamicMetadata(std::string filter_name, std::string key, absl::string_view buf); + CAPIStatus setStringFilterState(absl::string_view key, absl::string_view value, int state_type, + int life_span, int stream_sharing); private: bool hasDestroyed() { @@ -259,6 +261,15 @@ class FilterLogger : Logger::Loggable { void log(uint32_t level, absl::string_view message) const; }; +class GoStringFilterState : public StreamInfo::FilterState::Object { +public: + GoStringFilterState(absl::string_view value) : value_(value) {} + const std::string& value() const { return value_; } + +private: + const std::string value_; +}; + } // namespace Golang } // namespace HttpFilters } // namespace Extensions diff --git a/contrib/golang/filters/http/test/golang_integration_test.cc b/contrib/golang/filters/http/test/golang_integration_test.cc index 0ec933742933..71c8f8fee378 100644 --- a/contrib/golang/filters/http/test/golang_integration_test.cc +++ b/contrib/golang/filters/http/test/golang_integration_test.cc @@ -34,6 +34,8 @@ class RetrieveDynamicMetadataFilter : public Http::StreamEncoderFilter { const auto& fields = filter_it->second.fields(); std::string val = fields.at("foo").string_value(); EXPECT_EQ(val, "bar"); + EXPECT_TRUE( + decoder_callbacks_->streamInfo().filterState()->hasDataWithName("go_state_test_key")); return Http::FilterHeadersStatus::Continue; } diff --git a/contrib/golang/filters/http/test/test_data/basic/filter.go b/contrib/golang/filters/http/test/test_data/basic/filter.go index 91d6d80ac200..93870973acd2 100644 --- a/contrib/golang/filters/http/test/test_data/basic/filter.go +++ b/contrib/golang/filters/http/test/test_data/basic/filter.go @@ -111,6 +111,9 @@ func (f *filter) decodeHeaders(header api.RequestHeaderMap, endStream bool) api. md.Set("filter.go", "foo", "bar") } + fs := f.callbacks.StreamInfo().FilterState() + fs.SetString("go_state_test_key", "go_state_test_value", api.StateTypeReadOnly, api.LifeSpanRequest, api.SharedWithUpstreamConnection) + if strings.Contains(f.localreplay, "decode-header") { return f.sendLocalReply("decode-header") } From 2419cff0a333d0f19ecfccb1517985b1157d8956 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Thu, 25 May 2023 10:20:28 -0400 Subject: [PATCH 371/740] matcher: Introduce abstract interface `CustomMatchData` to matching API (#27587) * Introduce `CustomMatchData` to matching API Signed-off-by: tyxia --- envoy/matcher/matcher.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/envoy/matcher/matcher.h b/envoy/matcher/matcher.h index 32fdd76e2065..44f49681965a 100644 --- a/envoy/matcher/matcher.h +++ b/envoy/matcher/matcher.h @@ -25,7 +25,16 @@ class ServerFactoryContext; namespace Matcher { -using MatchingDataType = absl::variant; +// Abstract interface for custom matching data. +// Overrides this interface to provide custom matcher specific implementation. +class CustomMatchData { +public: + virtual ~CustomMatchData() = default; +}; + +using MatchingDataType = + absl::variant>; + inline constexpr absl::string_view DefaultMatchingDataType = "string"; // This file describes a MatchTree, which traverses a tree of matches until it From 2b3fbe5fe10e6e2e406def953558779d36df28cf Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Thu, 25 May 2023 10:21:25 -0400 Subject: [PATCH 372/740] Assert(valid) when ext_proc filter apply header mutations (#27547) * ASSERT(valid()) when ext_proc filter apply header mutations. Signed-off-by: Yanjun Xiang --- .../filters/http/ext_proc/ext_proc.cc | 5 ++- .../filters/http/ext_proc/mutation_utils.cc | 13 ++++++ .../ext_proc/ext_proc_integration_test.cc | 15 +++++++ .../http/ext_proc/mutation_utils_test.cc | 40 +++++++++++++++++++ 4 files changed, 72 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index fd45f89921c8..1b2a4c10f9db 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -775,7 +775,10 @@ void Filter::sendImmediateResponse(const ImmediateResponse& response) { const auto mut_status = MutationUtils::applyHeaderMutations( response.headers(), headers, false, immediateResponseChecker().checker(), stats_.rejected_header_mutations_); - ENVOY_BUG(mut_status.ok(), "Immediate response mutations should not fail"); + if (!mut_status.ok()) { + ENVOY_LOG_EVERY_POW_2(error, "Immediate response mutations failed with {}", + mut_status.message()); + } } }; diff --git a/source/extensions/filters/http/ext_proc/mutation_utils.cc b/source/extensions/filters/http/ext_proc/mutation_utils.cc index 08dbcd6eedf9..6ac7297004ec 100644 --- a/source/extensions/filters/http/ext_proc/mutation_utils.cc +++ b/source/extensions/filters/http/ext_proc/mutation_utils.cc @@ -40,6 +40,11 @@ absl::Status MutationUtils::applyHeaderMutations(const HeaderMutation& mutation, const Checker& checker, Counter& rejected_mutations) { for (const auto& hdr : mutation.remove_headers()) { + if (!Http::HeaderUtility::headerNameIsValid(hdr)) { + ENVOY_LOG(debug, "remove_headers contain invalid character, may not be removed."); + rejected_mutations.inc(); + return absl::InvalidArgumentError("Invalid character in remove_headers mutation."); + } const LowerCaseString remove_header(hdr); switch (checker.check(CheckOperation::REMOVE, remove_header, "")) { case CheckResult::OK: @@ -62,6 +67,13 @@ absl::Status MutationUtils::applyHeaderMutations(const HeaderMutation& mutation, if (!sh.has_header()) { continue; } + if (!Http::HeaderUtility::headerNameIsValid(sh.header().key()) || + !Http::HeaderUtility::headerValueIsValid(sh.header().value())) { + ENVOY_LOG(debug, + "set_headers contain invalid character in key or value, may not be appended."); + rejected_mutations.inc(); + return absl::InvalidArgumentError("Invalid character in set_headers mutation."); + } const LowerCaseString header_name(sh.header().key()); const bool append = PROTOBUF_GET_WRAPPED_OR_DEFAULT(sh, append, false); const auto check_op = (append && !headers.get(header_name).empty()) ? CheckOperation::APPEND @@ -87,6 +99,7 @@ absl::Status MutationUtils::applyHeaderMutations(const HeaderMutation& mutation, break; case CheckResult::FAIL: ENVOY_LOG(debug, "Header {} may not be modified. Returning error", header_name); + rejected_mutations.inc(); return absl::InvalidArgumentError( absl::StrCat("Invalid attempt to modify ", static_cast(header_name))); } diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc index 3c488aaa0e27..d137b3e5cda8 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -1079,6 +1079,21 @@ TEST_P(ExtProcIntegrationTest, GetAndRespondImmediately) { EXPECT_EQ("{\"reason\": \"Not authorized\"}", response->body()); } +TEST_P(ExtProcIntegrationTest, GetAndRespondImmediatelyWithInvalidCharacter) { + initializeConfig(); + HttpIntegrationTest::initialize(); + auto response = sendDownstreamRequest(absl::nullopt); + + processAndRespondImmediately(*grpc_upstreams_[0], true, [](ImmediateResponse& immediate) { + immediate.mutable_status()->set_code(envoy::type::v3::StatusCode::Unauthorized); + auto* hdr = immediate.mutable_headers()->add_set_headers(); + hdr->mutable_header()->set_key("x-failure-reason\n"); + hdr->mutable_header()->set_value("testing"); + }); + + verifyDownstreamResponse(*response, 401); +} + // Test the filter using the default configuration by connecting to // an ext_proc server that responds to the request_headers message // by sending back an immediate_response message after the diff --git a/test/extensions/filters/http/ext_proc/mutation_utils_test.cc b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc index 09dc91254794..669dcbd28b60 100644 --- a/test/extensions/filters/http/ext_proc/mutation_utils_test.cc +++ b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc @@ -188,6 +188,46 @@ TEST(MutationUtils, TestNonAppendableHeaders) { EXPECT_THAT(&headers, HeaderMapEqualIgnoreOrder(&expected_headers)); } +TEST(MutationUtils, TestSetHeaderWithInvalidCharacter) { + Http::TestRequestHeaderMapImpl headers{ + {":method", "GET"}, + {"host", "localhost:1000"}, + }; + Checker checker(HeaderMutationRules::default_instance()); + Envoy::Stats::MockCounter rejections; + envoy::service::ext_proc::v3::HeaderMutation mutation; + auto* s = mutation.add_set_headers(); + // Test header key contains invalid character. + s->mutable_header()->set_key("x-append-this\n"); + s->mutable_header()->set_value("value"); + EXPECT_CALL(rejections, inc()); + EXPECT_FALSE( + MutationUtils::applyHeaderMutations(mutation, headers, false, checker, rejections).ok()); + + mutation.Clear(); + s = mutation.add_set_headers(); + // Test header value contains invalid character. + s->mutable_header()->set_key("x-append-this"); + s->mutable_header()->set_value("value\r"); + EXPECT_CALL(rejections, inc()); + EXPECT_FALSE( + MutationUtils::applyHeaderMutations(mutation, headers, false, checker, rejections).ok()); +} + +TEST(MutationUtils, TestRemoveHeaderWithInvalidCharacter) { + Http::TestRequestHeaderMapImpl headers{ + {":method", "GET"}, + {"host", "localhost:1000"}, + }; + envoy::service::ext_proc::v3::HeaderMutation mutation; + mutation.add_remove_headers("host\n"); + Checker checker(HeaderMutationRules::default_instance()); + Envoy::Stats::MockCounter rejections; + EXPECT_CALL(rejections, inc()); + EXPECT_FALSE( + MutationUtils::applyHeaderMutations(mutation, headers, false, checker, rejections).ok()); +} + // Ensure that we actually replace the body TEST(MutationUtils, TestBodyMutationReplace) { Buffer::OwnedImpl buf; From 058db60ba54c3791193d2264a848d7295bb4e521 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Thu, 25 May 2023 10:22:41 -0400 Subject: [PATCH 373/740] http: Ensure that multiple referer header entries are added (#27532) * Ensure that there are multiple entries created with referer header Signed-off-by: tyxia --- test/common/http/conn_manager_utility_test.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index de2306e0a813..9d6aadb157be 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -260,6 +260,11 @@ TEST_F(ConnectionManagerUtilityTest, RemoveRefererIfMultipleEntriesAreFound) { ON_CALL(config_, useRemoteAddress()).WillByDefault(Return(true)); TestRequestHeaderMapImpl headers{{"referer", "https://example.com/"}, {"referer", "https://google.com/"}}; + + // If the referer header is registered with `CustomInlineHeaderRegistry` somewhere else already, + // multiple headers will be placed into same slot. (i.e., single entry). Thus, assert here + // to ensure that two entries are created before proceeding to test multiple entries removal. + ASSERT_EQ(headers.size(), 2); EXPECT_EQ((MutateRequestRet{"10.0.0.1:0", true, Tracing::Reason::NotTraceable}), callMutateRequestHeaders(headers, Protocol::Http2)); EXPECT_TRUE(headers.get(Http::CustomHeaders::get().Referer).empty()); From 547ef8858f71ce7e997b53eadf5fd75b32395ea6 Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Thu, 25 May 2023 10:36:44 -0400 Subject: [PATCH 374/740] Implement ext_proc header allow list feature (#27524) * Implement ext_proc header scrubbing feature. Signed-off-by: Yanjun Xiang --- .../filters/http/ext_proc/v3/ext_proc.proto | 2 +- changelogs/current.yaml | 5 ++ .../filters/http/ext_proc/ext_proc.cc | 6 +- .../filters/http/ext_proc/ext_proc.h | 17 +++++- .../filters/http/ext_proc/mutation_utils.cc | 19 +++++-- .../filters/http/ext_proc/mutation_utils.h | 8 ++- .../ext_proc/ext_proc_integration_test.cc | 57 +++++++++++++++++++ .../http/ext_proc/mutation_utils_test.cc | 52 ++++++++++++++++- 8 files changed, 155 insertions(+), 11 deletions(-) diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto index 02fa12d7455c..38feceb22706 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto @@ -187,7 +187,6 @@ message ExternalProcessor { // field is set in an external processor response. bool disable_clear_route_cache = 11; - // [#not-implemented-hide:] // Allow headers matching the ``forward_rules`` to be forwarded to the external processing server. // If not set, all headers are forwarded to the external processing server. HeaderForwardingRules forward_rules = 12; @@ -209,6 +208,7 @@ message ExternalProcessor { // The HeaderForwardingRules structure specifies what headers are // allowed to be forwarded to the external processing server. message HeaderForwardingRules { + // If not set, all headers are forwarded to the external processing server. type.matcher.v3.ListStringMatcher allowed_headers = 1; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 7f7af0bd7f0f..f868184568ec 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -203,6 +203,11 @@ new_features: :ref:`disable_clear_route_cache ` to force the ext_proc filter from clearing the route cache. Failures to clear from setting this field will be counted under the clear_route_cache_disabled stat. +- area: ext_proc + change: | + added new configuration field + :ref:`forward_rules ` + to only allow headers matchinging the forward rules to be forwarded to the external processing server. - area: redis_proxy change: | added new field :ref:`connection_rate_limit diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index 1b2a4c10f9db..deb93c625667 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -179,7 +179,8 @@ FilterHeadersStatus Filter::onHeaders(ProcessorState& state, state.setHasNoBody(end_stream); ProcessingRequest req; auto* headers_req = state.mutableHeaders(req); - MutationUtils::headersToProto(headers, *headers_req->mutable_headers()); + MutationUtils::headersToProto(headers, config_->headerMatchers(), + *headers_req->mutable_headers()); headers_req->set_end_of_stream(end_stream); state.onStartProcessorCall(std::bind(&Filter::onMessageTimeout, this), config_->messageTimeout(), ProcessorState::CallbackState::HeadersCallback); @@ -551,7 +552,8 @@ void Filter::sendBufferedData(ProcessorState& state, ProcessorState::CallbackSta void Filter::sendTrailers(ProcessorState& state, const Http::HeaderMap& trailers) { ProcessingRequest req; auto* trailers_req = state.mutableTrailers(req); - MutationUtils::headersToProto(trailers, *trailers_req->mutable_trailers()); + MutationUtils::headersToProto(trailers, config_->headerMatchers(), + *trailers_req->mutable_trailers()); state.onStartProcessorCall(std::bind(&Filter::onMessageTimeout, this), config_->messageTimeout(), ProcessorState::CallbackState::TrailersCallback); ENVOY_LOG(debug, "Sending trailers message"); diff --git a/source/extensions/filters/http/ext_proc/ext_proc.h b/source/extensions/filters/http/ext_proc/ext_proc.h index f3a380547a8b..ecbf61dcaae8 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.h +++ b/source/extensions/filters/http/ext_proc/ext_proc.h @@ -19,6 +19,7 @@ #include "envoy/upstream/upstream.h" #include "source/common/common/logger.h" +#include "source/common/common/matchers.h" #include "source/common/protobuf/protobuf.h" #include "source/extensions/filters/common/mutation_rules/mutation_rules.h" #include "source/extensions/filters/http/common/pass_through_filter.h" @@ -110,7 +111,7 @@ class FilterConfig { message_timeout_(message_timeout), max_message_timeout_ms_(max_message_timeout_ms), stats_(generateStats(stats_prefix, config.stat_prefix(), scope)), processing_mode_(config.processing_mode()), mutation_checker_(config.mutation_rules()), - filter_metadata_(config.filter_metadata()) {} + filter_metadata_(config.filter_metadata()), header_matchers_(initHeaderMatchers(config)) {} bool failureModeAllow() const { return failure_mode_allow_; } @@ -130,6 +131,8 @@ class FilterConfig { bool disableClearRouteCache() const { return disable_clear_route_cache_; } + const std::vector& headerMatchers() const { return header_matchers_; } + const Envoy::ProtobufWkt::Struct& filterMetadata() const { return filter_metadata_; } private: @@ -138,6 +141,16 @@ class FilterConfig { const std::string final_prefix = absl::StrCat(prefix, "ext_proc.", filter_stats_prefix); return {ALL_EXT_PROC_FILTER_STATS(POOL_COUNTER_PREFIX(scope, final_prefix))}; } + const std::vector initHeaderMatchers( + const envoy::extensions::filters::http::ext_proc::v3::ExternalProcessor& config) { + std::vector header_matchers; + for (const auto& matcher : config.forward_rules().allowed_headers().patterns()) { + header_matchers.push_back( + std::make_unique>( + matcher)); + } + return header_matchers; + } const bool failure_mode_allow_; const bool disable_clear_route_cache_; @@ -148,6 +161,8 @@ class FilterConfig { const envoy::extensions::filters::http::ext_proc::v3::ProcessingMode processing_mode_; const Filters::Common::MutationRules::Checker mutation_checker_; const Envoy::ProtobufWkt::Struct filter_metadata_; + // Empty header_matchers_ means allow all. + const std::vector header_matchers_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/http/ext_proc/mutation_utils.cc b/source/extensions/filters/http/ext_proc/mutation_utils.cc index 6ac7297004ec..390e6b139e78 100644 --- a/source/extensions/filters/http/ext_proc/mutation_utils.cc +++ b/source/extensions/filters/http/ext_proc/mutation_utils.cc @@ -24,13 +24,22 @@ using Stats::Counter; using envoy::service::ext_proc::v3::BodyMutation; using envoy::service::ext_proc::v3::HeaderMutation; +bool MutationUtils::headerInAllowList( + absl::string_view key, const std::vector& header_matchers) { + return std::any_of(header_matchers.begin(), header_matchers.end(), + [&key](auto& matcher) { return matcher->match(key); }); +} + void MutationUtils::headersToProto(const Http::HeaderMap& headers_in, + const std::vector& header_matchers, envoy::config::core::v3::HeaderMap& proto_out) { - headers_in.iterate([&proto_out](const Http::HeaderEntry& e) -> Http::HeaderMap::Iterate { - auto* new_header = proto_out.add_headers(); - new_header->set_key(std::string(e.key().getStringView())); - new_header->set_value(MessageUtil::sanitizeUtf8String(e.value().getStringView())); - + headers_in.iterate([&proto_out, + &header_matchers](const Http::HeaderEntry& e) -> Http::HeaderMap::Iterate { + if (header_matchers.empty() || headerInAllowList(e.key().getStringView(), header_matchers)) { + auto* new_header = proto_out.add_headers(); + new_header->set_key(std::string(e.key().getStringView())); + new_header->set_value(MessageUtil::sanitizeUtf8String(e.value().getStringView())); + } return Http::HeaderMap::Iterate::Continue; }); } diff --git a/source/extensions/filters/http/ext_proc/mutation_utils.h b/source/extensions/filters/http/ext_proc/mutation_utils.h index ff328ba7e9a1..767973458a8f 100644 --- a/source/extensions/filters/http/ext_proc/mutation_utils.h +++ b/source/extensions/filters/http/ext_proc/mutation_utils.h @@ -17,8 +17,9 @@ namespace ExternalProcessing { class MutationUtils : public Logger::Loggable { public: - // Convert a header map until a protobuf + // Convert a header map into a protobuf static void headersToProto(const Http::HeaderMap& headers_in, + const std::vector& header_matchers, envoy::config::core::v3::HeaderMap& proto_out); // Modify header map based on a set of mutations from a protobuf. An error will be @@ -38,6 +39,11 @@ class MutationUtils : public Logger::Loggable { // Determine if a particular HTTP status code is valid. static bool isValidHttpStatus(int code); + +private: + // Check whether header:key is in header_matchers. + static bool headerInAllowList(absl::string_view key, + const std::vector& header_matchers); }; } // namespace ExternalProcessing diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc index d137b3e5cda8..74f011c3a541 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -2029,4 +2029,61 @@ TEST_P(ExtProcIntegrationTest, SendHeaderAndTrailerInBufferedMode) { verifyDownstreamResponse(*response, 200); } +// Test the filter with header allow list configured and verify only the allowed headers +// in request or response headers and trailers are sent to the ext_proc server. +TEST_P(ExtProcIntegrationTest, GetAndSetHeadersAndTrailersWithHeaderScrubbing) { + auto* forward_rules = proto_config_.mutable_forward_rules(); + auto* list = forward_rules->mutable_allowed_headers(); + list->add_patterns()->set_exact(":method"); + list->add_patterns()->set_exact(":authority"); + list->add_patterns()->set_exact(":status"); + list->add_patterns()->set_exact("x-test-trailers"); + proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SEND); + proto_config_.mutable_processing_mode()->set_request_trailer_mode(ProcessingMode::SEND); + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SEND); + proto_config_.mutable_processing_mode()->set_response_trailer_mode(ProcessingMode::SEND); + + initializeConfig(); + HttpIntegrationTest::initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + auto encoder_decoder = codec_client_->startRequest(headers); + request_encoder_ = &encoder_decoder.first; + IntegrationStreamDecoderPtr response = std::move(encoder_decoder.second); + Http::TestRequestTrailerMapImpl request_trailers{{"x-trailer-foo", "yes"}}; + codec_client_->sendTrailers(*request_encoder_, request_trailers); + + processRequestHeadersMessage( + *grpc_upstreams_[0], true, [](const HttpHeaders& headers, HeadersResponse&) { + Http::TestRequestHeaderMapImpl expected_request_headers{{":method", "GET"}, + {":authority", "host"}}; + // Verify only allowed request headers is received by ext_proc server. + EXPECT_THAT(headers.headers(), HeaderProtosEqual(expected_request_headers)); + return true; + }); + processRequestTrailersMessage(*grpc_upstreams_[0], false, + [](const HttpTrailers& trailers, TrailersResponse&) { + // The request trailer header is not in the allow list. + EXPECT_EQ(trailers.trailers().headers_size(), 0); + return true; + }); + // Send back response with :status 200 and trailer header: x-test-trailers. + handleUpstreamRequestWithTrailer(); + processResponseHeadersMessage( + *grpc_upstreams_[0], false, [](const HttpHeaders& headers, HeadersResponse&) { + Http::TestRequestHeaderMapImpl expected_response_headers{{":status", "200"}}; + EXPECT_THAT(headers.headers(), HeaderProtosEqual(expected_response_headers)); + return true; + }); + processResponseTrailersMessage( + *grpc_upstreams_[0], false, [](const HttpTrailers& trailers, TrailersResponse&) { + // The response trailer header is in the allow list. + Http::TestResponseTrailerMapImpl expected_trailers{{"x-test-trailers", "Yes"}}; + EXPECT_THAT(trailers.trailers(), HeaderProtosEqual(expected_trailers)); + return true; + }); + verifyDownstreamResponse(*response, 200); +} + } // namespace Envoy diff --git a/test/extensions/filters/http/ext_proc/mutation_utils_test.cc b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc index 669dcbd28b60..bc8d36d9d77e 100644 --- a/test/extensions/filters/http/ext_proc/mutation_utils_test.cc +++ b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc @@ -34,7 +34,8 @@ TEST(MutationUtils, TestBuildHeaders) { headers.addCopy(LowerCaseString("x-number"), 9999); envoy::config::core::v3::HeaderMap proto_headers; - MutationUtils::headersToProto(headers, proto_headers); + std::vector header_matchers; + MutationUtils::headersToProto(headers, header_matchers, proto_headers); Http::TestRequestHeaderMapImpl expected{{":method", "GET"}, {":path", "/foo/the/bar?size=123"}, @@ -284,6 +285,55 @@ TEST(MutationUtils, TestBodyMutationNothing) { EXPECT_TRUE(TestUtility::buffersEqual(buf, bufCopy)); } +TEST(MutationUtils, TestAllowHeadersExactCaseSensitive) { + Http::TestRequestHeaderMapImpl headers{ + {":method", "GET"}, + {":path", "/foo/the/bar?size=123"}, + {"content-type", "text/plain; encoding=UTF8"}, + {"x-something-else", "yes"}, + }; + + envoy::config::core::v3::HeaderMap proto_headers; + std::vector header_matchers; + envoy::type::matcher::v3::StringMatcher string_matcher; + string_matcher.set_exact(":method"); + header_matchers.push_back( + std::make_unique>( + string_matcher)); + string_matcher.set_exact(":Path"); + header_matchers.push_back( + std::make_unique>( + string_matcher)); + MutationUtils::headersToProto(headers, header_matchers, proto_headers); + + Http::TestRequestHeaderMapImpl expected{{":method", "GET"}}; + EXPECT_THAT(proto_headers, HeaderProtosEqual(expected)); +} + +TEST(MutationUtils, TestAllowHeadersExactIgnoreCase) { + Http::TestRequestHeaderMapImpl headers{ + {":method", "GET"}, + {":path", "/foo/the/bar?size=123"}, + {"content-type", "text/plain; encoding=UTF8"}, + {"x-something-else", "yes"}, + }; + envoy::config::core::v3::HeaderMap proto_headers; + std::vector header_matchers; + envoy::type::matcher::v3::StringMatcher string_matcher; + string_matcher.set_exact(":method"); + header_matchers.push_back( + std::make_unique>( + string_matcher)); + string_matcher.set_exact(":Path"); + string_matcher.set_ignore_case(true); + header_matchers.push_back( + std::make_unique>( + string_matcher)); + MutationUtils::headersToProto(headers, header_matchers, proto_headers); + Http::TestRequestHeaderMapImpl expected{{":method", "GET"}, {":path", "/foo/the/bar?size=123"}}; + EXPECT_THAT(proto_headers, HeaderProtosEqual(expected)); +} + } // namespace } // namespace ExternalProcessing } // namespace HttpFilters From 881bc030a8f1cdc904311de6c1cdba38ef8cc98a Mon Sep 17 00:00:00 2001 From: botengyao Date: Thu, 25 May 2023 10:40:54 -0400 Subject: [PATCH 375/740] hc: add health event sink interface and file sink implementation (#27419) * add access log to health check Signed-off-by: Boteng Yao --- CODEOWNERS | 2 + api/BUILD | 1 + api/envoy/config/core/v3/health_check.proto | 18 ++- .../health_check/event_sinks/file/v3/BUILD | 9 ++ .../event_sinks/file/v3/file.proto | 22 ++++ api/versioning/BUILD | 1 + changelogs/current.yaml | 4 + docs/root/api-v3/config/config.rst | 1 + .../health_check_event_sinks.rst | 10 ++ .../upstream/health_checking.rst | 9 +- envoy/server/health_checker_config.h | 12 ++ envoy/upstream/BUILD | 9 ++ envoy/upstream/health_check_event_sink.h | 38 +++++++ source/common/upstream/BUILD | 7 ++ .../upstream/health_checker_event_logger.cc | 8 ++ .../upstream/health_checker_event_logger.h | 28 ++++- source/common/upstream/health_checker_impl.cc | 46 ++------ source/common/upstream/health_checker_impl.h | 37 ++++++ source/extensions/extensions_build_config.bzl | 6 + source/extensions/extensions_metadata.yaml | 7 ++ .../health_check/event_sinks/file/BUILD | 25 +++++ .../event_sinks/file/file_sink_impl.cc | 31 +++++ .../event_sinks/file/file_sink_impl.h | 43 +++++++ test/common/upstream/BUILD | 2 + ...se-health_check_fuzz_test-5704648171978752 | 1 - ...ed-health_check_fuzz_test-5678121129607168 | 1 - ...ed-health_check_fuzz_test-5748071634567168 | 1 - .../health_check_corpus/custom_health_check | 1 - .../health_check_corpus/grpc_double_reset | 1 - .../grpc_generalized-crash | 1 - ...h-449c4bf2d000d6e56b782fdd26a86e20a7f87b4f | 1 - .../health_check_corpus/tcp_crash-test | 1 - .../health_check_corpus/tcp_crash-test-1 | 1 - .../upstream/health_checker_impl_test.cc | 88 ++++++++++++++- .../health_check/event_sinks/file/BUILD | 26 +++++ .../event_sinks/file/file_sink_impl_test.cc | 106 ++++++++++++++++++ test/mocks/server/BUILD | 1 + .../server/health_checker_factory_context.cc | 1 + .../server/health_checker_factory_context.h | 5 + ...testcase-config_fuzz_test-4788023076847616 | 1 - ...inimized-config_fuzz_test-5666128418832384 | 1 - ...inimized-server_fuzz_test-5733243234811904 | 1 - tools/code_format/config.yaml | 1 + tools/extensions/extensions_schema.yaml | 1 + 44 files changed, 560 insertions(+), 57 deletions(-) create mode 100644 api/envoy/extensions/health_check/event_sinks/file/v3/BUILD create mode 100644 api/envoy/extensions/health_check/event_sinks/file/v3/file.proto create mode 100644 docs/root/api-v3/config/health_check_event_sinks/health_check_event_sinks.rst create mode 100644 envoy/upstream/health_check_event_sink.h create mode 100644 source/extensions/health_check/event_sinks/file/BUILD create mode 100644 source/extensions/health_check/event_sinks/file/file_sink_impl.cc create mode 100644 source/extensions/health_check/event_sinks/file/file_sink_impl.h create mode 100644 test/extensions/health_check/event_sinks/file/BUILD create mode 100644 test/extensions/health_check/event_sinks/file/file_sink_impl_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index efa25078addf..e9a63bb15179 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -309,6 +309,8 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/health_checkers/grpc @snowp @zuercher /*/extensions/health_checkers/http @snowp @zuercher /*/extensions/health_checkers/tcp @snowp @zuercher +# Health check event sinks +/*/extensions/health_check/event_sinks/file @botengyao @yanavlasov # IP Geolocation /*/extensions/filters/http/geoip @nezdolik @ravenblackx diff --git a/api/BUILD b/api/BUILD index cc3a5068531d..0a9934e0207b 100644 --- a/api/BUILD +++ b/api/BUILD @@ -235,6 +235,7 @@ proto_library( "//envoy/extensions/formatter/cel/v3:pkg", "//envoy/extensions/formatter/metadata/v3:pkg", "//envoy/extensions/formatter/req_without_query/v3:pkg", + "//envoy/extensions/health_check/event_sinks/file/v3:pkg", "//envoy/extensions/health_checkers/redis/v3:pkg", "//envoy/extensions/health_checkers/thrift/v3:pkg", "//envoy/extensions/http/cache/file_system_http_cache/v3:pkg", diff --git a/api/envoy/config/core/v3/health_check.proto b/api/envoy/config/core/v3/health_check.proto index f4c9d857b459..2ec258d8ac09 100644 --- a/api/envoy/config/core/v3/health_check.proto +++ b/api/envoy/config/core/v3/health_check.proto @@ -4,6 +4,7 @@ package envoy.config.core.v3; import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/event_service_config.proto"; +import "envoy/config/core/v3/extension.proto"; import "envoy/type/matcher/v3/string.proto"; import "envoy/type/v3/http.proto"; import "envoy/type/v3/range.proto"; @@ -13,6 +14,7 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; +import "envoy/annotations/deprecation.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -60,7 +62,7 @@ message HealthStatusSet { [(validate.rules).repeated = {items {enum {defined_only: true}}}]; } -// [#next-free-field: 25] +// [#next-free-field: 26] message HealthCheck { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.core.HealthCheck"; @@ -366,9 +368,19 @@ message HealthCheck { // The default value for "healthy edge interval" is the same as the default interval. google.protobuf.Duration healthy_edge_interval = 16 [(validate.rules).duration = {gt {}}]; + // .. attention:: + // This field is deprecated in favor of the extension + // :ref:`event_logger ` and + // :ref:`event_log_path ` + // in the file sink extension. + // // Specifies the path to the :ref:`health check event log `. - // If empty, no event log will be written. - string event_log_path = 17; + string event_log_path = 17 + [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; + + // A list of event log sinks to process the health check event. + // [#extension-category: envoy.health_check.event_sinks] + repeated TypedExtensionConfig event_logger = 25; // [#not-implemented-hide:] // The gRPC service for the health check event service. diff --git a/api/envoy/extensions/health_check/event_sinks/file/v3/BUILD b/api/envoy/extensions/health_check/event_sinks/file/v3/BUILD new file mode 100644 index 000000000000..ee92fb652582 --- /dev/null +++ b/api/envoy/extensions/health_check/event_sinks/file/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/health_check/event_sinks/file/v3/file.proto b/api/envoy/extensions/health_check/event_sinks/file/v3/file.proto new file mode 100644 index 000000000000..d436474a729a --- /dev/null +++ b/api/envoy/extensions/health_check/event_sinks/file/v3/file.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package envoy.extensions.health_check.event_sinks.file.v3; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.health_check.event_sinks.file.v3"; +option java_outer_classname = "FileProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/health_check/event_sinks/file/v3;filev3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Health Check Log File Sink] +// [#extension: envoy.health_check.event_sinks.file] + +// Health check event file sink. +// The health check event will be converted to JSON. +message HealthCheckEventFileSink { + // Specifies the path to the health check event log. + string event_log_path = 1 [(validate.rules).string = {min_len: 1}]; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 4f4a25d60e82..a256f1a4cada 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -173,6 +173,7 @@ proto_library( "//envoy/extensions/formatter/cel/v3:pkg", "//envoy/extensions/formatter/metadata/v3:pkg", "//envoy/extensions/formatter/req_without_query/v3:pkg", + "//envoy/extensions/health_check/event_sinks/file/v3:pkg", "//envoy/extensions/health_checkers/redis/v3:pkg", "//envoy/extensions/health_checkers/thrift/v3:pkg", "//envoy/extensions/http/cache/file_system_http_cache/v3:pkg", diff --git a/changelogs/current.yaml b/changelogs/current.yaml index f868184568ec..f537a751f08d 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -241,3 +241,7 @@ deprecated: change: | deprecated (1.25.0) :ref:`intermediate_log_entry ` in favour of :ref:`access_log_type `. +- area: health_check + change: | + deprecated the :ref:`HealthCheck event_log_path ` in favor of + :ref:`HealthCheck event_logger extension `. diff --git a/docs/root/api-v3/config/config.rst b/docs/root/api-v3/config/config.rst index 10a4752b2640..ca859c388e68 100644 --- a/docs/root/api-v3/config/config.rst +++ b/docs/root/api-v3/config/config.rst @@ -19,6 +19,7 @@ Extensions endpoint/endpoint filter/filter grpc_credential/grpc_credential + health_check_event_sinks/health_check_event_sinks health_checker/health_checker http/early_header_mutation http/custom_response diff --git a/docs/root/api-v3/config/health_check_event_sinks/health_check_event_sinks.rst b/docs/root/api-v3/config/health_check_event_sinks/health_check_event_sinks.rst new file mode 100644 index 000000000000..b05bea13b7fd --- /dev/null +++ b/docs/root/api-v3/config/health_check_event_sinks/health_check_event_sinks.rst @@ -0,0 +1,10 @@ +.. _api_v3_health_check_event_sinks: + +Health check event sinks +======================== + +.. toctree:: + :glob: + :maxdepth: 2 + + ../../extensions/health_check/event_sinks/*/v3/* diff --git a/docs/root/intro/arch_overview/upstream/health_checking.rst b/docs/root/intro/arch_overview/upstream/health_checking.rst index c3d674d4d019..9fbc8e126c6f 100644 --- a/docs/root/intro/arch_overview/upstream/health_checking.rst +++ b/docs/root/intro/arch_overview/upstream/health_checking.rst @@ -85,10 +85,17 @@ Health check event logging -------------------------- A per-healthchecker log of ejection and addition events can optionally be produced by Envoy by -specifying a log file path in :ref:`the HealthCheck config `. +specifying a log file path in :ref:`the HealthCheck config event_log_path `. The log is structured as JSON dumps of :ref:`HealthCheckEvent messages `. +Note: :ref:`the HealthCheck config event_log_path ` is deperated in favor of +:ref:`HealthCheck event_logger extension `. +The :ref:`event_log_path ` is used in the file sink extension for the JSON dumps. + +A new event sink extension catalog +`envoy.health_check.event_sinks` is created, and APIs can be found :ref:`here `. + Envoy can be configured to log all health check failure events by setting the :ref:`always_log_health_check_failures flag ` to true. diff --git a/envoy/server/health_checker_config.h b/envoy/server/health_checker_config.h index 82f27123db8c..9ee5477420fa 100644 --- a/envoy/server/health_checker_config.h +++ b/envoy/server/health_checker_config.h @@ -46,6 +46,18 @@ class HealthCheckerFactoryContext { * @return Api::Api& the API used by the server. */ virtual Api::Api& api() PURE; + + /** + * @return AccessLogManager for use by the entire server. + */ + virtual AccessLog::AccessLogManager& accessLogManager() PURE; + + /** + * Set the event logger to the context, nullptr is accepted since + * the default in the context is nullptr. + * @param event_logger the health check event logger. + */ + virtual void setEventLogger(Upstream::HealthCheckEventLoggerPtr event_logger) PURE; }; /** diff --git a/envoy/upstream/BUILD b/envoy/upstream/BUILD index fb52a403c2f7..8178b9d494e8 100644 --- a/envoy/upstream/BUILD +++ b/envoy/upstream/BUILD @@ -41,6 +41,15 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "health_check_event_sink_interface", + hdrs = ["health_check_event_sink.h"], + deps = [ + "//envoy/config:typed_config_interface", + "//envoy/server:health_checker_config_interface", + ], +) + envoy_cc_library( name = "health_checker_interface", hdrs = ["health_checker.h"], diff --git a/envoy/upstream/health_check_event_sink.h b/envoy/upstream/health_check_event_sink.h new file mode 100644 index 000000000000..aed7792d057d --- /dev/null +++ b/envoy/upstream/health_check_event_sink.h @@ -0,0 +1,38 @@ +#include "envoy/config/typed_config.h" +#include "envoy/server/health_checker_config.h" + +namespace Envoy { +namespace Upstream { + +/** + * Sink for health check event. + */ +class HealthCheckEventSink { +public: + virtual ~HealthCheckEventSink() = default; + + virtual void log(envoy::data::core::v3::HealthCheckEvent event) PURE; +}; + +using HealthCheckEventSinkPtr = std::unique_ptr; +using HealthCheckEventSinkSharedPtr = std::shared_ptr; + +/** + * A factory abstract class for creating instances of HealthCheckEventSink. + */ +class HealthCheckEventSinkFactory : public Config::TypedFactory { +public: + ~HealthCheckEventSinkFactory() override = default; + + /** + * Creates an HealthCheckEventSink using the given config. + */ + virtual HealthCheckEventSinkPtr + createHealthCheckEventSink(const ProtobufWkt::Any& config, + Server::Configuration::HealthCheckerFactoryContext& context) PURE; + + std::string category() const override { return "envoy.health_check.event_sinks"; } +}; + +} // namespace Upstream +} // namespace Envoy diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 23562f7aaa7f..86daa4417864 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -209,8 +209,14 @@ envoy_cc_library( srcs = ["health_checker_event_logger.cc"], hdrs = ["health_checker_event_logger.h"], deps = [ + "//envoy/server:factory_context_interface", + "//envoy/server:health_checker_config_interface", + "//envoy/upstream:health_check_event_sink_interface", "//envoy/upstream:health_checker_interface", + "//source/common/access_log:access_log_lib", "//source/common/network:utility_lib", + "//source/common/protobuf:utility_lib", + "@envoy_api//envoy/config/accesslog/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/data/core/v3:pkg_cc_proto", "@envoy_api//envoy/type/matcher:pkg_cc_proto", @@ -231,6 +237,7 @@ envoy_cc_library( "@envoy_api//envoy/type/v3:pkg_cc_proto", # TODO(dio): Remove dependency to server. "//envoy/server:health_checker_config_interface", + "//envoy/server:factory_context_interface", "//source/common/grpc:codec_lib", "//source/common/router:router_lib", "//source/common/http:codec_client_lib", diff --git a/source/common/upstream/health_checker_event_logger.cc b/source/common/upstream/health_checker_event_logger.cc index d4ed440c61d6..a293bcaa3be3 100644 --- a/source/common/upstream/health_checker_event_logger.cc +++ b/source/common/upstream/health_checker_event_logger.cc @@ -65,13 +65,21 @@ void HealthCheckEventLoggerImpl::createHealthCheckEvent( TimestampUtil::systemClockToTimestamp(time_source_.systemTime(), *event.mutable_timestamp()); callback(event); + for (const auto& event_sink : event_sinks_) { + event_sink->log(event); + } + #ifdef ENVOY_ENABLE_YAML + if (file_ == nullptr) { + return; + } // Make sure the type enums make it into the JSON const auto json = MessageUtil::getJsonStringFromMessageOrError(event, /* pretty_print */ false, /* always_print_primitive_fields */ true); file_->write(fmt::format("{}\n", json)); + #endif } } // namespace Upstream diff --git a/source/common/upstream/health_checker_event_logger.h b/source/common/upstream/health_checker_event_logger.h index d26be58d56cd..fe67cd9a8f87 100644 --- a/source/common/upstream/health_checker_event_logger.h +++ b/source/common/upstream/health_checker_event_logger.h @@ -2,15 +2,23 @@ #include "envoy/access_log/access_log.h" #include "envoy/common/callback.h" +#include "envoy/config/accesslog/v3/accesslog.pb.h" +#include "envoy/config/accesslog/v3/accesslog.pb.validate.h" #include "envoy/config/core/v3/health_check.pb.h" #include "envoy/data/core/v3/health_check_event.pb.h" #include "envoy/event/timer.h" #include "envoy/runtime/runtime.h" +#include "envoy/server/factory_context.h" +#include "envoy/server/health_checker_config.h" #include "envoy/stats/scope.h" #include "envoy/type/matcher/string.pb.h" +#include "envoy/upstream/health_check_event_sink.h" #include "envoy/upstream/health_checker.h" +#include "source/common/access_log/access_log_impl.h" #include "source/common/common/logger.h" +#include "source/common/config/utility.h" +#include "source/common/protobuf/utility.h" namespace Envoy { namespace Upstream { @@ -18,9 +26,22 @@ namespace Upstream { class HealthCheckEventLoggerImpl : public HealthCheckEventLogger { public: HealthCheckEventLoggerImpl(AccessLog::AccessLogManager& log_manager, TimeSource& time_source, - const std::string& file_name) - : time_source_(time_source), file_(log_manager.createAccessLog(Filesystem::FilePathAndType{ - Filesystem::DestinationType::File, file_name})) {} + const envoy::config::core::v3::HealthCheck& health_check_config, + Server::Configuration::HealthCheckerFactoryContext& context) + : time_source_(time_source) { + + // TODO(botengyao): Remove the file_ creation here into the file based health check + // event sink. In this way you can remove the file_ based code from the createHealthCheckEvent + if (!health_check_config.event_log_path().empty() /* deprecated */) { + file_ = log_manager.createAccessLog(Filesystem::FilePathAndType{ + Filesystem::DestinationType::File, health_check_config.event_log_path()}); + } + + for (const auto& config : health_check_config.event_logger()) { + auto& factory = Config::Utility::getAndCheckFactory(config); + event_sinks_.push_back(factory.createHealthCheckEventSink(config.typed_config(), context)); + } + } void logEjectUnhealthy(envoy::data::core::v3::HealthCheckerType health_checker_type, const HostDescriptionConstSharedPtr& host, @@ -42,6 +63,7 @@ class HealthCheckEventLoggerImpl : public HealthCheckEventLogger { std::function callback) const; TimeSource& time_source_; AccessLog::AccessLogFileSharedPtr file_; + std::vector event_sinks_; }; } // namespace Upstream diff --git a/source/common/upstream/health_checker_impl.cc b/source/common/upstream/health_checker_impl.cc index 6b97ecc0c3a8..3ccae085a020 100644 --- a/source/common/upstream/health_checker_impl.cc +++ b/source/common/upstream/health_checker_impl.cc @@ -47,44 +47,11 @@ const std::string& HealthCheckerFactory::getHostname(const HostSharedPtr& host, return cluster->name(); } -class HealthCheckerFactoryContextImpl : public Server::Configuration::HealthCheckerFactoryContext { -public: - HealthCheckerFactoryContextImpl(Upstream::Cluster& cluster, Envoy::Runtime::Loader& runtime, - Event::Dispatcher& dispatcher, - HealthCheckEventLoggerPtr&& event_logger, - ProtobufMessage::ValidationVisitor& validation_visitor, - Api::Api& api) - : cluster_(cluster), runtime_(runtime), dispatcher_(dispatcher), - event_logger_(std::move(event_logger)), validation_visitor_(validation_visitor), api_(api) { - } - Upstream::Cluster& cluster() override { return cluster_; } - Envoy::Runtime::Loader& runtime() override { return runtime_; } - Event::Dispatcher& mainThreadDispatcher() override { return dispatcher_; } - HealthCheckEventLoggerPtr eventLogger() override { return std::move(event_logger_); } - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { - return validation_visitor_; - } - Api::Api& api() override { return api_; } - -private: - Upstream::Cluster& cluster_; - Envoy::Runtime::Loader& runtime_; - Event::Dispatcher& dispatcher_; - HealthCheckEventLoggerPtr event_logger_; - ProtobufMessage::ValidationVisitor& validation_visitor_; - Api::Api& api_; -}; - HealthCheckerSharedPtr HealthCheckerFactory::create( const envoy::config::core::v3::HealthCheck& health_check_config, Upstream::Cluster& cluster, Runtime::Loader& runtime, Event::Dispatcher& dispatcher, AccessLog::AccessLogManager& log_manager, ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api) { - HealthCheckEventLoggerPtr event_logger; - if (!health_check_config.event_log_path().empty()) { - event_logger = std::make_unique( - log_manager, dispatcher.timeSource(), health_check_config.event_log_path()); - } Server::Configuration::CustomHealthCheckerFactory* factory = nullptr; switch (health_check_config.health_checker_case()) { @@ -112,9 +79,18 @@ HealthCheckerSharedPtr HealthCheckerFactory::create( health_check_config.custom_health_check()); } } + std::unique_ptr context( - new HealthCheckerFactoryContextImpl(cluster, runtime, dispatcher, std::move(event_logger), - validation_visitor, api)); + new HealthCheckerFactoryContextImpl(cluster, runtime, dispatcher, validation_visitor, api, + log_manager)); + + if (!health_check_config.event_log_path().empty() /* deprecated */ || + !health_check_config.event_logger().empty()) { + HealthCheckEventLoggerPtr event_logger; + event_logger = std::make_unique( + log_manager, dispatcher.timeSource(), health_check_config, *context); + context->setEventLogger(std::move(event_logger)); + } return factory->createCustomHealthChecker(health_check_config, *context); } diff --git a/source/common/upstream/health_checker_impl.h b/source/common/upstream/health_checker_impl.h index 32bcc69d557e..e6233a9b5773 100644 --- a/source/common/upstream/health_checker_impl.h +++ b/source/common/upstream/health_checker_impl.h @@ -9,9 +9,11 @@ #include "envoy/data/core/v3/health_check_event.pb.h" #include "envoy/grpc/status.h" #include "envoy/network/socket.h" +#include "envoy/server/factory_context.h" #include "envoy/server/health_checker_config.h" #include "envoy/type/v3/http.pb.h" #include "envoy/type/v3/range.pb.h" +#include "envoy/upstream/health_checker.h" #include "source/common/common/dump_state_utils.h" #include "source/common/common/logger.h" @@ -45,6 +47,41 @@ struct HealthCheckerEqualTo { } }; +/** + * Health checker factory context. + */ +class HealthCheckerFactoryContextImpl : public Server::Configuration::HealthCheckerFactoryContext { +public: + HealthCheckerFactoryContextImpl(Upstream::Cluster& cluster, Envoy::Runtime::Loader& runtime, + Event::Dispatcher& dispatcher, + ProtobufMessage::ValidationVisitor& validation_visitor, + Api::Api& api, AccessLog::AccessLogManager& log_manager) + : cluster_(cluster), runtime_(runtime), dispatcher_(dispatcher), + validation_visitor_(validation_visitor), log_manager_(log_manager), api_(api) {} + Upstream::Cluster& cluster() override { return cluster_; } + Envoy::Runtime::Loader& runtime() override { return runtime_; } + Event::Dispatcher& mainThreadDispatcher() override { return dispatcher_; } + HealthCheckEventLoggerPtr eventLogger() override { return std::move(event_logger_); } + ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { + return validation_visitor_; + } + Api::Api& api() override { return api_; } + + AccessLog::AccessLogManager& accessLogManager() override { return log_manager_; } + void setEventLogger(HealthCheckEventLoggerPtr event_logger) override { + event_logger_ = std::move(event_logger); + } + +private: + Upstream::Cluster& cluster_; + Envoy::Runtime::Loader& runtime_; + Event::Dispatcher& dispatcher_; + ProtobufMessage::ValidationVisitor& validation_visitor_; + AccessLog::AccessLogManager& log_manager_; + Api::Api& api_; + HealthCheckEventLoggerPtr event_logger_; +}; + /** * Factory for creating health checker implementations. */ diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index e0370c799dc2..1cccfe090b8b 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -66,6 +66,12 @@ EXTENSIONS = { "envoy.health_checkers.http": "//source/extensions/health_checkers/http:health_checker_lib", "envoy.health_checkers.grpc": "//source/extensions/health_checkers/grpc:health_checker_lib", + # + # Health check event sinks + # + + "envoy.health_check.event_sinks.file": "//source/extensions/health_check/event_sinks/file:file_sink_lib", + # # Input Matchers # diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index c0113843a8dd..0a19fb24bfbc 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -795,6 +795,13 @@ envoy.health_checkers.thrift: status: alpha type_urls: - envoy.extensions.health_checkers.thrift.v3.Thrift +envoy.health_check.event_sinks.file: + categories: + - envoy.health_check.event_sinks + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable + type_urls: + - envoy.extensions.health_check.event_sinks.file.v3.HealthCheckEventFileSink envoy.http.original_ip_detection.custom_header: categories: - envoy.http.original_ip_detection diff --git a/source/extensions/health_check/event_sinks/file/BUILD b/source/extensions/health_check/event_sinks/file/BUILD new file mode 100644 index 000000000000..4aa9f785662c --- /dev/null +++ b/source/extensions/health_check/event_sinks/file/BUILD @@ -0,0 +1,25 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "file_sink_lib", + srcs = ["file_sink_impl.cc"], + hdrs = ["file_sink_impl.h"], + extra_visibility = [ + # previously considered core code. + "//test:__subpackages__", + ], + deps = [ + "//envoy/registry", + "//envoy/upstream:health_check_event_sink_interface", + "//source/common/protobuf:utility_lib", + "@envoy_api//envoy/extensions/health_check/event_sinks/file/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/health_check/event_sinks/file/file_sink_impl.cc b/source/extensions/health_check/event_sinks/file/file_sink_impl.cc new file mode 100644 index 000000000000..61f0113c75c4 --- /dev/null +++ b/source/extensions/health_check/event_sinks/file/file_sink_impl.cc @@ -0,0 +1,31 @@ +#include "source/extensions/health_check/event_sinks/file/file_sink_impl.h" + +#include "envoy/registry/registry.h" + +#include "source/common/protobuf/utility.h" + +namespace Envoy { +namespace Upstream { + +void HealthCheckEventFileSink::log(envoy::data::core::v3::HealthCheckEvent event) { +#ifdef ENVOY_ENABLE_YAML + // Make sure the type enums make it into the JSON + const auto json = + MessageUtil::getJsonStringFromMessageOrError(event, /* pretty_print */ false, + /* always_print_primitive_fields */ true); + file_->write(fmt::format("{}\n", json)); +#endif +}; + +HealthCheckEventSinkPtr HealthCheckEventFileSinkFactory::createHealthCheckEventSink( + const ProtobufWkt::Any& config, Server::Configuration::HealthCheckerFactoryContext& context) { + const auto& validator_config = Envoy::MessageUtil::anyConvertAndValidate< + envoy::extensions::health_check::event_sinks::file::v3::HealthCheckEventFileSink>( + config, context.messageValidationVisitor()); + return std::make_unique(validator_config, context.accessLogManager()); +} + +REGISTER_FACTORY(HealthCheckEventFileSinkFactory, HealthCheckEventSinkFactory); + +} // namespace Upstream +} // namespace Envoy diff --git a/source/extensions/health_check/event_sinks/file/file_sink_impl.h b/source/extensions/health_check/event_sinks/file/file_sink_impl.h new file mode 100644 index 000000000000..79e8652157eb --- /dev/null +++ b/source/extensions/health_check/event_sinks/file/file_sink_impl.h @@ -0,0 +1,43 @@ +#pragma once + +#include "envoy/access_log/access_log.h" +#include "envoy/extensions/health_check/event_sinks/file/v3/file.pb.h" +#include "envoy/extensions/health_check/event_sinks/file/v3/file.pb.validate.h" +#include "envoy/upstream/health_check_event_sink.h" + +namespace Envoy { +namespace Upstream { + +class HealthCheckEventFileSink : public HealthCheckEventSink { +public: + explicit HealthCheckEventFileSink( + const envoy::extensions::health_check::event_sinks::file::v3::HealthCheckEventFileSink& + config, + AccessLog::AccessLogManager& log_manager) + : file_(log_manager.createAccessLog(Filesystem::FilePathAndType{ + Filesystem::DestinationType::File, config.event_log_path()})) {} + + void log(envoy::data::core::v3::HealthCheckEvent event) override; + +private: + AccessLog::AccessLogFileSharedPtr file_; +}; + +class HealthCheckEventFileSinkFactory : public HealthCheckEventSinkFactory { +public: + HealthCheckEventFileSinkFactory() = default; + + HealthCheckEventSinkPtr + createHealthCheckEventSink(const ProtobufWkt::Any& config, + Server::Configuration::HealthCheckerFactoryContext& context) override; + + std::string name() const override { return "envoy.health_check.event_sink.file"; } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr{ + new envoy::extensions::health_check::event_sinks::file::v3::HealthCheckEventFileSink()}; + } +}; + +} // namespace Upstream +} // namespace Envoy diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index d52e677e5fec..341abe8ab9f0 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -182,6 +182,7 @@ envoy_cc_test( "//source/common/protobuf:utility_lib", "//source/common/upstream:health_checker_lib", "//source/common/upstream:upstream_lib", + "//source/extensions/health_check/event_sinks/file:file_sink_lib", "//source/extensions/health_checkers/grpc:health_checker_lib", "//source/extensions/health_checkers/http:health_checker_lib", "//source/extensions/health_checkers/tcp:health_checker_lib", @@ -204,6 +205,7 @@ envoy_cc_test( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", "@envoy_api//envoy/data/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/health_check/event_sinks/file/v3:pkg_cc_proto", ], ) diff --git a/test/common/upstream/health_check_corpus/clusterfuzz-testcase-health_check_fuzz_test-5704648171978752 b/test/common/upstream/health_check_corpus/clusterfuzz-testcase-health_check_fuzz_test-5704648171978752 index 31117664f39d..5d505878716a 100644 --- a/test/common/upstream/health_check_corpus/clusterfuzz-testcase-health_check_fuzz_test-5704648171978752 +++ b/test/common/upstream/health_check_corpus/clusterfuzz-testcase-health_check_fuzz_test-5704648171978752 @@ -20,7 +20,6 @@ health_check_config { unhealthy_interval { seconds: 4227858432 } - event_log_path: "?(" tls_options { } } diff --git a/test/common/upstream/health_check_corpus/clusterfuzz-testcase-minimized-health_check_fuzz_test-5678121129607168 b/test/common/upstream/health_check_corpus/clusterfuzz-testcase-minimized-health_check_fuzz_test-5678121129607168 index b1530f2bc1d2..f48a0a08bd75 100644 --- a/test/common/upstream/health_check_corpus/clusterfuzz-testcase-minimized-health_check_fuzz_test-5678121129607168 +++ b/test/common/upstream/health_check_corpus/clusterfuzz-testcase-minimized-health_check_fuzz_test-5678121129607168 @@ -14,7 +14,6 @@ health_check_config { http_health_check { path: "\003" } - event_log_path: "(" interval_jitter_percent: 654311422 } actions { diff --git a/test/common/upstream/health_check_corpus/clusterfuzz-testcase-minimized-health_check_fuzz_test-5748071634567168 b/test/common/upstream/health_check_corpus/clusterfuzz-testcase-minimized-health_check_fuzz_test-5748071634567168 index 05ce7426fbec..bac5a09e0e7e 100644 --- a/test/common/upstream/health_check_corpus/clusterfuzz-testcase-minimized-health_check_fuzz_test-5748071634567168 +++ b/test/common/upstream/health_check_corpus/clusterfuzz-testcase-minimized-health_check_fuzz_test-5748071634567168 @@ -15,7 +15,6 @@ health_check_config { http_health_check { path: "(" } - event_log_path: "(" } actions { raise_event: 355888746 diff --git a/test/common/upstream/health_check_corpus/custom_health_check b/test/common/upstream/health_check_corpus/custom_health_check index d994dc6033bf..742f06b206c9 100644 --- a/test/common/upstream/health_check_corpus/custom_health_check +++ b/test/common/upstream/health_check_corpus/custom_health_check @@ -36,7 +36,6 @@ health_check_config { seconds: 131072 nanos: 128 } - event_log_path: "A(" interval_jitter_percent: 641007544 initial_jitter { seconds: 8960 diff --git a/test/common/upstream/health_check_corpus/grpc_double_reset b/test/common/upstream/health_check_corpus/grpc_double_reset index aaeca5b1d548..601ad42a77e5 100644 --- a/test/common/upstream/health_check_corpus/grpc_double_reset +++ b/test/common/upstream/health_check_corpus/grpc_double_reset @@ -15,7 +15,6 @@ health_check_config { service_name: "service" authority: "wwnvoyproxy.io" } - event_log_path: "200" } actions { respond { diff --git a/test/common/upstream/health_check_corpus/grpc_generalized-crash b/test/common/upstream/health_check_corpus/grpc_generalized-crash index 0336f7d7e42e..2943aafd2d6c 100644 --- a/test/common/upstream/health_check_corpus/grpc_generalized-crash +++ b/test/common/upstream/health_check_corpus/grpc_generalized-crash @@ -20,7 +20,6 @@ health_check_config { unhealthy_interval { seconds: 4227858432 } - event_log_path: "?(" tls_options { } } diff --git a/test/common/upstream/health_check_corpus/tcp_crash-449c4bf2d000d6e56b782fdd26a86e20a7f87b4f b/test/common/upstream/health_check_corpus/tcp_crash-449c4bf2d000d6e56b782fdd26a86e20a7f87b4f index 77bff0079e8d..271c8fbacbb4 100644 --- a/test/common/upstream/health_check_corpus/tcp_crash-449c4bf2d000d6e56b782fdd26a86e20a7f87b4f +++ b/test/common/upstream/health_check_corpus/tcp_crash-449c4bf2d000d6e56b782fdd26a86e20a7f87b4f @@ -19,7 +19,6 @@ health_check_config { no_traffic_interval { seconds: 1 } - event_log_path: "200" initial_jitter { seconds: 1 } diff --git a/test/common/upstream/health_check_corpus/tcp_crash-test b/test/common/upstream/health_check_corpus/tcp_crash-test index 77bff0079e8d..271c8fbacbb4 100644 --- a/test/common/upstream/health_check_corpus/tcp_crash-test +++ b/test/common/upstream/health_check_corpus/tcp_crash-test @@ -19,7 +19,6 @@ health_check_config { no_traffic_interval { seconds: 1 } - event_log_path: "200" initial_jitter { seconds: 1 } diff --git a/test/common/upstream/health_check_corpus/tcp_crash-test-1 b/test/common/upstream/health_check_corpus/tcp_crash-test-1 index 5f7c0bdf7bc3..06be75b1fa7e 100644 --- a/test/common/upstream/health_check_corpus/tcp_crash-test-1 +++ b/test/common/upstream/health_check_corpus/tcp_crash-test-1 @@ -25,7 +25,6 @@ health_check_config { no_traffic_interval { seconds: 1 } - event_log_path: "200" initial_jitter { seconds: 1 } diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index dfc6177cdcc7..8e97475b2e73 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -8,6 +8,7 @@ #include "envoy/config/core/v3/health_check.pb.validate.h" #include "envoy/config/endpoint/v3/endpoint_components.pb.h" #include "envoy/data/core/v3/health_check_event.pb.h" +#include "envoy/extensions/health_check/event_sinks/file/v3/file.pb.h" #include "envoy/upstream/health_check_host_monitor.h" #include "source/common/buffer/buffer_impl.h" @@ -6392,15 +6393,27 @@ TEST(HealthCheckEventLoggerImplTest, All) { Filesystem::DestinationType::File, "foo"})) .WillOnce(Return(file)); + envoy::config::core::v3::HealthCheck health_check_config; + health_check_config.set_event_log_path("foo"); + + NiceMock cluster; + Runtime::MockLoader runtime; + Event::MockDispatcher dispatcher; + NiceMock validation_visitor; + NiceMock api; + std::shared_ptr host(new NiceMock()); - NiceMock cluster; - ON_CALL(*host, cluster()).WillByDefault(ReturnRef(cluster)); + NiceMock cluster_info; + ON_CALL(*host, cluster()).WillByDefault(ReturnRef(cluster_info)); + + HealthCheckerFactoryContextImpl context(cluster, runtime, dispatcher, validation_visitor, api, + log_manager); Event::SimulatedTimeSystem time_system; // This is rendered as "2009-02-13T23:31:31.234Z".a time_system.setSystemTime(std::chrono::milliseconds(1234567891234)); - HealthCheckEventLoggerImpl event_logger(log_manager, time_system, "foo"); + HealthCheckEventLoggerImpl event_logger(log_manager, time_system, health_check_config, context); EXPECT_CALL(*file, write(absl::string_view{ "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{" @@ -6445,6 +6458,75 @@ TEST(HealthCheckEventLoggerImplTest, All) { event_logger.logNoLongerDegraded(envoy::data::core::v3::HTTP, host); } +TEST(HealthCheckEventLoggerImplTest, OneEventLogger) { + envoy::config::core::v3::HealthCheck health_check_config; + auto event_log = health_check_config.mutable_event_logger()->Add(); + envoy::extensions::health_check::event_sinks::file::v3::HealthCheckEventFileSink config; + config.set_event_log_path("foo"); + event_log->mutable_typed_config()->PackFrom(config); + + NiceMock log_manager; + StringViewSaver file_log_data; + ON_CALL(*log_manager.file_, write(_)).WillByDefault(SaveArg<0>(&file_log_data)); + + NiceMock cluster; + Runtime::MockLoader runtime; + Event::MockDispatcher dispatcher; + NiceMock validation_visitor; + NiceMock api; + + std::shared_ptr host(new NiceMock()); + NiceMock cluster_info; + ON_CALL(*host, cluster()).WillByDefault(ReturnRef(cluster_info)); + + HealthCheckerFactoryContextImpl context(cluster, runtime, dispatcher, validation_visitor, api, + log_manager); + + Event::SimulatedTimeSystem time_system; + // This is rendered as "2009-02-13T23:31:31.234Z".a + time_system.setSystemTime(std::chrono::milliseconds(1234567891234)); + + HealthCheckEventLoggerImpl event_logger(log_manager, time_system, health_check_config, context); + + event_logger.logEjectUnhealthy(envoy::data::core::v3::HTTP, host, envoy::data::core::v3::ACTIVE); + EXPECT_EQ(file_log_data, "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{" + "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\"," + "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_" + "cluster\",\"eject_unhealthy_event\":{\"failure_type\":\"ACTIVE\"}," + "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"); + + event_logger.logAddHealthy(envoy::data::core::v3::HTTP, host, false); + EXPECT_EQ(file_log_data, "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{" + "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\"," + "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_" + "cluster\",\"add_healthy_event\":{\"first_check\":false},\"timestamp\":" + "\"2009-02-13T23:31:31.234Z\"}\n"); + + event_logger.logUnhealthy(envoy::data::core::v3::HTTP, host, envoy::data::core::v3::ACTIVE, + false); + EXPECT_EQ(file_log_data.value(), + "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{" + "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\"," + "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_" + "cluster\",\"health_check_failure_event\":{\"failure_type\":\"ACTIVE\"," + "\"first_check\":false}," + "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"); + + event_logger.logDegraded(envoy::data::core::v3::HTTP, host); + EXPECT_EQ(file_log_data, "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{" + "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\"," + "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_" + "cluster\",\"degraded_healthy_host\":{}," + "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"); + + event_logger.logNoLongerDegraded(envoy::data::core::v3::HTTP, host); + EXPECT_EQ(file_log_data, "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{" + "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\"," + "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_" + "cluster\",\"no_longer_degraded_host\":{}," + "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"); +} + // Validate that the proto constraints don't allow zero length edge durations. TEST(HealthCheckProto, Validation) { { diff --git a/test/extensions/health_check/event_sinks/file/BUILD b/test/extensions/health_check/event_sinks/file/BUILD new file mode 100644 index 000000000000..34a4e41306c7 --- /dev/null +++ b/test/extensions/health_check/event_sinks/file/BUILD @@ -0,0 +1,26 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "file_sink_impl_test", + srcs = ["file_sink_impl_test.cc"], + extension_names = ["envoy.health_check.event_sinks.file"], + deps = [ + "//source/extensions/health_check/event_sinks/file:file_sink_lib", + "//test/mocks/access_log:access_log_mocks", + "//test/mocks/server:health_checker_factory_context_mocks", + "//test/test_common:environment_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/health_check/event_sinks/file/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/health_check/event_sinks/file/file_sink_impl_test.cc b/test/extensions/health_check/event_sinks/file/file_sink_impl_test.cc new file mode 100644 index 000000000000..a7945dcb6951 --- /dev/null +++ b/test/extensions/health_check/event_sinks/file/file_sink_impl_test.cc @@ -0,0 +1,106 @@ +#include "envoy/extensions/health_check/event_sinks/file/v3/file.pb.h" +#include "envoy/extensions/health_check/event_sinks/file/v3/file.pb.validate.h" +#include "envoy/registry/registry.h" + +#include "source/common/protobuf/message_validator_impl.h" +#include "source/extensions/health_check/event_sinks/file/file_sink_impl.h" + +#include "test/mocks/access_log/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/server/health_checker_factory_context.h" +#include "test/mocks/stats/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::SaveArg; + +namespace Envoy { +namespace Upstream { + +TEST(HealthCheckEventFileSinkFactory, createHealthCheckEventSink) { + auto factory = Envoy::Registry::FactoryRegistry::getFactory( + "envoy.health_check.event_sink.file"); + EXPECT_NE(factory, nullptr); + + envoy::extensions::health_check::event_sinks::file::v3::HealthCheckEventFileSink config; + config.set_event_log_path("test_path"); + Envoy::ProtobufWkt::Any typed_config; + typed_config.PackFrom(config); + + NiceMock context; + EXPECT_NE(factory->createHealthCheckEventSink(typed_config, context), nullptr); +} + +TEST(HealthCheckEventFileSinkFactory, createEmptyHealthCheckEventSink) { + auto factory = Envoy::Registry::FactoryRegistry::getFactory( + "envoy.health_check.event_sink.file"); + EXPECT_NE(factory, nullptr); + auto empty_proto = factory->createEmptyConfigProto(); + auto config = *dynamic_cast< + envoy::extensions::health_check::event_sinks::file::v3::HealthCheckEventFileSink*>( + empty_proto.get()); + EXPECT_TRUE(config.event_log_path().empty()); +} + +TEST(HealthCheckEventFileSink, logTest) { + envoy::extensions::health_check::event_sinks::file::v3::HealthCheckEventFileSink config; + config.set_event_log_path("test_path"); + NiceMock log_manager; + StringViewSaver file_log_data; + ON_CALL(*log_manager.file_, write(_)).WillByDefault(SaveArg<0>(&file_log_data)); + + HealthCheckEventFileSink file_sink(config, log_manager); + + envoy::data::core::v3::HealthCheckEvent event; + TestUtility::loadFromYaml(R"EOF( + health_checker_type: HTTP + host: + socket_address: + protocol: TCP + address: 10.0.0.1 + resolver_name: '' + ipv4_compat: false + port_value: 443 + cluster_name: fake_cluster + eject_unhealthy_event: + failure_type: ACTIVE + timestamp: '2009-02-13T23:31:31.234Z' + )EOF", + event); + + file_sink.log(event); + EXPECT_EQ(file_log_data, "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{" + "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\"," + "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_" + "cluster\",\"eject_unhealthy_event\":{\"failure_type\":\"ACTIVE\"}," + "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"); + + envoy::data::core::v3::HealthCheckEvent add_event; + TestUtility::loadFromYaml(R"EOF( + health_checker_type: HTTP + host: + socket_address: + protocol: TCP + address: 10.0.0.1 + resolver_name: '' + ipv4_compat: false + port_value: 443 + cluster_name: fake_cluster + add_healthy_event: + first_check: false + timestamp: '2009-02-13T23:31:31.234Z' + )EOF", + add_event); + + file_sink.log(add_event); + EXPECT_EQ(file_log_data, "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{" + "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\"," + "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_" + "cluster\",\"add_healthy_event\":{\"first_check\":false},\"timestamp\":" + "\"2009-02-13T23:31:31.234Z\"}\n"); +} + +} // namespace Upstream +} // namespace Envoy diff --git a/test/mocks/server/BUILD b/test/mocks/server/BUILD index 6ee47f2e1d7c..8ebe31e1ec73 100644 --- a/test/mocks/server/BUILD +++ b/test/mocks/server/BUILD @@ -275,6 +275,7 @@ envoy_cc_mock( deps = [ "//envoy/server:health_checker_config_interface", "//test/mocks:common_lib", + "//test/mocks/access_log:access_log_mocks", "//test/mocks/api:api_mocks", "//test/mocks/event:event_mocks", "//test/mocks/protobuf:protobuf_mocks", diff --git a/test/mocks/server/health_checker_factory_context.cc b/test/mocks/server/health_checker_factory_context.cc index 45d2de4315a1..f8a3ed362352 100644 --- a/test/mocks/server/health_checker_factory_context.cc +++ b/test/mocks/server/health_checker_factory_context.cc @@ -19,6 +19,7 @@ MockHealthCheckerFactoryContext::MockHealthCheckerFactoryContext() { ON_CALL(*this, messageValidationVisitor()) .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); + ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); } MockHealthCheckerFactoryContext::~MockHealthCheckerFactoryContext() = default; diff --git a/test/mocks/server/health_checker_factory_context.h b/test/mocks/server/health_checker_factory_context.h index d4bc31b438d2..85d8af23b9bd 100644 --- a/test/mocks/server/health_checker_factory_context.h +++ b/test/mocks/server/health_checker_factory_context.h @@ -2,6 +2,7 @@ #include "envoy/server/health_checker_config.h" +#include "test/mocks/access_log/mocks.h" #include "test/mocks/api/mocks.h" #include "test/mocks/common.h" #include "test/mocks/event/mocks.h" @@ -29,6 +30,9 @@ class MockHealthCheckerFactoryContext : public virtual HealthCheckerFactoryConte MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); MOCK_METHOD(Api::Api&, api, ()); + MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, ()); + MOCK_METHOD(void, setEventLogger, (Upstream::HealthCheckEventLoggerPtr)); + Upstream::HealthCheckEventLoggerPtr eventLogger() override { if (!event_logger_) { event_logger_ = std::make_unique>(); @@ -41,6 +45,7 @@ class MockHealthCheckerFactoryContext : public virtual HealthCheckerFactoryConte testing::NiceMock random_; testing::NiceMock runtime_; testing::NiceMock api_{}; + testing::NiceMock access_log_manager_; std::unique_ptr> event_logger_; }; diff --git a/test/server/server_corpus/clusterfuzz-testcase-config_fuzz_test-4788023076847616 b/test/server/server_corpus/clusterfuzz-testcase-config_fuzz_test-4788023076847616 index 37d9bdc530f1..60c47ab5943c 100644 --- a/test/server/server_corpus/clusterfuzz-testcase-config_fuzz_test-4788023076847616 +++ b/test/server/server_corpus/clusterfuzz-testcase-config_fuzz_test-4788023076847616 @@ -46,7 +46,6 @@ static_resources { seconds: 2559 nanos: 16384 } - event_log_path: "]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]" interval_jitter_percent: 524288 tls_options { alpn_protocols: "/" diff --git a/test/server/server_corpus/clusterfuzz-testcase-minimized-config_fuzz_test-5666128418832384 b/test/server/server_corpus/clusterfuzz-testcase-minimized-config_fuzz_test-5666128418832384 index 70e8a1ecb3eb..59693aa628f5 100644 --- a/test/server/server_corpus/clusterfuzz-testcase-minimized-config_fuzz_test-5666128418832384 +++ b/test/server/server_corpus/clusterfuzz-testcase-minimized-config_fuzz_test-5666128418832384 @@ -85,7 +85,6 @@ node { id: " " cluster: " " build_version: " " } static_resou service_name: "0" use_http2: true } - event_log_path: "c" } tls_context { common_tls_context { diff --git a/test/server/server_corpus/clusterfuzz-testcase-minimized-server_fuzz_test-5733243234811904 b/test/server/server_corpus/clusterfuzz-testcase-minimized-server_fuzz_test-5733243234811904 index a66f1e06095d..d58632ba4e42 100644 --- a/test/server/server_corpus/clusterfuzz-testcase-minimized-server_fuzz_test-5733243234811904 +++ b/test/server/server_corpus/clusterfuzz-testcase-minimized-server_fuzz_test-5733243234811904 @@ -49,7 +49,6 @@ static_resources { seconds: 2299 nanos: 16384 } - event_log_path: "%" always_log_health_check_failures: true } typed_extension_protocol_options { diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index 0aff6d44d5ae..fed59e924743 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -436,6 +436,7 @@ visibility_excludes: - source/extensions/quic/server_preferred_address/BUILD - source/extensions/listener_managers/listener_manager/BUILD - source/extensions/upstreams/tcp/BUILD +- source/extensions/health_check/event_sinks/BUILD - source/extensions/health_checkers/BUILD - source/extensions/health_checkers/BUILD - source/extensions/health_checkers/BUILD diff --git a/tools/extensions/extensions_schema.yaml b/tools/extensions/extensions_schema.yaml index 7558fb7f8068..030939c667b9 100644 --- a/tools/extensions/extensions_schema.yaml +++ b/tools/extensions/extensions_schema.yaml @@ -76,6 +76,7 @@ categories: - envoy.grpc_credentials - envoy.guarddog_actions - envoy.health_checkers +- envoy.health_check.event_sinks - envoy.http.cache - envoy.http.header_validators - envoy.http.stateful_header_formatters From c6292584f59eac45bf43f3dba7b0f8c36b3ab0bc Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 25 May 2023 15:55:34 +0100 Subject: [PATCH 376/740] ci: Add github workflow for verify examples (#27371) Signed-off-by: Ryan Northey --- .azure-pipelines/stage/verify.yml | 81 ++++++++++++----------- .azure-pipelines/stages.yml | 6 ++ .github/workflows/check-deps.yml | 8 ++- .github/workflows/envoy-verify.yml | 88 +++++++++++++++++++++++++ .github/workflows/mobile_release.yml | 7 +- .github/workflows/pr_notifier.yml | 7 +- .github/workflows/stale.yml | 7 +- .github/workflows/workflow-complete.yml | 61 +++++++++++++++++ .github/workflows/workflow-start.yml | 38 +++++++++++ bazel/repositories.bzl | 10 +++ ci/do_ci.sh | 33 ++++++++++ ci/run_envoy_docker.sh | 6 ++ ci/upload_gcs_artifact.sh | 4 +- tools/base/requirements.in | 2 +- 14 files changed, 312 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/envoy-verify.yml create mode 100644 .github/workflows/workflow-complete.yml create mode 100644 .github/workflows/workflow-start.yml diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml index 5f7ddabe3d02..238d7b4da974 100644 --- a/.azure-pipelines/stage/verify.yml +++ b/.azure-pipelines/stage/verify.yml @@ -9,55 +9,59 @@ parameters: - name: authGCP type: string default: "" +- name: authGithubWorkflow + type: string + default: "" +- name: authGithubWorkflowAppId + type: string + default: "" +- name: authGithubWorkflowInstallId + type: string + default: "" + +- name: runDocker + displayName: "Run Docker" + type: string + default: true + +# TODO(phlax): improve docker publishing job and move this there jobs: - job: examples displayName: Examples (Docker/x64) - condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true')) + condition: | + and(not(canceled()), + eq(${{ parameters.runDocker }}, 'true')) + timeoutInMinutes: 120 pool: vmImage: "ubuntu-20.04" steps: - - bash: .azure-pipelines/cleanup.sh - displayName: "Removing tools from agent" - - bash: | - set -e - - if [[ "$BUILD_REASON" == "PullRequest" ]]; then - DOWNLOAD_PATH="$(git rev-parse HEAD | head -c7)" - else - DOWNLOAD_PATH="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-${BUILD_SOURCEBRANCHNAME}}" - fi - - tmpdir=$(mktemp -d) - cd "$tmpdir" - images=("" "contrib" "google-vrp") - for image in "${images[@]}"; do - if [[ -n "$image" ]]; then - variant="${image}-dev" - filename="envoy-${image}.tar" - else - variant=dev - filename="envoy.tar" - fi - echo "Download docker image (https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/docker/${filename}) ..." - curl -sLO "https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/docker/${filename}" - echo "Copy oci image: oci-archive:${filename} docker-daemon:envoyproxy/envoy:${variant}" - skopeo copy -q "oci-archive:${filename}" "docker-daemon:envoyproxy/envoy:${variant}" - rm "$filename" - done - docker images | grep envoy + - task: DownloadSecureFile@1 + name: WorkflowTriggerKey + displayName: 'Download workflow trigger key' + inputs: + secureFile: '${{ parameters.authGithubWorkflow }}' - bash: | set -e - export DEBIAN_FRONTEND=noninteractive - sudo apt-get -qq update -y - sudo apt-get -qq install -y --no-install-recommends expect + KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)" + echo "##vso[task.setvariable variable=value;isoutput=true]$KEY" + name: key - - bash: ./ci/do_ci.sh verify_examples - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - NO_BUILD_SETUP: 1 - # ENVOY_EXAMPLES_DEBUG: 1 + - template: ../bazel.yml + parameters: + ciTarget: verify.trigger + authGithub: "$(key.value)" + # Please see `ci/do_ci.sh` for notes on required vars. + env: + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + ENVOY_BRANCH: "$(System.PullRequest.TargetBranch)" + ENVOY_COMMIT: "$(System.PullRequest.SourceCommitId)" + ENVOY_HEAD_REF: "$(Build.SourceBranch)" + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + ENVOY_BRANCH: "$(Build.SourceBranch)" + GITHUB_APP_ID: ${{ parameters.authGithubWorkflowAppId }} + GITHUB_INSTALL_ID: ${{ parameters.authGithubWorkflowInstallId }} - job: packages_x64 displayName: Debs (x64) @@ -83,6 +87,7 @@ jobs: BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} displayName: "Verify packages" + - job: packages_arm64 displayName: Debs (arm64) condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true')) diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index 6a36bb8da161..e678f892e541 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -169,11 +169,17 @@ stages: - stage: verify displayName: Verify dependsOn: ["env", "publish"] + variables: + RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']] jobs: - template: stage/verify.yml parameters: authGCP: $(GcpServiceAccountKey) bucketGCP: $(GcsArtifactBucket) + authGithubWorkflow: $(GitHubPublicRepoWorkflowKey) + authGithubWorkflowAppId: $(GitHubPublicRepoWorkflowAppId) + authGithubWorkflowInstallId: $(GitHubPublicRepoWorkflowInstallId) + runDocker: variables['RUN_DOCKER'] - stage: macos displayName: macOS diff --git a/.github/workflows/check-deps.yml b/.github/workflows/check-deps.yml index 1744e3fa332e..bb98ec2f36d7 100644 --- a/.github/workflows/check-deps.yml +++ b/.github/workflows/check-deps.yml @@ -3,7 +3,6 @@ name: Check dependencies on: schedule: - cron: '0 8 * * *' - workflow_dispatch: permissions: read-all @@ -11,7 +10,12 @@ permissions: read-all jobs: build: runs-on: ubuntu-20.04 - if: github.repository == 'envoyproxy/envoy' + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || !contains(github.actor, '[bot]')) + }} permissions: contents: read # to fetch code (actions/checkout) issues: write # required to open/close dependency issues diff --git a/.github/workflows/envoy-verify.yml b/.github/workflows/envoy-verify.yml new file mode 100644 index 000000000000..be10ceb57304 --- /dev/null +++ b/.github/workflows/envoy-verify.yml @@ -0,0 +1,88 @@ +name: Verify/examples + +on: + # This runs untrusted code, do not expose secrets in the verify job + workflow_dispatch: + inputs: + ref: + description: "Git SHA ref to checkout" + sha: + description: "Git SHA of commit HEAD (ie last commit of PR)" + head_ref: + description: "Ref for grouping PRs" + +concurrency: + group: ${{ github.event.inputs.head_ref || github.run_id }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check: + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (!contains(github.actor, '[bot]') + || github.actor == trigger-workflow-envoy[bot]) + }} + uses: ./.github/workflows/workflow-start.yml + permissions: + contents: read + statuses: write + with: + workflowName: ${{ github.workflow }} + + # Runs untrusted code + verify-examples: + runs-on: ubuntu-20.04 + needs: check + steps: + # Checkout the repo at provided commit + - name: 'Checkout Repository' + uses: actions/checkout@v3 + with: + ref: "${{ inputs.ref }}" + + - run: | + set -e + + BUCKET="envoy-postsubmit" + PULL_REGEX="^refs/pull/*" + if [[ "${{ inputs.head_ref }}" =~ ${PULL_REGEX} ]]; then + BUCKET="envoy-pr" + fi + + DOWNLOAD_PATH="$(echo "${{ github.event.inputs.sha }}" | head -c 7)" + + tmpdir=$(mktemp -d) + cd "$tmpdir" + images=("" "contrib" "google-vrp") + for image in "${images[@]}"; do + if [[ -n "$image" ]]; then + variant="${image}-dev" + filename="envoy-${image}.tar" + else + variant=dev + filename="envoy.tar" + fi + fileurl="https://storage.googleapis.com/${BUCKET}/${DOWNLOAD_PATH}/docker/${filename}" + echo "Download docker image (${fileurl}) ..." + curl -sLO "$fileurl" + echo "Copy oci image: oci-archive:${filename} docker-daemon:envoyproxy/envoy:${variant}" + skopeo copy -q "oci-archive:${filename}" "docker-daemon:envoyproxy/envoy:${variant}" + rm "$filename" + done + docker images | grep envoy + + - run: | + set -e + export DEBIAN_FRONTEND=noninteractive + sudo apt-get -qq update -y + sudo apt-get -qq install -y --no-install-recommends expect + + - run: | + ./ci/do_ci.sh verify_examples + env: + NO_BUILD_SETUP: 1 + # ENVOY_EXAMPLES_DEBUG: 1 diff --git a/.github/workflows/mobile_release.yml b/.github/workflows/mobile_release.yml index 8beed1ab6268..df91222186e8 100644 --- a/.github/workflows/mobile_release.yml +++ b/.github/workflows/mobile_release.yml @@ -8,7 +8,12 @@ on: jobs: android_release_artifacts: - if: github.repository == 'envoyproxy/envoy' + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || !contains(github.actor, '[bot]')) + }} name: android_release_artifacts runs-on: ubuntu-20.04 timeout-minutes: 120 diff --git a/.github/workflows/pr_notifier.yml b/.github/workflows/pr_notifier.yml index 782ad94075da..ac913f34f2e1 100644 --- a/.github/workflows/pr_notifier.yml +++ b/.github/workflows/pr_notifier.yml @@ -14,7 +14,12 @@ jobs: pull-requests: read # for pr_notifier.py name: PR Notifier runs-on: ubuntu-20.04 - if: github.repository == 'envoyproxy/envoy' + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || !contains(github.actor, '[bot]')) + }} steps: - uses: actions/checkout@v3 - name: Set up Python 3.8 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c544b7374e6f..faec7e043697 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,7 +10,12 @@ jobs: pull-requests: write # for actions/stale to close stale PRs name: Prune Stale runs-on: ubuntu-20.04 - if: github.repository == 'envoyproxy/envoy' + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || !contains(github.actor, '[bot]')) + }} steps: - name: Prune Stale diff --git a/.github/workflows/workflow-complete.yml b/.github/workflows/workflow-complete.yml new file mode 100644 index 000000000000..8b0d7f8c98d1 --- /dev/null +++ b/.github/workflows/workflow-complete.yml @@ -0,0 +1,61 @@ +name: Workflow complete +# This workflow is only required for externally triggered jobs that have manually +# set the check status for a commit/PR + +on: + # Do not run untrusted code here + workflow_run: + workflows: + - Verify/examples + types: + - completed + +permissions: + contents: read + +jobs: + complete: + runs-on: ubuntu-20.04 + permissions: + statuses: write + steps: + - name: 'Download artifact' + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "state_sha" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/state_sha.zip`, Buffer.from(download.data)); + + - run: | + set -e + unzip state_sha.zip + STATE_SHA="$(cat state_sha)" + echo "state_sha=$STATE_SHA" >> "$GITHUB_OUTPUT" + STATE="${{ github.event.workflow_run.conclusion }}" + if [[ ${STATE} != "success" ]]; then + STATE=failure + fi + echo "state=${STATE}" >> "$GITHUB_OUTPUT" + id: job + - name: Complete status check + uses: envoyproxy/toolshed/gh-actions/status@a6e1c951217efae1ac6b2bf32c5a9729976442b8 + with: + authToken: ${{ secrets.GITHUB_TOKEN }} + context: ${{ github.event.workflow.name }} + state: ${{ steps.job.outputs.state }} + sha: ${{ steps.job.outputs.state_sha }} + target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }} diff --git a/.github/workflows/workflow-start.yml b/.github/workflows/workflow-start.yml new file mode 100644 index 000000000000..64c2999c1201 --- /dev/null +++ b/.github/workflows/workflow-start.yml @@ -0,0 +1,38 @@ +name: Workflow start +# This workflow is only required for externally triggered jobs that need to manually +# set the check status for a commit/PR + +on: + workflow_call: + inputs: + workflowName: + required: true + type: string + +permissions: + contents: read + +jobs: + start: + runs-on: ubuntu-20.04 + permissions: + statuses: write + steps: + - name: Start status check + uses: envoyproxy/toolshed/gh-actions/status@a6e1c951217efae1ac6b2bf32c5a9729976442b8 + with: + authToken: ${{ secrets.GITHUB_TOKEN }} + context: ${{ inputs.workflowName }} + state: 'pending' + sha: ${{ inputs.sha }} + target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: Save the SHA + env: + STATE_SHA: ${{ inputs.sha }} + run: | + mkdir -p ./sha + echo $STATE_SHA > ./sha/state_sha + - uses: actions/upload-artifact@v3 + with: + name: state_sha + path: sha/ diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index c77baa9b132c..c52716a570cd 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -164,6 +164,16 @@ envoy_entry_point( script = "envoy.project", ) +envoy_entry_point( + name = "trigger", + args = [ + "trigger", + PATH, + ], + pkg = "envoy.base.utils", + script = "envoy.project", +) + ''') _envoy_repo = repository_rule( diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 4d516321753c..21fd47eae66d 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -717,6 +717,39 @@ case $CI_TARGET in run_ci_verify "*" "win32-front-proxy|shared" ;; + verify.trigger) + setup_clang_toolchain + WORKFLOW="envoy-verify.yml" + # * Note on vars * + # `ENVOY_REPO`: Should always be envoyproxy/envoy unless testing + # `ENVOY_BRANCH`: Target branch for PRs, source branch for others + # `COMMIT`: This may be a merge commit in a PR + # `ENVOY_COMMIT`: The actual last commit of branch/PR + # `ENVOY_HEAD_REF`: must also be set in PRs to provide a unique key for job grouping, + # cancellation, and to discriminate from other branch CI + COMMIT="$(git rev-parse HEAD)" + ENVOY_COMMIT="${ENVOY_COMMIT:-${COMMIT}}" + ENVOY_REPO="${ENVOY_REPO:-envoyproxy/envoy}" + echo "Trigger workflow (${WORKFLOW})" + echo " Repo: ${ENVOY_REPO}" + echo " Branch: ${ENVOY_BRANCH}" + echo " Ref: ${COMMIT}" + echo " Inputs:" + echo " sha: ${ENVOY_COMMIT}" + echo " head_ref: ${ENVOY_HEAD_REF}" + GITHUB_APP_KEY="$(echo "$GITHUB_TOKEN" | base64 -d -w0)" + export GITHUB_APP_KEY + INPUTS="{\"ref\":\"$COMMIT\",\"sha\":\"$ENVOY_COMMIT\",\"head_ref\":\"$ENVOY_HEAD_REF\"}" + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + @envoy_repo//:trigger \ + -- --repo="$ENVOY_REPO" \ + --trigger-app-id="$GITHUB_APP_ID" \ + --trigger-installation-id="$GITHUB_INSTALL_ID" \ + --trigger-ref="$ENVOY_BRANCH" \ + --trigger-workflow="$WORKFLOW" \ + --trigger-inputs="$INPUTS" + ;; + *) echo "Invalid do_ci.sh target (${CI_TARGET}), see ci/README.md for valid targets." exit 1 diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 30cbe0094613..7fd9bac73335 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -124,15 +124,21 @@ docker run --rm \ -e GOOGLE_BES_PROJECT_ID \ -e GCP_SERVICE_ACCOUNT_KEY \ -e NUM_CPUS \ + -e ENVOY_BRANCH \ -e ENVOY_RBE \ -e ENVOY_BUILD_IMAGE \ -e ENVOY_SRCDIR \ -e ENVOY_BUILD_TARGET \ -e ENVOY_BUILD_DEBUG_INFORMATION \ -e ENVOY_BUILD_FILTER_EXAMPLE \ + -e ENVOY_COMMIT \ + -e ENVOY_HEAD_REF \ + -e ENVOY_REPO \ -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ -e GCS_ARTIFACT_BUCKET \ -e GITHUB_TOKEN \ + -e GITHUB_APP_ID \ + -e GITHUB_INSTALL_ID \ -e NETLIFY_TRIGGER_URL \ -e BUILD_SOURCEBRANCHNAME \ -e BAZELISK_BASE_URL \ diff --git a/ci/upload_gcs_artifact.sh b/ci/upload_gcs_artifact.sh index 43c0032cc936..2c54feb13528 100755 --- a/ci/upload_gcs_artifact.sh +++ b/ci/upload_gcs_artifact.sh @@ -38,9 +38,9 @@ if [ ! -d "${SOURCE_DIRECTORY}" ]; then exit 1 fi -if [[ "$BUILD_REASON" == "PullRequest" ]] || [[ "$TARGET_SUFFIX" == "docs" ]]; then +if [[ "$BUILD_REASON" == "PullRequest" ]] || [[ "$TARGET_SUFFIX" == "docs" ]] || [[ "$TARGET_SUFFIX" == "docker" ]]; then # upload to the last commit sha (first 7 chars), either - # - docs build on main + # - docs/docker build on main, eg docs # -> https://storage.googleapis.com/envoy-postsubmit/$UPLOAD_PATH/docs/envoy-docs-rst.tar.gz # - PR build (commit sha from the developers branch) # -> https://storage.googleapis.com/envoy-pr/$UPLOAD_PATH/$TARGET_SUFFIX diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 69fd8d5d441f..75ee140ce992 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -5,7 +5,7 @@ cffi>=1.15.0 colorama coloredlogs dependatool>=0.2.2 -envoy.base.utils>=0.4.10 +envoy.base.utils>=0.4.11 envoy.code.check>=0.5.4 envoy.dependency.check>=0.1.7 envoy.distribution.release>=0.0.9 From 649057d0e138fcfed035e35959e2197fcb809c48 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 25 May 2023 12:30:13 -0400 Subject: [PATCH 377/740] test: reducing yaml in e2e tests (#27545) Signed-off-by: Alyssa Wilk --- .../integration/cds_integration_test.cc | 2 +- test/config/utility.cc | 243 ++++++++---------- test/config/utility.h | 18 +- test/integration/ads_integration.cc | 9 +- test/integration/ads_integration.h | 5 +- test/integration/ads_integration_test.cc | 10 +- test/integration/cds_integration_test.cc | 22 +- .../xds_config_tracker_integration_test.cc | 4 +- test/server/config_validation/xds_fuzz.cc | 2 +- 9 files changed, 143 insertions(+), 172 deletions(-) diff --git a/mobile/test/common/integration/cds_integration_test.cc b/mobile/test/common/integration/cds_integration_test.cc index f1a8c794e6d5..0720e4a9ce9c 100644 --- a/mobile/test/common/integration/cds_integration_test.cc +++ b/mobile/test/common/integration/cds_integration_test.cc @@ -41,7 +41,7 @@ class CdsIntegrationTest : public XdsIntegrationTest { use_xdstp_ ? cds_namespace_ + "/my_cluster?xds.node.cluster=envoy-mobile" : "my_cluster"; envoy::config::cluster::v3::Cluster cluster1 = ConfigHelper::buildStaticCluster( cluster_name, fake_upstreams_[0]->localAddress()->ip()->port(), - Network::Test::getLoopbackAddressString(ipVersion()), "ROUND_ROBIN"); + Network::Test::getLoopbackAddressString(ipVersion())); initializeXdsStream(); int cluster_count = getGaugeValue("cluster_manager.active_clusters"); // Do the initial compareDiscoveryRequest / sendDiscoveryResponse for cluster_1. diff --git a/test/config/utility.cc b/test/config/utility.cc index ab1d97d89575..ca33b39059f4 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -490,140 +490,109 @@ std::string ConfigHelper::adsBootstrap(const std::string& api_type) { } // TODO(samflattery): bundle this up with buildCluster -envoy::config::cluster::v3::Cluster ConfigHelper::buildStaticCluster(const std::string& name, - int port, - const std::string& address, - const std::string& lb_policy) { - return TestUtility::parseYaml( - fmt::format(R"EOF( - name: {} - connect_timeout: 5s - type: STATIC - load_assignment: - cluster_name: {} - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: {} - health_check_config: - address: - socket_address: - address: {} - port_value: {} - lb_policy: {} - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {{}} - )EOF", - name, name, address, port, address, port, lb_policy)); +envoy::config::cluster::v3::Cluster +ConfigHelper::buildStaticCluster(const std::string& name, int port, const std::string& address, + const envoy::config::cluster::v3::Cluster::LbPolicy lb_policy) { + envoy::config::cluster::v3::Cluster cluster; + cluster.mutable_connect_timeout()->set_seconds(5); + cluster.set_type(envoy::config::cluster::v3::Cluster::STATIC); + cluster.set_name(name); + cluster.mutable_load_assignment()->set_cluster_name(name); + auto* endpoint = + cluster.mutable_load_assignment()->add_endpoints()->add_lb_endpoints()->mutable_endpoint(); + auto* addr = endpoint->mutable_address(); + addr->mutable_socket_address()->set_address(address); + addr->mutable_socket_address()->set_port_value(port); + addr = endpoint->mutable_health_check_config()->mutable_address(); + addr->mutable_socket_address()->set_address(address); + addr->mutable_socket_address()->set_port_value(port); + cluster.set_lb_policy(lb_policy); + envoy::extensions::upstreams::http::v3::HttpProtocolOptions protocol_options; + protocol_options.mutable_explicit_http_config()->mutable_http2_protocol_options(); + + (*cluster.mutable_typed_extension_protocol_options()) + ["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"] + .PackFrom(protocol_options); + + return cluster; } envoy::config::cluster::v3::Cluster ConfigHelper::buildH1ClusterWithHighCircuitBreakersLimits( - const std::string& name, int port, const std::string& address, const std::string& lb_policy) { - return TestUtility::parseYaml( - fmt::format(R"EOF( - name: {} - connect_timeout: 50s - type: STATIC - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 10000 - max_pending_requests: 10000 - max_requests: 10000 - max_retries: 10000 - load_assignment: - cluster_name: {} - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: {} - lb_policy: {} - )EOF", - name, name, address, port, lb_policy)); + const std::string& name, int port, const std::string& address, + const envoy::config::cluster::v3::Cluster::LbPolicy lb_policy) { + envoy::config::cluster::v3::Cluster cluster; + cluster.set_name(name); + cluster.mutable_connect_timeout()->set_seconds(50); + cluster.set_type(envoy::config::cluster::v3::Cluster::STATIC); + auto* threshold = cluster.mutable_circuit_breakers()->mutable_thresholds()->Add(); + threshold->set_priority(envoy::config::core::v3::RoutingPriority::DEFAULT); + threshold->mutable_max_connections()->set_value(10000); + threshold->mutable_max_pending_requests()->set_value(10000); + threshold->mutable_max_requests()->set_value(10000); + threshold->mutable_max_retries()->set_value(10000); + cluster.mutable_load_assignment()->set_cluster_name(name); + auto* endpoint = + cluster.mutable_load_assignment()->add_endpoints()->add_lb_endpoints()->mutable_endpoint(); + cluster.set_lb_policy(lb_policy); + auto* addr = endpoint->mutable_address(); + addr->mutable_socket_address()->set_address(address); + addr->mutable_socket_address()->set_port_value(port); + return cluster; } -envoy::config::cluster::v3::Cluster ConfigHelper::buildCluster(const std::string& name, - const std::string& lb_policy) { - API_NO_BOOST(envoy::config::cluster::v3::Cluster) cluster; - TestUtility::loadFromYaml(fmt::format(R"EOF( - name: {} - connect_timeout: 5s - type: EDS - eds_cluster_config: - eds_config: - resource_api_version: V3 - ads: {{}} - lb_policy: {} - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {{}} - )EOF", - name, lb_policy), - cluster); +envoy::config::cluster::v3::Cluster +ConfigHelper::buildCluster(const std::string& name, + const envoy::config::cluster::v3::Cluster::LbPolicy lb_policy) { + envoy::config::cluster::v3::Cluster cluster; + cluster.mutable_connect_timeout()->set_seconds(5); + cluster.set_type(envoy::config::cluster::v3::Cluster::EDS); + cluster.set_name(name); + cluster.set_lb_policy(lb_policy); + + auto* eds = cluster.mutable_eds_cluster_config()->mutable_eds_config(); + eds->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + eds->mutable_ads(); + + envoy::extensions::upstreams::http::v3::HttpProtocolOptions protocol_options; + protocol_options.mutable_explicit_http_config()->mutable_http2_protocol_options(); + (*cluster.mutable_typed_extension_protocol_options()) + ["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"] + .PackFrom(protocol_options); + return cluster; } -envoy::config::cluster::v3::Cluster ConfigHelper::buildTlsCluster(const std::string& name, - const std::string& lb_policy) { - API_NO_BOOST(envoy::config::cluster::v3::Cluster) cluster; - TestUtility::loadFromYaml( - fmt::format(R"EOF( - name: {} - connect_timeout: 5s - type: EDS - eds_cluster_config: - eds_config: - resource_api_version: V3 - ads: {{}} - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext - common_tls_context: - validation_context: - trusted_ca: - filename: {} - lb_policy: {} - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {{}} - )EOF", - name, - TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem"), - lb_policy), - cluster); +envoy::config::cluster::v3::Cluster +ConfigHelper::buildTlsCluster(const std::string& name, + const envoy::config::cluster::v3::Cluster::LbPolicy lb_policy) { + envoy::config::cluster::v3::Cluster cluster = buildCluster(name, lb_policy); + + auto* socket = cluster.mutable_transport_socket(); + envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_socket; + tls_socket.mutable_common_tls_context() + ->mutable_validation_context() + ->mutable_trusted_ca() + ->set_filename( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); + socket->set_name("envoy.transport_sockets.tls"); + socket->mutable_typed_config()->PackFrom(tls_socket); + return cluster; } envoy::config::endpoint::v3::ClusterLoadAssignment -ConfigHelper::buildClusterLoadAssignment(const std::string& name, const std::string& address, +ConfigHelper::buildClusterLoadAssignment(const std::string& name, const std::string& address_str, uint32_t port) { API_NO_BOOST(envoy::config::endpoint::v3::ClusterLoadAssignment) cluster_load_assignment; - TestUtility::loadFromYaml(fmt::format(R"EOF( - cluster_name: {} - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: {} - )EOF", - name, address, port), - cluster_load_assignment); + cluster_load_assignment.set_cluster_name(name); + auto* address = cluster_load_assignment.add_endpoints() + ->add_lb_endpoints() + ->mutable_endpoint() + ->mutable_address() + ->mutable_socket_address(); + address->set_address(address_str); + address->set_port_value(port); + return cluster_load_assignment; } @@ -631,32 +600,22 @@ envoy::config::endpoint::v3::ClusterLoadAssignment ConfigHelper::buildClusterLoadAssignmentWithLeds(const std::string& name, const std::string& leds_collection_name) { API_NO_BOOST(envoy::config::endpoint::v3::ClusterLoadAssignment) cluster_load_assignment; - TestUtility::loadFromYaml(fmt::format(R"EOF( - cluster_name: {} - endpoints: - leds_cluster_locality_config: - leds_config: - resource_api_version: V3 - ads: {{}} - leds_collection_name: {} - )EOF", - name, leds_collection_name), - cluster_load_assignment); + + cluster_load_assignment.set_cluster_name(name); + auto* lclc = cluster_load_assignment.add_endpoints()->mutable_leds_cluster_locality_config(); + auto* leds = lclc->mutable_leds_config(); + leds->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + leds->mutable_ads(); + lclc->set_leds_collection_name(leds_collection_name); return cluster_load_assignment; } -envoy::config::endpoint::v3::LbEndpoint ConfigHelper::buildLbEndpoint(const std::string& address, - uint32_t port) { +envoy::config::endpoint::v3::LbEndpoint +ConfigHelper::buildLbEndpoint(const std::string& address_str, uint32_t port) { API_NO_BOOST(envoy::config::endpoint::v3::LbEndpoint) lb_endpoint; - TestUtility::loadFromYaml(fmt::format(R"EOF( - endpoint: - address: - socket_address: - address: {} - port_value: {} - )EOF", - address, port), - lb_endpoint); + auto* address = lb_endpoint.mutable_endpoint()->mutable_address()->mutable_socket_address(); + address->set_address(address_str); + address->set_port_value(port); return lb_endpoint; } diff --git a/test/config/utility.h b/test/config/utility.h index 61c7ee881a75..9f8101b7c604 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -208,19 +208,23 @@ class ConfigHelper { // Builds a standard Cluster config fragment, with a single endpoint (at address:port). static envoy::config::cluster::v3::Cluster buildStaticCluster(const std::string& name, int port, const std::string& address, - const std::string& lb_policy = "ROUND_ROBIN"); + const envoy::config::cluster::v3::Cluster::LbPolicy lb_policy = + envoy::config::cluster::v3::Cluster::ROUND_ROBIN); - static envoy::config::cluster::v3::Cluster - buildH1ClusterWithHighCircuitBreakersLimits(const std::string& name, int port, - const std::string& address, - const std::string& lb_policy = "ROUND_ROBIN"); + static envoy::config::cluster::v3::Cluster buildH1ClusterWithHighCircuitBreakersLimits( + const std::string& name, int port, const std::string& address, + const envoy::config::cluster::v3::Cluster::LbPolicy lb_policy); // ADS configurations static envoy::config::cluster::v3::Cluster - buildCluster(const std::string& name, const std::string& lb_policy = "ROUND_ROBIN"); + buildCluster(const std::string& name, + const envoy::config::cluster::v3::Cluster::LbPolicy lb_policy = + envoy::config::cluster::v3::Cluster::ROUND_ROBIN); static envoy::config::cluster::v3::Cluster - buildTlsCluster(const std::string& name, const std::string& lb_policy = "ROUND_ROBIN"); + buildTlsCluster(const std::string& name, + const envoy::config::cluster::v3::Cluster::LbPolicy lb_policy = + envoy::config::cluster::v3::Cluster::ROUND_ROBIN); static envoy::config::endpoint::v3::ClusterLoadAssignment buildClusterLoadAssignment(const std::string& name, const std::string& ip_version, uint32_t port); diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index bff190ca146d..8272c56be18f 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -47,17 +47,18 @@ AdsIntegrationTest::AdsIntegrationTest() void AdsIntegrationTest::TearDown() { cleanUpXdsConnection(); } -envoy::config::cluster::v3::Cluster AdsIntegrationTest::buildCluster(const std::string& name, - const std::string& lb_policy) { +envoy::config::cluster::v3::Cluster +AdsIntegrationTest::buildCluster(const std::string& name, + envoy::config::cluster::v3::Cluster::LbPolicy lb_policy) { return ConfigHelper::buildCluster(name, lb_policy); } envoy::config::cluster::v3::Cluster AdsIntegrationTest::buildTlsCluster(const std::string& name) { - return ConfigHelper::buildTlsCluster(name, "ROUND_ROBIN"); + return ConfigHelper::buildTlsCluster(name, envoy::config::cluster::v3::Cluster::ROUND_ROBIN); } envoy::config::cluster::v3::Cluster AdsIntegrationTest::buildRedisCluster(const std::string& name) { - return ConfigHelper::buildCluster(name, "MAGLEV"); + return ConfigHelper::buildCluster(name, envoy::config::cluster::v3::Cluster::MAGLEV); } envoy::config::endpoint::v3::ClusterLoadAssignment diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h index 43232c8ab070..57e9f65e2c2e 100644 --- a/test/integration/ads_integration.h +++ b/test/integration/ads_integration.h @@ -49,8 +49,9 @@ class AdsIntegrationTest : public AdsDeltaSotwIntegrationSubStateParamTest, void TearDown() override; - envoy::config::cluster::v3::Cluster buildCluster(const std::string& name, - const std::string& lb_policy = "ROUND_ROBIN"); + envoy::config::cluster::v3::Cluster + buildCluster(const std::string& name, envoy::config::cluster::v3::Cluster::LbPolicy lb_policy = + envoy::config::cluster::v3::Cluster::ROUND_ROBIN); envoy::config::cluster::v3::Cluster buildTlsCluster(const std::string& name); diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 66242df004ba..1c6c060800ca 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -109,8 +109,8 @@ TEST_P(AdsIntegrationTest, ClusterInitializationUpdateTheOnlyWarmingCluster) { test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); // Update lb policy to MAGLEV so that cluster update is not skipped due to the same hash. sendDiscoveryResponse( - cds_type_url, {buildCluster("cluster_0", "MAGLEV")}, {buildCluster("cluster_0", "MAGLEV")}, - {}, "2"); + cds_type_url, {buildCluster("cluster_0", envoy::config::cluster::v3::Cluster::MAGLEV)}, + {buildCluster("cluster_0", envoy::config::cluster::v3::Cluster::MAGLEV)}, {}, "2"); EXPECT_TRUE(compareDiscoveryRequest(eds_type_url, "", {"cluster_0"}, {"cluster_0"}, {})); sendDiscoveryResponse( eds_type_url, {buildClusterLoadAssignment("cluster_0")}, @@ -190,9 +190,11 @@ TEST_P(AdsIntegrationTest, ClusterInitializationUpdateOneOfThe2Warming) { sendDiscoveryResponse( cds_type_url, {ConfigHelper::buildStaticCluster("primary_cluster", 8000, "127.0.0.1"), - buildCluster("cluster_0", "MAGLEV"), buildCluster("cluster_1")}, + buildCluster("cluster_0", envoy::config::cluster::v3::Cluster::MAGLEV), + buildCluster("cluster_1")}, {ConfigHelper::buildStaticCluster("primary_cluster", 8000, "127.0.0.1"), - buildCluster("cluster_0", "MAGLEV"), buildCluster("cluster_1")}, + buildCluster("cluster_0", envoy::config::cluster::v3::Cluster::MAGLEV), + buildCluster("cluster_1")}, {}, "2"); EXPECT_TRUE(compareDiscoveryRequest(eds_type_url, "", {"cluster_0", "cluster_1"}, {"cluster_0", "cluster_1"}, {})); diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index 8e03b0421201..f3b0db0ce4b4 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -83,12 +83,14 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht // cluster in the bootstrap config - which we don't want since we're testing dynamic CDS! addFakeUpstream(upstream_codec_type_); addFakeUpstream(upstream_codec_type_); - cluster1_ = cluster_creator_( - ClusterName1, fake_upstreams_[UpstreamIndex1]->localAddress()->ip()->port(), - Network::Test::getLoopbackAddressString(ipVersion()), "ROUND_ROBIN"); - cluster2_ = cluster_creator_( - ClusterName2, fake_upstreams_[UpstreamIndex2]->localAddress()->ip()->port(), - Network::Test::getLoopbackAddressString(ipVersion()), "ROUND_ROBIN"); + cluster1_ = cluster_creator_(ClusterName1, + fake_upstreams_[UpstreamIndex1]->localAddress()->ip()->port(), + Network::Test::getLoopbackAddressString(ipVersion()), + envoy::config::cluster::v3::Cluster::ROUND_ROBIN); + cluster2_ = cluster_creator_(ClusterName2, + fake_upstreams_[UpstreamIndex2]->localAddress()->ip()->port(), + Network::Test::getLoopbackAddressString(ipVersion()), + envoy::config::cluster::v3::Cluster::ROUND_ROBIN); // Let Envoy establish its connection to the CDS server. acceptXdsConnection(); @@ -136,8 +138,9 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht // True if we decided not to run the test after all. bool test_skipped_{true}; Http::CodecType upstream_codec_type_{Http::CodecType::HTTP2}; - std::function + std::function cluster_creator_; }; @@ -239,7 +242,8 @@ TEST_P(CdsIntegrationTest, CdsClusterWithThreadAwareLbCycleUpDownUp) { // Update cluster1_ to use MAGLEV load balancer policy. cluster1_ = ConfigHelper::buildStaticCluster( ClusterName1, fake_upstreams_[UpstreamIndex1]->localAddress()->ip()->port(), - Network::Test::getLoopbackAddressString(ipVersion()), "MAGLEV"); + Network::Test::getLoopbackAddressString(ipVersion()), + envoy::config::cluster::v3::Cluster::MAGLEV); // Cyclically add and remove cluster with ThreadAwareLb. for (int i = 42; i < 142; i += 2) { diff --git a/test/integration/xds_config_tracker_integration_test.cc b/test/integration/xds_config_tracker_integration_test.cc index 96f58bdaf79c..bbc6fc04920e 100644 --- a/test/integration/xds_config_tracker_integration_test.cc +++ b/test/integration/xds_config_tracker_integration_test.cc @@ -138,10 +138,10 @@ class XdsConfigTrackerIntegrationTest : public Grpc::DeltaSotwIntegrationParamTe addFakeUpstream(Http::CodecType::HTTP2); cluster1_ = ConfigHelper::buildStaticCluster( ClusterName1, fake_upstreams_[UpstreamIndex1]->localAddress()->ip()->port(), - Network::Test::getLoopbackAddressString(ipVersion()), "ROUND_ROBIN"); + Network::Test::getLoopbackAddressString(ipVersion())); cluster2_ = ConfigHelper::buildStaticCluster( ClusterName2, fake_upstreams_[UpstreamIndex2]->localAddress()->ip()->port(), - Network::Test::getLoopbackAddressString(ipVersion()), "ROUND_ROBIN"); + Network::Test::getLoopbackAddressString(ipVersion())); acceptXdsConnection(); registerTestServerPorts({}); diff --git a/test/server/config_validation/xds_fuzz.cc b/test/server/config_validation/xds_fuzz.cc index 43ec342341c4..295c2b8b0d8b 100644 --- a/test/server/config_validation/xds_fuzz.cc +++ b/test/server/config_validation/xds_fuzz.cc @@ -12,7 +12,7 @@ namespace Envoy { // Helper functions to build API responses. envoy::config::cluster::v3::Cluster XdsFuzzTest::buildCluster(const std::string& name) { - return ConfigHelper::buildCluster(name, "ROUND_ROBIN"); + return ConfigHelper::buildCluster(name); }; envoy::config::endpoint::v3::ClusterLoadAssignment From 4168a90c65e1022073d154eb6ad24552d2cd9f4b Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 25 May 2023 17:47:56 +0100 Subject: [PATCH 378/740] ci: Cleanup/fix for verify move (#27635) Signed-off-by: Ryan Northey --- .github/workflows/envoy-verify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/envoy-verify.yml b/.github/workflows/envoy-verify.yml index be10ceb57304..adeec2c8487c 100644 --- a/.github/workflows/envoy-verify.yml +++ b/.github/workflows/envoy-verify.yml @@ -24,7 +24,7 @@ jobs: ${{ github.repository == 'envoyproxy/envoy' && (!contains(github.actor, '[bot]') - || github.actor == trigger-workflow-envoy[bot]) + || github.actor == 'trigger-workflow-envoy[bot]') }} uses: ./.github/workflows/workflow-start.yml permissions: From dde960032500887867a20557d8e5c7e29753fae0 Mon Sep 17 00:00:00 2001 From: "Dr. Andre Vehreschild" <101638173+vehre-x41@users.noreply.github.com> Date: Thu, 25 May 2023 19:14:39 +0200 Subject: [PATCH 379/740] [fuzz] Remove conn_manager from writefilter allow list (#27633) Signed-off-by: Andre Vehreschild --- .../filters/network/common/fuzz/uber_per_writefilter.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc b/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc index 696935566fb4..9ea866b9ab40 100644 --- a/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc +++ b/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc @@ -15,6 +15,10 @@ std::vector UberWriteFilterFuzzer::filterNames() { // See test/extensions/filters/network/common/fuzz/BUILD for more information. filter_names = Registry::FactoryRegistry< Server::Configuration::NamedNetworkFilterConfigFactory>::registeredNames(); + // http_connection_manager gets into the build by dependencies, but shall not be + // fuzzed in the write filter. + filter_names.erase(std::remove(filter_names.begin(), filter_names.end(), + "envoy.filters.network.http_connection_manager")); } return filter_names; } From e2ec5a085e67109f7062bbbe92aa05deb1ae6c62 Mon Sep 17 00:00:00 2001 From: "Dr. Andre Vehreschild" <101638173+vehre-x41@users.noreply.github.com> Date: Thu, 25 May 2023 20:04:47 +0200 Subject: [PATCH 380/740] [fuzz] Prevent using too deeply nested configurations by cut-off (#27631) Signed-off-by: Andre Vehreschild --- test/fuzz/mutable_visitor.cc | 2 +- test/fuzz/mutable_visitor.h | 4 +-- test/fuzz/validated_input_generator.cc | 37 ++++++++++++++------------ test/fuzz/validated_input_generator.h | 11 ++++---- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/test/fuzz/mutable_visitor.cc b/test/fuzz/mutable_visitor.cc index b011d2caff42..4e61a07e5dc9 100644 --- a/test/fuzz/mutable_visitor.cc +++ b/test/fuzz/mutable_visitor.cc @@ -16,7 +16,7 @@ namespace { void traverseMessageWorkerExt(ProtoVisitor& visitor, Protobuf::Message& message, std::vector& parents, bool was_any_or_top_level, bool recurse_into_any, - absl::string_view const& field_name) { + absl::string_view field_name) { visitor.onEnterMessage(message, parents, was_any_or_top_level, field_name); absl::Cleanup message_leaver = [&visitor, &parents, &message, was_any_or_top_level, field_name] { visitor.onLeaveMessage(message, parents, was_any_or_top_level, field_name); diff --git a/test/fuzz/mutable_visitor.h b/test/fuzz/mutable_visitor.h index 789c1dfbd8de..6cc6407de4ee 100644 --- a/test/fuzz/mutable_visitor.h +++ b/test/fuzz/mutable_visitor.h @@ -22,9 +22,9 @@ class ProtoVisitor { // Any before being unpacked for further recursion. The latter can // only be achieved by using recurse_into_any. virtual void onEnterMessage(Protobuf::Message&, absl::Span, - bool was_any_or_top_level, absl::string_view const& field_name) PURE; + bool was_any_or_top_level, absl::string_view field_name) PURE; virtual void onLeaveMessage(Protobuf::Message&, absl::Span, - bool was_any_or_top_level, absl::string_view const& field_name) PURE; + bool was_any_or_top_level, absl::string_view field_name) PURE; }; void traverseMessage(ProtoVisitor& visitor, Protobuf::Message& message, bool recurse_into_any); diff --git a/test/fuzz/validated_input_generator.cc b/test/fuzz/validated_input_generator.cc index 0eb859d10859..db9e4f3700c7 100644 --- a/test/fuzz/validated_input_generator.cc +++ b/test/fuzz/validated_input_generator.cc @@ -206,7 +206,7 @@ void ValidatedInputGenerator::handleAnyRules( void ValidatedInputGenerator::handleMessageTypedField( Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, const Protobuf::Reflection* reflection, const validate::FieldRules& rules, - const absl::Span& parents, const bool force_create) { + const absl::Span& parents, bool force_create, bool cut_off) { if (field.is_repeated()) { const validate::RepeatedRules& repeated_rules = rules.repeated(); @@ -250,6 +250,9 @@ void ValidatedInputGenerator::handleMessageTypedField( break; } default: + if (cut_off) { + value->Clear(); + } break; } } @@ -264,7 +267,7 @@ void ValidatedInputGenerator::handleIntrinsicTypedField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, const Protobuf::Reflection* reflection, const validate::FieldRules& rules, - const bool force) { + bool force) { if (field.is_repeated()) { const validate::RepeatedRules& repeated_rules = rules.repeated(); @@ -314,13 +317,13 @@ void ValidatedInputGenerator::handleIntrinsicTypedField(Protobuf::Message& msg, void ValidatedInputGenerator::onField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, const absl::Span parents) { - onField(msg, field, parents, false); + onField(msg, field, parents, false, false); } void ValidatedInputGenerator::onField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, const absl::Span parents, - const bool force_create) { + bool force_create, bool cut_off) { const Protobuf::Reflection* reflection = msg.GetReflection(); if (!field.options().HasExtension(validate::rules) && !force_create) { @@ -393,7 +396,7 @@ void ValidatedInputGenerator::onField(Protobuf::Message& msg, break; } case Protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { - handleMessageTypedField(msg, field, reflection, rules, parents, force_create); + handleMessageTypedField(msg, field, reflection, rules, parents, force_create, cut_off); break; } default: @@ -403,7 +406,7 @@ void ValidatedInputGenerator::onField(Protobuf::Message& msg, void ValidatedInputGenerator::onEnterMessage(Protobuf::Message& msg, absl::Span parents, - bool, absl::string_view const& field_name) { + bool, absl::string_view field_name) { ++current_depth_; const Protobuf::Reflection* reflection = msg.GetReflection(); const Protobuf::Descriptor* descriptor = msg.GetDescriptor(); @@ -416,28 +419,28 @@ void ValidatedInputGenerator::onEnterMessage(Protobuf::Message& msg, any_message->Clear(); } } + const bool max_depth_exceeded = max_depth_ > 0 && current_depth_ > max_depth_; for (int oneof_index = 0; oneof_index < descriptor->oneof_decl_count(); ++oneof_index) { const Protobuf::OneofDescriptor* oneof_desc = descriptor->oneof_decl(oneof_index); - if (oneof_desc->options().HasExtension(validate::required) && - oneof_desc->options().GetExtension(validate::required) && - !reflection->HasOneof(msg, descriptor->oneof_decl(oneof_index))) { + if (max_depth_exceeded || (oneof_desc->options().HasExtension(validate::required) && + oneof_desc->options().GetExtension(validate::required) && + !reflection->HasOneof(msg, descriptor->oneof_decl(oneof_index)))) { // No required member in one of set, so create one. for (int index = 0; index < oneof_desc->field_count(); ++index) { const std::string parents_class_name = parents.back()->GetDescriptor()->full_name(); // Treat matchers special, because in their oneof they reference themselves, which may // create long chains. Prefer the first alternative, which does not reference itself. // Nevertheless do it randomly to allow for some nesting. - if ((max_depth_ > 0 && current_depth_ > max_depth_) || - ((parents_class_name == "xds.type.matcher.v3.Matcher.MatcherList.Predicate" || - parents_class_name == - "xds.type.matcher.v3.Matcher.MatcherList.Predicate.SinglePredicate") && - (random_() % 200) > 0)) { - onField(msg, *oneof_desc->field(0), parents, true); + if ((parents_class_name == "xds.type.matcher.v3.Matcher.MatcherList.Predicate" || + parents_class_name == + "xds.type.matcher.v3.Matcher.MatcherList.Predicate.SinglePredicate") && + (random_() % 200) > 0) { + onField(msg, *oneof_desc->field(0), parents, true, max_depth_exceeded); } else { // Do not use the first available alternative all the time, because of cyclic // dependencies. const int rnd_index = random_() % oneof_desc->field_count(); - onField(msg, *oneof_desc->field(rnd_index), parents, true); + onField(msg, *oneof_desc->field(rnd_index), parents, true, max_depth_exceeded); } // Check if for the above field an entry could be created and quit the inner loop if so. // It might not be possible, when the datatype is not supported (yet). @@ -451,7 +454,7 @@ void ValidatedInputGenerator::onEnterMessage(Protobuf::Message& msg, void ValidatedInputGenerator::onLeaveMessage(Protobuf::Message&, absl::Span, bool, - absl::string_view const&) { + absl::string_view) { message_path_.pop_back(); --current_depth_; } diff --git a/test/fuzz/validated_input_generator.h b/test/fuzz/validated_input_generator.h index d7844217bbfe..7074e43c0852 100644 --- a/test/fuzz/validated_input_generator.h +++ b/test/fuzz/validated_input_generator.h @@ -42,7 +42,7 @@ class ValidatedInputGenerator : public ProtobufMessage::ProtoVisitor, private pg const Protobuf::Reflection* reflection, const validate::FieldRules& rules, const absl::Span& parents, - const bool force_create); + bool force_create, bool cut_off); // Handle all validation rules for intrinsic types like int, uint and string. // Messages are more complicated to handle and can not be handled here. @@ -52,19 +52,20 @@ class ValidatedInputGenerator : public ProtobufMessage::ProtoVisitor, private pg T, typename std::result_of::type>> void handleIntrinsicTypedField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, const Protobuf::Reflection* reflection, - const validate::FieldRules& rules, const bool force); + const validate::FieldRules& rules, bool force); void onField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, const absl::Span parents) override; void onField(Protobuf::Message& msg, const Protobuf::FieldDescriptor& field, - const absl::Span parents, const bool force_create); + const absl::Span parents, bool force_create, + bool cut_off); void onEnterMessage(Protobuf::Message& msg, absl::Span parents, - bool, absl::string_view const& field_name) override; + bool, absl::string_view field_name) override; void onLeaveMessage(Protobuf::Message&, absl::Span, bool, - absl::string_view const&) override; + absl::string_view) override; private: Mutator mutator_; From 693dab2c88a43c55185b373bbc8e955cf9afa7ad Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 25 May 2023 19:09:10 +0100 Subject: [PATCH 381/740] ci: Further cleanup/fix for verify move (#27636) Signed-off-by: Ryan Northey --- .github/workflows/envoy-verify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/envoy-verify.yml b/.github/workflows/envoy-verify.yml index adeec2c8487c..7682fe649507 100644 --- a/.github/workflows/envoy-verify.yml +++ b/.github/workflows/envoy-verify.yml @@ -53,7 +53,7 @@ jobs: BUCKET="envoy-pr" fi - DOWNLOAD_PATH="$(echo "${{ github.event.inputs.sha }}" | head -c 7)" + DOWNLOAD_PATH="$(echo "${{ inputs.ref }}" | head -c 7)" tmpdir=$(mktemp -d) cd "$tmpdir" From 17b40953f28219caf3f40e273a0ee361f6bc0826 Mon Sep 17 00:00:00 2001 From: Christoph Pakulski Date: Thu, 25 May 2023 16:46:28 -0400 Subject: [PATCH 382/740] router: add unit tests for router/delegating_route_impl (#27616) add unit tests for router/delegating_route_impl Additional Description: This is to increase coverage for common/router code which dropped after #27360. Risk Level: Low. Testing: Docs Changes: No Release Notes: No Platform Specific Features: N Fixes #27406 Signed-off-by: Christoph Pakulski --- source/common/router/delegating_route_impl.cc | 4 + source/common/router/delegating_route_impl.h | 1 + test/common/router/BUILD | 11 +++ .../router/delegating_route_impl_test.cc | 96 +++++++++++++++++++ test/mocks/router/mocks.cc | 6 ++ test/mocks/router/mocks.h | 20 ++-- test/per_file_coverage.sh | 2 +- 7 files changed, 133 insertions(+), 7 deletions(-) create mode 100644 test/common/router/delegating_route_impl_test.cc diff --git a/source/common/router/delegating_route_impl.cc b/source/common/router/delegating_route_impl.cc index fe0df35d47b0..fb117da3bcd3 100644 --- a/source/common/router/delegating_route_impl.cc +++ b/source/common/router/delegating_route_impl.cc @@ -189,5 +189,9 @@ const EarlyDataPolicy& DelegatingRouteEntry::earlyDataPolicy() const { return base_route_->routeEntry()->earlyDataPolicy(); } +const RouteStatsContextOptRef DelegatingRouteEntry::routeStatsContext() const { + return base_route_->routeEntry()->routeStatsContext(); +} + } // namespace Router } // namespace Envoy diff --git a/source/common/router/delegating_route_impl.h b/source/common/router/delegating_route_impl.h index 9677945ed23c..422f770bded6 100644 --- a/source/common/router/delegating_route_impl.h +++ b/source/common/router/delegating_route_impl.h @@ -112,6 +112,7 @@ class DelegatingRouteEntry : public Router::RouteEntry { const ConnectConfigOptRef connectConfig() const override; const std::string& routeName() const override; const EarlyDataPolicy& earlyDataPolicy() const override; + const RouteStatsContextOptRef routeStatsContext() const override; private: const Router::RouteConstSharedPtr base_route_; diff --git a/test/common/router/BUILD b/test/common/router/BUILD index a01f8c91bf96..affe611e5150 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -488,6 +488,17 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "delegating_route_impl_test", + srcs = ["delegating_route_impl_test.cc"], + deps = [ + "//source/common/router:config_lib", + "//source/common/router:delegating_route_lib", + "//test/mocks/router:router_mocks", + "//test/mocks/stream_info:stream_info_mocks", + ], +) + envoy_proto_library( name = "router_fuzz_proto", srcs = ["router_fuzz.proto"], diff --git a/test/common/router/delegating_route_impl_test.cc b/test/common/router/delegating_route_impl_test.cc new file mode 100644 index 000000000000..b09cd55899d5 --- /dev/null +++ b/test/common/router/delegating_route_impl_test.cc @@ -0,0 +1,96 @@ +#include "source/common/router/delegating_route_impl.h" + +#include "test/mocks/router/mocks.h" +#include "test/mocks/stream_info/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Router { +namespace { + +using testing::Return; + +#define TEST_METHOD(method, ...) \ + EXPECT_CALL(*inner_object_ptr, method); \ + wrapper_object.method(__VA_ARGS__); + +// Verify that DelegatingRoute class forwards all calls to internal base route. +TEST(DelegatingRoute, DelegatingRouteTest) { + const std::shared_ptr inner_object_ptr = std::make_shared(); + DelegatingRoute wrapper_object(inner_object_ptr); + + TEST_METHOD(directResponseEntry); + TEST_METHOD(routeEntry); + TEST_METHOD(decorator); + TEST_METHOD(tracingConfig); + TEST_METHOD(metadata); + TEST_METHOD(typedMetadata); + + std::string name; + TEST_METHOD(mostSpecificPerFilterConfig, name); + + std::function cb; + TEST_METHOD(traversePerFilterConfig, name, cb); +} + +// Verify that DelegatingRouteEntry class forwards all calls to internal base route. +TEST(DelegatingRouteEntry, DelegatingRouteEntryTest) { + const std::shared_ptr base_route_ptr = std::make_shared(); + const std::shared_ptr inner_object_ptr = std::make_shared(); + DelegatingRouteEntry wrapper_object(base_route_ptr); + EXPECT_CALL(*base_route_ptr, routeEntry).WillRepeatedly(Return(inner_object_ptr.get())); + + Http::TestRequestHeaderMapImpl request_headers; + Http::TestResponseHeaderMapImpl response_headers; + StreamInfo::MockStreamInfo stream_info; + + TEST_METHOD(finalizeResponseHeaders, response_headers, stream_info); + TEST_METHOD(responseHeaderTransforms, stream_info); + TEST_METHOD(clusterName); + TEST_METHOD(clusterNotFoundResponseCode); + TEST_METHOD(corsPolicy); + TEST_METHOD(currentUrlPathAfterRewrite, request_headers); + TEST_METHOD(finalizeRequestHeaders, request_headers, stream_info, true); + TEST_METHOD(requestHeaderTransforms, stream_info); + TEST_METHOD(hashPolicy); + TEST_METHOD(hedgePolicy); + TEST_METHOD(priority); + TEST_METHOD(rateLimitPolicy); + TEST_METHOD(retryPolicy); + TEST_METHOD(pathMatcher); + TEST_METHOD(pathRewriter); + TEST_METHOD(internalRedirectPolicy); + TEST_METHOD(retryShadowBufferLimit); + TEST_METHOD(shadowPolicies); + TEST_METHOD(timeout); + TEST_METHOD(idleTimeout); + TEST_METHOD(usingNewTimeouts); + TEST_METHOD(maxStreamDuration); + TEST_METHOD(grpcTimeoutHeaderMax); + TEST_METHOD(grpcTimeoutHeaderOffset); + TEST_METHOD(maxGrpcTimeout); + TEST_METHOD(grpcTimeoutOffset); + TEST_METHOD(virtualCluster, request_headers); + TEST_METHOD(virtualHost); + TEST_METHOD(autoHostRewrite); + TEST_METHOD(appendXfh); + TEST_METHOD(metadataMatchCriteria); + TEST_METHOD(opaqueConfig); + TEST_METHOD(includeVirtualHostRateLimits); + TEST_METHOD(tlsContextMatchCriteria); + TEST_METHOD(pathMatchCriterion); + TEST_METHOD(includeAttemptCountInRequest); + TEST_METHOD(includeAttemptCountInResponse); + TEST_METHOD(upgradeMap); + TEST_METHOD(connectConfig); + TEST_METHOD(routeName); + TEST_METHOD(earlyDataPolicy); + TEST_METHOD(routeStatsContext); +} + +} // namespace +} // namespace Router +} // namespace Envoy diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index 3c77af507b18..caeed352f87f 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -115,6 +115,11 @@ MockRouteEntry::MockRouteEntry() { return connect_config_.has_value() ? makeOptRef(connect_config_.value()) : absl::nullopt; })); ON_CALL(*this, earlyDataPolicy()).WillByDefault(ReturnRef(early_data_policy_)); + path_matcher_ = std::make_shared>(); + ON_CALL(*this, pathMatcher()).WillByDefault(ReturnRef(path_matcher_)); + path_rewriter_ = std::make_shared>(); + ON_CALL(*this, pathRewriter()).WillByDefault(ReturnRef(path_rewriter_)); + ON_CALL(*this, routeStatsContext()).WillByDefault(Return(RouteStatsContextOptRef())); } MockRouteEntry::~MockRouteEntry() = default; @@ -143,6 +148,7 @@ MockRoute::MockRoute() { ON_CALL(*this, decorator()).WillByDefault(Return(&decorator_)); ON_CALL(*this, tracingConfig()).WillByDefault(Return(nullptr)); ON_CALL(*this, metadata()).WillByDefault(ReturnRef(metadata_)); + ON_CALL(*this, typedMetadata()).WillByDefault(ReturnRef(typed_metadata_)); } MockRoute::~MockRoute() = default; diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 9f5976bd814f..8acad60e6201 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -438,10 +438,7 @@ class MockRouteEntry : public RouteEntry { MOCK_METHOD(const UpgradeMap&, upgradeMap, (), (const)); MOCK_METHOD(const std::string&, routeName, (), (const)); MOCK_METHOD(const EarlyDataPolicy&, earlyDataPolicy, (), (const)); - - const RouteStatsContextOptRef routeStatsContext() const override { - return RouteStatsContextOptRef(); - } + MOCK_METHOD(const RouteStatsContextOptRef, routeStatsContext, (), (const)); std::string cluster_name_{"fake_cluster"}; std::string route_name_{"fake_route_name"}; @@ -449,8 +446,8 @@ class MockRouteEntry : public RouteEntry { TestVirtualCluster virtual_cluster_; TestRetryPolicy retry_policy_; testing::NiceMock internal_redirect_policy_; - testing::NiceMock path_matcher_; - testing::NiceMock path_rewriter_; + PathMatcherSharedPtr path_matcher_; + PathRewriterSharedPtr path_rewriter_; TestHedgePolicy hedge_policy_; testing::NiceMock rate_limit_policy_; std::vector shadow_policies_; @@ -492,6 +489,16 @@ class MockRouteTracing : public RouteTracing { class MockRoute : public Route { public: + struct MockRouteMetadataObj : public Envoy::Config::TypedMetadata::Object {}; + class MockRouteMetadata : public Envoy::Config::TypedMetadata { + public: + const Envoy::Config::TypedMetadata::Object* getData(const std::string&) const override { + return &object_; + } + + private: + MockRouteMetadataObj object_; + }; MockRoute(); ~MockRoute() override; @@ -513,6 +520,7 @@ class MockRoute : public Route { testing::NiceMock decorator_; testing::NiceMock route_tracing_; envoy::config::core::v3::Metadata metadata_; + MockRouteMetadata typed_metadata_; }; class MockConfig : public Config { diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index fcc07960f9f5..0f2d76724055 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -18,7 +18,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/network/dns_resolver:91.6" # A few lines of MacOS code not tested in linux scripts. Tested in MacOS scripts "source/common/protobuf:96.3" "source/common/quic:93.5" -"source/common/router:96.2" +"source/common/router:96.6" "source/common/secret:95.0" "source/common/signal:87.2" # Death tests don't report LCOV "source/common/singleton:95.7" From 263cb1c29ef434b08e16355ff3a236f93eb995d2 Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 26 May 2023 08:42:25 +0100 Subject: [PATCH 383/740] deps: Bump `edenhill_librdkafka` -> 2.1.1 (#27507) Signed-off-by: Ryan Northey Co-authored-by: Adam Kotwasinski --- bazel/foreign_cc/BUILD | 2 +- bazel/foreign_cc/librdkafka.patch | 57 +++++++++++++++++++ bazel/repositories.bzl | 4 ++ bazel/repository_locations.bzl | 6 +- .../filters/network/test/mesh/kafka_mocks.h | 18 +++++- 5 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 bazel/foreign_cc/librdkafka.patch diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index e450ecfee7b2..67caf394a107 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -55,7 +55,7 @@ cc_library( configure_make( name = "librdkafka_build", configure_in_place = True, - configure_options = ["--disable-ssl --disable-gssapi --disable-lz4-ext --disable-zstd && cp Makefile.config src/.. && cp config.h src/.."], + configure_options = ["--disable-ssl --disable-gssapi --disable-lz4-ext --disable-zstd --disable-curl && cp Makefile.config src/.. && cp config.h src/.."], lib_source = "@edenhill_librdkafka//:all", out_static_libs = [ "librdkafka.a", diff --git a/bazel/foreign_cc/librdkafka.patch b/bazel/foreign_cc/librdkafka.patch new file mode 100644 index 000000000000..a1321f77da23 --- /dev/null +++ b/bazel/foreign_cc/librdkafka.patch @@ -0,0 +1,57 @@ +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 33481ba1..681d0c5c 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -66,7 +66,6 @@ set( + tinycthread.c + tinycthread_extra.c + rdxxhash.c +- cJSON.c + ) + + if(WITH_SSL) +diff --git a/src/Makefile b/src/Makefile +index 26df5723..69bdb427 100644 +--- a/src/Makefile ++++ b/src/Makefile +@@ -43,7 +43,7 @@ SRCS= rdkafka.c rdkafka_broker.c rdkafka_msg.c rdkafka_topic.c \ + rdkafka_assignor.c rdkafka_range_assignor.c \ + rdkafka_roundrobin_assignor.c rdkafka_sticky_assignor.c \ + rdkafka_feature.c \ +- rdcrc32.c crc32c.c rdmurmur2.c rdfnv1a.c cJSON.c \ ++ rdcrc32.c crc32c.c rdmurmur2.c rdfnv1a.c \ + rdaddr.c rdrand.c rdlist.c \ + tinycthread.c tinycthread_extra.c \ + rdlog.c rdstring.c rdkafka_event.c rdkafka_metadata.c \ +diff --git a/src/rdkafka.c b/src/rdkafka.c +index 33147ccd..5ed33b29 100644 +--- a/src/rdkafka.c ++++ b/src/rdkafka.c +@@ -71,9 +71,6 @@ + #include + #endif + +-#define CJSON_HIDE_SYMBOLS +-#include "cJSON.h" +- + #if WITH_CURL + #include "rdhttp.h" + #endif +@@ -139,8 +136,6 @@ void rd_kafka_set_thread_sysname(const char *fmt, ...) { + } + + static void rd_kafka_global_init0(void) { +- cJSON_Hooks json_hooks = {.malloc_fn = rd_malloc, .free_fn = rd_free}; +- + mtx_init(&rd_kafka_global_lock, mtx_plain); + #if ENABLE_DEVEL + rd_atomic32_init(&rd_kafka_op_cnt, 0); +@@ -153,8 +148,6 @@ static void rd_kafka_global_init0(void) { + rd_kafka_ssl_init(); + #endif + +- cJSON_InitHooks(&json_hooks); +- + #if WITH_CURL + rd_http_global_init(); + #endif diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index c52716a570cd..fd13e7347949 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1308,6 +1308,10 @@ filegroup( external_http_archive( name = "edenhill_librdkafka", build_file_content = BUILD_ALL_CONTENT, + # (adam.kotwasinski) librdkafka bundles in cJSON, which is also bundled in by libvppinfra. + # For now, let's just drop this dependency from Kafka, as it's used only for monitoring. + patches = ["@envoy//bazel/foreign_cc:librdkafka.patch"], + patch_args = ["-p1"], ) native.bind( name = "librdkafka", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 387ae12c3aaa..1236add57d89 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1202,13 +1202,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Kafka (C/C++ client)", project_desc = "C/C++ client for Apache Kafka (open-source distributed event streaming platform)", project_url = "https://github.com/edenhill/librdkafka", - version = "1.8.2", - sha256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6", + version = "2.1.1", + sha256 = "7be1fc37ab10ebdc037d5c5a9b35b48931edafffae054b488faaff99e60e0108", strip_prefix = "librdkafka-{version}", urls = ["https://github.com/edenhill/librdkafka/archive/v{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.network.kafka_mesh"], - release_date = "2021-10-18", + release_date = "2023-05-02", cpe = "N/A", license = "librdkafka", license_url = "https://github.com/edenhill/librdkafka/blob/v{version}/LICENSE", diff --git a/contrib/kafka/filters/network/test/mesh/kafka_mocks.h b/contrib/kafka/filters/network/test/mesh/kafka_mocks.h index 2e2a7efb758f..212e473101cb 100644 --- a/contrib/kafka/filters/network/test/mesh/kafka_mocks.h +++ b/contrib/kafka/filters/network/test/mesh/kafka_mocks.h @@ -42,8 +42,8 @@ class MockLibRdKafkaUtils : public LibRdKafkaUtils { // Base class for librdkafka objects. class MockKafkaHandle : public virtual RdKafka::Handle { public: - MOCK_METHOD(const std::string, name, (), (const)); - MOCK_METHOD(const std::string, memberid, (), (const)); + MOCK_METHOD(std::string, name, (), (const)); + MOCK_METHOD(std::string, memberid, (), (const)); MOCK_METHOD(int, poll, (int), ()); MOCK_METHOD(int, outq_len, (), ()); MOCK_METHOD(RdKafka::ErrorCode, metadata, @@ -59,7 +59,7 @@ class MockKafkaHandle : public virtual RdKafka::Handle { MOCK_METHOD(RdKafka::Queue*, get_partition_queue, (const RdKafka::TopicPartition*), ()); MOCK_METHOD(RdKafka::ErrorCode, set_log_queue, (RdKafka::Queue*), ()); MOCK_METHOD(void, yield, (), ()); - MOCK_METHOD(const std::string, clusterid, (int), ()); + MOCK_METHOD(std::string, clusterid, (int), ()); MOCK_METHOD(struct rd_kafka_s*, c_ptr, (), ()); MOCK_METHOD(int32_t, controllerid, (int), ()); MOCK_METHOD(RdKafka::ErrorCode, fatal_error, (std::string&), (const)); @@ -98,6 +98,10 @@ class MockKafkaProducer : public RdKafka::Producer, public MockKafkaHandle { ()); MOCK_METHOD(RdKafka::Error*, commit_transaction, (int), ()); MOCK_METHOD(RdKafka::Error*, abort_transaction, (int), ()); + MOCK_METHOD(RdKafka::Error*, sasl_background_callbacks_enable, (), ()); + MOCK_METHOD(RdKafka::Queue*, get_sasl_queue, (), ()); + MOCK_METHOD(RdKafka::Queue*, get_background_queue, (), ()); + MOCK_METHOD(RdKafka::Error*, sasl_set_credentials, (const std::string&, const std::string&), ()); }; class MockKafkaMessage : public RdKafka::Message { @@ -121,6 +125,8 @@ class MockKafkaMessage : public RdKafka::Message { MOCK_METHOD(RdKafka::Headers*, headers, ()); MOCK_METHOD(RdKafka::Headers*, headers, (RdKafka::ErrorCode*)); MOCK_METHOD(int32_t, broker_id, (), (const)); + MOCK_METHOD(int32_t, leader_epoch, (), (const)); + MOCK_METHOD(RdKafka::Error*, offset_store, ()); }; class MockKafkaConsumer : public RdKafka::KafkaConsumer, public MockKafkaHandle { @@ -144,6 +150,8 @@ class MockKafkaConsumer : public RdKafka::KafkaConsumer, public MockKafkaHandle MOCK_METHOD(RdKafka::ErrorCode, committed, (std::vector&, int), ()); MOCK_METHOD(RdKafka::ErrorCode, position, (std::vector&), ()); MOCK_METHOD(RdKafka::ErrorCode, close, (), ()); + MOCK_METHOD(RdKafka::Error*, close, (RdKafka::Queue*), ()); + MOCK_METHOD(bool, closed, (), ()); MOCK_METHOD(RdKafka::ErrorCode, seek, (const RdKafka::TopicPartition&, int), ()); MOCK_METHOD(RdKafka::ErrorCode, offsets_store, (std::vector&), ()); MOCK_METHOD(RdKafka::ConsumerGroupMetadata*, groupMetadata, (), ()); @@ -153,6 +161,10 @@ class MockKafkaConsumer : public RdKafka::KafkaConsumer, public MockKafkaHandle ()); MOCK_METHOD(RdKafka::Error*, incremental_unassign, (const std::vector&), ()); + MOCK_METHOD(RdKafka::Error*, sasl_background_callbacks_enable, (), ()); + MOCK_METHOD(RdKafka::Queue*, get_sasl_queue, (), ()); + MOCK_METHOD(RdKafka::Queue*, get_background_queue, (), ()); + MOCK_METHOD(RdKafka::Error*, sasl_set_credentials, (const std::string&, const std::string&), ()); }; } // namespace Mesh From 3cde58784b639c5e408ad793b18460859a7a5fae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 May 2023 08:46:42 +0100 Subject: [PATCH 384/740] build(deps): bump nginx from `a2f6ffd` to `af296b1` in /examples/local_ratelimit (#27629) build(deps): bump nginx in /examples/local_ratelimit Bumps nginx from `a2f6ffd` to `af296b1`. --- updated-dependencies: - dependency-name: nginx dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/local_ratelimit/Dockerfile-nginx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/local_ratelimit/Dockerfile-nginx b/examples/local_ratelimit/Dockerfile-nginx index d48af7252182..d4d5b1681b33 100644 --- a/examples/local_ratelimit/Dockerfile-nginx +++ b/examples/local_ratelimit/Dockerfile-nginx @@ -1 +1 @@ -FROM nginx@sha256:a2f6ffddd272625464710d6189f1bf5d02e068392facae344762a512209e74d5 +FROM nginx@sha256:af296b188c7b7df99ba960ca614439c99cb7cf252ed7bbc23e90cfda59092305 From 0c4f903e5a63665cedac818d4f83e17ee82062ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 May 2023 08:54:11 +0100 Subject: [PATCH 385/740] build(deps): bump orjson from 3.8.13 to 3.8.14 in /tools/base (#27648) Bumps [orjson](https://github.com/ijl/orjson) from 3.8.13 to 3.8.14. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.8.13...3.8.14) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 94 ++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index ab3a456306a0..a901bba652a5 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -926,53 +926,53 @@ oauth2client==4.1.3 \ # via # gcs-oauth2-boto-plugin # google-apitools -orjson==3.8.13 \ - --hash=sha256:0055168bc38c9caf7211e66e7c06d7f127d2c1dd1cd1d806c58f3a81d6074a6c \ - --hash=sha256:05bfef2719d68b44ab38061f9cd2b3a58d9994f7230734ba6d3c16db97c5e94a \ - --hash=sha256:0a3bc7e12f69f7bcefe522c4e4dac33a9b3b450aae0b3170ab61fbce0a6e1b37 \ - --hash=sha256:0ca2aced3fa6ce6d440a2a2e55bb7618fd24fce146068523472f349598e992ee \ - --hash=sha256:0d2e9a8ea45db847864868f7a566bece7d425c06627e5dbdd5fc8399a9c3330b \ - --hash=sha256:11a457fafdd207f361986750a5229fc36911fc9fdfc274d078fdf1654845ef45 \ - --hash=sha256:1234110f782af5e81893b5419b9374ca2559dbd976cbd515e6c3afc292cdfb6a \ - --hash=sha256:1316c60c0f55440e765b0211e94d171ab2c11d00fe8dcf0ac70c9bd1d9818e6b \ - --hash=sha256:14e54713703d5436a7be54ff50d780b6b09358f1a0be6107a3ea4f3537a4f6d8 \ - --hash=sha256:156bd6325a4f4a0c88556b7d774e3e18713c8134b6f807571a3eec14dfcafff6 \ - --hash=sha256:17788155c50f47d9fd037e12ac82a57381c157ea4de50e8946df8519da0f7f02 \ - --hash=sha256:246e22d167ede9ebf09685587187bde9e2440a515bd5eab2e97f029b9de57677 \ - --hash=sha256:24f923cf8d7e2e9a975f4507f93e93c262f26b4a1a4f72e4d6e75eda45de8f40 \ - --hash=sha256:2e362090bdd4261608eceefd8ed127cd2bfc48643601f9c0cf5d162ca6a7c4cd \ - --hash=sha256:305ffd227857cede7318c056020d1a3f3295e8adf8e7f2cbd78c26c530a0f234 \ - --hash=sha256:386e60a09585b2b5db84879ebad6d49427ae5a9677f86a90bff9cbbec42b03be \ - --hash=sha256:41585f90cfe24d0ae7d5bc96968617b8bcebb618e19db5b0bbadce6bc82f3455 \ - --hash=sha256:47cb98386a7ff79d0ace6a7c9d5c49ca2b4ea42e4339c565f5efe7757790dd04 \ - --hash=sha256:4b98fbca0ea0f5e56b3c1d050b78460ca9708419780ec218cef1eca424db2ee5 \ - --hash=sha256:50cfa3449157c4a4ad017a041dbb5fe37091800220fd5e651c0e5fff63bdac61 \ - --hash=sha256:5492a1d9eea5a1cb33ae6d225091c69dc79f16d952885625c00070388489d412 \ - --hash=sha256:59d79e5de4a1de246517b4c92dcf6a7ef1fb12e3ce4bbfc6c0f99d1d905405fd \ - --hash=sha256:59d81f5b9e280ac3ced615e726bfba722785cc5f7fc3aa1e0ea304c5a4114e94 \ - --hash=sha256:5e56ca7bd82b25f40955184df21c977369debe51c4b83fc3113b6427726312f3 \ - --hash=sha256:637d55ba6b48b698973d7e647b9de6bb2b424c445f51c86df4e976e672300b21 \ - --hash=sha256:6b323bb4af76c16636ac1fec403331208f978ae8a2c6bcab904ee1683c05ad7a \ - --hash=sha256:6dcccda35f11f12ebb36db0ebdca9854327530e1fffe02331cde78177851ae7f \ - --hash=sha256:6fe2981bd0f6959d821253604e9ba2c5ffa03c6202d11f0e3c190e5712b6835b \ - --hash=sha256:8075487b7b2e7cc2c44d8ee7950845b6854cd08a04df80b36055cc0236c28edd \ - --hash=sha256:8d2ce41c5992dbe9962ef75db1e70ed33636959f2f4b929f9d8cbb2e30472a08 \ - --hash=sha256:97d8444cf48f8fe2718fd3b99484906c29a909dc3a8177e8751170a9a28bcf33 \ - --hash=sha256:9b5b005841394e563f1ca3314a6884101a1b1f1dd30c569b4a0335e1ebf49fbf \ - --hash=sha256:a4a182e7a58114a81d52d67bdc034eb83571690158c4b8d3f1bf5c5f772f77b1 \ - --hash=sha256:b05ef096362c8a96fdcd85392c68156c9b680271aea350b490c2d0f3ef1b6b6a \ - --hash=sha256:b2325d8471867c99c432c96861d72d8b7336293860ebb17c9d70e1d377cc2b32 \ - --hash=sha256:bf934a036dafe63c3b1d630efaf996b85554e7ab03754019a18cc0fe2bdcc3a9 \ - --hash=sha256:c97be6a6ff4d546579f08f1d67aad92715313a06b214e3f2df9bb9f1b45765c2 \ - --hash=sha256:ce737ddf9d5f960996b63c12dbcc82ae2315c45f19165b2fe14a5b33ab8705bb \ - --hash=sha256:cf79f51a7ca59ac322a1e65430142ab1cb9c9a845e893e0e3958deaefe1c9873 \ - --hash=sha256:d30b8b9fe1ff56fb6ff64d2c2e227d49819b58ae8dac51089f393e31b39a4080 \ - --hash=sha256:e62f14f3eabccdd2108e3d5884fb66197255accc42b9ffa7f04d9dbf7336b479 \ - --hash=sha256:e792d286ad175d36f6b77b7ba77f1654a13f705a7ccfef7819e9b6d49277120d \ - --hash=sha256:ea6899624661d2258a71bde33266c3c08c8d9596865acf0ac19a9552c08fa1a6 \ - --hash=sha256:f084ce58b3fd496429deb3435aa7295ab57e349a33cdb99b3cb5f0a66a610a84 \ - --hash=sha256:f8aa77df01c60b7d8b0ff5501d6b8583a4acb06c4373c59bf769025ff8b8b4cb \ - --hash=sha256:fc42b2006abaa4fb72c9193931a9236dd85ce0483cc74079c315ce8529568ca1 +orjson==3.8.14 \ + --hash=sha256:01640ab79111dd97515cba9fab7c66cb3b0967b0892cc74756a801ff681a01b6 \ + --hash=sha256:017de5ba22e58dfa6f41914f5edb8cd052d23f171000684c26b2d2ab219db31e \ + --hash=sha256:04c70dc8ca79b0072a16d82f94b9d9dd6598a43dd753ab20039e9f7d2b14f017 \ + --hash=sha256:062829b5e20cd8648bf4c11c3a5ee7cf196fa138e573407b5312c849b0cf354d \ + --hash=sha256:087c0dc93379e8ba2d59e9f586fab8de8c137d164fccf8afd5523a2137570917 \ + --hash=sha256:09a3bf3154f40299b8bc95e9fb8da47436a59a2106fc22cae15f76d649e062da \ + --hash=sha256:0bc6b7abf27f1dc192dadad249df9b513912506dd420ce50fd18864a33789b71 \ + --hash=sha256:0bf00c42333412a9338297bf888d7428c99e281e20322070bde8c2314775508b \ + --hash=sha256:19415aaf30525a5baff0d72a089fcdd68f19a3674998263c885c3908228c1086 \ + --hash=sha256:20b7ffc7736000ea205f9143df322b03961f287b4057606291c62c842ff3c5b5 \ + --hash=sha256:27967be4c16bd09f4aeff8896d9be9cbd00fd72f5815d5980e4776f821e2f77c \ + --hash=sha256:31a2a29be559e92dcc5c278787b4166da6f0d45675b59a11c4867f5d1455ebf4 \ + --hash=sha256:33bc310da4ad2ffe8f7f1c9e89692146d9ec5aec2d1c9ef6b67f8dc5e2d63241 \ + --hash=sha256:38ca39bae7fbc050332a374062d4cdec28095540fa8bb245eada467897a3a0bb \ + --hash=sha256:3ee09bfbf1d54c127d3061f6721a1a11d2ce502b50597c3d0d2e1bd2d235b764 \ + --hash=sha256:5ea93fd3ef7be7386f2516d728c877156de1559cda09453fc7dd7b696d0439b3 \ + --hash=sha256:5fb66f0ac23e861b817c858515ac1f74d1cd9e72e3f82a5b2c9bae9f92286adc \ + --hash=sha256:6112194c11e611596eed72f46efb0e6b4812682eff3c7b48473d1146c3fa0efb \ + --hash=sha256:64b4fca0531030040e611c6037aaf05359e296877ab0a8e744c26ef9c32738b9 \ + --hash=sha256:67a7e883b6f782b106683979ccc43d89b98c28a1f4a33fe3a22e253577499bb1 \ + --hash=sha256:716a3994e039203f0a59056efa28185d4cac51b922cc5bf27ab9182cfa20e12e \ + --hash=sha256:739f9f633e1544f2a477fa3bef380f488c8dca6e2521c8dc36424b12554ee31e \ + --hash=sha256:7a7b0fead2d0115ef927fa46ad005d7a3988a77187500bf895af67b365c10d1f \ + --hash=sha256:7cb35dd3ba062c1d984d57e6477768ed7b62ed9260f31362b2d69106f9c60ebd \ + --hash=sha256:7d3d8faded5a514b80b56d0429eb38b429d7a810f8749d25dc10a0cc15b8a3c8 \ + --hash=sha256:7e2f75b7d9285e35c3d4dff9811185535ff2ea637f06b2b242cb84385f8ffe63 \ + --hash=sha256:87ba7882e146e24a7d8b4a7971c20212c2af75ead8096fc3d55330babb1015fb \ + --hash=sha256:8a896a12b38fe201a72593810abc1f4f1597e65b8c869d5fc83bbcf75d93398f \ + --hash=sha256:8b206cca6836a4c6683bcaa523ab467627b5f03902e5e1082dc59cd010e6925f \ + --hash=sha256:92374bc35b6da344a927d5a850f7db80a91c7b837de2f0ea90fc870314b1ff44 \ + --hash=sha256:9393a63cb0424515ec5e434078b3198de6ec9e057f1d33bad268683935f0a5d5 \ + --hash=sha256:9725226478d1dafe46d26f758eadecc6cf98dcbb985445e14a9c74aaed6ccfea \ + --hash=sha256:97ebb7fab5f1ae212a6501f17cb7750a6838ffc2f1cebbaa5dec1a90038ca3c6 \ + --hash=sha256:9df820e6c8c84c52ec39ea2cc9c79f7999c839c7d1481a056908dce3b90ce9f9 \ + --hash=sha256:9f5cf61b6db68f213c805c55bf0aab9b4cb75a4e9c7f5bfbd4deb3a0aef0ec53 \ + --hash=sha256:aedba48264fe87e5060c0e9c2b28909f1e60626e46dc2f77e0c8c16939e2e1f7 \ + --hash=sha256:bf6825e160e4eb0ef65ce37d8c221edcab96ff2ffba65e5da2437a60a12b3ad1 \ + --hash=sha256:ca90db8f551b8960da95b0d4cad6c0489df52ea03585b6979595be7b31a3f946 \ + --hash=sha256:d03f29b0369bb1ab55c8a67103eb3a9675daaf92f04388568034fe16be48fa5d \ + --hash=sha256:d66966fd94719beb84e8ed84833bc59c3c005d3d2d0c42f11d7552d3267c6de7 \ + --hash=sha256:de1ee13d6b6727ee1db38722695250984bae81b8fc9d05f1176c74d14b1322d9 \ + --hash=sha256:e53bc5beb612df8ddddb065f079d3fd30b5b4e73053518524423549d61177f3f \ + --hash=sha256:ebca14ae80814219ea3327e3dfa7ff618621ff335e45781fac26f5cd0b48f2b4 \ + --hash=sha256:ee0299b2dda9afce351a5e8c148ea7a886de213f955aa0288fb874fb44829c36 \ + --hash=sha256:f4ac01a3db4e6a98a8ad1bb1a3e8bfc777928939e87c04e93e0d5006df574a4b \ + --hash=sha256:f80e62afe49e6bfc706e041faa351d7520b5f86572b8e31455802251ea989613 # via # -r requirements.in # envoy-base-utils From cab680b0ef801e269f9a50cd745728d4ae8b86e6 Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 26 May 2023 09:51:00 +0100 Subject: [PATCH 386/740] ci: Use `bazel.yml` for Docker publishing (#27047) Signed-off-by: Ryan Northey --- .azure-pipelines/bazel.yml | 24 ++++- .azure-pipelines/stage/publish.yml | 136 ++++++++++++++++++----------- .azure-pipelines/stage/verify.yml | 58 +----------- .azure-pipelines/stages.yml | 8 +- 4 files changed, 110 insertions(+), 116 deletions(-) diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index 7a3c701ac8ac..37c7d4735fd0 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -7,12 +7,30 @@ parameters: displayName: "Suffix of artifact" type: string default: "" + +# caching +- name: cacheKeyDocker + type: string + default: ".devcontainer/Dockerfile" +- name: cacheKeyDockerVersion + type: string + default: $(cacheKeyBuildImage) +- name: cacheKeyDockerName + type: string + default: envoy_build_image +- name: cacheKeyDockerPath + type: string + default: /mnt/docker +- name: cacheKeyDockerTmpDir + type: string + default: /mnt/docker_cache - name: cacheKey type: string default: $(cacheKeyBazelFiles) - name: cacheVersion type: string default: $(cacheKeyBazel) + - name: rbe displayName: "Enable RBE" type: boolean @@ -128,7 +146,11 @@ steps: condition: and(not(canceled()), eq(variables.BAZEL_CACHE_RESTORED, 'true')) - template: cached.yml parameters: - version: "$(cacheKeyBuildImage)" + key: "${{ parameters.cacheKeyDocker }}" + version: "${{ parameters.cacheKeyDockerVersion }}" + name: "${{ parameters.cacheKeyDockerName }}" + path: "${{ parameters.cacheKeyDockerPath }}" + tmpDirectory: "${{ parameters.cacheKeyDockerTmpDir }}" arch: "${{ parameters.artifactSuffix }}" - ${{ each step in parameters.stepsPre }}: diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index d7c79e7cabf4..b028aa88c9ea 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -24,6 +24,15 @@ parameters: - name: authGithub type: string default: "" +- name: authGithubWorkflow + type: string + default: "" +- name: authGithubWorkflowAppId + type: string + default: "" +- name: authGithubWorkflowInstallId + type: string + default: "" - name: authGPGPassphrase type: string default: "" @@ -93,70 +102,91 @@ jobs: pool: vmImage: "ubuntu-20.04" steps: - - template: ../cached.yml - parameters: - name: publish_docker - # VERSION.txt is included to refresh Docker images for release - key: "ci/Dockerfile-envoy | VERSION.txt| $(cacheKeyBazelFiles)" - version: "$(cacheKeyDockerBuild)" - path: "" - - bash: | - echo "disk space at beginning of build:" - df -h - displayName: "Check disk space at beginning" - task: DownloadBuildArtifacts@0 inputs: buildType: current artifactName: "bazel.release" itemPattern: "bazel.release/**/bin/*" targetPath: $(Build.StagingDirectory) - - bash: | - set -e - - mkdir -p linux/amd64 linux/arm64 - - # x64 - cp -a $(Build.StagingDirectory)/bazel.release/x64/bin/release.tar.zst linux/amd64/release.tar.zst - cp -a $(Build.StagingDirectory)/bazel.release/x64/bin/schema_validator_tool linux/amd64/schema_validator_tool - - # arm64 - cp -a $(Build.StagingDirectory)/bazel.release/arm64/bin/release.tar.zst linux/arm64/release.tar.zst - cp -a $(Build.StagingDirectory)/bazel.release/arm64/bin/schema_validator_tool linux/arm64/schema_validator_tool + - template: ../bazel.yml + parameters: + ciTarget: docker-upload + # cacheVersion: $(cacheKeyBazel) + publishEnvoy: false + publishTestResults: false + # VERSION.txt is included to refresh Docker images for release + cacheKeyDocker: "ci/Dockerfile-envoy | VERSION.txt| $(cacheKeyBazelFiles)" + cacheKeyDockerName: publish_docker + cacheKeyDockerTmpDir: /var/azpcache + cacheKeyDockerPath: "" + cacheKeyDockerVersion: "$(cacheKeyDockerBuild)" + env: + GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + stepsPre: + - bash: | + echo "disk space at beginning of Docker build:" + df -h + displayName: "Check disk space before Docker build" + - bash: | + set -e - # Debug what files appear to have been downloaded - find linux -type f -name "*" | xargs ls -l + mkdir -p linux/amd64 linux/arm64 - ci/docker_ci.sh - timeoutInMinutes: ${{ parameters.timeoutDockerPublish }} - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} - DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} - DOCKER_BUILD_TIMEOUT: ${{ parameters.timeoutDockerBuild }} + # x64 + cp -a $(Build.StagingDirectory)/bazel.release/x64/bin/release.tar.zst linux/amd64/release.tar.zst + cp -a $(Build.StagingDirectory)/bazel.release/x64/bin/schema_validator_tool linux/amd64/schema_validator_tool - - bash: | - echo "disk space at end of build:" - df -h - displayName: "Check disk space at end" - condition: not(canceled()) + # arm64 + cp -a $(Build.StagingDirectory)/bazel.release/arm64/bin/release.tar.zst linux/arm64/release.tar.zst + cp -a $(Build.StagingDirectory)/bazel.release/arm64/bin/schema_validator_tool linux/arm64/schema_validator_tool - # Publish docker - - script: | - ci/run_envoy_docker.sh 'ci/do_ci.sh docker-upload' - displayName: "Upload Docker to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} - GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + # Debug what files appear to have been downloaded + find linux -type f -name "*" | xargs ls -l - - script: sudo .azure-pipelines/docker/save_cache.sh /mnt/docker_cache - displayName: "Cache/save (publish_docker)" + ci/docker_ci.sh + displayName: Build Docker images + timeoutInMinutes: ${{ parameters.timeoutDockerPublish }} + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} + DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} + DOCKER_BUILD_TIMEOUT: ${{ parameters.timeoutDockerBuild }} + stepsPost: + - task: DownloadSecureFile@1 + name: WorkflowTriggerKey + displayName: 'Download workflow trigger key' + inputs: + secureFile: '${{ parameters.authGithubWorkflow }}' + - bash: | + set -e + KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)" + echo "##vso[task.setvariable variable=value;isoutput=true]$KEY" + name: key + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh verify.trigger' || echo "trigger not ready yet ..." + # Please see `ci/do_ci.sh` for notes on required vars. + env: + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + ENVOY_BRANCH: "$(System.PullRequest.TargetBranch)" + ENVOY_COMMIT: "$(System.PullRequest.SourceCommitId)" + ENVOY_HEAD_REF: "$(Build.SourceBranch)" + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + ENVOY_BRANCH: "$(Build.SourceBranch)" + # github auth + GITHUB_TOKEN: "$(key.value)" + GITHUB_APP_ID: ${{ parameters.authGithubWorkflowAppId }} + GITHUB_INSTALL_ID: ${{ parameters.authGithubWorkflowInstallId }} + # rbe env + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} + GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_RBE: "1" + - script: sudo .azure-pipelines/docker/save_cache.sh /var/azpcache + displayName: "Cache/save (publish_docker)" - job: package_x64 displayName: Linux debs (x64) diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml index 238d7b4da974..127a2148ac8f 100644 --- a/.azure-pipelines/stage/verify.yml +++ b/.azure-pipelines/stage/verify.yml @@ -1,68 +1,13 @@ parameters: -- name: bucketGCP - type: string - default: "" - # Auth - name: authGCP type: string default: "" -- name: authGithubWorkflow - type: string - default: "" -- name: authGithubWorkflowAppId - type: string - default: "" -- name: authGithubWorkflowInstallId - type: string - default: "" - -- name: runDocker - displayName: "Run Docker" - type: string - default: true -# TODO(phlax): improve docker publishing job and move this there jobs: -- job: examples - displayName: Examples (Docker/x64) - condition: | - and(not(canceled()), - eq(${{ parameters.runDocker }}, 'true')) - timeoutInMinutes: 120 - pool: - vmImage: "ubuntu-20.04" - steps: - - task: DownloadSecureFile@1 - name: WorkflowTriggerKey - displayName: 'Download workflow trigger key' - inputs: - secureFile: '${{ parameters.authGithubWorkflow }}' - - - bash: | - set -e - KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)" - echo "##vso[task.setvariable variable=value;isoutput=true]$KEY" - name: key - - - template: ../bazel.yml - parameters: - ciTarget: verify.trigger - authGithub: "$(key.value)" - # Please see `ci/do_ci.sh` for notes on required vars. - env: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - ENVOY_BRANCH: "$(System.PullRequest.TargetBranch)" - ENVOY_COMMIT: "$(System.PullRequest.SourceCommitId)" - ENVOY_HEAD_REF: "$(Build.SourceBranch)" - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - ENVOY_BRANCH: "$(Build.SourceBranch)" - GITHUB_APP_ID: ${{ parameters.authGithubWorkflowAppId }} - GITHUB_INSTALL_ID: ${{ parameters.authGithubWorkflowInstallId }} - - job: packages_x64 displayName: Debs (x64) condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true')) @@ -114,7 +59,7 @@ jobs: - job: verified displayName: Verification complete - dependsOn: ["examples", "packages_x64", "packages_arm64"] + dependsOn: ["packages_x64", "packages_arm64"] pool: vmImage: "ubuntu-20.04" # This condition ensures that this (required) check passes if all of @@ -124,7 +69,6 @@ jobs: condition: | and( eq(variables['Build.Reason'], 'PullRequest'), - in(dependencies.examples.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), in(dependencies.packages_x64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), in(dependencies.packages_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) steps: diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index e678f892e541..caab5bae0fad 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -148,6 +148,9 @@ stages: authDockerPassword: $(DockerPassword) authGCP: $(GcpServiceAccountKey) authGithub: $(GitHubPublicRepoOnlyAccessToken) + authGithubWorkflow: $(GitHubPublicRepoWorkflowKey) + authGithubWorkflowAppId: $(GitHubPublicRepoWorkflowAppId) + authGithubWorkflowInstallId: $(GitHubPublicRepoWorkflowInstallId) authGPGPassphrase: $(MaintainerGPGKeyPassphrase) authGPGKey: $(MaintainerGPGKeySecureFileDownloadPath) authGPGPath: $(MaintainerGPGKey.secureFilePath) @@ -175,11 +178,6 @@ stages: - template: stage/verify.yml parameters: authGCP: $(GcpServiceAccountKey) - bucketGCP: $(GcsArtifactBucket) - authGithubWorkflow: $(GitHubPublicRepoWorkflowKey) - authGithubWorkflowAppId: $(GitHubPublicRepoWorkflowAppId) - authGithubWorkflowInstallId: $(GitHubPublicRepoWorkflowInstallId) - runDocker: variables['RUN_DOCKER'] - stage: macos displayName: macOS From 0d79cc8a107a86c2f6880fff42b7c5c3d30f87ad Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Fri, 26 May 2023 14:36:57 -0400 Subject: [PATCH 387/740] ext_proc filter allow_mode_override API implemenation (#27618) * ext_proc filter allow_mode_override API implemenation Signed-off-by: Yanjun Xiang --- .../filters/http/ext_proc/v3/ext_proc.proto | 4 +- changelogs/current.yaml | 9 ++++ .../filters/http/ext_proc/ext_proc.cc | 5 +- .../filters/http/ext_proc/ext_proc.h | 8 +++- .../filters/http/ext_proc/filter_test.cc | 47 ++++++++++++++++++- .../ext_proc/streaming_integration_test.cc | 2 +- 6 files changed, 68 insertions(+), 7 deletions(-) diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto index 38feceb22706..2198a2d6f25e 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto @@ -76,7 +76,8 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // By default, the processor sends only the request and response headers messages. // This may be changed to include any of the six steps by changing the processing_mode // setting of the filter configuration, or by setting the mode_override of any response -// from the external processor. This way, a processor may, for example, use information +// from the external processor. The latter is only enabled if allow_mode_override is +// set to true. This way, a processor may, for example, use information // in the request header to determine whether the message body must be examined, or whether // the proxy should simply stream it straight through. // @@ -196,7 +197,6 @@ message ExternalProcessor { // ext_proc filter name. google.protobuf.Struct filter_metadata = 13; - // [#not-implemented-hide:] // If ``allow_mode_override`` is set to true, the filter config :ref:`processing_mode // ` // can be overridden by the response message from the external processing server diff --git a/changelogs/current.yaml b/changelogs/current.yaml index f537a751f08d..6cb35b3eaa98 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -206,6 +206,15 @@ new_features: - area: ext_proc change: | added new configuration field + :ref:`allow_mode_override ` + If set to true, the filter config + :ref:`processing_mode ` + can be overridden by the + :ref:`mode_override ` + in the response message from the external processing server. + If not set, the ``mode_override`` API in the response message will be ignored. +- area: ext_proc + change: | :ref:`forward_rules ` to only allow headers matchinging the forward rules to be forwarded to the external processing server. - area: redis_proxy diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index deb93c625667..7661048511ac 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -607,8 +607,9 @@ void Filter::onReceiveMessage(std::unique_ptr&& r) { // Update processing mode now because filter callbacks check it // and the various "handle" methods below may result in callbacks - // being invoked in line. - if (response->has_mode_override()) { + // being invoked in line. This only happens when filter has allow_mode_override + // set to true. Otherwise, the response mode_override proto field is ignored. + if (config_->allowModeOverride() && response->has_mode_override()) { ENVOY_LOG(debug, "Processing mode overridden by server for this request"); decoding_state_.setProcessingMode(response->mode_override()); encoding_state_.setProcessingMode(response->mode_override()); diff --git a/source/extensions/filters/http/ext_proc/ext_proc.h b/source/extensions/filters/http/ext_proc/ext_proc.h index ecbf61dcaae8..f3917364d197 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.h +++ b/source/extensions/filters/http/ext_proc/ext_proc.h @@ -111,7 +111,9 @@ class FilterConfig { message_timeout_(message_timeout), max_message_timeout_ms_(max_message_timeout_ms), stats_(generateStats(stats_prefix, config.stat_prefix(), scope)), processing_mode_(config.processing_mode()), mutation_checker_(config.mutation_rules()), - filter_metadata_(config.filter_metadata()), header_matchers_(initHeaderMatchers(config)) {} + filter_metadata_(config.filter_metadata()), + allow_mode_override_(config.allow_mode_override()), + header_matchers_(initHeaderMatchers(config)) {} bool failureModeAllow() const { return failure_mode_allow_; } @@ -125,6 +127,8 @@ class FilterConfig { return processing_mode_; } + bool allowModeOverride() const { return allow_mode_override_; } + const Filters::Common::MutationRules::Checker& mutationChecker() const { return mutation_checker_; } @@ -161,6 +165,8 @@ class FilterConfig { const envoy::extensions::filters::http::ext_proc::v3::ProcessingMode processing_mode_; const Filters::Common::MutationRules::Checker mutation_checker_; const Envoy::ProtobufWkt::Struct filter_metadata_; + // If set to true, allow the processing mode to be modified by the ext_proc response. + const bool allow_mode_override_; // Empty header_matchers_ means allow all. const std::vector header_matchers_; }; diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc index 009458d53c6d..3f54f761d261 100644 --- a/test/extensions/filters/http/ext_proc/filter_test.cc +++ b/test/extensions/filters/http/ext_proc/filter_test.cc @@ -1395,6 +1395,7 @@ TEST_F(HttpFilterTest, GetStreamingBodyAndChangeMode) { response_body_mode: "STREAMED" request_trailer_mode: "SKIP" response_trailer_mode: "SKIP" + allow_mode_override: true )EOF"); // Create synthetic HTTP request @@ -1482,6 +1483,7 @@ TEST_F(HttpFilterTest, GetStreamingBodyAndChangeModeDifferentOrder) { response_body_mode: "STREAMED" request_trailer_mode: "SKIP" response_trailer_mode: "SKIP" + allow_mode_override: true )EOF"); // Create synthetic HTTP request @@ -1885,8 +1887,9 @@ TEST_F(HttpFilterTest, ProcessingModeOverrideResponseHeaders) { grpc_service: envoy_grpc: cluster_name: "ext_proc_server" + allow_mode_override: true )EOF"); - + EXPECT_EQ(filter_->config().allowModeOverride(), true); EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, false)); processRequestHeaders( @@ -1920,6 +1923,48 @@ TEST_F(HttpFilterTest, ProcessingModeOverrideResponseHeaders) { EXPECT_EQ(1, config_->stats().streams_closed_.value()); } +// Leaving the allow_mode_override in filter config to be default, which is false. +// In such case, the mode_override in the response will be ignored. +TEST_F(HttpFilterTest, DisableResponseModeOverride) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" + processing_mode: + request_header_mode: "SEND" + response_header_mode: "SEND" + )EOF"); + + EXPECT_EQ(filter_->config().allowModeOverride(), false); + EXPECT_EQ(filter_->config().processingMode().response_header_mode(), ProcessingMode::SEND); + EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, true)); + + // When ext_proc server sends back the request header response, it contains the + // mode_override for the response_header_mode to be SKIP. + processRequestHeaders( + false, [](const HttpHeaders&, ProcessingResponse& response, HeadersResponse&) { + response.mutable_mode_override()->set_response_header_mode(ProcessingMode::SKIP); + }); + + response_headers_.addCopy(LowerCaseString(":status"), "200"); + response_headers_.addCopy(LowerCaseString("content-type"), "text/plain"); + EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->encodeHeaders(response_headers_, true)); + + // Such mode_override is ignored. The response header is still sent to the ext_proc server. + processResponseHeaders(false, + [](const HttpHeaders& header_resp, ProcessingResponse&, HeadersResponse&) { + EXPECT_TRUE(header_resp.end_of_stream()); + Http::TestRequestHeaderMapImpl expected_response{ + {":status", "200"}, {"content-type", "text/plain"}}; + EXPECT_THAT(header_resp.headers(), HeaderProtosEqual(expected_response)); + }); + + Http::TestRequestHeaderMapImpl final_expected_response{{":status", "200"}, + {"content-type", "text/plain"}}; + EXPECT_THAT(&response_headers_, HeaderMapEqualIgnoreOrder(&final_expected_response)); + filter_->onDestroy(); +} + // Using a processing mode, configure the filter to only send the response_headers // message. TEST_F(HttpFilterTest, ProcessingModeResponseHeadersOnly) { diff --git a/test/extensions/filters/http/ext_proc/streaming_integration_test.cc b/test/extensions/filters/http/ext_proc/streaming_integration_test.cc index 76ae7902ac8c..5b6d46413e21 100644 --- a/test/extensions/filters/http/ext_proc/streaming_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/streaming_integration_test.cc @@ -44,7 +44,7 @@ class StreamingIntegrationTest : public HttpIntegrationTest, void initializeConfig() { // This enables a built-in automatic upstream server. autonomous_upstream_ = true; - + proto_config_.set_allow_mode_override(true); config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { // Create a cluster for our gRPC server pointing to the address that is running the gRPC // server. From 92be7e1e7d668ddac6e282e0ceaff38ffe0f2737 Mon Sep 17 00:00:00 2001 From: Ketankumar Rana <124616929+ktn-nv@users.noreply.github.com> Date: Fri, 26 May 2023 11:51:23 -0700 Subject: [PATCH 388/740] Oauth2: Marking token holding cookie as httpOnly to prevent attacks (#27297) * Oauth2: Marking token holiding cookie as httpOnly to prevent attacks Signed-off-by: Ketankumar Rana --- changelogs/current.yaml | 7 +++ source/common/runtime/runtime_features.cc | 1 + .../extensions/filters/http/oauth2/filter.cc | 22 +++++----- .../filters/http/oauth2/filter_test.cc | 44 ++++++++++++++++--- 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 6cb35b3eaa98..0a5f3aadd377 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -114,6 +114,13 @@ bug_fixes: - area: boringssl change: | Fixed the crash that occurs when contrib is compiled with ``boringssl=fips`` defined. +- area: oauth2 + change: | + The httpOnly attribute for Set-Cookie for tokens in HTTP response header was missing, + causing tokens to be accessible from the JavaScript making the apps vulnerable. + This was fixed now by marking the cookie as httpOnly. + This behavioral change can be temporarily reverted by setting runtime guard + ``envoy.reloadable_features.oauth_make_token_cookie_httponly`` to false. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 14e980901d74..5d6d318e0b55 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -55,6 +55,7 @@ RUNTIME_GUARD(envoy_reloadable_features_initialize_upstream_filters); RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name); RUNTIME_GUARD(envoy_reloadable_features_no_full_scan_certs_on_sni_mismatch); RUNTIME_GUARD(envoy_reloadable_features_oauth_header_passthrough_fix); +RUNTIME_GUARD(envoy_reloadable_features_oauth_make_token_cookie_httponly); RUNTIME_GUARD(envoy_reloadable_features_oauth_use_standard_max_age_value); RUNTIME_GUARD(envoy_reloadable_features_oauth_use_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_original_dst_rely_on_idle_timeout); diff --git a/source/extensions/filters/http/oauth2/filter.cc b/source/extensions/filters/http/oauth2/filter.cc index af111b3c26b1..d3422448d46b 100644 --- a/source/extensions/filters/http/oauth2/filter.cc +++ b/source/extensions/filters/http/oauth2/filter.cc @@ -36,9 +36,7 @@ Http::RegisterCustomInlineHeadercookieNames(); headers.addReferenceKey( @@ -535,18 +532,23 @@ void OAuth2Filter::addResponseCookies(Http::ResponseHeaderMap& headers, // If opted-in, we also create a new Bearer cookie for the authorization token provided by the // auth server. if (config_->forwardBearerToken()) { + std::string cookie_attribute_httponly = + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.oauth_make_token_cookie_httponly") + ? cookie_tail_http_only + : cookie_tail; headers.addReferenceKey( Http::Headers::get().SetCookie, - absl::StrCat(cookie_names.bearer_token_, "=", access_token_, cookie_tail)); + absl::StrCat(cookie_names.bearer_token_, "=", access_token_, cookie_attribute_httponly)); if (id_token_ != EMPTY_STRING) { - headers.addReferenceKey(Http::Headers::get().SetCookie, - absl::StrCat(cookie_names.id_token_, "=", id_token_, cookie_tail)); + headers.addReferenceKey( + Http::Headers::get().SetCookie, + absl::StrCat(cookie_names.id_token_, "=", id_token_, cookie_attribute_httponly)); } if (refresh_token_ != EMPTY_STRING) { - headers.addReferenceKey( - Http::Headers::get().SetCookie, - absl::StrCat(cookie_names.refresh_token_, "=", refresh_token_, cookie_tail)); + headers.addReferenceKey(Http::Headers::get().SetCookie, + absl::StrCat(cookie_names.refresh_token_, "=", refresh_token_, + cookie_attribute_httponly)); } } } diff --git a/test/extensions/filters/http/oauth2/filter_test.cc b/test/extensions/filters/http/oauth2/filter_test.cc index 970bbf360e81..e0445410e3ec 100644 --- a/test/extensions/filters/http/oauth2/filter_test.cc +++ b/test/extensions/filters/http/oauth2/filter_test.cc @@ -1104,7 +1104,8 @@ TEST_F(OAuth2Test, OAuthTestFullFlowPostWithParametersLegacyEncoding) { "version=1;path=/;Max-Age=;secure;HttpOnly"}, {Http::Headers::get().SetCookie.get(), "OauthExpires=;version=1;path=/;Max-Age=;secure;HttpOnly"}, - {Http::Headers::get().SetCookie.get(), "BearerToken=;version=1;path=/;Max-Age=;secure"}, + {Http::Headers::get().SetCookie.get(), + "BearerToken=;version=1;path=/;Max-Age=;secure;HttpOnly"}, {Http::Headers::get().Location.get(), "https://traffic.example.com/test?name=admin&level=trace"}, }; @@ -1195,7 +1196,8 @@ TEST_F(OAuth2Test, OAuthTestFullFlowPostWithParameters) { "version=1;path=/;Max-Age=;secure;HttpOnly"}, {Http::Headers::get().SetCookie.get(), "OauthExpires=;version=1;path=/;Max-Age=;secure;HttpOnly"}, - {Http::Headers::get().SetCookie.get(), "BearerToken=;version=1;path=/;Max-Age=;secure"}, + {Http::Headers::get().SetCookie.get(), + "BearerToken=;version=1;path=/;Max-Age=;secure;HttpOnly"}, {Http::Headers::get().Location.get(), "https://traffic.example.com/test/utf8%C3%83?name=admin&level=trace"}, }; @@ -1226,11 +1228,11 @@ TEST_F(OAuth2Test, OAuthAccessTokenSucessWithTokens) { {Http::Headers::get().SetCookie.get(), "OauthExpires=1600;version=1;path=/;Max-Age=600;secure;HttpOnly"}, {Http::Headers::get().SetCookie.get(), - "BearerToken=access_code;version=1;path=/;Max-Age=600;secure"}, + "BearerToken=access_code;version=1;path=/;Max-Age=600;secure;HttpOnly"}, {Http::Headers::get().SetCookie.get(), - "IdToken=some-id-token;version=1;path=/;Max-Age=600;secure"}, + "IdToken=some-id-token;version=1;path=/;Max-Age=600;secure;HttpOnly"}, {Http::Headers::get().SetCookie.get(), - "RefreshToken=some-refresh-token;version=1;path=/;Max-Age=600;secure"}, + "RefreshToken=some-refresh-token;version=1;path=/;Max-Age=600;secure;HttpOnly"}, {Http::Headers::get().Location.get(), ""}, }; @@ -1258,6 +1260,38 @@ TEST_F(OAuth2Test, OAuthAccessTokenSucessWithTokens_oauth_use_standard_max_age_v "version=1;path=/;Max-Age=600;secure;HttpOnly"}, {Http::Headers::get().SetCookie.get(), "OauthExpires=600;version=1;path=/;Max-Age=600;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "BearerToken=access_code;version=1;path=/;Max-Age=600;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "IdToken=some-id-token;version=1;path=/;Max-Age=600;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "RefreshToken=some-refresh-token;version=1;path=/;Max-Age=600;secure;HttpOnly"}, + {Http::Headers::get().Location.get(), ""}, + }; + + EXPECT_CALL(decoder_callbacks_, encodeHeaders_(HeaderMapEqualRef(&expected_headers), true)); + + filter_->onGetAccessTokenSuccess("access_code", "some-id-token", "some-refresh-token", + std::chrono::seconds(600)); +} +TEST_F(OAuth2Test, OAuthAccessTokenSucessWithTokens_oauth_make_token_cookie_httponly) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({ + {"envoy.reloadable_features.oauth_make_token_cookie_httponly", "false"}, + }); + + // Set SystemTime to a fixed point so we get consistent HMAC encodings between test runs. + test_time_.setSystemTime(SystemTime(std::chrono::seconds(1000))); + + // Expected response after the callback is complete. + Http::TestRequestHeaderMapImpl expected_headers{ + {Http::Headers::get().Status.get(), "302"}, + {Http::Headers::get().SetCookie.get(), + "OauthHMAC=" + "MjI2YmI5YTRiZjJlNTFlNDUzZWVjOWUzYmU1MThlNGQyNDgyNzA0ZTBkMGQyY2M3M2QyMzg3NTRkZTY0YmU5YQ==;" + "version=1;path=/;Max-Age=600;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "OauthExpires=1600;version=1;path=/;Max-Age=600;secure;HttpOnly"}, {Http::Headers::get().SetCookie.get(), "BearerToken=access_code;version=1;path=/;Max-Age=600;secure"}, {Http::Headers::get().SetCookie.get(), From 0e822607588b6313931154a5ae8680c0df2af08a Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Fri, 26 May 2023 15:07:26 -0400 Subject: [PATCH 389/740] Fix ext_proc fuzzer test issues. (#27619) * Fix ext_proc fuzzer test issues. Signed-off-by: Yanjun Xiang --- ...h-df781e3c1eede590dddbb6d3fd7e58f87b8d89cd | 28 +++++++++++++++++++ .../unit_test_fuzz/ext_proc_unit_test_fuzz.cc | 14 +++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-df781e3c1eede590dddbb6d3fd7e58f87b8d89cd diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-df781e3c1eede590dddbb6d3fd7e58f87b8d89cd b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-df781e3c1eede590dddbb6d3fd7e58f87b8d89cd new file mode 100644 index 000000000000..8d60ca08e841 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-df781e3c1eede590dddbb6d3fd7e58f87b8d89cd @@ -0,0 +1,28 @@ +config { + grpc_service { + envoy_grpc { + cluster_name: ":" + } + } + processing_mode { + response_header_mode: SKIP + request_trailer_mode: SKIP + response_trailer_mode: SKIP + } + max_message_timeout { + nanos: 89 + } +} +request { + http_body { + data: "\177\177\177\177" + } +} +response { + request_headers { + } + mode_override { + request_header_mode: SKIP + request_body_mode: BUFFERED + } +} diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc index 477a04bb2841..8ca81765cb27 100644 --- a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_unit_test_fuzz.cc @@ -18,13 +18,18 @@ namespace UnitTestFuzz { class FuzzerMocks { public: - FuzzerMocks() : addr_(std::make_shared("/test/test.sock")) { + FuzzerMocks() + : addr_(std::make_shared("/test/test.sock")), buffer_("foo") { ON_CALL(decoder_callbacks_, connection()) .WillByDefault(Return(OptRef{connection_})); connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress(addr_); connection_.stream_info_.downstream_connection_info_provider_->setLocalAddress(addr_); ON_CALL(decoder_callbacks_, addDecodedTrailers()).WillByDefault(ReturnRef(request_trailers_)); ON_CALL(encoder_callbacks_, addEncodedTrailers()).WillByDefault(ReturnRef(response_trailers_)); + ON_CALL(decoder_callbacks_, decodingBuffer()).WillByDefault(Return(&buffer_)); + ON_CALL(encoder_callbacks_, encodingBuffer()).WillByDefault(Return(&buffer_)); + ON_CALL(decoder_callbacks_, decoderBufferLimit()).WillByDefault(Return(1024)); + ON_CALL(encoder_callbacks_, encoderBufferLimit()).WillByDefault(Return(1024)); } NiceMock decoder_callbacks_; @@ -33,6 +38,7 @@ class FuzzerMocks { NiceMock connection_; NiceMock request_trailers_; NiceMock response_trailers_; + NiceMock buffer_; testing::NiceMock async_client_stream_info_; }; @@ -72,8 +78,8 @@ DEFINE_PROTO_FUZZER( filter->setDecoderFilterCallbacks(mocks.decoder_callbacks_); filter->setEncoderFilterCallbacks(mocks.encoder_callbacks_); - ON_CALL(*client, start(_, _, _)) - .WillByDefault(Invoke( + EXPECT_CALL(*client, start(_, _, _)) + .WillRepeatedly(Invoke( [&](ExternalProcessing::ExternalProcessorCallbacks&, const envoy::config::core::v3::GrpcService&, const StreamInfo::StreamInfo&) -> ExternalProcessing::ExternalProcessorStreamPtr { @@ -88,7 +94,7 @@ DEFINE_PROTO_FUZZER( })); EXPECT_CALL(*stream, streamInfo()) .WillRepeatedly(ReturnRef(mocks.async_client_stream_info_)); - ON_CALL(*stream, close()).WillByDefault(Return(false)); + EXPECT_CALL(*stream, close()).WillRepeatedly(Return(false)); return stream; })); From 71a50bea7bc46e965eae3a12253d9021de40fcf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20R=2E=20Sede=C3=B1o?= Date: Fri, 26 May 2023 15:07:59 -0400 Subject: [PATCH 390/740] Update QUICHE from 4a5e9e692 to 4506f8d26 (#27638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update QUICHE from 4a5e9e692 to 4506f8d26 https://github.com/google/quiche/compare/4a5e9e692..4506f8d26 ``` $ git log 4a5e9e692..4506f8d26 --date=short --no-merges --format="%ad %al %s" 2023-05-24 bnc Internal change 2023-05-24 danzh Internal change 2023-05-23 martinduke Use QuicPacketWriter::SupportsEcn interface to abandon ECN attempts. 2023-05-23 martinduke Refactor ECN in QuicConnection. 2023-05-23 martinduke Add SupportsEcn() interface to QuicPacketWriter. A follow-on will use this interface in QuicConnection. 2023-05-23 martinduke Refactor PerPacketOptions for QuicWriter. From the bug: 2023-05-22 elburrito Use public metadata expiry time as BlindSignAuth token expiry 2023-05-19 elburrito Add ClearCache function to CachedBlindSignAuth. This function will be called when the cached tokens are invalid and need to be deleted. 2023-05-19 danzh Internal change ``` Signed-off-by: Alejandro R Sedeño --- bazel/repository_locations.bzl | 6 ++--- .../common/quic/envoy_quic_packet_writer.cc | 8 +++---- source/common/quic/envoy_quic_packet_writer.h | 5 +++- source/common/quic/udp_gso_batch_writer.cc | 3 ++- .../quic/envoy_quic_client_session_test.cc | 2 +- .../quic/envoy_quic_client_stream_test.cc | 5 ++-- .../common/quic/envoy_quic_dispatcher_test.cc | 2 +- .../quic/envoy_quic_server_session_test.cc | 5 ++-- .../quic/envoy_quic_server_stream_test.cc | 5 ++-- test/common/quic/envoy_quic_writer_test.cc | 24 +++++++++++-------- test/per_file_coverage.sh | 2 +- 11 files changed, 39 insertions(+), 28 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 1236add57d89..f6769dc0c707 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1056,12 +1056,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "4a5e9e692040ff0786563c30488a7d29fd58f40d", - sha256 = "59e4712549c8f39fa31b5ded59ab548538384785bcc0854ee59221160f91a16d", + version = "4506f8d266853c2b039bec1ee4bc8ed0f22d861c", + sha256 = "f532f981381b8e0f817b4426d6dae085c32bdfe75ba01c55ad09a6d9341f91fb", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2023-05-18", + release_date = "2023-05-24", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", diff --git a/source/common/quic/envoy_quic_packet_writer.cc b/source/common/quic/envoy_quic_packet_writer.cc index 74663900d619..75178b9cdf27 100644 --- a/source/common/quic/envoy_quic_packet_writer.cc +++ b/source/common/quic/envoy_quic_packet_writer.cc @@ -25,10 +25,10 @@ quic::WriteResult convertToQuicWriteResult(Api::IoCallUint64Result& result) { EnvoyQuicPacketWriter::EnvoyQuicPacketWriter(Network::UdpPacketWriterPtr envoy_udp_packet_writer) : envoy_udp_packet_writer_(std::move(envoy_udp_packet_writer)) {} -quic::WriteResult EnvoyQuicPacketWriter::WritePacket(const char* buffer, size_t buffer_len, - const quic::QuicIpAddress& self_ip, - const quic::QuicSocketAddress& peer_address, - quic::PerPacketOptions* options) { +quic::WriteResult EnvoyQuicPacketWriter::WritePacket( + const char* buffer, size_t buffer_len, const quic::QuicIpAddress& self_ip, + const quic::QuicSocketAddress& peer_address, quic::PerPacketOptions* options, + [[maybe_unused]] const quic::QuicPacketWriterParams& params) { ASSERT(options == nullptr, "Per packet option is not supported yet."); Buffer::BufferFragmentImpl fragment(buffer, buffer_len, nullptr); diff --git a/source/common/quic/envoy_quic_packet_writer.h b/source/common/quic/envoy_quic_packet_writer.h index 2b50c8aa2c95..80b4b6bfc02e 100644 --- a/source/common/quic/envoy_quic_packet_writer.h +++ b/source/common/quic/envoy_quic_packet_writer.h @@ -14,7 +14,8 @@ class EnvoyQuicPacketWriter : public quic::QuicPacketWriter { quic::WriteResult WritePacket(const char* buffer, size_t buf_len, const quic::QuicIpAddress& self_address, const quic::QuicSocketAddress& peer_address, - quic::PerPacketOptions* options) override; + quic::PerPacketOptions* options, + const quic::QuicPacketWriterParams& params) override; // quic::QuicPacketWriter bool IsWriteBlocked() const override { return envoy_udp_packet_writer_->isWriteBlocked(); } @@ -22,6 +23,8 @@ class EnvoyQuicPacketWriter : public quic::QuicPacketWriter { bool IsBatchMode() const override { return envoy_udp_packet_writer_->isBatchMode(); } // Currently this writer doesn't support pacing offload. bool SupportsReleaseTime() const override { return false; } + // Currently this writer doesn't support Explicit Congestion Notification. + bool SupportsEcn() const override { return false; } absl::optional MessageTooBigErrorCode() const override; quic::QuicByteCount GetMaxPacketSize(const quic::QuicSocketAddress& peer_address) const override; diff --git a/source/common/quic/udp_gso_batch_writer.cc b/source/common/quic/udp_gso_batch_writer.cc index fa08f95b29b6..7171806d7640 100644 --- a/source/common/quic/udp_gso_batch_writer.cc +++ b/source/common/quic/udp_gso_batch_writer.cc @@ -55,9 +55,10 @@ UdpGsoBatchWriter::writePacket(const Buffer::Instance& buffer, const Network::Ad // TODO(yugant): Currently we do not use PerPacketOptions with Quic, we may want to // specify this parameter here at a later stage. + quic::QuicPacketWriterParams params; quic::WriteResult quic_result = WritePacket(static_cast(buffer.frontSlice().mem_), payload_len, self_addr.host(), peer_addr, - /*quic::PerPacketOptions=*/nullptr); + /*quic::PerPacketOptions=*/nullptr, params); updateUdpGsoBatchWriterStats(quic_result); return convertQuicWriteResult(quic_result, payload_len); diff --git a/test/common/quic/envoy_quic_client_session_test.cc b/test/common/quic/envoy_quic_client_session_test.cc index e9c30b2fde0b..921c0237f117 100644 --- a/test/common/quic/envoy_quic_client_session_test.cc +++ b/test/common/quic/envoy_quic_client_session_test.cc @@ -91,7 +91,7 @@ class EnvoyQuicClientSessionTest : public testing::TestWithParam) { return quic::QuicConsumedData{write_length, state != quic::NO_FIN}; })); - EXPECT_CALL(writer_, WritePacket(_, _, _, _, _)) + EXPECT_CALL(writer_, WritePacket(_, _, _, _, _, _)) .WillRepeatedly(Invoke([](const char*, size_t buf_len, const quic::QuicIpAddress&, - const quic::QuicSocketAddress&, quic::PerPacketOptions*) { + const quic::QuicSocketAddress&, quic::PerPacketOptions*, + const quic::QuicPacketWriterParams&) { return quic::WriteResult{quic::WRITE_STATUS_OK, static_cast(buf_len)}; })); } diff --git a/test/common/quic/envoy_quic_dispatcher_test.cc b/test/common/quic/envoy_quic_dispatcher_test.cc index ce0bb7d806de..2bf102ad948c 100644 --- a/test/common/quic/envoy_quic_dispatcher_test.cc +++ b/test/common/quic/envoy_quic_dispatcher_test.cc @@ -79,7 +79,7 @@ class EnvoyQuicDispatcherTest : public testing::TestWithParam>()) { auto writer = new testing::NiceMock(); envoy_quic_dispatcher_.InitializeWithWriter(writer); - EXPECT_CALL(*writer, WritePacket(_, _, _, _, _)) + EXPECT_CALL(*writer, WritePacket(_, _, _, _, _, _)) .WillRepeatedly(Return(quic::WriteResult(quic::WRITE_STATUS_OK, 0))); EXPECT_CALL(proof_source_->filterChain(), transportSocketFactory()) .WillRepeatedly(ReturnRef(transport_socket_factory_)); diff --git a/test/common/quic/envoy_quic_server_session_test.cc b/test/common/quic/envoy_quic_server_session_test.cc index 210abdda140e..43608d9f8cde 100644 --- a/test/common/quic/envoy_quic_server_session_test.cc +++ b/test/common/quic/envoy_quic_server_session_test.cc @@ -177,9 +177,10 @@ class EnvoyQuicServerSessionTest : public testing::Test { Event::Dispatcher::RunType::NonBlock); connection_helper_.GetClock()->Now(); - ON_CALL(writer_, WritePacket(_, _, _, _, _)) + ON_CALL(writer_, WritePacket(_, _, _, _, _, _)) .WillByDefault(Invoke([](const char*, size_t buf_len, const quic::QuicIpAddress&, - const quic::QuicSocketAddress&, quic::PerPacketOptions*) { + const quic::QuicSocketAddress&, quic::PerPacketOptions*, + const quic::QuicPacketWriterParams&) { return quic::WriteResult{quic::WRITE_STATUS_OK, static_cast(buf_len)}; })); ON_CALL(crypto_stream_helper_, CanAcceptClientHello(_, _, _, _, _)).WillByDefault(Return(true)); diff --git a/test/common/quic/envoy_quic_server_stream_test.cc b/test/common/quic/envoy_quic_server_stream_test.cc index f2a79a6c6b67..7b2f5f9e05ef 100644 --- a/test/common/quic/envoy_quic_server_stream_test.cc +++ b/test/common/quic/envoy_quic_server_stream_test.cc @@ -73,9 +73,10 @@ class EnvoyQuicServerStreamTest : public testing::Test { quic::StreamSendingState state, bool, absl::optional) { return quic::QuicConsumedData{write_length, state != quic::NO_FIN}; })); - EXPECT_CALL(writer_, WritePacket(_, _, _, _, _)) + EXPECT_CALL(writer_, WritePacket(_, _, _, _, _, _)) .WillRepeatedly(Invoke([](const char*, size_t buf_len, const quic::QuicIpAddress&, - const quic::QuicSocketAddress&, quic::PerPacketOptions*) { + const quic::QuicSocketAddress&, quic::PerPacketOptions*, + const quic::QuicPacketWriterParams&) { return quic::WriteResult{quic::WRITE_STATUS_OK, static_cast(buf_len)}; })); } diff --git a/test/common/quic/envoy_quic_writer_test.cc b/test/common/quic/envoy_quic_writer_test.cc index f6a21ffce892..3dc8a748d4bf 100644 --- a/test/common/quic/envoy_quic_writer_test.cc +++ b/test/common/quic/envoy_quic_writer_test.cc @@ -59,22 +59,23 @@ class EnvoyQuicWriterTest : public ::testing::Test { TEST_F(EnvoyQuicWriterTest, AssertOnNonNullPacketOption) { std::string str("Hello World!"); - EXPECT_DEBUG_DEATH(envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, - peer_address_, - reinterpret_cast(0x1)), - ""); + quic::QuicPacketWriterParams params; + EXPECT_DEBUG_DEATH( + envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, peer_address_, + reinterpret_cast(0x1), params), + ""); } TEST_F(EnvoyQuicWriterTest, SendSuccessfully) { std::string str("Hello World!"); - + quic::QuicPacketWriterParams params; EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) .WillOnce(testing::Invoke([this, str](int, const msghdr* message, int) { verifySendData(str, message); return Api::SysCallSizeResult{static_cast(str.length()), 0}; })); quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, - peer_address_, nullptr); + peer_address_, nullptr, params); EXPECT_EQ(quic::WRITE_STATUS_OK, result.status); EXPECT_EQ(str.length(), result.bytes_written); EXPECT_FALSE(envoy_quic_writer_.IsWriteBlocked()); @@ -82,13 +83,14 @@ TEST_F(EnvoyQuicWriterTest, SendSuccessfully) { TEST_F(EnvoyQuicWriterTest, SendBlocked) { std::string str("Hello World!"); + quic::QuicPacketWriterParams params; EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) .WillOnce(testing::Invoke([this, str](int, const msghdr* message, int) { verifySendData(str, message); return Api::SysCallSizeResult{-1, SOCKET_ERROR_AGAIN}; })); quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, - peer_address_, nullptr); + peer_address_, nullptr, params); EXPECT_EQ(quic::WRITE_STATUS_BLOCKED, result.status); EXPECT_EQ(SOCKET_ERROR_AGAIN, result.error_code); EXPECT_TRUE(envoy_quic_writer_.IsWriteBlocked()); @@ -101,7 +103,7 @@ TEST_F(EnvoyQuicWriterTest, SendBlocked) { })); #endif EXPECT_DEBUG_DEATH(envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, - peer_address_, nullptr), + peer_address_, nullptr, params), ""); envoy_quic_writer_.SetWritable(); EXPECT_FALSE(envoy_quic_writer_.IsWriteBlocked()); @@ -109,13 +111,14 @@ TEST_F(EnvoyQuicWriterTest, SendBlocked) { TEST_F(EnvoyQuicWriterTest, SendFailure) { std::string str("Hello World!"); + quic::QuicPacketWriterParams params; EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) .WillOnce(testing::Invoke([this, str](int, const msghdr* message, int) { verifySendData(str, message); return Api::SysCallSizeResult{-1, SOCKET_ERROR_NOT_SUP}; })); quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, - peer_address_, nullptr); + peer_address_, nullptr, params); EXPECT_EQ(quic::WRITE_STATUS_ERROR, result.status); EXPECT_EQ(SOCKET_ERROR_NOT_SUP, result.error_code); EXPECT_FALSE(envoy_quic_writer_.IsWriteBlocked()); @@ -123,13 +126,14 @@ TEST_F(EnvoyQuicWriterTest, SendFailure) { TEST_F(EnvoyQuicWriterTest, SendFailureMessageTooBig) { std::string str("Hello World!"); + quic::QuicPacketWriterParams params; EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) .WillOnce(testing::Invoke([this, str](int, const msghdr* message, int) { verifySendData(str, message); return Api::SysCallSizeResult{-1, SOCKET_ERROR_MSG_SIZE}; })); quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, - peer_address_, nullptr); + peer_address_, nullptr, params); // Currently MessageSize should be propagated through error_code. This test // would fail if QUICHE changes to propagate through status in the future. EXPECT_EQ(quic::WRITE_STATUS_ERROR, result.status); diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 0f2d76724055..3c31ef8fed8a 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -17,7 +17,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/network:94.4" # Flaky, `activateFileEvents`, `startSecureTransport` and `ioctl`, listener_socket do not always report LCOV "source/common/network/dns_resolver:91.6" # A few lines of MacOS code not tested in linux scripts. Tested in MacOS scripts "source/common/protobuf:96.3" -"source/common/quic:93.5" +"source/common/quic:93.4" "source/common/router:96.6" "source/common/secret:95.0" "source/common/signal:87.2" # Death tests don't report LCOV From 3f0b409ec062baec62d650d86333250d0b893df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E6=B3=BD=E8=BD=A9?= Date: Sun, 28 May 2023 22:52:57 +0800 Subject: [PATCH 391/740] docs: we can only choose bazelisk via Homebrew (#27644) If we try to install bazelisk & bazel, there will be an error: Error: Cannot install bazel because conflicting formulae are installed. bazelisk: because Bazelisk replaces the bazel binary Please `brew unlink bazelisk` before continuing. The rest of this doc recommends using bazelisk, so we replace bazel here to bazelisk. Signed-off-by: spacewander --- bazel/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/README.md b/bazel/README.md index 116399e4904c..38cd9a9f0df3 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -122,7 +122,7 @@ for how to update or override dependencies. ### macOS On macOS, you'll need to install several dependencies. This can be accomplished via [Homebrew](https://brew.sh/): ```console - brew install coreutils wget libtool go bazel clang-format autoconf aspell + brew install coreutils wget libtool go bazelisk clang-format autoconf aspell ``` _notes_: `coreutils` is used for `realpath`, `gmd5sum` and `gsha256sum` From 5e7900dc7b88faa78d2343e45d58412616320a2e Mon Sep 17 00:00:00 2001 From: Zhewei Hu Date: Sun, 28 May 2023 18:29:22 -0700 Subject: [PATCH 392/740] [ZK filter] update doc for *_resp_fast metrics (#27613) Signed-off-by: Zhewei Hu Co-authored-by: Zhewei Hu --- .../zookeeper_proxy_filter.rst | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst index 8f914333364e..b3785d9ec37d 100644 --- a/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst @@ -97,33 +97,33 @@ The following counters are available: getallchildrennumber_resp, Counter, Number of getallchildrennumber responses close_resp, Counter, Number of close responses watch_event, Counter, Number of watch events fired by the server - connect_resp_fast, Counter, Number of connect responses faster than the threshold - ping_resp_fast, Counter, Number of ping responses faster than the threshold - auth_resp_fast, Counter, Number of auth responses faster than the threshold - getdata_resp_fast, Counter, Number of getdata responses faster than the threshold - create_resp_fast, Counter, Number of create responses faster than the threshold - create2_resp_fast, Counter, Number of create2 responses faster than the threshold - createcontainer_resp_fast, Counter, Number of createcontainer responses faster than the threshold - createttl_resp_fast, Counter, Number of createttl responses faster than the threshold - setdata_resp_fast, Counter, Number of setdata responses faster than the threshold - getchildren_resp_fast, Counter, Number of getchildren responses faster than the threshold - getchildren2_resp_fast, Counter, Number of getchildren2 responses faster than the threshold - delete_resp_fast, Counter, Number of delete responses faster than the threshold - exists_resp_fast, Counter, Number of exists responses faster than the threshold - getacl_resp_fast, Counter, Number of getacl responses faster than the threshold - setacl_resp_fast, Counter, Number of setacl responses faster than the threshold - sync_resp_fast, Counter, Number of sync responses faster than the threshold - check_resp_fast, Counter, Number of check responses faster than the threshold - multi_resp_fast, Counter, Number of multi responses faster than the threshold - reconfig_resp_fast, Counter, Number of reconfig responses faster than the threshold - setauth_resp_fast, Counter, Number of setauth responses faster than the threshold - setwatches_resp_fast, Counter, Number of setwatches responses faster than the threshold - setwatches2_resp_fast, Counter, Number of setwatches2 responses faster than the threshold - checkwatches_resp_fast, Counter, Number of checkwatches responses faster than the threshold - removewatches_resp_fast, Counter, Number of removewatches responses faster than the threshold - getephemerals_resp_fast, Counter, Number of getephemerals responses faster than the threshold - getallchildrennumber_resp_fast, Counter, Number of getallchildrennumber responses faster than the threshold - close_resp_fast, Counter, Number of close responses faster than the threshold + connect_resp_fast, Counter, Number of connect responses faster than or equal to the threshold + ping_resp_fast, Counter, Number of ping responses faster than or equal to the threshold + auth_resp_fast, Counter, Number of auth responses faster than or equal to the threshold + getdata_resp_fast, Counter, Number of getdata responses faster than or equal to the threshold + create_resp_fast, Counter, Number of create responses faster than or equal to the threshold + create2_resp_fast, Counter, Number of create2 responses faster than or equal to the threshold + createcontainer_resp_fast, Counter, Number of createcontainer responses faster than or equal to the threshold + createttl_resp_fast, Counter, Number of createttl responses faster than or equal to the threshold + setdata_resp_fast, Counter, Number of setdata responses faster than or equal to the threshold + getchildren_resp_fast, Counter, Number of getchildren responses faster than or equal to the threshold + getchildren2_resp_fast, Counter, Number of getchildren2 responses faster than or equal to the threshold + delete_resp_fast, Counter, Number of delete responses faster than or equal to the threshold + exists_resp_fast, Counter, Number of exists responses faster than or equal to the threshold + getacl_resp_fast, Counter, Number of getacl responses faster than or equal to the threshold + setacl_resp_fast, Counter, Number of setacl responses faster than or equal to the threshold + sync_resp_fast, Counter, Number of sync responses faster than or equal to the threshold + check_resp_fast, Counter, Number of check responses faster than or equal to the threshold + multi_resp_fast, Counter, Number of multi responses faster than or equal to the threshold + reconfig_resp_fast, Counter, Number of reconfig responses faster than or equal to the threshold + setauth_resp_fast, Counter, Number of setauth responses faster than or equal to the threshold + setwatches_resp_fast, Counter, Number of setwatches responses faster than or equal to the threshold + setwatches2_resp_fast, Counter, Number of setwatches2 responses faster than or equal to the threshold + checkwatches_resp_fast, Counter, Number of checkwatches responses faster than or equal to the threshold + removewatches_resp_fast, Counter, Number of removewatches responses faster than or equal to the threshold + getephemerals_resp_fast, Counter, Number of getephemerals responses faster than or equal to the threshold + getallchildrennumber_resp_fast, Counter, Number of getallchildrennumber responses faster than or equal to the threshold + close_resp_fast, Counter, Number of close responses faster than or equal to the threshold connect_resp_slow, Counter, Number of connect responses slower than the threshold ping_resp_slow, Counter, Number of ping responses slower than the threshold auth_resp_slow, Counter, Number of auth responses slower than the threshold From 9b27cf63fb62d0e12ae44d417b1815c299b67b58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 09:26:47 +0100 Subject: [PATCH 393/740] build(deps): bump openpolicyagent/opa from 0.52.0-istio to 0.53.0-istio in /examples/ext_authz (#27671) build(deps): bump openpolicyagent/opa in /examples/ext_authz Bumps openpolicyagent/opa from 0.52.0-istio to 0.53.0-istio. --- updated-dependencies: - dependency-name: openpolicyagent/opa dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/ext_authz/Dockerfile-opa | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ext_authz/Dockerfile-opa b/examples/ext_authz/Dockerfile-opa index 950698be7305..70847d7d5ae2 100644 --- a/examples/ext_authz/Dockerfile-opa +++ b/examples/ext_authz/Dockerfile-opa @@ -1 +1 @@ -FROM openpolicyagent/opa:0.52.0-istio@sha256:03fce2c713c1b10c01371aea70aa2630b7a464cce45fe458286a5734ec492a3b +FROM openpolicyagent/opa:0.53.0-istio@sha256:dcf28d23b3d1919b6314ef34342687c558cec64fe4f93fa484e8c8e82824d56d From d8126fa32df3433f57cf95d7f4cabb13c771c4b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 11:30:00 +0100 Subject: [PATCH 394/740] build(deps): bump jaegertracing/all-in-one from `308b14d` to `114816a` in /examples/shared/jaeger (#27670) build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger Bumps jaegertracing/all-in-one from `308b14d` to `114816a`. --- updated-dependencies: - dependency-name: jaegertracing/all-in-one dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/jaeger/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile index 2c87ffde174e..57dfe7cb60a4 100644 --- a/examples/shared/jaeger/Dockerfile +++ b/examples/shared/jaeger/Dockerfile @@ -1,4 +1,4 @@ -FROM jaegertracing/all-in-one@sha256:308b14daf2ddbc171bd5ce882dd5b12c1bac6446913fbb60b69424f4a34b7ade +FROM jaegertracing/all-in-one@sha256:114816a5808fddda2bdcf85f335d306b279b2cebb1e48d9d1d0f0e9ab53beeb9 HEALTHCHECK \ --interval=1s \ --timeout=1s \ From 32724b8808d451491a7afe32f685769816c6673f Mon Sep 17 00:00:00 2001 From: zhaohuabing Date: Mon, 29 May 2023 18:34:45 +0800 Subject: [PATCH 395/740] support fips-compliant build for arm (#27622) Signed-off-by: Huabing Zhao --- bazel/BUILD | 1 - bazel/external/boringssl_fips.genrule_cmd | 59 +++++++++++++++-------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/bazel/BUILD b/bazel/BUILD index c1557d082179..ca8db723eb3e 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -464,7 +464,6 @@ config_setting( name = "boringssl_fips", constraint_values = [ "@platforms//os:linux", - "@platforms//cpu:x86_64", ], values = {"define": "boringssl=fips"}, ) diff --git a/bazel/external/boringssl_fips.genrule_cmd b/bazel/external/boringssl_fips.genrule_cmd index 37fc301a9f9f..46526a9a84de 100755 --- a/bazel/external/boringssl_fips.genrule_cmd +++ b/bazel/external/boringssl_fips.genrule_cmd @@ -2,15 +2,21 @@ set -e -# BoringSSL build as described in the Security Policy for BoringCrypto module (2020-07-02): -# https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp3678.pdf +export CXXFLAGS='' +export LDFLAGS='' -# This works only on Linux-x86_64. -if [[ `uname` != "Linux" || `uname -m` != "x86_64" ]]; then - echo "ERROR: BoringSSL FIPS is currently supported only on Linux-x86_64." +# BoringSSL build as described in the Security Policy for BoringCrypto module (2022-05-06): +# https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp4407.pdf + +OS=`uname` +ARCH=`uname -m` +# This works only on Linux-x86_64 and Linux-aarch64. +if [[ "$OS" != "Linux" || ("$ARCH" != "x86_64" && "$ARCH" != "aarch64") ]]; then + echo "ERROR: BoringSSL FIPS is currently supported only on Linux-x86_64 and Linux-aarch64." exit 1 fi + # Bazel magic. # ROOT=$(dirname $(rootpath boringssl/BUILDING.md))/.. ROOT=./external/boringssl_fips @@ -27,11 +33,15 @@ export PATH="$(dirname `which cmake`):/usr/bin:/bin" # Clang VERSION=12.0.0 -SHA256=a9ff205eb0b73ca7c86afc6432eed1c2d49133bd0d49e47b15be59bbf0dd292e -PLATFORM="x86_64-linux-gnu-ubuntu-20.04" +if [[ "$ARCH" == "x86_64" ]]; then + PLATFORM="x86_64-linux-gnu-ubuntu-20.04" + SHA256=a9ff205eb0b73ca7c86afc6432eed1c2d49133bd0d49e47b15be59bbf0dd292e +else + PLATFORM="aarch64-linux-gnu" + SHA256=d05f0b04fb248ce1e7a61fcd2087e6be8bc4b06b2cc348792f383abf414dec48 +fi -curl -sLO https://github.com/llvm/llvm-project/releases/download/llvmorg-"$VERSION"/clang+llvm-"$VERSION"-"$PLATFORM".tar.xz \ - && echo "$SHA256" clang+llvm-"$VERSION"-"$PLATFORM".tar.xz | sha256sum --check +curl -sLO https://github.com/llvm/llvm-project/releases/download/llvmorg-"$VERSION"/clang+llvm-"$VERSION"-"$PLATFORM".tar.xz tar xf clang+llvm-"$VERSION"-"$PLATFORM".tar.xz export HOME="$PWD" @@ -45,8 +55,13 @@ fi # Go VERSION=1.16.5 -SHA256=b12c23023b68de22f74c0524f10b753e7b08b1504cb7e417eccebdd3fae49061 -PLATFORM="linux-amd64" +if [[ "$ARCH" == "x86_64" ]]; then + PLATFORM="linux-amd64" + SHA256=b12c23023b68de22f74c0524f10b753e7b08b1504cb7e417eccebdd3fae49061 +else + PLATFORM="linux-arm64" + SHA256=d5446b46ef6f36fdffa852f73dfbbe78c1ddf010b99fa4964944b9ae8b4d6799 +fi curl -sLO https://dl.google.com/go/go"$VERSION"."$PLATFORM".tar.gz \ && echo "$SHA256" go"$VERSION"."$PLATFORM".tar.gz | sha256sum --check @@ -63,12 +78,12 @@ fi # Ninja VERSION=1.10.2 -SHA256=763464859c7ef2ea3a0a10f4df40d2025d3bb9438fcb1228404640410c0ec22d -PLATFORM="linux" - -curl -sLO https://github.com/ninja-build/ninja/releases/download/v"$VERSION"/ninja-"$PLATFORM".zip \ - && echo "$SHA256" ninja-"$PLATFORM".zip | sha256sum --check -unzip -o ninja-"$PLATFORM".zip +SHA256=ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed +curl -sLO https://github.com/ninja-build/ninja/archive/refs/tags/v"$VERSION".tar.gz \ + && echo "$SHA256" v"$VERSION".tar.gz | sha256sum --check +tar -xvf v"$VERSION".tar.gz +cd ninja-"$VERSION" +python3 ./configure.py --bootstrap export PATH="$PWD:$PATH" @@ -76,11 +91,17 @@ if [[ `ninja --version` != "$VERSION" ]]; then echo "ERROR: Ninja version doesn't match." exit 1 fi +cd .. # CMake VERSION=3.20.1 -SHA256=b8c141bd7a6d335600ab0a8a35e75af79f95b837f736456b5532f4d717f20a09 -PLATFORM="linux-x86_64" +if [[ "$ARCH" == "x86_64" ]]; then + PLATFORM="linux-x86_64" + SHA256=b8c141bd7a6d335600ab0a8a35e75af79f95b837f736456b5532f4d717f20a09 +else + PLATFORM="linux-aarch64" + SHA256=5ad1f8139498a1956df369c401658ec787f63c8cb4e9759f2edaa51626a86512 +fi curl -sLO https://github.com/Kitware/CMake/releases/download/v"$VERSION"/cmake-"$VERSION"-"$PLATFORM".tar.gz \ && echo "$SHA256" cmake-"$VERSION"-"$PLATFORM".tar.gz | sha256sum --check From f80fd48f5f7417c5aa77ee4f5ca4df6e0f8e625d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 12:27:45 +0100 Subject: [PATCH 396/740] build(deps): bump protobuf from 4.23.1 to 4.23.2 in /examples/grpc-bridge/client (#27668) build(deps): bump protobuf in /examples/grpc-bridge/client Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 4.23.1 to 4.23.2. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v4.23.1...v4.23.2) --- updated-dependencies: - dependency-name: protobuf dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/client/requirements.txt | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index 41ca3dbc8d03..4044bb58fc62 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -112,20 +112,20 @@ idna==3.2 \ --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \ --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3 # via requests -protobuf==4.23.1 \ - --hash=sha256:2036a3a1e7fc27f973fa0a7888dce712393af644f4695385f117886abc792e39 \ - --hash=sha256:32e78beda26d7a101fecf15d7a4a792278a0d26a31bc327ff05564a9d68ab8ee \ - --hash=sha256:346990f634272caac1f09efbcfbbacb23098b1f606d172534c6fa2d9758bb436 \ - --hash=sha256:3b8905eafe4439076e1f58e9d1fa327025fd2777cf90f14083092ae47f77b0aa \ - --hash=sha256:3ce113b3f3362493bddc9069c2163a38f240a9ed685ff83e7bcb756b05e1deb0 \ - --hash=sha256:410bcc0a5b279f634d3e16082ce221dfef7c3392fac723500e2e64d1806dd2be \ - --hash=sha256:5b9cd6097e6acae48a68cb29b56bc79339be84eca65b486910bb1e7a30e2b7c1 \ - --hash=sha256:65f0ac96ef67d7dd09b19a46aad81a851b6f85f89725577f16de38f2d68ad477 \ - --hash=sha256:91fac0753c3c4951fbb98a93271c43cc7cf3b93cf67747b3e600bb1e5cc14d61 \ - --hash=sha256:95789b569418a3e32a53f43d7763be3d490a831e9c08042539462b6d972c2d7e \ - --hash=sha256:ac50be82491369a9ec3710565777e4da87c6d2e20404e0abb1f3a8f10ffd20f0 \ - --hash=sha256:decf119d54e820f298ee6d89c72d6b289ea240c32c521f00433f9dc420595f38 \ - --hash=sha256:f9510cac91e764e86acd74e2b7f7bc5e6127a7f3fb646d7c8033cfb84fd1176a +protobuf==4.23.2 \ + --hash=sha256:09310bce43353b46d73ba7e3bca78273b9bc50349509b9698e64d288c6372c2a \ + --hash=sha256:20874e7ca4436f683b64ebdbee2129a5a2c301579a67d1a7dda2cdf62fb7f5f7 \ + --hash=sha256:25e3370eda26469b58b602e29dff069cfaae8eaa0ef4550039cc5ef8dc004511 \ + --hash=sha256:281342ea5eb631c86697e1e048cb7e73b8a4e85f3299a128c116f05f5c668f8f \ + --hash=sha256:384dd44cb4c43f2ccddd3645389a23ae61aeb8cfa15ca3a0f60e7c3ea09b28b3 \ + --hash=sha256:54a533b971288af3b9926e53850c7eb186886c0c84e61daa8444385a4720297f \ + --hash=sha256:6c081863c379bb1741be8f8193e893511312b1d7329b4a75445d1ea9955be69e \ + --hash=sha256:86df87016d290143c7ce3be3ad52d055714ebaebb57cc659c387e76cfacd81aa \ + --hash=sha256:8da6070310d634c99c0db7df48f10da495cc283fd9e9234877f0cd182d43ab7f \ + --hash=sha256:b2cfab63a230b39ae603834718db74ac11e52bccaaf19bf20f5cce1a84cf76df \ + --hash=sha256:c52cfcbfba8eb791255edd675c1fe6056f723bf832fa67f0442218f8817c076e \ + --hash=sha256:ce744938406de1e64b91410f473736e815f28c3b71201302612a68bf01517fea \ + --hash=sha256:efabbbbac1ab519a514579ba9ec52f006c28ae19d97915951f69fa70da2c9e91 # via # -r requirements.in # grpcio-tools From af8aef48b5395d32d4bbac5aa3ed0a84d3bf31d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 12:31:22 +0100 Subject: [PATCH 397/740] build(deps): bump github/codeql-action from 2.3.4 to 2.3.5 (#27647) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.3.4 to 2.3.5. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/f0e3dfb30302f8a0881bb509b044e0de4f6ef589...0225834cc549ee0ca93cb085b92954821a145866) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 660eec1f12a5..b81e86623169 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -33,7 +33,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@f0e3dfb30302f8a0881bb509b044e0de4f6ef589 + uses: github/codeql-action/init@0225834cc549ee0ca93cb085b92954821a145866 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -64,4 +64,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@f0e3dfb30302f8a0881bb509b044e0de4f6ef589 + uses: github/codeql-action/analyze@0225834cc549ee0ca93cb085b92954821a145866 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index fcb710edb6e4..b1ab668242b7 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -45,7 +45,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@f0e3dfb30302f8a0881bb509b044e0de4f6ef589 + uses: github/codeql-action/init@0225834cc549ee0ca93cb085b92954821a145866 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -78,4 +78,4 @@ jobs: - name: Perform CodeQL Analysis if: env.BUILD_TARGETS != '' - uses: github/codeql-action/analyze@f0e3dfb30302f8a0881bb509b044e0de4f6ef589 + uses: github/codeql-action/analyze@0225834cc549ee0ca93cb085b92954821a145866 From 69ebcab163608f172fe5ac5e7092353e1264b1a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 08:53:23 +0100 Subject: [PATCH 398/740] build(deps): bump github.com/envoyproxy/go-control-plane from 0.11.0 to 0.11.1 in /examples/ext_authz/auth/grpc-service (#27683) build(deps): bump github.com/envoyproxy/go-control-plane Bumps [github.com/envoyproxy/go-control-plane](https://github.com/envoyproxy/go-control-plane) from 0.11.0 to 0.11.1. - [Release notes](https://github.com/envoyproxy/go-control-plane/releases) - [Changelog](https://github.com/envoyproxy/go-control-plane/blob/main/CHANGELOG.md) - [Commits](https://github.com/envoyproxy/go-control-plane/compare/v0.11.0...v0.11.1) --- updated-dependencies: - dependency-name: github.com/envoyproxy/go-control-plane dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/ext_authz/auth/grpc-service/go.mod | 4 +- examples/ext_authz/auth/grpc-service/go.sum | 134 +++++++++++++++++++- 2 files changed, 129 insertions(+), 9 deletions(-) diff --git a/examples/ext_authz/auth/grpc-service/go.mod b/examples/ext_authz/auth/grpc-service/go.mod index a1a9385404d8..8bc653ebc6e5 100644 --- a/examples/ext_authz/auth/grpc-service/go.mod +++ b/examples/ext_authz/auth/grpc-service/go.mod @@ -3,8 +3,8 @@ module github.com/envoyproxy/envoy/examples/ext_authz/auth/grpc-service go 1.14 require ( - github.com/envoyproxy/go-control-plane v0.11.0 + github.com/envoyproxy/go-control-plane v0.11.1 github.com/golang/protobuf v1.5.3 - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 + google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e google.golang.org/grpc v1.55.0 ) diff --git a/examples/ext_authz/auth/grpc-service/go.sum b/examples/ext_authz/auth/grpc-service/go.sum index ed297be4e25b..3eb080a7e70a 100644 --- a/examples/ext_authz/auth/grpc-service/go.sum +++ b/examples/ext_authz/auth/grpc-service/go.sum @@ -42,13 +42,18 @@ cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wx cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= @@ -57,25 +62,35 @@ cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= @@ -96,6 +111,7 @@ cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oe cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -105,12 +121,16 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= @@ -122,9 +142,12 @@ cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5v cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= @@ -133,6 +156,7 @@ cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uX cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -146,6 +170,7 @@ cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARy cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= @@ -156,9 +181,12 @@ cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iW cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= @@ -166,6 +194,7 @@ cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= @@ -173,6 +202,7 @@ cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KF cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= @@ -182,6 +212,7 @@ cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxB cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= @@ -191,14 +222,17 @@ cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZW cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= @@ -206,6 +240,7 @@ cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= @@ -214,12 +249,14 @@ cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= @@ -227,15 +264,19 @@ cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aU cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= @@ -250,6 +291,7 @@ cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+o cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= @@ -265,19 +307,26 @@ cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQE cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= @@ -296,6 +345,7 @@ cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtq cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= @@ -312,22 +362,26 @@ cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJP cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= @@ -353,9 +407,11 @@ cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2om cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -363,8 +419,10 @@ cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjp cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= @@ -372,6 +430,7 @@ cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7d cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= @@ -388,6 +447,8 @@ cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0 cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= @@ -399,11 +460,13 @@ cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQk cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= @@ -414,35 +477,44 @@ cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3s cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -453,9 +525,11 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= @@ -470,12 +544,18 @@ cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV6 cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= @@ -487,11 +567,14 @@ cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiC cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= @@ -522,6 +605,7 @@ github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGW github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -547,8 +631,9 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 h1:58f1tJ1ra+zFINPlwLWvQsR9CzAKt2e+EWV2yX9oXQ4= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74 h1:zlUubfBUxApscKFsF4VSvvfhsBNTBu0eF/ddvpo96yk= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -564,13 +649,15 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= -github.com/envoyproxy/go-control-plane v0.11.0 h1:jtLewhRR2vMRNnq2ZZUoCjUlgut+Y0+sDDWPOfwOi1o= github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= -github.com/envoyproxy/protoc-gen-validate v0.10.0 h1:oIfnZFdC0YhpNNEX+SuIqko4cqqVZeN9IGTrhZje83Y= github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.1 h1:kt9FtLiooDc0vbwTLhdg3dyNX1K9Qwa1EK9LcD4jVUQ= +github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -681,6 +768,7 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -710,6 +798,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -728,6 +817,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -750,6 +840,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -836,6 +927,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -891,8 +983,10 @@ golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1012,8 +1106,10 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -1021,6 +1117,8 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1035,13 +1133,15 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1103,6 +1203,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1174,6 +1275,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/ google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1305,8 +1408,23 @@ google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1346,6 +1464,7 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= @@ -1364,6 +1483,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 3a8ad2fe8afa7a6ab428ded663b3b34cf8f02500 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 09:39:49 +0100 Subject: [PATCH 399/740] build(deps): bump github.com/envoyproxy/go-control-plane from 0.11.0 to 0.11.1 in /examples/load-reporting-service (#27682) build(deps): bump github.com/envoyproxy/go-control-plane Bumps [github.com/envoyproxy/go-control-plane](https://github.com/envoyproxy/go-control-plane) from 0.11.0 to 0.11.1. - [Release notes](https://github.com/envoyproxy/go-control-plane/releases) - [Changelog](https://github.com/envoyproxy/go-control-plane/blob/main/CHANGELOG.md) - [Commits](https://github.com/envoyproxy/go-control-plane/compare/v0.11.0...v0.11.1) --- updated-dependencies: - dependency-name: github.com/envoyproxy/go-control-plane dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/load-reporting-service/go.mod | 2 +- examples/load-reporting-service/go.sum | 134 +++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 8 deletions(-) diff --git a/examples/load-reporting-service/go.mod b/examples/load-reporting-service/go.mod index 99b9d340b241..a7de6e377294 100644 --- a/examples/load-reporting-service/go.mod +++ b/examples/load-reporting-service/go.mod @@ -3,7 +3,7 @@ module github.com/envoyproxy/envoy/examples/load-reporting-service go 1.13 require ( - github.com/envoyproxy/go-control-plane v0.11.0 + github.com/envoyproxy/go-control-plane v0.11.1 github.com/golang/protobuf v1.5.3 google.golang.org/grpc v1.55.0 ) diff --git a/examples/load-reporting-service/go.sum b/examples/load-reporting-service/go.sum index ed297be4e25b..3eb080a7e70a 100644 --- a/examples/load-reporting-service/go.sum +++ b/examples/load-reporting-service/go.sum @@ -42,13 +42,18 @@ cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wx cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= @@ -57,25 +62,35 @@ cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= @@ -96,6 +111,7 @@ cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oe cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -105,12 +121,16 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= @@ -122,9 +142,12 @@ cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5v cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= @@ -133,6 +156,7 @@ cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uX cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -146,6 +170,7 @@ cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARy cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= @@ -156,9 +181,12 @@ cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iW cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= @@ -166,6 +194,7 @@ cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= @@ -173,6 +202,7 @@ cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KF cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= @@ -182,6 +212,7 @@ cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxB cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= @@ -191,14 +222,17 @@ cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZW cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= @@ -206,6 +240,7 @@ cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= @@ -214,12 +249,14 @@ cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= @@ -227,15 +264,19 @@ cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aU cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= @@ -250,6 +291,7 @@ cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+o cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= @@ -265,19 +307,26 @@ cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQE cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= @@ -296,6 +345,7 @@ cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtq cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= @@ -312,22 +362,26 @@ cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJP cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= @@ -353,9 +407,11 @@ cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2om cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -363,8 +419,10 @@ cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjp cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= @@ -372,6 +430,7 @@ cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7d cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= @@ -388,6 +447,8 @@ cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0 cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= @@ -399,11 +460,13 @@ cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQk cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= @@ -414,35 +477,44 @@ cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3s cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -453,9 +525,11 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= @@ -470,12 +544,18 @@ cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV6 cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= @@ -487,11 +567,14 @@ cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiC cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= @@ -522,6 +605,7 @@ github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGW github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -547,8 +631,9 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 h1:58f1tJ1ra+zFINPlwLWvQsR9CzAKt2e+EWV2yX9oXQ4= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74 h1:zlUubfBUxApscKFsF4VSvvfhsBNTBu0eF/ddvpo96yk= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -564,13 +649,15 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= -github.com/envoyproxy/go-control-plane v0.11.0 h1:jtLewhRR2vMRNnq2ZZUoCjUlgut+Y0+sDDWPOfwOi1o= github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= -github.com/envoyproxy/protoc-gen-validate v0.10.0 h1:oIfnZFdC0YhpNNEX+SuIqko4cqqVZeN9IGTrhZje83Y= github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.1 h1:kt9FtLiooDc0vbwTLhdg3dyNX1K9Qwa1EK9LcD4jVUQ= +github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -681,6 +768,7 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -710,6 +798,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -728,6 +817,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -750,6 +840,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -836,6 +927,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -891,8 +983,10 @@ golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1012,8 +1106,10 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -1021,6 +1117,8 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1035,13 +1133,15 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1103,6 +1203,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1174,6 +1275,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/ google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1305,8 +1408,23 @@ google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1346,6 +1464,7 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= @@ -1364,6 +1483,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 08ddc06c9469825f2ce3ae423947b8e5b170143c Mon Sep 17 00:00:00 2001 From: Zhewei Hu Date: Tue, 30 May 2023 02:01:01 -0700 Subject: [PATCH 400/740] [ZK filter] pass std::chrono::milliseconds by value (#27533) Signed-off-by: Zhewei Hu --- .../extensions/filters/network/zookeeper_proxy/decoder.cc | 2 +- source/extensions/filters/network/zookeeper_proxy/decoder.h | 6 +++--- source/extensions/filters/network/zookeeper_proxy/filter.cc | 4 ++-- source/extensions/filters/network/zookeeper_proxy/filter.h | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.cc b/source/extensions/filters/network/zookeeper_proxy/decoder.cc index cce503acb9f9..18a9ee1129da 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.cc @@ -617,7 +617,7 @@ void DecoderImpl::decode(Buffer::Instance& data, DecodeType dtype, uint64_t full } void DecoderImpl::parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len, - const std::chrono::milliseconds& latency) { + const std::chrono::milliseconds latency) { ensureMinLength(len, PROTOCOL_VERSION_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH + INT_LENGTH); const auto timeout = helper_.peekInt32(data, offset); diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.h b/source/extensions/filters/network/zookeeper_proxy/decoder.h index af737a6753d3..540cff9d8e23 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.h @@ -101,9 +101,9 @@ class DecoderCallbacks { virtual void onCloseRequest() PURE; virtual void onResponseBytes(uint64_t bytes) PURE; virtual void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly, - const std::chrono::milliseconds& latency) PURE; + const std::chrono::milliseconds latency) PURE; virtual void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error, - const std::chrono::milliseconds& latency) PURE; + const std::chrono::milliseconds latency) PURE; virtual void onWatchEvent(int32_t event_type, int32_t client_state, const std::string& path, int64_t zxid, int32_t error) PURE; }; @@ -174,7 +174,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { void ensureMaxLength(int32_t len) const; std::string pathOnlyRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len, - const std::chrono::milliseconds& latency); + const std::chrono::milliseconds latency); void parseWatchEvent(Buffer::Instance& data, uint64_t& offset, uint32_t len, int64_t zxid, int32_t error); bool maybeReadBool(Buffer::Instance& data, uint64_t& offset); diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.cc b/source/extensions/filters/network/zookeeper_proxy/filter.cc index 665c61ce09e0..a6369e84444a 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/filter.cc @@ -368,7 +368,7 @@ void ZooKeeperFilter::onCloseRequest() { void ZooKeeperFilter::onConnectResponse(const int32_t proto_version, const int32_t timeout, const bool readonly, - const std::chrono::milliseconds& latency) { + const std::chrono::milliseconds latency) { config_->stats_.connect_resp_.inc(); switch (config_->errorBudgetDecision(OpCodes::Connect, latency)) { @@ -394,7 +394,7 @@ void ZooKeeperFilter::onConnectResponse(const int32_t proto_version, const int32 } void ZooKeeperFilter::onResponse(const OpCodes opcode, const int32_t xid, const int64_t zxid, - const int32_t error, const std::chrono::milliseconds& latency) { + const int32_t error, const std::chrono::milliseconds latency) { Stats::StatName opcode_latency = config_->unknown_opcode_latency_; std::string opname = ""; auto iter = config_->op_code_map_.find(opcode); diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.h b/source/extensions/filters/network/zookeeper_proxy/filter.h index 85dbb04cb322..483c5653a59f 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/filter.h @@ -291,9 +291,9 @@ class ZooKeeperFilter : public Network::Filter, void onCloseRequest() override; void onResponseBytes(uint64_t bytes) override; void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly, - const std::chrono::milliseconds& latency) override; + const std::chrono::milliseconds latency) override; void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error, - const std::chrono::milliseconds& latency) override; + const std::chrono::milliseconds latency) override; void onWatchEvent(int32_t event_type, int32_t client_state, const std::string& path, int64_t zxid, int32_t error) override; From 891329516d9e68c248763308f3c8a52c5ea32261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E6=B3=BD=E8=BD=A9?= Date: Tue, 30 May 2023 18:53:05 +0800 Subject: [PATCH 401/740] golang filter: reject if the cgocheck is not disabled (#27677) Signed-off-by: spacewander --- .../filters/http/source/go/pkg/api/BUILD | 1 + .../http/source/go/pkg/api/cgocheck.go | 32 +++++++++++++++++++ .../filters/http/source/go/pkg/http/config.go | 5 +++ 3 files changed, 38 insertions(+) create mode 100644 contrib/golang/filters/http/source/go/pkg/api/cgocheck.go diff --git a/contrib/golang/filters/http/source/go/pkg/api/BUILD b/contrib/golang/filters/http/source/go/pkg/api/BUILD index 45b8bc46cb1a..392f1cab149d 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/BUILD +++ b/contrib/golang/filters/http/source/go/pkg/api/BUILD @@ -6,6 +6,7 @@ go_library( name = "api", srcs = [ "capi.go", + "cgocheck.go", "filter.go", "type.go", ], diff --git a/contrib/golang/filters/http/source/go/pkg/api/cgocheck.go b/contrib/golang/filters/http/source/go/pkg/api/cgocheck.go new file mode 100644 index 000000000000..01c6f84c8c40 --- /dev/null +++ b/contrib/golang/filters/http/source/go/pkg/api/cgocheck.go @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package api + +import ( + "os" + "strings" +) + +func CgoCheckDisabled() bool { + env := os.Getenv("GODEBUG") + // TODO: handle compile-time GODEBUG var after Go 1.21 is released + if strings.Index(env, "cgocheck=0") != -1 { + return true + } + return false +} diff --git a/contrib/golang/filters/http/source/go/pkg/http/config.go b/contrib/golang/filters/http/source/go/pkg/http/config.go index 703c5bdf97e5..b58b2dfd9f0d 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/config.go +++ b/contrib/golang/filters/http/source/go/pkg/http/config.go @@ -50,6 +50,11 @@ var ( //export envoyGoFilterNewHttpPluginConfig func envoyGoFilterNewHttpPluginConfig(namePtr, nameLen, configPtr, configLen uint64) uint64 { + if !api.CgoCheckDisabled() { + cAPI.HttpLog(api.Error, "The Envoy Golang filter requires the `GODEBUG=cgocheck=0` environment variable set.") + return 0 + } + buf := utils.BytesToSlice(configPtr, configLen) var any anypb.Any proto.Unmarshal(buf, &any) From 86537d1640e47b386e2f466e58931ccfa8aca770 Mon Sep 17 00:00:00 2001 From: norbjd Date: Tue, 30 May 2023 15:02:02 +0200 Subject: [PATCH 402/740] tooling: include protoc-gen-jsonschema repository (#27661) Signed-off-by: norbjd --- api/bazel/repositories.bzl | 4 ++++ api/bazel/repository_locations.bzl | 11 +++++++++++ bazel/dependency_imports.bzl | 3 +++ 3 files changed, 18 insertions(+) diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index b49db8c59a0c..8142c827e981 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -49,6 +49,10 @@ def api_dependencies(): build_file_content = BUF_BUILD_CONTENT, ) + external_http_archive( + name = "com_github_chrusty_protoc_gen_jsonschema", + ) + PROMETHEUSMETRICS_BUILD_CONTENT = """ load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index c259e3633f93..670c0ad4fc04 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -140,4 +140,15 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "Apache-2.0", license_url = "https://github.com/bufbuild/buf/blob/v{version}/LICENSE", ), + com_github_chrusty_protoc_gen_jsonschema = dict( + project_name = "protoc-gen-jsonschema", + project_desc = "Protobuf to JSON-Schema compiler", + project_url = "https://github.com/norbjd/protoc-gen-jsonschema", + strip_prefix = "protoc-gen-jsonschema-{version}", + sha256 = "ba3e313b10a1b50a6c1232d994c13f6e23d3669be4ae7fea13762f42bb3b2abc", + version = "7680e4998426e62b6896995ff73d4d91cc5fb13c", + urls = ["https://github.com/norbjd/protoc-gen-jsonschema/archive/{version}.zip"], + use_category = ["build"], + release_date = "2023-05-30", + ), ) diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index 72c90bf19d74..b743a1936d0d 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -15,6 +15,7 @@ load("@emsdk//:emscripten_deps.bzl", "emscripten_deps") load("@com_github_aignas_rules_shellcheck//:deps.bzl", "shellcheck_dependencies") load("@aspect_bazel_lib//lib:repositories.bzl", "register_jq_toolchains", "register_yq_toolchains") load("@com_google_cel_cpp//bazel:deps.bzl", "parser_deps") +load("@com_github_chrusty_protoc_gen_jsonschema//:deps.bzl", protoc_gen_jsonschema_go_dependencies = "go_dependencies") # go version for rules_go GO_VERSION = "1.18" @@ -146,6 +147,8 @@ def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, y # source = "https://github.com/bufbuild/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L23-L28" ) + protoc_gen_jsonschema_go_dependencies() + def envoy_download_go_sdks(go_version): go_download_sdk( name = "go_linux_amd64", From da913546f6647ffd91939faa54cecc19868db449 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 30 May 2023 14:48:32 +0100 Subject: [PATCH 403/740] deps: Bump `rules_python` -> 0.22.0 (#27688) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index f6769dc0c707..938da6a0bf74 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -884,9 +884,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Python rules for Bazel", project_desc = "Bazel rules for the Python language", project_url = "https://github.com/bazelbuild/rules_python", - version = "0.20.0", - sha256 = "a644da969b6824cc87f8fe7b18101a8a6c57da5db39caa6566ec6109f37d2141", - release_date = "2023-03-20", + version = "0.22.0", + sha256 = "863ba0fa944319f7e3d695711427d9ad80ba92c6edd0b7c7443b84e904689539", + release_date = "2023-05-25", strip_prefix = "rules_python-{version}", urls = ["https://github.com/bazelbuild/rules_python/archive/{version}.tar.gz"], use_category = ["build"], From 8d85d3534dde5e3a1ba4b21617b8a04492383225 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 30 May 2023 14:48:55 +0100 Subject: [PATCH 404/740] deps: Bump `com_github_c_ares_c_ares` -> 1.19.1 (#27685) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 938da6a0bf74..b99962c6db20 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -207,12 +207,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "c-ares", project_desc = "C library for asynchronous DNS requests", project_url = "https://c-ares.haxx.se/", - version = "1.19.0", - sha256 = "bfceba37e23fd531293829002cac0401ef49a6dc55923f7f92236585b7ad1dd3", + version = "1.19.1", + sha256 = "321700399b72ed0e037d0074c629e7741f6b2ec2dda92956abe3e9671d3e268e", strip_prefix = "c-ares-{version}", urls = ["https://github.com/c-ares/c-ares/releases/download/cares-{underscore_version}/c-ares-{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2023-01-28", + release_date = "2023-05-22", cpe = "cpe:2.3:a:c-ares_project:c-ares:*", license = "c-ares", license_url = "https://github.com/c-ares/c-ares/blob/cares-{underscore_version}/LICENSE.md", From e9574b229551d3f3509315475ad60451d10579a1 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 30 May 2023 14:49:15 +0100 Subject: [PATCH 405/740] deps: Bump `rules_rust` -> 0.22.0 (#27689) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index b99962c6db20..350791fd267e 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1321,12 +1321,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Bazel rust rules", project_desc = "Bazel rust rules (used by Wasm)", project_url = "https://github.com/bazelbuild/rules_rust", - version = "0.21.1", - sha256 = "25209daff2ba21e818801c7b2dab0274c43808982d6aea9f796d899db6319146", + version = "0.22.0", + sha256 = "50272c39f20a3a3507cb56dcb5c3b348bda697a7d868708449e2fa6fb893444c", urls = ["https://github.com/bazelbuild/rules_rust/releases/download/{version}/rules_rust-v{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.wasmtime"], - release_date = "2023-04-24", + release_date = "2023-05-22", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/bazelbuild/rules_rust/blob/{version}/LICENSE.txt", From abeed2f37528901d22bdb8f92fe1acad4bacae0b Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 30 May 2023 14:49:41 +0100 Subject: [PATCH 406/740] deps: Bump `com_github_google_perfetto` -> 34.0 (#27691) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 350791fd267e..123d1b13a58f 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -193,12 +193,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Perfetto", project_desc = "Perfetto Tracing SDK", project_url = "https://perfetto.dev/", - version = "32.1", - sha256 = "0d1088b4758b3d5f3813178c6de22386329d42407d23aa1479f20dce96e49d78", + version = "34.0", + sha256 = "81dbf2fac446a0389c80e309b2060dcccd926012ce2a61621a47e3e432aee8c1", strip_prefix = "perfetto-{version}/sdk", urls = ["https://github.com/google/perfetto/archive/v{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2023-02-01", + release_date = "2023-05-02", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/google/perfetto/blob/v{version}/LICENSE", From cac2ac36ebb92f9602401700d9181ecc33811d56 Mon Sep 17 00:00:00 2001 From: StarryNight Date: Tue, 30 May 2023 23:06:17 +0800 Subject: [PATCH 407/740] add go extension trailer get raw api (#27676) Signed-off-by: wangkai19 --- contrib/golang/filters/http/source/go/pkg/http/type.go | 4 +++- contrib/golang/filters/http/test/golang_integration_test.cc | 5 +++++ contrib/golang/filters/http/test/test_data/basic/filter.go | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/contrib/golang/filters/http/source/go/pkg/http/type.go b/contrib/golang/filters/http/source/go/pkg/http/type.go index 411f7db12d7a..a9c35dd85ca1 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/type.go +++ b/contrib/golang/filters/http/source/go/pkg/http/type.go @@ -180,7 +180,9 @@ func (h *requestOrResponseTrailerMapImpl) initTrailers() { } func (h *requestOrResponseTrailerMapImpl) GetRaw(key string) string { - panic("unsupported yet") + var value string + cAPI.HttpGetHeader(unsafe.Pointer(h.request.req), &key, &value) + return value } func (h *requestOrResponseTrailerMapImpl) Get(key string) (string, bool) { diff --git a/contrib/golang/filters/http/test/golang_integration_test.cc b/contrib/golang/filters/http/test/golang_integration_test.cc index 71c8f8fee378..efda0c3f4541 100644 --- a/contrib/golang/filters/http/test/golang_integration_test.cc +++ b/contrib/golang/filters/http/test/golang_integration_test.cc @@ -298,6 +298,11 @@ name: golang true, upstream_request_->trailers()->get(Http::LowerCaseString("x-test-trailer-1")).empty()); + // check trailer value which add in golang: x-test-trailer-2 + entries = upstream_request_->trailers()->get(Http::LowerCaseString("x-test-trailer-2")); + + EXPECT_EQ("bar", entries[0]->value().getStringView()); + Http::TestResponseHeaderMapImpl response_headers{ {":status", "200"}, {"x-test-header-0", "foo"}, diff --git a/contrib/golang/filters/http/test/test_data/basic/filter.go b/contrib/golang/filters/http/test/test_data/basic/filter.go index 93870973acd2..3d73cd9df187 100644 --- a/contrib/golang/filters/http/test/test_data/basic/filter.go +++ b/contrib/golang/filters/http/test/test_data/basic/filter.go @@ -196,6 +196,10 @@ func (f *filter) decodeTrailers(trailers api.RequestTrailerMap) api.StatusType { trailers.Set("x-test-trailer-0", "bar") trailers.Del("x-test-trailer-1") + if trailers.GetRaw("existed-trailer") == "foo" { + trailers.Add("x-test-trailer-2", "bar") + } + if f.panic == "decode-trailer" { badcode() } From 2f511b5d528622d0d2c2e0d592b09b746a4369b9 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 30 May 2023 16:16:13 +0100 Subject: [PATCH 408/740] examples: Fix for cache example (#27695) Not sure why this has not failed before, but the current code is definitely not correct Signed-off-by: Ryan Northey --- examples/cache/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/cache/service.py b/examples/cache/service.py index 08c98df447fd..aff199544db3 100644 --- a/examples/cache/service.py +++ b/examples/cache/service.py @@ -1,5 +1,6 @@ import hashlib import os +import re import datetime from typing import Optional From f0a949c403a41b7ff2a3e72a7a232e0179373cf7 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 30 May 2023 16:38:54 +0100 Subject: [PATCH 409/740] deps: Bump `bazel_gazelle` -> 0.31.0 (#27694) Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 123d1b13a58f..a401e54c11af 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -33,10 +33,10 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Gazelle", project_desc = "Bazel BUILD file generator for Go projects", project_url = "https://github.com/bazelbuild/bazel-gazelle", - version = "0.30.0", - sha256 = "727f3e4edd96ea20c29e8c2ca9e8d2af724d8c7778e7923a854b2c80952bc405", + version = "0.31.0", + sha256 = "29d5dafc2a5582995488c6735115d1d366fcd6a0fc2e2a153f02988706349825", urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v{version}/bazel-gazelle-v{version}.tar.gz"], - release_date = "2023-03-30", + release_date = "2023-05-27", use_category = ["build"], license = "Apache-2.0", license_url = "https://github.com/bazelbuild/bazel-gazelle/blob/v{version}/LICENSE", From ddac503792e67cabf9e593ed89ce829fb693b3b5 Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 30 May 2023 16:39:08 +0100 Subject: [PATCH 410/740] deps: Bump `com_github_bufbuild_buf` -> 1.19.0 (#27690) Signed-off-by: Ryan Northey --- api/bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 670c0ad4fc04..4a95ca496959 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -131,11 +131,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "buf", project_desc = "A new way of working with Protocol Buffers.", # Used for breaking change detection in API protobufs project_url = "https://buf.build", - version = "1.14.0", - sha256 = "9ab382081872df03faaf192cfa82566d32436cfd78782035e94b4d04a982620f", + version = "1.19.0", + sha256 = "ff35aa96b54037d492d30a21dce8d96d47693a761487f98551d750407c27c285", strip_prefix = "buf", urls = ["https://github.com/bufbuild/buf/releases/download/v{version}/buf-Linux-x86_64.tar.gz"], - release_date = "2023-02-09", + release_date = "2023-05-17", use_category = ["api"], license = "Apache-2.0", license_url = "https://github.com/bufbuild/buf/blob/v{version}/LICENSE", From 7f51511d154e53eda1d93a856bf23aec7f75eb4a Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 30 May 2023 12:00:24 -0400 Subject: [PATCH 411/740] mobile: moving test filters to test directory (#27611) Signed-off-by: Alyssa Wilk --- mobile/envoy_build_config/BUILD | 16 ++++++------- mobile/envoy_build_config/test_extensions.cc | 18 ++++++++------ .../extensions/filters/http/assertion/BUILD | 23 ------------------ .../common/http/filters}/assertion/BUILD | 24 +++++++++++++++---- .../assertion/assertion_filter_test.cc | 6 ++--- .../common/http/filters}/assertion/config.cc | 4 ++-- .../common/http/filters}/assertion/config.h | 4 ++-- .../common/http/filters}/assertion/filter.cc | 2 +- .../common/http/filters}/assertion/filter.h | 2 +- .../http/filters}/assertion/filter.proto | 0 .../http/filters}/route_cache_reset/BUILD | 10 ++++---- .../http/filters}/route_cache_reset/config.cc | 4 ++-- .../http/filters}/route_cache_reset/config.h | 4 ++-- .../http/filters}/route_cache_reset/filter.cc | 2 +- .../http/filters}/route_cache_reset/filter.h | 2 +- .../filters}/route_cache_reset/filter.proto | 0 .../common/http/filters}/test_accessor/BUILD | 10 ++++---- .../http/filters}/test_accessor/config.cc | 4 ++-- .../http/filters}/test_accessor/config.h | 6 ++--- .../http/filters}/test_accessor/filter.cc | 2 +- .../http/filters}/test_accessor/filter.h | 3 ++- .../http/filters}/test_accessor/filter.proto | 0 .../http/filters}/test_event_tracker/BUILD | 10 ++++---- .../filters}/test_event_tracker/config.cc | 4 ++-- .../http/filters}/test_event_tracker/config.h | 6 ++--- .../filters}/test_event_tracker/filter.cc | 2 +- .../http/filters}/test_event_tracker/filter.h | 3 ++- .../filters}/test_event_tracker/filter.proto | 0 .../common/http/filters}/test_kv_store/BUILD | 10 ++++---- .../http/filters}/test_kv_store/config.cc | 4 ++-- .../http/filters}/test_kv_store/config.h | 6 ++--- .../http/filters}/test_kv_store/filter.cc | 2 +- .../http/filters}/test_kv_store/filter.h | 3 ++- .../http/filters}/test_kv_store/filter.proto | 0 .../common/http/filters}/test_logger/BUILD | 10 ++++---- .../http/filters}/test_logger/config.cc | 4 ++-- .../common/http/filters}/test_logger/config.h | 6 ++--- .../common/http/filters}/test_logger/filter.h | 2 +- .../http/filters}/test_logger/filter.proto | 0 .../common/http/filters}/test_read/BUILD | 10 ++++---- .../common/http/filters}/test_read/config.cc | 4 ++-- .../common/http/filters}/test_read/config.h | 4 ++-- .../common/http/filters}/test_read/filter.cc | 2 +- .../common/http/filters}/test_read/filter.h | 2 +- .../http/filters}/test_read/filter.proto | 0 .../http/filters}/test_remote_response/BUILD | 10 ++++---- .../filters}/test_remote_response/config.cc | 4 ++-- .../filters}/test_remote_response/config.h | 4 ++-- .../filters}/test_remote_response/filter.cc | 2 +- .../filters}/test_remote_response/filter.h | 2 +- .../test_remote_response/filter.proto | 0 tools/code_format/check_format.py | 2 +- 52 files changed, 131 insertions(+), 133 deletions(-) delete mode 100644 mobile/test/common/extensions/filters/http/assertion/BUILD rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/assertion/BUILD (67%) rename mobile/test/common/{extensions/filters/http => http/filters}/assertion/assertion_filter_test.cc (98%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/assertion/config.cc (86%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/assertion/config.h (85%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/assertion/filter.cc (99%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/assertion/filter.h (96%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/assertion/filter.proto (100%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/route_cache_reset/BUILD (86%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/route_cache_reset/config.cc (83%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/route_cache_reset/config.h (84%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/route_cache_reset/filter.cc (93%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/route_cache_reset/filter.h (94%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/route_cache_reset/filter.proto (100%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_accessor/BUILD (88%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_accessor/config.cc (85%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_accessor/config.h (79%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_accessor/filter.cc (93%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_accessor/filter.h (94%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_accessor/filter.proto (100%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_event_tracker/BUILD (88%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_event_tracker/config.cc (85%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_event_tracker/config.h (79%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_event_tracker/filter.cc (93%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_event_tracker/filter.h (95%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_event_tracker/filter.proto (100%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_kv_store/BUILD (89%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_kv_store/config.cc (86%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_kv_store/config.h (80%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_kv_store/filter.cc (95%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_kv_store/filter.h (96%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_kv_store/filter.proto (100%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_logger/BUILD (86%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_logger/config.cc (83%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_logger/config.h (78%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_logger/filter.h (88%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_logger/filter.proto (100%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_read/BUILD (87%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_read/config.cc (83%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_read/config.h (85%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_read/filter.cc (96%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_read/filter.h (93%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_read/filter.proto (100%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_remote_response/BUILD (90%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_remote_response/config.cc (83%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_remote_response/config.h (84%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_remote_response/filter.cc (96%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_remote_response/filter.h (92%) rename mobile/{library/common/extensions/filters/http => test/common/http/filters}/test_remote_response/filter.proto (100%) diff --git a/mobile/envoy_build_config/BUILD b/mobile/envoy_build_config/BUILD index 3b3c27bddcf1..49edbf5a85ff 100644 --- a/mobile/envoy_build_config/BUILD +++ b/mobile/envoy_build_config/BUILD @@ -115,14 +115,14 @@ envoy_cc_library( deps = [ "@envoy//source/extensions/clusters/static:static_cluster_lib", "@envoy//source/extensions/filters/http/buffer:config", - "@envoy_mobile//library/common/extensions/filters/http/assertion:config", - "@envoy_mobile//library/common/extensions/filters/http/route_cache_reset:config", - "@envoy_mobile//library/common/extensions/filters/http/test_accessor:config", - "@envoy_mobile//library/common/extensions/filters/http/test_event_tracker:config", - "@envoy_mobile//library/common/extensions/filters/http/test_kv_store:config", - "@envoy_mobile//library/common/extensions/filters/http/test_logger:config", - "@envoy_mobile//library/common/extensions/filters/http/test_read:config", - "@envoy_mobile//library/common/extensions/filters/http/test_remote_response:config", + "@envoy_mobile//test/common/http/filters/assertion:config", + "@envoy_mobile//test/common/http/filters/route_cache_reset:config", + "@envoy_mobile//test/common/http/filters/test_accessor:config", + "@envoy_mobile//test/common/http/filters/test_event_tracker:config", + "@envoy_mobile//test/common/http/filters/test_kv_store:config", + "@envoy_mobile//test/common/http/filters/test_logger:config", + "@envoy_mobile//test/common/http/filters/test_read:config", + "@envoy_mobile//test/common/http/filters/test_remote_response:config", ], alwayslink = 1, ) diff --git a/mobile/envoy_build_config/test_extensions.cc b/mobile/envoy_build_config/test_extensions.cc index d0bb9c56087b..b18f120c95fc 100644 --- a/mobile/envoy_build_config/test_extensions.cc +++ b/mobile/envoy_build_config/test_extensions.cc @@ -1,14 +1,16 @@ #include "source/extensions/clusters/static/static_cluster.h" #include "source/extensions/filters/http/buffer/config.h" +#include "test/common/http/filters/assertion/config.h" +#include "test/common/http/filters/route_cache_reset/config.h" +#include "test/common/http/filters/test_accessor/config.h" +#include "test/common/http/filters/test_event_tracker/config.h" +#include "test/common/http/filters/test_kv_store/config.h" +#include "test/common/http/filters/test_logger/config.h" +#include "test/common/http/filters/test_read/config.h" +#include "test/common/http/filters/test_remote_response/config.h" + #include "external/envoy_build_config/test_extensions.h" -#include "library/common/extensions/filters/http/assertion/config.h" -#include "library/common/extensions/filters/http/route_cache_reset/config.h" -#include "library/common/extensions/filters/http/test_accessor/config.h" -#include "library/common/extensions/filters/http/test_event_tracker/config.h" -#include "library/common/extensions/filters/http/test_kv_store/config.h" -#include "library/common/extensions/filters/http/test_logger/config.h" -#include "library/common/extensions/filters/http/test_read/config.h" void register_test_extensions() { Envoy::Extensions::HttpFilters::Assertion::forceRegisterAssertionFilterFactory(); @@ -18,6 +20,8 @@ void register_test_extensions() { Envoy::Extensions::HttpFilters::TestEventTracker::forceRegisterTestEventTrackerFilterFactory(); Envoy::Extensions::HttpFilters::TestKeyValueStore::forceRegisterTestKeyValueStoreFilterFactory(); Envoy::Extensions::HttpFilters::TestLogger::forceRegisterFactory(); + Envoy::Extensions::HttpFilters::TestRemoteResponse:: + forceRegisterTestRemoteResponseFilterFactory(); Envoy::HttpFilters::TestRead::forceRegisterTestReadFilterFactory(); Envoy::Upstream::forceRegisterStaticClusterFactory(); } diff --git a/mobile/test/common/extensions/filters/http/assertion/BUILD b/mobile/test/common/extensions/filters/http/assertion/BUILD deleted file mode 100644 index 5081eebf6d63..000000000000 --- a/mobile/test/common/extensions/filters/http/assertion/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") -load( - "@envoy//test/extensions:extensions_build_system.bzl", - "envoy_extension_cc_test", -) - -licenses(["notice"]) # Apache 2 - -envoy_package() - -envoy_extension_cc_test( - name = "assertion_filter_test", - srcs = ["assertion_filter_test.cc"], - extension_names = ["envoy.filters.http.assertion"], - repository = "@envoy", - deps = [ - "//library/common/extensions/filters/http/assertion:config", - "//library/common/extensions/filters/http/assertion:filter_cc_proto", - "@envoy//test/mocks/http:http_mocks", - "@envoy//test/mocks/server:factory_context_mocks", - "@envoy//test/test_common:utility_lib", - ], -) diff --git a/mobile/library/common/extensions/filters/http/assertion/BUILD b/mobile/test/common/http/filters/assertion/BUILD similarity index 67% rename from mobile/library/common/extensions/filters/http/assertion/BUILD rename to mobile/test/common/http/filters/assertion/BUILD index 8e858b5b4ccf..30e054af6339 100644 --- a/mobile/library/common/extensions/filters/http/assertion/BUILD +++ b/mobile/test/common/http/filters/assertion/BUILD @@ -1,13 +1,14 @@ load( "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", + "envoy_cc_library", + "envoy_cc_test", + "envoy_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_extension_package() +envoy_package() envoy_proto_library( name = "filter", @@ -17,7 +18,7 @@ envoy_proto_library( ], ) -envoy_cc_extension( +envoy_cc_library( name = "assertion_filter_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], @@ -32,7 +33,7 @@ envoy_cc_extension( ], ) -envoy_cc_extension( +envoy_cc_library( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], @@ -42,3 +43,16 @@ envoy_cc_extension( "@envoy//source/extensions/filters/http/common:factory_base_lib", ], ) + +envoy_cc_test( + name = "assertion_filter_test", + srcs = ["assertion_filter_test.cc"], + repository = "@envoy", + deps = [ + ":config", + ":filter_cc_proto", + "@envoy//test/mocks/http:http_mocks", + "@envoy//test/mocks/server:factory_context_mocks", + "@envoy//test/test_common:utility_lib", + ], +) diff --git a/mobile/test/common/extensions/filters/http/assertion/assertion_filter_test.cc b/mobile/test/common/http/filters/assertion/assertion_filter_test.cc similarity index 98% rename from mobile/test/common/extensions/filters/http/assertion/assertion_filter_test.cc rename to mobile/test/common/http/filters/assertion/assertion_filter_test.cc index ea1d66dbba06..9623ccc6787e 100644 --- a/mobile/test/common/extensions/filters/http/assertion/assertion_filter_test.cc +++ b/mobile/test/common/http/filters/assertion/assertion_filter_test.cc @@ -1,11 +1,11 @@ +#include "test/common/http/filters/assertion/config.h" +#include "test/common/http/filters/assertion/filter.h" +#include "test/common/http/filters/assertion/filter.pb.h" #include "test/mocks/http/mocks.h" #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" -#include "library/common/extensions/filters/http/assertion/config.h" -#include "library/common/extensions/filters/http/assertion/filter.h" -#include "library/common/extensions/filters/http/assertion/filter.pb.h" using testing::ByMove; using testing::Return; diff --git a/mobile/library/common/extensions/filters/http/assertion/config.cc b/mobile/test/common/http/filters/assertion/config.cc similarity index 86% rename from mobile/library/common/extensions/filters/http/assertion/config.cc rename to mobile/test/common/http/filters/assertion/config.cc index 14e45062cc5c..0637ba36855f 100644 --- a/mobile/library/common/extensions/filters/http/assertion/config.cc +++ b/mobile/test/common/http/filters/assertion/config.cc @@ -1,6 +1,6 @@ -#include "library/common/extensions/filters/http/assertion/config.h" +#include "test/common/http/filters/assertion/config.h" -#include "library/common/extensions/filters/http/assertion/filter.h" +#include "test/common/http/filters/assertion/filter.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/assertion/config.h b/mobile/test/common/http/filters/assertion/config.h similarity index 85% rename from mobile/library/common/extensions/filters/http/assertion/config.h rename to mobile/test/common/http/filters/assertion/config.h index 4b4e69a786a9..4c3df493b976 100644 --- a/mobile/library/common/extensions/filters/http/assertion/config.h +++ b/mobile/test/common/http/filters/assertion/config.h @@ -4,8 +4,8 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "library/common/extensions/filters/http/assertion/filter.pb.h" -#include "library/common/extensions/filters/http/assertion/filter.pb.validate.h" +#include "test/common/http/filters/assertion/filter.pb.h" +#include "test/common/http/filters/assertion/filter.pb.validate.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/assertion/filter.cc b/mobile/test/common/http/filters/assertion/filter.cc similarity index 99% rename from mobile/library/common/extensions/filters/http/assertion/filter.cc rename to mobile/test/common/http/filters/assertion/filter.cc index aac5f232bb92..b01770762a89 100644 --- a/mobile/library/common/extensions/filters/http/assertion/filter.cc +++ b/mobile/test/common/http/filters/assertion/filter.cc @@ -1,4 +1,4 @@ -#include "library/common/extensions/filters/http/assertion/filter.h" +#include "test/common/http/filters/assertion/filter.h" #include "envoy/http/codes.h" #include "envoy/server/filter_config.h" diff --git a/mobile/library/common/extensions/filters/http/assertion/filter.h b/mobile/test/common/http/filters/assertion/filter.h similarity index 96% rename from mobile/library/common/extensions/filters/http/assertion/filter.h rename to mobile/test/common/http/filters/assertion/filter.h index d235eb5ac8f7..19f5c14abb43 100644 --- a/mobile/library/common/extensions/filters/http/assertion/filter.h +++ b/mobile/test/common/http/filters/assertion/filter.h @@ -5,7 +5,7 @@ #include "source/extensions/common/matcher/matcher.h" #include "source/extensions/filters/http/common/pass_through_filter.h" -#include "library/common/extensions/filters/http/assertion/filter.pb.h" +#include "test/common/http/filters/assertion/filter.pb.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/assertion/filter.proto b/mobile/test/common/http/filters/assertion/filter.proto similarity index 100% rename from mobile/library/common/extensions/filters/http/assertion/filter.proto rename to mobile/test/common/http/filters/assertion/filter.proto diff --git a/mobile/library/common/extensions/filters/http/route_cache_reset/BUILD b/mobile/test/common/http/filters/route_cache_reset/BUILD similarity index 86% rename from mobile/library/common/extensions/filters/http/route_cache_reset/BUILD rename to mobile/test/common/http/filters/route_cache_reset/BUILD index 44d4f60f2437..866454f1e06b 100644 --- a/mobile/library/common/extensions/filters/http/route_cache_reset/BUILD +++ b/mobile/test/common/http/filters/route_cache_reset/BUILD @@ -1,20 +1,20 @@ load( "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", + "envoy_cc_library", + "envoy_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_extension_package() +envoy_package() envoy_proto_library( name = "filter", srcs = ["filter.proto"], ) -envoy_cc_extension( +envoy_cc_library( name = "route_cache_reset_filter_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], @@ -28,7 +28,7 @@ envoy_cc_extension( ], ) -envoy_cc_extension( +envoy_cc_library( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], diff --git a/mobile/library/common/extensions/filters/http/route_cache_reset/config.cc b/mobile/test/common/http/filters/route_cache_reset/config.cc similarity index 83% rename from mobile/library/common/extensions/filters/http/route_cache_reset/config.cc rename to mobile/test/common/http/filters/route_cache_reset/config.cc index 84b6dfedb065..80455061b043 100644 --- a/mobile/library/common/extensions/filters/http/route_cache_reset/config.cc +++ b/mobile/test/common/http/filters/route_cache_reset/config.cc @@ -1,6 +1,6 @@ -#include "library/common/extensions/filters/http/route_cache_reset/config.h" +#include "test/common/http/filters/route_cache_reset/config.h" -#include "library/common/extensions/filters/http/route_cache_reset/filter.h" +#include "test/common/http/filters/route_cache_reset/filter.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/route_cache_reset/config.h b/mobile/test/common/http/filters/route_cache_reset/config.h similarity index 84% rename from mobile/library/common/extensions/filters/http/route_cache_reset/config.h rename to mobile/test/common/http/filters/route_cache_reset/config.h index b29a9c4605c3..6a5734258e37 100644 --- a/mobile/library/common/extensions/filters/http/route_cache_reset/config.h +++ b/mobile/test/common/http/filters/route_cache_reset/config.h @@ -2,8 +2,8 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "library/common/extensions/filters/http/route_cache_reset/filter.pb.h" -#include "library/common/extensions/filters/http/route_cache_reset/filter.pb.validate.h" +#include "test/common/http/filters/route_cache_reset/filter.pb.h" +#include "test/common/http/filters/route_cache_reset/filter.pb.validate.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/route_cache_reset/filter.cc b/mobile/test/common/http/filters/route_cache_reset/filter.cc similarity index 93% rename from mobile/library/common/extensions/filters/http/route_cache_reset/filter.cc rename to mobile/test/common/http/filters/route_cache_reset/filter.cc index 0ff7e304c95a..25f0cf3a0227 100644 --- a/mobile/library/common/extensions/filters/http/route_cache_reset/filter.cc +++ b/mobile/test/common/http/filters/route_cache_reset/filter.cc @@ -1,4 +1,4 @@ -#include "library/common/extensions/filters/http/route_cache_reset/filter.h" +#include "test/common/http/filters/route_cache_reset/filter.h" #include "envoy/server/filter_config.h" diff --git a/mobile/library/common/extensions/filters/http/route_cache_reset/filter.h b/mobile/test/common/http/filters/route_cache_reset/filter.h similarity index 94% rename from mobile/library/common/extensions/filters/http/route_cache_reset/filter.h rename to mobile/test/common/http/filters/route_cache_reset/filter.h index 52f0d8ed6c3f..e6a744523261 100644 --- a/mobile/library/common/extensions/filters/http/route_cache_reset/filter.h +++ b/mobile/test/common/http/filters/route_cache_reset/filter.h @@ -4,7 +4,7 @@ #include "source/common/common/logger.h" -#include "library/common/extensions/filters/http/route_cache_reset/filter.pb.h" +#include "test/common/http/filters/route_cache_reset/filter.pb.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/route_cache_reset/filter.proto b/mobile/test/common/http/filters/route_cache_reset/filter.proto similarity index 100% rename from mobile/library/common/extensions/filters/http/route_cache_reset/filter.proto rename to mobile/test/common/http/filters/route_cache_reset/filter.proto diff --git a/mobile/library/common/extensions/filters/http/test_accessor/BUILD b/mobile/test/common/http/filters/test_accessor/BUILD similarity index 88% rename from mobile/library/common/extensions/filters/http/test_accessor/BUILD rename to mobile/test/common/http/filters/test_accessor/BUILD index 977df4ab6783..90c55313131d 100644 --- a/mobile/library/common/extensions/filters/http/test_accessor/BUILD +++ b/mobile/test/common/http/filters/test_accessor/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", + "envoy_cc_library", + "envoy_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_extension_package() +envoy_package() envoy_proto_library( name = "filter", @@ -17,7 +17,7 @@ envoy_proto_library( ], ) -envoy_cc_extension( +envoy_cc_library( name = "test_accessor_filter_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], @@ -32,7 +32,7 @@ envoy_cc_extension( ], ) -envoy_cc_extension( +envoy_cc_library( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], diff --git a/mobile/library/common/extensions/filters/http/test_accessor/config.cc b/mobile/test/common/http/filters/test_accessor/config.cc similarity index 85% rename from mobile/library/common/extensions/filters/http/test_accessor/config.cc rename to mobile/test/common/http/filters/test_accessor/config.cc index 44deda4b9561..9ff030337689 100644 --- a/mobile/library/common/extensions/filters/http/test_accessor/config.cc +++ b/mobile/test/common/http/filters/test_accessor/config.cc @@ -1,6 +1,6 @@ -#include "library/common/extensions/filters/http/test_accessor/config.h" +#include "test/common/http/filters/test_accessor/config.h" -#include "library/common/extensions/filters/http/test_accessor/filter.h" +#include "test/common/http/filters/test_accessor/filter.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_accessor/config.h b/mobile/test/common/http/filters/test_accessor/config.h similarity index 79% rename from mobile/library/common/extensions/filters/http/test_accessor/config.h rename to mobile/test/common/http/filters/test_accessor/config.h index 44bd26007b76..a1ae5603426c 100644 --- a/mobile/library/common/extensions/filters/http/test_accessor/config.h +++ b/mobile/test/common/http/filters/test_accessor/config.h @@ -4,9 +4,9 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "library/common/extensions/filters/http/test_accessor/filter.h" -#include "library/common/extensions/filters/http/test_accessor/filter.pb.h" -#include "library/common/extensions/filters/http/test_accessor/filter.pb.validate.h" +#include "test/common/http/filters/test_accessor/filter.h" +#include "test/common/http/filters/test_accessor/filter.pb.h" +#include "test/common/http/filters/test_accessor/filter.pb.validate.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_accessor/filter.cc b/mobile/test/common/http/filters/test_accessor/filter.cc similarity index 93% rename from mobile/library/common/extensions/filters/http/test_accessor/filter.cc rename to mobile/test/common/http/filters/test_accessor/filter.cc index e35f152dd004..274dab56ea23 100644 --- a/mobile/library/common/extensions/filters/http/test_accessor/filter.cc +++ b/mobile/test/common/http/filters/test_accessor/filter.cc @@ -1,4 +1,4 @@ -#include "library/common/extensions/filters/http/test_accessor/filter.h" +#include "test/common/http/filters/test_accessor/filter.h" #include "envoy/server/filter_config.h" diff --git a/mobile/library/common/extensions/filters/http/test_accessor/filter.h b/mobile/test/common/http/filters/test_accessor/filter.h similarity index 94% rename from mobile/library/common/extensions/filters/http/test_accessor/filter.h rename to mobile/test/common/http/filters/test_accessor/filter.h index d5a7355c19e3..599702db5259 100644 --- a/mobile/library/common/extensions/filters/http/test_accessor/filter.h +++ b/mobile/test/common/http/filters/test_accessor/filter.h @@ -4,9 +4,10 @@ #include "source/extensions/filters/http/common/pass_through_filter.h" +#include "test/common/http/filters/test_accessor/filter.pb.h" + #include "library/common/api/c_types.h" #include "library/common/api/external.h" -#include "library/common/extensions/filters/http/test_accessor/filter.pb.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_accessor/filter.proto b/mobile/test/common/http/filters/test_accessor/filter.proto similarity index 100% rename from mobile/library/common/extensions/filters/http/test_accessor/filter.proto rename to mobile/test/common/http/filters/test_accessor/filter.proto diff --git a/mobile/library/common/extensions/filters/http/test_event_tracker/BUILD b/mobile/test/common/http/filters/test_event_tracker/BUILD similarity index 88% rename from mobile/library/common/extensions/filters/http/test_event_tracker/BUILD rename to mobile/test/common/http/filters/test_event_tracker/BUILD index f015f035bba8..9c1315425c81 100644 --- a/mobile/library/common/extensions/filters/http/test_event_tracker/BUILD +++ b/mobile/test/common/http/filters/test_event_tracker/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", + "envoy_cc_library", + "envoy_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_extension_package() +envoy_package() envoy_proto_library( name = "filter", @@ -17,7 +17,7 @@ envoy_proto_library( ], ) -envoy_cc_extension( +envoy_cc_library( name = "test_event_tracker_filter_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], @@ -33,7 +33,7 @@ envoy_cc_extension( ], ) -envoy_cc_extension( +envoy_cc_library( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], diff --git a/mobile/library/common/extensions/filters/http/test_event_tracker/config.cc b/mobile/test/common/http/filters/test_event_tracker/config.cc similarity index 85% rename from mobile/library/common/extensions/filters/http/test_event_tracker/config.cc rename to mobile/test/common/http/filters/test_event_tracker/config.cc index 8f68091d65fc..21854dfbff0c 100644 --- a/mobile/library/common/extensions/filters/http/test_event_tracker/config.cc +++ b/mobile/test/common/http/filters/test_event_tracker/config.cc @@ -1,6 +1,6 @@ -#include "library/common/extensions/filters/http/test_event_tracker/config.h" +#include "test/common/http/filters/test_event_tracker/config.h" -#include "library/common/extensions/filters/http/test_event_tracker/filter.h" +#include "test/common/http/filters/test_event_tracker/filter.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_event_tracker/config.h b/mobile/test/common/http/filters/test_event_tracker/config.h similarity index 79% rename from mobile/library/common/extensions/filters/http/test_event_tracker/config.h rename to mobile/test/common/http/filters/test_event_tracker/config.h index 8d4746728fc3..d246c93ad750 100644 --- a/mobile/library/common/extensions/filters/http/test_event_tracker/config.h +++ b/mobile/test/common/http/filters/test_event_tracker/config.h @@ -4,9 +4,9 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "library/common/extensions/filters/http/test_event_tracker/filter.h" -#include "library/common/extensions/filters/http/test_event_tracker/filter.pb.h" -#include "library/common/extensions/filters/http/test_event_tracker/filter.pb.validate.h" +#include "test/common/http/filters/test_event_tracker/filter.h" +#include "test/common/http/filters/test_event_tracker/filter.pb.h" +#include "test/common/http/filters/test_event_tracker/filter.pb.validate.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_event_tracker/filter.cc b/mobile/test/common/http/filters/test_event_tracker/filter.cc similarity index 93% rename from mobile/library/common/extensions/filters/http/test_event_tracker/filter.cc rename to mobile/test/common/http/filters/test_event_tracker/filter.cc index 24a76a53dd0d..5f5d1733c7dc 100644 --- a/mobile/library/common/extensions/filters/http/test_event_tracker/filter.cc +++ b/mobile/test/common/http/filters/test_event_tracker/filter.cc @@ -1,4 +1,4 @@ -#include "library/common/extensions/filters/http/test_event_tracker/filter.h" +#include "test/common/http/filters/test_event_tracker/filter.h" #include "library/common/api/external.h" #include "library/common/bridge/utility.h" diff --git a/mobile/library/common/extensions/filters/http/test_event_tracker/filter.h b/mobile/test/common/http/filters/test_event_tracker/filter.h similarity index 95% rename from mobile/library/common/extensions/filters/http/test_event_tracker/filter.h rename to mobile/test/common/http/filters/test_event_tracker/filter.h index 93bb5850104d..ffc5e18086d2 100644 --- a/mobile/library/common/extensions/filters/http/test_event_tracker/filter.h +++ b/mobile/test/common/http/filters/test_event_tracker/filter.h @@ -4,8 +4,9 @@ #include "source/extensions/filters/http/common/pass_through_filter.h" +#include "test/common/http/filters/test_event_tracker/filter.pb.h" + #include "library/common/api/c_types.h" -#include "library/common/extensions/filters/http/test_event_tracker/filter.pb.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_event_tracker/filter.proto b/mobile/test/common/http/filters/test_event_tracker/filter.proto similarity index 100% rename from mobile/library/common/extensions/filters/http/test_event_tracker/filter.proto rename to mobile/test/common/http/filters/test_event_tracker/filter.proto diff --git a/mobile/library/common/extensions/filters/http/test_kv_store/BUILD b/mobile/test/common/http/filters/test_kv_store/BUILD similarity index 89% rename from mobile/library/common/extensions/filters/http/test_kv_store/BUILD rename to mobile/test/common/http/filters/test_kv_store/BUILD index 418364d8c9ac..076bbb8fa330 100644 --- a/mobile/library/common/extensions/filters/http/test_kv_store/BUILD +++ b/mobile/test/common/http/filters/test_kv_store/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", + "envoy_cc_library", + "envoy_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_extension_package() +envoy_package() envoy_proto_library( name = "filter", @@ -17,7 +17,7 @@ envoy_proto_library( ], ) -envoy_cc_extension( +envoy_cc_library( name = "test_kv_store_filter_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], @@ -33,7 +33,7 @@ envoy_cc_extension( ], ) -envoy_cc_extension( +envoy_cc_library( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], diff --git a/mobile/library/common/extensions/filters/http/test_kv_store/config.cc b/mobile/test/common/http/filters/test_kv_store/config.cc similarity index 86% rename from mobile/library/common/extensions/filters/http/test_kv_store/config.cc rename to mobile/test/common/http/filters/test_kv_store/config.cc index 54bfd918d6d3..8135dac6c91f 100644 --- a/mobile/library/common/extensions/filters/http/test_kv_store/config.cc +++ b/mobile/test/common/http/filters/test_kv_store/config.cc @@ -1,6 +1,6 @@ -#include "library/common/extensions/filters/http/test_kv_store/config.h" +#include "test/common/http/filters/test_kv_store/config.h" -#include "library/common/extensions/filters/http/test_kv_store/filter.h" +#include "test/common/http/filters/test_kv_store/filter.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_kv_store/config.h b/mobile/test/common/http/filters/test_kv_store/config.h similarity index 80% rename from mobile/library/common/extensions/filters/http/test_kv_store/config.h rename to mobile/test/common/http/filters/test_kv_store/config.h index 67b84091ba6c..07b1c59e827b 100644 --- a/mobile/library/common/extensions/filters/http/test_kv_store/config.h +++ b/mobile/test/common/http/filters/test_kv_store/config.h @@ -4,9 +4,9 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "library/common/extensions/filters/http/test_kv_store/filter.h" -#include "library/common/extensions/filters/http/test_kv_store/filter.pb.h" -#include "library/common/extensions/filters/http/test_kv_store/filter.pb.validate.h" +#include "test/common/http/filters/test_kv_store/filter.h" +#include "test/common/http/filters/test_kv_store/filter.pb.h" +#include "test/common/http/filters/test_kv_store/filter.pb.validate.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_kv_store/filter.cc b/mobile/test/common/http/filters/test_kv_store/filter.cc similarity index 95% rename from mobile/library/common/extensions/filters/http/test_kv_store/filter.cc rename to mobile/test/common/http/filters/test_kv_store/filter.cc index d20a0ecf805f..07e0d5905ac3 100644 --- a/mobile/library/common/extensions/filters/http/test_kv_store/filter.cc +++ b/mobile/test/common/http/filters/test_kv_store/filter.cc @@ -1,4 +1,4 @@ -#include "library/common/extensions/filters/http/test_kv_store/filter.h" +#include "test/common/http/filters/test_kv_store/filter.h" #include "envoy/server/filter_config.h" diff --git a/mobile/library/common/extensions/filters/http/test_kv_store/filter.h b/mobile/test/common/http/filters/test_kv_store/filter.h similarity index 96% rename from mobile/library/common/extensions/filters/http/test_kv_store/filter.h rename to mobile/test/common/http/filters/test_kv_store/filter.h index 6dfc29bd6ea7..912f0f961c86 100644 --- a/mobile/library/common/extensions/filters/http/test_kv_store/filter.h +++ b/mobile/test/common/http/filters/test_kv_store/filter.h @@ -4,9 +4,10 @@ #include "source/extensions/filters/http/common/pass_through_filter.h" +#include "test/common/http/filters/test_kv_store/filter.pb.h" + #include "library/common/api/c_types.h" #include "library/common/api/external.h" -#include "library/common/extensions/filters/http/test_kv_store/filter.pb.h" #include "library/common/extensions/key_value/platform/c_types.h" namespace Envoy { diff --git a/mobile/library/common/extensions/filters/http/test_kv_store/filter.proto b/mobile/test/common/http/filters/test_kv_store/filter.proto similarity index 100% rename from mobile/library/common/extensions/filters/http/test_kv_store/filter.proto rename to mobile/test/common/http/filters/test_kv_store/filter.proto diff --git a/mobile/library/common/extensions/filters/http/test_logger/BUILD b/mobile/test/common/http/filters/test_logger/BUILD similarity index 86% rename from mobile/library/common/extensions/filters/http/test_logger/BUILD rename to mobile/test/common/http/filters/test_logger/BUILD index 2a31e88bba96..93203f0d3a31 100644 --- a/mobile/library/common/extensions/filters/http/test_logger/BUILD +++ b/mobile/test/common/http/filters/test_logger/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", + "envoy_cc_library", + "envoy_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_extension_package() +envoy_package() envoy_proto_library( name = "filter", @@ -17,7 +17,7 @@ envoy_proto_library( ], ) -envoy_cc_extension( +envoy_cc_library( name = "test_event_tracker_filter_lib", hdrs = ["filter.h"], repository = "@envoy", @@ -28,7 +28,7 @@ envoy_cc_extension( ], ) -envoy_cc_extension( +envoy_cc_library( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], diff --git a/mobile/library/common/extensions/filters/http/test_logger/config.cc b/mobile/test/common/http/filters/test_logger/config.cc similarity index 83% rename from mobile/library/common/extensions/filters/http/test_logger/config.cc rename to mobile/test/common/http/filters/test_logger/config.cc index 4e8c0086c110..e3f94b59eb71 100644 --- a/mobile/library/common/extensions/filters/http/test_logger/config.cc +++ b/mobile/test/common/http/filters/test_logger/config.cc @@ -1,6 +1,6 @@ -#include "library/common/extensions/filters/http/test_logger/config.h" +#include "test/common/http/filters/test_logger/config.h" -#include "library/common/extensions/filters/http/test_logger/filter.h" +#include "test/common/http/filters/test_logger/filter.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_logger/config.h b/mobile/test/common/http/filters/test_logger/config.h similarity index 78% rename from mobile/library/common/extensions/filters/http/test_logger/config.h rename to mobile/test/common/http/filters/test_logger/config.h index b77bccb7eb06..91fcb8decdb7 100644 --- a/mobile/library/common/extensions/filters/http/test_logger/config.h +++ b/mobile/test/common/http/filters/test_logger/config.h @@ -4,9 +4,9 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "library/common/extensions/filters/http/test_logger/filter.h" -#include "library/common/extensions/filters/http/test_logger/filter.pb.h" -#include "library/common/extensions/filters/http/test_logger/filter.pb.validate.h" +#include "test/common/http/filters/test_logger/filter.h" +#include "test/common/http/filters/test_logger/filter.pb.h" +#include "test/common/http/filters/test_logger/filter.pb.validate.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_logger/filter.h b/mobile/test/common/http/filters/test_logger/filter.h similarity index 88% rename from mobile/library/common/extensions/filters/http/test_logger/filter.h rename to mobile/test/common/http/filters/test_logger/filter.h index 28f1ece15281..3777407a0b76 100644 --- a/mobile/library/common/extensions/filters/http/test_logger/filter.h +++ b/mobile/test/common/http/filters/test_logger/filter.h @@ -4,7 +4,7 @@ #include "source/extensions/filters/http/common/pass_through_filter.h" -#include "library/common/extensions/filters/http/test_logger/filter.pb.h" +#include "test/common/http/filters/test_logger/filter.pb.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_logger/filter.proto b/mobile/test/common/http/filters/test_logger/filter.proto similarity index 100% rename from mobile/library/common/extensions/filters/http/test_logger/filter.proto rename to mobile/test/common/http/filters/test_logger/filter.proto diff --git a/mobile/library/common/extensions/filters/http/test_read/BUILD b/mobile/test/common/http/filters/test_read/BUILD similarity index 87% rename from mobile/library/common/extensions/filters/http/test_read/BUILD rename to mobile/test/common/http/filters/test_read/BUILD index 54324cca68ac..96743b875e87 100644 --- a/mobile/library/common/extensions/filters/http/test_read/BUILD +++ b/mobile/test/common/http/filters/test_read/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", + "envoy_cc_library", + "envoy_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_extension_package() +envoy_package() envoy_proto_library( name = "filter", @@ -17,7 +17,7 @@ envoy_proto_library( ], ) -envoy_cc_extension( +envoy_cc_library( name = "test_read_filter_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], @@ -30,7 +30,7 @@ envoy_cc_extension( ], ) -envoy_cc_extension( +envoy_cc_library( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], diff --git a/mobile/library/common/extensions/filters/http/test_read/config.cc b/mobile/test/common/http/filters/test_read/config.cc similarity index 83% rename from mobile/library/common/extensions/filters/http/test_read/config.cc rename to mobile/test/common/http/filters/test_read/config.cc index 3c5ce3c402f8..ce75a5f1a60c 100644 --- a/mobile/library/common/extensions/filters/http/test_read/config.cc +++ b/mobile/test/common/http/filters/test_read/config.cc @@ -1,6 +1,6 @@ -#include "library/common/extensions/filters/http/test_read/config.h" +#include "test/common/http/filters/test_read/config.h" -#include "library/common/extensions/filters/http/test_read/filter.h" +#include "test/common/http/filters/test_read/filter.h" namespace Envoy { namespace HttpFilters { diff --git a/mobile/library/common/extensions/filters/http/test_read/config.h b/mobile/test/common/http/filters/test_read/config.h similarity index 85% rename from mobile/library/common/extensions/filters/http/test_read/config.h rename to mobile/test/common/http/filters/test_read/config.h index 24e22d706cb1..291c63211ee7 100644 --- a/mobile/library/common/extensions/filters/http/test_read/config.h +++ b/mobile/test/common/http/filters/test_read/config.h @@ -4,8 +4,8 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "library/common/extensions/filters/http/test_read/filter.pb.h" -#include "library/common/extensions/filters/http/test_read/filter.pb.validate.h" +#include "test/common/http/filters/test_read/filter.pb.h" +#include "test/common/http/filters/test_read/filter.pb.validate.h" namespace Envoy { namespace HttpFilters { diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.cc b/mobile/test/common/http/filters/test_read/filter.cc similarity index 96% rename from mobile/library/common/extensions/filters/http/test_read/filter.cc rename to mobile/test/common/http/filters/test_read/filter.cc index fb7c42752727..50b20cdf3a06 100644 --- a/mobile/library/common/extensions/filters/http/test_read/filter.cc +++ b/mobile/test/common/http/filters/test_read/filter.cc @@ -1,4 +1,4 @@ -#include "library/common/extensions/filters/http/test_read/filter.h" +#include "test/common/http/filters/test_read/filter.h" #include "envoy/server/filter_config.h" diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.h b/mobile/test/common/http/filters/test_read/filter.h similarity index 93% rename from mobile/library/common/extensions/filters/http/test_read/filter.h rename to mobile/test/common/http/filters/test_read/filter.h index dc5c2e73cea2..c59fdc4203ec 100644 --- a/mobile/library/common/extensions/filters/http/test_read/filter.h +++ b/mobile/test/common/http/filters/test_read/filter.h @@ -7,7 +7,7 @@ #include "source/common/stream_info/stream_info_impl.h" #include "source/extensions/filters/http/common/pass_through_filter.h" -#include "library/common/extensions/filters/http/test_read/filter.pb.h" +#include "test/common/http/filters/test_read/filter.pb.h" namespace Envoy { namespace HttpFilters { diff --git a/mobile/library/common/extensions/filters/http/test_read/filter.proto b/mobile/test/common/http/filters/test_read/filter.proto similarity index 100% rename from mobile/library/common/extensions/filters/http/test_read/filter.proto rename to mobile/test/common/http/filters/test_read/filter.proto diff --git a/mobile/library/common/extensions/filters/http/test_remote_response/BUILD b/mobile/test/common/http/filters/test_remote_response/BUILD similarity index 90% rename from mobile/library/common/extensions/filters/http/test_remote_response/BUILD rename to mobile/test/common/http/filters/test_remote_response/BUILD index 64ac60a4b6be..94c9f6df790b 100644 --- a/mobile/library/common/extensions/filters/http/test_remote_response/BUILD +++ b/mobile/test/common/http/filters/test_remote_response/BUILD @@ -1,20 +1,20 @@ load( "@envoy//bazel:envoy_build_system.bzl", - "envoy_cc_extension", - "envoy_extension_package", + "envoy_cc_library", + "envoy_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_extension_package() +envoy_package() envoy_proto_library( name = "filter", srcs = ["filter.proto"], ) -envoy_cc_extension( +envoy_cc_library( name = "test_remote_response_filter_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], @@ -36,7 +36,7 @@ envoy_cc_extension( ], ) -envoy_cc_extension( +envoy_cc_library( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], diff --git a/mobile/library/common/extensions/filters/http/test_remote_response/config.cc b/mobile/test/common/http/filters/test_remote_response/config.cc similarity index 83% rename from mobile/library/common/extensions/filters/http/test_remote_response/config.cc rename to mobile/test/common/http/filters/test_remote_response/config.cc index abdcd38f18ef..384611e10fce 100644 --- a/mobile/library/common/extensions/filters/http/test_remote_response/config.cc +++ b/mobile/test/common/http/filters/test_remote_response/config.cc @@ -1,6 +1,6 @@ -#include "library/common/extensions/filters/http/test_remote_response/config.h" +#include "test/common/http/filters/test_remote_response/config.h" -#include "library/common/extensions/filters/http/test_remote_response/filter.h" +#include "test/common/http/filters/test_remote_response/filter.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_remote_response/config.h b/mobile/test/common/http/filters/test_remote_response/config.h similarity index 84% rename from mobile/library/common/extensions/filters/http/test_remote_response/config.h rename to mobile/test/common/http/filters/test_remote_response/config.h index 11a8c60ad84a..09642e21a59d 100644 --- a/mobile/library/common/extensions/filters/http/test_remote_response/config.h +++ b/mobile/test/common/http/filters/test_remote_response/config.h @@ -2,8 +2,8 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "library/common/extensions/filters/http/test_remote_response/filter.pb.h" -#include "library/common/extensions/filters/http/test_remote_response/filter.pb.validate.h" +#include "test/common/http/filters/test_remote_response/filter.pb.h" +#include "test/common/http/filters/test_remote_response/filter.pb.validate.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_remote_response/filter.cc b/mobile/test/common/http/filters/test_remote_response/filter.cc similarity index 96% rename from mobile/library/common/extensions/filters/http/test_remote_response/filter.cc rename to mobile/test/common/http/filters/test_remote_response/filter.cc index c350853ac497..7b1eb4235797 100644 --- a/mobile/library/common/extensions/filters/http/test_remote_response/filter.cc +++ b/mobile/test/common/http/filters/test_remote_response/filter.cc @@ -1,4 +1,4 @@ -#include "library/common/extensions/filters/http/test_remote_response/filter.h" +#include "test/common/http/filters/test_remote_response/filter.h" #include "envoy/http/header_map.h" #include "envoy/server/filter_config.h" diff --git a/mobile/library/common/extensions/filters/http/test_remote_response/filter.h b/mobile/test/common/http/filters/test_remote_response/filter.h similarity index 92% rename from mobile/library/common/extensions/filters/http/test_remote_response/filter.h rename to mobile/test/common/http/filters/test_remote_response/filter.h index 8e982982dd10..1ac0bf1a1ca3 100644 --- a/mobile/library/common/extensions/filters/http/test_remote_response/filter.h +++ b/mobile/test/common/http/filters/test_remote_response/filter.h @@ -5,7 +5,7 @@ #include "source/common/common/logger.h" #include "source/extensions/filters/http/common/pass_through_filter.h" -#include "library/common/extensions/filters/http/test_remote_response/filter.pb.h" +#include "test/common/http/filters/test_remote_response/filter.pb.h" namespace Envoy { namespace Extensions { diff --git a/mobile/library/common/extensions/filters/http/test_remote_response/filter.proto b/mobile/test/common/http/filters/test_remote_response/filter.proto similarity index 100% rename from mobile/library/common/extensions/filters/http/test_remote_response/filter.proto rename to mobile/test/common/http/filters/test_remote_response/filter.proto diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index 8484e89752ac..18297edba85e 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -551,7 +551,7 @@ def check_source_line(self, line, file_path, report_error): report_error( "Don't use ambiguous duration(value), use an explicit duration type, e.g. Event::TimeSystem::Milliseconds(value)" ) - if not self.allow_listed_for_register_factory(file_path): + if file_path.startswith("mobile") and not self.allow_listed_for_register_factory(file_path): if "Registry::RegisterFactory<" in line or "REGISTER_FACTORY" in line: report_error( "Don't use Registry::RegisterFactory or REGISTER_FACTORY in tests, " From 420b1e911bef4ea2355a6c0c43830d7caf5b3ec4 Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Tue, 30 May 2023 13:43:02 -0400 Subject: [PATCH 412/740] tests: Eliminate retry backoffs in the SDS integration tests (#27679) When the xDS cluster is specified after the cluster that depends on it (e.g. for SDS) in the Bootstrap config, the cluster isn't available when making the xDS request, causing a backoff retry after [0,1s). This adds an delays to the tests. This commit fixes the issue by specifying the xDS cluster first in order, before the static cluster that depends on it, in the Bootstrap config. There are about 120 tests in this file, so we save on average about 60 seconds of test runtime everytime the tests in this file are executed. Signed-off-by: Ali Beyad --- .../sds_dynamic_integration_test.cc | 198 +++++++++++------- 1 file changed, 120 insertions(+), 78 deletions(-) diff --git a/test/integration/sds_dynamic_integration_test.cc b/test/integration/sds_dynamic_integration_test.cc index 8b080d7c0064..92929dbf74bf 100644 --- a/test/integration/sds_dynamic_integration_test.cc +++ b/test/integration/sds_dynamic_integration_test.cc @@ -111,6 +111,9 @@ class SdsDynamicIntegrationBaseTest : public Grpc::BaseGrpcClientIntegrationPara Network::Address::IpVersion ipVersion() const override { return GetParam().ip_version; } Grpc::ClientType clientType() const override { return GetParam().sds_grpc_type; } + virtual std::unique_ptr& sdsUpstream() { return fake_upstreams_[0]; } + // Index in fake_upstreams_ vector of the data plane upstream. + int dataPlaneUpstreamIndex() { return 1; } protected: void createSdsStream(FakeUpstream&) { @@ -130,7 +133,7 @@ class SdsDynamicIntegrationBaseTest : public Grpc::BaseGrpcClientIntegrationPara api_config_source->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC); api_config_source->set_transport_api_version(envoy::config::core::v3::V3); auto* grpc_service = api_config_source->add_grpc_services(); - setGrpcService(*grpc_service, "sds_cluster.lyft.com", fake_upstreams_.back()->localAddress()); + setGrpcService(*grpc_service, "sds_cluster.lyft.com", sdsUpstream()->localAddress()); } envoy::extensions::transport_sockets::tls::v3::Secret getServerSecretRsa() { @@ -230,10 +233,14 @@ class SdsDynamicDownstreamIntegrationTest : public SdsDynamicIntegrationBaseTest }); config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - // Add a static sds cluster - auto* sds_cluster = bootstrap.mutable_static_resources()->add_clusters(); - sds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + // Add a static SDS cluster as the first cluster in the list. + // The SDS cluster needs to appear before the cluster that uses it for secrets, so that it + // gets initialized first. + bootstrap.mutable_static_resources()->add_clusters()->MergeFrom( + bootstrap.static_resources().clusters(0)); + auto* sds_cluster = bootstrap.mutable_static_resources()->mutable_clusters(0); sds_cluster->set_name("sds_cluster.lyft.com"); + sds_cluster->mutable_load_assignment()->set_cluster_name("sds_cluster.lyft.com"); ConfigHelper::setHttp2(*sds_cluster); }); @@ -321,8 +328,10 @@ version_info: "0" } void createUpstreams() override { - create_xds_upstream_ = true; - HttpIntegrationTest::createUpstreams(); + // SDS cluster is H2, while the data cluster is H1. + addFakeUpstream(Http::CodecType::HTTP2); + addFakeUpstream(Http::CodecType::HTTP1); + xds_upstream_ = fake_upstreams_.front().get(); } void waitForSdsUpdateStats(size_t times) { @@ -386,7 +395,7 @@ TEST_P(SdsDynamicKeyRotationIntegrationTest, BasicRotation) { {TestEnvironment::runfilesPath("test/integration/sds_dynamic_key_rotation_setup.sh")}); on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getCurrentServerSecret()); }; initialize(); @@ -398,7 +407,7 @@ TEST_P(SdsDynamicKeyRotationIntegrationTest, BasicRotation) { return makeSslClientConnection(); }; // First request with server{cert,key}.pem. - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); cleanupUpstreamAndDownstream(); // Rotate. TestEnvironment::renameFile(TestEnvironment::temporaryPath("root/new"), @@ -409,7 +418,7 @@ TEST_P(SdsDynamicKeyRotationIntegrationTest, BasicRotation) { EXPECT_EQ(0, test_server_->counter("sds.server_cert_rsa.update_rejected")->value()); // First request with server_ecdsa{cert,key}.pem. - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); } // Validate that rotating to a directory with missing certs is handled. @@ -419,7 +428,7 @@ TEST_P(SdsDynamicKeyRotationIntegrationTest, EmptyRotation) { {TestEnvironment::runfilesPath("test/integration/sds_dynamic_key_rotation_setup.sh")}); on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getCurrentServerSecret()); }; initialize(); @@ -431,7 +440,7 @@ TEST_P(SdsDynamicKeyRotationIntegrationTest, EmptyRotation) { return makeSslClientConnection(); }; // First request with server{cert,key}.pem. - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); cleanupUpstreamAndDownstream(); // Rotate to an empty directory, this should fail. @@ -444,14 +453,14 @@ TEST_P(SdsDynamicKeyRotationIntegrationTest, EmptyRotation) { EXPECT_EQ(0, test_server_->counter("sds.server_cert_rsa.update_rejected")->value()); // Requests continue to work with key/cert pair. - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); } // A test that SDS server send a good server secret for a static listener. // The first ssl request should be OK. TEST_P(SdsDynamicDownstreamIntegrationTest, BasicSuccess) { on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getServerSecretRsa()); }; initialize(); @@ -459,7 +468,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, BasicSuccess) { ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection(); }; - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); // Success EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_success")->value()); @@ -468,7 +477,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, BasicSuccess) { TEST_P(SdsDynamicDownstreamIntegrationTest, DualCert) { on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getServerSecretRsa()); }; @@ -484,7 +493,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, DualCert) { .setTlsVersion(envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLSv1_2) .setCipherSuites({"ECDHE-ECDSA-AES128-GCM-SHA256"}), context_manager_, *api_); - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); cleanupUpstreamAndDownstream(); client_ssl_ctx_ = createClientSslTransportSocketFactory( @@ -492,7 +501,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, DualCert) { .setTlsVersion(envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLSv1_2) .setCipherSuites({"ECDHE-RSA-AES128-GCM-SHA256"}), context_manager_, *api_); - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); // Success EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_success")->value()); @@ -520,7 +529,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, DualCert) { // via sds and give a simple case of selecting cert based on SNI. TEST_P(SdsDynamicDownstreamIntegrationTest, MultipleCerts) { on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getServerSecretRsa()); }; @@ -576,7 +585,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, MultipleCerts) { // then SDS send a good server secret, the second request should be OK. TEST_P(SdsDynamicDownstreamIntegrationTest, WrongSecretFirst) { on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getWrongSecret(server_cert_rsa_)); }; initialize(); @@ -598,7 +607,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, WrongSecretFirst) { ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection(); }; - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); // Success EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_success")->value()); @@ -621,10 +630,14 @@ class SdsDynamicDownstreamCertValidationContextTest : public SdsDynamicDownstrea configureInlinedCerts(&common_tls_context); }); - // Add a static sds cluster - auto* sds_cluster = bootstrap.mutable_static_resources()->add_clusters(); - sds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + // Add a static SDS cluster as the first cluster in the list. + // The SDS cluster needs to appear before the cluster that uses it for secrets, so that it + // gets initialized first. + bootstrap.mutable_static_resources()->add_clusters()->MergeFrom( + bootstrap.static_resources().clusters(0)); + auto* sds_cluster = bootstrap.mutable_static_resources()->mutable_clusters(0); sds_cluster->set_name("sds_cluster.lyft.com"); + sds_cluster->mutable_load_assignment()->set_cluster_name("sds_cluster.lyft.com"); ConfigHelper::setHttp2(*sds_cluster); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext upstream_tls_context; @@ -641,8 +654,9 @@ class SdsDynamicDownstreamCertValidationContextTest : public SdsDynamicDownstrea TestEnvironment::runfilesPath("test/config/integration/certs/clientcert.pem")); upstream_tls_certificate->mutable_private_key()->set_filename( TestEnvironment::runfilesPath("test/config/integration/certs/clientkey.pem")); - auto* upstream_transport_socket = - bootstrap.mutable_static_resources()->mutable_clusters(0)->mutable_transport_socket(); + auto* upstream_transport_socket = bootstrap.mutable_static_resources() + ->mutable_clusters(dataPlaneUpstreamIndex()) + ->mutable_transport_socket(); upstream_transport_socket->set_name("envoy.transport_sockets.tls"); upstream_transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); }); @@ -681,9 +695,11 @@ class SdsDynamicDownstreamCertValidationContextTest : public SdsDynamicDownstrea } void createUpstreams() override { - // Fake upstream with SSL/TLS for the first cluster. + // SDS cluster for the first cluster. + addFakeUpstream(Http::CodecType::HTTP2); + // Fake upstream with SSL/TLS for the second cluster. addFakeUpstream(createUpstreamSslContext(), upstreamProtocol(), /*autonomous_upstream=*/false); - create_xds_upstream_ = true; + xds_upstream_ = fake_upstreams_.front().get(); } Network::DownstreamTransportSocketFactoryPtr createUpstreamSslContext() { @@ -726,7 +742,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, SdsDynamicDownstreamCertValidatio // The first ssl request should be OK. TEST_P(SdsDynamicDownstreamCertValidationContextTest, BasicSuccess) { on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getCvcSecret()); }; initialize(); @@ -734,7 +750,7 @@ TEST_P(SdsDynamicDownstreamCertValidationContextTest, BasicSuccess) { ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection(); }; - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); // Success EXPECT_EQ(1, test_server_->counter("sds.validation_secret.update_success")->value()); @@ -747,7 +763,7 @@ TEST_P(SdsDynamicDownstreamCertValidationContextTest, BasicSuccess) { TEST_P(SdsDynamicDownstreamCertValidationContextTest, CombinedCertValidationContextSuccess) { enableCombinedValidationContext(true); on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getCvcSecretWithOnlyTrustedCa()); }; initialize(); @@ -755,7 +771,7 @@ TEST_P(SdsDynamicDownstreamCertValidationContextTest, CombinedCertValidationCont ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection(); }; - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); // Success EXPECT_EQ(1, test_server_->counter("sds.validation_secret.update_success")->value()); @@ -767,7 +783,7 @@ TEST_P(SdsDynamicDownstreamCertValidationContextTest, CombinedCertValidationCont TEST_P(SdsDynamicDownstreamCertValidationContextTest, BasicWithSharedSecret) { shareValidationSecret(true); on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getCvcSecret()); }; initialize(); @@ -781,7 +797,7 @@ TEST_P(SdsDynamicDownstreamCertValidationContextTest, BasicWithSharedSecret) { ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection(); }; - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); // Success EXPECT_EQ(1, test_server_->counter("sds.validation_secret.update_success")->value()); @@ -794,7 +810,7 @@ TEST_P(SdsDynamicDownstreamCertValidationContextTest, CombinedValidationContextW enableCombinedValidationContext(true); shareValidationSecret(true); on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getCvcSecretWithOnlyTrustedCa()); }; initialize(); @@ -808,7 +824,7 @@ TEST_P(SdsDynamicDownstreamCertValidationContextTest, CombinedValidationContextW ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection(); }; - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); // Success EXPECT_EQ(1, test_server_->counter("sds.validation_secret.update_success")->value()); @@ -826,20 +842,25 @@ class SdsDynamicUpstreamIntegrationTest : public SdsDynamicIntegrationBaseTest { setUpstreamProtocol(Http::CodecType::HTTP3); } config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - // add sds cluster first. - auto* sds_cluster = bootstrap.mutable_static_resources()->add_clusters(); - sds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + // Add a static SDS cluster as the first cluster in the list. + // The SDS cluster needs to appear before the cluster that uses it for secrets, so that it + // gets initialized first. + bootstrap.mutable_static_resources()->add_clusters()->MergeFrom( + bootstrap.static_resources().clusters(0)); + auto* sds_cluster = bootstrap.mutable_static_resources()->mutable_clusters(0); sds_cluster->set_name("sds_cluster.lyft.com"); + sds_cluster->mutable_load_assignment()->set_cluster_name("sds_cluster.lyft.com"); ConfigHelper::setHttp2(*sds_cluster); - // Unwind Quic for sds cluster. + // Unwind Quic for SDS cluster. if (test_quic_) { sds_cluster->clear_transport_socket(); } // change the first cluster with ssl and sds. - auto* transport_socket = - bootstrap.mutable_static_resources()->mutable_clusters(0)->mutable_transport_socket(); + auto* transport_socket = bootstrap.mutable_static_resources() + ->mutable_clusters(dataPlaneUpstreamIndex()) + ->mutable_transport_socket(); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; tls_context.set_sni("lyft.com"); auto* secret_config = @@ -872,10 +893,12 @@ class SdsDynamicUpstreamIntegrationTest : public SdsDynamicIntegrationBaseTest { } void createUpstreams() override { - // This is for backend with ssl + // SDS cluster for the first cluster. + addFakeUpstream(Http::CodecType::HTTP2); + // FakeUpstream with SSL/TLS for the second cluster. addFakeUpstream(createUpstreamSslContext(context_manager_, *api_, test_quic_), upstreamProtocol(), /*autonomous_upstream=*/false); - create_xds_upstream_ = true; + xds_upstream_ = fake_upstreams_.front().get(); } }; @@ -886,7 +909,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, SdsDynamicUpstreamIntegrationTest, // The first request should work. TEST_P(SdsDynamicUpstreamIntegrationTest, BasicSuccess) { on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getClientSecret()); }; @@ -902,19 +925,19 @@ TEST_P(SdsDynamicUpstreamIntegrationTest, BasicSuccess) { test_server_->waitForCounterGe( "cluster.cluster_0.client_ssl_socket_factory.ssl_context_update_by_sds", 1); - testRouterHeaderOnlyRequestAndResponse(); + testRouterHeaderOnlyRequestAndResponse(/*create_connection=*/nullptr, dataPlaneUpstreamIndex()); // Success EXPECT_EQ(1, test_server_->counter("sds.client_cert.update_success")->value()); EXPECT_EQ(0, test_server_->counter("sds.client_cert.update_rejected")->value()); } -// To test a static cluster with sds. SDS send a bad client secret first. -// The first request should fail with 503, then SDS sends a good client secret, -// the second request should work. +// Tests a static cluster with SDS. +// SDS sends a bad client secret first, so the request fails with a 503. +// Then, SDS sends a good client secret, so the second request is successful. TEST_P(SdsDynamicUpstreamIntegrationTest, WrongSecretFirst) { on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getWrongSecret(client_cert_)); }; initialize(); @@ -928,7 +951,8 @@ TEST_P(SdsDynamicUpstreamIntegrationTest, WrongSecretFirst) { // Wait for the raw TCP connection with bad credentials and close it. if (upstreamProtocol() != Http::CodecType::HTTP3) { FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE( + fake_upstreams_[dataPlaneUpstreamIndex()]->waitForRawConnection(fake_upstream_connection)); ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); } @@ -943,7 +967,7 @@ TEST_P(SdsDynamicUpstreamIntegrationTest, WrongSecretFirst) { EXPECT_EQ(1, test_server_->counter("sds.client_cert.update_rejected")->value()); // Verify the update succeeded. - testRouterHeaderOnlyRequestAndResponse(); + testRouterHeaderOnlyRequestAndResponse(/*create_connection=*/nullptr, dataPlaneUpstreamIndex()); } // Test CDS with SDS. A cluster provided by CDS raises new SDS request for upstream cert. @@ -952,8 +976,34 @@ class SdsCdsIntegrationTest : public SdsDynamicIntegrationBaseTest { public: void initialize() override { config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - // Create the dynamic cluster. This cluster will be using sds. - dynamic_cluster_ = bootstrap.mutable_static_resources()->clusters(0); + // The 1st cluster, which already exists in the Bootstrap config, will be the CDS cluster. + // The 2nd cluster, which will be added, will be the SDS cluster. + // The 3rd cluster, which will be added after that, will be the dynamic cluster, which will + // use SDS. + // The xDS clusters need to appear before the cluster that uses it, so that they get + // initialized first. + + // The SDS cluster. + auto* sds_cluster = bootstrap.mutable_static_resources()->add_clusters(); + sds_cluster->MergeFrom(bootstrap.static_resources().clusters(0)); + // The static cluster. + auto* static_cluster = bootstrap.mutable_static_resources()->add_clusters(); + static_cluster->MergeFrom(bootstrap.static_resources().clusters(0)); + // Make a copy of the static cluster to create and set up the dynamic cluster. + dynamic_cluster_ = *static_cluster; + + // Set up the first cluster, the CDS cluster. + auto* cds_cluster = bootstrap.mutable_static_resources()->mutable_clusters(0); + cds_cluster->set_name("cds_cluster"); + cds_cluster->mutable_load_assignment()->set_cluster_name("cds_cluster"); + ConfigHelper::setHttp2(*cds_cluster); + + // Set up the second cluster, the SDS cluster. + sds_cluster->set_name("sds_cluster.lyft.com"); + sds_cluster->mutable_load_assignment()->set_cluster_name("sds_cluster.lyft.com"); + ConfigHelper::setHttp2(*sds_cluster); + + // Set up the dynamic cluster to use SDS. dynamic_cluster_.set_name("dynamic"); dynamic_cluster_.mutable_connect_timeout()->MergeFrom( ProtobufUtil::TimeUtil::MillisecondsToDuration(500000)); @@ -966,17 +1016,7 @@ class SdsCdsIntegrationTest : public SdsDynamicIntegrationBaseTest { transport_socket->set_name("envoy.transport_sockets.tls"); transport_socket->mutable_typed_config()->PackFrom(tls_context); - // Add cds cluster first. - auto* cds_cluster = bootstrap.mutable_static_resources()->add_clusters(); - cds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); - cds_cluster->set_name("cds_cluster"); - ConfigHelper::setHttp2(*cds_cluster); - // Then add sds cluster. - auto* sds_cluster = bootstrap.mutable_static_resources()->add_clusters(); - sds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); - sds_cluster->set_name("sds_cluster.lyft.com"); - ConfigHelper::setHttp2(*sds_cluster); - + // Set up the Bootstrap's CDS config to fetch from the CDS cluster. const std::string cds_yaml = R"EOF( resource_api_version: V3 api_config_source: @@ -1010,14 +1050,18 @@ class SdsCdsIntegrationTest : public SdsDynamicIntegrationBaseTest { } void createUpstreams() override { - // Static cluster. - addFakeUpstream(Http::CodecType::HTTP1); - // Cds Cluster. + // CDS Cluster. addFakeUpstream(Http::CodecType::HTTP2); - // Sds Cluster. + // SDS Cluster. addFakeUpstream(Http::CodecType::HTTP2); + // Static cluster. + addFakeUpstream(Http::CodecType::HTTP1); + xds_upstream_ = fake_upstreams_.front().get(); } + std::unique_ptr& cdsUpstream() { return fake_upstreams_[0]; } + std::unique_ptr& sdsUpstream() override { return fake_upstreams_[1]; } + void sendCdsResponse() { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse( @@ -1032,6 +1076,7 @@ class SdsCdsIntegrationTest : public SdsDynamicIntegrationBaseTest { discovery_response.add_resources()->PackFrom(secret); sds_stream.sendGrpcMessage(discovery_response); } + envoy::config::cluster::v3::Cluster dynamic_cluster_; FakeHttpConnectionPtr sds_connection_; FakeStreamPtr sds_stream_; @@ -1044,8 +1089,7 @@ TEST_P(SdsCdsIntegrationTest, BasicSuccess) { on_server_init_function_ = [this]() { { // CDS. - AssertionResult result = - fake_upstreams_[1]->waitForHttpConnection(*dispatcher_, xds_connection_); + AssertionResult result = cdsUpstream()->waitForHttpConnection(*dispatcher_, xds_connection_); RELEASE_ASSERT(result, result.message()); result = xds_connection_->waitForNewStream(*dispatcher_, xds_stream_); RELEASE_ASSERT(result, result.message()); @@ -1054,8 +1098,7 @@ TEST_P(SdsCdsIntegrationTest, BasicSuccess) { } { // SDS. - AssertionResult result = - fake_upstreams_[2]->waitForHttpConnection(*dispatcher_, sds_connection_); + AssertionResult result = sdsUpstream()->waitForHttpConnection(*dispatcher_, sds_connection_); RELEASE_ASSERT(result, result.message()); result = sds_connection_->waitForNewStream(*dispatcher_, sds_stream_); @@ -1122,7 +1165,7 @@ TEST_P(SdsDynamicDownstreamPrivateKeyIntegrationTest, BasicPrivateKeyProvider) { test_private_key_method_factory(test_factory); on_server_init_function_ = [this]() { - createSdsStream(*(fake_upstreams_[1])); + createSdsStream(*sdsUpstream()); sendSdsResponse(getCurrentServerPrivateKeyProviderSecret()); }; initialize(); @@ -1133,7 +1176,7 @@ TEST_P(SdsDynamicDownstreamPrivateKeyIntegrationTest, BasicPrivateKeyProvider) { ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection(); }; - testRouterHeaderOnlyRequestAndResponse(&creator); + testRouterHeaderOnlyRequestAndResponse(&creator, dataPlaneUpstreamIndex()); cleanupUpstreamAndDownstream(); } @@ -1184,8 +1227,7 @@ TEST_P(SdsCdsPrivateKeyIntegrationTest, BasicSdsCdsPrivateKeyProvider) { on_server_init_function_ = [this]() { { // CDS. - AssertionResult result = - fake_upstreams_[1]->waitForHttpConnection(*dispatcher_, xds_connection_); + AssertionResult result = cdsUpstream()->waitForHttpConnection(*dispatcher_, xds_connection_); EXPECT_TRUE(result); result = xds_connection_->waitForNewStream(*dispatcher_, xds_stream_); EXPECT_TRUE(result); @@ -1194,8 +1236,7 @@ TEST_P(SdsCdsPrivateKeyIntegrationTest, BasicSdsCdsPrivateKeyProvider) { } { // SDS. - AssertionResult result = - fake_upstreams_[2]->waitForHttpConnection(*dispatcher_, sds_connection_); + AssertionResult result = sdsUpstream()->waitForHttpConnection(*dispatcher_, sds_connection_); EXPECT_TRUE(result); result = sds_connection_->waitForNewStream(*dispatcher_, sds_stream_); @@ -1216,5 +1257,6 @@ TEST_P(SdsCdsPrivateKeyIntegrationTest, BasicSdsCdsPrivateKeyProvider) { // Successfully removed the dynamic cluster. test_server_->waitForGaugeEq("cluster_manager.active_clusters", 3); } + } // namespace Ssl } // namespace Envoy From b5acd8404aab2e495b7a6dcff9e7e0f0e7a0b440 Mon Sep 17 00:00:00 2001 From: danzh Date: Tue, 30 May 2023 13:58:59 -0400 Subject: [PATCH 413/740] dfp: request failed when cached in dns cache (#27641) In light of #27407, the DNS matched matches against host:port. But an entry is found in DNS cache, DFP filter call getHost() again on the cache with raw host. That call failed the lookup and causes dns_resolution_failure. Additional Description: remove mobile/test/kotlin/integration/MixedSchemeTest.kt which doesn't work under CI; And make Hello Envoy app on android runs with Android certification validation APIs. Risk Level: low Testing: add new test Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A Further Fixes #27331 Signed-off-by: Dan Zhang Co-authored-by: Alyssa Wilk --- .../java/hello_world/MainActivity.java | 22 ++-- .../kotlin/integration/MixedSchemeTest.kt | 101 ------------------ .../dynamic_forward_proxy/proxy_filter.cc | 6 +- .../proxy_filter_integration_test.cc | 30 +++++- .../proxy_filter_test.cc | 21 ---- 5 files changed, 47 insertions(+), 133 deletions(-) delete mode 100644 mobile/test/kotlin/integration/MixedSchemeTest.kt diff --git a/mobile/examples/java/hello_world/MainActivity.java b/mobile/examples/java/hello_world/MainActivity.java index 7f5d81f207d8..920508a04de9 100644 --- a/mobile/examples/java/hello_world/MainActivity.java +++ b/mobile/examples/java/hello_world/MainActivity.java @@ -35,7 +35,8 @@ public class MainActivity extends Activity { private static final String ENVOY_SERVER_HEADER = "server"; private static final String REQUEST_AUTHORITY = "api.lyft.com"; private static final String REQUEST_PATH = "/ping"; - private static final String REQUEST_SCHEME = "http"; + private static final String REQUEST_SCHEME_HTTP = "http"; + private static final String REQUEST_SCHEME_HTTPS = "https"; private static final Set FILTERED_HEADERS = new HashSet() { { add("server"); @@ -49,6 +50,7 @@ public class MainActivity extends Activity { private HandlerThread thread = new HandlerThread(REQUEST_HANDLER_THREAD_NAME); private ResponseRecyclerViewAdapter viewAdapter; + private Boolean clear_text = true; @Override protected void onCreate(Bundle savedInstanceState) { @@ -61,6 +63,7 @@ protected void onCreate(Bundle savedInstanceState) { Log.d("MainActivity", "Envoy async internal setup completed"); return null; }) + .enablePlatformCertificatesValidation(true) .build(); recyclerView = (RecyclerView)findViewById(R.id.recycler_view); @@ -95,14 +98,15 @@ private void makeRequest() { // Note: this request will use an http/1.1 stream for the upstream request. // The Kotlin example uses h2. This is done on purpose to test both paths in // end-to-end tests in CI. - RequestHeaders requestHeaders = new RequestHeadersBuilder(RequestMethod.GET, REQUEST_SCHEME, - REQUEST_AUTHORITY, REQUEST_PATH) - .build(); + String scheme = clear_text ? REQUEST_SCHEME_HTTP : REQUEST_SCHEME_HTTPS; + RequestHeaders requestHeaders = + new RequestHeadersBuilder(RequestMethod.GET, scheme, REQUEST_AUTHORITY, REQUEST_PATH) + .build(); engine.streamClient() .newStreamPrototype() .setOnResponseHeaders((responseHeaders, endStream, ignored) -> { Integer status = responseHeaders.getHttpStatus(); - String message = "received headers with status " + status; + String message = "received headers with status " + status + " for " + scheme + " request"; StringBuilder sb = new StringBuilder(); for (Map.Entry> entry : @@ -115,7 +119,9 @@ private void makeRequest() { String headerText = sb.toString(); Log.d("MainActivity", message); - if (status == 301) { + if ((scheme == REQUEST_SCHEME_HTTP && status == 301) || + (scheme == REQUEST_SCHEME_HTTPS && status == 200)) { + // The server returns 301 to http request and 200 to https request. String serverHeaderField = responseHeaders.value(ENVOY_SERVER_HEADER).get(0); recyclerView.post(() -> viewAdapter.add(new Success(message, headerText))); } else { @@ -125,13 +131,15 @@ private void makeRequest() { }) .setOnError((error, ignored) -> { String message = "failed with error after " + error.getAttemptCount() + - " attempts: " + error.getMessage(); + " attempts: " + error.getMessage() + " for " + scheme + " request"; Log.d("MainActivity", message); recyclerView.post(() -> viewAdapter.add(new Failure(message))); return Unit.INSTANCE; }) .start(Executors.newSingleThreadExecutor()) .sendHeaders(requestHeaders, true); + + clear_text = !clear_text; } private void recordStats() { diff --git a/mobile/test/kotlin/integration/MixedSchemeTest.kt b/mobile/test/kotlin/integration/MixedSchemeTest.kt deleted file mode 100644 index ec054681e1c4..000000000000 --- a/mobile/test/kotlin/integration/MixedSchemeTest.kt +++ /dev/null @@ -1,101 +0,0 @@ -package test.kotlin.integration - -import io.envoyproxy.envoymobile.LogLevel -import io.envoyproxy.envoymobile.Standard -import io.envoyproxy.envoymobile.EngineBuilder -import io.envoyproxy.envoymobile.RequestHeadersBuilder -import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.engine.JniLibrary -import java.nio.ByteBuffer -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.fail -import org.junit.Test - -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" - -class ReceiveDataTest { - - init { - JniLibrary.loadTestLibrary() - } - - @Test - fun `response headers and response data call onResponseHeaders and onResponseData`() { - - val engine = EngineBuilder(Standard()) - .addLogLevel(LogLevel.TRACE) - .build() - val client = engine.streamClient() - - // httpi - val headersExpectation2 = CountDownLatch(1) - val dataExpectation2 = CountDownLatch(1) - - var status2: Int? = null - - val requestHeaders2 = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "http", - authority = "example.com", - path = "/" - ) - .build() - - client.newStreamPrototype() - .setOnResponseHeaders { responseHeaders, _, _ -> - status2 = responseHeaders.httpStatus - headersExpectation2.countDown() - } - .setOnResponseData { data, _, _ -> - dataExpectation2.countDown() - } - .setOnError { _, _ -> - fail("Unexpected error") - } - .start() - .sendHeaders(requestHeaders2, true) - - headersExpectation2.await(10, TimeUnit.SECONDS) - dataExpectation2.await(10, TimeUnit.SECONDS) - - assertThat(headersExpectation2.count).isEqualTo(0) - assertThat(dataExpectation2.count).isEqualTo(0) - - // https - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/" - ) - .build() - - val headersExpectation = CountDownLatch(1) - val dataExpectation = CountDownLatch(1) - - var status: Int? = null - client.newStreamPrototype() - .setOnResponseHeaders { responseHeaders, _, _ -> - status = responseHeaders.httpStatus - headersExpectation.countDown() - } - .setOnResponseData { data, _, _ -> - dataExpectation.countDown() - } - .setOnError { _, _ -> fail("Unexpected error") } - .start() - .sendHeaders(requestHeaders, true) - - headersExpectation.await(10, TimeUnit.SECONDS) - dataExpectation.await(10, TimeUnit.SECONDS) - engine.terminate() - - assertThat(headersExpectation.count).isEqualTo(0) - assertThat(dataExpectation.count).isEqualTo(0) - - assertThat(status).isEqualTo(200) - assertThat(status2).isEqualTo(200) - } -} diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc index 7b851c589df2..3ab49ca0bf26 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc @@ -84,7 +84,7 @@ LoadClusterEntryHandlePtr ProxyFilterConfig::addDynamicCluster( std::string version_info = ""; ENVOY_LOG(debug, "deliver dynamic cluster {} creation to main thread", cluster_name); main_thread_dispatcher_.post([this, cluster, version_info]() { - ENVOY_LOG(debug, "initilizing dynamic cluster {} creation in main thread", cluster.name()); + ENVOY_LOG(debug, "initializing dynamic cluster {} creation in main thread", cluster.name()); cluster_manager_.addOrUpdateCluster(cluster, version_info); }); } else { @@ -223,7 +223,7 @@ Http::FilterHeadersStatus ProxyFilter::decodeHeaders(Http::RequestHeaderMap& hea // using it, which is not a good usage, will end with ServiceUnavailable. // Thread local cluster is existing due to the thread local cache, and the main thread notify // work thread is on the way. - ENVOY_STREAM_LOG(debug, "dynamic foward cluster is gone", *this->decoder_callbacks_); + ENVOY_STREAM_LOG(debug, "dynamic forward cluster is gone", *this->decoder_callbacks_); this->decoder_callbacks_->sendLocalReply(Http::Code::ServiceUnavailable, ResponseStrings::get().DFPClusterIsGone, nullptr, absl::nullopt, RcDetails::get().DFPClusterIsGone); @@ -263,7 +263,7 @@ Http::FilterHeadersStatus ProxyFilter::decodeHeaders(Http::RequestHeaderMap& hea ASSERT(cache_load_handle_ == nullptr); ENVOY_STREAM_LOG(debug, "DNS cache entry already loaded, continuing", *decoder_callbacks_); - auto const& host = config_->cache().getHost(headers.Host()->value().getStringView()); + auto const& host = result.host_info_; latchTime(decoder_callbacks_, DNS_END); if (!host.has_value() || !host.value()->address()) { onDnsResolutionFail(); diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 8a4885eedf14..093e2ef2b782 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -258,7 +258,7 @@ name: envoy.clusters.dynamic_forward_proxy }; int64_t getHeaderValue(const Http::ResponseHeaderMap& headers, absl::string_view name) { - EXPECT_FALSE(headers.get(Http::LowerCaseString(name)).empty()) << "Mising " << name; + EXPECT_FALSE(headers.get(Http::LowerCaseString(name)).empty()) << "Missing " << name; int64_t val; if (!headers.get(Http::LowerCaseString(name)).empty() && absl::SimpleAtoi(headers.get(Http::LowerCaseString(name))[0]->value().getStringView(), @@ -488,6 +488,34 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamCleartext) { checkSimpleRequestSuccess(0, 0, response.get()); } +// Regression test a bug where the host header was used for cache lookups rather than host:port key +TEST_P(ProxyFilterIntegrationTest, CacheSansPort) { + useAccessLog("%RESPONSE_CODE_DETAILS%"); + initializeWithArgs(); + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestRequestHeaderMapImpl request_headers{{":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "localhost"}}; + + // Send a request to localhost, with no port specified. The cluster will + // default to localhost:443, and the connection will fail. + auto response = codec_client_->makeHeaderOnlyRequest(request_headers); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("503", response->headers().getStatusValue()); + EXPECT_THAT(waitForAccessLog(access_log_name_), + HasSubstr("upstream_reset_before_response_started")); + EXPECT_EQ(1, test_server_->counter("dns_cache.foo.host_added")->value()); + + // Now try a second request and make sure it encounters the same error rather + // than dns_resolution_failure. + auto response2 = codec_client_->makeHeaderOnlyRequest(request_headers); + ASSERT_TRUE(response2->waitForEndStream()); + EXPECT_EQ("503", response2->headers().getStatusValue()); + EXPECT_THAT(waitForAccessLog(access_log_name_, 1), + HasSubstr("upstream_reset_before_response_started")); +} + // Verify that `override_auto_sni_header` can be used along with auto_sni to set // SNI from an arbitrary header. TEST_P(ProxyFilterIntegrationTest, UpstreamTlsWithAltHeaderSni) { diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc index d4db4d0a1c08..fe546b87e102 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc @@ -560,13 +560,6 @@ TEST_F(UpstreamResolvedHostFilterStateHelper, AddResolvedHostFilterStateMetadata return MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::InCache, nullptr, host_info}; })); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, getHost(_)) - .WillOnce( - Invoke([&](absl::string_view) - -> absl::optional { - return host_info; - })); - EXPECT_CALL(*host_info, address()).Times(2).WillRepeatedly(Return(host_info->address_)); // Host was resolved successfully, so continue filter iteration. @@ -618,13 +611,6 @@ TEST_F(UpstreamResolvedHostFilterStateHelper, UpdateResolvedHostFilterStateMetad return MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::InCache, nullptr, host_info}; })); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, getHost(_)) - .WillOnce( - Invoke([&](absl::string_view) - -> absl::optional { - return host_info; - })); - EXPECT_CALL(*host_info, address()).Times(2).WillRepeatedly(Return(host_info->address_)); // Host was resolved successfully, so continue filter iteration. @@ -673,13 +659,6 @@ TEST_F(UpstreamResolvedHostFilterStateHelper, IgnoreFilterStateMetadataNullAddre return MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::InCache, nullptr, host_info}; })); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, getHost(_)) - .WillOnce( - Invoke([&](absl::string_view) - -> absl::optional { - return host_info; - })); - EXPECT_CALL(*host_info, address()); EXPECT_CALL(callbacks_, sendLocalReply(Http::Code::ServiceUnavailable, Eq("DNS resolution failure"), _, _, From 047a8ff65774ed3239a0d14f2dc73a5793a43e8b Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 30 May 2023 15:19:46 -0400 Subject: [PATCH 414/740] test: making E-M e2e test work sans yaml (#27634) Signed-off-by: Alyssa Wilk --- .../workflows/mobile-compile_time_options.yml | 37 ++++++----- test/common/upstream/utility.h | 29 +++++--- test/config/utility.cc | 66 ++++++++++++++++++- .../health_checkers/redis/redis_test.cc | 4 +- test/integration/BUILD | 18 ++--- test/integration/base_integration_test.cc | 62 ++++++++++------- test/integration/ssl_utility.cc | 53 +++++---------- 7 files changed, 172 insertions(+), 97 deletions(-) diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index fc8ff2073781..5d2ee4083d5c 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -31,9 +31,9 @@ jobs: - name: 'Build C++ library' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Envoy Mobile build verifies that the build configuration where HTTP/3 is enabled and - # HTTP Datagrams are disabled works. run: | + # Envoy Mobile build which verifies that the build configuration where HTTP/3 is enabled and + # HTTP Datagrams are disabled works. cd mobile && ./bazelw test \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-linux") \ --define=signal_trace=disabled \ @@ -43,7 +43,22 @@ jobs: --define=google_grpc=disabled \ --@envoy//bazel:http3=False \ --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //test/cc/... + //test/cc/... \ + # Envoy Mobile build which verifies that the build configuration where YAML is disabled. + && ./bazelw test \ + $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ + --config=ci \ + --fat_apk_cpu=x86_64 \ + --define=signal_trace=disabled \ + --define=envoy_mobile_request_compression=disabled \ + --define=envoy_enable_http_datagrams=disabled \ + --define=google_grpc=disabled \ + --define=envoy_yaml=disabled \ + --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ + //test/java/integration:android_engine_socket_tag_test \ + //test/java/integration:android_engine_start_test \ + //test/java/io/envoyproxy/envoymobile/utilities:certificate_verification_tests + //test/common/integration:client_integration_test swift_build: if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} needs: env @@ -103,18 +118,4 @@ jobs: --define=google_grpc=disabled \ --define=envoy_yaml=disabled \ --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //:android_dist \ - && ./bazelw test \ - $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ - --config=ci \ - --fat_apk_cpu=x86_64 \ - --define=signal_trace=disabled \ - --define=admin_html=enabled \ - --define=envoy_mobile_request_compression=disabled \ - --define=envoy_enable_http_datagrams=disabled \ - --define=google_grpc=disabled \ - --define=envoy_yaml=disabled \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //test/java/integration:android_engine_socket_tag_test \ - //test/java/integration:android_engine_start_test \ - //test/java/io/envoyproxy/envoymobile/utilities:certificate_verification_tests + //:android_dist diff --git a/test/common/upstream/utility.h b/test/common/upstream/utility.h index 503b794b570a..0458f90a15db 100644 --- a/test/common/upstream/utility.h +++ b/test/common/upstream/utility.h @@ -57,24 +57,41 @@ inline std::string defaultStaticClusterJson(const std::string& name) { inline envoy::config::bootstrap::v3::Bootstrap parseBootstrapFromV3Json(const std::string& json_string) { envoy::config::bootstrap::v3::Bootstrap bootstrap; + // TODO(alyssawilk) rename to JSON +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromJson(json_string, bootstrap); +#else + PANIC("JSON compiled out: cannot parse " + json_string); +#endif return bootstrap; } inline envoy::config::cluster::v3::Cluster parseClusterFromV3Json(const std::string& json_string) { envoy::config::cluster::v3::Cluster cluster; +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromJson(json_string, cluster); +#else + PANIC("JSON compiled out: cannot parse " + json_string); +#endif return cluster; } inline envoy::config::cluster::v3::Cluster parseClusterFromV3Yaml(const std::string& yaml) { envoy::config::cluster::v3::Cluster cluster; +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(yaml, cluster); +#else + PANIC("JSON compiled out: cannot parse " + yaml); +#endif return cluster; } inline envoy::config::cluster::v3::Cluster defaultStaticCluster(const std::string& name) { +#ifdef ENVOY_ENABLE_YAML return parseClusterFromV3Json(defaultStaticClusterJson(name)); +#else + PANIC("JSON compiled out: cannot parse " + name); +#endif } inline HostSharedPtr makeTestHost(ClusterInfoConstSharedPtr cluster, const std::string& hostname, @@ -167,15 +184,11 @@ makeLocalityWeights(std::initializer_list locality_weights) { inline envoy::config::core::v3::HealthCheck parseHealthCheckFromV3Yaml(const std::string& yaml_string) { envoy::config::core::v3::HealthCheck health_check; +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYamlAndValidate(yaml_string, health_check); - return health_check; -} - -// For DEPRECATED TEST CASES -inline envoy::config::core::v3::HealthCheck -parseHealthCheckFromV2Yaml(const std::string& yaml_string) { - envoy::config::core::v3::HealthCheck health_check; - TestUtility::loadFromYamlAndValidate(yaml_string, health_check); +#else + PANIC("JSON compiled out: cannot parse " + yaml_string); +#endif return health_check; } diff --git a/test/config/utility.cc b/test/config/utility.cc index ca33b39059f4..b9b15c102ad3 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -33,7 +33,13 @@ namespace Envoy { namespace { envoy::config::bootstrap::v3::Bootstrap& basicBootstrap(envoy::config::bootstrap::v3::Bootstrap& bootstrap, const std::string& config) { +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(config, bootstrap); +#else + UNREFERENCED_PARAMETER(config); + UNREFERENCED_PARAMETER(bootstrap); + PANIC("JSON compiled out: can't load config"); +#endif return bootstrap; } } // namespace @@ -513,7 +519,6 @@ ConfigHelper::buildStaticCluster(const std::string& name, int port, const std::s (*cluster.mutable_typed_extension_protocol_options()) ["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"] .PackFrom(protocol_options); - return cluster; } @@ -576,7 +581,6 @@ ConfigHelper::buildTlsCluster(const std::string& name, TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); socket->set_name("envoy.transport_sockets.tls"); socket->mutable_typed_config()->PackFrom(tls_socket); - return cluster; } @@ -623,6 +627,7 @@ envoy::config::listener::v3::Listener ConfigHelper::buildBaseListener(const std::string& name, const std::string& address, const std::string& filter_chains) { API_NO_BOOST(envoy::config::listener::v3::Listener) listener; +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(fmt::format( R"EOF( name: {} @@ -636,6 +641,12 @@ ConfigHelper::buildBaseListener(const std::string& name, const std::string& addr name, address, filter_chains), listener); return listener; +#else + UNREFERENCED_PARAMETER(name); + UNREFERENCED_PARAMETER(address); + UNREFERENCED_PARAMETER(filter_chains); + PANIC("YAML support compiled out"); +#endif } envoy::config::listener::v3::Listener ConfigHelper::buildListener(const std::string& name, @@ -667,6 +678,7 @@ envoy::config::listener::v3::Listener ConfigHelper::buildListener(const std::str envoy::config::route::v3::RouteConfiguration ConfigHelper::buildRouteConfig(const std::string& name, const std::string& cluster) { API_NO_BOOST(envoy::config::route::v3::RouteConfiguration) route; +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(fmt::format(R"EOF( name: "{}" virtual_hosts: @@ -679,6 +691,11 @@ ConfigHelper::buildRouteConfig(const std::string& name, const std::string& clust name, cluster), route); return route; +#else + UNREFERENCED_PARAMETER(name); + UNREFERENCED_PARAMETER(cluster); + PANIC("YAML support compiled out"); +#endif } envoy::config::endpoint::v3::Endpoint ConfigHelper::buildEndpoint(const std::string& address) { @@ -770,6 +787,7 @@ void ConfigHelper::addListenerTypedMetadata(absl::string_view key, ProtobufWkt:: void ConfigHelper::addClusterFilterMetadata(absl::string_view metadata_yaml, absl::string_view cluster_name) { +#ifdef ENVOY_ENABLE_YAML RELEASE_ASSERT(!finalized_, ""); ProtobufWkt::Struct cluster_metadata; TestUtility::loadFromYaml(std::string(metadata_yaml), cluster_metadata); @@ -787,6 +805,11 @@ void ConfigHelper::addClusterFilterMetadata(absl::string_view metadata_yaml, } break; } +#else + UNREFERENCED_PARAMETER(metadata_yaml); + UNREFERENCED_PARAMETER(cluster_name); + PANIC("YAML support compiled out"); +#endif } void ConfigHelper::setConnectConfig( @@ -1163,7 +1186,13 @@ void ConfigHelper::prependFilter(const std::string& config, bool downstream) { loadHttpConnectionManager(hcm_config); auto* filter_list_back = hcm_config.add_http_filters(); +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(config, *filter_list_back); +#else + UNREFERENCED_PARAMETER(config); + UNREFERENCED_PARAMETER(filter_list_back); + PANIC("YAML support compiled out"); +#endif // Now move it to the front. for (int i = hcm_config.http_filters_size() - 1; i > 0; --i) { @@ -1188,7 +1217,12 @@ void ConfigHelper::prependFilter(const std::string& config, bool downstream) { old_protocol_options.add_http_filters()->set_name("envoy.filters.http.upstream_codec"); } auto* filter_list_back = old_protocol_options.add_http_filters(); +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(config, *filter_list_back); +#else + UNREFERENCED_PARAMETER(filter_list_back); + PANIC("YAML support compiled out"); +#endif for (int i = old_protocol_options.http_filters_size() - 1; i > 0; --i) { old_protocol_options.mutable_http_filters()->SwapElements(i, i - 1); } @@ -1400,7 +1434,12 @@ void ConfigHelper::initializeTls( filename: "{{ test_rundir }}/test/config/integration/certs/intermediate_partial_ca_cert_chain.pem" )EOF"; } +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(TestEnvironment::substitute(cert_yaml), *validation_context); +#else + UNREFERENCED_PARAMETER(cert_yaml); + PANIC("YAML support compiled out"); +#endif if (options.max_verify_depth_.has_value()) { validation_context->mutable_max_verify_depth()->set_value( options.max_verify_depth_.value()); @@ -1478,7 +1517,13 @@ void ConfigHelper::addNetworkFilter(const std::string& filter_yaml) { auto* filter_chain = bootstrap_.mutable_static_resources()->mutable_listeners(0)->mutable_filter_chains(0); auto* filter_list_back = filter_chain->add_filters(); +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(filter_yaml, *filter_list_back); +#else + UNREFERENCED_PARAMETER(filter_list_back); + UNREFERENCED_PARAMETER(filter_yaml); + PANIC("YAML support compiled out"); +#endif // Now move it to the front. for (int i = filter_chain->filters_size() - 1; i > 0; --i) { @@ -1490,7 +1535,13 @@ void ConfigHelper::addListenerFilter(const std::string& filter_yaml) { RELEASE_ASSERT(!finalized_, ""); auto* listener = bootstrap_.mutable_static_resources()->mutable_listeners(0); auto* filter_list_back = listener->add_listener_filters(); +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(filter_yaml, *filter_list_back); +#else + UNREFERENCED_PARAMETER(filter_list_back); + UNREFERENCED_PARAMETER(filter_yaml); + PANIC("YAML support compiled out"); +#endif // Now move it to the front. for (int i = listener->listener_filters_size() - 1; i > 0; --i) { @@ -1501,7 +1552,13 @@ void ConfigHelper::addListenerFilter(const std::string& filter_yaml) { void ConfigHelper::addBootstrapExtension(const std::string& config) { RELEASE_ASSERT(!finalized_, ""); auto* extension = bootstrap_.add_bootstrap_extensions(); +#ifdef ENVOY_ENABLE_YAML TestUtility::loadFromYaml(config, *extension); +#else + UNREFERENCED_PARAMETER(extension); + UNREFERENCED_PARAMETER(config); + PANIC("YAML support compiled out"); +#endif } bool ConfigHelper::loadHttpConnectionManager( @@ -1548,9 +1605,14 @@ void ConfigHelper::setLds(absl::string_view version_info) { const std::string lds_filename = bootstrap().dynamic_resources().lds_config().path_config_source().path(); +#ifdef ENVOY_ENABLE_YAML std::string file = TestEnvironment::writeStringToFileForTest( "new_lds_file", MessageUtil::getJsonStringFromMessageOrError(lds)); TestEnvironment::renameFile(file, lds_filename); +#else + UNREFERENCED_PARAMETER(lds_filename); + PANIC("YAML support compiled out"); +#endif } void ConfigHelper::setDownstreamOutboundFramesLimits(uint32_t max_all_frames, diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index 1bcb4e5ba783..c564838a30bb 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -79,7 +79,7 @@ class RedisHealthCheckerTest "@type": type.googleapis.com/envoy.extensions.health_checkers.redis.v3.Redis )EOF"; - const auto& health_check_config = Upstream::parseHealthCheckFromV2Yaml(yaml); + const auto& health_check_config = Upstream::parseHealthCheckFromV3Yaml(yaml); const auto& redis_config = getRedisHealthCheckConfig( health_check_config, ProtobufMessage::getStrictValidationVisitor()); @@ -164,7 +164,7 @@ class RedisHealthCheckerTest key: foo )EOF"; - const auto& health_check_config = Upstream::parseHealthCheckFromV2Yaml(yaml); + const auto& health_check_config = Upstream::parseHealthCheckFromV3Yaml(yaml); const auto& redis_config = getRedisHealthCheckConfig( health_check_config, ProtobufMessage::getStrictValidationVisitor()); diff --git a/test/integration/BUILD b/test/integration/BUILD index fbcbfbfaf828..c918f09fadf1 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1053,7 +1053,6 @@ envoy_cc_test_library( "//source/common/config:api_version_lib", "//source/extensions/clusters/eds:eds_lib", "//source/extensions/clusters/static:static_cluster_lib", - "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib", "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//source/extensions/load_balancing_policies/maglev:config", @@ -1081,7 +1080,9 @@ envoy_cc_test_library( ] + select({ "//bazel:apple": ["//source/extensions/network/dns_resolver/apple:config"], "//conditions:default": [], - }), + }) + envoy_select_enable_yaml([ + "//source/extensions/config_subscription/filesystem:filesystem_subscription_lib", + ]), ) envoy_cc_test_library( @@ -1101,12 +1102,14 @@ envoy_cc_test_library( name = "utility_lib", srcs = [ "server.cc", + "ssl_utility.cc", "utility.cc", - ] + envoy_select_enable_yaml(["ssl_utility.cc"]), + ], hdrs = [ "server.h", + "ssl_utility.h", "utility.h", - ] + envoy_select_enable_yaml(["ssl_utility.h"]), + ], data = ["//test/common/runtime:filesystem_test_data"], deps = [ ":server_stats_interface", @@ -1140,6 +1143,8 @@ envoy_cc_test_library( "//source/server:options_lib", "//source/server:process_context_lib", "//source/server:server_lib", + "//test/common/upstream:utility_lib", + "//test/config:utility_lib", "//test/mocks:common_lib", "//test/mocks/event:event_mocks", "//test/mocks/runtime:runtime_mocks", @@ -1157,10 +1162,7 @@ envoy_cc_test_library( "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/quic/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", - ] + envoy_select_enable_yaml([ - "//test/common/upstream:utility_lib", - "//test/config:utility_lib", - ]), + ], ) envoy_cc_test_library( diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index a90bf2210a43..5a2bbfa161fa 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -30,9 +30,14 @@ namespace Envoy { envoy::config::bootstrap::v3::Bootstrap configToBootstrap(const std::string& config) { +#ifdef ENVOY_ENABLE_YAML envoy::config::bootstrap::v3::Bootstrap bootstrap; TestUtility::loadFromYaml(config, bootstrap); return bootstrap; +#else + UNREFERENCED_PARAMETER(config); + PANIC("YAML support compiled out: can't parse YAML"); +#endif } using ::testing::_; @@ -131,19 +136,17 @@ void BaseIntegrationTest::initialize() { Network::DownstreamTransportSocketFactoryPtr BaseIntegrationTest::createUpstreamTlsContext(const FakeUpstreamConfig& upstream_config) { envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; - const std::string yaml = absl::StrFormat( - R"EOF( -common_tls_context: - tls_certificates: - - certificate_chain: { filename: "%s" } - private_key: { filename: "%s" } - validation_context: - trusted_ca: { filename: "%s" } -)EOF", - TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcert.pem"), - TestEnvironment::runfilesPath("test/config/integration/certs/upstreamkey.pem"), - TestEnvironment::runfilesPath("test/config/integration/certs/cacert.pem")); - TestUtility::loadFromYaml(yaml, tls_context); + const std::string rundir = TestEnvironment::runfilesDirectory(); + tls_context.mutable_common_tls_context() + ->mutable_validation_context() + ->mutable_trusted_ca() + ->set_filename(rundir + "/test/config/integration/certs/cacert.pem"); + auto* certs = tls_context.mutable_common_tls_context()->add_tls_certificates(); + certs->mutable_certificate_chain()->set_filename( + rundir + "/test/config/integration/certs/upstreamcert.pem"); + certs->mutable_private_key()->set_filename(rundir + + "/test/config/integration/certs/upstreamkey.pem"); + if (upstream_config.upstream_protocol_ == Http::CodecType::HTTP2) { tls_context.mutable_common_tls_context()->add_alpn_protocols("h2"); } else if (upstream_config.upstream_protocol_ == Http::CodecType::HTTP1) { @@ -221,15 +224,22 @@ std::string BaseIntegrationTest::finalizeConfigWithPorts(ConfigHelper& config_he ProtobufWkt::Any* resource = lds.add_resources(); resource->PackFrom(listener); } +#ifdef ENVOY_ENABLE_YAML TestEnvironment::writeStringToFileForTest( lds_path, MessageUtil::getJsonStringFromMessageOrError(lds), true); - +#else + PANIC("YAML support compiled out: can't parse YAML"); +#endif // Now that the listeners have been written to the lds file, remove them from static resources // or they will not be reloadable. bootstrap.mutable_static_resources()->mutable_listeners()->Clear(); } +#ifdef ENVOY_ENABLE_YAML ENVOY_LOG_MISC(debug, "Running Envoy with configuration:\n{}", MessageUtil::getYamlStringFromMessage(bootstrap)); +#else + ENVOY_LOG_MISC(debug, "Running Envoy with configuration:\n{}", bootstrap.DebugString()); +#endif const std::string bootstrap_path = TestEnvironment::writeStringToFileForTest( "bootstrap.pb", TestUtility::getProtobufBinaryStringFromMessage(bootstrap)); @@ -426,7 +436,11 @@ std::string getListenerDetails(Envoy::Server::Instance& server) { const auto& cbs_maps = server.admin()->getConfigTracker().getCallbacksMap(); ProtobufTypes::MessagePtr details = cbs_maps.at("listeners")(Matchers::UniversalStringMatcher()); auto listener_info = Protobuf::down_cast(*details); +#ifdef ENVOY_ENABLE_YAML return MessageUtil::getYamlStringFromMessage(listener_info.dynamic_listeners(0).error_state()); +#else + return listener_info.dynamic_listeners(0).error_state().DebugString(); +#endif } void BaseIntegrationTest::createGeneratedApiTestServer( @@ -766,16 +780,16 @@ AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( // Attempt to heuristically discover missing tag-extraction rules when new stats are added. // This is done by looking through the entire config for fields named `stat_prefix`, and then // validating that those values do not appear in the tag-extracted name of any stat. The alternate -// approach of looking for the prefix in the extracted tags was more difficult because in the tests -// some prefix values are reused (leading to false negatives) and some tests have base configuration -// that sets a stat_prefix but don't produce any stats at all with that configuration (leading to -// false positives). +// approach of looking for the prefix in the extracted tags was more difficult because in the +// tests some prefix values are reused (leading to false negatives) and some tests have base +// configuration that sets a stat_prefix but don't produce any stats at all with that +// configuration (leading to false positives). // // To add a rule, see `source/common/config/well_known_names.cc`. // -// This is done in all integration tests because it is testing new stats and scopes that are created -// for which the author isn't aware that tag extraction rules need to be written, and thus the -// author wouldn't think to write tests for that themselves. +// This is done in all integration tests because it is testing new stats and scopes that are +// created for which the author isn't aware that tag extraction rules need to be written, and thus +// the author wouldn't think to write tests for that themselves. void BaseIntegrationTest::checkForMissingTagExtractionRules() { BufferingStreamDecoderPtr response = IntegrationUtil::makeSingleRequest( test_server_->adminAddress(), "GET", "/config_dump", "", Http::CodecType::HTTP1); @@ -791,9 +805,9 @@ void BaseIntegrationTest::checkForMissingTagExtractionRules() { std::vector stat_prefixes; Json::ObjectCallback find_stat_prefix = [&](const std::string& name, const Json::Object& root) -> bool { - // Looking for `stat_prefix` is based on precedent for how this is usually named in the config. - // If there are other names used for a similar purpose, this check could be expanded to add them - // also. + // Looking for `stat_prefix` is based on precedent for how this is usually named in the + // config. If there are other names used for a similar purpose, this check could be expanded + // to add them also. if (name == "stat_prefix") { auto prefix = root.asString(); if (!prefix.empty()) { diff --git a/test/integration/ssl_utility.cc b/test/integration/ssl_utility.cc index af93ec2c3a48..6727632d99f1 100644 --- a/test/integration/ssl_utility.cc +++ b/test/integration/ssl_utility.cc @@ -25,47 +25,30 @@ namespace Ssl { void initializeUpstreamTlsContextConfig( const ClientSslTransportOptions& options, envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext& tls_context) { - std::string yaml_plain = R"EOF( - common_tls_context: - validation_context: - trusted_ca: - filename: "{{ test_rundir }}/test/config/integration/certs/cacert.pem" -)EOF"; + const std::string rundir = TestEnvironment::runfilesDirectory(); + tls_context.mutable_common_tls_context() + ->mutable_validation_context() + ->mutable_trusted_ca() + ->set_filename(rundir + "/test/config/integration/certs/cacert.pem"); + auto* certs = tls_context.mutable_common_tls_context()->add_tls_certificates(); + std::string chain; + std::string key; if (options.client_ecdsa_cert_) { - yaml_plain += R"EOF( - tls_certificates: - certificate_chain: - filename: "{{ test_rundir }}/test/config/integration/certs/client_ecdsacert.pem" - private_key: - filename: "{{ test_rundir }}/test/config/integration/certs/client_ecdsakey.pem" -)EOF"; + chain = rundir + "/test/config/integration/certs/client_ecdsacert.pem"; + key = rundir + "/test/config/integration/certs/client_ecdsakey.pem"; } else if (options.use_expired_spiffe_cert_) { - yaml_plain += R"EOF( - tls_certificates: - certificate_chain: - filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_cert.pem" - private_key: - filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_key.pem" -)EOF"; + chain = rundir + "/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_cert.pem"; + key = rundir + "/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_key.pem"; } else if (options.client_with_intermediate_cert_) { - yaml_plain += R"EOF( - tls_certificates: - certificate_chain: - filename: "{{ test_rundir }}/test/config/integration/certs/client2_chain.pem" - private_key: - filename: "{{ test_rundir }}/test/config/integration/certs/client2key.pem" -)EOF"; + chain = rundir + "/test/config/integration/certs/client2_chain.pem"; + key = rundir + "/test/config/integration/certs/client2key.pem"; } else { - yaml_plain += R"EOF( - tls_certificates: - certificate_chain: - filename: "{{ test_rundir }}/test/config/integration/certs/clientcert.pem" - private_key: - filename: "{{ test_rundir }}/test/config/integration/certs/clientkey.pem" -)EOF"; + chain = rundir + "/test/config/integration/certs/clientcert.pem"; + key = rundir + "/test/config/integration/certs/clientkey.pem"; } + certs->mutable_certificate_chain()->set_filename(chain); + certs->mutable_private_key()->set_filename(key); - TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), tls_context); auto* common_context = tls_context.mutable_common_tls_context(); if (options.alpn_) { From 5673f1adcdf9b79a23c011d902e2a21c5db45bd1 Mon Sep 17 00:00:00 2001 From: doujiang24 Date: Wed, 31 May 2023 03:36:01 +0800 Subject: [PATCH 415/740] golang filter: always use GC finalizer to delete the req object in C++ (#27630) It could be safer since people may still use the request object in Go even after it told C++ it had already finished processing. Signed-off-by: doujiang24 --- contrib/golang/filters/http/source/go/pkg/http/shim.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/contrib/golang/filters/http/source/go/pkg/http/shim.go b/contrib/golang/filters/http/source/go/pkg/http/shim.go index fc208a77c982..fe0cb3751792 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/shim.go +++ b/contrib/golang/filters/http/source/go/pkg/http/shim.go @@ -215,10 +215,4 @@ func envoyGoFilterOnHttpDestroy(r *C.httpRequest, reason uint64) { f.OnDestroy(v) Requests.DeleteReq(r) - - // no one is using req now, we can remove it manually, for better performance. - if v == api.Normal { - runtime.SetFinalizer(req, nil) - req.Finalize(api.GCFinalize) - } } From 1c85fe6a6d4f94313384b953c7ad610b65aaa1f2 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 31 May 2023 08:11:45 +0100 Subject: [PATCH 416/740] ci/examples: Minor cleanup (#27686) Signed-off-by: Ryan Northey --- ci/do_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 21fd47eae66d..b0f178293f05 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -163,7 +163,7 @@ function run_ci_verify () { # by a strong umask (ie only group readable by default). umask 027 chmod -R o-rwx examples/ - "${ENVOY_SRCDIR}/ci/verify_examples.sh" "${@}" || exit + "${ENVOY_SRCDIR}/ci/verify_examples.sh" "${@}" } CI_TARGET=$1 From fac75f32d961a9ae7264935ef983e8bfa027cf96 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 31 May 2023 09:26:40 +0100 Subject: [PATCH 417/740] ci: Add github workflow POLICY (#27600) Signed-off-by: Ryan Northey --- .github/workflows/POLICY.md | 59 +++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/POLICY.md diff --git a/.github/workflows/POLICY.md b/.github/workflows/POLICY.md new file mode 100644 index 000000000000..86d775493dc9 --- /dev/null +++ b/.github/workflows/POLICY.md @@ -0,0 +1,59 @@ +# Envoy Github workflows + +## Trusted workflows + +Github workflows that are **not** triggered by a `pull_request` generally run with +the repository context/permissions. + +In various ways, these workflows can be triggered as the result of a `pull_request` +and/or be made to run untrusted code (ie PR code). + +This can be useful, but carries significant risks. + +In particular this can effect: + +- `pull_request_target` +- `workflow_run` +- `workflow_dispatch` + +Do not use these trigger events unless they are required. + +## Restrict global permissions and secrets in trusted workflows + +If a job requires specific permissions, these should be added on per-job basis. + +Global permissions should be set as follows: + +```yaml +permissions: + contents: read +``` + +Likewise, any secrets that a job requires should be set per-job. + +## Restrict access to `workflow_dispatch` + +It is important to restrict who can trigger these types of workflow. + +Do not allow any bots or app users to do so, unless this is specifically required. + +For example, you could add a `job` condition to prevent any bots from triggering the workflow: + +```yaml + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || !contains(github.actor, '[bot]')) + }} +``` + +## Trusted/untrusted CI jobs + +If a trusted workflow is used to run untrusted code, then the entire job that runs this code +should be treated as untrusted. + +In this case, it is **essential** to ensure: + +- no write permissions in the untrusted job +- no secrets in the untrusted job From 13e5514911a6e63220fa991b64ae7691533875ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 May 2023 09:31:26 +0100 Subject: [PATCH 418/740] build(deps): bump otel/opentelemetry-collector from `5c3a03d` to `32edac6` in /examples/opentelemetry (#27716) build(deps): bump otel/opentelemetry-collector Bumps otel/opentelemetry-collector from `5c3a03d` to `32edac6`. --- updated-dependencies: - dependency-name: otel/opentelemetry-collector dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/opentelemetry/Dockerfile-opentelemetry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry index 177882fbdf92..ad03819c59e4 100644 --- a/examples/opentelemetry/Dockerfile-opentelemetry +++ b/examples/opentelemetry/Dockerfile-opentelemetry @@ -1,7 +1,7 @@ FROM alpine:3.18@sha256:02bb6f428431fbc2809c5d1b41eab5a68350194fb508869a33cb1af4444c9b11 as otelc_curl RUN apk --update add curl -FROM otel/opentelemetry-collector:latest@sha256:5c3a03dac40b1f37069c63a2ee75d7bd8c7ba81fc42831085053b1c70e528349 +FROM otel/opentelemetry-collector:latest@sha256:32edac6fe9479c4a33a60be226af0a3ee9f69a5b79d935835b37bf9b25e4b5ce COPY --from=otelc_curl / / From de2ef460d88b2f703a5bfcae9e6b35304ec83906 Mon Sep 17 00:00:00 2001 From: code Date: Wed, 31 May 2023 19:07:42 +0800 Subject: [PATCH 419/740] generic proxy: add support to the original dst cluster and subset lb (#27502) Signed-off-by: wbpcode --- .../filters/network/source/interface/filter.h | 5 ++ .../filters/network/source/proxy.cc | 4 ++ .../filters/network/source/proxy.h | 1 + .../filters/network/source/router/BUILD | 2 + .../filters/network/source/router/router.cc | 23 ++++++++ .../filters/network/source/router/router.h | 6 +++ .../filters/network/test/mocks/filter.h | 1 + .../filters/network/test/proxy_test.cc | 21 ++++++++ .../network/test/router/router_test.cc | 54 +++++++++++++++++++ 9 files changed, 117 insertions(+) diff --git a/contrib/generic_proxy/filters/network/source/interface/filter.h b/contrib/generic_proxy/filters/network/source/interface/filter.h index 36f0f30698d0..e8bcdb06c861 100644 --- a/contrib/generic_proxy/filters/network/source/interface/filter.h +++ b/contrib/generic_proxy/filters/network/source/interface/filter.h @@ -75,6 +75,11 @@ class StreamFilterCallbacks { * @return absl::optional the extended options from upstream response. */ virtual absl::optional responseOptions() const PURE; + + /** + * @return const Network::Connection* downstream connection. + */ + virtual const Network::Connection* connection() const PURE; }; class UpstreamBindingCallback { diff --git a/contrib/generic_proxy/filters/network/source/proxy.cc b/contrib/generic_proxy/filters/network/source/proxy.cc index c8e03e87c13d..7925d27b54a5 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.cc +++ b/contrib/generic_proxy/filters/network/source/proxy.cc @@ -137,6 +137,10 @@ OptRef ActiveStream::ActiveDecoderFilter::boundUpstreamConn() { return parent_.parent_.boundUpstreamConn(); } +const Network::Connection* ActiveStream::ActiveFilterBase::connection() const { + return &parent_.parent_.connection(); +} + void ActiveStream::continueEncoding() { if (active_stream_reset_ || local_or_upstream_response_stream_ == nullptr) { return; diff --git a/contrib/generic_proxy/filters/network/source/proxy.h b/contrib/generic_proxy/filters/network/source/proxy.h index 3de767b737e9..964815908dc3 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.h +++ b/contrib/generic_proxy/filters/network/source/proxy.h @@ -130,6 +130,7 @@ class ActiveStream : public FilterChainManager, absl::optional responseOptions() const override { return parent_.local_or_upstream_response_options_; } + const Network::Connection* connection() const override; bool isDualFilter() const { return is_dual_; } diff --git a/contrib/generic_proxy/filters/network/source/router/BUILD b/contrib/generic_proxy/filters/network/source/router/BUILD index e78d581f9cf0..0b06176425b0 100644 --- a/contrib/generic_proxy/filters/network/source/router/BUILD +++ b/contrib/generic_proxy/filters/network/source/router/BUILD @@ -25,6 +25,8 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/common:linked_object", "//source/common/common:minimal_logger_lib", + "//source/common/config:well_known_names", + "//source/common/router:metadatamatchcriteria_lib", "//source/common/stream_info:stream_info_lib", "//source/common/tracing:tracer_lib", "//source/common/upstream:load_balancer_lib", diff --git a/contrib/generic_proxy/filters/network/source/router/router.cc b/contrib/generic_proxy/filters/network/source/router/router.cc index 8a24acbf3f67..eff84ef125a0 100644 --- a/contrib/generic_proxy/filters/network/source/router/router.cc +++ b/contrib/generic_proxy/filters/network/source/router/router.cc @@ -4,6 +4,8 @@ #include "envoy/network/connection.h" #include "source/common/common/assert.h" +#include "source/common/config/well_known_names.h" +#include "source/common/router/metadatamatchcriteria_impl.h" #include "source/common/tracing/tracer_impl.h" #include "contrib/generic_proxy/filters/network/source/interface/filter.h" @@ -395,6 +397,27 @@ FilterStatus RouterFilter::onStreamDecoded(Request& request) { return FilterStatus::StopIteration; } +const Envoy::Router::MetadataMatchCriteria* RouterFilter::metadataMatchCriteria() { + // Have we been called before? If so, there's no need to recompute. + if (metadata_match_ != nullptr) { + return metadata_match_.get(); + } + + const auto& request_metadata = callbacks_->streamInfo().dynamicMetadata().filter_metadata(); + const auto filter_it = request_metadata.find(Envoy::Config::MetadataFilters::get().ENVOY_LB); + + if (filter_it == request_metadata.end()) { + return nullptr; + } + + metadata_match_ = std::make_unique(filter_it->second); + return metadata_match_.get(); +} + +const Network::Connection* RouterFilter::downstreamConnection() const { + return callbacks_ != nullptr ? callbacks_->connection() : nullptr; +} + } // namespace Router } // namespace GenericProxy } // namespace NetworkFilters diff --git a/contrib/generic_proxy/filters/network/source/router/router.h b/contrib/generic_proxy/filters/network/source/router/router.h index 9d6905c9519b..8666fb44124a 100644 --- a/contrib/generic_proxy/filters/network/source/router/router.h +++ b/contrib/generic_proxy/filters/network/source/router/router.h @@ -142,6 +142,10 @@ class RouterFilter : public DecoderFilter, std::list& upstreamRequestsForTest() { return upstream_requests_; } + // Upstream::LoadBalancerContextBase + const Envoy::Router::MetadataMatchCriteria* metadataMatchCriteria() override; + const Network::Connection* downstreamConnection() const override; + private: friend class UpstreamRequest; friend class UpstreamManagerImpl; @@ -159,6 +163,8 @@ class RouterFilter : public DecoderFilter, Upstream::ClusterInfoConstSharedPtr cluster_; Request* request_{}; + Envoy::Router::MetadataMatchCriteriaConstPtr metadata_match_; + RequestEncoderPtr request_encoder_; std::list upstream_requests_; diff --git a/contrib/generic_proxy/filters/network/test/mocks/filter.h b/contrib/generic_proxy/filters/network/test/mocks/filter.h index bf861429c15d..4f563c6416dc 100644 --- a/contrib/generic_proxy/filters/network/test/mocks/filter.h +++ b/contrib/generic_proxy/filters/network/test/mocks/filter.h @@ -115,6 +115,7 @@ template class MockStreamFilterCallbacks : public Base { MOCK_METHOD(OptRef, tracingConfig, (), (const)); MOCK_METHOD(absl::optional, requestOptions, (), (const)); MOCK_METHOD(absl::optional, responseOptions, (), (const)); + MOCK_METHOD(const Network::Connection*, connection, (), (const)); }; class MockUpstreamManager : public UpstreamManager { diff --git a/contrib/generic_proxy/filters/network/test/proxy_test.cc b/contrib/generic_proxy/filters/network/test/proxy_test.cc index 2a547fe1abc7..4020b27542c7 100644 --- a/contrib/generic_proxy/filters/network/test/proxy_test.cc +++ b/contrib/generic_proxy/filters/network/test/proxy_test.cc @@ -427,6 +427,27 @@ TEST_F(FilterTest, ActiveStreamPerFilterConfigNoRouteEntry) { EXPECT_EQ(nullptr, active_stream->decoderFiltersForTest()[0]->perFilterConfig()); } +TEST_F(FilterTest, ActiveStreamConnection) { + mock_stream_filters_.push_back( + {"fake_test_filter_name_0", std::make_shared>()}); + + initializeFilter(); + + auto request = std::make_unique(); + filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + EXPECT_EQ(1, filter_->activeStreamsForTest().size()); + + auto active_stream = filter_->activeStreamsForTest().begin()->get(); + + EXPECT_EQ(1, active_stream->decoderFiltersForTest().size()); + EXPECT_EQ(1, active_stream->encoderFiltersForTest().size()); + EXPECT_EQ(1, active_stream->nextDecoderFilterIndexForTest()); + EXPECT_EQ(0, active_stream->nextEncoderFilterIndexForTest()); + + EXPECT_EQ(&filter_callbacks_.connection_, + active_stream->decoderFiltersForTest()[0]->connection()); +} + TEST_F(FilterTest, ActiveStreamAddFilters) { initializeFilter(); diff --git a/contrib/generic_proxy/filters/network/test/router/router_test.cc b/contrib/generic_proxy/filters/network/test/router/router_test.cc index ff979fdecca6..62384272689e 100644 --- a/contrib/generic_proxy/filters/network/test/router/router_test.cc +++ b/contrib/generic_proxy/filters/network/test/router/router_test.cc @@ -276,6 +276,42 @@ class RouterFilterTest : public testing::TestWithParam { EXPECT_EQ(1, filter_->upstreamRequestsForTest().size()); } + void verifyMetadataMatchCriteria() { + ProtobufWkt::Struct request_struct; + ProtobufWkt::Value val; + + // Populate metadata like StreamInfo.setDynamicMetadata() would. + auto& fields_map = *request_struct.mutable_fields(); + val.set_string_value("v3.1"); + fields_map["version"] = val; + val.set_string_value("devel"); + fields_map["stage"] = val; + val.set_string_value("1"); + fields_map["xkey_in_request"] = val; + (*mock_stream_info_.metadata_ + .mutable_filter_metadata())[Envoy::Config::MetadataFilters::get().ENVOY_LB] = + request_struct; + + auto match = filter_->metadataMatchCriteria()->metadataMatchCriteria(); + + EXPECT_EQ(match.size(), 3); + auto it = match.begin(); + + // Note: metadataMatchCriteria() keeps its entries sorted, so the order for checks + // below matters. + + EXPECT_EQ((*it)->name(), "stage"); + EXPECT_EQ((*it)->value().value().string_value(), "devel"); + it++; + + EXPECT_EQ((*it)->name(), "version"); + EXPECT_EQ((*it)->value().value().string_value(), "v3.1"); + it++; + + EXPECT_EQ((*it)->name(), "xkey_in_request"); + EXPECT_EQ((*it)->value().value().string_value(), "1"); + } + NiceMock factory_context_; NiceMock dispatcher_; @@ -707,6 +743,24 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponseDecodingFailure) { notifyDecodingFailure(); } +TEST_P(RouterFilterTest, LoadBalancerContextDownstreamConnection) { + setup(); + EXPECT_CALL(mock_filter_callback_, connection()); + filter_->downstreamConnection(); +} + +TEST_P(RouterFilterTest, LoadBalancerContextNoMetadataMatchCriteria) { + setup(); + + // No metadata match criteria by default. + EXPECT_EQ(nullptr, filter_->metadataMatchCriteria()); +} + +TEST_P(RouterFilterTest, LoadBalancerContextMetadataMatchCriteria) { + setup(); + verifyMetadataMatchCriteria(); +} + } // namespace } // namespace Router } // namespace GenericProxy From 26797819a34cc4461d8cf2c0019463ab4d04509f Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 31 May 2023 13:52:55 +0100 Subject: [PATCH 420/740] mobile/ci: Fix/cleanup yaml config (#27720) Signed-off-by: Ryan Northey --- .github/workflows/mobile-compile_time_options.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index 5d2ee4083d5c..ae6b82e4ec57 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -34,7 +34,8 @@ jobs: run: | # Envoy Mobile build which verifies that the build configuration where HTTP/3 is enabled and # HTTP Datagrams are disabled works. - cd mobile && ./bazelw test \ + cd mobile + ./bazelw test \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-linux") \ --define=signal_trace=disabled \ --define=admin_html=enabled \ @@ -43,9 +44,9 @@ jobs: --define=google_grpc=disabled \ --@envoy//bazel:http3=False \ --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //test/cc/... \ - # Envoy Mobile build which verifies that the build configuration where YAML is disabled. - && ./bazelw test \ + //test/cc/... + # Envoy Mobile build which verifies that the build configuration where YAML is disabled. + ./bazelw test \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --config=ci \ --fat_apk_cpu=x86_64 \ @@ -57,7 +58,7 @@ jobs: --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ //test/java/integration:android_engine_socket_tag_test \ //test/java/integration:android_engine_start_test \ - //test/java/io/envoyproxy/envoymobile/utilities:certificate_verification_tests + //test/java/io/envoyproxy/envoymobile/utilities:certificate_verification_tests \ //test/common/integration:client_integration_test swift_build: if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} From 0ae464feb4b3d1655baed1087cc97339fc644712 Mon Sep 17 00:00:00 2001 From: James Peach Date: Thu, 1 Jun 2023 00:37:35 +1000 Subject: [PATCH 421/740] docs: fix xDS RPC variant list (#27715) In restructured text, nested lists need a blank line separator to make them render as lists rather than as paragraph content. Signed-off-by: James Peach --- docs/root/api-docs/xds_protocol.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/root/api-docs/xds_protocol.rst b/docs/root/api-docs/xds_protocol.rst index bc2a3db82cac..0ba5abb942ed 100644 --- a/docs/root/api-docs/xds_protocol.rst +++ b/docs/root/api-docs/xds_protocol.rst @@ -162,27 +162,42 @@ Each of these RPC services can provide a method for each of the SotW and Increme variants. Here are the RPC services and methods for each resource type: - Listener: Listener Discovery Service (LDS) + - SotW: ListenerDiscoveryService.StreamListeners - Incremental: ListenerDiscoveryService.DeltaListeners + - RouteConfiguration: Route Discovery Service (RDS) + - SotW: RouteDiscoveryService.StreamRoutes - Incremental: RouteDiscoveryService.DeltaRoutes + - ScopedRouteConfiguration: Scoped Route Discovery Service (SRDS) + - SotW: ScopedRouteDiscoveryService.StreamScopedRoutes - Incremental: ScopedRouteDiscoveryService.DeltaScopedRoutes + - VirtualHost: Virtual Host Discovery Service (VHDS) + - SotW: N/A - Incremental: VirtualHostDiscoveryService.DeltaVirtualHosts + - Cluster: Cluster Discovery Service (CDS) + - SotW: ClusterDiscoveryService.StreamClusters - Incremental: ClusterDiscoveryService.DeltaClusters + - ClusterLoadAssignment: Endpoint Discovery Service (EDS) + - SotW: EndpointDiscoveryService.StreamEndpoints - Incremental: EndpointDiscoveryService.DeltaEndpoints + - Secret: Secret Discovery Service (SDS) + - SotW: SecretDiscoveryService.StreamSecrets - Incremental: SecretDiscoveryService.DeltaSecrets + - Runtime: Runtime Discovery Service (RTDS) + - SotW: RuntimeDiscoveryService.StreamRuntime - Incremental: RuntimeDiscoveryService.DeltaRuntime From 644e991d2b3a6bfdfed4da7c55da957cd18db57e Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 31 May 2023 19:05:24 +0100 Subject: [PATCH 422/740] ci/examples: Add diskspace hack (github version) (#27734) Signed-off-by: Ryan Northey --- .github/workflows/envoy-verify.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/envoy-verify.yml b/.github/workflows/envoy-verify.yml index 7682fe649507..5460797233b5 100644 --- a/.github/workflows/envoy-verify.yml +++ b/.github/workflows/envoy-verify.yml @@ -38,6 +38,23 @@ jobs: runs-on: ubuntu-20.04 needs: check steps: + - run: | + echo "Disk space before cruft removal" + df -h + + TO_REMOVE=( + /opt/hostedtoolcache + /usr/local/lib/android + /usr/local/.ghcup) + + for removal in "${TO_REMOVE[@]}"; do + echo "Removing: ${removal} ..." + sudo rm -rf "$removal" + done + + echo "Disk space after cruft removal" + df -h + # Checkout the repo at provided commit - name: 'Checkout Repository' uses: actions/checkout@v3 From 702edcd142ce1cd3b9b325159bca9a51de6d4451 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Wed, 31 May 2023 14:06:02 -0400 Subject: [PATCH 423/740] matcher: Implement CEL matcher (#27527) Signed-off-by: tyxia --- CODEOWNERS | 4 + bazel/repository_locations.bzl | 6 +- changelogs/current.yaml | 4 + .../advanced/matching/matching_api.rst | 10 + source/extensions/extensions_build_config.bzl | 6 + source/extensions/extensions_metadata.yaml | 14 + .../extensions/filters/common/expr/context.h | 32 +- .../filters/common/expr/evaluator.h | 26 +- .../extensions/matching/http/cel_input/BUILD | 25 + .../matching/http/cel_input/cel_input.cc | 18 + .../matching/http/cel_input/cel_input.h | 82 +++ .../matching/input_matchers/cel_matcher/BUILD | 34 ++ .../input_matchers/cel_matcher/config.cc | 15 + .../input_matchers/cel_matcher/config.h | 53 ++ .../input_matchers/cel_matcher/matcher.cc | 54 ++ .../input_matchers/cel_matcher/matcher.h | 54 ++ .../matching/input_matchers/cel_matcher/BUILD | 36 ++ .../cel_matcher/cel_matcher_test.cc | 401 +++++++++++++ .../cel_matcher/cel_matcher_test.h | 554 ++++++++++++++++++ test/per_file_coverage.sh | 1 + tools/extensions/extensions_schema.yaml | 1 + 21 files changed, 1401 insertions(+), 29 deletions(-) create mode 100644 source/extensions/matching/http/cel_input/BUILD create mode 100644 source/extensions/matching/http/cel_input/cel_input.cc create mode 100644 source/extensions/matching/http/cel_input/cel_input.h create mode 100644 source/extensions/matching/input_matchers/cel_matcher/BUILD create mode 100644 source/extensions/matching/input_matchers/cel_matcher/config.cc create mode 100644 source/extensions/matching/input_matchers/cel_matcher/config.h create mode 100644 source/extensions/matching/input_matchers/cel_matcher/matcher.cc create mode 100644 source/extensions/matching/input_matchers/cel_matcher/matcher.h create mode 100644 test/extensions/matching/input_matchers/cel_matcher/BUILD create mode 100644 test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc create mode 100644 test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h diff --git a/CODEOWNERS b/CODEOWNERS index e9a63bb15179..33b257f22307 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -167,10 +167,14 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/matching/input_matchers/consistent_hashing @snowp @donyu # runtime fraction input matcher /*/extensions/matching/input_matchers/runtime_fraction @ravenblackx @UNOWNED +# CEL input matcher +/*/extensions/matching/input_matchers/cel_matcher @tyxia @UNOWNED # environment generic input /*/extensions/matching/common_inputs/environment @snowp @donyu # format string matching /*/extensions/matching/actions/format_string @kyessenov @UNOWNED +# CEL data input +/*/extensions/matching/http/cel_input @tyxia @UNOWNED # user space socket pair, event, connection and listener /*/extensions/io_socket/user_space @kyessenov @UNOWNED /*/extensions/bootstrap/internal_listener @kyessenov @adisuissa diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index a401e54c11af..6ea53a224d09 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1102,13 +1102,15 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.stat_sinks.wasm", "envoy.rbac.matchers.upstream_ip_port", "envoy.formatter.cel", + "envoy.matching.inputs.cel_data_input", + "envoy.matching.matchers.cel_matcher", ], release_date = "2023-03-08", cpe = "N/A", ), com_github_google_flatbuffers = dict( project_name = "FlatBuffers", - project_desc = "Cross platform serialization library architected for maximum memory efficiency", + project_desc = "FlatBuffers is a cross platform serialization library architected for maximum memory efficiency", project_url = "https://github.com/google/flatbuffers", version = "23.3.3", sha256 = "8aff985da30aaab37edf8e5b02fda33ed4cbdd962699a8e2af98fdef306f4e4d", @@ -1127,6 +1129,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", "envoy.rbac.matchers.upstream_ip_port", + "envoy.matching.inputs.cel_data_input", + "envoy.matching.matchers.cel_matcher", ], release_date = "2023-03-03", cpe = "cpe:2.3:a:google:flatbuffers:*", diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 0a5f3aadd377..3db18423ae73 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -251,6 +251,10 @@ new_features: change: | added new field ``filter_metadata ` + and :ref:`CEL input matcher `. deprecated: - area: access_log diff --git a/docs/root/intro/arch_overview/advanced/matching/matching_api.rst b/docs/root/intro/arch_overview/advanced/matching/matching_api.rst index 3566c9176fff..767261840667 100644 --- a/docs/root/intro/arch_overview/advanced/matching/matching_api.rst +++ b/docs/root/intro/arch_overview/advanced/matching/matching_api.rst @@ -88,6 +88,16 @@ are available in some contexts: * :ref:`Trie-based IP matcher ` applies to network inputs. +* `Common Expression Language `_ (CEL) based matching: + +.. _extension_envoy.matching.inputs.cel_data_input: + + * CEL matching data input: :ref:`CEL data input value `. + +.. _extension_envoy.matching.matchers.cel_matcher: + + * CEL matching input matcher: :ref:`CEL input matcher `. + Matching actions ################ diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 1cccfe090b8b..49f995c7bc32 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -79,6 +79,7 @@ EXTENSIONS = { "envoy.matching.matchers.consistent_hashing": "//source/extensions/matching/input_matchers/consistent_hashing:config", "envoy.matching.matchers.ip": "//source/extensions/matching/input_matchers/ip:config", "envoy.matching.matchers.runtime_fraction": "//source/extensions/matching/input_matchers/runtime_fraction:config", + "envoy.matching.matchers.cel_matcher": "//source/extensions/matching/input_matchers/cel_matcher:config", # # Network Matchers @@ -102,6 +103,11 @@ EXTENSIONS = { "envoy.matching.common_inputs.environment_variable": "//source/extensions/matching/common_inputs/environment_variable:config", + # + # CEL Matching Input + # + "envoy.matching.inputs.cel_data_input": "//source/extensions/matching/http/cel_input:cel_input_lib", + # # Matching actions # diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 0a19fb24bfbc..0d8f55d9fe26 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -885,6 +885,13 @@ envoy.matching.matchers.runtime_fraction: status: stable type_urls: - envoy.extensions.matching.input_matchers.runtime_fraction.v3.RuntimeFraction +envoy.matching.matchers.cel_matcher: + categories: + - envoy.matching.input_matchers + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable + type_urls: + - xds.type.matcher.v3.CelMatcher envoy.path.match.uri_template.uri_template_matcher: categories: - envoy.path.match @@ -1349,6 +1356,13 @@ envoy.matching.inputs.query_params: status: alpha type_urls: - envoy.type.matcher.v3.HttpRequestQueryParamMatchInput +envoy.matching.inputs.cel_data_input: + categories: + - envoy.matching.http.input + security_posture: unknown + status: alpha + type_urls: + - xds.type.matcher.v3.HttpAttributesCelMatchInput envoy.matching.inputs.destination_ip: categories: - envoy.matching.http.input diff --git a/source/extensions/filters/common/expr/context.h b/source/extensions/filters/common/expr/context.h index 187f8455619d..7257655ffb77 100644 --- a/source/extensions/filters/common/expr/context.h +++ b/source/extensions/filters/common/expr/context.h @@ -99,10 +99,10 @@ using WrapperFields = ConstSingleton; class RequestWrapper; -absl::optional convertHeaderEntry(const Http::HeaderEntry* header); +absl::optional convertHeaderEntry(const ::Envoy::Http::HeaderEntry* header); absl::optional convertHeaderEntry(Protobuf::Arena& arena, - Http::HeaderUtility::GetAllOfHeaderAsStringResult&& result); + ::Envoy::Http::HeaderUtility::GetAllOfHeaderAsStringResult&& result); template class HeadersWrapper : public google::api::expr::runtime::CelMap { public: @@ -112,12 +112,12 @@ template class HeadersWrapper : public google::api::expr::runtime::Cel return {}; } auto str = std::string(key.StringOrDie().value()); - if (!Http::validHeaderString(str)) { + if (!::Envoy::Http::validHeaderString(str)) { // Reject key if it is an invalid header string return {}; } - return convertHeaderEntry( - arena_, Http::HeaderUtility::getAllOfHeaderAsString(*value_, Http::LowerCaseString(str))); + return convertHeaderEntry(arena_, ::Envoy::Http::HeaderUtility::getAllOfHeaderAsString( + *value_, ::Envoy::Http::LowerCaseString(str))); } int size() const override { return ListKeys().value()->size(); } bool empty() const override { return value_ == nullptr ? true : value_->empty(); } @@ -126,10 +126,11 @@ template class HeadersWrapper : public google::api::expr::runtime::Cel return &WrapperFields::get().Empty; } absl::flat_hash_set keys; - value_->iterate([&keys](const Http::HeaderEntry& header) -> Http::HeaderMap::Iterate { - keys.insert(header.key().getStringView()); - return Http::HeaderMap::Iterate::Continue; - }); + value_->iterate( + [&keys](const ::Envoy::Http::HeaderEntry& header) -> ::Envoy::Http::HeaderMap::Iterate { + keys.insert(header.key().getStringView()); + return ::Envoy::Http::HeaderMap::Iterate::Continue; + }); std::vector values; values.reserve(keys.size()); for (const auto& key : keys) { @@ -163,26 +164,27 @@ class BaseWrapper : public google::api::expr::runtime::CelMap { class RequestWrapper : public BaseWrapper { public: - RequestWrapper(Protobuf::Arena& arena, const Http::RequestHeaderMap* headers, + RequestWrapper(Protobuf::Arena& arena, const ::Envoy::Http::RequestHeaderMap* headers, const StreamInfo::StreamInfo& info) : BaseWrapper(arena), headers_(arena, headers), info_(info) {} absl::optional operator[](CelValue key) const override; private: - const HeadersWrapper headers_; + const HeadersWrapper<::Envoy::Http::RequestHeaderMap> headers_; const StreamInfo::StreamInfo& info_; }; class ResponseWrapper : public BaseWrapper { public: - ResponseWrapper(Protobuf::Arena& arena, const Http::ResponseHeaderMap* headers, - const Http::ResponseTrailerMap* trailers, const StreamInfo::StreamInfo& info) + ResponseWrapper(Protobuf::Arena& arena, const ::Envoy::Http::ResponseHeaderMap* headers, + const ::Envoy::Http::ResponseTrailerMap* trailers, + const StreamInfo::StreamInfo& info) : BaseWrapper(arena), headers_(arena, headers), trailers_(arena, trailers), info_(info) {} absl::optional operator[](CelValue key) const override; private: - const HeadersWrapper headers_; - const HeadersWrapper trailers_; + const HeadersWrapper<::Envoy::Http::ResponseHeaderMap> headers_; + const HeadersWrapper<::Envoy::Http::ResponseTrailerMap> trailers_; const StreamInfo::StreamInfo& info_; }; diff --git a/source/extensions/filters/common/expr/evaluator.h b/source/extensions/filters/common/expr/evaluator.h index 4df508dae2ae..63ed1e1266b8 100644 --- a/source/extensions/filters/common/expr/evaluator.h +++ b/source/extensions/filters/common/expr/evaluator.h @@ -27,9 +27,9 @@ using ExpressionPtr = std::unique_ptr; class StreamActivation : public google::api::expr::runtime::BaseActivation { public: StreamActivation(const StreamInfo::StreamInfo& info, - const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers) + const ::Envoy::Http::RequestHeaderMap* request_headers, + const ::Envoy::Http::ResponseHeaderMap* response_headers, + const ::Envoy::Http::ResponseTrailerMap* response_trailers) : activation_info_(&info), activation_request_headers_(request_headers), activation_response_headers_(response_headers), activation_response_trailers_(response_trailers) {} @@ -45,17 +45,17 @@ class StreamActivation : public google::api::expr::runtime::BaseActivation { protected: void resetActivation() const; mutable const StreamInfo::StreamInfo* activation_info_{nullptr}; - mutable const Http::RequestHeaderMap* activation_request_headers_{nullptr}; - mutable const Http::ResponseHeaderMap* activation_response_headers_{nullptr}; - mutable const Http::ResponseTrailerMap* activation_response_trailers_{nullptr}; + mutable const ::Envoy::Http::RequestHeaderMap* activation_request_headers_{nullptr}; + mutable const ::Envoy::Http::ResponseHeaderMap* activation_response_headers_{nullptr}; + mutable const ::Envoy::Http::ResponseTrailerMap* activation_response_trailers_{nullptr}; }; // Creates an activation providing the common context attributes. // The activation lazily creates wrappers during an evaluation using the evaluation arena. ActivationPtr createActivation(const StreamInfo::StreamInfo& info, - const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers); + const ::Envoy::Http::RequestHeaderMap* request_headers, + const ::Envoy::Http::ResponseHeaderMap* response_headers, + const ::Envoy::Http::ResponseTrailerMap* response_trailers); // Creates an expression builder. The optional arena is used to enable constant folding // for intermediate evaluation results. @@ -70,14 +70,14 @@ ExpressionPtr createExpression(Builder& builder, const google::api::expr::v1alph // results and potentially the final value. absl::optional evaluate(const Expression& expr, Protobuf::Arena& arena, const StreamInfo::StreamInfo& info, - const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers); + const ::Envoy::Http::RequestHeaderMap* request_headers, + const ::Envoy::Http::ResponseHeaderMap* response_headers, + const ::Envoy::Http::ResponseTrailerMap* response_trailers); // Evaluates an expression and returns true if the expression evaluates to "true". // Returns false if the expression fails to evaluate. bool matches(const Expression& expr, const StreamInfo::StreamInfo& info, - const Http::RequestHeaderMap& headers); + const ::Envoy::Http::RequestHeaderMap& headers); // Returns a string for a CelValue. std::string print(CelValue value); diff --git a/source/extensions/matching/http/cel_input/BUILD b/source/extensions/matching/http/cel_input/BUILD new file mode 100644 index 000000000000..0c283b322866 --- /dev/null +++ b/source/extensions/matching/http/cel_input/BUILD @@ -0,0 +1,25 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "cel_input_lib", + srcs = ["cel_input.cc"], + hdrs = ["cel_input.h"], + extra_visibility = [ + ], + deps = [ + "//envoy/http:filter_interface", + "//envoy/http:header_map_interface", + "//source/common/http:header_utility_lib", + "//source/common/http:utility_lib", + "//source/extensions/filters/common/expr:evaluator_lib", + "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/matching/http/cel_input/cel_input.cc b/source/extensions/matching/http/cel_input/cel_input.cc new file mode 100644 index 000000000000..2e4ceeb0ad25 --- /dev/null +++ b/source/extensions/matching/http/cel_input/cel_input.cc @@ -0,0 +1,18 @@ +#include "source/extensions/matching/http/cel_input/cel_input.h" + +#include "envoy/registry/registry.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace Http { +namespace CelInput { + +REGISTER_FACTORY(HttpCelDataInputFactory, + Matcher::DataInputFactory<::Envoy::Http::HttpMatchingData>); + +} // namespace CelInput +} // namespace Http +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/matching/http/cel_input/cel_input.h b/source/extensions/matching/http/cel_input/cel_input.h new file mode 100644 index 000000000000..0e5707c4e2e4 --- /dev/null +++ b/source/extensions/matching/http/cel_input/cel_input.h @@ -0,0 +1,82 @@ +#pragma once + +#include "envoy/http/filter.h" +#include "envoy/matcher/matcher.h" +#include "envoy/server/factory_context.h" +#include "envoy/type/matcher/v3/http_inputs.pb.h" +#include "envoy/type/matcher/v3/http_inputs.pb.validate.h" + +#include "source/common/http/header_utility.h" +#include "source/common/http/utility.h" +#include "source/extensions/filters/common/expr/evaluator.h" + +#include "xds/type/matcher/v3/http_inputs.pb.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace Http { +namespace CelInput { + +using ::Envoy::Http::RequestHeaderMapOptConstRef; +using ::Envoy::Http::ResponseHeaderMapOptConstRef; +using ::Envoy::Http::ResponseTrailerMapOptConstRef; + +using BaseActivationPtr = std::unique_ptr; + +// CEL matcher specific matching data +class CelMatchData : public ::Envoy::Matcher::CustomMatchData { +public: + explicit CelMatchData(BaseActivationPtr activation) : activation_(std::move(activation)) {} + BaseActivationPtr activation_; +}; + +class HttpCelDataInput : public Matcher::DataInput { +public: + HttpCelDataInput() = default; + Matcher::DataInputGetResult get(const Envoy::Http::HttpMatchingData& data) const override { + RequestHeaderMapOptConstRef maybe_request_headers = data.requestHeaders(); + ResponseHeaderMapOptConstRef maybe_response_headers = data.responseHeaders(); + ResponseTrailerMapOptConstRef maybe_response_trailers = data.responseTrailers(); + + // Returns NotAvailable state when all of three below are empty. + if (!maybe_request_headers && !maybe_response_headers && !maybe_response_trailers) { + return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()}; + } + + // CEL library supports mixed matching condition of request headers, response headers and + // response trailers. + std::unique_ptr activation = + Extensions::Filters::Common::Expr::createActivation( + data.streamInfo(), maybe_request_headers.ptr(), maybe_response_headers.ptr(), + maybe_response_trailers.ptr()); + + return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, + std::make_unique(std::move(activation))}; + } + + absl::string_view dataInputType() const override { return "cel_data_input"; } +}; + +class HttpCelDataInputFactory : public Matcher::DataInputFactory { +public: + HttpCelDataInputFactory() = default; + std::string name() const override { return "envoy.matching.inputs.cel_data_input"; } + + Matcher::DataInputFactoryCb + createDataInputFactoryCb(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override { + return [] { return std::make_unique(); }; + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } +}; + +DECLARE_FACTORY(HttpCelDataInputFactory); + +} // namespace CelInput +} // namespace Http +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/matching/input_matchers/cel_matcher/BUILD b/source/extensions/matching/input_matchers/cel_matcher/BUILD new file mode 100644 index 000000000000..748641ef45e6 --- /dev/null +++ b/source/extensions/matching/input_matchers/cel_matcher/BUILD @@ -0,0 +1,34 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "cel_matcher_lib", + srcs = ["matcher.cc"], + hdrs = ["matcher.h"], + deps = [ + "//envoy/matcher:matcher_interface", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/common/expr:evaluator_lib", + "//source/extensions/matching/http/cel_input:cel_input_lib", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":cel_matcher_lib", + "//envoy/matcher:matcher_interface", + "//envoy/registry", + "//envoy/server:factory_context_interface", + ], +) diff --git a/source/extensions/matching/input_matchers/cel_matcher/config.cc b/source/extensions/matching/input_matchers/cel_matcher/config.cc new file mode 100644 index 000000000000..d2f961a314c7 --- /dev/null +++ b/source/extensions/matching/input_matchers/cel_matcher/config.cc @@ -0,0 +1,15 @@ +#include "source/extensions/matching/input_matchers/cel_matcher/config.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace CelMatcher { + +REGISTER_FACTORY(CelInputMatcherFactory, Envoy::Matcher::InputMatcherFactory); + +} // namespace CelMatcher +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/matching/input_matchers/cel_matcher/config.h b/source/extensions/matching/input_matchers/cel_matcher/config.h new file mode 100644 index 000000000000..d894ebe7cb8a --- /dev/null +++ b/source/extensions/matching/input_matchers/cel_matcher/config.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include + +#include "envoy/matcher/matcher.h" + +#include "source/common/protobuf/utility.h" +#include "source/extensions/matching/input_matchers/cel_matcher/matcher.h" + +#include "xds/type/matcher/v3/cel.pb.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace CelMatcher { + +class CelInputMatcherFactory : public ::Envoy::Matcher::InputMatcherFactory { +public: + InputMatcherFactoryCb + createInputMatcherFactoryCb(const Protobuf::Message& config, + Server::Configuration::ServerFactoryContext&) override { + if (expr_builder_ == nullptr) { + expr_builder_ = Extensions::Filters::Common::Expr::createBuilder(nullptr); + } + + const auto& cel_matcher_config = + dynamic_cast(config); + CelMatcherSharedPtr cel_matcher = + std::make_shared<::xds::type::matcher::v3::CelMatcher>(cel_matcher_config); + + return [cel_matcher = std::move(cel_matcher), this] { + return std::make_unique(cel_matcher, *expr_builder_); + }; + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return "envoy.matching.matchers.cel_matcher"; } + +private: + // Expression builder must outlive the compiled expression. + BuilderPtr expr_builder_; +}; + +} // namespace CelMatcher +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/matching/input_matchers/cel_matcher/matcher.cc b/source/extensions/matching/input_matchers/cel_matcher/matcher.cc new file mode 100644 index 000000000000..d6d61bc81142 --- /dev/null +++ b/source/extensions/matching/input_matchers/cel_matcher/matcher.cc @@ -0,0 +1,54 @@ +#include "source/extensions/matching/input_matchers/cel_matcher/matcher.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace CelMatcher { + +using ::Envoy::Extensions::Matching::Http::CelInput::CelMatchData; +using ::xds::type::v3::CelExpression; + +CelInputMatcher::CelInputMatcher(CelMatcherSharedPtr cel_matcher, Builder& builder) + : cel_matcher_(std::move(cel_matcher)) { + const CelExpression& input_expr = cel_matcher_->expr_match(); + switch (input_expr.expr_specifier_case()) { + case CelExpression::ExprSpecifierCase::kParsedExpr: + compiled_expr_ = + Filters::Common::Expr::createExpression(builder, input_expr.parsed_expr().expr()); + return; + case CelExpression::ExprSpecifierCase::kCheckedExpr: + compiled_expr_ = + Filters::Common::Expr::createExpression(builder, input_expr.checked_expr().expr()); + return; + case CelExpression::ExprSpecifierCase::EXPR_SPECIFIER_NOT_SET: + PANIC_DUE_TO_PROTO_UNSET; + } + PANIC_DUE_TO_CORRUPT_ENUM; +} + +bool CelInputMatcher::match(const MatchingDataType& input) { + Protobuf::Arena arena; + if (auto* ptr = absl::get_if>(&input); + ptr != nullptr) { + CelMatchData* cel_data = dynamic_cast((*ptr).get()); + // Compiled expression should not be nullptr at this point because the program should have + // encountered a panic in the constructor earlier if any such error cases occurred. CEL matching + // data should also not be nullptr since any errors should have been thrown by the CEL library + // already. + ASSERT(compiled_expr_ != nullptr && cel_data != nullptr); + + auto eval_result = compiled_expr_->Evaluate(*cel_data->activation_, &arena); + if (eval_result.ok() && eval_result.value().IsBool()) { + return eval_result.value().BoolOrDie(); + } + } + + return false; +} + +} // namespace CelMatcher +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/matching/input_matchers/cel_matcher/matcher.h b/source/extensions/matching/input_matchers/cel_matcher/matcher.h new file mode 100644 index 000000000000..5ac9e366c675 --- /dev/null +++ b/source/extensions/matching/input_matchers/cel_matcher/matcher.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + +#include "envoy/matcher/matcher.h" + +#include "source/common/protobuf/utility.h" +#include "source/extensions/filters/common/expr/evaluator.h" +#include "source/extensions/matching/http/cel_input/cel_input.h" + +#include "absl/types/variant.h" +#include "xds/type/matcher/v3/cel.pb.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace CelMatcher { + +using ::Envoy::Matcher::InputMatcher; +using ::Envoy::Matcher::InputMatcherFactoryCb; +using ::Envoy::Matcher::MatchingDataType; + +using CelMatcher = ::xds::type::matcher::v3::CelMatcher; +using CompiledExpressionPtr = std::unique_ptr; +using BaseActivationPtr = std::unique_ptr; +using Builder = google::api::expr::runtime::CelExpressionBuilder; +using BuilderPtr = std::unique_ptr; +using CelMatcherSharedPtr = std::shared_ptr<::xds::type::matcher::v3::CelMatcher>; + +class CelInputMatcher : public InputMatcher, public Logger::Loggable { +public: + CelInputMatcher(CelMatcherSharedPtr cel_matcher, Builder& builder); + + bool match(const MatchingDataType& input) override; + + // TODO(tyxia) Formalize the validation approach. Use fixed string for now. + absl::flat_hash_set supportedDataInputTypes() const override { + return absl::flat_hash_set{"cel_data_input"}; + } + +private: + // Expression proto must outlive the compiled expression. + CelMatcherSharedPtr cel_matcher_; + CompiledExpressionPtr compiled_expr_; +}; + +} // namespace CelMatcher +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/matching/input_matchers/cel_matcher/BUILD b/test/extensions/matching/input_matchers/cel_matcher/BUILD new file mode 100644 index 000000000000..bd175a3dea4b --- /dev/null +++ b/test/extensions/matching/input_matchers/cel_matcher/BUILD @@ -0,0 +1,36 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "cel_matcher_test", + srcs = [ + "cel_matcher_test.cc", + "cel_matcher_test.h", + ], + extension_names = ["envoy.matching.matchers.cel_matcher"], + deps = [ + "//source/common/matcher:matcher_lib", + "//source/extensions/matching/http/cel_input:cel_input_lib", + "//source/extensions/matching/input_matchers/cel_matcher:cel_matcher_lib", + "//source/extensions/matching/input_matchers/cel_matcher:config", + "//test/common/matcher:test_utility_lib", + "//test/mocks/http:http_mocks", + "//test/mocks/matcher:matcher_mocks", + "//test/mocks/server:factory_context_mocks", + "//test/mocks/stream_info:stream_info_mocks", + "//test/test_common:registry_lib", + "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto", + "@envoy_api//envoy/config/common/matcher/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc new file mode 100644 index 000000000000..f1d3378a800d --- /dev/null +++ b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc @@ -0,0 +1,401 @@ +#include "test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h" + +#include +#include + +#include "envoy/config/common/matcher/v3/matcher.pb.validate.h" +#include "envoy/config/core/v3/extension.pb.h" +#include "envoy/matcher/matcher.h" +#include "envoy/registry/registry.h" + +#include "source/common/matcher/matcher.h" +#include "source/common/protobuf/utility.h" +#include "source/extensions/matching/http/cel_input/cel_input.h" +#include "source/extensions/matching/input_matchers/cel_matcher/config.h" +#include "source/extensions/matching/input_matchers/cel_matcher/matcher.h" + +#include "test/common/matcher/test_utility.h" +#include "test/mocks/matcher/mocks.h" +#include "test/mocks/server/factory_context.h" +#include "test/test_common/registry.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" +#include "xds/type/matcher/v3/matcher.pb.validate.h" + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace CelMatcher { + +using ::Envoy::Http::LowerCaseString; +using ::Envoy::Http::TestRequestHeaderMapImpl; +using ::Envoy::Http::TestResponseHeaderMapImpl; +using ::Envoy::Http::TestResponseTrailerMapImpl; + +enum class ExpressionType { + CheckedExpression = 0, + ParsedExpression = 1, + NoExpression = 2, +}; + +class CelMatcherTest : public ::testing::Test { +public: + CelMatcherTest() + : inject_action_(action_factory_), + data_(Envoy::Http::Matching::HttpMatchingDataImpl(stream_info_)) {} + + void buildCustomHeader(const absl::flat_hash_map& custom_value_pairs, + TestRequestHeaderMapImpl& headers) { + // Add custom_value_pairs to the request header. + for (auto const& pair : custom_value_pairs) { + headers.setCopy(LowerCaseString(pair.first), pair.second); + } + } + + Matcher::MatchTreeSharedPtr + buildMatcherTree(const std::string& cel_expr_config, + ExpressionType expr_type = ExpressionType::CheckedExpression) { + xds::type::matcher::v3::CelMatcher cel_matcher; + switch (expr_type) { + case ExpressionType::CheckedExpression: { + google::api::expr::v1alpha1::CheckedExpr checked_expr; + Protobuf::TextFormat::ParseFromString(cel_expr_config, &checked_expr); + cel_matcher.mutable_expr_match()->mutable_checked_expr()->MergeFrom(checked_expr); + break; + } + case ExpressionType::ParsedExpression: { + google::api::expr::v1alpha1::ParsedExpr parsed_expr; + Protobuf::TextFormat::ParseFromString(cel_expr_config, &parsed_expr); + cel_matcher.mutable_expr_match()->mutable_parsed_expr()->MergeFrom(parsed_expr); + break; + } + case ExpressionType::NoExpression: + break; + } + + xds::type::matcher::v3::Matcher matcher; + auto* inner_matcher = matcher.mutable_matcher_list()->add_matchers(); + auto* single_predicate = inner_matcher->mutable_predicate()->mutable_single_predicate(); + + xds::type::matcher::v3::HttpAttributesCelMatchInput cel_match_input; + single_predicate->mutable_input()->set_name("envoy.matching.inputs.cel_data_input"); + single_predicate->mutable_input()->mutable_typed_config()->PackFrom(cel_match_input); + + auto* custom_matcher = single_predicate->mutable_custom_match(); + custom_matcher->mutable_typed_config()->PackFrom(cel_matcher); + + xds::type::matcher::v3::Matcher::OnMatch on_match; + std::string on_match_config = R"EOF( + action: + name: test_action + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + value: match!! + )EOF"; + MessageUtil::loadFromYaml(on_match_config, on_match, + ProtobufMessage::getStrictValidationVisitor()); + + inner_matcher->mutable_on_match()->MergeFrom(on_match); + + auto string_factory_on_match = Matcher::TestDataInputStringFactory("value"); + + Matcher::MockMatchTreeValidationVisitor validation_visitor; + EXPECT_CALL(validation_visitor, + performDataInputValidation( + _, "type.googleapis.com/xds.type.matcher.v3.HttpAttributesCelMatchInput")); + Matcher::MatchTreeFactory matcher_factory( + context_, factory_context_, validation_visitor); + auto match_tree = matcher_factory.create(matcher); + + return match_tree(); + } + + Matcher::StringActionFactory action_factory_; + Registry::InjectFactory> inject_action_; + testing::NiceMock stream_info_; + absl::string_view context_ = ""; + testing::NiceMock factory_context_; + + TestRequestHeaderMapImpl default_headers_{ + {":method", "GET"}, {":scheme", "http"}, {":authority", "host"}}; + + Envoy::Http::Matching::HttpMatchingDataImpl data_; +}; + +TEST_F(CelMatcherTest, CelMatcherRequestHeaderMatched) { + auto matcher_tree = buildMatcherTree(RequestHeaderCelExprString); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"authenticated_user", "staging"}}, request_headers); + data_.onRequestHeaders(request_headers); + // data.onRequestHeaders(request_headers); + + const auto result = matcher_tree->match(data_); + // The match was complete, match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_TRUE(result.on_match_.has_value()); + EXPECT_NE(result.on_match_->action_cb_, nullptr); +} + +TEST_F(CelMatcherTest, CelMatcherRequestHeaderNotMatched) { + auto matcher_tree = buildMatcherTree(RequestHeaderCelExprString); + + // Build header with request header value field mismatched case. + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"authenticated_user", "NOT_MATCHED"}}, request_headers); + data_.onRequestHeaders(request_headers); + + const auto result = matcher_tree->match(data_); + // The match was completed, no match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_EQ(result.on_match_, absl::nullopt); + + // Build header with request header key field mismatched case. + TestRequestHeaderMapImpl request_headers_2 = default_headers_; + buildCustomHeader({{"NOT_MATCHED", "staging"}}, request_headers_2); + Envoy::Http::Matching::HttpMatchingDataImpl data_2 = + Envoy::Http::Matching::HttpMatchingDataImpl(stream_info_); + data_2.onRequestHeaders(request_headers_2); + const auto result_2 = matcher_tree->match(data_2); + // The match was completed, no match found. + EXPECT_EQ(result_2.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_EQ(result_2.on_match_, absl::nullopt); +} + +TEST_F(CelMatcherTest, CelMatcherNoRequestAttributes) { + auto matcher_tree = buildMatcherTree(RequestHeaderCelExprString); + + // No request attributes added to matching data. + const auto result = matcher_tree->match(data_); + // The match was completed, no match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::UnableToMatch); + EXPECT_EQ(result.on_match_, absl::nullopt); +} + +TEST_F(CelMatcherTest, CelMatcherRequestHeaderPathMatched) { + auto matcher_tree = buildMatcherTree(RequestPathCelExprString); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{":path", "/foo"}}, request_headers); + data_.onRequestHeaders(request_headers); + + const auto result = matcher_tree->match(data_); + // The match was complete, match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_TRUE(result.on_match_.has_value()); + EXPECT_NE(result.on_match_->action_cb_, nullptr); +} + +TEST_F(CelMatcherTest, CelMatcherRequestHeaderPathNotMatched) { + auto matcher_tree = buildMatcherTree(RequestPathCelExprString); + + TestRequestHeaderMapImpl request_headers = default_headers_; + // The matching condition is: request.path == '/foo'. + buildCustomHeader({{":path", "/bar"}}, request_headers); + data_.onRequestHeaders(request_headers); + + const auto result = matcher_tree->match(data_); + // The match was completed, no match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_EQ(result.on_match_, absl::nullopt); +} + +TEST_F(CelMatcherTest, CelMatcherResponseHeaderMatched) { + auto matcher_tree = buildMatcherTree(ReponseHeaderCelExprString); + + TestResponseHeaderMapImpl response_headers; + response_headers.addCopy(LowerCaseString(":status"), "200"); + response_headers.addCopy(LowerCaseString("content-type"), "text/plain"); + response_headers.addCopy(LowerCaseString("content-length"), "3"); + data_.onResponseHeaders(response_headers); + + const auto result = matcher_tree->match(data_); + // The match was complete, match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_TRUE(result.on_match_.has_value()); + EXPECT_NE(result.on_match_->action_cb_, nullptr); +} + +TEST_F(CelMatcherTest, CelMatcherResponseHeaderNotMatched) { + auto matcher_tree = buildMatcherTree(ReponseHeaderCelExprString); + + TestResponseHeaderMapImpl response_headers = {{"content-type", "text/html"}}; + data_.onResponseHeaders(response_headers); + + const auto result = matcher_tree->match(data_); + // The match was completed, no match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_EQ(result.on_match_, absl::nullopt); +} + +TEST_F(CelMatcherTest, CelMatcherResponseTrailerMatched) { + auto matcher_tree = buildMatcherTree(ReponseTrailerCelExprString); + + TestResponseTrailerMapImpl response_trailers = {{"transfer-encoding", "chunked"}}; + data_.onResponseTrailers(response_trailers); + + const auto result = matcher_tree->match(data_); + // The match was complete, match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_TRUE(result.on_match_.has_value()); + EXPECT_NE(result.on_match_->action_cb_, nullptr); +} + +TEST_F(CelMatcherTest, CelMatcherResponseTrailerNotMatched) { + auto matcher_tree = buildMatcherTree(ReponseTrailerCelExprString); + + TestResponseTrailerMapImpl response_trailers = {{"transfer-encoding", "chunked_not_matched"}}; + data_.onResponseTrailers(response_trailers); + + const auto result = matcher_tree->match(data_); + // The match was completed, no match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_EQ(result.on_match_, absl::nullopt); +} + +TEST_F(CelMatcherTest, CelMatcherRequestHeaderAndPathMatched) { + auto matcher_tree = buildMatcherTree(RequestHeaderAndPathCelString); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"user", "staging"}, {":path", "/foo"}}, request_headers); + data_.onRequestHeaders(request_headers); + + const auto result = matcher_tree->match(data_); + // The match was complete, match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_TRUE(result.on_match_.has_value()); + EXPECT_NE(result.on_match_->action_cb_, nullptr); +} + +TEST_F(CelMatcherTest, CelMatcherRequestHeaderAndPathNotMatched) { + auto matcher_tree = buildMatcherTree(RequestHeaderAndPathCelString); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"user", "prod"}, {":path", "/foo"}}, request_headers); + data_.onRequestHeaders(request_headers); + + const auto result = matcher_tree->match(data_); + // The match was completed, no match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_EQ(result.on_match_, absl::nullopt); +} + +TEST_F(CelMatcherTest, CelMatcherRequestHeaderOrPathMatched) { + auto matcher_tree = buildMatcherTree(RequestHeaderOrPathCelString); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"user", "prod"}, {":path", "/foo"}}, request_headers); + data_.onRequestHeaders(request_headers); + + const auto result = matcher_tree->match(data_); + // The match was complete, match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_TRUE(result.on_match_.has_value()); + EXPECT_NE(result.on_match_->action_cb_, nullptr); +} + +TEST_F(CelMatcherTest, CelMatcherRequestHeaderOrPathNotMatched) { + auto matcher_tree = buildMatcherTree(RequestHeaderOrPathCelString); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"user", "prod"}, {":path", "/bar"}}, request_headers); + data_.onRequestHeaders(request_headers); + + const auto result = matcher_tree->match(data_); + // The match was completed, no match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_EQ(result.on_match_, absl::nullopt); +} + +TEST_F(CelMatcherTest, CelMatcherRequestResponseMatched) { + auto matcher_tree = buildMatcherTree(RequestAndResponseCelString); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"user", "staging"}}, request_headers); + data_.onRequestHeaders(request_headers); + + TestResponseHeaderMapImpl response_headers = {{"content-type", "text/plain"}}; + data_.onResponseHeaders(response_headers); + + TestResponseTrailerMapImpl response_trailers = {{"transfer-encoding", "chunked"}}; + data_.onResponseTrailers(response_trailers); + + const auto result = matcher_tree->match(data_); + // The match was complete, match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_TRUE(result.on_match_.has_value()); + EXPECT_NE(result.on_match_->action_cb_, nullptr); +} + +TEST_F(CelMatcherTest, CelMatcherRequestResponseNotMatched) { + auto matcher_tree = buildMatcherTree(RequestAndResponseCelString); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"user", "staging"}}, request_headers); + data_.onRequestHeaders(request_headers); + + TestResponseHeaderMapImpl response_headers = {{"content-type", "text/html"}}; + data_.onResponseHeaders(response_headers); + + TestResponseTrailerMapImpl response_trailers = {{"transfer-encoding", "chunked"}}; + data_.onResponseTrailers(response_trailers); + + const auto result = matcher_tree->match(data_); + // The match was completed, no match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_EQ(result.on_match_, absl::nullopt); +} + +TEST_F(CelMatcherTest, CelMatcherRequestResponseMatchedWithParsedExpr) { + auto matcher_tree = + buildMatcherTree(RequestAndResponseCelString, ExpressionType::ParsedExpression); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"user", "staging"}}, request_headers); + data_.onRequestHeaders(request_headers); + + TestResponseHeaderMapImpl response_headers = {{"content-type", "text/plain"}}; + data_.onResponseHeaders(response_headers); + + TestResponseTrailerMapImpl response_trailers = {{"transfer-encoding", "chunked"}}; + data_.onResponseTrailers(response_trailers); + + const auto result = matcher_tree->match(data_); + // The match was complete, match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_TRUE(result.on_match_.has_value()); + EXPECT_NE(result.on_match_->action_cb_, nullptr); +} + +TEST_F(CelMatcherTest, CelMatcherRequestResponseNotMatchedWithParsedExpr) { + auto matcher_tree = + buildMatcherTree(RequestAndResponseCelString, ExpressionType::ParsedExpression); + + TestRequestHeaderMapImpl request_headers = default_headers_; + buildCustomHeader({{"user", "staging"}}, request_headers); + data_.onRequestHeaders(request_headers); + + TestResponseHeaderMapImpl response_headers = {{"content-type", "text/html"}}; + data_.onResponseHeaders(response_headers); + + TestResponseTrailerMapImpl response_trailers = {{"transfer-encoding", "chunked"}}; + data_.onResponseTrailers(response_trailers); + + const auto result = matcher_tree->match(data_); + // The match was completed, no match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_EQ(result.on_match_, absl::nullopt); +} + +TEST_F(CelMatcherTest, NoCelExpression) { + EXPECT_DEATH(buildMatcherTree(RequestHeaderCelExprString, ExpressionType::NoExpression), + ".*panic: unset oneof.*"); +} + +} // namespace CelMatcher +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h new file mode 100644 index 000000000000..efadd414fe34 --- /dev/null +++ b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h @@ -0,0 +1,554 @@ +#pragma once + +namespace Envoy { +namespace Extensions { +namespace Matching { +namespace InputMatchers { +namespace CelMatcher { + +// Compiled CEL expression string: request.headers['authenticated_user'] == 'staging' +inline constexpr char RequestHeaderCelExprString[] = R"pb( + expr { + id: 8 + call_expr { + function: "_==_" + args { + id: 6 + call_expr { + function: "_[_]" + args { + id: 5 + select_expr { + operand { + id: 4 + ident_expr {name: "request"} + } + field: "headers" + } + } + args { + id: 7 + const_expr { + string_value: "authenticated_user" + } + } + } + } + args { + id: 9 + const_expr { string_value: "staging" } + } + } + } +)pb"; + +// Compiled CEL expression string: request.path == '/foo' +inline constexpr char RequestPathCelExprString[] = R"pb( + expr { + id: 3 + call_expr { + function: "_==_" + args { + id: 2 + select_expr { + operand { + id: 1 + ident_expr { + name: "request" + } + } + field: "path" + } + } + args { + id: 4 + const_expr { + string_value: "/foo" + } + } + } + } +)pb"; + +// Compiled CEL expression string: response.headers['content-type'] == 'text/plain' +inline constexpr char ReponseHeaderCelExprString[] = R"pb( + expr { + id: 8 + call_expr { + function: "_==_" + args { + id: 6 + call_expr { + function: "_[_]" + args { + id: 5 + select_expr { + operand { + id: 4 + ident_expr {name: "response"} + } + field: "headers" + } + } + args { + id: 7 + const_expr { + string_value: "content-type" + } + } + } + } + args { + id: 9 + const_expr { string_value: "text/plain" } + } + } + } +)pb"; + +// Compiled CEL expression string: request.path == '/foo' && request.headers['user'] == +// 'staging'. +// Note, source_info is not required for evaluation process that happens in data plane, which is +// confirmed by test cases above. However, it will be included as part of result from parsing and +// checking process passed to data plane. Thus, the entire compiled CEL result is also tested. +inline constexpr char RequestHeaderAndPathCelString[] = R"pb( + expr { + id: 11 + call_expr { + function: "_&&_" + args { + id: 3 + call_expr { + function: "_==_" + args { + id: 2 + select_expr { + operand { + id: 1 + ident_expr { + name: "request" + } + } + field: "path" + } + } + args { + id: 4 + const_expr { + string_value: "/foo" + } + } + } + } + args { + id: 9 + call_expr { + function: "_==_" + args { + id: 7 + call_expr { + function: "_[_]" + args { + id: 6 + select_expr { + operand { + id: 5 + ident_expr { + name: "request" + } + } + field: "headers" + } + } + args { + id: 8 + const_expr { + string_value: "user" + } + } + } + } + args { + id: 10 + const_expr { + string_value: "staging" + } + } + } + } + } + } + source_info { + location: "" + line_offsets: 57 + positions { + key: 1 + value: 0 + } + positions { + key: 2 + value: 7 + } + positions { + key: 3 + value: 12 + } + positions { + key: 4 + value: 14 + } + positions { + key: 5 + value: 22 + } + positions { + key: 6 + value: 29 + } + positions { + key: 7 + value: 37 + } + positions { + key: 8 + value: 38 + } + positions { + key: 9 + value: 45 + } + positions { + key: 10 + value: 47 + } + positions { + key: 11 + value: 20 + } + } +)pb"; + +// Compiled CEL expression string: response.trailers['transfer-encoding']=='chunked' +inline constexpr char ReponseTrailerCelExprString[] = R"pb( + expr { + id: 5 + call_expr { + function: "_==_" + args { + id: 3 + call_expr { + function: "_[_]" + args { + id: 2 + select_expr { + operand { + id: 1 + ident_expr { + name: "response" + } + } + field: "trailers" + } + } + args { + id: 4 + const_expr { + string_value: "transfer-encoding" + } + } + } + } + args { + id: 6 + const_expr { + string_value: "chunked" + } + } + } + } +)pb"; + +// Compiled CEL expression string: request.path == '/foo' || request.headers['user'] == +// 'staging'. +inline constexpr char RequestHeaderOrPathCelString[] = R"pb( + expr { + id: 11 + call_expr { + function: "_||_" + args { + id: 3 + call_expr { + function: "_==_" + args { + id: 2 + select_expr { + operand { + id: 1 + ident_expr { + name: "request" + } + } + field: "path" + } + } + args { + id: 4 + const_expr { + string_value: "/foo" + } + } + } + } + args { + id: 9 + call_expr { + function: "_==_" + args { + id: 7 + call_expr { + function: "_[_]" + args { + id: 6 + select_expr { + operand { + id: 5 + ident_expr { + name: "request" + } + } + field: "headers" + } + } + args { + id: 8 + const_expr { + string_value: "user" + } + } + } + } + args { + id: 10 + const_expr { + string_value: "staging" + } + } + } + } + } + } +)pb"; + +// request.headers['user']=='staging'&&response.headers['content-type']=='text/plain'&&response.trailers['transfer-encoding']=='chunked' +inline constexpr char RequestAndResponseCelString[] = R"pb( + expr { + id: 20 + call_expr { + function: "_&&_" + args { + id: 13 + call_expr { + function: "_&&_" + args { + id: 5 + call_expr { + function: "_==_" + args { + id: 3 + call_expr { + function: "_[_]" + args { + id: 2 + select_expr { + operand { + id: 1 + ident_expr { + name: "request" + } + } + field: "headers" + } + } + args { + id: 4 + const_expr { + string_value: "user" + } + } + } + } + args { + id: 6 + const_expr { + string_value: "staging" + } + } + } + } + args { + id: 11 + call_expr { + function: "_==_" + args { + id: 9 + call_expr { + function: "_[_]" + args { + id: 8 + select_expr { + operand { + id: 7 + ident_expr { + name: "response" + } + } + field: "headers" + } + } + args { + id: 10 + const_expr { + string_value: "content-type" + } + } + } + } + args { + id: 12 + const_expr { + string_value: "text/plain" + } + } + } + } + } + } + args { + id: 18 + call_expr { + function: "_==_" + args { + id: 16 + call_expr { + function: "_[_]" + args { + id: 15 + select_expr { + operand { + id: 14 + ident_expr { + name: "response" + } + } + field: "trailers" + } + } + args { + id: 17 + const_expr { + string_value: "transfer-encoding" + } + } + } + } + args { + id: 19 + const_expr { + string_value: "chunked" + } + } + } + } + } + } + source_info { + location: "" + line_offsets: 134 + positions { + key: 1 + value: 0 + } + positions { + key: 2 + value: 7 + } + positions { + key: 3 + value: 15 + } + positions { + key: 4 + value: 16 + } + positions { + key: 5 + value: 23 + } + positions { + key: 6 + value: 25 + } + positions { + key: 7 + value: 36 + } + positions { + key: 8 + value: 44 + } + positions { + key: 9 + value: 52 + } + positions { + key: 10 + value: 53 + } + positions { + key: 11 + value: 68 + } + positions { + key: 12 + value: 70 + } + positions { + key: 13 + value: 34 + } + positions { + key: 14 + value: 84 + } + positions { + key: 15 + value: 92 + } + positions { + key: 16 + value: 101 + } + positions { + key: 17 + value: 102 + } + positions { + key: 18 + value: 122 + } + positions { + key: 19 + value: 124 + } + positions { + key: 20 + value: 82 + } + } +)pb"; + +} // namespace CelMatcher +} // namespace InputMatchers +} // namespace Matching +} // namespace Extensions +} // namespace Envoy diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 3c31ef8fed8a..e26c99615b49 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -83,6 +83,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/config_subscription/rest:94.3" "source/extensions/config_subscription:94.8" "source/extensions/config_subscription/grpc:94.0" +"source/extensions/matching/input_matchers/cel_matcher:90.7" #Death tests don't report LCOV ) [[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" diff --git a/tools/extensions/extensions_schema.yaml b/tools/extensions/extensions_schema.yaml index 030939c667b9..6262a972cd70 100644 --- a/tools/extensions/extensions_schema.yaml +++ b/tools/extensions/extensions_schema.yaml @@ -11,6 +11,7 @@ builtin: - envoy.matching.inputs.response_headers - envoy.matching.inputs.response_trailers - envoy.matching.inputs.query_params +- envoy.matching.inputs.cel_data_input - envoy.matching.inputs.destination_ip - envoy.matching.inputs.destination_port - envoy.matching.inputs.source_ip From 7e6031d2a16210660b01531b54bc2dc14b1b6e61 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 31 May 2023 19:56:34 +0100 Subject: [PATCH 424/740] ci/mobile: Set build images in one place (#27733) Signed-off-by: Ryan Northey --- .github/workflows/env.yml | 49 +++++++++++++++------- .github/workflows/mobile-android_build.yml | 2 +- .github/workflows/mobile-android_tests.yml | 2 +- .github/workflows/mobile-asan.yml | 2 +- .github/workflows/mobile-cc_tests.yml | 2 +- .github/workflows/mobile-core.yml | 8 +++- .github/workflows/mobile-coverage.yml | 2 +- .github/workflows/mobile-docs.yml | 8 +++- .github/workflows/mobile-format.yml | 2 +- .github/workflows/mobile-perf.yml | 6 +-- .github/workflows/mobile-tsan.yml | 2 +- .github/workflows/mobile_release.yml | 2 +- 12 files changed, 60 insertions(+), 27 deletions(-) diff --git a/.github/workflows/env.yml b/.github/workflows/env.yml index a0caaa67cfb6..27149302a3d6 100644 --- a/.github/workflows/env.yml +++ b/.github/workflows/env.yml @@ -3,42 +3,48 @@ name: Environment on: workflow_call: outputs: + build_image_ubuntu: + value: ${{ jobs.repo.outputs.build_image_ubuntu }} + build_image_ubuntu_mobile: + value: ${{ jobs.repo.outputs.build_image_ubuntu_mobile }} mobile_android_build: - value: ${{ jobs.mobile.outputs.mobile_android_build }} + value: ${{ jobs.repo.outputs.mobile_android_build }} mobile_android_build_all: - value: ${{ jobs.mobile.outputs.mobile_android_build_all }} + value: ${{ jobs.repo.outputs.mobile_android_build_all }} mobile_android_tests: - value: ${{ jobs.mobile.outputs.mobile_android_tests }} + value: ${{ jobs.repo.outputs.mobile_android_tests }} mobile_asan: - value: ${{ jobs.mobile.outputs.mobile_asan }} + value: ${{ jobs.repo.outputs.mobile_asan }} mobile_cc_tests: - value: ${{ jobs.mobile.outputs.mobile_cc_tests }} + value: ${{ jobs.repo.outputs.mobile_cc_tests }} mobile_compile_time_options: - value: ${{ jobs.mobile.outputs.mobile_compile_time_options }} + value: ${{ jobs.repo.outputs.mobile_compile_time_options }} mobile_coverage: - value: ${{ jobs.mobile.outputs.mobile_coverage }} + value: ${{ jobs.repo.outputs.mobile_coverage }} mobile_formatting: - value: ${{ jobs.mobile.outputs.mobile_formatting }} + value: ${{ jobs.repo.outputs.mobile_formatting }} mobile_ios_build: - value: ${{ jobs.mobile.outputs.mobile_ios_build }} + value: ${{ jobs.repo.outputs.mobile_ios_build }} mobile_ios_build_all: - value: ${{ jobs.mobile.outputs.mobile_ios_build_all }} + value: ${{ jobs.repo.outputs.mobile_ios_build_all }} mobile_ios_tests: - value: ${{ jobs.mobile.outputs.mobile_ios_tests }} + value: ${{ jobs.repo.outputs.mobile_ios_tests }} mobile_release_validation: - value: ${{ jobs.mobile.outputs.mobile_release_validation }} + value: ${{ jobs.repo.outputs.mobile_release_validation }} mobile_tsan: - value: ${{ jobs.mobile.outputs.mobile_tsan }} + value: ${{ jobs.repo.outputs.mobile_tsan }} concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }}-env cancel-in-progress: true jobs: - mobile: + repo: if: github.repository == 'envoyproxy/envoy' runs-on: ubuntu-20.04 outputs: + build_image_ubuntu: ${{ steps.build_image.outputs.build_image_ubuntu }} + build_image_ubuntu_mobile: ${{ steps.build_image.outputs.build_image_ubuntu_mobile }} mobile_android_build: ${{ steps.should_run.outputs.mobile_android_build }} mobile_android_build_all: ${{ steps.should_run.outputs.mobile_android_build_all }} mobile_android_tests: ${{ steps.should_run.outputs.mobile_android_tests }} @@ -58,6 +64,21 @@ jobs: fetch-depth: 0 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy + + - id: build_image + name: 'Check current build images' + run: | + { + echo "build_image_ubuntu=${BUILD_IMAGE_UBUNTU_REPO}:${BUILD_IMAGE_UBUNTU}@sha256:${BUILD_IMAGE_UBUNTU_SHA}" + echo "build_image_ubuntu_mobile=${BUILD_IMAGE_UBUNTU_REPO}:mobile-${BUILD_IMAGE_UBUNTU}@sha256:${BUILD_IMAGE_UBUNTU_MOBILE_SHA}" + } >> "$GITHUB_OUTPUT" + env: + # TODO(phlax): derive these from a config file + BUILD_IMAGE_UBUNTU_REPO: envoyproxy/envoy-build-ubuntu + BUILD_IMAGE_UBUNTU: 41c5a05d708972d703661b702a63ef5060125c33 + BUILD_IMAGE_UBUNTU_SHA: 50337314a150ed12447c87c1622eac6f611a069888722fb9a426e21ed161cc26 + BUILD_IMAGE_UBUNTU_MOBILE_SHA: ca26ff05bd3f3a09468242faaf38ae48315e57f0a87c102352162f95ac620e6f + - id: should_run name: 'Check what to run' run: | diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml index 8f5c71a7d978..f791676c9084 100644 --- a/.github/workflows/mobile-android_build.yml +++ b/.github/workflows/mobile-android_build.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml index 9baf8b9922a7..93129f5b9e86 100644 --- a/.github/workflows/mobile-android_tests.yml +++ b/.github/workflows/mobile-android_tests.yml @@ -83,7 +83,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-asan.yml b/.github/workflows/mobile-asan.yml index e5a9378c262e..e6e72fcb3967 100644 --- a/.github/workflows/mobile-asan.yml +++ b/.github/workflows/mobile-asan.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 180 container: - image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile-cc_tests.yml b/.github/workflows/mobile-cc_tests.yml index c824fc8f6ece..99e1951fde1b 100644 --- a/.github/workflows/mobile-cc_tests.yml +++ b/.github/workflows/mobile-cc_tests.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu }} steps: - uses: actions/checkout@v3 - name: Add safe directory diff --git a/.github/workflows/mobile-core.yml b/.github/workflows/mobile-core.yml index 7c111e04a88b..31383adf4752 100644 --- a/.github/workflows/mobile-core.yml +++ b/.github/workflows/mobile-core.yml @@ -11,13 +11,19 @@ concurrency: cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + unittests: if: ${{ github.repository == 'envoyproxy/envoy' }} + needs: env name: unit_tests runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu }} steps: - uses: actions/checkout@v3 - name: Ensure no listener leaks diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml index 8ff027449a9e..f628f2d62134 100644 --- a/.github/workflows/mobile-coverage.yml +++ b/.github/workflows/mobile-coverage.yml @@ -26,7 +26,7 @@ jobs: run: shell: bash container: - image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu }} steps: - uses: actions/checkout@v3 - name: Add safe directory diff --git a/.github/workflows/mobile-docs.yml b/.github/workflows/mobile-docs.yml index 093b5f29740c..31554d189288 100644 --- a/.github/workflows/mobile-docs.yml +++ b/.github/workflows/mobile-docs.yml @@ -11,12 +11,18 @@ concurrency: cancel-in-progress: true jobs: + env: + if: ${{ github.repository == 'envoyproxy/envoy' }} + uses: ./.github/workflows/env.yml + secrets: inherit + docs: if: ${{ github.repository == 'envoyproxy/envoy' }} + needs: env runs-on: ubuntu-20.04 timeout-minutes: 20 container: - image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu }} steps: - uses: actions/checkout@v3 - name: Add safe directory diff --git a/.github/workflows/mobile-format.yml b/.github/workflows/mobile-format.yml index ac48d4308063..54b57cddbcf4 100644 --- a/.github/workflows/mobile-format.yml +++ b/.github/workflows/mobile-format.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 45 container: - image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu }} env: CLANG_FORMAT: /opt/llvm/bin/clang-format BUILDIFIER_BIN: /usr/local/bin/buildifier diff --git a/.github/workflows/mobile-perf.yml b/.github/workflows/mobile-perf.yml index b1c142bf2c1b..9da97dbb0016 100644 --- a/.github/workflows/mobile-perf.yml +++ b/.github/workflows/mobile-perf.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu }} env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu }} env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ @@ -76,7 +76,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 30 container: - image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu }} steps: - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 diff --git a/.github/workflows/mobile-tsan.yml b/.github/workflows/mobile-tsan.yml index 918613f15146..713e9a27d720 100644 --- a/.github/workflows/mobile-tsan.yml +++ b/.github/workflows/mobile-tsan.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 container: - image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ diff --git a/.github/workflows/mobile_release.yml b/.github/workflows/mobile_release.yml index df91222186e8..e2f6f7b947b6 100644 --- a/.github/workflows/mobile_release.yml +++ b/.github/workflows/mobile_release.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:mobile-41c5a05d708972d703661b702a63ef5060125c33 + image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} env: CC: /opt/llvm/bin/clang CXX: /opt/llvm/bin/clang++ From 842c20238440f88f33167e4bc024e2e113cbb0e8 Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Wed, 31 May 2023 15:07:43 -0400 Subject: [PATCH 425/740] Overload Manager: Validate configured overload actions. (#27711) Validate configured actions to prevent silent failures. Signed-off-by: Kevin Baichoo --- changelogs/current.yaml | 7 ++ envoy/server/overload/overload_manager.h | 11 +++ source/common/runtime/runtime_features.cc | 1 + source/server/overload_manager_impl.cc | 11 +++ test/server/overload_manager_impl_test.cc | 95 ++++++++++++++++------- 5 files changed, 96 insertions(+), 29 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 3db18423ae73..478fd07c9b7e 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -70,6 +70,13 @@ minor_behavior_changes: the cookie is still accepted, but is planned to be obsoleted in the future. This behavior change can be reverted by setting ``envoy.reloadable_features.stateful_session_encode_ttl_in_cookie`` to ``false``. +- area: overload manager + change: | + Changed behavior of the overload manager to error on unknown overload + manager actions. Prior it would silently fail. This change can be reverted + temporarily by setting the runtime guard + ``envoy.reloadable_features.overload_manager_error_unknown_action`` to + false. - area: router change: | Added check for existing metadata before setting metadata due to 'auto_sni', 'auto_san_validation', or diff --git a/envoy/server/overload/overload_manager.h b/envoy/server/overload/overload_manager.h index f65163c0ad15..bc71798a5ac8 100644 --- a/envoy/server/overload/overload_manager.h +++ b/envoy/server/overload/overload_manager.h @@ -38,6 +38,17 @@ class OverloadActionNameValues { // Overload action to reset streams using excessive memory. const std::string ResetStreams = "envoy.overload_actions.reset_high_memory_stream"; + + // This should be kept current with the Overload actions available. + // This is the last member of this class to duplicating the strings with + // proper lifetime guarantees. + const std::array WellKnownActions = {StopAcceptingRequests, + DisableHttpKeepAlive, + StopAcceptingConnections, + RejectIncomingConnections, + ShrinkHeap, + ReduceTimeouts, + ResetStreams}; }; using OverloadActionNames = ConstSingleton; diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 5d6d318e0b55..0571c4ef20fe 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -59,6 +59,7 @@ RUNTIME_GUARD(envoy_reloadable_features_oauth_make_token_cookie_httponly); RUNTIME_GUARD(envoy_reloadable_features_oauth_use_standard_max_age_value); RUNTIME_GUARD(envoy_reloadable_features_oauth_use_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_original_dst_rely_on_idle_timeout); +RUNTIME_GUARD(envoy_reloadable_features_overload_manager_error_unknown_action); RUNTIME_GUARD(envoy_reloadable_features_prohibit_route_refresh_after_response_headers_sent); RUNTIME_GUARD(envoy_reloadable_features_quic_defer_logging_to_ack_listener); RUNTIME_GUARD(envoy_reloadable_features_reject_require_client_certificate_with_quic); diff --git a/source/server/overload_manager_impl.cc b/source/server/overload_manager_impl.cc index e277abd50b43..0eea1311f14f 100644 --- a/source/server/overload_manager_impl.cc +++ b/source/server/overload_manager_impl.cc @@ -413,6 +413,17 @@ OverloadManagerImpl::OverloadManagerImpl(Event::Dispatcher& dispatcher, Stats::S const auto& name = action.name(); const auto symbol = action_symbol_table_.get(name); ENVOY_LOG(debug, "Adding overload action {}", name); + + // Validate that this is a well known overload action. + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.overload_manager_error_unknown_action")) { + auto& well_known_actions = OverloadActionNames::get().WellKnownActions; + if (std::find(well_known_actions.begin(), well_known_actions.end(), name) == + well_known_actions.end()) { + throw EnvoyException(absl::StrCat("Unknown Overload Manager Action ", name)); + } + } + // TODO: use in place construction once https://github.com/abseil/abseil-cpp/issues/388 is // addressed // We cannot currently use in place construction as the OverloadAction constructor may throw, diff --git a/test/server/overload_manager_impl_test.cc b/test/server/overload_manager_impl_test.cc index b43e8f41c388..081f66f6dc44 100644 --- a/test/server/overload_manager_impl_test.cc +++ b/test/server/overload_manager_impl_test.cc @@ -262,7 +262,7 @@ constexpr char kRegularStateConfig[] = R"YAML( - name: envoy.resource_monitors.fake_resource3 - name: envoy.resource_monitors.fake_resource4 actions: - - name: envoy.overload_actions.dummy_action + - name: envoy.overload_actions.stop_accepting_requests triggers: - name: envoy.resource_monitors.fake_resource1 threshold: @@ -287,7 +287,7 @@ constexpr char proactiveResourceConfig[] = R"YAML( - name: envoy.resource_monitors.fake_resource1 - name: envoy.resource_monitors.global_downstream_max_connections actions: - - name: envoy.overload_actions.dummy_action + - name: envoy.overload_actions.shrink_heap triggers: - name: envoy.resource_monitors.fake_resource1 threshold: @@ -300,11 +300,13 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { auto manager(createOverloadManager(kRegularStateConfig)); bool is_active = false; int cb_count = 0; - manager->registerForAction("envoy.overload_actions.dummy_action", dispatcher_, + manager->registerForAction("envoy.overload_actions.stop_accepting_requests", dispatcher_, [&](OverloadActionState state) { is_active = state.isSaturated(); cb_count++; }); + // This overload action callback should never be fired as the action is + // unknown and unconfigured to a trigger. manager->registerForAction("envoy.overload_actions.unknown_action", dispatcher_, [&](OverloadActionState) { EXPECT_TRUE(false); }); manager->start(); @@ -312,10 +314,11 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { EXPECT_FALSE(manager->getThreadLocalOverloadState().isResourceMonitorEnabled( OverloadProactiveResourceName::GlobalDownstreamMaxConnections)); - Stats::Gauge& active_gauge = stats_.gauge("overload.envoy.overload_actions.dummy_action.active", - Stats::Gauge::ImportMode::Accumulate); + Stats::Gauge& active_gauge = + stats_.gauge("overload.envoy.overload_actions.stop_accepting_requests.active", + Stats::Gauge::ImportMode::Accumulate); Stats::Gauge& scale_percent_gauge = - stats_.gauge("overload.envoy.overload_actions.dummy_action.scale_percent", + stats_.gauge("overload.envoy.overload_actions.stop_accepting_requests.scale_percent", Stats::Gauge::ImportMode::Accumulate); Stats::Gauge& pressure_gauge1 = stats_.gauge("overload.envoy.resource_monitors.fake_resource1.pressure", @@ -323,8 +326,8 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { Stats::Gauge& pressure_gauge2 = stats_.gauge("overload.envoy.resource_monitors.fake_resource2.pressure", Stats::Gauge::ImportMode::NeverImport); - const OverloadActionState& action_state = - manager->getThreadLocalOverloadState().getState("envoy.overload_actions.dummy_action"); + const OverloadActionState& action_state = manager->getThreadLocalOverloadState().getState( + "envoy.overload_actions.stop_accepting_requests"); // Update does not exceed fake_resource1 trigger threshold, no callback expected factory1_.monitor_->setPressure(0.5); @@ -410,12 +413,13 @@ TEST_F(OverloadManagerImplTest, ScaledTrigger) { auto manager(createOverloadManager(kRegularStateConfig)); manager->start(); - const auto& action_state = - manager->getThreadLocalOverloadState().getState("envoy.overload_actions.dummy_action"); - Stats::Gauge& active_gauge = stats_.gauge("overload.envoy.overload_actions.dummy_action.active", - Stats::Gauge::ImportMode::Accumulate); + const auto& action_state = manager->getThreadLocalOverloadState().getState( + "envoy.overload_actions.stop_accepting_requests"); + Stats::Gauge& active_gauge = + stats_.gauge("overload.envoy.overload_actions.stop_accepting_requests.active", + Stats::Gauge::ImportMode::Accumulate); Stats::Gauge& scale_percent_gauge = - stats_.gauge("overload.envoy.overload_actions.dummy_action.scale_percent", + stats_.gauge("overload.envoy.overload_actions.stop_accepting_requests.scale_percent", Stats::Gauge::ImportMode::Accumulate); factory3_.monitor_->setPressure(0.5); @@ -471,8 +475,8 @@ TEST_F(OverloadManagerImplTest, AggregatesMultipleResourceUpdates) { auto manager(createOverloadManager(kRegularStateConfig)); manager->start(); - const OverloadActionState& action_state = - manager->getThreadLocalOverloadState().getState("envoy.overload_actions.dummy_action"); + const OverloadActionState& action_state = manager->getThreadLocalOverloadState().getState( + "envoy.overload_actions.stop_accepting_requests"); factory1_.monitor_->setUpdateAsync(true); @@ -493,8 +497,8 @@ TEST_F(OverloadManagerImplTest, DelayedUpdatesAreCoalesced) { auto manager(createOverloadManager(kRegularStateConfig)); manager->start(); - const OverloadActionState& action_state = - manager->getThreadLocalOverloadState().getState("envoy.overload_actions.dummy_action"); + const OverloadActionState& action_state = manager->getThreadLocalOverloadState().getState( + "envoy.overload_actions.stop_accepting_requests"); factory3_.monitor_->setUpdateAsync(true); factory4_.monitor_->setUpdateAsync(true); @@ -517,8 +521,8 @@ TEST_F(OverloadManagerImplTest, FlushesUpdatesEvenWithOneUnresponsive) { auto manager(createOverloadManager(kRegularStateConfig)); manager->start(); - const OverloadActionState& action_state = - manager->getThreadLocalOverloadState().getState("envoy.overload_actions.dummy_action"); + const OverloadActionState& action_state = manager->getThreadLocalOverloadState().getState( + "envoy.overload_actions.stop_accepting_requests"); // Set monitor 1 to async, but never publish updates for it. factory1_.monitor_->setUpdateAsync(true); @@ -662,8 +666,8 @@ TEST_F(OverloadManagerImplTest, DuplicateProactiveResourceMonitor) { TEST_F(OverloadManagerImplTest, DuplicateOverloadAction) { const std::string config = R"EOF( actions: - - name: "envoy.overload_actions.dummy_action" - - name: "envoy.overload_actions.dummy_action" + - name: "envoy.overload_actions.shrink_heap" + - name: "envoy.overload_actions.shrink_heap" )EOF"; EXPECT_THROW_WITH_REGEX(createOverloadManager(config), EnvoyException, @@ -673,7 +677,7 @@ TEST_F(OverloadManagerImplTest, DuplicateOverloadAction) { TEST_F(OverloadManagerImplTest, ActionWithUnexpectedTypedConfig) { const std::string config = R"EOF( actions: - - name: "envoy.overload_actions.dummy_action" + - name: "envoy.overload_actions.shrink_heap" typed_config: "@type": type.googleapis.com/google.protobuf.Empty )EOF"; @@ -722,7 +726,7 @@ TEST_F(OverloadManagerImplTest, ScaledTriggerSaturationLessThanScalingThreshold) resource_monitors: - name: "envoy.resource_monitors.fake_resource1" actions: - - name: "envoy.overload_actions.dummy_action" + - name: "envoy.overload_actions.shrink_heap" triggers: - name: "envoy.resource_monitors.fake_resource1" scaled: @@ -740,7 +744,7 @@ TEST_F(OverloadManagerImplTest, ScaledTriggerThresholdsEqual) { resource_monitors: - name: "envoy.resource_monitors.fake_resource1" actions: - - name: "envoy.overload_actions.dummy_action" + - name: "envoy.overload_actions.shrink_heap" triggers: - name: "envoy.resource_monitors.fake_resource1" scaled: @@ -752,10 +756,43 @@ TEST_F(OverloadManagerImplTest, ScaledTriggerThresholdsEqual) { "scaling_threshold must be less than saturation_threshold.*"); } +TEST_F(OverloadManagerImplTest, UnknownActionShouldError) { + const std::string config = R"EOF( + resource_monitors: + - name: "envoy.resource_monitors.fake_resource1" + actions: + - name: "envoy.overload_actions.not_a_valid_action" + triggers: + - name: "envoy.resource_monitors.fake_resource1" + threshold: + value: 0.9 + )EOF"; + + EXPECT_THROW_WITH_REGEX(createOverloadManager(config), EnvoyException, + "Unknown Overload Manager Action .*"); +} + +TEST_F(OverloadManagerImplTest, LegacyUnknownActionShouldSilentlyFail) { + scoped_runtime_.mergeValues( + {{"envoy.reloadable_features.overload_manager_error_unknown_action", "false"}}); + const std::string config = R"EOF( + resource_monitors: + - name: "envoy.resource_monitors.fake_resource1" + actions: + - name: "envoy.overload_actions.not_a_valid_action" + triggers: + - name: "envoy.resource_monitors.fake_resource1" + threshold: + value: 0.9 + )EOF"; + + auto overload_manager = createOverloadManager(config); +} + TEST_F(OverloadManagerImplTest, UnknownTrigger) { const std::string config = R"EOF( actions: - - name: "envoy.overload_actions.dummy_action" + - name: "envoy.overload_actions.shrink_heap" triggers: - name: "envoy.resource_monitors.fake_resource1" threshold: @@ -771,7 +808,7 @@ TEST_F(OverloadManagerImplTest, DuplicateTrigger) { resource_monitors: - name: "envoy.resource_monitors.fake_resource1" actions: - - name: "envoy.overload_actions.dummy_action" + - name: "envoy.overload_actions.shrink_heap" triggers: - name: "envoy.resource_monitors.fake_resource1" threshold: @@ -807,7 +844,7 @@ TEST_F(OverloadManagerImplTest, Shutdown) { TEST_F(OverloadManagerImplTest, MissingConfigTriggerType) { constexpr char missingTriggerTypeConfig[] = R"YAML( actions: - - name: envoy.overload_actions.dummy_action + - name: envoy.overload_actions.shrink_heap triggers: - name: envoy.resource_monitors.fake_resource1 )YAML"; @@ -1048,7 +1085,7 @@ TEST_F(OverloadManagerLoadShedPointImplTest, LoadShedPointShouldUseCurrentReadin scaling_threshold: 0.5 saturation_threshold: 1.0 actions: - - name: envoy.overload_actions.dummy_action + - name: envoy.overload_actions.shrink_heap triggers: - name: envoy.resource_monitors.fake_resource1 scaled: @@ -1062,7 +1099,7 @@ TEST_F(OverloadManagerLoadShedPointImplTest, LoadShedPointShouldUseCurrentReadin std::vector overload_action_states; manager->registerForAction( - "envoy.overload_actions.dummy_action", *other_dispatcher, + "envoy.overload_actions.shrink_heap", *other_dispatcher, [&](OverloadActionState state) { overload_action_states.push_back(state.value()); }); manager->start(); From d4ce0e93af517d388dce5b398f123e9d03121a1d Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 31 May 2023 16:19:49 -0400 Subject: [PATCH 426/740] mobile: compile time options fix (#27732) reinstating compile time options build (broken by bad yaml) removing linux !h3 tests as we have swift e2e. I could add it to the kotlin build if we have concerns otherwise moving linux compile time options build to mimic common test options, and simplifying down to client integration test (avoid a bunch of flakey android deps and macos constraints) now that the client integration test works. Signed-off-by: Alyssa Wilk --- .../workflows/mobile-compile_time_options.yml | 33 ++++--------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index ae6b82e4ec57..c6c246432436 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -23,43 +23,22 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 120 container: - image: envoyproxy/envoy-build-ubuntu:b0ff77ae3f25b0bf595f9b8bba46b489723ab446 + image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33 steps: - uses: actions/checkout@v3 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Build C++ library' + - name: 'Building C++ library' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Envoy Mobile build which verifies that the build configuration where YAML is disabled. run: | - # Envoy Mobile build which verifies that the build configuration where HTTP/3 is enabled and - # HTTP Datagrams are disabled works. - cd mobile - ./bazelw test \ + cd mobile && ./bazelw test \ $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-linux") \ - --define=signal_trace=disabled \ - --define=admin_html=enabled \ - --define=envoy_mobile_request_compression=disabled \ - --define=envoy_enable_http_datagrams=disabled \ - --define=google_grpc=disabled \ - --@envoy//bazel:http3=False \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //test/cc/... - # Envoy Mobile build which verifies that the build configuration where YAML is disabled. - ./bazelw test \ - $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --config=ci \ - --fat_apk_cpu=x86_64 \ - --define=signal_trace=disabled \ - --define=envoy_mobile_request_compression=disabled \ - --define=envoy_enable_http_datagrams=disabled \ - --define=google_grpc=disabled \ --define=envoy_yaml=disabled \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //test/java/integration:android_engine_socket_tag_test \ - //test/java/integration:android_engine_start_test \ - //test/java/io/envoyproxy/envoymobile/utilities:certificate_verification_tests \ - //test/common/integration:client_integration_test + --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ + //test/common/integration:client_integration_test --test_output=all swift_build: if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} needs: env From 9ecf7611bdace7d7ff77959fdb607b59012c6a6d Mon Sep 17 00:00:00 2001 From: code Date: Thu, 1 Jun 2023 10:07:38 +0800 Subject: [PATCH 427/740] router: deprecate optional http filters and add route/vh level `is_optional` support in the typed_per_filter_config (#27263) * router: remove optional http filters because it's hcm dependent Signed-off-by: wbpcode * Revert "router: remove optional http filters because it's hcm dependent" This reverts commit 1725ce7d259d8586b0ba298e7e9a50953163fa73. Signed-off-by: wbpcode * add runtime guard/tests/change log Signed-off-by: wbpcode * fix more test Signed-off-by: wbpcode * add route/virual host level optional flag support and test and change log Signed-off-by: wbpcode * fix format Signed-off-by: wbpcode * fix docs Signed-off-by: wbpcode * fix test Signed-off-by: wbpcode * Update changelogs/current.yaml Co-authored-by: Adi (Suissa) Peleg Signed-off-by: code * address all comments Signed-off-by: wbpcode * fix conflict Signed-off-by: wbpcode * address comment Signed-off-by: wbpcode * address comments Signed-off-by: wbpcode * fix test and add new test to cover the new exception throwing Signed-off-by: wbpcode --------- Signed-off-by: wbpcode Signed-off-by: code Co-authored-by: Adi (Suissa) Peleg --- .../config/route/v3/route_components.proto | 3 +- .../v3/http_connection_manager.proto | 1 - changelogs/current.yaml | 18 +++ source/common/router/config_impl.cc | 57 ++++++-- source/common/router/config_impl.h | 3 +- source/common/runtime/runtime_features.cc | 1 + test/common/router/BUILD | 2 + test/common/router/config_impl_test.cc | 137 +++++++++++++++++- test/common/router/rds_impl_test.cc | 116 ++++++++++++++- test/common/router/scoped_rds_test.cc | 6 + .../http_typed_per_filter_config_test.cc | 30 +++- .../scoped_rds_integration_test.cc | 4 + 12 files changed, 361 insertions(+), 17 deletions(-) diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index a38bdccac97b..56686a096050 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -2376,7 +2376,6 @@ message InternalRedirectPolicy { // :ref:`Route.typed_per_filter_config`, // or :ref:`WeightedCluster.ClusterWeight.typed_per_filter_config` // to add additional flags to the filter. -// [#not-implemented-hide:] message FilterConfig { // The filter config. google.protobuf.Any config = 1; @@ -2398,5 +2397,7 @@ message FilterConfig { // created and it is too late to change the chain. // // This field only make sense for the downstream HTTP filters for now. + // + // [#not-implemented-hide:] bool disabled = 3; } diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 041db6ff60bf..f86be41f0493 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -1145,7 +1145,6 @@ message HttpFilter { // If true, clients that do not support this filter may ignore the // filter but otherwise accept the config. // Otherwise, clients that do not support this filter must reject the config. - // This is also same with typed per filter config. bool is_optional = 6; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 478fd07c9b7e..c6eff7753bf3 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -113,6 +113,18 @@ bug_fixes: Fixed the bug that updating :ref:`scope_key_builder ` of SRDS config doesn't work and multiple HCM share the same ``scope_key_builder``. +- area: http + change: | + The :ref:`is_optional + ` + field of HTTP filter can only be used for configuration loading of + :ref:`HTTP filter ` + and will be ignored for loading of route or virtual host level filter config. This behavioral change + can be temporarily reverted by setting runtime guard + ``envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config`` to false. + You can also use + :ref:`route/virtual host optional flag ` + as a replacement of the feature. - area: logging change: | Do not display GRPC_STATUS_NUMBER for non gRPC requests. @@ -245,6 +257,12 @@ new_features: - area: admin change: | Adds a new admin stats html bucket-mode ``detailed`` to generate all recorded buckets and summary percentiles. +- area: http + change: | + Add support to the route/virtual host level + :ref:`is_optional ` field. + A route/virtual host level per filter config can be marked as optional, which means that if + the filter fails to load, the configuration will no be rejected. - area: upstream change: | Added :ref:`cluster provided extension diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 014920a35bcd..e854bd3507b6 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -2104,11 +2104,10 @@ RouteConstSharedPtr ConfigImpl::route(const RouteCallback& cb, } RouteSpecificFilterConfigConstSharedPtr PerFilterConfigs::createRouteSpecificFilterConfig( - const std::string& name, const ProtobufWkt::Any& typed_config, - const OptionalHttpFilters& optional_http_filters, + const std::string& name, const ProtobufWkt::Any& typed_config, bool is_optional, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) { - bool is_optional = (optional_http_filters.find(name) != optional_http_filters.end()); + Server::Configuration::NamedHttpFilterConfigFactory* factory = Envoy::Config::Utility::getFactoryByType( typed_config); @@ -2148,12 +2147,52 @@ PerFilterConfigs::PerFilterConfigs( const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) { - for (const auto& it : typed_configs) { - const auto& name = it.first; - auto object = createRouteSpecificFilterConfig(name, it.second, optional_http_filters, - factory_context, validator); - if (object != nullptr) { - configs_[name] = std::move(object); + + const bool ignore_optional_option_from_hcm_for_route_config(Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config")); + + absl::string_view filter_config_type = + envoy::config::route::v3::FilterConfig::default_instance().GetDescriptor()->full_name(); + + for (const auto& per_filter_config : typed_configs) { + const std::string& name = per_filter_config.first; + RouteSpecificFilterConfigConstSharedPtr config; + + // There are two ways to mark a route/virtual host per filter configuration as optional: + // 1. Mark it as optional in the HTTP filter of HCM. This way is deprecated but still works + // when the runtime flag + // `envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config` + // is explicitly set to false. + // 2. Mark it as optional in the route/virtual host per filter configuration. This way is + // recommended. + // + // We check the first way first to ensure if this filter configuration is marked as optional + // or not. This will be true if the runtime flag is explicitly reverted to false and the + // config name is in the optional http filter list. + bool is_optional_by_hcm = !ignore_optional_option_from_hcm_for_route_config && + (optional_http_filters.find(name) != optional_http_filters.end()); + + if (TypeUtil::typeUrlToDescriptorFullName(per_filter_config.second.type_url()) == + filter_config_type) { + envoy::config::route::v3::FilterConfig filter_config; + Envoy::Config::Utility::translateOpaqueConfig(per_filter_config.second, validator, + filter_config); + + if (!filter_config.has_config()) { + throw EnvoyException( + fmt::format("Empty route/virtual host per filter configuration for {} filter", name)); + } + + config = createRouteSpecificFilterConfig(name, filter_config.config(), + is_optional_by_hcm || filter_config.is_optional(), + factory_context, validator); + } else { + config = createRouteSpecificFilterConfig(name, per_filter_config.second, is_optional_by_hcm, + factory_context, validator); + } + + if (config != nullptr) { + configs_[name] = std::move(config); } } } diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index e5f570e3cfe8..3f31ce43a986 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -91,10 +91,9 @@ class PerFilterConfigs : public Logger::Loggable { private: RouteSpecificFilterConfigConstSharedPtr createRouteSpecificFilterConfig(const std::string& name, const ProtobufWkt::Any& typed_config, - const OptionalHttpFilters& optional_http_filters, + bool is_optional, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); - absl::node_hash_map configs_; }; diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 0571c4ef20fe..f0e2429dccce 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -51,6 +51,7 @@ RUNTIME_GUARD(envoy_reloadable_features_http_ext_auth_failure_mode_allow_header_ RUNTIME_GUARD(envoy_reloadable_features_http_filter_avoid_reentrant_local_reply); RUNTIME_GUARD(envoy_reloadable_features_http_reject_path_with_fragment); RUNTIME_GUARD(envoy_reloadable_features_http_strip_fragment_from_path_unsafe_if_disabled); +RUNTIME_GUARD(envoy_reloadable_features_ignore_optional_option_from_hcm_for_route_config); RUNTIME_GUARD(envoy_reloadable_features_initialize_upstream_filters); RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name); RUNTIME_GUARD(envoy_reloadable_features_no_full_scan_certs_on_sni_mismatch); diff --git a/test/common/router/BUILD b/test/common/router/BUILD index affe611e5150..f7a93e02fd26 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -123,6 +123,7 @@ envoy_cc_test( "//test/mocks/server:instance_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/admin/v3:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", @@ -169,6 +170,7 @@ envoy_cc_test( "//test/mocks/router:router_mocks", "//test/mocks/server:instance_mocks", "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/admin/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 122c00f7ccc5..dae94a11bb91 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -10288,6 +10288,11 @@ TEST_F(PerFilterConfigsTest, DefaultFilterImplementationAnyWithCheckPerVirtualHo } TEST_F(PerFilterConfigsTest, OptionalDefaultFilterImplementationAnyWithCheckPerVirtualHost) { + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); + const std::string yaml = R"EOF( virtual_hosts: - name: bar @@ -10330,6 +10335,11 @@ TEST_F(PerFilterConfigsTest, DefaultFilterImplementationAnyWithCheckPerRoute) { } TEST_F(PerFilterConfigsTest, OptionalDefaultFilterImplementationAnyWithCheckPerRoute) { + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); + const std::string yaml = R"EOF( virtual_hosts: - name: bar @@ -10369,6 +10379,11 @@ TEST_F(PerFilterConfigsTest, PerVirtualHostWithUnknownFilter) { } TEST_F(PerFilterConfigsTest, PerVirtualHostWithOptionalUnknownFilter) { + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); + const std::string yaml = R"EOF( virtual_hosts: - name: bar @@ -10406,7 +10421,12 @@ TEST_F(PerFilterConfigsTest, PerRouteWithUnknownFilter) { "'google.protobuf.BoolValue'"); } -TEST_F(PerFilterConfigsTest, PerRouteWithOptionalUnknownFilter) { +TEST_F(PerFilterConfigsTest, PerRouteWithHcmOptionalUnknownFilterLegacy) { + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); + const std::string yaml = R"EOF( virtual_hosts: - name: bar @@ -10425,6 +10445,121 @@ TEST_F(PerFilterConfigsTest, PerRouteWithOptionalUnknownFilter) { checkNoPerFilterConfig(yaml, "filter.unknown", optional_http_filters); } +TEST_F(PerFilterConfigsTest, PerRouteWithHcmOptionalUnknownFilter) { + const std::string yaml = R"EOF( +virtual_hosts: + - name: bar + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: baz } + typed_per_filter_config: + filter.unknown: + "@type": type.googleapis.com/google.protobuf.BoolValue +)EOF"; + + factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); + OptionalHttpFilters optional_http_filters; + optional_http_filters.insert("filter.unknown"); + + EXPECT_THROW_WITH_MESSAGE( + TestConfigImpl(parseRouteConfigurationFromYaml(yaml), factory_context_, true, + optional_http_filters), + EnvoyException, + "Didn't find a registered implementation for 'filter.unknown' with type URL: " + "'google.protobuf.BoolValue'"); +} + +TEST_F(PerFilterConfigsTest, OptionalDefaultFilterImplementationAny) { + const std::string yaml = R"EOF( +typed_per_filter_config: + test.default.filter: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + is_optional: true + config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + seconds: 123 +virtual_hosts: + - name: bar + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: baz } + typed_per_filter_config: + test.default.filter: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + is_optional: true + config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + seconds: 123 + typed_per_filter_config: + test.default.filter: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + is_optional: true + config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + seconds: 123 +)EOF"; + + factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); + checkNoPerFilterConfig(yaml, "filter.unknown"); +} + +TEST_F(PerFilterConfigsTest, OptionalUnknownFilter) { + const std::string yaml = R"EOF( +typed_per_filter_config: + filter.unknown: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + is_optional: true + config: + "@type": type.googleapis.com/google.protobuf.BoolValue +virtual_hosts: + - name: bar + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: baz } + typed_per_filter_config: + filter.unknown: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + is_optional: true + config: + "@type": type.googleapis.com/google.protobuf.BoolValue + typed_per_filter_config: + filter.unknown: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + is_optional: true + config: + "@type": type.googleapis.com/google.protobuf.BoolValue +)EOF"; + + factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); + checkNoPerFilterConfig(yaml, "filter.unknown"); +} + +TEST_F(PerFilterConfigsTest, FilterConfigWithoutConfig) { + const std::string yaml = R"EOF( +virtual_hosts: + - name: bar + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: baz } + typed_per_filter_config: + filter.unknown: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + is_optional: true +)EOF"; + + EXPECT_THROW_WITH_MESSAGE( + TestConfigImpl(parseRouteConfigurationFromYaml(yaml), factory_context_, false), + EnvoyException, + "Empty route/virtual host per filter configuration for filter.unknown filter"); +} + TEST_F(PerFilterConfigsTest, RouteLocalTypedConfig) { const std::string yaml = R"EOF( typed_per_filter_config: diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index 881e43983cc7..981094f1b32d 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -26,6 +26,7 @@ #include "test/test_common/printers.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" +#include "test/test_common/test_runtime.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -180,7 +181,12 @@ stat_prefix: foo "'google.protobuf.Struct'"); } -TEST_F(RdsImplTest, RdsAndStaticWithOptionalUnknownFilterPerVirtualHostConfig) { +TEST_F(RdsImplTest, RdsAndStaticWithHcmOptionalUnknownFilterPerVirtualHostConfig) { + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); + const std::string config_yaml = R"EOF( route_config: virtual_hosts: @@ -205,6 +211,34 @@ stat_prefix: foo "foo.", *route_config_provider_manager_); } +TEST_F(RdsImplTest, RdsAndStaticWithOptionalUnknownFilterPerVirtualHostConfig) { + const std::string config_yaml = R"EOF( +route_config: + virtual_hosts: + - name: bar + domains: ["*"] + routes: + - match: { prefix: "/" } + typed_per_filter_config: + filter.unknown: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + is_optional: true + config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + seconds: 123 +codec_type: auto +stat_prefix: foo +http_filters: +- name: filter.unknown + is_optional: true + )EOF"; + + RouteConfigProviderUtil::create(parseHttpConnectionManagerFromYaml(config_yaml), + server_factory_context_, validation_visitor_, outer_init_manager_, + "foo.", *route_config_provider_manager_); +} + TEST_F(RdsImplTest, DestroyDuringInitialize) { InSequence s; setup(); @@ -358,7 +392,12 @@ TEST_F(RdsImplTest, UnknownFacotryForPerVirtualHostTypedConfig) { } // validate the optional unknown factory will be ignored for per virtualhost typed config. -TEST_F(RdsImplTest, OptionalUnknownFacotryForPerVirtualHostTypedConfig) { +TEST_F(RdsImplTest, HcmOptionalUnknownFacotryForPerVirtualHostTypedConfig) { + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); + InSequence s; const std::string config_yaml = R"EOF( rds: @@ -421,6 +460,74 @@ stat_prefix: foo rds_callbacks_->onConfigUpdate(decoded_resources.refvec_, response1.version_info()); } +// Validate the optional unknown factory will be ignored for per virtualhost typed config. +TEST_F(RdsImplTest, OptionalUnknownFacotryForPerVirtualHostTypedConfig) { + InSequence s; + const std::string config_yaml = R"EOF( +rds: + config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: 1s + route_config_name: foo_route_config +codec_type: auto +stat_prefix: foo +http_filters: +- name: filter.unknown + is_optional: true + )EOF"; + + setup(config_yaml); + + const std::string response1_json = R"EOF( +{ + "version_info": "1", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "foo_route_config", + "virtual_hosts": [ + { + "name": "integration", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/foo" + }, + "route": { + "cluster_header": ":authority" + } + } + ], + "typed_per_filter_config": { + "filter.unknown": { + "@type": "type.googleapis.com/envoy.config.route.v3.FilterConfig", + "is_optional": true, + "config": { + "@type": "type.googleapis.com/google.protobuf.Struct" + } + } + } + } + ] + } + ] +} +)EOF"; + auto response1 = + TestUtility::parseYaml(response1_json); + const auto decoded_resources = + TestUtility::decodeResources(response1); + + EXPECT_CALL(init_watcher_, ready()); + rds_callbacks_->onConfigUpdate(decoded_resources.refvec_, response1.version_info()); +} + // validate there will be exception throw when unknown factory found for per route typed config. TEST_F(RdsImplTest, UnknownFacotryForPerRouteTypedConfig) { InSequence s; @@ -476,6 +583,11 @@ TEST_F(RdsImplTest, UnknownFacotryForPerRouteTypedConfig) { // validate the optional unknown factory will be ignored for per route typed config. TEST_F(RdsImplTest, OptionalUnknownFacotryForPerRouteTypedConfig) { + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); + InSequence s; const std::string config_yaml = R"EOF( rds: diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index d4b8b16f0b5d..f392bdd26d60 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -24,6 +24,7 @@ #include "test/mocks/router/mocks.h" #include "test/mocks/server/instance.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "absl/strings/string_view.h" @@ -570,6 +571,11 @@ route_configuration_name: foo_routes // Test ignoring the optional unknown factory in the per-virtualhost typed config. TEST_F(ScopedRdsTest, OptionalUnknownFactoryForPerVirtualHostTypedConfig) { + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); + OptionalHttpFilters optional_http_filters; optional_http_filters.insert("filter.unknown"); setup(optional_http_filters); diff --git a/test/integration/http_typed_per_filter_config_test.cc b/test/integration/http_typed_per_filter_config_test.cc index 43b82354dbd1..d6a07c91b89f 100644 --- a/test/integration/http_typed_per_filter_config_test.cc +++ b/test/integration/http_typed_per_filter_config_test.cc @@ -1,3 +1,5 @@ +#include "envoy/config/route/v3/route_components.pb.h" + #include "test/integration/filters/set_response_code_filter_config.pb.h" #include "test/integration/http_integration.h" @@ -47,7 +49,11 @@ TEST_F(HTTPTypedPerFilterConfigTest, RejectUnknownHttpFilterInTypedPerFilterConf "'google.protobuf.Struct'"); } -TEST_F(HTTPTypedPerFilterConfigTest, IgnoreUnknownOptionalHttpFilterInTypedPerFilterConfig) { +TEST_F(HTTPTypedPerFilterConfigTest, HcmIgnoreUnknownOptionalHttpFilterInTypedPerFilterConfig) { + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + config_helper_.addRuntimeOverride( + "envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"); + config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { @@ -65,5 +71,27 @@ TEST_F(HTTPTypedPerFilterConfigTest, IgnoreUnknownOptionalHttpFilterInTypedPerFi initialize(); } +TEST_F(HTTPTypedPerFilterConfigTest, IgnoreUnknownOptionalHttpFilterInTypedPerFilterConfig) { + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { + auto* virtual_host = hcm.mutable_route_config()->mutable_virtual_hosts(0); + auto* config = virtual_host->mutable_typed_per_filter_config(); + + envoy::config::route::v3::FilterConfig filter_config; + filter_config.set_is_optional(true); + filter_config.mutable_config()->PackFrom(Envoy::ProtobufWkt::Struct()); + (*config)["filter.unknown"].PackFrom(filter_config); + + auto* filter = hcm.mutable_http_filters()->Add(); + filter->set_name("filter.unknown"); + filter->set_is_optional(true); + // keep router the last + auto size = hcm.http_filters_size(); + hcm.mutable_http_filters()->SwapElements(size - 2, size - 1); + }); + initialize(); +} + } // namespace } // namespace Envoy diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 4b5aedffa47e..118725693f2b 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -465,6 +465,10 @@ route_configuration_name: {} sendRdsResponse(fmt::format(route_config_tmpl, "foo_route", "cluster_0"), "1"); }; + // TODO(wbpcode): This test should be removed once the deprecated flag is removed. + config_helper_.addRuntimeOverride( + "envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"); + config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& http_connection_manager) { From 58ec492086e290c2a94b818206c86bce8201f3c0 Mon Sep 17 00:00:00 2001 From: cui fliter Date: Thu, 1 Jun 2023 16:08:19 +0800 Subject: [PATCH 428/740] fix problematic link (#27673) Signed-off-by: cui fliter --- api/envoy/extensions/transport_sockets/s2a/v3/s2a.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/transport_sockets/s2a/v3/s2a.proto b/api/envoy/extensions/transport_sockets/s2a/v3/s2a.proto index a8042c0417f0..03326c43e4db 100644 --- a/api/envoy/extensions/transport_sockets/s2a/v3/s2a.proto +++ b/api/envoy/extensions/transport_sockets/s2a/v3/s2a.proto @@ -14,7 +14,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#not-implemented-hide:] // Configuration for S2A transport socket. This allows Envoy clients to // configure how to offload mTLS handshakes to the S2A service. -// https://github.com/google/s2a-core#readme +// https://github.com/google/s2a-go#readme message S2AConfiguration { // The address of the S2A. This can be an IP address or a hostname, // followed by a port number. From 9406ed05cc7442b6c7ae8f8bfaf66e9514d7a7b7 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 1 Jun 2023 09:16:26 +0100 Subject: [PATCH 429/740] ci: Shift downstream sync azp -> gh (#27294) Signed-off-by: Ryan Northey --- .azure-pipelines/stage/sync.yml | 95 -------------------------------- .azure-pipelines/stages.yml | 15 ----- .github/workflows/envoy-sync.yml | 29 ++++++++++ ci/api_mirror.sh | 44 --------------- ci/filter_example_mirror.sh | 37 ------------- ci/go_mirror.sh | 12 ---- 6 files changed, 29 insertions(+), 203 deletions(-) delete mode 100644 .azure-pipelines/stage/sync.yml create mode 100644 .github/workflows/envoy-sync.yml delete mode 100755 ci/api_mirror.sh delete mode 100755 ci/filter_example_mirror.sh delete mode 100755 ci/go_mirror.sh diff --git a/.azure-pipelines/stage/sync.yml b/.azure-pipelines/stage/sync.yml deleted file mode 100644 index 39eb91180336..000000000000 --- a/.azure-pipelines/stage/sync.yml +++ /dev/null @@ -1,95 +0,0 @@ - -parameters: -- name: authGCP - type: string - default: "" -- name: authSSHKeyPassphrase - type: string - default: "" -- name: authSSHDataPlaneApiKey - type: string - default: "" -- name: authSSHDataPlaneApiKeyPublic - type: string - default: "" -- name: authSSHGoControlPlaneKey - type: string - default: "" -- name: authSSHGoControlPlaneKeyPublic - type: string - default: "" -- name: authSSHFilterExampleKey - type: string - default: "" -- name: authSSHFilterExampleKeyPublic - type: string - default: "" - - -jobs: -- job: filter_example - dependsOn: [] - pool: - vmImage: "ubuntu-20.04" - steps: - - checkout: self - fetchDepth: 0 - fetchTags: true - - task: InstallSSHKey@0 - inputs: - hostName: $(authGithubSSHKeyPublic) - sshPublicKey: "${{ parameters.authSSHFilterExampleKeyPublic }}" - sshPassphrase: "${{ parameters.authSSHKeyPassphrase }}" - sshKeySecureFile: "${{ parameters.authSSHFilterExampleKey }}" - - bash: ci/filter_example_mirror.sh - displayName: "Sync envoy-filter-example" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - -- job: data_plane_api - dependsOn: [] - pool: - vmImage: "ubuntu-20.04" - steps: - - checkout: self - fetchDepth: 0 - fetchTags: true - - task: InstallSSHKey@0 - inputs: - hostName: $(authGithubSSHKeyPublic) - sshPublicKey: "${{ parameters.authSSHDataPlaneApiKeyPublic }}" - sshPassphrase: "${{ parameters.authSSHKeyPassphrase }}" - sshKeySecureFile: "${{ parameters.authSSHDataPlaneApiKey }}" - - bash: ci/api_mirror.sh - displayName: "Sync data-plane-api" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - -- job: go_control_plane - dependsOn: [] - pool: - vmImage: "ubuntu-20.04" - steps: - - checkout: self - fetchDepth: 0 - fetchTags: true - - task: InstallSSHKey@0 - inputs: - hostName: $(authGithubSSHKeyPublic) - sshPublicKey: "${{ parameters.authSSHGoControlPlaneKeyPublic }}" - sshPassphrase: "${{ parameters.authSSHKeyPassphrase }}" - sshKeySecureFile: "${{ parameters.authSSHGoControlPlaneKey }}" - - bash: | - cp -a ~/.ssh $(Build.StagingDirectory)/ - ci/run_envoy_docker.sh 'ci/go_mirror.sh' - displayName: "Sync go-control-plane" - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - # TODO: make into parameters - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} - AZP_BRANCH: $(Build.SourceBranch) diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index caab5bae0fad..de69153bfbf7 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -75,21 +75,6 @@ stages: bucketGCP: $(GcsArtifactBucket) checkDeps: variables['CHECK_DEPS'] -- stage: sync - condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true)) - dependsOn: [] - jobs: - - template: stage/sync.yml - parameters: - authGCP: $(GcpServiceAccountKey) - authSSHDataPlaneApiKeyPublic: $(DataPlaneApiPublicKey) - authSSHDataPlaneApiKey: $(DataPlaneApiPrivateKey) - authSSHGoControlPlaneKeyPublic: $(GoControlPlanePublicKey) - authSSHGoControlPlaneKey: $(GoControlPlanePrivateKey) - authSSHFilterExampleKeyPublic: $(FilterExamplePublicKey) - authSSHFilterExampleKey: $(FilterExamplePrivateKey) - authSSHKeyPassphrase: $(SshDeployKeyPassphrase) - - stage: linux_x64 displayName: Linux x64 dependsOn: ${{ parameters.buildStageDeps }} diff --git a/.github/workflows/envoy-sync.yml b/.github/workflows/envoy-sync.yml new file mode 100644 index 000000000000..da65ccc55984 --- /dev/null +++ b/.github/workflows/envoy-sync.yml @@ -0,0 +1,29 @@ +name: 'Sync downstream' + +on: + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + sync: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + downstream: + - go-control-plane + - envoy-filter-example + - data-plane-api + steps: + - uses: envoyproxy/toolshed/gh-actions/dispatch@1f1feae1e372dde41ecc6830028989bb6037c480 + with: + repository: "envoyproxy/${{ matrix.downstream }}" + ref: main + key: "${{ secrets.ENVOY_CI_SYNC_APP_KEY }}" + workflow: envoy-sync.yaml + app_id: ${{ secrets.ENVOY_CI_SYNC_APP_ID }} diff --git a/ci/api_mirror.sh b/ci/api_mirror.sh deleted file mode 100755 index 8a3022b72431..000000000000 --- a/ci/api_mirror.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -set -e - -CHECKOUT_DIR=../data-plane-api -MAIN_BRANCH="refs/heads/main" -API_MAIN_BRANCH="main" - -if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then - echo "Cloning..." - git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" -b "${API_MAIN_BRANCH}" - - git -C "$CHECKOUT_DIR" config user.name "data-plane-api(Azure Pipelines)" - git -C "$CHECKOUT_DIR" config user.email data-plane-api@users.noreply.github.com - - # Determine last envoyproxy/envoy SHA in envoyproxy/data-plane-api - MIRROR_MSG="Mirrored from https://github.com/envoyproxy/envoy" - LAST_ENVOY_SHA=$(git -C "$CHECKOUT_DIR" log --grep="$MIRROR_MSG" -n 1 | grep "$MIRROR_MSG" | \ - tail -n 1 | sed -e "s#.*$MIRROR_MSG @ ##") - - echo "Last mirrored envoyproxy/envoy SHA is $LAST_ENVOY_SHA" - - # Compute SHA sequence to replay in envoyproxy/data-plane-api - SHAS=$(git rev-list --reverse "$LAST_ENVOY_SHA"..HEAD api/) - - # For each SHA, hard reset, rsync api/ and generate commit in - # envoyproxy/data-plane-api - API_WORKING_DIR="../envoy-api-mirror" - git worktree add "$API_WORKING_DIR" - for sha in $SHAS - do - git -C "$API_WORKING_DIR" reset --hard "$sha" - COMMIT_MSG=$(git -C "$API_WORKING_DIR" log --format=%B -n 1) - QUALIFIED_COMMIT_MSG=$(echo -e "$COMMIT_MSG\n\n$MIRROR_MSG @ $sha") - rsync -acv --delete --exclude "ci/" --exclude ".*" --exclude LICENSE \ - "$API_WORKING_DIR"/api/ "$CHECKOUT_DIR"/ - git -C "$CHECKOUT_DIR" add . - git -C "$CHECKOUT_DIR" commit -m "$QUALIFIED_COMMIT_MSG" - done - - echo "Pushing..." - git -C "$CHECKOUT_DIR" push origin "${API_MAIN_BRANCH}" - echo "Done" -fi diff --git a/ci/filter_example_mirror.sh b/ci/filter_example_mirror.sh deleted file mode 100755 index ed26317c49a2..000000000000 --- a/ci/filter_example_mirror.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -e - -ENVOY_SRCDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd) -CHECKOUT_DIR=../envoy-filter-example -MAIN_BRANCH="refs/heads/main" -FILTER_EXAMPLE_MAIN_BRANCH="main" - -if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then - echo "Cloning..." - git clone git@github.com:envoyproxy/envoy-filter-example "$CHECKOUT_DIR" -b "${FILTER_EXAMPLE_MAIN_BRANCH}" - - git -C "$CHECKOUT_DIR" config user.name "envoy-filter-example(Azure Pipelines)" - git -C "$CHECKOUT_DIR" config user.email envoy-filter-example@users.noreply.github.com - - echo "Updating Submodule..." - # Update submodule to latest Envoy SHA - ENVOY_SHA=$(git rev-parse HEAD) - CURRENT_SHA="$(git -C "$CHECKOUT_DIR" ls-files -s envoy | cut -d' ' -f2)" - - if [[ "$CURRENT_SHA" == "$ENVOY_SHA" ]]; then - echo "Submodule already up to date (${ENVOY_SHA})" - exit 0 - fi - - git -C "$CHECKOUT_DIR" submodule update --init - git -C "$CHECKOUT_DIR/envoy" checkout "$ENVOY_SHA" - - echo "Updating Workspace file." - sed -e "s|{ENVOY_SRCDIR}|envoy|" "${ENVOY_SRCDIR}"/ci/WORKSPACE.filter.example > "${CHECKOUT_DIR}"/WORKSPACE - - echo "Committing, and Pushing..." - git -C "$CHECKOUT_DIR" commit -a -m "Update Envoy submodule to $ENVOY_SHA" - git -C "$CHECKOUT_DIR" push origin "${FILTER_EXAMPLE_MAIN_BRANCH}" - echo "Done" -fi diff --git a/ci/go_mirror.sh b/ci/go_mirror.sh deleted file mode 100755 index 6e69c68a0a93..000000000000 --- a/ci/go_mirror.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e - -MAIN_BRANCH="refs/heads/main" - -# shellcheck source=ci/setup_cache.sh -. "$(dirname "$0")"/setup_cache.sh - -if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then - BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_EXTRA_OPTIONS}" tools/api/generate_go_protobuf.py --sync -fi From 62da62f6f8503a9f37b978ba2cef5adff9723c9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 09:27:53 +0100 Subject: [PATCH 430/740] build(deps): bump actions/dependency-review-action from 3.0.4 to 3.0.6 (#27741) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 3.0.4 to 3.0.6. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/f46c48ed6d4f1227fb2d9ea62bf6bcbed315589e...1360a344ccb0ab6e9475edef90ad2f46bf8003b1) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/depsreview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index 007b2e497174..542888585103 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -13,4 +13,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@v3 - name: 'Dependency Review' - uses: actions/dependency-review-action@f46c48ed6d4f1227fb2d9ea62bf6bcbed315589e + uses: actions/dependency-review-action@1360a344ccb0ab6e9475edef90ad2f46bf8003b1 From ae5758f28c4fe4e3e9935b0d1480393fb4cc1782 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 09:43:31 +0100 Subject: [PATCH 431/740] build(deps): bump jaegertracing/all-in-one from `114816a` to `1cb8093` in /examples/shared/jaeger (#27745) build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger Bumps jaegertracing/all-in-one from `114816a` to `1cb8093`. --- updated-dependencies: - dependency-name: jaegertracing/all-in-one dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/jaeger/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile index 57dfe7cb60a4..b9fcb0029145 100644 --- a/examples/shared/jaeger/Dockerfile +++ b/examples/shared/jaeger/Dockerfile @@ -1,4 +1,4 @@ -FROM jaegertracing/all-in-one@sha256:114816a5808fddda2bdcf85f335d306b279b2cebb1e48d9d1d0f0e9ab53beeb9 +FROM jaegertracing/all-in-one@sha256:1cb80937ebe12ed06267d2f773623d33a4610df9ef59d82e92c15613f31772ed HEALTHCHECK \ --interval=1s \ --timeout=1s \ From bdaebba6b3e77701b487761813272cc3ecd3625e Mon Sep 17 00:00:00 2001 From: John Bucy Date: Wed, 31 May 2023 23:07:21 -1000 Subject: [PATCH 432/740] Fix examples/cache (#27743) examples/cache: fix reference to aiohttp exception Signed-off-by: John Bucy --- examples/cache/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cache/service.py b/examples/cache/service.py index aff199544db3..37e18a370959 100644 --- a/examples/cache/service.py +++ b/examples/cache/service.py @@ -63,7 +63,7 @@ async def get(request): stored_response = yaml.safe_load(open('/etc/responses.yaml', 'r')).get(response_id) if stored_response is None: - raise aiohttp.web.HTTPNotFound(reason="No response found with the given id") + raise web.HTTPNotFound(reason="No response found with the given id") request_date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") response = web.Response( From b39d6a09a1af8af96c6c9a9ec283fa581f33dd66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jun 2023 10:13:45 +0100 Subject: [PATCH 433/740] build(deps): bump openzipkin/zipkin from `2d797c2` to `46ee38f` in /examples/zipkin (#27746) build(deps): bump openzipkin/zipkin in /examples/zipkin Bumps openzipkin/zipkin from `2d797c2` to `46ee38f`. --- updated-dependencies: - dependency-name: openzipkin/zipkin dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/zipkin/Dockerfile-zipkin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/zipkin/Dockerfile-zipkin b/examples/zipkin/Dockerfile-zipkin index c09f6bb90525..4bf28ce777b1 100644 --- a/examples/zipkin/Dockerfile-zipkin +++ b/examples/zipkin/Dockerfile-zipkin @@ -1 +1 @@ -FROM openzipkin/zipkin:latest@sha256:2d797c2da4aa98aee166eef82852cfb3cf09c1c3c3e48905be3564ce7e6acfc4 +FROM openzipkin/zipkin:latest@sha256:46ee38f01382e72de617f704441fd935390c6eeb762e88869530e661f1339765 From d69008bf49b9eb4b4327d4cb4bc8f74dbcef93a8 Mon Sep 17 00:00:00 2001 From: "Dr. Andre Vehreschild" <101638173+vehre-x41@users.noreply.github.com> Date: Thu, 1 Jun 2023 14:09:45 +0200 Subject: [PATCH 434/740] [fuzz] Look at the correct class name for cut off. (#27729) Signed-off-by: Andre Vehreschild --- test/fuzz/validated_input_generator.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/fuzz/validated_input_generator.cc b/test/fuzz/validated_input_generator.cc index db9e4f3700c7..6970b111afcb 100644 --- a/test/fuzz/validated_input_generator.cc +++ b/test/fuzz/validated_input_generator.cc @@ -427,13 +427,12 @@ void ValidatedInputGenerator::onEnterMessage(Protobuf::Message& msg, !reflection->HasOneof(msg, descriptor->oneof_decl(oneof_index)))) { // No required member in one of set, so create one. for (int index = 0; index < oneof_desc->field_count(); ++index) { - const std::string parents_class_name = parents.back()->GetDescriptor()->full_name(); + const std::string class_name = descriptor->full_name(); // Treat matchers special, because in their oneof they reference themselves, which may // create long chains. Prefer the first alternative, which does not reference itself. // Nevertheless do it randomly to allow for some nesting. - if ((parents_class_name == "xds.type.matcher.v3.Matcher.MatcherList.Predicate" || - parents_class_name == - "xds.type.matcher.v3.Matcher.MatcherList.Predicate.SinglePredicate") && + if ((class_name == "xds.type.matcher.v3.Matcher.MatcherList.Predicate" || + class_name == "xds.type.matcher.v3.Matcher.MatcherList.Predicate.SinglePredicate") && (random_() % 200) > 0) { onField(msg, *oneof_desc->field(0), parents, true, max_depth_exceeded); } else { From c43460d62489948a36a6eb36b5c6dc018e96902f Mon Sep 17 00:00:00 2001 From: doujiang24 Date: Thu, 1 Jun 2023 20:38:40 +0800 Subject: [PATCH 435/740] warn about `GODEBUG=cgocheck=0` env variable in golang plugin example docs. (#27744) Signed-off-by: doujiang24 --- .../root/configuration/http/cluster_specifier/golang.rst | 2 +- .../configuration/http/http_filters/golang_filter.rst | 2 +- docs/root/start/sandboxes/golang.rst | 9 +++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/root/configuration/http/cluster_specifier/golang.rst b/docs/root/configuration/http/cluster_specifier/golang.rst index 6b1938c5559d..efe9e1e984ff 100644 --- a/docs/root/configuration/http/cluster_specifier/golang.rst +++ b/docs/root/configuration/http/cluster_specifier/golang.rst @@ -9,7 +9,7 @@ and makes it easier to extend Envoy. Go cluster specifier plugins can be recompiled independently of Envoy. .. warning:: - The Envoy Golang cluster specifier is designed to be run with the `GODEBUG=cgocheck=0` environment variable set. + The Envoy Golang cluster specifier is designed to be run with the ``GODEBUG=cgocheck=0`` environment variable set. This disables the cgo pointer check. diff --git a/docs/root/configuration/http/http_filters/golang_filter.rst b/docs/root/configuration/http/http_filters/golang_filter.rst index e4e7aec82e1e..9a1106d8bfcd 100644 --- a/docs/root/configuration/http/http_filters/golang_filter.rst +++ b/docs/root/configuration/http/http_filters/golang_filter.rst @@ -13,7 +13,7 @@ See the `Envoy's Golang extension proposal documentation for more details on the filter's implementation. .. warning:: - The Envoy Golang filter is designed to be run with the `GODEBUG=cgocheck=0` environment variable set. + The Envoy Golang filter is designed to be run with the ``GODEBUG=cgocheck=0`` environment variable set. This disables the cgo pointer check. diff --git a/docs/root/start/sandboxes/golang.rst b/docs/root/start/sandboxes/golang.rst index 50bc2c8e878e..d81adedf48ea 100644 --- a/docs/root/start/sandboxes/golang.rst +++ b/docs/root/start/sandboxes/golang.rst @@ -40,6 +40,15 @@ Step 2: Start all of our containers Start all the containers. +.. warning:: + The Envoy Golang filter is designed to be run with the ``GODEBUG=cgocheck=0`` environment variable set. + + This disables the cgo pointer check. + + Failure to set this environment variable will cause Envoy to crash! + + Here, we have set this environment variable in :repo:`Dockerfile ` + .. code-block:: console $ docker compose pull From 487cedbc8bdccff80d497be95ac58a71b5244b64 Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 1 Jun 2023 14:45:53 +0100 Subject: [PATCH 436/740] deps: Bump `proxy_wasm_cpp_host` -> 5d76116 (+ related deps) (#27718) * deps: Bump `com_github_wamr` -> 1.2.2 * deps: Bump `proxy_wasm_cpp_host` -> 5d76116 * deps: Bump `com_github_wasmtime` -> 9.0.3 Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 6ea53a224d09..1deaabc0ec69 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -930,11 +930,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Webassembly Micro Runtime", project_desc = "A standalone runtime with a small footprint for WebAssembly", project_url = "https://github.com/bytecodealliance/wasm-micro-runtime", - version = "WAMR-1.1.2", - sha256 = "976b928f420040a77e793051e4d742208adf157370b9ad0f5535e126adb31eb0", + version = "WAMR-1.2.2", + sha256 = "d328fc1e19c54cfdb4248b861de54b62977b9b85c0a40eaaeb9cd9b628c0c788", strip_prefix = "wasm-micro-runtime-{version}", urls = ["https://github.com/bytecodealliance/wasm-micro-runtime/archive/{version}.tar.gz"], - release_date = "2022-12-16", + release_date = "2023-05-16", use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.wamr"], cpe = "N/A", @@ -958,11 +958,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "wasmtime", project_desc = "A standalone runtime for WebAssembly", project_url = "https://github.com/bytecodealliance/wasmtime", - version = "6.0.1", - sha256 = "7ed6359faa385c40fc77e324301e5c70c7fdaeeca8a5ab58e25b1f75035b2cd6", + version = "9.0.3", + sha256 = "917da461249b11a3176a39573723f78c627259576d0ca10b00d6e7f7fa047081", strip_prefix = "wasmtime-{version}", urls = ["https://github.com/bytecodealliance/wasmtime/archive/v{version}.tar.gz"], - release_date = "2023-03-08", + release_date = "2023-05-31", use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.wasmtime"], cpe = "cpe:2.3:a:bytecodealliance:wasmtime:*", @@ -1271,8 +1271,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "WebAssembly for Proxies (C++ host implementation)", project_desc = "WebAssembly for Proxies (C++ host implementation)", project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host", - version = "8aa8053609d893e25287551295bd9cd7dd4c3dcc", - sha256 = "e588c077705ed2c5aa77edd19151cec8c75525ec5d0973bcb8b45a7a27109dd9", + version = "5d76116c449d6892b298b7ae79a84ef1cf5752bf", + sha256 = "a5825a1a5bbd5b0178c6189b227d5cf4370ac713a883b41f6a54edd768a03cb7", strip_prefix = "proxy-wasm-cpp-host-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-host/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -1288,7 +1288,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.wasm.runtime.wavm", "envoy.wasm.runtime.wasmtime", ], - release_date = "2023-03-20", + release_date = "2023-06-01", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host/blob/{version}/LICENSE", From 94223d8d6e075ce254bb79ac76321788e160e9e6 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 1 Jun 2023 09:59:14 -0400 Subject: [PATCH 437/740] xds: moving xds only files into the relevant extension directory. (#27703) move gRPC files Signed-off-by: Alyssa Wilk --- contrib/config/test/BUILD | 4 +- .../kv_store_xds_delegate_integration_test.cc | 2 +- .../config/test/kv_store_xds_delegate_test.cc | 2 +- envoy/config/BUILD | 1 - source/common/config/BUILD | 86 +++---------------- .../extensions/config_subscription/grpc/BUILD | 76 ++++++++++++++-- .../config_subscription/grpc/grpc_mux_impl.cc | 2 +- .../config_subscription/grpc/grpc_mux_impl.h | 2 +- .../config_subscription/grpc}/grpc_stream.h | 0 .../grpc/new_delta_subscription_state.h | 4 +- .../grpc/new_grpc_mux_impl.h | 6 +- .../grpc/old_delta_subscription_state.h | 4 +- .../grpc}/pausable_ack_queue.cc | 2 +- .../grpc}/pausable_ack_queue.h | 2 +- .../config_subscription/grpc}/update_ack.h | 0 .../config_subscription/grpc}/watch_map.cc | 2 +- .../config_subscription/grpc}/watch_map.h | 0 .../config_subscription/grpc/xds_mux/BUILD | 10 +-- .../grpc/xds_mux/grpc_mux_impl.h | 6 +- .../grpc/xds_mux/sotw_subscription_state.cc | 2 +- .../grpc/xds_mux/subscription_state.h | 2 +- .../grpc}/xds_source_id.cc | 2 +- .../config_subscription/grpc}/xds_source_id.h | 0 test/common/config/BUILD | 46 ---------- .../extensions/config_subscription/grpc/BUILD | 46 ++++++++++ .../grpc}/grpc_stream_test.cc | 2 +- .../grpc}/pausable_ack_queue_test.cc | 2 +- .../grpc}/watch_map_test.cc | 2 +- .../grpc}/xds_source_id_test.cc | 2 +- test/per_file_coverage.sh | 2 +- 30 files changed, 159 insertions(+), 160 deletions(-) rename source/{common/config => extensions/config_subscription/grpc}/grpc_stream.h (100%) rename source/{common/config => extensions/config_subscription/grpc}/pausable_ack_queue.cc (95%) rename source/{common/config => extensions/config_subscription/grpc}/pausable_ack_queue.h (94%) rename source/{common/config => extensions/config_subscription/grpc}/update_ack.h (100%) rename source/{common/config => extensions/config_subscription/grpc}/watch_map.cc (99%) rename source/{common/config => extensions/config_subscription/grpc}/watch_map.h (100%) rename source/{common/config => extensions/config_subscription/grpc}/xds_source_id.cc (90%) rename source/{common/config => extensions/config_subscription/grpc}/xds_source_id.h (100%) rename test/{common/config => extensions/config_subscription/grpc}/grpc_stream_test.cc (99%) rename test/{common/config => extensions/config_subscription/grpc}/pausable_ack_queue_test.cc (97%) rename test/{common/config => extensions/config_subscription/grpc}/watch_map_test.cc (99%) rename test/{common/config => extensions/config_subscription/grpc}/xds_source_id_test.cc (92%) diff --git a/contrib/config/test/BUILD b/contrib/config/test/BUILD index adcfd31ef533..36a4dc4e4464 100644 --- a/contrib/config/test/BUILD +++ b/contrib/config/test/BUILD @@ -16,7 +16,7 @@ envoy_cc_test( ], deps = [ "//contrib/config/source:kv_store_xds_delegate", - "//source/common/config:xds_source_id_lib", + "//source/extensions/config_subscription/grpc:xds_source_id_lib", "//source/extensions/key_value/file_based:config_lib", "//test/mocks/api:api_mocks", "//test/mocks/event:event_mocks", @@ -43,7 +43,7 @@ envoy_cc_test( ":invalid_proto_kv_store_config_proto_cc_proto", "//contrib/config/source:kv_store_xds_delegate", "//envoy/common:key_value_store_interface", - "//source/common/config:xds_source_id_lib", + "//source/extensions/config_subscription/grpc:xds_source_id_lib", "//source/extensions/key_value/file_based:config_lib", "//test/common/grpc:grpc_client_integration_lib", "//test/integration:http_integration_lib", diff --git a/contrib/config/test/kv_store_xds_delegate_integration_test.cc b/contrib/config/test/kv_store_xds_delegate_integration_test.cc index 18a8836293b6..ebdfb279833a 100644 --- a/contrib/config/test/kv_store_xds_delegate_integration_test.cc +++ b/contrib/config/test/kv_store_xds_delegate_integration_test.cc @@ -6,7 +6,7 @@ #include "envoy/service/runtime/v3/rtds.pb.h" #include "envoy/service/secret/v3/sds.pb.h" -#include "source/common/config/xds_source_id.h" +#include "source/extensions/config_subscription/grpc/xds_source_id.h" #include "test/common/grpc/grpc_client_integration.h" #include "test/integration/http_integration.h" diff --git a/contrib/config/test/kv_store_xds_delegate_test.cc b/contrib/config/test/kv_store_xds_delegate_test.cc index 89747e100008..d69a5ad507f9 100644 --- a/contrib/config/test/kv_store_xds_delegate_test.cc +++ b/contrib/config/test/kv_store_xds_delegate_test.cc @@ -6,7 +6,7 @@ #include "envoy/service/runtime/v3/rtds.pb.h" #include "envoy/service/runtime/v3/rtds.pb.validate.h" -#include "source/common/config/xds_source_id.h" +#include "source/extensions/config_subscription/grpc/xds_source_id.h" #include "test/common/stats/stat_test_utility.h" #include "test/mocks/api/mocks.h" diff --git a/envoy/config/BUILD b/envoy/config/BUILD index 23fd71a5c19a..a660c39c59c7 100644 --- a/envoy/config/BUILD +++ b/envoy/config/BUILD @@ -135,7 +135,6 @@ envoy_cc_library( ":subscription_interface", ":typed_config_interface", "//envoy/protobuf:message_validator_interface", - "//source/common/config:update_ack_lib", "//source/common/protobuf", "@com_google_googleapis//google/rpc:status_cc_proto", ], diff --git a/source/common/config/BUILD b/source/common/config/BUILD index f7a25c71a45e..8f527194970c 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -85,23 +85,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "grpc_stream_lib", - hdrs = ["grpc_stream.h"], - deps = [ - ":utility_lib", - "//envoy/config:grpc_mux_interface", - "//envoy/config:subscription_interface", - "//envoy/grpc:async_client_interface", - "//envoy/upstream:cluster_manager_interface", - "//source/common/common:backoff_lib", - "//source/common/common:minimal_logger_lib", - "//source/common/common:token_bucket_impl_lib", - "//source/common/grpc:async_client_lib", - "//source/common/protobuf", - ], -) - envoy_cc_library( name = "null_grpc_mux_lib", hdrs = ["null_grpc_mux_impl.h"], @@ -137,16 +120,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "pausable_ack_queue_lib", - srcs = ["pausable_ack_queue.cc"], - hdrs = ["pausable_ack_queue.h"], - deps = [ - ":update_ack_lib", - "//source/common/common:assert_lib", - ], -) - envoy_cc_library( name = "protobuf_link_hacks", hdrs = ["protobuf_link_hacks.h"], @@ -224,18 +197,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "xds_context_params_lib", - srcs = ["xds_context_params.cc"], - hdrs = ["xds_context_params.h"], - deps = [ - "//source/common/common:macros", - "//source/common/protobuf:utility_lib", - "@com_github_cncf_udpa//xds/core/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - ], -) - envoy_cc_library( name = "xds_resource_lib", srcs = ["xds_resource.cc"], @@ -246,21 +207,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "xds_source_id_lib", - srcs = ["xds_source_id.cc"], - hdrs = ["xds_source_id.h"], - deps = [ - "//envoy/config:xds_resources_delegate_interface", - ], -) - -envoy_cc_library( - name = "update_ack_lib", - hdrs = ["update_ack.h"], - deps = ["@com_google_googleapis//google/rpc:status_cc_proto"], -) - envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], @@ -296,26 +242,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "watch_map_lib", - srcs = ["watch_map.cc"], - hdrs = ["watch_map.h"], - deps = [ - ":decoded_resource_lib", - ":utility_lib", - ":xds_resource_lib", - "//envoy/config:custom_config_validators_interface", - "//envoy/config:subscription_interface", - "//envoy/config:xds_config_tracker_interface", - "//source/common/common:assert_lib", - "//source/common/common:cleanup_lib", - "//source/common/common:minimal_logger_lib", - "//source/common/common:utility_lib", - "//source/common/protobuf", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_cc_library( name = "subscription_base_interface", hdrs = ["subscription_base.h"], @@ -351,6 +277,18 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "xds_context_params_lib", + srcs = ["xds_context_params.cc"], + hdrs = ["xds_context_params.h"], + deps = [ + "//source/common/common:macros", + "//source/common/protobuf:utility_lib", + "@com_github_cncf_udpa//xds/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "well_known_names", srcs = ["well_known_names.cc"], diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD index 9736956f4938..09f73cbc92e4 100644 --- a/source/extensions/config_subscription/grpc/BUILD +++ b/source/extensions/config_subscription/grpc/BUILD @@ -14,6 +14,8 @@ envoy_cc_library( srcs = ["grpc_mux_impl.cc"], hdrs = ["grpc_mux_impl.h"], deps = [ + ":grpc_stream_lib", + ":xds_source_id_lib", "//envoy/config:custom_config_validators_interface", "//envoy/config:grpc_mux_interface", "//envoy/config:subscription_interface", @@ -25,12 +27,10 @@ envoy_cc_library( "//source/common/common:utility_lib", "//source/common/config:api_version_lib", "//source/common/config:decoded_resource_lib", - "//source/common/config:grpc_stream_lib", "//source/common/config:ttl_lib", "//source/common/config:utility_lib", "//source/common/config:xds_context_params_lib", "//source/common/config:xds_resource_lib", - "//source/common/config:xds_source_id_lib", "//source/common/memory:utils_lib", "//source/common/protobuf", "@com_google_absl//absl/container:btree", @@ -44,13 +44,13 @@ envoy_cc_library( hdrs = ["new_grpc_mux_impl.h"], deps = [ ":delta_subscription_state_lib", + ":grpc_stream_lib", + ":pausable_ack_queue_lib", + ":watch_map_lib", "//envoy/config:custom_config_validators_interface", "//envoy/config:xds_config_tracker_interface", "//envoy/event:dispatcher_interface", "//envoy/grpc:async_client_interface", - "//source/common/config:grpc_stream_lib", - "//source/common/config:pausable_ack_queue_lib", - "//source/common/config:watch_map_lib", "//source/common/config:xds_context_params_lib", "//source/common/config:xds_resource_lib", "//source/common/memory:utils_lib", @@ -138,6 +138,8 @@ envoy_cc_library( "old_delta_subscription_state.h", ], deps = [ + ":pausable_ack_queue_lib", + ":watch_map_lib", "//envoy/config:subscription_interface", "//envoy/event:dispatcher_interface", "//source/common/common:assert_lib", @@ -145,12 +147,72 @@ envoy_cc_library( "//source/common/common:minimal_logger_lib", "//source/common/common:token_bucket_impl_lib", "//source/common/config:api_version_lib", - "//source/common/config:pausable_ack_queue_lib", "//source/common/config:ttl_lib", "//source/common/config:utility_lib", - "//source/common/config:watch_map_lib", "//source/common/grpc:common_lib", "//source/common/protobuf", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) + +envoy_cc_library( + name = "grpc_stream_lib", + hdrs = ["grpc_stream.h"], + deps = [ + "//envoy/config:grpc_mux_interface", + "//envoy/config:subscription_interface", + "//envoy/grpc:async_client_interface", + "//envoy/upstream:cluster_manager_interface", + "//source/common/common:backoff_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/common:token_bucket_impl_lib", + "//source/common/config:utility_lib", + "//source/common/grpc:async_client_lib", + "//source/common/protobuf", + ], +) + +envoy_cc_library( + name = "pausable_ack_queue_lib", + srcs = ["pausable_ack_queue.cc"], + hdrs = ["pausable_ack_queue.h"], + deps = [ + ":update_ack_lib", + "//source/common/common:assert_lib", + ], +) + +envoy_cc_library( + name = "xds_source_id_lib", + srcs = ["xds_source_id.cc"], + hdrs = ["xds_source_id.h"], + deps = [ + "//envoy/config:xds_resources_delegate_interface", + ], +) + +envoy_cc_library( + name = "update_ack_lib", + hdrs = ["update_ack.h"], + deps = ["@com_google_googleapis//google/rpc:status_cc_proto"], +) + +envoy_cc_library( + name = "watch_map_lib", + srcs = ["watch_map.cc"], + hdrs = ["watch_map.h"], + deps = [ + "//envoy/config:custom_config_validators_interface", + "//envoy/config:subscription_interface", + "//envoy/config:xds_config_tracker_interface", + "//source/common/common:assert_lib", + "//source/common/common:cleanup_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/common:utility_lib", + "//source/common/config:decoded_resource_lib", + "//source/common/config:utility_lib", + "//source/common/config:xds_resource_lib", + "//source/common/protobuf", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc index 3dce1268b18d..c5ba997b6f5b 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc @@ -4,9 +4,9 @@ #include "source/common/config/decoded_resource_impl.h" #include "source/common/config/utility.h" -#include "source/common/config/xds_source_id.h" #include "source/common/memory/utils.h" #include "source/common/protobuf/protobuf.h" +#include "source/extensions/config_subscription/grpc/xds_source_id.h" #include "absl/container/btree_map.h" #include "absl/container/node_hash_set.h" diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/grpc_mux_impl.h index 280a37af6e87..83ac9c558c7e 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.h @@ -20,11 +20,11 @@ #include "source/common/common/logger.h" #include "source/common/common/utility.h" #include "source/common/config/api_version.h" -#include "source/common/config/grpc_stream.h" #include "source/common/config/ttl.h" #include "source/common/config/utility.h" #include "source/common/config/xds_context_params.h" #include "source/common/config/xds_resource.h" +#include "source/extensions/config_subscription/grpc/grpc_stream.h" #include "absl/container/node_hash_map.h" #include "xds/core/v3/resource_name.pb.h" diff --git a/source/common/config/grpc_stream.h b/source/extensions/config_subscription/grpc/grpc_stream.h similarity index 100% rename from source/common/config/grpc_stream.h rename to source/extensions/config_subscription/grpc/grpc_stream.h diff --git a/source/extensions/config_subscription/grpc/new_delta_subscription_state.h b/source/extensions/config_subscription/grpc/new_delta_subscription_state.h index dae05387428f..03d949bac1b8 100644 --- a/source/extensions/config_subscription/grpc/new_delta_subscription_state.h +++ b/source/extensions/config_subscription/grpc/new_delta_subscription_state.h @@ -10,9 +10,9 @@ #include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "source/common/config/api_version.h" -#include "source/common/config/pausable_ack_queue.h" #include "source/common/config/ttl.h" -#include "source/common/config/watch_map.h" +#include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" +#include "source/extensions/config_subscription/grpc/watch_map.h" #include "absl/container/node_hash_map.h" diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h index fd694c542102..39a27eeed84b 100644 --- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h @@ -11,12 +11,12 @@ #include "source/common/common/logger.h" #include "source/common/config/api_version.h" -#include "source/common/config/grpc_stream.h" -#include "source/common/config/pausable_ack_queue.h" -#include "source/common/config/watch_map.h" #include "source/common/grpc/common.h" #include "source/common/runtime/runtime_features.h" #include "source/extensions/config_subscription/grpc/delta_subscription_state.h" +#include "source/extensions/config_subscription/grpc/grpc_stream.h" +#include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" +#include "source/extensions/config_subscription/grpc/watch_map.h" namespace Envoy { namespace Config { diff --git a/source/extensions/config_subscription/grpc/old_delta_subscription_state.h b/source/extensions/config_subscription/grpc/old_delta_subscription_state.h index f1efcb554445..d5997c953bd9 100644 --- a/source/extensions/config_subscription/grpc/old_delta_subscription_state.h +++ b/source/extensions/config_subscription/grpc/old_delta_subscription_state.h @@ -10,9 +10,9 @@ #include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "source/common/config/api_version.h" -#include "source/common/config/pausable_ack_queue.h" #include "source/common/config/ttl.h" -#include "source/common/config/watch_map.h" +#include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" +#include "source/extensions/config_subscription/grpc/watch_map.h" #include "absl/container/node_hash_map.h" diff --git a/source/common/config/pausable_ack_queue.cc b/source/extensions/config_subscription/grpc/pausable_ack_queue.cc similarity index 95% rename from source/common/config/pausable_ack_queue.cc rename to source/extensions/config_subscription/grpc/pausable_ack_queue.cc index 6208fd909fb4..fca69a07851c 100644 --- a/source/common/config/pausable_ack_queue.cc +++ b/source/extensions/config_subscription/grpc/pausable_ack_queue.cc @@ -1,4 +1,4 @@ -#include "source/common/config/pausable_ack_queue.h" +#include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" #include diff --git a/source/common/config/pausable_ack_queue.h b/source/extensions/config_subscription/grpc/pausable_ack_queue.h similarity index 94% rename from source/common/config/pausable_ack_queue.h rename to source/extensions/config_subscription/grpc/pausable_ack_queue.h index 07af25fa9bfd..a3140b7831ad 100644 --- a/source/common/config/pausable_ack_queue.h +++ b/source/extensions/config_subscription/grpc/pausable_ack_queue.h @@ -2,7 +2,7 @@ #include -#include "source/common/config/update_ack.h" +#include "source/extensions/config_subscription/grpc/update_ack.h" #include "absl/container/flat_hash_map.h" diff --git a/source/common/config/update_ack.h b/source/extensions/config_subscription/grpc/update_ack.h similarity index 100% rename from source/common/config/update_ack.h rename to source/extensions/config_subscription/grpc/update_ack.h diff --git a/source/common/config/watch_map.cc b/source/extensions/config_subscription/grpc/watch_map.cc similarity index 99% rename from source/common/config/watch_map.cc rename to source/extensions/config_subscription/grpc/watch_map.cc index 803bc729d210..b9a1907fd3d3 100644 --- a/source/common/config/watch_map.cc +++ b/source/extensions/config_subscription/grpc/watch_map.cc @@ -1,4 +1,4 @@ -#include "source/common/config/watch_map.h" +#include "source/extensions/config_subscription/grpc/watch_map.h" #include "envoy/service/discovery/v3/discovery.pb.h" diff --git a/source/common/config/watch_map.h b/source/extensions/config_subscription/grpc/watch_map.h similarity index 100% rename from source/common/config/watch_map.h rename to source/extensions/config_subscription/grpc/watch_map.h diff --git a/source/extensions/config_subscription/grpc/xds_mux/BUILD b/source/extensions/config_subscription/grpc/xds_mux/BUILD index 9dd3a7318c7b..50cba69c40a6 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/BUILD +++ b/source/extensions/config_subscription/grpc/xds_mux/BUILD @@ -32,9 +32,9 @@ envoy_cc_library( "//source/common/config:api_version_lib", "//source/common/config:decoded_resource_lib", "//source/common/config:utility_lib", - "//source/common/config:xds_source_id_lib", "//source/common/grpc:common_lib", "//source/common/protobuf", + "//source/extensions/config_subscription/grpc:xds_source_id_lib", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) @@ -50,8 +50,8 @@ envoy_cc_library( "//envoy/local_info:local_info_interface", "//source/common/common:minimal_logger_lib", "//source/common/config:ttl_lib", - "//source/common/config:update_ack_lib", "//source/common/config:utility_lib", + "//source/extensions/config_subscription/grpc:update_ack_lib", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) @@ -65,12 +65,12 @@ envoy_cc_extension( ":sotw_subscription_state_lib", "//envoy/event:dispatcher_interface", "//envoy/grpc:async_client_interface", - "//source/common/config:grpc_stream_lib", - "//source/common/config:pausable_ack_queue_lib", - "//source/common/config:watch_map_lib", "//source/common/config:xds_context_params_lib", "//source/common/config:xds_resource_lib", "//source/common/memory:utils_lib", + "//source/extensions/config_subscription/grpc:grpc_stream_lib", + "//source/extensions/config_subscription/grpc:pausable_ack_queue_lib", + "//source/extensions/config_subscription/grpc:watch_map_lib", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h index e5434f9eaaeb..7669ca73cffd 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h @@ -20,10 +20,10 @@ #include "source/common/common/logger.h" #include "source/common/common/utility.h" #include "source/common/config/api_version.h" -#include "source/common/config/grpc_stream.h" -#include "source/common/config/pausable_ack_queue.h" -#include "source/common/config/watch_map.h" #include "source/common/grpc/common.h" +#include "source/extensions/config_subscription/grpc/grpc_stream.h" +#include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" +#include "source/extensions/config_subscription/grpc/watch_map.h" #include "source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h" #include "source/extensions/config_subscription/grpc/xds_mux/sotw_subscription_state.h" diff --git a/source/extensions/config_subscription/grpc/xds_mux/sotw_subscription_state.cc b/source/extensions/config_subscription/grpc/xds_mux/sotw_subscription_state.cc index e1a887663ffa..18bafc1a8015 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/sotw_subscription_state.cc +++ b/source/extensions/config_subscription/grpc/xds_mux/sotw_subscription_state.cc @@ -1,7 +1,7 @@ #include "source/extensions/config_subscription/grpc/xds_mux/sotw_subscription_state.h" #include "source/common/config/utility.h" -#include "source/common/config/xds_source_id.h" +#include "source/extensions/config_subscription/grpc/xds_source_id.h" namespace Envoy { namespace Config { diff --git a/source/extensions/config_subscription/grpc/xds_mux/subscription_state.h b/source/extensions/config_subscription/grpc/xds_mux/subscription_state.h index 41223d0829d4..3c7ec8d0c542 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/subscription_state.h +++ b/source/extensions/config_subscription/grpc/xds_mux/subscription_state.h @@ -11,9 +11,9 @@ #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/config/ttl.h" -#include "source/common/config/update_ack.h" #include "source/common/config/utility.h" #include "source/common/protobuf/protobuf.h" +#include "source/extensions/config_subscription/grpc/update_ack.h" #include "absl/strings/string_view.h" diff --git a/source/common/config/xds_source_id.cc b/source/extensions/config_subscription/grpc/xds_source_id.cc similarity index 90% rename from source/common/config/xds_source_id.cc rename to source/extensions/config_subscription/grpc/xds_source_id.cc index 090f1d4bff1e..8dcd7da287f3 100644 --- a/source/common/config/xds_source_id.cc +++ b/source/extensions/config_subscription/grpc/xds_source_id.cc @@ -1,4 +1,4 @@ -#include "source/common/config/xds_source_id.h" +#include "source/extensions/config_subscription/grpc/xds_source_id.h" #include "absl/strings/str_cat.h" diff --git a/source/common/config/xds_source_id.h b/source/extensions/config_subscription/grpc/xds_source_id.h similarity index 100% rename from source/common/config/xds_source_id.h rename to source/extensions/config_subscription/grpc/xds_source_id.h diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 60f8f952f964..dd5035c98579 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -31,22 +31,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "grpc_stream_test", - srcs = ["grpc_stream_test.cc"], - deps = [ - "//source/common/config:grpc_stream_lib", - "//test/common/stats:stat_test_utility_lib", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/test_common:logging_lib", - "//test/test_common:simulated_time_system_lib", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "grpc_subscription_impl_test", srcs = ["grpc_subscription_impl_test.cc"], @@ -112,14 +96,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "pausable_ack_queue_test", - srcs = ["pausable_ack_queue_test.cc"], - deps = [ - "//source/common/config:pausable_ack_queue_lib", - ], -) - envoy_cc_test( name = "metadata_test", srcs = ["metadata_test.cc"], @@ -182,20 +158,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "watch_map_test", - srcs = ["watch_map_test.cc"], - deps = [ - "//envoy/config:xds_config_tracker_interface", - "//source/common/config:watch_map_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/config:custom_config_validators_mocks", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_proto_library( name = "dummy_config_proto", srcs = ["dummy_config.proto"], @@ -289,11 +251,3 @@ envoy_cc_test( "//test/test_common:test_runtime_lib", ], ) - -envoy_cc_test( - name = "xds_source_id_test", - srcs = ["xds_source_id_test.cc"], - deps = [ - "//source/common/config:xds_source_id_lib", - ], -) diff --git a/test/extensions/config_subscription/grpc/BUILD b/test/extensions/config_subscription/grpc/BUILD index a6ee6147e4f4..b352ca281b13 100644 --- a/test/extensions/config_subscription/grpc/BUILD +++ b/test/extensions/config_subscription/grpc/BUILD @@ -195,3 +195,49 @@ envoy_cc_test( "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) + +envoy_cc_test( + name = "grpc_stream_test", + srcs = ["grpc_stream_test.cc"], + deps = [ + "//source/extensions/config_subscription/grpc:grpc_stream_lib", + "//test/common/stats:stat_test_utility_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/test_common:logging_lib", + "//test/test_common:simulated_time_system_lib", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + +envoy_cc_test( + name = "pausable_ack_queue_test", + srcs = ["pausable_ack_queue_test.cc"], + deps = [ + "//source/extensions/config_subscription/grpc:pausable_ack_queue_lib", + ], +) + +envoy_cc_test( + name = "watch_map_test", + srcs = ["watch_map_test.cc"], + deps = [ + "//envoy/config:xds_config_tracker_interface", + "//source/extensions/config_subscription/grpc:watch_map_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/config:custom_config_validators_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + +envoy_cc_test( + name = "xds_source_id_test", + srcs = ["xds_source_id_test.cc"], + deps = [ + "//source/extensions/config_subscription/grpc:xds_source_id_lib", + ], +) diff --git a/test/common/config/grpc_stream_test.cc b/test/extensions/config_subscription/grpc/grpc_stream_test.cc similarity index 99% rename from test/common/config/grpc_stream_test.cc rename to test/extensions/config_subscription/grpc/grpc_stream_test.cc index 259d2619cecd..6d594c2ff2f6 100644 --- a/test/common/config/grpc_stream_test.cc +++ b/test/extensions/config_subscription/grpc/grpc_stream_test.cc @@ -1,7 +1,7 @@ #include "envoy/service/discovery/v3/discovery.pb.h" -#include "source/common/config/grpc_stream.h" #include "source/common/protobuf/protobuf.h" +#include "source/extensions/config_subscription/grpc/grpc_stream.h" #include "test/common/stats/stat_test_utility.h" #include "test/mocks/common.h" diff --git a/test/common/config/pausable_ack_queue_test.cc b/test/extensions/config_subscription/grpc/pausable_ack_queue_test.cc similarity index 97% rename from test/common/config/pausable_ack_queue_test.cc rename to test/extensions/config_subscription/grpc/pausable_ack_queue_test.cc index f984193bf837..e52e3a32b414 100644 --- a/test/common/config/pausable_ack_queue_test.cc +++ b/test/extensions/config_subscription/grpc/pausable_ack_queue_test.cc @@ -1,4 +1,4 @@ -#include "source/common/config/pausable_ack_queue.h" +#include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" #include "gtest/gtest.h" diff --git a/test/common/config/watch_map_test.cc b/test/extensions/config_subscription/grpc/watch_map_test.cc similarity index 99% rename from test/common/config/watch_map_test.cc rename to test/extensions/config_subscription/grpc/watch_map_test.cc index 76cfe3176836..dcdbe9a418ee 100644 --- a/test/common/config/watch_map_test.cc +++ b/test/extensions/config_subscription/grpc/watch_map_test.cc @@ -6,7 +6,7 @@ #include "envoy/service/discovery/v3/discovery.pb.h" #include "envoy/stats/scope.h" -#include "source/common/config/watch_map.h" +#include "source/extensions/config_subscription/grpc/watch_map.h" #include "test/mocks/config/custom_config_validators.h" #include "test/mocks/config/mocks.h" diff --git a/test/common/config/xds_source_id_test.cc b/test/extensions/config_subscription/grpc/xds_source_id_test.cc similarity index 92% rename from test/common/config/xds_source_id_test.cc rename to test/extensions/config_subscription/grpc/xds_source_id_test.cc index c138d56cd3b1..c3ad0a37bf7c 100644 --- a/test/common/config/xds_source_id_test.cc +++ b/test/extensions/config_subscription/grpc/xds_source_id_test.cc @@ -1,4 +1,4 @@ -#include "source/common/config/xds_source_id.h" +#include "source/extensions/config_subscription/grpc/xds_source_id.h" #include "gtest/gtest.h" diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index e26c99615b49..76f798634a85 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -7,7 +7,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/api:82.6" "source/common/api/posix:81.3" "source/common/common/posix:92.7" -"source/common/config:95.6" +"source/common/config:94.8" "source/common/crypto:88.1" "source/common/event:95.1" # Emulated edge events guards don't report LCOV "source/common/filesystem/posix:96.2" # FileReadToEndNotReadable fails in some env; createPath can't test all failure branches. From d732f9a5a58ca412d5dc13835c8f06897cfb1c5f Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 1 Jun 2023 13:38:30 -0400 Subject: [PATCH 438/740] router: cleaning up 1xx coalescing now done in codec filter (#27736) Signed-off-by: Alyssa Wilk --- source/common/router/router.cc | 12 +------- source/common/router/router.h | 8 ++--- test/common/router/router_test.cc | 50 ------------------------------- 3 files changed, 4 insertions(+), 66 deletions(-) diff --git a/source/common/router/router.cc b/source/common/router/router.cc index b03ea8b89a5a..6e50cd72c999 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -1397,17 +1397,7 @@ void Filter::onUpstream1xxHeaders(Http::ResponseHeaderMapPtr&& headers, // the complexity until someone asks for it. retry_state_.reset(); - // We coalesce 1xx headers here, to prevent encoder filters and HCM from having to worry - // about this. This is done in the router filter, rather than UpstreamRequest, since we want to - // potentially coalesce across retries and multiple upstream requests in the future, even though - // we currently don't support retry after 1xx. - // It's plausible that this functionality might need to move to HCM in the future for internal - // redirects, but we would need to maintain the "only call encode1xxHeaders() once" - // invariant. - if (!downstream_1xx_headers_encoded_) { - downstream_1xx_headers_encoded_ = true; - callbacks_->encode1xxHeaders(std::move(headers)); - } + callbacks_->encode1xxHeaders(std::move(headers)); } void Filter::resetAll() { diff --git a/source/common/router/router.h b/source/common/router/router.h index 3245b1452147..c3c3b61c8180 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -380,10 +380,9 @@ class Filter : Logger::Loggable, public: Filter(FilterConfig& config, FilterStats& stats) : config_(config), stats_(stats), grpc_request_(false), exclude_http_code_stats_(false), - downstream_1xx_headers_encoded_(false), downstream_response_started_(false), - downstream_end_stream_(false), is_retry_(false), request_buffer_overflowed_(false), - streaming_shadows_( - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.streaming_shadow")) {} + downstream_response_started_(false), downstream_end_stream_(false), is_retry_(false), + request_buffer_overflowed_(false), streaming_shadows_(Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.streaming_shadow")) {} ~Filter() override; @@ -672,7 +671,6 @@ class Filter : Logger::Loggable, FilterUtility::HedgingParams hedging_params_; bool grpc_request_ : 1; bool exclude_http_code_stats_ : 1; - bool downstream_1xx_headers_encoded_ : 1; bool downstream_response_started_ : 1; bool downstream_end_stream_ : 1; bool is_retry_ : 1; diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index de30e05444be..82f7711cb50c 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -3504,56 +3504,6 @@ TEST_F(RouterTest, RetryUpstreamResetResponseStarted) { callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); } -// The router filter is responsible for not propagating 100-continue headers after the initial 100. -// TODO(alyssawilk) remove coalescing with old code deprecation. -TEST_F(RouterTest, Coalesce1xxHeaders) { - // Setup. - NiceMock encoder1; - Http::ResponseDecoder* response_decoder = nullptr; - expectNewStreamWithImmediateEncoder(encoder1, &response_decoder, Http::Protocol::Http10); - - expectResponseTimerCreate(); - - Http::TestRequestHeaderMapImpl headers; - HttpTestUtility::addDefaultHeaders(headers); - router_->decodeHeaders(headers, true); - EXPECT_EQ(1U, - callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); - - // Initial 100-continue, this is processed normally. - EXPECT_CALL(callbacks_, encode1xxHeaders_(_)); - { - Http::ResponseHeaderMapPtr continue_headers( - new Http::TestResponseHeaderMapImpl{{":status", "100"}}); - // NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) - response_decoder->decode1xxHeaders(std::move(continue_headers)); - } - EXPECT_EQ( - 1U, - cm_.thread_local_cluster_.cluster_.info_->stats_store_.counter("upstream_rq_100").value()); - - // No encode1xxHeaders() invocation for the second 100-continue (but we continue to track - // stats from upstream). - EXPECT_CALL(callbacks_, encode1xxHeaders_(_)).Times(0); - { - Http::ResponseHeaderMapPtr continue_headers( - new Http::TestResponseHeaderMapImpl{{":status", "100"}}); - // NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) - response_decoder->decode1xxHeaders(std::move(continue_headers)); - } - // With the filter manager coalescing, the router only sees 1 100. - EXPECT_EQ( - 1U, - cm_.thread_local_cluster_.cluster_.info_->stats_store_.counter("upstream_rq_100").value()); - - // Reset stream and cleanup. - EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); - encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); - EXPECT_EQ(1U, - callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); -} - TEST_F(RouterTest, RetryUpstreamReset1xxResponseStarted) { NiceMock encoder1; Http::ResponseDecoder* response_decoder = nullptr; From 5fefd9d4f78d169e3cb71f4e2c53a1b5d575334b Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 1 Jun 2023 20:48:43 +0100 Subject: [PATCH 439/740] changelog: Add `wasmtime`, `c-ares` CVE updates (#27753) Signed-off-by: Ryan Northey --- changelogs/current.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index c6eff7753bf3..aff04bd00c7d 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -140,6 +140,18 @@ bug_fixes: This was fixed now by marking the cookie as httpOnly. This behavioral change can be temporarily reverted by setting runtime guard ``envoy.reloadable_features.oauth_make_token_cookie_httponly`` to false. +- area: dependency + change: | + update Wasmtime and related deps -> 9.0.3 to resolve + `CVE-2023-30624 `_. +- area: dependency + change: | + update C-ares -> 1.91.1 to resolve: + + - `CVE-2023-31130 `_. + - `CVE-2023-31147 `_. + - `CVE-2023-31124 `_. + - `CVE-2023-32067 `_. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` From 517d2dba98ba7691d1642caf2ef4dd42a6daf029 Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Thu, 1 Jun 2023 23:11:24 -0400 Subject: [PATCH 440/740] docs: Add info about xDS cluster ordering in the Bootstrap (#27706) Clarification for a workaround to #27702. Signed-off-by: Ali Beyad --- docs/root/api-docs/xds_protocol.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/root/api-docs/xds_protocol.rst b/docs/root/api-docs/xds_protocol.rst index 0ba5abb942ed..a9c36ef1f4f6 100644 --- a/docs/root/api-docs/xds_protocol.rst +++ b/docs/root/api-docs/xds_protocol.rst @@ -252,6 +252,14 @@ how to contact the ADS server, which will be used whenever a :ref:`ConfigSource management server) contains an :ref:`AggregatedConfigSource ` message. +A current limitation in Envoy is that any xDS :ref:`Cluster ` resources +should be specified first in the `static_resources` field of the Bootstrap configuration prior to any static +:ref:`Cluster ` resources that depend on the xDS cluster. Failure to do +so will result in slower Envoy initialization (see the `GitHub issue `_ +for details). As an example, if a cluster depends on an xDS :ref:`Cluster ` +for SDS to configure the secrets on a transport socket, the xDS :ref:`Cluster ` +should be specified first in the `static_resources` field, before the cluster with the transport socket secret is specified. + In a gRPC client that uses xDS, only ADS is supported, and the bootstrap file contains the name of the ADS server, which will be used for all resources. The :ref:`ConfigSource ` messages in the :ref:`Listener ` and From 4942648504249567f41691ca8fe3258ad1acd3f6 Mon Sep 17 00:00:00 2001 From: code Date: Fri, 2 Jun 2023 16:37:07 +0800 Subject: [PATCH 441/740] generic proxy: fix flaky integration test (#27763) Signed-off-by: wbpcode --- .../generic_proxy/filters/network/test/integration_test.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/generic_proxy/filters/network/test/integration_test.cc b/contrib/generic_proxy/filters/network/test/integration_test.cc index 53c948f247da..d68f2d3b154e 100644 --- a/contrib/generic_proxy/filters/network/test/integration_test.cc +++ b/contrib/generic_proxy/filters/network/test/integration_test.cc @@ -410,8 +410,12 @@ TEST_P(IntegrationTest, MultipleRequests) { request_2.protocol_ = "fake_fake_fake"; request_2.data_ = {{"version", "v1"}, {"stream_id", "2"}}; - // Send the second request with the same stream id and expect the connection to be closed. + // Reset request encoder callback. + request_encoder_callback_ = std::make_shared(); + + // Send the second request with the different stream id and expect the connection to be alive. sendRequestForTest(request_2); + waitForUpstreamRequestForTest(request_encoder_callback_->request_bytes_, nullptr); FakeStreamCodecFactory::FakeResponse response_2; response_2.protocol_ = "fake_fake_fake"; From 08b7a8922702b49c0f2191ee58a48a4850206a35 Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Fri, 2 Jun 2023 15:37:04 +0300 Subject: [PATCH 442/740] sds: Add support to hot-reload CRL files (#27751) * sds: Add support to hot-reload CRL files Signed-off-by: Tero Saarni --- .../transport_sockets/tls/v3/common.proto | 5 +++++ changelogs/current.yaml | 6 ++++++ source/common/secret/sds_api.cc | 16 +++++++++++----- source/common/secret/sds_api.h | 3 +++ test/common/secret/sds_api_test.cc | 16 ++++++++++++++-- 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/api/envoy/extensions/transport_sockets/tls/v3/common.proto b/api/envoy/extensions/transport_sockets/tls/v3/common.proto index 4780858efa36..66c8c7978def 100644 --- a/api/envoy/extensions/transport_sockets/tls/v3/common.proto +++ b/api/envoy/extensions/transport_sockets/tls/v3/common.proto @@ -505,6 +505,11 @@ message CertificateValidationContext { // from that chain. This default behavior can be altered by setting // :ref:`only_verify_leaf_cert_crl ` to // true. + // + // If ``crl`` is a filesystem path, a watch will be added to the parent + // directory for any file moves to support rotation. This currently only + // applies to dynamic secrets, when the ``CertificateValidationContext`` is + // delivered via SDS. config.core.v3.DataSource crl = 7; // If specified, Envoy will not reject expired certificates. diff --git a/changelogs/current.yaml b/changelogs/current.yaml index aff04bd00c7d..7fc60f892d9a 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -292,6 +292,12 @@ new_features: change: | added CEL(Common Expression Language) matcher support :ref:`CEL data input ` and :ref:`CEL input matcher `. +- area: tls + change: | + Added support for hot-reloading CRL file when the file changes on disk. + This works with dynamic secrets when + :ref:`CertificateValidationContext ` + is delivered via SDS. deprecated: - area: access_log diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index 253391f44940..e20ca282f22e 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -195,11 +195,17 @@ std::vector TlsCertificateSdsApi::getDataSourceFilenames() { std::vector CertificateValidationContextSdsApi::getDataSourceFilenames() { std::vector files; - if (sds_certificate_validation_context_secrets_ && - sds_certificate_validation_context_secrets_->has_trusted_ca() && - sds_certificate_validation_context_secrets_->trusted_ca().specifier_case() == - envoy::config::core::v3::DataSource::SpecifierCase::kFilename) { - files.push_back(sds_certificate_validation_context_secrets_->trusted_ca().filename()); + if (sds_certificate_validation_context_secrets_) { + if (sds_certificate_validation_context_secrets_->has_trusted_ca() && + sds_certificate_validation_context_secrets_->trusted_ca().specifier_case() == + envoy::config::core::v3::DataSource::SpecifierCase::kFilename) { + files.push_back(sds_certificate_validation_context_secrets_->trusted_ca().filename()); + } + if (sds_certificate_validation_context_secrets_->has_crl() && + sds_certificate_validation_context_secrets_->crl().specifier_case() == + envoy::config::core::v3::DataSource::SpecifierCase::kFilename) { + files.push_back(sds_certificate_validation_context_secrets_->crl().filename()); + } } return files; } diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index edc5db1e0ad8..2c5a08bc5281 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -284,6 +284,9 @@ class CertificateValidationContextSdsApi : public SdsApi, // We replace path based secrets with inlined secrets on update. resolveDataSource(files, *resolved_certificate_validation_context_secrets_->mutable_trusted_ca()); + if (sds_certificate_validation_context_secrets_->has_crl()) { + resolveDataSource(files, *resolved_certificate_validation_context_secrets_->mutable_crl()); + } } void diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index de86349a0783..6cb640f23464 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -293,6 +293,7 @@ class CertificateValidationContextSdsRotationApiTest : public testing::TestWithP } void onConfigUpdate(const std::string& trusted_ca_path, const std::string& trusted_ca_value, + const std::string& crl_path, const std::string& crl_value, const std::string& watch_path) { const std::string yaml = fmt::format( R"EOF( @@ -300,17 +301,24 @@ class CertificateValidationContextSdsRotationApiTest : public testing::TestWithP validation_context: trusted_ca: filename: "{}" + crl: + filename: "{}" allow_expired_certificate: true )EOF", - trusted_ca_path); + trusted_ca_path, crl_path); envoy::extensions::transport_sockets::tls::v3::Secret typed_secret; TestUtility::loadFromYaml(yaml, typed_secret); const auto decoded_resources = TestUtility::decodeResources({typed_secret}); auto* watcher = new Filesystem::MockWatcher(); EXPECT_CALL(filesystem_, fileReadToEnd(trusted_ca_path)).WillOnce(Return(trusted_ca_value)); + EXPECT_CALL(filesystem_, fileReadToEnd(crl_path)).WillOnce(Return(crl_value)); EXPECT_CALL(secret_callback_, onAddOrUpdateSecret()); EXPECT_CALL(mock_dispatcher_, createFilesystemWatcher_()).WillOnce(Return(watcher)); + EXPECT_CALL(*watcher, addWatch(watch_path, Filesystem::Watcher::Events::MovedTo, _)) + .WillOnce(Invoke([this](absl::string_view, uint32_t, Filesystem::Watcher::OnChangedCb cb) { + watch_cbs_.push_back(cb); + })); EXPECT_CALL(*watcher, addWatch(watch_path, Filesystem::Watcher::Events::MovedTo, _)) .WillOnce(Invoke([this](absl::string_view, uint32_t, Filesystem::Watcher::OnChangedCb cb) { watch_cbs_.push_back(cb); @@ -417,15 +425,19 @@ TEST_P(TlsCertificateSdsRotationApiTest, FailedRotation) { // Basic rotation of CertificateValidationContext. TEST_P(CertificateValidationContextSdsRotationApiTest, CertificateValidationContext) { InSequence s; - onConfigUpdate("/foo/bar/ca.pem", "a", "/foo/bar/"); + onConfigUpdate("/foo/bar/ca.pem", "a", "/foo/bar/crl.pem", "b", "/foo/bar/"); EXPECT_CALL(filesystem_, fileReadToEnd("/foo/bar/ca.pem")).WillOnce(Return("c")); + EXPECT_CALL(filesystem_, fileReadToEnd("/foo/bar/crl.pem")).WillOnce(Return("d")); EXPECT_CALL(filesystem_, fileReadToEnd("/foo/bar/ca.pem")).WillOnce(Return("c")); + EXPECT_CALL(filesystem_, fileReadToEnd("/foo/bar/crl.pem")).WillOnce(Return("d")); + EXPECT_CALL(secret_callback_, onAddOrUpdateSecret()); watch_cbs_[0](Filesystem::Watcher::Events::MovedTo); const auto& secret = *sds_api_->secret(); EXPECT_EQ("c", secret.trusted_ca().inline_bytes()); + EXPECT_EQ("d", secret.crl().inline_bytes()); } // Hash consistency verification prevents races. From d29dcb1ce058edcfcd80737c0a097f776b6759fa Mon Sep 17 00:00:00 2001 From: phlax Date: Fri, 2 Jun 2023 18:53:52 +0100 Subject: [PATCH 443/740] ci: Replace python go proto script with ci target (#27675) Signed-off-by: Ryan Northey --- ci/build_setup.sh | 6 ++ ci/do_ci.sh | 55 ++++++++-- tools/api/generate_go_protobuf.py | 161 +----------------------------- 3 files changed, 55 insertions(+), 167 deletions(-) diff --git a/ci/build_setup.sh b/ci/build_setup.sh index 9ad3a8dd0577..f0de186abd60 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -7,6 +7,10 @@ set -e +if [[ -n "$NO_BUILD_SETUP" ]]; then + exit +fi + export PPROF_PATH=/thirdparty_build/bin/pprof [ -z "${NUM_CPUS}" ] && NUM_CPUS=$(grep -c ^processor /proc/cpuinfo) @@ -196,3 +200,5 @@ if [[ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "true" ]]; then else echo "Skip setting up Envoy Filter Example." fi + +export NO_BUILD_SETUP=1 diff --git a/ci/do_ci.sh b/ci/do_ci.sh index b0f178293f05..2d7b61298396 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -8,17 +8,15 @@ set -e # TODO(phlax): Clarify and/or integrate SRCDIR and ENVOY_SRCDIR export SRCDIR="${SRCDIR:-$PWD}" export ENVOY_SRCDIR="${ENVOY_SRCDIR:-$PWD}" -NO_BUILD_SETUP="${NO_BUILD_SETUP:-}" -if [[ -z "$NO_BUILD_SETUP" ]]; then - # shellcheck source=ci/setup_cache.sh - . "$(dirname "$0")"/setup_cache.sh - # shellcheck source=ci/build_setup.sh - . "$(dirname "$0")"/build_setup.sh +# shellcheck source=ci/setup_cache.sh +. "$(dirname "$0")"/setup_cache.sh +# shellcheck source=ci/build_setup.sh +. "$(dirname "$0")"/build_setup.sh + +echo "building using ${NUM_CPUS} CPUs" +echo "building for ${ENVOY_BUILD_ARCH}" - echo "building using ${NUM_CPUS} CPUs" - echo "building for ${ENVOY_BUILD_ARCH}" -fi cd "${SRCDIR}" if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then @@ -202,8 +200,6 @@ case $CI_TARGET in //tools/protoprint:protoprint_test echo "Validating API structure..." "${ENVOY_SRCDIR}"/tools/api/validate_structure.py - echo "Validate Golang protobuf generation..." - "${ENVOY_SRCDIR}"/tools/api/generate_go_protobuf.py echo "Testing API..." bazel_with_collection \ test "${BAZEL_BUILD_OPTIONS[@]}" \ @@ -215,6 +211,43 @@ case $CI_TARGET in echo "Building API..." bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ -c fastbuild @envoy_api//envoy/... + if [[ -n "$ENVOY_API_ONLY" ]]; then + exit 0 + fi + ;& + + api.go) + if [[ -z "$NO_BUILD_SETUP" ]]; then + setup_clang_toolchain + fi + GO_IMPORT_BASE="github.com/envoyproxy/go-control-plane" + GO_TARGETS=(@envoy_api//...) + read -r -a GO_PROTOS <<< "$(bazel query "${BAZEL_GLOBAL_OPTIONS[@]}" "kind('go_proto_library', ${GO_TARGETS[*]})" | tr '\n' ' ')" + echo "${GO_PROTOS[@]}" | grep -q envoy_api || echo "No go proto targets found" + bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ + --experimental_proto_descriptor_sets_include_source_info \ + --remote_download_outputs=all \ + "${GO_PROTOS[@]}" + rm -rf build_go + mkdir -p build_go + echo "Copying go protos -> build_go" + BAZEL_BIN="$(bazel info "${BAZEL_BUILD_OPTIONS[@]}" bazel-bin)" + for GO_PROTO in "${GO_PROTOS[@]}"; do + # strip @envoy_api// + RULE_DIR="$(echo "${GO_PROTO:12}" | cut -d: -f1)" + PROTO="$(echo "${GO_PROTO:12}" | cut -d: -f2)" + INPUT_DIR="${BAZEL_BIN}/external/envoy_api/${RULE_DIR}/${PROTO}_/${GO_IMPORT_BASE}/${RULE_DIR}" + OUTPUT_DIR="build_go/${RULE_DIR}" + mkdir -p "$OUTPUT_DIR" + if [[ ! -e "$INPUT_DIR" ]]; then + echo "Unable to find input ${INPUT_DIR}" >&2 + exit 1 + fi + # echo "Copying go files ${INPUT_DIR} -> ${OUTPUT_DIR}" + while read -r GO_FILE; do + cp -a "$GO_FILE" "$OUTPUT_DIR" + done <<< "$(find "$INPUT_DIR" -name "*.go")" + done ;; api_compat) diff --git a/tools/api/generate_go_protobuf.py b/tools/api/generate_go_protobuf.py index 2fc3728f56b5..af3a85b29129 100755 --- a/tools/api/generate_go_protobuf.py +++ b/tools/api/generate_go_protobuf.py @@ -1,164 +1,13 @@ #!/usr/bin/env python3 -from subprocess import check_output, STDOUT, CalledProcessError -import argparse -import glob -import os -import shlex -import shutil import sys -import re -# Needed for CI to pass down bazel options. -BAZEL_BUILD_OPTIONS = shlex.split(os.environ.get('BAZEL_BUILD_OPTION_LIST', '')) -BAZEL_GLOBAL_OPTIONS = shlex.split(os.environ.get('BAZEL_GLOBAL_OPTION_LIST', '')) -BAZEL_STARTUP_OPTIONS = shlex.split(os.environ.get('BAZEL_STARTUP_OPTION_LIST', '')) -TARGETS = '@envoy_api//...' -IMPORT_BASE = 'github.com/envoyproxy/go-control-plane' -REPO_BASE = 'go-control-plane' -BRANCH = 'main' -MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ ' -USER_NAME = 'go-control-plane(Azure Pipelines)' -USER_EMAIL = 'go-control-plane@users.noreply.github.com' - - -def generate_protobufs(targets, output, api_repo): - bazel_bin = check_output( - ['bazel', *BAZEL_STARTUP_OPTIONS, 'info', *BAZEL_BUILD_OPTIONS, - 'bazel-bin']).decode().strip() - go_protos = check_output([ - 'bazel', - *BAZEL_STARTUP_OPTIONS, - 'query', - *BAZEL_GLOBAL_OPTIONS, - 'kind("go_proto_library", %s)' % targets, - ]).split() - - # Each rule has the form @envoy_api//foo/bar:baz_go_proto. - # First build all the rules to ensure we have the output files. - # We preserve source info so comments are retained on generated code. - try: - check_output([ - 'bazel', *BAZEL_STARTUP_OPTIONS, 'build', '-c', 'fastbuild', - '--experimental_proto_descriptor_sets_include_source_info' - ] + BAZEL_BUILD_OPTIONS + go_protos, - stderr=STDOUT) - except CalledProcessError as e: - print(e.output) - raise e - - for rule in go_protos: - # Example rule: - # @envoy_api//envoy/config/bootstrap/v2:pkg_go_proto - # - # Example generated directory: - # bazel-bin/external/envoy_api/envoy/config/bootstrap/v2/linux_amd64_stripped/pkg_go_proto%/github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2/ - # - # Example output directory: - # go_out/envoy/config/bootstrap/v2 - rule_dir, proto = rule.decode().rsplit('//', 1)[1].rsplit(':', 1) - - prefix = '' if not api_repo else os.path.join('external', api_repo) - input_dir = os.path.join(bazel_bin, prefix, rule_dir, proto + '_', IMPORT_BASE, rule_dir) - input_files = glob.glob(os.path.join(input_dir, '*.go')) - output_dir = os.path.join(output, rule_dir) - - # Ensure the output directory exists - os.makedirs(output_dir, 0o755, exist_ok=True) - for generated_file in input_files: - shutil.copy(generated_file, output_dir) - print('Go artifacts placed into: ' + output) - - -def git(repo, *args): - cmd = ['git'] - if repo: - cmd = cmd + ['-C', repo] - for arg in args: - cmd = cmd + [arg] - return check_output(cmd).decode() - - -def clone_go_protobufs(repo): - # Create a local clone of go-control-plane - git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo, '-b', BRANCH) - - -def find_last_sync_sha(repo): - # Determine last envoyproxy/envoy SHA in envoyproxy/go-control-plane - last_commit = git(repo, 'log', '--grep=' + MIRROR_MSG, '-n', '1', '--format=%B').strip() - # Initial SHA from which the APIs start syncing. Prior to that it was done manually. - if last_commit == "": - return 'e7f0b7176efdc65f96eb1697b829d1e6187f4502' - m = re.search(MIRROR_MSG + '(\w+)', last_commit) - return m.group(1) - - -def updated_since_sha(repo, last_sha): - # Determine if there are changes to API since last SHA - return git(None, 'rev-list', '%s..HEAD' % last_sha).split() - - -def write_revision_info(repo, sha): - # Put a file in the generated code root containing the latest mirrored SHA - dst = os.path.join(repo, 'envoy', 'COMMIT') - with open(dst, 'w') as fh: - fh.write(sha) - - -def sync_go_protobufs(output, repo): - for folder in ['envoy', 'contrib']: - # Sync generated content against repo and return true if there is a commit necessary - dst = os.path.join(repo, folder) - # Remove subtree in repo - git(repo, 'rm', '-r', '--ignore-unmatch', folder) - # Copy subtree from output to repo - shutil.copytree(os.path.join(output, folder), dst) - git(repo, 'add', folder) - - -def publish_go_protobufs(repo, sha): - # Publish generated files with the last SHA changes to API - git(repo, 'config', 'user.name', USER_NAME) - git(repo, 'config', 'user.email', USER_EMAIL) - git(repo, 'add', 'envoy') - git(repo, 'add', 'contrib') - git(repo, 'commit', '--allow-empty', '-s', '-m', MIRROR_MSG + sha) - git(repo, 'push', 'origin', BRANCH) - - -def updated(repo): - return len([ - f for f in git(repo, 'diff', 'HEAD', '--name-only').splitlines() if f != 'envoy/COMMIT' - ]) > 0 +def main(): + # TODO(phlax): remove this 12/23 + sys.stderr.write("This script has been removed, please use `ci/do_ci.sh api.go` instead\n") + sys.exit(1) if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='Generate Go protobuf files and sync with go-control-plane') - parser.add_argument('--sync', action='store_true') - parser.add_argument('--output_base', default='build_go') - parser.add_argument('--targets', default=TARGETS) - parser.add_argument('--api_repo', default="envoy_api") - args = parser.parse_args() - - workspace = check_output( - ['bazel', *BAZEL_STARTUP_OPTIONS, 'info', *BAZEL_BUILD_OPTIONS, - 'workspace']).decode().strip() - output = os.path.join(workspace, args.output_base) - generate_protobufs(args.targets, output, args.api_repo) - if not args.sync: - print('Skipping sync with go-control-plane') - sys.exit() - - repo = os.path.join(workspace, REPO_BASE) - clone_go_protobufs(repo) - sync_go_protobufs(output, repo) - last_sha = find_last_sync_sha(repo) - changes = updated_since_sha(repo, last_sha) - if updated(repo): - print('Changes detected: %s' % changes) - new_sha = changes[0] - write_revision_info(repo, new_sha) - publish_go_protobufs(repo, new_sha) + main() From d0fd0258e52de718ef29b52699fe0fb82b29fafc Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Fri, 2 Jun 2023 15:05:04 -0400 Subject: [PATCH 444/740] Fuzzer: Fix ext_proc fuzz bug (#27639) * ext_proc should not open an additional stream if we've completed processing. Signed-off-by: Yanjun Xiang --- .../filters/http/ext_proc/ext_proc.cc | 8 ++- ...h-232906f11069ff930e29c13c1a3ae5a2c3202c57 | 70 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-232906f11069ff930e29c13c1a3ae5a2c3202c57 diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index 7661048511ac..0ef4f84c8274 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -118,7 +118,13 @@ void Filter::setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks& callb } Filter::StreamOpenState Filter::openStream() { - ENVOY_BUG(!processing_complete_, "openStream should not have been called"); + // External processing is completed. This means there is no need to send any further + // message to the server for processing. Just return IgnoreError so the filter + // will return FilterHeadersStatus::Continue. + if (processing_complete_) { + ENVOY_LOG(debug, "External processing is completed when trying to open the gRPC stream"); + return StreamOpenState::IgnoreError; + } if (!stream_) { ENVOY_LOG(debug, "Opening gRPC stream to external processor"); stream_ = client_->start(*this, grpc_service_, decoder_callbacks_->streamInfo()); diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-232906f11069ff930e29c13c1a3ae5a2c3202c57 b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-232906f11069ff930e29c13c1a3ae5a2c3202c57 new file mode 100644 index 000000000000..9bfa96c62f75 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/crash-232906f11069ff930e29c13c1a3ae5a2c3202c57 @@ -0,0 +1,70 @@ +config { + grpc_service { + google_grpc { + target_uri: "test_uri" + channel_credentials { + ssl_credentials { + private_key { + inline_string: "/" + } + } + } + stat_prefix: "test_uri" + credentials_factory_name: "test_uri" + config { + } + } + initial_metadata { + key: "test_uri" + } + } + failure_mode_allow: true + processing_mode { + response_header_mode: SKIP + } + request_attributes: "(" + response_attributes: "test_uri" + response_attributes: "9\000\000\000\000\000\000\000" + stat_prefix: "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000." +} +request { + headers { + headers { + key: "(" + } + headers { + key: "test_uri" + } + } + http_body { + data: "\360\270\270\270\360\270\270\270\360\270\270\270\360\270\270\270\360\270\270\270\360\270\270\270\360\270\270\270\360\270\270\270\360\270\270\270\360\270\270\270\360\270\270\270\360\270\270\270\350\270\270" + } +} +response { + request_headers { + response { + header_mutation { + remove_headers: "f" + remove_headers: "f" + } + } + } + dynamic_metadata { + fields { + key: "" + value { + } + } + fields { + key: "" + value { + } + } + } + mode_override { + request_header_mode: SKIP + request_body_mode: BUFFERED + response_body_mode: BUFFERED + request_trailer_mode: SEND + } +} From e20ba06ff3719862212780d413b2405de8f06460 Mon Sep 17 00:00:00 2001 From: botengyao Date: Fri, 2 Jun 2023 15:12:00 -0400 Subject: [PATCH 445/740] nit: setOverrideConfigForTests comment clean up (#27770) comment clean up Signed-off-by: Boteng Yao --- mobile/test/common/integration/test_engine_builder.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/test/common/integration/test_engine_builder.h b/mobile/test/common/integration/test_engine_builder.h index 1c969046249d..7fb6d6be3e0a 100644 --- a/mobile/test/common/integration/test_engine_builder.h +++ b/mobile/test/common/integration/test_engine_builder.h @@ -7,8 +7,8 @@ namespace Envoy { // A wrapper class around EngineBuilder, specifically for mobile tests. // // Mobile tests often supply their own configuration for convenience, instead of using the -// EngineBuilder APIs. This wrapper class builds an Envoy Mobile Engine with the ability to -// access setOverrideConfigForTests(), which is a protected method inside EngineBuilder. +// EngineBuilder APIs. This wrapper class can build an Envoy Mobile Engine through createEngine +// with a customized bootstrap configuration. class TestEngineBuilder : public Platform::EngineBuilder { public: virtual ~TestEngineBuilder() {} From ae44069e7185531fdcb92d776eb9f62d1721ff6d Mon Sep 17 00:00:00 2001 From: danzh Date: Fri, 2 Jun 2023 15:36:19 -0400 Subject: [PATCH 446/740] deprecate runtime guard `envoy.reloadable_features.tls_async_cert_validation` (#27656) Risk Level: low Testing: existing tests Docs Changes: release note Release Notes: removed_config_or_runtime Platform Specific Features: N/A Fixes #26908 Signed-off-by: Dan Zhang --- changelogs/current.yaml | 3 + .../platform_bridge_cert_validator.h | 8 - .../common/quic/envoy_quic_proof_verifier.cc | 68 +---- .../common/quic/envoy_quic_proof_verifier.h | 7 - source/common/runtime/runtime_features.cc | 1 - .../tls/cert_validator/cert_validator.h | 12 - .../tls/cert_validator/default_validator.cc | 31 --- .../tls/cert_validator/default_validator.h | 4 - .../cert_validator/spiffe/spiffe_validator.cc | 17 -- .../cert_validator/spiffe/spiffe_validator.h | 3 - .../transport_sockets/tls/context_impl.cc | 77 +----- .../transport_sockets/tls/context_impl.h | 6 - .../transport_sockets/tls/ssl_handshaker.cc | 1 - .../quic/envoy_quic_proof_verifier_test.cc | 71 ++---- .../cert_validator/default_validator_test.cc | 26 +- .../spiffe/spiffe_validator_test.cc | 239 ++++++------------ .../tls/cert_validator/timed_cert_validator.h | 7 - .../tls/integration/ssl_integration_test.cc | 12 - .../transport_sockets/tls/ssl_socket_test.cc | 83 ++---- .../integration/quic_http_integration_test.cc | 12 - 20 files changed, 165 insertions(+), 523 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 7fc60f892d9a..932d78b1e19f 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -179,6 +179,9 @@ removed_config_or_runtime: - area: header_formatters change: | removed runtime key ``envoy.reloadable_features.unified_header_formatter`` and legacy code paths. +- area: tls + change: | + remove runtime key ``envoy.reloadable_features.tls_async_cert_validation`` and legacy code paths. - area: config change: | removed runtime key ``envoy.reloadable_features.delta_xds_subscription_state_tracking_fix`` and legacy code paths. diff --git a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.h b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.h index c2912c77799b..db190635e03e 100644 --- a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.h +++ b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.h @@ -37,14 +37,6 @@ class PlatformBridgeCertValidator : public CertValidator, Logger::Loggable daysUntilFirstCertExpires() const override { return absl::nullopt; } Envoy::Ssl::CertificateDetailsPtr getCaCertInformation() const override { return nullptr; } // Return empty string diff --git a/source/common/quic/envoy_quic_proof_verifier.cc b/source/common/quic/envoy_quic_proof_verifier.cc index 0f2404e90132..b971b0a5ced1 100644 --- a/source/common/quic/envoy_quic_proof_verifier.cc +++ b/source/common/quic/envoy_quic_proof_verifier.cc @@ -43,16 +43,14 @@ class QuicValidateResultCallback : public Ssl::ValidateResultCallback { void onCertValidationResult(bool succeeded, Ssl::ClientValidationStatus /*detailed_status*/, const std::string& error_details, uint8_t /*tls_alert*/) override { + std::string error; if (!succeeded) { - std::unique_ptr details = std::make_unique(false); - quic_callback_->Run(succeeded, error_details, &details); - return; + error = error_details; + } else { + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(leaf_cert_); + succeeded = verifyLeafCertMatchesHostname(*cert_view, hostname_, &error); } - std::string error; - - std::unique_ptr cert_view = - quic::CertificateView::ParseSingleCertificate(leaf_cert_); - succeeded = verifyLeafCertMatchesHostname(*cert_view, hostname_, &error); std::unique_ptr details = std::make_unique(succeeded); quic_callback_->Run(succeeded, error, &details); @@ -69,8 +67,8 @@ class QuicValidateResultCallback : public Ssl::ValidateResultCallback { } // namespace quic::QuicAsyncStatus EnvoyQuicProofVerifier::VerifyCertChain( - const std::string& hostname, const uint16_t port, const std::vector& certs, - const std::string& ocsp_response, const std::string& cert_sct, + const std::string& hostname, const uint16_t /*port*/, const std::vector& certs, + const std::string& /*ocsp_response*/, const std::string& /*cert_sct*/, const quic::ProofVerifyContext* context, std::string* error_details, std::unique_ptr* details, uint8_t* out_alert, std::unique_ptr callback) { @@ -83,16 +81,6 @@ quic::QuicAsyncStatus EnvoyQuicProofVerifier::VerifyCertChain( } ENVOY_BUG(!verify_context->isServer(), "Client certificates are not supported in QUIC yet."); - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - if (doVerifyCertChain(hostname, port, certs, ocsp_response, cert_sct, context, error_details, - out_alert, std::move(callback))) { - *details = std::make_unique(true); - return quic::QUIC_SUCCESS; - } - *details = std::make_unique(false); - return quic::QUIC_FAILURE; - } - bssl::UniquePtr cert_chain(sk_X509_new_null()); for (const auto& cert_str : certs) { bssl::UniquePtr cert = parseDERCertificate(cert_str, error_details); @@ -143,45 +131,5 @@ quic::QuicAsyncStatus EnvoyQuicProofVerifier::VerifyCertChain( return quic::QUIC_FAILURE; } -bool EnvoyQuicProofVerifier::doVerifyCertChain( - const std::string& hostname, const uint16_t /*port*/, const std::vector& certs, - const std::string& /*ocsp_response*/, const std::string& /*cert_sct*/, - const quic::ProofVerifyContext* /*context*/, std::string* error_details, uint8_t* /*out_alert*/, - std::unique_ptr /*callback*/) { - bssl::UniquePtr intermediates(sk_X509_new_null()); - bssl::UniquePtr leaf; - for (size_t i = 0; i < certs.size(); i++) { - bssl::UniquePtr cert = parseDERCertificate(certs[i], error_details); - if (!cert) { - return false; - } - if (i == 0) { - leaf = std::move(cert); - } else { - sk_X509_push(intermediates.get(), cert.release()); - } - } - std::unique_ptr cert_view = - quic::CertificateView::ParseSingleCertificate(certs[0]); - ASSERT(cert_view != nullptr); - int sign_alg = deduceSignatureAlgorithmFromPublicKey(cert_view->public_key(), error_details); - if (sign_alg == 0) { - return false; - } - // We down cast rather than add verifyCertChain to Envoy::Ssl::Context because - // verifyCertChain uses a bunch of SSL-specific structs which we want to keep - // out of the interface definition. - bool success = static_cast(context_.get()) - ->verifyCertChain(*leaf, *intermediates, *error_details); - if (!success) { - return false; - } - if (verifyLeafCertMatchesHostname(*cert_view, hostname, error_details)) { - return true; - } - *error_details = absl::StrCat("Leaf certificate doesn't match hostname: ", hostname); - return false; -} - } // namespace Quic } // namespace Envoy diff --git a/source/common/quic/envoy_quic_proof_verifier.h b/source/common/quic/envoy_quic_proof_verifier.h index 50a3b9e66a44..fc03fbca15d0 100644 --- a/source/common/quic/envoy_quic_proof_verifier.h +++ b/source/common/quic/envoy_quic_proof_verifier.h @@ -54,13 +54,6 @@ class EnvoyQuicProofVerifier : public EnvoyQuicProofVerifierBase { std::unique_ptr callback) override; private: - // TODO(danzh) remove when deprecating envoy.reloadable_features.tls_async_cert_validation. - bool doVerifyCertChain(const std::string& hostname, const uint16_t port, - const std::vector& certs, const std::string& ocsp_response, - const std::string& cert_sct, const quic::ProofVerifyContext* context, - std::string* error_details, uint8_t* out_alert, - std::unique_ptr callback); - Envoy::Ssl::ClientContextSharedPtr context_; }; diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index f0e2429dccce..c7ce0d53bc49 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -73,7 +73,6 @@ RUNTIME_GUARD(envoy_reloadable_features_tcp_pool_idle_timeout); RUNTIME_GUARD(envoy_reloadable_features_test_feature_true); RUNTIME_GUARD(envoy_reloadable_features_thrift_allow_negative_field_ids); RUNTIME_GUARD(envoy_reloadable_features_thrift_connection_draining); -RUNTIME_GUARD(envoy_reloadable_features_tls_async_cert_validation); RUNTIME_GUARD(envoy_reloadable_features_uhv_allow_malformed_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_uhv_preserve_url_encoded_case); RUNTIME_GUARD(envoy_reloadable_features_uhv_translate_backslash_to_slash); diff --git a/source/extensions/transport_sockets/tls/cert_validator/cert_validator.h b/source/extensions/transport_sockets/tls/cert_validator/cert_validator.h index 0034cc7dff02..99e6945923fd 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/cert_validator.h +++ b/source/extensions/transport_sockets/tls/cert_validator/cert_validator.h @@ -58,18 +58,6 @@ class CertValidator { */ virtual void addClientValidationContext(SSL_CTX* context, bool require_client_cert) PURE; - /** - * Called by verifyCallback to do the actual cert chain verification. - * - * @param store_ctx the store context - * @param ssl_extended_info the info for storing the validation status - * @param leaf_cert the peer certificate to verify - * @return 1 to indicate verification success and 0 to indicate verification failure. - */ - virtual int doSynchronousVerifyCertChain( - X509_STORE_CTX* store_ctx, Ssl::SslExtendedSocketInfo* ssl_extended_info, X509& leaf_cert, - const Network::TransportSocketOptions* transport_socket_options) PURE; - /** * Called by customVerifyCallback to do the actual cert chain verification which could be * asynchronous. If the verification is asynchronous, Pending will be returned. After the diff --git a/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc b/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc index 74ff6984d9a4..c618b2531583 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc +++ b/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc @@ -194,37 +194,6 @@ int DefaultCertValidator::initializeSslContexts(std::vector contexts, return verify_mode; } -int DefaultCertValidator::doSynchronousVerifyCertChain( - X509_STORE_CTX* store_ctx, Ssl::SslExtendedSocketInfo* ssl_extended_info, X509& leaf_cert, - const Network::TransportSocketOptions* transport_socket_options) { - if (verify_trusted_ca_) { - int ret = X509_verify_cert(store_ctx); - if (ssl_extended_info) { - ssl_extended_info->setCertificateValidationStatus( - ret == 1 ? Envoy::Ssl::ClientValidationStatus::Validated - : Envoy::Ssl::ClientValidationStatus::Failed); - } - - if (ret <= 0) { - stats_.fail_verify_error_.inc(); - ENVOY_LOG(debug, "{}", Utility::getX509VerificationErrorInfo(store_ctx)); - return allow_untrusted_certificate_ ? 1 : ret; - } - } - Envoy::Ssl::ClientValidationStatus detailed_status = - Envoy::Ssl::ClientValidationStatus::NotValidated; - bool success = verifyCertAndUpdateStatus(&leaf_cert, transport_socket_options, detailed_status, - nullptr, nullptr); - if (ssl_extended_info) { - ssl_extended_info->setCertificateValidationStatus(detailed_status); - } - if (!success) { - X509_STORE_CTX_set_error(store_ctx, X509_V_ERR_APPLICATION_VERIFICATION); - return 0; - } - return 1; -} - bool DefaultCertValidator::verifyCertAndUpdateStatus( X509* leaf_cert, const Network::TransportSocketOptions* transport_socket_options, Envoy::Ssl::ClientValidationStatus& detailed_status, std::string* error_details, diff --git a/source/extensions/transport_sockets/tls/cert_validator/default_validator.h b/source/extensions/transport_sockets/tls/cert_validator/default_validator.h index 06ad4b7f887e..704af90c1053 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/default_validator.h +++ b/source/extensions/transport_sockets/tls/cert_validator/default_validator.h @@ -42,10 +42,6 @@ class DefaultCertValidator : public CertValidator, Logger::Loggable, bool) { return SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT; } -int SPIFFEValidator::doSynchronousVerifyCertChain(X509_STORE_CTX* store_ctx, - Ssl::SslExtendedSocketInfo* ssl_extended_info, - X509& leaf_cert, - const Network::TransportSocketOptions*) { - STACK_OF(X509)* cert_chain = X509_STORE_CTX_get0_untrusted(store_ctx); - X509_VERIFY_PARAM* verify_param = X509_STORE_CTX_get0_param(store_ctx); - std::string error_details; - bool verified = - verifyCertChainUsingTrustBundleStore(leaf_cert, cert_chain, verify_param, error_details); - if (ssl_extended_info) { - ssl_extended_info->setCertificateValidationStatus( - verified ? Envoy::Ssl::ClientValidationStatus::Validated - : Envoy::Ssl::ClientValidationStatus::Failed); - } - return verified ? 1 : 0; -} - bool SPIFFEValidator::verifyCertChainUsingTrustBundleStore(X509& leaf_cert, STACK_OF(X509)* cert_chain, X509_VERIFY_PARAM* verify_param, diff --git a/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h b/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h index d252620d2e02..3850fa5533be 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h +++ b/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h @@ -41,9 +41,6 @@ class SPIFFEValidator : public CertValidator { // Tls::CertValidator void addClientValidationContext(SSL_CTX* context, bool require_client_cert) override; - int doSynchronousVerifyCertChain( - X509_STORE_CTX* store_ctx, Ssl::SslExtendedSocketInfo* ssl_extended_info, X509& leaf_cert, - const Network::TransportSocketOptions* transport_socket_options) override; ValidationResults doVerifyCertChain(STACK_OF(X509)& cert_chain, Ssl::ValidateResultCallbackPtr callback, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index ea8dff81ea39..13389b0c8a17 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -172,22 +172,17 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c if (!capabilities_.verifies_peer_certificates) { for (auto ctx : ssl_contexts) { if (verify_mode != SSL_VERIFY_NONE) { - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - // TODO(danzh) Envoy's use of SSL_VERIFY_NONE does not quite match the actual semantics as - // a client. As a client, SSL_VERIFY_NONE means to verify the certificate (which will fail - // without trust anchors), save the result in the session ticket, but otherwise continue - // with the handshake. But Envoy actually wants it to accept all certificates. The - // disadvantage of using SSL_VERIFY_NONE is that it records the verify_result, which Envoy - // never queries but gets saved in session tickets, and tries to find an anchor that isn't - // there. And also it differs from server side behavior of SSL_VERIFY_NONE which won't - // even request client certs. So, instead, we should configure a callback to skip - // validation and always supply the callback to boring SSL. - SSL_CTX_set_custom_verify(ctx, verify_mode, customVerifyCallback); - SSL_CTX_set_reverify_on_resume(ctx, /*reverify_on_resume_enabled)=*/1); - } else { - SSL_CTX_set_verify(ctx, verify_mode, nullptr); - SSL_CTX_set_cert_verify_callback(ctx, verifyCallback, this); - } + // TODO(danzh) Envoy's use of SSL_VERIFY_NONE does not quite match the actual semantics as + // a client. As a client, SSL_VERIFY_NONE means to verify the certificate (which will fail + // without trust anchors), save the result in the session ticket, but otherwise continue + // with the handshake. But Envoy actually wants it to accept all certificates. The + // disadvantage of using SSL_VERIFY_NONE is that it records the verify_result, which Envoy + // never queries but gets saved in session tickets, and tries to find an anchor that isn't + // there. And also it differs from server side behavior of SSL_VERIFY_NONE which won't + // even request client certs. So, instead, we should configure a callback to skip + // validation and always supply the callback to boring SSL. + SSL_CTX_set_custom_verify(ctx, verify_mode, customVerifyCallback); + SSL_CTX_set_reverify_on_resume(ctx, /*reverify_on_resume_enabled)=*/1); } } } @@ -467,23 +462,6 @@ ContextImpl::newSsl(const Network::TransportSocketOptionsConstSharedPtr& options return ssl_con; } -int ContextImpl::verifyCallback(X509_STORE_CTX* store_ctx, void* arg) { - ContextImpl* impl = reinterpret_cast(arg); - SSL* ssl = reinterpret_cast( - X509_STORE_CTX_get_ex_data(store_ctx, SSL_get_ex_data_X509_STORE_CTX_idx())); - auto cert = bssl::UniquePtr(SSL_get_peer_certificate(ssl)); - auto transport_socket_options_shared_ptr_ptr = - static_cast(SSL_get_app_data(ssl)); - ASSERT(transport_socket_options_shared_ptr_ptr); - const Network::TransportSocketOptions* transport_socket_options = - (*transport_socket_options_shared_ptr_ptr).get(); - return impl->cert_validator_->doSynchronousVerifyCertChain( - store_ctx, - reinterpret_cast( - SSL_get_ex_data(ssl, ContextImpl::sslExtendedSocketInfoIndex())), - *cert, transport_socket_options); -} - enum ssl_verify_result_t ContextImpl::customVerifyCallback(SSL* ssl, uint8_t* out_alert) { auto* extended_socket_info = reinterpret_cast( SSL_get_ex_data(ssl, ContextImpl::sslExtendedSocketInfoIndex())); @@ -525,7 +503,6 @@ enum ssl_verify_result_t ContextImpl::customVerifyCallback(SSL* ssl, uint8_t* ou ValidationResults ContextImpl::customVerifyCertChain( Envoy::Ssl::SslExtendedSocketInfo* extended_socket_info, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, SSL* ssl) { - ASSERT(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")); ASSERT(extended_socket_info); STACK_OF(X509)* cert_chain = SSL_get_peer_full_cert_chain(ssl); if (cert_chain == nullptr) { @@ -1373,42 +1350,10 @@ bool TlsContext::isCipherEnabled(uint16_t cipher_id, uint16_t client_version) { return false; } -bool ContextImpl::verifyCertChain(X509& leaf_cert, STACK_OF(X509)& intermediates, - std::string& error_details) { - bssl::UniquePtr ctx(X509_STORE_CTX_new()); - - ASSERT(!tls_contexts_.empty()); - // It doesn't matter which SSL context is used, because they share the same - // cert validation config. - const SSL_CTX* ssl_ctx = tls_contexts_[0].ssl_ctx_.get(); - X509_STORE* store = SSL_CTX_get_cert_store(ssl_ctx); - if (!X509_STORE_CTX_init(ctx.get(), store, &leaf_cert, &intermediates)) { - error_details = "Failed to verify certificate chain: X509_STORE_CTX_init"; - return false; - } - // Currently this method is only used to verify server certs, so hard-code "ssl_server" for now. - if (!X509_STORE_CTX_set_default(ctx.get(), "ssl_server") || - !X509_VERIFY_PARAM_set1(X509_STORE_CTX_get0_param(ctx.get()), - SSL_CTX_get0_param(const_cast(ssl_ctx)))) { - error_details = - "Failed to verify certificate chain: fail to setup X509_STORE_CTX or its param."; - return false; - } - - int res = cert_validator_->doSynchronousVerifyCertChain(ctx.get(), nullptr, leaf_cert, nullptr); - // If |SSL_VERIFY_NONE|, the error is non-fatal, but we keep the error details. - if (res <= 0 && SSL_CTX_get_verify_mode(ssl_ctx) != SSL_VERIFY_NONE) { - error_details = Utility::getX509VerificationErrorInfo(ctx.get()); - return false; - } - return true; -} - ValidationResults ContextImpl::customVerifyCertChainForQuic( STACK_OF(X509)& cert_chain, Ssl::ValidateResultCallbackPtr callback, bool is_server, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, const CertValidator::ExtraValidationContext& validation_context, const std::string& host_name) { - ASSERT(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")); ASSERT(!tls_contexts_.empty()); // It doesn't matter which SSL context is used, because they share the same cert validation // config. diff --git a/source/extensions/transport_sockets/tls/context_impl.h b/source/extensions/transport_sockets/tls/context_impl.h index 45c2a08e7ee7..c786f3738662 100644 --- a/source/extensions/transport_sockets/tls/context_impl.h +++ b/source/extensions/transport_sockets/tls/context_impl.h @@ -91,9 +91,6 @@ class ContextImpl : public virtual Envoy::Ssl::Context, std::vector getPrivateKeyMethodProviders(); - // TODO(danzh) remove when deprecate envoy.reloadable_features.tls_async_cert_validation - bool verifyCertChain(X509& leaf_cert, STACK_OF(X509)& intermediates, std::string& error_details); - // Validate cert asynchronously for a QUIC connection. ValidationResults customVerifyCertChainForQuic( STACK_OF(X509)& cert_chain, Ssl::ValidateResultCallbackPtr callback, bool is_server, @@ -115,9 +112,6 @@ class ContextImpl : public virtual Envoy::Ssl::Context, */ static int sslContextIndex(); - // A SSL_CTX_set_cert_verify_callback for custom cert validation. - static int verifyCallback(X509_STORE_CTX* store_ctx, void* arg); - // A SSL_CTX_set_custom_verify callback for asynchronous cert validation. static enum ssl_verify_result_t customVerifyCallback(SSL* ssl, uint8_t* out_alert); diff --git a/source/extensions/transport_sockets/tls/ssl_handshaker.cc b/source/extensions/transport_sockets/tls/ssl_handshaker.cc index 3528f5f5f1df..2a0ad63dcc4e 100644 --- a/source/extensions/transport_sockets/tls/ssl_handshaker.cc +++ b/source/extensions/transport_sockets/tls/ssl_handshaker.cc @@ -48,7 +48,6 @@ void SslExtendedSocketInfoImpl::onCertificateValidationCompleted(bool succeeded) cert_validation_result_ = succeeded ? Ssl::ValidateStatus::Successful : Ssl::ValidateStatus::Failed; if (cert_validate_result_callback_.has_value()) { - ASSERT(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")); // This is an async cert validation. cert_validate_result_callback_.reset(); // Resume handshake. diff --git a/test/common/quic/envoy_quic_proof_verifier_test.cc b/test/common/quic/envoy_quic_proof_verifier_test.cc index 93d1d025403e..04957d0e2905 100644 --- a/test/common/quic/envoy_quic_proof_verifier_test.cc +++ b/test/common/quic/envoy_quic_proof_verifier_test.cc @@ -31,8 +31,7 @@ class MockProofVerifierCallback : public quic::ProofVerifierCallback { MOCK_METHOD(void, Run, (bool, const std::string&, std::unique_ptr*)); }; -class EnvoyQuicProofVerifierTest : public ::testing::Test, - public testing::WithParamInterface { +class EnvoyQuicProofVerifierTest : public testing::Test { public: EnvoyQuicProofVerifierTest() : root_ca_cert_(cert_chain_.substr(cert_chain_.rfind("-----BEGIN CERTIFICATE-----"))), @@ -41,8 +40,6 @@ class EnvoyQuicProofVerifierTest : public ::testing::Test, std::vector chain = quic::CertificateView::LoadPemFromStream(&pem_stream); return chain[0]; }()) { - Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.tls_async_cert_validation", - GetParam()); ON_CALL(client_context_config_, cipherSuites) .WillByDefault(ReturnRef( Extensions::TransportSockets::Tls::ClientContextConfigImpl::DEFAULT_CIPHER_SUITES)); @@ -111,9 +108,7 @@ class EnvoyQuicProofVerifierTest : public ::testing::Test, NiceMock verify_context_; }; -INSTANTIATE_TEST_SUITE_P(EnvoyQuicProofVerifierTests, EnvoyQuicProofVerifierTest, testing::Bool()); - -TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainSuccess) { +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainSuccess) { configCertVerificationDetails(true); std::unique_ptr cert_view = quic::CertificateView::ParseSingleCertificate(leaf_cert_); @@ -132,10 +127,7 @@ TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainSuccess) { EXPECT_TRUE(cloned->isValid()); } -TEST_P(EnvoyQuicProofVerifierTest, AsyncVerifyCertChainSuccess) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } +TEST_F(EnvoyQuicProofVerifierTest, AsyncVerifyCertChainSuccess) { custom_validator_config_ = envoy::config::core::v3::TypedExtensionConfig(); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "envoy.tls.cert_validator.timed_cert_validator" @@ -171,7 +163,7 @@ name: "envoy.tls.cert_validator.timed_cert_validator" verify_timer->invokeCallback(); } -TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureFromSsl) { +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureFromSsl) { configCertVerificationDetails(false); std::unique_ptr cert_view = quic::CertificateView::ParseSingleCertificate(leaf_cert_); @@ -184,26 +176,20 @@ TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureFromSsl) { {leaf_cert_}, ocsp_response, cert_sct, &verify_context_, &error_details, &verify_details, nullptr, nullptr)) << error_details; - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - EXPECT_EQ("verify cert failed: X509_verify_cert: certificate verification error at depth 1: " - "certificate has expired", - error_details); - } else { - EXPECT_EQ("X509_verify_cert: certificate verification error at depth 1: " - "certificate has expired", - error_details); - } + EXPECT_EQ("verify cert failed: X509_verify_cert: certificate verification error at depth 1: " + "certificate has expired", + error_details); EXPECT_NE(verify_details, nullptr); EXPECT_FALSE(static_cast(*verify_details).isValid()); } -TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidCA) { +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidCA) { root_ca_cert_ = "invalid root CA"; EXPECT_THROW_WITH_REGEX(configCertVerificationDetails(true), EnvoyException, "Failed to load trusted CA certificates from"); } -TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidLeafCert) { +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidLeafCert) { configCertVerificationDetails(true); const std::string ocsp_response; const std::string cert_sct; @@ -217,7 +203,7 @@ TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidLeafCert) { EXPECT_EQ("d2i_X509: fail to parse DER", error_details); } -TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureLeafCertWithGarbage) { +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureLeafCertWithGarbage) { configCertVerificationDetails(true); std::unique_ptr cert_view = quic::CertificateView::ParseSingleCertificate(leaf_cert_); @@ -235,7 +221,7 @@ TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureLeafCertWithGarbage) { EXPECT_EQ("There is trailing garbage in DER.", error_details); } -TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidHost) { +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidHost) { configCertVerificationDetails(true); const std::string ocsp_response; const std::string cert_sct; @@ -249,11 +235,7 @@ TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidHost) { EXPECT_EQ("Leaf certificate doesn't match hostname: unknown.org", error_details); } -TEST_P(EnvoyQuicProofVerifierTest, AsyncVerifyCertChainFailureInvalidHost) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - +TEST_F(EnvoyQuicProofVerifierTest, AsyncVerifyCertChainFailureInvalidHost) { custom_validator_config_ = envoy::config::core::v3::TypedExtensionConfig(); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "envoy.tls.cert_validator.timed_cert_validator" @@ -288,7 +270,7 @@ name: "envoy.tls.cert_validator.timed_cert_validator" verify_timer->invokeCallback(); } -TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureUnsupportedECKey) { +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureUnsupportedECKey) { configCertVerificationDetails(true); const std::string ocsp_response; const std::string cert_sct; @@ -323,7 +305,7 @@ VdGXMAjeXhnOnPvmDi5hUz/uvI+Pg6cNmUoCRwSCnK/DazhA EXPECT_EQ("Invalid leaf cert, only P-256 ECDSA certificates are supported", error_details); } -TEST_P(EnvoyQuicProofVerifierTest, VerifyCertChainFailureNonServerAuthEKU) { +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureNonServerAuthEKU) { // Override the CA cert with cert copied from test/config/integration/certs/cacert.pem. root_ca_cert_ = R"(-----BEGIN CERTIFICATE----- MIID3TCCAsWgAwIBAgIUdCu/mLip3X/We37vh3BA9u/nxakwDQYJKoZIhvcNAQEL @@ -391,24 +373,13 @@ ZCFbredVxDBZuoVsfrKPSQa407Jj1Q== verifier_->VerifyCertChain("lyft.com", 54321, chain, ocsp_response, cert_sct, &verify_context_, &error_details, &verify_details, nullptr, nullptr)); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - EXPECT_EQ("verify cert failed: X509_verify_cert: certificate verification error at depth 0: " - "unsupported certificate " - "purpose", - error_details); - } else { - EXPECT_EQ("X509_verify_cert: certificate verification error at depth 0: " - "unsupported certificate " - "purpose", - error_details); - } + EXPECT_EQ("verify cert failed: X509_verify_cert: certificate verification error at depth 0: " + "unsupported certificate " + "purpose", + error_details); } -TEST_P(EnvoyQuicProofVerifierTest, VerifySubjectAltNameListOverrideFailure) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - +TEST_F(EnvoyQuicProofVerifierTest, VerifySubjectAltNameListOverrideFailure) { transport_socket_options_.reset(new Network::TransportSocketOptionsImpl("", {"non-example.com"})); configCertVerificationDetails(true); std::unique_ptr cert_view = @@ -427,14 +398,14 @@ TEST_P(EnvoyQuicProofVerifierTest, VerifySubjectAltNameListOverrideFailure) { EXPECT_FALSE(static_cast(*verify_details).isValid()); } -TEST_P(EnvoyQuicProofVerifierTest, VerifyProof) { +TEST_F(EnvoyQuicProofVerifierTest, VerifyProof) { configCertVerificationDetails(true); EXPECT_DEATH(verifier_->VerifyProof("", 0, "", quic::QUIC_VERSION_IETF_RFC_V1, "", {}, "", "", nullptr, nullptr, nullptr, {}), "not implemented"); } -TEST_P(EnvoyQuicProofVerifierTest, CreateDefaultContext) { +TEST_F(EnvoyQuicProofVerifierTest, CreateDefaultContext) { configCertVerificationDetails(true); EXPECT_EQ(nullptr, verifier_->CreateDefaultContext()); } diff --git a/test/extensions/transport_sockets/tls/cert_validator/default_validator_test.cc b/test/extensions/transport_sockets/tls/cert_validator/default_validator_test.cc index 12bc395a3109..2920617947c3 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/default_validator_test.cc +++ b/test/extensions/transport_sockets/tls/cert_validator/default_validator_test.cc @@ -180,12 +180,6 @@ TEST(DefaultCertValidatorTest, TestCertificateVerificationWithNoValidationContex nullptr), Envoy::Ssl::ClientValidationStatus::NotValidated); bssl::UniquePtr cert(X509_new()); - bssl::UniquePtr store_ctx(X509_STORE_CTX_new()); - EXPECT_EQ(default_validator->doSynchronousVerifyCertChain(/*store_ctx=*/store_ctx.get(), - /*ssl_extended_info=*/nullptr, - /*leaf_cert=*/*cert, - /*transport_socket_options=*/nullptr), - 0); SSLContextPtr ssl_ctx = SSL_CTX_new(TLS_method()); bssl::UniquePtr cert_chain(sk_X509_new_null()); ASSERT_TRUE(bssl::PushToStack(cert_chain.get(), std::move(cert))); @@ -291,7 +285,9 @@ TEST(DefaultCertValidatorTest, WithVerifyDepth) { class MockCertificateValidationContextConfig : public Ssl::CertificateValidationContextConfig { public: - MockCertificateValidationContextConfig() { + MockCertificateValidationContextConfig() : MockCertificateValidationContextConfig("") {} + + explicit MockCertificateValidationContextConfig(const std::string& s) : s_(s) { auto matcher = envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher(); matcher.set_san_type( static_cast( @@ -340,6 +336,22 @@ TEST(DefaultCertValidatorTest, TestUnexpectedSanMatcherType) { "Failed to create string SAN matcher of type.*"); } +TEST(DefaultCertValidatorTest, TestInitializeSslContextFailure) { + auto mock_context_config = std::make_unique( + "-----BEGIN CERTIFICATE-----\nincomplete payload"); + EXPECT_CALL(*mock_context_config.get(), trustChainVerification()) + .WillRepeatedly(testing::Return(envoy::extensions::transport_sockets::tls::v3:: + CertificateValidationContext::ACCEPT_UNTRUSTED)); + + Stats::TestUtil::TestStore store; + auto ssl_stats = generateSslStats(*store.rootScope()); + auto validator = std::make_unique(mock_context_config.get(), ssl_stats, + Event::GlobalTimeSystem().timeSystem()); + auto ctx = std::vector(); + EXPECT_THROW_WITH_REGEX(validator->initializeSslContexts(ctx, false), EnvoyException, + "Failed to load trusted CA certificates from.*"); +} + } // namespace Tls } // namespace TransportSockets } // namespace Extensions diff --git a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_test.cc b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_test.cc index 9f67f7978654..c6a06acab352 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_test.cc +++ b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_test.cc @@ -36,12 +36,9 @@ using X509StoreContextPtr = CSmartPtr; using X509Ptr = CSmartPtr; using SSLContextPtr = CSmartPtr; -class TestSPIFFEValidator : public ::testing::Test, public testing::WithParamInterface { +class TestSPIFFEValidator : public testing::Test { public: - TestSPIFFEValidator() : stats_(generateSslStats(*store_.rootScope())) { - Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.tls_async_cert_validation", - GetParam()); - } + TestSPIFFEValidator() : stats_(generateSslStats(*store_.rootScope())) {} void initialize(std::string yaml, TimeSource& time_source) { envoy::config::core::v3::TypedExtensionConfig typed_conf; TestUtility::loadFromYaml(yaml, typed_conf); @@ -102,9 +99,7 @@ class TestSPIFFEValidator : public ::testing::Test, public testing::WithParamInt SPIFFEValidatorPtr validator_; }; -INSTANTIATE_TEST_SUITE_P(TestSPIFFEValidators, TestSPIFFEValidator, testing::Bool()); - -TEST_P(TestSPIFFEValidator, InvalidCA) { +TEST_F(TestSPIFFEValidator, InvalidCA) { // Invalid trust bundle. EXPECT_THROW_WITH_MESSAGE(initialize(TestEnvironment::substitute(R"EOF( name: envoy.tls.cert_validator.spiffe @@ -119,7 +114,7 @@ name: envoy.tls.cert_validator.spiffe } // Multiple trust bundles are given for the same trust domain. -TEST_P(TestSPIFFEValidator, Constructor) { +TEST_F(TestSPIFFEValidator, Constructor) { EXPECT_THROW_WITH_MESSAGE(initialize(TestEnvironment::substitute(R"EOF( name: envoy.tls.cert_validator.spiffe typed_config: @@ -204,13 +199,13 @@ TEST(SPIFFEValidator, TestCertificatePrecheck) { EXPECT_TRUE(SPIFFEValidator::certificatePrecheck(cert.get())); } -TEST_P(TestSPIFFEValidator, TestInitializeSslContexts) { +TEST_F(TestSPIFFEValidator, TestInitializeSslContexts) { initialize(); EXPECT_EQ(SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, validator().initializeSslContexts({}, false)); } -TEST_P(TestSPIFFEValidator, TestGetTrustBundleStore) { +TEST_F(TestSPIFFEValidator, TestGetTrustBundleStore) { initialize(); // No SAN @@ -236,7 +231,7 @@ TEST_P(TestSPIFFEValidator, TestGetTrustBundleStore) { EXPECT_TRUE(validator().getTrustBundleStore(cert.get())); } -TEST_P(TestSPIFFEValidator, TestDoVerifyCertChainWithEmptyChain) { +TEST_F(TestSPIFFEValidator, TestDoVerifyCertChainWithEmptyChain) { initialize(); TestSslExtendedSocketInfo info; SSLContextPtr ssl_ctx = SSL_CTX_new(TLS_method()); @@ -249,30 +244,24 @@ TEST_P(TestSPIFFEValidator, TestDoVerifyCertChainWithEmptyChain) { EXPECT_EQ(1, stats().fail_verify_error_.value()); } -TEST_P(TestSPIFFEValidator, TestDoVerifyCertChainPrecheckFailure) { +TEST_F(TestSPIFFEValidator, TestDoVerifyCertChainPrecheckFailure) { initialize(); bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( // basicConstraints: CA:True "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem")); TestSslExtendedSocketInfo info; - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - SSLContextPtr ssl_ctx = SSL_CTX_new(TLS_method()); - bssl::UniquePtr cert_chain(sk_X509_new_null()); - sk_X509_push(cert_chain.get(), cert.release()); - ValidationResults results = validator().doVerifyCertChain( - *cert_chain, info.createValidateResultCallback(), - /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, ""); - EXPECT_EQ(ValidationResults::ValidationStatus::Failed, results.status); - EXPECT_EQ(Envoy::Ssl::ClientValidationStatus::Failed, results.detailed_status); - } else { - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_FALSE(validator().doSynchronousVerifyCertChain(store_ctx.get(), &info, *cert, nullptr)); - EXPECT_EQ(info.certificateValidationStatus(), Envoy::Ssl::ClientValidationStatus::Failed); - } + SSLContextPtr ssl_ctx = SSL_CTX_new(TLS_method()); + bssl::UniquePtr cert_chain(sk_X509_new_null()); + sk_X509_push(cert_chain.get(), cert.release()); + ValidationResults results = + validator().doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), + /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, ""); + EXPECT_EQ(ValidationResults::ValidationStatus::Failed, results.status); + EXPECT_EQ(Envoy::Ssl::ClientValidationStatus::Failed, results.detailed_status); EXPECT_EQ(1, stats().fail_verify_error_.value()); } -TEST_P(TestSPIFFEValidator, TestDoVerifyCertChainSingleTrustDomain) { +TEST_F(TestSPIFFEValidator, TestDoVerifyCertChainSingleTrustDomain) { initialize(TestEnvironment::substitute(R"EOF( name: envoy.tls.cert_validator.spiffe typed_config: @@ -286,10 +275,10 @@ name: envoy.tls.cert_validator.spiffe X509StorePtr store = X509_STORE_new(); SSLContextPtr ssl_ctx = SSL_CTX_new(TLS_method()); TestSslExtendedSocketInfo info; - // Trust domain matches so should be accepted. - auto cert = readCertFromFile(TestEnvironment::substitute( - "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_cert.pem")); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { + { + // Trust domain matches so should be accepted. + auto cert = readCertFromFile(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_cert.pem")); bssl::UniquePtr cert_chain(sk_X509_new_null()); sk_X509_push(cert_chain.get(), cert.release()); EXPECT_EQ(ValidationResults::ValidationStatus::Successful, @@ -297,16 +286,12 @@ name: envoy.tls.cert_validator.spiffe .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") .status); - } else { - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), nullptr)); - EXPECT_TRUE(validator().doSynchronousVerifyCertChain(store_ctx.get(), nullptr, *cert, nullptr)); } - // Different trust domain so should be rejected. - cert = readCertFromFile(TestEnvironment::substitute( - "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/spiffe_san_cert.pem")); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { + { + // Different trust domain so should be rejected. + auto cert = readCertFromFile(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/spiffe_san_cert.pem")); bssl::UniquePtr cert_chain(sk_X509_new_null()); sk_X509_push(cert_chain.get(), cert.release()); EXPECT_EQ(ValidationResults::ValidationStatus::Failed, @@ -314,17 +299,11 @@ name: envoy.tls.cert_validator.spiffe .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") .status); - } else { - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), nullptr)); - EXPECT_FALSE( - validator().doSynchronousVerifyCertChain(store_ctx.get(), nullptr, *cert, nullptr)); } - - // Does not have san. - cert = readCertFromFile(TestEnvironment::substitute( - "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/extensions_cert.pem")); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { + { + // Does not have san. + auto cert = readCertFromFile(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/extensions_cert.pem")); bssl::UniquePtr cert_chain(sk_X509_new_null()); sk_X509_push(cert_chain.get(), cert.release()); EXPECT_EQ(ValidationResults::ValidationStatus::Failed, @@ -332,17 +311,12 @@ name: envoy.tls.cert_validator.spiffe .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") .status); - } else { - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), nullptr)); - EXPECT_FALSE( - validator().doSynchronousVerifyCertChain(store_ctx.get(), nullptr, *cert, nullptr)); } EXPECT_EQ(2, stats().fail_verify_error_.value()); } -TEST_P(TestSPIFFEValidator, TestDoVerifyCertChainMultipleTrustDomain) { +TEST_F(TestSPIFFEValidator, TestDoVerifyCertChainMultipleTrustDomain) { initialize(TestEnvironment::substitute(R"EOF( name: envoy.tls.cert_validator.spiffe typed_config: @@ -360,10 +334,10 @@ name: envoy.tls.cert_validator.spiffe SSLContextPtr ssl_ctx = SSL_CTX_new(TLS_method()); TestSslExtendedSocketInfo info; - // Trust domain matches so should be accepted. - auto cert = readCertFromFile(TestEnvironment::substitute( - "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_cert.pem")); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { + { + // Trust domain matches so should be accepted. + auto cert = readCertFromFile(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_cert.pem")); bssl::UniquePtr cert_chain(sk_X509_new_null()); sk_X509_push(cert_chain.get(), cert.release()); EXPECT_EQ(ValidationResults::ValidationStatus::Successful, @@ -371,15 +345,11 @@ name: envoy.tls.cert_validator.spiffe .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") .status); - } else { - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), nullptr)); - EXPECT_TRUE(validator().doSynchronousVerifyCertChain(store_ctx.get(), nullptr, *cert, nullptr)); } - cert = readCertFromFile(TestEnvironment::substitute( - "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/spiffe_san_cert.pem")); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { + { + auto cert = readCertFromFile(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/spiffe_san_cert.pem")); bssl::UniquePtr cert_chain(sk_X509_new_null()); sk_X509_push(cert_chain.get(), cert.release()); EXPECT_EQ(ValidationResults::ValidationStatus::Successful, @@ -387,17 +357,13 @@ name: envoy.tls.cert_validator.spiffe .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") .status); - } else { - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), nullptr)); - EXPECT_TRUE(validator().doSynchronousVerifyCertChain(store_ctx.get(), nullptr, *cert, nullptr)); } - // Trust domain matches but it has expired. - cert = readCertFromFile(TestEnvironment::substitute( - "{{ test_rundir " - "}}/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_cert.pem")); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { + { + // Trust domain matches but it has expired. + auto cert = readCertFromFile(TestEnvironment::substitute( + "{{ test_rundir " + "}}/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_cert.pem")); bssl::UniquePtr cert_chain(sk_X509_new_null()); sk_X509_push(cert_chain.get(), cert.release()); EXPECT_EQ(ValidationResults::ValidationStatus::Failed, @@ -405,17 +371,12 @@ name: envoy.tls.cert_validator.spiffe .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") .status); - } else { - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), nullptr)); - EXPECT_FALSE( - validator().doSynchronousVerifyCertChain(store_ctx.get(), nullptr, *cert, nullptr)); } - // Does not have san. - cert = readCertFromFile(TestEnvironment::substitute( - "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/extensions_cert.pem")); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { + { + // Does not have san. + auto cert = readCertFromFile(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/extensions_cert.pem")); bssl::UniquePtr cert_chain(sk_X509_new_null()); sk_X509_push(cert_chain.get(), cert.release()); EXPECT_EQ(ValidationResults::ValidationStatus::Failed, @@ -423,17 +384,12 @@ name: envoy.tls.cert_validator.spiffe .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") .status); - } else { - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), nullptr)); - EXPECT_FALSE( - validator().doSynchronousVerifyCertChain(store_ctx.get(), nullptr, *cert, nullptr)); } EXPECT_EQ(2, stats().fail_verify_error_.value()); } -TEST_P(TestSPIFFEValidator, TestDoVerifyCertChainMultipleTrustDomainAllowExpired) { +TEST_F(TestSPIFFEValidator, TestDoVerifyCertChainMultipleTrustDomainAllowExpired) { setAllowExpiredCertificate(true); initialize(TestEnvironment::substitute(R"EOF( name: envoy.tls.cert_validator.spiffe @@ -453,24 +409,18 @@ name: envoy.tls.cert_validator.spiffe auto cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir " "}}/test/extensions/transport_sockets/tls/test_data/expired_spiffe_san_cert.pem")); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - bssl::UniquePtr cert_chain(sk_X509_new_null()); - sk_X509_push(cert_chain.get(), cert.release()); - EXPECT_EQ(ValidationResults::ValidationStatus::Successful, - validator() - .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), - /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") - .status); - } else { - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), nullptr)); - EXPECT_TRUE(validator().doSynchronousVerifyCertChain(store_ctx.get(), nullptr, *cert, nullptr)); - } + bssl::UniquePtr cert_chain(sk_X509_new_null()); + sk_X509_push(cert_chain.get(), cert.release()); + EXPECT_EQ(ValidationResults::ValidationStatus::Successful, + validator() + .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), + /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") + .status); EXPECT_EQ(0, stats().fail_verify_error_.value()); } -TEST_P(TestSPIFFEValidator, TestDoVerifyCertChainSANMatching) { +TEST_F(TestSPIFFEValidator, TestDoVerifyCertChainSANMatching) { const auto config = TestEnvironment::substitute(R"EOF( name: envoy.tls.cert_validator.spiffe typed_config: @@ -489,9 +439,7 @@ name: envoy.tls.cert_validator.spiffe X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), nullptr)); bssl::UniquePtr cert_chain(sk_X509_new_null()); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - sk_X509_push(cert_chain.get(), cert.release()); - } + sk_X509_push(cert_chain.get(), cert.release()); TestSslExtendedSocketInfo info; info.setCertificateValidationStatus(Envoy::Ssl::ClientValidationStatus::NotValidated); { @@ -499,39 +447,28 @@ name: envoy.tls.cert_validator.spiffe matcher.set_prefix("spiffe://lyft.com/"); setSanMatchers({matcher}); initialize(config); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - ValidationResults results = validator().doVerifyCertChain( - *cert_chain, info.createValidateResultCallback(), - /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, ""); - EXPECT_EQ(ValidationResults::ValidationStatus::Successful, results.status); - EXPECT_EQ(Envoy::Ssl::ClientValidationStatus::Validated, results.detailed_status); - } else { - EXPECT_TRUE(validator().doSynchronousVerifyCertChain(store_ctx.get(), &info, *cert, nullptr)); - EXPECT_EQ(info.certificateValidationStatus(), Envoy::Ssl::ClientValidationStatus::Validated); - } + ValidationResults results = validator().doVerifyCertChain( + *cert_chain, info.createValidateResultCallback(), + /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, ""); + EXPECT_EQ(ValidationResults::ValidationStatus::Successful, results.status); + EXPECT_EQ(Envoy::Ssl::ClientValidationStatus::Validated, results.detailed_status); } { envoy::type::matcher::v3::StringMatcher matcher; matcher.set_prefix("spiffe://example.com/"); setSanMatchers({matcher}); initialize(config); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - ValidationResults results = validator().doVerifyCertChain( - *cert_chain, info.createValidateResultCallback(), - /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, ""); - EXPECT_EQ(ValidationResults::ValidationStatus::Failed, results.status); - EXPECT_EQ(Envoy::Ssl::ClientValidationStatus::Failed, results.detailed_status); - } else { - EXPECT_FALSE( - validator().doSynchronousVerifyCertChain(store_ctx.get(), &info, *cert, nullptr)); - EXPECT_EQ(info.certificateValidationStatus(), Envoy::Ssl::ClientValidationStatus::Failed); - } + ValidationResults results = validator().doVerifyCertChain( + *cert_chain, info.createValidateResultCallback(), + /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, ""); + EXPECT_EQ(ValidationResults::ValidationStatus::Failed, results.status); + EXPECT_EQ(Envoy::Ssl::ClientValidationStatus::Failed, results.detailed_status); EXPECT_EQ(1, stats().fail_verify_san_.value()); stats().fail_verify_san_.reset(); } } -TEST_P(TestSPIFFEValidator, TestDoVerifyCertChainIntermediateCerts) { +TEST_F(TestSPIFFEValidator, TestDoVerifyCertChainIntermediateCerts) { initialize(TestEnvironment::substitute(R"EOF( name: envoy.tls.cert_validator.spiffe typed_config: @@ -551,25 +488,15 @@ name: envoy.tls.cert_validator.spiffe "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/" "intermediate_ca_cert.pem")); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - SSLContextPtr ssl_ctx = SSL_CTX_new(TLS_method()); - bssl::UniquePtr cert_chain(sk_X509_new_null()); - sk_X509_push(cert_chain.get(), cert.release()); - sk_X509_push(cert_chain.get(), intermediate_ca_cert.release()); - EXPECT_EQ(ValidationResults::ValidationStatus::Successful, - validator() - .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), - /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") - .status); - } else { - X509StorePtr store = X509_STORE_new(); - STACK_OF(X509)* intermediates = sk_X509_new_null(); - sk_X509_push(intermediates, intermediate_ca_cert.release()); - X509StoreContextPtr store_ctx = X509_STORE_CTX_new(); - EXPECT_TRUE(X509_STORE_CTX_init(store_ctx.get(), store.get(), cert.get(), intermediates)); - EXPECT_TRUE(validator().doSynchronousVerifyCertChain(store_ctx.get(), nullptr, *cert, nullptr)); - sk_X509_pop_free(intermediates, X509_free); - } + SSLContextPtr ssl_ctx = SSL_CTX_new(TLS_method()); + bssl::UniquePtr cert_chain(sk_X509_new_null()); + sk_X509_push(cert_chain.get(), cert.release()); + sk_X509_push(cert_chain.get(), intermediate_ca_cert.release()); + EXPECT_EQ(ValidationResults::ValidationStatus::Successful, + validator() + .doVerifyCertChain(*cert_chain, info.createValidateResultCallback(), + /*transport_socket_options=*/nullptr, *ssl_ctx, {}, false, "") + .status); } void addIA5StringGenNameExt(X509* cert, int type, const std::string name) { @@ -582,7 +509,7 @@ void addIA5StringGenNameExt(X509* cert, int type, const std::string name) { EXPECT_TRUE(X509_add1_ext_i2d(cert, NID_subject_alt_name, gens.get(), 0, X509V3_ADD_DEFAULT)); } -TEST_P(TestSPIFFEValidator, TestMatchSubjectAltNameWithURISan) { +TEST_F(TestSPIFFEValidator, TestMatchSubjectAltNameWithURISan) { envoy::type::matcher::v3::StringMatcher exact_matcher, prefix_matcher, regex_matcher; exact_matcher.set_exact("spiffe://example.com/workload"); prefix_matcher.set_prefix("spiffe://envoy.com"); @@ -627,7 +554,7 @@ name: envoy.tls.cert_validator.spiffe } // SPIFFE validator ignores any SANs other than URI. -TEST_P(TestSPIFFEValidator, TestMatchSubjectAltNameWithoutURISan) { +TEST_F(TestSPIFFEValidator, TestMatchSubjectAltNameWithoutURISan) { envoy::type::matcher::v3::StringMatcher exact_matcher, prefix_matcher; exact_matcher.set_exact("spiffe://example.com/workload"); prefix_matcher.set_prefix("envoy"); @@ -659,7 +586,7 @@ name: envoy.tls.cert_validator.spiffe } } -TEST_P(TestSPIFFEValidator, TestGetCaCertInformation) { +TEST_F(TestSPIFFEValidator, TestGetCaCertInformation) { initialize(); // No cert is set so this should be nullptr. @@ -682,7 +609,7 @@ name: envoy.tls.cert_validator.spiffe EXPECT_TRUE(actual); } -TEST_P(TestSPIFFEValidator, TestDaysUntilFirstCertExpires) { +TEST_F(TestSPIFFEValidator, TestDaysUntilFirstCertExpires) { initialize(); EXPECT_EQ(std::numeric_limits::max(), validator().daysUntilFirstCertExpires().value()); @@ -707,7 +634,7 @@ name: envoy.tls.cert_validator.spiffe EXPECT_EQ(19946, validator().daysUntilFirstCertExpires().value()); } -TEST_P(TestSPIFFEValidator, TestDaysUntilFirstCertExpiresExpired) { +TEST_F(TestSPIFFEValidator, TestDaysUntilFirstCertExpiresExpired) { Event::SimulatedTimeSystem time_system; // 2033-05-18 03:33:20 UTC const time_t known_date_time = 2000000000; @@ -727,7 +654,7 @@ name: envoy.tls.cert_validator.spiffe EXPECT_EQ(absl::nullopt, validator().daysUntilFirstCertExpires()); } -TEST_P(TestSPIFFEValidator, TestAddClientValidationContext) { +TEST_F(TestSPIFFEValidator, TestAddClientValidationContext) { Event::TestRealTimeSystem time_system; initialize(TestEnvironment::substitute(R"EOF( name: envoy.tls.cert_validator.spiffe @@ -770,7 +697,7 @@ name: envoy.tls.cert_validator.spiffe EXPECT_TRUE(foundTestCA); } -TEST_P(TestSPIFFEValidator, TestUpdateDigestForSessionId) { +TEST_F(TestSPIFFEValidator, TestUpdateDigestForSessionId) { Event::TestRealTimeSystem time_system; initialize(TestEnvironment::substitute(R"EOF( name: envoy.tls.cert_validator.spiffe diff --git a/test/extensions/transport_sockets/tls/cert_validator/timed_cert_validator.h b/test/extensions/transport_sockets/tls/cert_validator/timed_cert_validator.h index 890a76a48c6e..72e3a721daae 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/timed_cert_validator.h +++ b/test/extensions/transport_sockets/tls/cert_validator/timed_cert_validator.h @@ -20,13 +20,6 @@ class TimedCertValidator : public DefaultCertValidator { : DefaultCertValidator(config, stats, time_source), validation_time_out_ms_(validation_time_out_ms), expected_host_name_(expected_host_name) {} - int doSynchronousVerifyCertChain( - X509_STORE_CTX* /*store_ctx*/, Ssl::SslExtendedSocketInfo* /*ssl_extended_info*/, - X509& /*leaf_cert*/, - const Network::TransportSocketOptions* /*transport_socket_options*/) override { - PANIC("unimplemented"); - } - ValidationResults doVerifyCertChain(STACK_OF(X509)& cert_chain, Ssl::ValidateResultCallbackPtr callback, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, diff --git a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc index f79a8218c223..5cf4a34ff6fb 100644 --- a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc +++ b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc @@ -382,10 +382,6 @@ TEST_P(SslIntegrationTest, RouterHeaderOnlyRequestAndResponseWithSni) { } TEST_P(SslIntegrationTest, AsyncCertValidationSucceeds) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - // Config client to use an async cert validator which defer the actual validation by 5ms. envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = new envoy::config::core::v3::TypedExtensionConfig(); @@ -416,10 +412,6 @@ name: "envoy.tls.cert_validator.timed_cert_validator" } TEST_P(SslIntegrationTest, AsyncCertValidationAfterTearDown) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = new envoy::config::core::v3::TypedExtensionConfig(); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( @@ -469,10 +461,6 @@ name: "envoy.tls.cert_validator.timed_cert_validator" } TEST_P(SslIntegrationTest, AsyncCertValidationAfterSslShutdown) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = new envoy::config::core::v3::TypedExtensionConfig(); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 65b366f01606..e036a7b6c1d4 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -921,25 +921,12 @@ TestUtilOptionsV2 createProtocolTestOptions( } // namespace -std::string ipCustomCertValidationTestParamsToString( - const ::testing::TestParamInfo>& params) { - return fmt::format( - "{}_{}", - TestUtility::ipTestParamsToString( - ::testing::TestParamInfo(std::get<0>(params.param), 0)), - std::get<1>(params.param) ? "with_custom_cert_validation" : "with_sync_cert_validation"); -} - -class SslSocketTest - : public SslCertsTest, - public testing::WithParamInterface> { +class SslSocketTest : public SslCertsTest, + public testing::WithParamInterface { protected: SslSocketTest() : dispatcher_(api_->allocateDispatcher("test_thread")), - stream_info_(api_->timeSource(), nullptr), version_(std::get<0>(GetParam())) { - Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.tls_async_cert_validation", - std::get<1>(GetParam())); - } + stream_info_(api_->timeSource(), nullptr), version_(GetParam()) {} void testClientSessionResumption(const std::string& server_ctx_yaml, const std::string& client_ctx_yaml, bool expect_reuse, @@ -951,10 +938,9 @@ class SslSocketTest Network::Address::IpVersion version_; }; -INSTANTIATE_TEST_SUITE_P( - IpVersions, SslSocketTest, - testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), testing::Bool()), - ipCustomCertValidationTestParamsToString); +INSTANTIATE_TEST_SUITE_P(IpVersions, SslSocketTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); TEST_P(SslSocketTest, ServerTransportSocketOptions) { Stats::TestUtil::TestStore server_stats_store; @@ -2093,13 +2079,9 @@ TEST_P(SslSocketTest, FailedClientCertAllowExpiredBadHashVerification) { configureServerAndExpiredClientCertificate(listener, client, server_config); TestUtilOptionsV2 test_options(listener, client, false, version_); - testUtilV2( - test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") - .setExpectedClientCertUri("spiffe://lyft.com/test-team") - .setExpectedTransportFailureReasonContains( - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation") - ? "TLSV1.*_BAD_CERTIFICATE_HASH_VALUE" - : "SSLV3_ALERT_HANDSHAKE_FAILURE")); + testUtilV2(test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") + .setExpectedClientCertUri("spiffe://lyft.com/test-team") + .setExpectedTransportFailureReasonContains("TLSV1.*_BAD_CERTIFICATE_HASH_VALUE")); } // Allow expired certificates, but use the wrong CA so it should fail still. @@ -2651,12 +2633,8 @@ TEST_P(SslSocketTest, FailedClientCertificateSpkiVerificationWrongClientCertific "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/no_san_key.pem")); TestUtilOptionsV2 test_options(listener, client, false, version_); - testUtilV2( - test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") - .setExpectedTransportFailureReasonContains( - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation") - ? "TLSV1.*_BAD_CERTIFICATE_HASH_VALUE" - : "SSLV3_ALERT_HANDSHAKE_FAILURE")); + testUtilV2(test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") + .setExpectedTransportFailureReasonContains("TLSV1.*_BAD_CERTIFICATE_HASH_VALUE")); // Fails even with client renegotiation. client.set_allow_renegotiation(true); @@ -2689,12 +2667,8 @@ TEST_P(SslSocketTest, FailedClientCertificateSpkiVerificationNoCAWrongClientCert "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/no_san_key.pem")); TestUtilOptionsV2 test_options(listener, client, false, version_); - testUtilV2( - test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") - .setExpectedTransportFailureReasonContains( - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation") - ? "TLSV1.*_BAD_CERTIFICATE_HASH_VALUE" - : "SSLV3_ALERT_HANDSHAKE_FAILURE")); + testUtilV2(test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") + .setExpectedTransportFailureReasonContains("TLSV1.*_BAD_CERTIFICATE_HASH_VALUE")); // Fails even with client renegotiation. client.set_allow_renegotiation(true); @@ -2899,12 +2873,8 @@ TEST_P(SslSocketTest, FailedClientCertificateHashAndSpkiVerificationWrongClientC "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/no_san_key.pem")); TestUtilOptionsV2 test_options(listener, client, false, version_); - testUtilV2( - test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") - .setExpectedTransportFailureReasonContains( - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation") - ? "TLSV1.*_BAD_CERTIFICATE_HASH_VALUE" - : "SSLV3_ALERT_HANDSHAKE_FAILURE")); + testUtilV2(test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") + .setExpectedTransportFailureReasonContains("TLSV1.*_BAD_CERTIFICATE_HASH_VALUE")); // Fails even with client renegotiation. client.set_allow_renegotiation(true); @@ -2938,12 +2908,8 @@ TEST_P(SslSocketTest, FailedClientCertificateHashAndSpkiVerificationNoCAWrongCli "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/no_san_key.pem")); TestUtilOptionsV2 test_options(listener, client, false, version_); - testUtilV2( - test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") - .setExpectedTransportFailureReasonContains( - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation") - ? "TLSV1.*_BAD_CERTIFICATE_HASH_VALUE" - : "SSLV3_ALERT_HANDSHAKE_FAILURE")); + testUtilV2(test_options.setExpectedServerStats("ssl.fail_verify_cert_hash") + .setExpectedTransportFailureReasonContains("TLSV1.*_BAD_CERTIFICATE_HASH_VALUE")); // Fails even with client renegotiation. client.set_allow_renegotiation(true); @@ -5983,10 +5949,9 @@ class SslReadBufferLimitTest : public SslSocketTest { Network::Address::InstanceConstSharedPtr source_address_; }; -INSTANTIATE_TEST_SUITE_P( - IpVersions, SslReadBufferLimitTest, - testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), testing::Bool()), - ipCustomCertValidationTestParamsToString); +INSTANTIATE_TEST_SUITE_P(IpVersions, SslReadBufferLimitTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); TEST_P(SslReadBufferLimitTest, NoLimit) { readBufferLimitTest(0, 256 * 1024, 256 * 1024, 1, false); @@ -6948,10 +6913,6 @@ TEST_P(SslSocketTest, Sni) { } TEST_P(SslSocketTest, AsyncCustomCertValidatorSucceeds) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - const std::string client_ctx_yaml = R"EOF( common_tls_context: tls_certificates: @@ -6995,10 +6956,6 @@ TEST_P(SslSocketTest, AsyncCustomCertValidatorSucceeds) { } TEST_P(SslSocketTest, AsyncCustomCertValidatorFails) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - const std::string server_ctx_yaml = R"EOF( common_tls_context: tls_certificates: diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index cb506c233f37..97f93cda361e 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -1066,10 +1066,6 @@ TEST_P(QuicHttpIntegrationTest, NoStreams) { } TEST_P(QuicHttpIntegrationTest, AsyncCertVerificationSucceeds) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - // Config the client to defer cert validation by 5ms. envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = new envoy::config::core::v3::TypedExtensionConfig(); @@ -1086,10 +1082,6 @@ name: "envoy.tls.cert_validator.timed_cert_validator" } TEST_P(QuicHttpIntegrationTest, AsyncCertVerificationAfterDisconnect) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = new envoy::config::core::v3::TypedExtensionConfig(); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( @@ -1129,10 +1121,6 @@ name: "envoy.tls.cert_validator.timed_cert_validator" } TEST_P(QuicHttpIntegrationTest, AsyncCertVerificationAfterTearDown) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) { - return; - } - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = new envoy::config::core::v3::TypedExtensionConfig(); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( From 19e0611a97a128cecd9caf4f0bee01da1b2acedb Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Fri, 2 Jun 2023 16:08:58 -0400 Subject: [PATCH 447/740] composite: add CEL matcher to allow list (#27771) Add CEL matcher to allow list Signed-off-by: tyxia --- source/extensions/filters/http/composite/config.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/http/composite/config.h b/source/extensions/filters/http/composite/config.h index 0ab7203f1093..8d088994378e 100644 --- a/source/extensions/filters/http/composite/config.h +++ b/source/extensions/filters/http/composite/config.h @@ -11,6 +11,8 @@ #include "source/common/protobuf/utility.h" #include "source/extensions/filters/http/common/factory_base.h" +#include "xds/type/matcher/v3/http_inputs.pb.h" + namespace Envoy { namespace Extensions { namespace HttpFilters { @@ -35,9 +37,12 @@ class CompositeFilterFactory // This ensure that trees are only allowed to match on request headers, avoiding configurations // where the matcher requires data that will be available too late for the delegation to work // correctly. - requirements->mutable_data_input_allow_list()->add_type_url( - TypeUtil::descriptorFullNameToTypeUrl( - envoy::type::matcher::v3::HttpRequestHeaderMatchInput::descriptor()->full_name())); + auto* allow_list = requirements->mutable_data_input_allow_list(); + allow_list->add_type_url(TypeUtil::descriptorFullNameToTypeUrl( + envoy::type::matcher::v3::HttpRequestHeaderMatchInput::descriptor()->full_name())); + // CEL matcher and its input is also allowed. + allow_list->add_type_url(TypeUtil::descriptorFullNameToTypeUrl( + xds::type::matcher::v3::HttpAttributesCelMatchInput::descriptor()->full_name())); return requirements; } From 2ab9d6b7f816acbbde3cecc9357efc4f0960c9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E6=B3=BD=E8=BD=A9?= Date: Sat, 3 Jun 2023 04:26:12 +0800 Subject: [PATCH 448/740] golang filter: support getting log level (#27652) Signed-off-by: spacewander --- contrib/golang/filters/http/source/cgo.cc | 2 ++ .../filters/http/source/go/pkg/api/api.h | 1 + .../filters/http/source/go/pkg/api/capi.go | 1 + .../filters/http/source/go/pkg/api/filter.go | 1 + .../filters/http/source/go/pkg/api/type.go | 18 ++++++++++++++++++ .../http/source/go/pkg/http/capi_impl.go | 4 ++++ .../filters/http/source/go/pkg/http/filter.go | 4 ++++ .../filters/http/source/golang_filter.cc | 2 ++ .../golang/filters/http/source/golang_filter.h | 1 + .../http/test/golang_integration_test.cc | 3 +++ .../http/test/test_data/basic/filter.go | 1 + 11 files changed, 38 insertions(+) diff --git a/contrib/golang/filters/http/source/cgo.cc b/contrib/golang/filters/http/source/cgo.cc index 56742609932a..74adb835d778 100644 --- a/contrib/golang/filters/http/source/cgo.cc +++ b/contrib/golang/filters/http/source/cgo.cc @@ -175,6 +175,8 @@ void envoyGoFilterHttpLog(uint32_t level, void* message) { getFilterLogger().log(level, mesg); } +uint32_t envoyGoFilterHttpLogLevel() { return getFilterLogger().level(); } + CAPIStatus envoyGoFilterHttpSetDynamicMetadata(void* r, void* name, void* key, void* buf) { return envoyGoFilterHandlerWrapper( r, [name, key, buf](std::shared_ptr& filter) -> CAPIStatus { diff --git a/contrib/golang/filters/http/source/go/pkg/api/api.h b/contrib/golang/filters/http/source/go/pkg/api/api.h index 3673e458d5c8..3620ff1dc454 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/api.h +++ b/contrib/golang/filters/http/source/go/pkg/api/api.h @@ -65,6 +65,7 @@ CAPIStatus envoyGoFilterHttpGetIntegerValue(void* r, int id, void* value); CAPIStatus envoyGoFilterHttpSetDynamicMetadata(void* r, void* name, void* key, void* buf); void envoyGoFilterHttpLog(uint32_t level, void* message); +uint32_t envoyGoFilterHttpLogLevel(); void envoyGoFilterHttpFinalize(void* r, int reason); diff --git a/contrib/golang/filters/http/source/go/pkg/api/capi.go b/contrib/golang/filters/http/source/go/pkg/api/capi.go index 764cf3845431..494bf72572be 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/capi.go +++ b/contrib/golang/filters/http/source/go/pkg/api/capi.go @@ -46,6 +46,7 @@ type HttpCAPI interface { HttpSetDynamicMetadata(r unsafe.Pointer, filterName string, key string, value interface{}) HttpLog(level LogType, message string) + HttpLogLevel() LogType HttpFinalize(r unsafe.Pointer, reason int) diff --git a/contrib/golang/filters/http/source/go/pkg/api/filter.go b/contrib/golang/filters/http/source/go/pkg/api/filter.go index 1979fc2935f0..1d3973a2f525 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/filter.go +++ b/contrib/golang/filters/http/source/go/pkg/api/filter.go @@ -93,6 +93,7 @@ type FilterCallbacks interface { // RecoverPanic recover panic in defer and terminate the request by SendLocalReply with 500 status code. RecoverPanic() Log(level LogType, msg string) + LogLevel() LogType // TODO add more for filter callbacks } diff --git a/contrib/golang/filters/http/source/go/pkg/api/type.go b/contrib/golang/filters/http/source/go/pkg/api/type.go index 24156b06a144..002cfd80a28d 100644 --- a/contrib/golang/filters/http/source/go/pkg/api/type.go +++ b/contrib/golang/filters/http/source/go/pkg/api/type.go @@ -70,6 +70,24 @@ const ( Critical LogType = 5 ) +func (self LogType) String() string { + switch self { + case Trace: + return "trace" + case Debug: + return "debug" + case Info: + return "info" + case Warn: + return "warn" + case Error: + return "error" + case Critical: + return "critical" + } + return "unknown" +} + //******************* log level end *******************// // ****************** HeaderMap start ******************// diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go index f043b5671224..3df76682028e 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go +++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go @@ -255,6 +255,10 @@ func (c *httpCApiImpl) HttpLog(level api.LogType, message string) { C.envoyGoFilterHttpLog(C.uint32_t(level), unsafe.Pointer(&message)) } +func (c *httpCApiImpl) HttpLogLevel() api.LogType { + return api.LogType(C.envoyGoFilterHttpLogLevel()) +} + func (c *httpCApiImpl) HttpFinalize(r unsafe.Pointer, reason int) { C.envoyGoFilterHttpFinalize(r, C.int(reason)) } diff --git a/contrib/golang/filters/http/source/go/pkg/http/filter.go b/contrib/golang/filters/http/source/go/pkg/http/filter.go index 881b2825b1b7..b035ab7a5e81 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/filter.go +++ b/contrib/golang/filters/http/source/go/pkg/http/filter.go @@ -115,6 +115,10 @@ func (r *httpRequest) Log(level api.LogType, message string) { cAPI.HttpLog(level, fmt.Sprintf("[go_plugin_http][%v] %v", r.pluginName(), message)) } +func (r *httpRequest) LogLevel() api.LogType { + return cAPI.HttpLogLevel() +} + func (r *httpRequest) StreamInfo() api.StreamInfo { return &streamInfo{ request: r, diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc index 8dfd65f14ca6..82aa29bdd97f 100644 --- a/contrib/golang/filters/http/source/golang_filter.cc +++ b/contrib/golang/filters/http/source/golang_filter.cc @@ -1255,6 +1255,8 @@ void FilterLogger::log(uint32_t level, absl::string_view message) const { PANIC_DUE_TO_CORRUPT_ENUM; } +uint32_t FilterLogger::level() const { return static_cast(ENVOY_LOGGER().level()); } + } // namespace Golang } // namespace HttpFilters } // namespace Extensions diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h index af1423103c15..7d175df45ed2 100644 --- a/contrib/golang/filters/http/source/golang_filter.h +++ b/contrib/golang/filters/http/source/golang_filter.h @@ -259,6 +259,7 @@ class FilterLogger : Logger::Loggable { FilterLogger() = default; void log(uint32_t level, absl::string_view message) const; + uint32_t level() const; }; class GoStringFilterState : public StreamInfo::FilterState::Object { diff --git a/contrib/golang/filters/http/test/golang_integration_test.cc b/contrib/golang/filters/http/test/golang_integration_test.cc index efda0c3f4541..1e4908e9986e 100644 --- a/contrib/golang/filters/http/test/golang_integration_test.cc +++ b/contrib/golang/filters/http/test/golang_integration_test.cc @@ -386,6 +386,9 @@ name: golang // verify host EXPECT_EQ("test.com", getHeader(response->headers(), "test-host")); + // verify log level + EXPECT_EQ("error", getHeader(response->headers(), "test-log-level")); + // upper("goodbye") EXPECT_EQ("GOODBYE", response->body()); diff --git a/contrib/golang/filters/http/test/test_data/basic/filter.go b/contrib/golang/filters/http/test/test_data/basic/filter.go index 3d73cd9df187..b1838067c1c3 100644 --- a/contrib/golang/filters/http/test/test_data/basic/filter.go +++ b/contrib/golang/filters/http/test/test_data/basic/filter.go @@ -256,6 +256,7 @@ func (f *filter) encodeHeaders(header api.ResponseHeaderMap, endStream bool) api header.Set("test-method", f.method) header.Set("test-path", f.path) header.Set("test-host", f.host) + header.Set("test-log-level", f.callbacks.LogLevel().String()) header.Set("rsp-route-name", f.callbacks.StreamInfo().GetRouteName()) header.Set("rsp-filter-chain-name", f.callbacks.StreamInfo().FilterChainName()) header.Set("rsp-attempt-count", strconv.Itoa(int(f.callbacks.StreamInfo().AttemptCount()))) From 62dcbc89cbd9ff67343c6fe3a2bbe70909b34864 Mon Sep 17 00:00:00 2001 From: phlax Date: Sat, 3 Jun 2023 16:30:48 +0100 Subject: [PATCH 449/740] ci: Minor cleanup for build script (#27780) Signed-off-by: Ryan Northey --- ci/build_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_setup.sh b/ci/build_setup.sh index f0de186abd60..f4a94398f1bf 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -8,7 +8,7 @@ set -e if [[ -n "$NO_BUILD_SETUP" ]]; then - exit + return fi export PPROF_PATH=/thirdparty_build/bin/pprof From 22af18a40f043ac46d8c9ef9b51cb50f2ce69d50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Jun 2023 17:24:25 +0100 Subject: [PATCH 450/740] build(deps): bump github/codeql-action from 2.3.5 to 2.3.6 (#27764) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.3.5 to 2.3.6. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/0225834cc549ee0ca93cb085b92954821a145866...83f0fe6c4988d98a455712a27f0255212bba9bd4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index b81e86623169..7c069b38723f 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -33,7 +33,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@0225834cc549ee0ca93cb085b92954821a145866 + uses: github/codeql-action/init@83f0fe6c4988d98a455712a27f0255212bba9bd4 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -64,4 +64,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@0225834cc549ee0ca93cb085b92954821a145866 + uses: github/codeql-action/analyze@83f0fe6c4988d98a455712a27f0255212bba9bd4 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index b1ab668242b7..f303a369c068 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -45,7 +45,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@0225834cc549ee0ca93cb085b92954821a145866 + uses: github/codeql-action/init@83f0fe6c4988d98a455712a27f0255212bba9bd4 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -78,4 +78,4 @@ jobs: - name: Perform CodeQL Analysis if: env.BUILD_TARGETS != '' - uses: github/codeql-action/analyze@0225834cc549ee0ca93cb085b92954821a145866 + uses: github/codeql-action/analyze@83f0fe6c4988d98a455712a27f0255212bba9bd4 From 4da4a1175e6774b610f5da4e736abc43ebcda9bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Jun 2023 17:40:18 +0100 Subject: [PATCH 451/740] build(deps): bump confluentinc/cp-zookeeper from `8fb8e15` to `90631f2` in /examples/kafka (#27766) build(deps): bump confluentinc/cp-zookeeper in /examples/kafka Bumps confluentinc/cp-zookeeper from `8fb8e15` to `90631f2`. --- updated-dependencies: - dependency-name: confluentinc/cp-zookeeper dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/kafka/Dockerfile-zookeeper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka/Dockerfile-zookeeper b/examples/kafka/Dockerfile-zookeeper index 1cd685cc6876..b5c8f9c05a9c 100644 --- a/examples/kafka/Dockerfile-zookeeper +++ b/examples/kafka/Dockerfile-zookeeper @@ -1 +1 @@ -FROM confluentinc/cp-zookeeper:latest@sha256:8fb8e15d702d6935d21622ac03e789ba79bcedd3fa809134556d9fed0f88f0e9 +FROM confluentinc/cp-zookeeper:latest@sha256:90631f224b4397ecfb4e824e43d93e0e42656841c6d55ef635c49df3975260ae From 7731469dbdd94e2599fd0188a7c29b35f4f0c280 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Jun 2023 18:15:59 +0100 Subject: [PATCH 452/740] build(deps): bump cryptography from 39.0.2 to 41.0.0 in /.github/actions/pr_notifier (#27781) build(deps): bump cryptography in /.github/actions/pr_notifier Bumps [cryptography](https://github.com/pyca/cryptography) from 39.0.2 to 41.0.0. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/39.0.2...41.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/pr_notifier/requirements.txt | 44 +++++++++----------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/.github/actions/pr_notifier/requirements.txt b/.github/actions/pr_notifier/requirements.txt index f241eeaa3fc0..0ffc7c83a529 100644 --- a/.github/actions/pr_notifier/requirements.txt +++ b/.github/actions/pr_notifier/requirements.txt @@ -138,30 +138,26 @@ charset-normalizer==3.1.0 \ --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab # via requests -cryptography==39.0.2 \ - --hash=sha256:103e8f7155f3ce2ffa0049fe60169878d47a4364b277906386f8de21c9234aa1 \ - --hash=sha256:23df8ca3f24699167daf3e23e51f7ba7334d504af63a94af468f468b975b7dd7 \ - --hash=sha256:2725672bb53bb92dc7b4150d233cd4b8c59615cd8288d495eaa86db00d4e5c06 \ - --hash=sha256:30b1d1bfd00f6fc80d11300a29f1d8ab2b8d9febb6ed4a38a76880ec564fae84 \ - --hash=sha256:35d658536b0a4117c885728d1a7032bdc9a5974722ae298d6c533755a6ee3915 \ - --hash=sha256:50cadb9b2f961757e712a9737ef33d89b8190c3ea34d0fb6675e00edbe35d074 \ - --hash=sha256:5f8c682e736513db7d04349b4f6693690170f95aac449c56f97415c6980edef5 \ - --hash=sha256:6236a9610c912b129610eb1a274bdc1350b5df834d124fa84729ebeaf7da42c3 \ - --hash=sha256:788b3921d763ee35dfdb04248d0e3de11e3ca8eb22e2e48fef880c42e1f3c8f9 \ - --hash=sha256:8bc0008ef798231fac03fe7d26e82d601d15bd16f3afaad1c6113771566570f3 \ - --hash=sha256:8f35c17bd4faed2bc7797d2a66cbb4f986242ce2e30340ab832e5d99ae60e011 \ - --hash=sha256:b49a88ff802e1993b7f749b1eeb31134f03c8d5c956e3c125c75558955cda536 \ - --hash=sha256:bc0521cce2c1d541634b19f3ac661d7a64f9555135e9d8af3980965be717fd4a \ - --hash=sha256:bc5b871e977c8ee5a1bbc42fa8d19bcc08baf0c51cbf1586b0e87a2694dde42f \ - --hash=sha256:c43ac224aabcbf83a947eeb8b17eaf1547bce3767ee2d70093b461f31729a480 \ - --hash=sha256:d15809e0dbdad486f4ad0979753518f47980020b7a34e9fc56e8be4f60702fac \ - --hash=sha256:d7d84a512a59f4412ca8549b01f94be4161c94efc598bf09d027d67826beddc0 \ - --hash=sha256:e029b844c21116564b8b61216befabca4b500e6816fa9f0ba49527653cae2108 \ - --hash=sha256:e8a0772016feeb106efd28d4a328e77dc2edae84dfbac06061319fdb669ff828 \ - --hash=sha256:e944fe07b6f229f4c1a06a7ef906a19652bdd9fd54c761b0ff87e83ae7a30354 \ - --hash=sha256:eb40fe69cfc6f5cdab9a5ebd022131ba21453cf7b8a7fd3631f45bbf52bed612 \ - --hash=sha256:fa507318e427169ade4e9eccef39e9011cdc19534f55ca2f36ec3f388c1f70f3 \ - --hash=sha256:ffd394c7896ed7821a6d13b24657c6a34b6e2650bd84ae063cf11ccffa4f1a97 +cryptography==41.0.0 \ + --hash=sha256:0ddaee209d1cf1f180f1efa338a68c4621154de0afaef92b89486f5f96047c55 \ + --hash=sha256:14754bcdae909d66ff24b7b5f166d69340ccc6cb15731670435efd5719294895 \ + --hash=sha256:344c6de9f8bda3c425b3a41b319522ba3208551b70c2ae00099c205f0d9fd3be \ + --hash=sha256:34d405ea69a8b34566ba3dfb0521379b210ea5d560fafedf9f800a9a94a41928 \ + --hash=sha256:3680248309d340fda9611498a5319b0193a8dbdb73586a1acf8109d06f25b92d \ + --hash=sha256:3c5ef25d060c80d6d9f7f9892e1d41bb1c79b78ce74805b8cb4aa373cb7d5ec8 \ + --hash=sha256:4ab14d567f7bbe7f1cdff1c53d5324ed4d3fc8bd17c481b395db224fb405c237 \ + --hash=sha256:5c1f7293c31ebc72163a9a0df246f890d65f66b4a40d9ec80081969ba8c78cc9 \ + --hash=sha256:6b71f64beeea341c9b4f963b48ee3b62d62d57ba93eb120e1196b31dc1025e78 \ + --hash=sha256:7d92f0248d38faa411d17f4107fc0bce0c42cae0b0ba5415505df72d751bf62d \ + --hash=sha256:8362565b3835ceacf4dc8f3b56471a2289cf51ac80946f9087e66dc283a810e0 \ + --hash=sha256:84a165379cb9d411d58ed739e4af3396e544eac190805a54ba2e0322feb55c46 \ + --hash=sha256:88ff107f211ea696455ea8d911389f6d2b276aabf3231bf72c8853d22db755c5 \ + --hash=sha256:9f65e842cb02550fac96536edb1d17f24c0a338fd84eaf582be25926e993dde4 \ + --hash=sha256:a4fc68d1c5b951cfb72dfd54702afdbbf0fb7acdc9b7dc4301bbf2225a27714d \ + --hash=sha256:b7f2f5c525a642cecad24ee8670443ba27ac1fab81bba4cc24c7b6b41f2d0c75 \ + --hash=sha256:b846d59a8d5a9ba87e2c3d757ca019fa576793e8758174d3868aecb88d6fc8eb \ + --hash=sha256:bf8fc66012ca857d62f6a347007e166ed59c0bc150cefa49f28376ebe7d992a2 \ + --hash=sha256:f5d0bf9b252f30a31664b6f64432b4730bb7038339bd18b1fafe129cfc2be9be # via pyjwt deprecated==1.2.13 \ --hash=sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d \ From ba9a2a7b0ee700117a49f493ff72fe080c5c2b9e Mon Sep 17 00:00:00 2001 From: phlax Date: Sun, 4 Jun 2023 08:30:28 +0100 Subject: [PATCH 453/740] deps: Bump tooling updates to resolve openssl vulns (#27779) Signed-off-by: Ryan Northey --- tools/base/requirements.in | 1 + tools/base/requirements.txt | 289 ++++++++++++++++++------------------ 2 files changed, 147 insertions(+), 143 deletions(-) diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 75ee140ce992..86532c679c85 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -4,6 +4,7 @@ aiohttp>=3.8.1 cffi>=1.15.0 colorama coloredlogs +cryptography>=41.0.1 dependatool>=0.2.2 envoy.base.utils>=0.4.11 envoy.code.check>=0.5.4 diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index a901bba652a5..b60e9117dd11 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -212,9 +212,9 @@ boto==2.49.0 \ --hash=sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8 \ --hash=sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a # via gcs-oauth2-boto-plugin -cachetools==5.3.0 \ - --hash=sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14 \ - --hash=sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4 +cachetools==5.3.1 \ + --hash=sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590 \ + --hash=sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b # via google-auth certifi==2023.5.7 \ --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \ @@ -383,36 +383,37 @@ coloredlogs==15.0.1 \ crcmod==1.7 \ --hash=sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e # via gsutil -cryptography==40.0.2 \ - --hash=sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440 \ - --hash=sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288 \ - --hash=sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b \ - --hash=sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958 \ - --hash=sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b \ - --hash=sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d \ - --hash=sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a \ - --hash=sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404 \ - --hash=sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b \ - --hash=sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e \ - --hash=sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2 \ - --hash=sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c \ - --hash=sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b \ - --hash=sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9 \ - --hash=sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b \ - --hash=sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636 \ - --hash=sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99 \ - --hash=sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e \ - --hash=sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9 +cryptography==41.0.1 \ + --hash=sha256:059e348f9a3c1950937e1b5d7ba1f8e968508ab181e75fc32b879452f08356db \ + --hash=sha256:1a5472d40c8f8e91ff7a3d8ac6dfa363d8e3138b961529c996f3e2df0c7a411a \ + --hash=sha256:1a8e6c2de6fbbcc5e14fd27fb24414507cb3333198ea9ab1258d916f00bc3039 \ + --hash=sha256:1fee5aacc7367487b4e22484d3c7e547992ed726d14864ee33c0176ae43b0d7c \ + --hash=sha256:5d092fdfedaec4cbbffbf98cddc915ba145313a6fdaab83c6e67f4e6c218e6f3 \ + --hash=sha256:5f0ff6e18d13a3de56f609dd1fd11470918f770c6bd5d00d632076c727d35485 \ + --hash=sha256:7bfc55a5eae8b86a287747053140ba221afc65eb06207bedf6e019b8934b477c \ + --hash=sha256:7fa01527046ca5facdf973eef2535a27fec4cb651e4daec4d043ef63f6ecd4ca \ + --hash=sha256:8dde71c4169ec5ccc1087bb7521d54251c016f126f922ab2dfe6649170a3b8c5 \ + --hash=sha256:8f4ab7021127a9b4323537300a2acfb450124b2def3756f64dc3a3d2160ee4b5 \ + --hash=sha256:948224d76c4b6457349d47c0c98657557f429b4e93057cf5a2f71d603e2fc3a3 \ + --hash=sha256:9a6c7a3c87d595608a39980ebaa04d5a37f94024c9f24eb7d10262b92f739ddb \ + --hash=sha256:b46e37db3cc267b4dea1f56da7346c9727e1209aa98487179ee8ebed09d21e43 \ + --hash=sha256:b4ceb5324b998ce2003bc17d519080b4ec8d5b7b70794cbd2836101406a9be31 \ + --hash=sha256:cb33ccf15e89f7ed89b235cff9d49e2e62c6c981a6061c9c8bb47ed7951190bc \ + --hash=sha256:d198820aba55660b4d74f7b5fd1f17db3aa5eb3e6893b0a41b75e84e4f9e0e4b \ + --hash=sha256:d34579085401d3f49762d2f7d6634d6b6c2ae1242202e860f4d26b046e3a1006 \ + --hash=sha256:eb8163f5e549a22888c18b0d53d6bb62a20510060a22fd5a995ec8a05268df8a \ + --hash=sha256:f73bff05db2a3e5974a6fd248af2566134d8981fd7ab012e5dd4ddb1d9a70699 # via + # -r requirements.in # pyjwt # pyopenssl dependatool==0.2.2 \ --hash=sha256:8e66850c79e37325735efa67ce06e2d5a939c0dab758f37b9bd3d09d0fb1f9a4 \ --hash=sha256:dff28853a7252d6a5d670c2519165506902c0d4746cbbdac99d2ad63ed96d82d # via -r requirements.in -deprecated==1.2.13 \ - --hash=sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d \ - --hash=sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d +deprecated==1.2.14 \ + --hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \ + --hash=sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3 # via pygithub docutils==0.19 \ --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ @@ -621,9 +622,9 @@ frozenlist==1.3.3 \ gcs-oauth2-boto-plugin==3.0 \ --hash=sha256:f4120b08b7f8d32904674c98f07d4caf4083a58343c0c0fa0016e0f0254dfe31 # via gsutil -gidgethub==5.2.1 \ - --hash=sha256:8ac9cf5314aac0f3a33f46127d4a538d0ed1a7030027eb2fc4aa3b55c66abcef \ - --hash=sha256:a533f85a57955261433c250701476304cc374037c0ab66b0e5372b4846749d5c +gidgethub==5.3.0 \ + --hash=sha256:4dd92f2252d12756b13f9dd15cde322bfb0d625b6fb5d680da1567ec74b462c0 \ + --hash=sha256:9ece7d37fbceb819b80560e7ed58f936e48a65d37ec5f56db79145156b426a25 # via # aio-api-github # envoy-dependency-check @@ -648,9 +649,9 @@ google-apitools==0.5.32 \ --hash=sha256:b78f74116558e0476e19501b5b4b2ac7c93261a69c5449c861ea95cbc853c688 \ --hash=sha256:c3763e52289f61e21c41d5531e20fbda9cc8484a088b8686fd460770db8bad13 # via gsutil -google-auth[aiohttp]==2.17.3 \ - --hash=sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc \ - --hash=sha256:f586b274d3eb7bd932ea424b1c702a30e0393a2e2bc4ca3eae8263ffd8be229f +google-auth[aiohttp]==2.19.1 \ + --hash=sha256:a9cfa88b3e16196845e64a3658eb953992129d13ac7337b064c6546f77c17183 \ + --hash=sha256:ea165e014c7cbd496558796b627c271aa8c18b4cba79dc1cc962b24c5efdfb85 # via # google-api-core # google-cloud-core @@ -781,57 +782,57 @@ jinja2==3.1.2 \ # envoy-base-utils # envoy-dependency-check # sphinx -markupsafe==2.1.2 \ - --hash=sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed \ - --hash=sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc \ - --hash=sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2 \ - --hash=sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460 \ - --hash=sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7 \ - --hash=sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0 \ - --hash=sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1 \ - --hash=sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa \ - --hash=sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03 \ - --hash=sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323 \ - --hash=sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65 \ - --hash=sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013 \ - --hash=sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036 \ - --hash=sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f \ - --hash=sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4 \ - --hash=sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419 \ - --hash=sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2 \ - --hash=sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619 \ - --hash=sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a \ - --hash=sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a \ - --hash=sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd \ - --hash=sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7 \ - --hash=sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666 \ - --hash=sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65 \ - --hash=sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859 \ - --hash=sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625 \ - --hash=sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff \ - --hash=sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156 \ - --hash=sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd \ - --hash=sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba \ - --hash=sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f \ - --hash=sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1 \ - --hash=sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094 \ - --hash=sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a \ - --hash=sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513 \ - --hash=sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed \ - --hash=sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d \ - --hash=sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3 \ - --hash=sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147 \ - --hash=sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c \ - --hash=sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603 \ - --hash=sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601 \ - --hash=sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a \ - --hash=sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1 \ - --hash=sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d \ - --hash=sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3 \ - --hash=sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54 \ - --hash=sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2 \ - --hash=sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6 \ - --hash=sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58 +markupsafe==2.1.3 \ + --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ + --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ + --hash=sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431 \ + --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ + --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ + --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ + --hash=sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c \ + --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ + --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ + --hash=sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9 \ + --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ + --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ + --hash=sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d \ + --hash=sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3 \ + --hash=sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00 \ + --hash=sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155 \ + --hash=sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac \ + --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ + --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ + --hash=sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8 \ + --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ + --hash=sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24 \ + --hash=sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea \ + --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ + --hash=sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0 \ + --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ + --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ + --hash=sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2 \ + --hash=sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707 \ + --hash=sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6 \ + --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ + --hash=sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779 \ + --hash=sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636 \ + --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ + --hash=sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad \ + --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ + --hash=sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc \ + --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ + --hash=sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48 \ + --hash=sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7 \ + --hash=sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e \ + --hash=sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b \ + --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ + --hash=sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5 \ + --hash=sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e \ + --hash=sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb \ + --hash=sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9 \ + --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ + --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \ + --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 # via jinja2 mccabe==0.7.0 \ --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \ @@ -926,53 +927,53 @@ oauth2client==4.1.3 \ # via # gcs-oauth2-boto-plugin # google-apitools -orjson==3.8.14 \ - --hash=sha256:01640ab79111dd97515cba9fab7c66cb3b0967b0892cc74756a801ff681a01b6 \ - --hash=sha256:017de5ba22e58dfa6f41914f5edb8cd052d23f171000684c26b2d2ab219db31e \ - --hash=sha256:04c70dc8ca79b0072a16d82f94b9d9dd6598a43dd753ab20039e9f7d2b14f017 \ - --hash=sha256:062829b5e20cd8648bf4c11c3a5ee7cf196fa138e573407b5312c849b0cf354d \ - --hash=sha256:087c0dc93379e8ba2d59e9f586fab8de8c137d164fccf8afd5523a2137570917 \ - --hash=sha256:09a3bf3154f40299b8bc95e9fb8da47436a59a2106fc22cae15f76d649e062da \ - --hash=sha256:0bc6b7abf27f1dc192dadad249df9b513912506dd420ce50fd18864a33789b71 \ - --hash=sha256:0bf00c42333412a9338297bf888d7428c99e281e20322070bde8c2314775508b \ - --hash=sha256:19415aaf30525a5baff0d72a089fcdd68f19a3674998263c885c3908228c1086 \ - --hash=sha256:20b7ffc7736000ea205f9143df322b03961f287b4057606291c62c842ff3c5b5 \ - --hash=sha256:27967be4c16bd09f4aeff8896d9be9cbd00fd72f5815d5980e4776f821e2f77c \ - --hash=sha256:31a2a29be559e92dcc5c278787b4166da6f0d45675b59a11c4867f5d1455ebf4 \ - --hash=sha256:33bc310da4ad2ffe8f7f1c9e89692146d9ec5aec2d1c9ef6b67f8dc5e2d63241 \ - --hash=sha256:38ca39bae7fbc050332a374062d4cdec28095540fa8bb245eada467897a3a0bb \ - --hash=sha256:3ee09bfbf1d54c127d3061f6721a1a11d2ce502b50597c3d0d2e1bd2d235b764 \ - --hash=sha256:5ea93fd3ef7be7386f2516d728c877156de1559cda09453fc7dd7b696d0439b3 \ - --hash=sha256:5fb66f0ac23e861b817c858515ac1f74d1cd9e72e3f82a5b2c9bae9f92286adc \ - --hash=sha256:6112194c11e611596eed72f46efb0e6b4812682eff3c7b48473d1146c3fa0efb \ - --hash=sha256:64b4fca0531030040e611c6037aaf05359e296877ab0a8e744c26ef9c32738b9 \ - --hash=sha256:67a7e883b6f782b106683979ccc43d89b98c28a1f4a33fe3a22e253577499bb1 \ - --hash=sha256:716a3994e039203f0a59056efa28185d4cac51b922cc5bf27ab9182cfa20e12e \ - --hash=sha256:739f9f633e1544f2a477fa3bef380f488c8dca6e2521c8dc36424b12554ee31e \ - --hash=sha256:7a7b0fead2d0115ef927fa46ad005d7a3988a77187500bf895af67b365c10d1f \ - --hash=sha256:7cb35dd3ba062c1d984d57e6477768ed7b62ed9260f31362b2d69106f9c60ebd \ - --hash=sha256:7d3d8faded5a514b80b56d0429eb38b429d7a810f8749d25dc10a0cc15b8a3c8 \ - --hash=sha256:7e2f75b7d9285e35c3d4dff9811185535ff2ea637f06b2b242cb84385f8ffe63 \ - --hash=sha256:87ba7882e146e24a7d8b4a7971c20212c2af75ead8096fc3d55330babb1015fb \ - --hash=sha256:8a896a12b38fe201a72593810abc1f4f1597e65b8c869d5fc83bbcf75d93398f \ - --hash=sha256:8b206cca6836a4c6683bcaa523ab467627b5f03902e5e1082dc59cd010e6925f \ - --hash=sha256:92374bc35b6da344a927d5a850f7db80a91c7b837de2f0ea90fc870314b1ff44 \ - --hash=sha256:9393a63cb0424515ec5e434078b3198de6ec9e057f1d33bad268683935f0a5d5 \ - --hash=sha256:9725226478d1dafe46d26f758eadecc6cf98dcbb985445e14a9c74aaed6ccfea \ - --hash=sha256:97ebb7fab5f1ae212a6501f17cb7750a6838ffc2f1cebbaa5dec1a90038ca3c6 \ - --hash=sha256:9df820e6c8c84c52ec39ea2cc9c79f7999c839c7d1481a056908dce3b90ce9f9 \ - --hash=sha256:9f5cf61b6db68f213c805c55bf0aab9b4cb75a4e9c7f5bfbd4deb3a0aef0ec53 \ - --hash=sha256:aedba48264fe87e5060c0e9c2b28909f1e60626e46dc2f77e0c8c16939e2e1f7 \ - --hash=sha256:bf6825e160e4eb0ef65ce37d8c221edcab96ff2ffba65e5da2437a60a12b3ad1 \ - --hash=sha256:ca90db8f551b8960da95b0d4cad6c0489df52ea03585b6979595be7b31a3f946 \ - --hash=sha256:d03f29b0369bb1ab55c8a67103eb3a9675daaf92f04388568034fe16be48fa5d \ - --hash=sha256:d66966fd94719beb84e8ed84833bc59c3c005d3d2d0c42f11d7552d3267c6de7 \ - --hash=sha256:de1ee13d6b6727ee1db38722695250984bae81b8fc9d05f1176c74d14b1322d9 \ - --hash=sha256:e53bc5beb612df8ddddb065f079d3fd30b5b4e73053518524423549d61177f3f \ - --hash=sha256:ebca14ae80814219ea3327e3dfa7ff618621ff335e45781fac26f5cd0b48f2b4 \ - --hash=sha256:ee0299b2dda9afce351a5e8c148ea7a886de213f955aa0288fb874fb44829c36 \ - --hash=sha256:f4ac01a3db4e6a98a8ad1bb1a3e8bfc777928939e87c04e93e0d5006df574a4b \ - --hash=sha256:f80e62afe49e6bfc706e041faa351d7520b5f86572b8e31455802251ea989613 +orjson==3.9.0 \ + --hash=sha256:04e61db09ff155846b69d07cf5aa21001f2010ea669ec3169c1fbad9c9e40cd5 \ + --hash=sha256:08cb43569198c1f5c89ecafcbfc62414f6115d894ff908d8cf8e5e24801364e6 \ + --hash=sha256:09522937479bd39d5bb32d11a5ecdf6926fda43ac2cbde21cc1a9508b4e4ea29 \ + --hash=sha256:09ee828572fadcd58bf356d2c1bad99a95c7c9c1f182b407abbc7dec1810f542 \ + --hash=sha256:0e7fe5d603ee9177ff2e45858b4fc47fea2da0688f23d9773654889d56dfbc82 \ + --hash=sha256:108c58d2c7648c991f82f9b2217c50981ad7cf6aaee3efbfaa9d807e49cd69b8 \ + --hash=sha256:128b1cd0f00a37ba64a12cceeba4e8070655d4400edd55a737513ee663c1ed5a \ + --hash=sha256:1e3bde77c1e0061eb34bae6fea44818b2198e043ee10a16ad7b160921fee26ea \ + --hash=sha256:21f6a6fdfbc13cd715c61e9fa9daeff732df6401ab7d6a2ebad0042313a40bd1 \ + --hash=sha256:2536a7f30fd4d77532769ea9285cd20c69bd2b40acf980de94bbc79b1c6fad5a \ + --hash=sha256:271b6f1018757fc6bca40ae72e6cdb6cf84584dde2d1e5eaac30e387a13d9e72 \ + --hash=sha256:2af7dff1c7ddb0c83eb5773acf6566b153f8cd32e4ba782ae9ccd6d0f324efd3 \ + --hash=sha256:3235c31d0fe674f6e3433e9ddfed212aa840c83a9b6ef5ae128950e2c808c303 \ + --hash=sha256:3a208d0bca609de3152eb8320d5093ad9c52979332f626c13500d1645c66bf8d \ + --hash=sha256:3f1193417b5a93deb41bcb8db27b61179b9b3e299b337b578c31f19159664da3 \ + --hash=sha256:44fa74b497e608a8cdca1ee37fe3533a30f17163c7e2872ab1b854900cf0dfcf \ + --hash=sha256:45df5bf6531ffda518331cc93cdcd4c84f4a4a0507d72af8fb698c7131a440a0 \ + --hash=sha256:46c9733330b75c116438f555c0b971a2388b5f502e2dd4ec3bf6bacb96f82741 \ + --hash=sha256:47d7e4a3effc0e9314bd5b06e7431f2490a5e64dcdcbbc4d60e713786fec327d \ + --hash=sha256:5afd22847b07b63f2b8fcfddd5b7a6f47c5aaa25e19b97a3d6d39508b8fd465a \ + --hash=sha256:6c50654e4870805e4b1a587c2c3c5ef2f36f3e67fc463a738339ff40d65f7db1 \ + --hash=sha256:721d47dffedb7795ffea8a06f2de7d192de7b58e085cf357a99abf0eb931f2c3 \ + --hash=sha256:748c1e8df0b0880c63d323e167ad17ab4db2e1178a40902c2fcb68cbe402d7c8 \ + --hash=sha256:7a3693fde44b2eeb80074ecbe8c504b25baf71e66c080af2a574193a5ba81960 \ + --hash=sha256:86da00836029b2a071229c8aecab998a2f316c1bc7de10ae020d7311de3a6d0d \ + --hash=sha256:88626d898c408450c57664899831cf072787898af4847fa4466607ad2a83f454 \ + --hash=sha256:8a1fcddcabe121e393f3c4a31ed6d3535214d42a4ece0f9dde2e250006d6a58d \ + --hash=sha256:949698bdddb1daff986d73e6bbe6cd68833cd80c4adc6b69fafbd46634d4672c \ + --hash=sha256:9de2129d40674007cb24164939e075b5b39fee768bf20801e08c0e3283bfb18e \ + --hash=sha256:9ee5f1ba82146a50d61fb58d310a37c0f406eda898172f9c98673b5d6f9461c3 \ + --hash=sha256:a901c432828c191332d75f358142736c433d4a192f7794123e1d30d68193de86 \ + --hash=sha256:bd89d63707ac616462832bfc5d16fa0c12483f86add2432ce55c8710c9531c03 \ + --hash=sha256:c41d1ef6ec308e9e3701764b3de889ed8c1c126eceaea881dd1027bffbed89fe \ + --hash=sha256:c4949fc1304b702197c0840882e84b86d8d5ca33c3d945cc60727bc1786c2b20 \ + --hash=sha256:c68af71b1110820c914f9df75842895b5528ff524d3286fde57097b2b5ed8f22 \ + --hash=sha256:c7b241c3229084035b38cac9b5c96b43644da829da41d9d5be0fefb96fb116e1 \ + --hash=sha256:d2fbf34667a8be48ec89d5ef479a00d4e7b3acda62d722c97377702da0c30ffd \ + --hash=sha256:d414fd0678e949779104f5b307f0f9fac861728e19d3cdde66759af77f892da0 \ + --hash=sha256:d4c2d31178e3027affd98eead033f1c406890df83a0ca2016604cc21f722a1d1 \ + --hash=sha256:d4fcf598bd5a99a94caa7ec92ce657939f12491e4753ea7e4d6c03faf5f7912e \ + --hash=sha256:e44ebe2129d43c5a48f3affa3fa59c6484ed16faf5b00486add1061a95384ab0 \ + --hash=sha256:ebe372e9f4e4f0335b7b4ebfab991b3734371e3d5b7f989ca3baa5da25185f4a \ + --hash=sha256:edd77183c154cbedaa6dac32fee9cb770b04e2a7f367a5864f444578554cc946 \ + --hash=sha256:f6476e2487c0b7387187de15e5b8f6635c29b75934f2e689ca8cad6550439f3d \ + --hash=sha256:f6ab80b60195f166a9d666b2eaf6d2c74202b6da2a1fb4b4d66b9cc0ce5c9957 \ + --hash=sha256:f6dd27c71cd6e146795f876449a8eae74f67ae1e4e244dfc1203489103eb2d94 # via # -r requirements.in # envoy-base-utils @@ -1075,9 +1076,9 @@ pynacl==1.5.0 \ --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \ --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543 # via pygithub -pyopenssl==23.1.1 \ - --hash=sha256:841498b9bec61623b1b6c47ebbc02367c07d60e0e195f19790817f10cc8db0b7 \ - --hash=sha256:9e0c526404a210df9d2b18cd33364beadb0dc858a739b885677bc65e105d4a4c +pyopenssl==23.2.0 \ + --hash=sha256:24f0dc5227396b3e831f4c7f602b950a5e9833d292c8e4a2e06b709292806ae2 \ + --hash=sha256:276f931f55a452e7dea69c7173e984eb2a4407ce413c918aa34b55f82f9b8bac # via # gcs-oauth2-boto-plugin # gsutil @@ -1257,18 +1258,20 @@ trycast==1.0.0 \ # via # aio-core # envoy-base-utils -typing-extensions==4.5.0 \ - --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \ - --hash=sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4 +typing-extensions==4.6.3 \ + --hash=sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26 \ + --hash=sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5 # via aiodocker uritemplate==4.1.1 \ --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e # via gidgethub -urllib3==2.0.2 \ - --hash=sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc \ - --hash=sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e - # via requests +urllib3==1.26.16 \ + --hash=sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f \ + --hash=sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14 + # via + # google-auth + # requests uvloop==0.17.0 \ --hash=sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d \ --hash=sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1 \ @@ -1388,9 +1391,9 @@ wrapt==1.15.0 \ --hash=sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559 \ --hash=sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639 # via deprecated -yamllint==1.31.0 \ - --hash=sha256:15f4bdb645e6a4a0a22fe5415bc38b4a934c51419b30104896d2f3f95e329185 \ - --hash=sha256:2d83f1d12f733e162a87e06b176149d7bb9c5bae4a9e5fce1c771d7f703f7a65 +yamllint==1.32.0 \ + --hash=sha256:d01dde008c65de5b235188ab3110bebc59d18e5c65fc8a58267cd211cd9df34a \ + --hash=sha256:d97a66e48da820829d96077d76b8dfbe6c6140f106e558dae87e81ac4e6b30b7 # via envoy-code-check yapf==0.33.0 \ --hash=sha256:4c2b59bd5ffe46f3a7da48df87596877189148226ce267c16e8b44240e51578d \ From 741aec091014f2f4374c77a064812a5ca9917ae4 Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 5 Jun 2023 07:31:34 +0100 Subject: [PATCH 454/740] release/ci: Prevent release if Docker job fails (#27782) Signed-off-by: Ryan Northey --- .azure-pipelines/stage/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index b028aa88c9ea..7d4e5fef9887 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -392,6 +392,7 @@ jobs: dependsOn: ["success"] condition: | and(not(canceled()), + in(dependencies.success.result, 'Succeeded', 'SucceededWithIssues'), eq(${{ parameters.publishGithubRelease }}, 'true')) pool: vmImage: "ubuntu-20.04" From 22866f4c3af0a7875bb4ee32722703202e6b441e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jun 2023 07:42:31 +0100 Subject: [PATCH 455/740] build(deps): bump confluentinc/cp-kafka from `328bb7b` to `187dac6` in /examples/kafka (#27767) build(deps): bump confluentinc/cp-kafka in /examples/kafka Bumps confluentinc/cp-kafka from `328bb7b` to `187dac6`. --- updated-dependencies: - dependency-name: confluentinc/cp-kafka dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/kafka/Dockerfile-kafka | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka/Dockerfile-kafka b/examples/kafka/Dockerfile-kafka index 46f63575c21e..d354e9a3650f 100644 --- a/examples/kafka/Dockerfile-kafka +++ b/examples/kafka/Dockerfile-kafka @@ -1 +1 @@ -FROM confluentinc/cp-kafka:latest@sha256:328bb7b00e15f173bee5f68a9da0c97651ca961e2f7736563dc55bbebac80e5e +FROM confluentinc/cp-kafka:latest@sha256:187dac6627e7906c350f5f8c982f80ce735ff1a0e571a20de6000a309a12ce63 From 20f12ed907c588d7d0ed0d3320d22f663c13864b Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 5 Jun 2023 08:42:50 +0100 Subject: [PATCH 456/740] release/ci: Add dry-run var for non-postsubmit (#27784) Signed-off-by: Ryan Northey --- .azure-pipelines/env.yml | 7 +++++++ .azure-pipelines/stage/publish.yml | 6 ++++++ .azure-pipelines/stages.yml | 2 ++ ci/do_ci.sh | 2 +- ci/run_envoy_docker.sh | 1 + 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/env.yml b/.azure-pipelines/env.yml index 01a9a78d081f..9511877caf0a 100644 --- a/.azure-pipelines/env.yml +++ b/.azure-pipelines/env.yml @@ -183,6 +183,8 @@ jobs: set -e PUBLISH_GITHUB_RELEASE=$(run.packaging) + # NB: leave this empty as it is checked `-n` + PUBLISH_GITHUB_RELEASE_DRY_RUN= PUBLISH_DOCKERHUB=false PUBLISH_DOCS=false PUBLISH_DOCS_LATEST=false @@ -212,8 +214,12 @@ jobs: PUBLISH_DOCS=false fi fi + if [[ -z "$POSTSUBMIT" ]]; then + PUBLISH_GITHUB_RELEASE_DRY_RUN=true + fi echo "##vso[task.setvariable variable=githubRelease;isoutput=true]${PUBLISH_GITHUB_RELEASE}" + echo "##vso[task.setvariable variable=githubReleaseDryRun;isoutput=true]${PUBLISH_GITHUB_RELEASE_DRY_RUN}" echo "##vso[task.setvariable variable=dockerhub;isoutput=true]${PUBLISH_DOCKERHUB}" echo "##vso[task.setvariable variable=docs;isoutput=true]${PUBLISH_DOCS}" echo "##vso[task.setvariable variable=docsLatest;isoutput=true]${PUBLISH_DOCS_LATEST}" @@ -240,6 +246,7 @@ jobs: echo "env.outputs['run.packaging']: $(run.packaging)" echo echo "env.outputs['publish.githubRelease']: $(publish.githubRelease)" + echo "env.outputs['publish.githubReleaseDryRun']: $(publish.githubReleaseDryRun)" echo "env.outputs['publish.dockerhub]: $(publish.dockerhub)" echo "env.outputs['publish.docs]: $(publish.docs)" echo "env.outputs['publish.docsLatest]: $(publish.docsLatest)" diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index 7d4e5fef9887..beb715a6b46d 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -90,6 +90,10 @@ parameters: displayName: "Publish Github release" type: string default: false +- name: publishGithubReleaseDryRun + displayName: "Publish Github release (dry run)" + type: string + default: false jobs: @@ -409,3 +413,5 @@ jobs: cacheVersion: $(cacheKeyBazel) publishEnvoy: false publishTestResults: false + env: + ENVOY_PUBLISH_DRY_RUN: ${{ parameters.publishGithubReleaseDryRun }} diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index de69153bfbf7..4100bd1258b9 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -122,6 +122,7 @@ stages: RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']] RUN_PACKAGING: $[stageDependencies.env.repo.outputs['run.packaging']] PUBLISH_GITHUB_RELEASE: $[stageDependencies.env.repo.outputs['publish.githubRelease']] + PUBLISH_GITHUB_RELEASE_DRY_RUN: $[stageDependencies.env.repo.outputs['publish.githubReleaseDryRun']] PUBLISH_DOCKERHUB: $[stageDependencies.env.repo.outputs['publish.dockerhub']] PUBLISH_DOCS: $[stageDependencies.env.repo.outputs['publish.docs']] PUBLISH_DOCS_LATEST: $[stageDependencies.env.repo.outputs['publish.docsLatest']] @@ -153,6 +154,7 @@ stages: publishDocsRelease: variables['PUBLISH_DOCS_RELEASE'] publishDockerhub: variables['PUBLISH_DOCKERHUB'] publishGithubRelease: variables['PUBLISH_GITHUB_RELEASE'] + publishGithubReleaseDryRun: variables['PUBLISH_GITHUB_RELEASE_DRY_RUN'] - stage: verify displayName: Verify diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 2d7b61298396..c320a7a29e4a 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -624,7 +624,7 @@ case $CI_TARGET in PUBLISH_ARGS=( --publish-commitish="$BUILD_SHA" --publish-assets=/build/release.signed/release.signed.tar.zst) - if [[ "$VERSION_DEV" == "dev" ]]; then + if [[ "$VERSION_DEV" == "dev" ]] || [[ -n "$ENVOY_PUBLISH_DRY_RUN" ]]; then PUBLISH_ARGS+=(--dry-run) fi bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 7fd9bac73335..fdda903eae86 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -133,6 +133,7 @@ docker run --rm \ -e ENVOY_BUILD_FILTER_EXAMPLE \ -e ENVOY_COMMIT \ -e ENVOY_HEAD_REF \ + -e ENVOY_PUBLISH_DRY_RUN \ -e ENVOY_REPO \ -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ -e GCS_ARTIFACT_BUCKET \ From fe8fd73ef6fe46644fe65098ec8d629f3f384530 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jun 2023 09:13:47 +0100 Subject: [PATCH 457/740] build(deps): bump mysql from `d6164ff` to `4bae986` in /examples/mysql (#27797) Bumps mysql from `d6164ff` to `4bae986`. --- updated-dependencies: - dependency-name: mysql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/mysql/Dockerfile-mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mysql/Dockerfile-mysql b/examples/mysql/Dockerfile-mysql index de8c80df6a3e..d1038cf5ca42 100644 --- a/examples/mysql/Dockerfile-mysql +++ b/examples/mysql/Dockerfile-mysql @@ -1 +1 @@ -FROM mysql:8.0.33@sha256:d6164ff4855b9b3f2c7748c6ec564ccff841f79a7023db0f9293143481a44b6e +FROM mysql:8.0.33@sha256:4bae98614cd6ad1aecbdd32ff1b37b93fb0ee22b069469e7bc9679bacef1abd2 From 0d6fab926bb9f166200e90153ca3b403b0cf9640 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jun 2023 09:16:27 +0100 Subject: [PATCH 458/740] build(deps): bump jaegertracing/all-in-one from `1cb8093` to `12e96c7` in /examples/shared/jaeger (#27798) build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger Bumps jaegertracing/all-in-one from `1cb8093` to `12e96c7`. --- updated-dependencies: - dependency-name: jaegertracing/all-in-one dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/jaeger/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile index b9fcb0029145..5c671c6b6cad 100644 --- a/examples/shared/jaeger/Dockerfile +++ b/examples/shared/jaeger/Dockerfile @@ -1,4 +1,4 @@ -FROM jaegertracing/all-in-one@sha256:1cb80937ebe12ed06267d2f773623d33a4610df9ef59d82e92c15613f31772ed +FROM jaegertracing/all-in-one@sha256:12e96c7396d758da7d300d162f2aab899c2f6cc51e037cc7bced8dbd6d3cfc5d HEALTHCHECK \ --interval=1s \ --timeout=1s \ From 1890fb1ea9101ea234e13de6dc43e37507081e79 Mon Sep 17 00:00:00 2001 From: Ali Beyad Date: Mon, 5 Jun 2023 12:10:24 -0400 Subject: [PATCH 459/740] xds: Add extension factory registration for EDS (#27793) Signed-off-by: Ali Beyad --- source/extensions/clusters/eds/BUILD | 1 + source/extensions/clusters/eds/eds.h | 3 +++ 2 files changed, 4 insertions(+) diff --git a/source/extensions/clusters/eds/BUILD b/source/extensions/clusters/eds/BUILD index cbdbbd402395..b7b2984fb8ec 100644 --- a/source/extensions/clusters/eds/BUILD +++ b/source/extensions/clusters/eds/BUILD @@ -20,6 +20,7 @@ envoy_cc_extension( "//envoy/config:subscription_factory_interface", "//envoy/config:subscription_interface", "//envoy/local_info:local_info_interface", + "//envoy/registry", "//envoy/secret:secret_manager_interface", "//envoy/upstream:cluster_factory_interface", "//envoy/upstream:locality_lib", diff --git a/source/extensions/clusters/eds/eds.h b/source/extensions/clusters/eds/eds.h index 7513f2f18e01..b362cd07325e 100644 --- a/source/extensions/clusters/eds/eds.h +++ b/source/extensions/clusters/eds/eds.h @@ -10,6 +10,7 @@ #include "envoy/config/subscription.h" #include "envoy/config/subscription_factory.h" #include "envoy/local_info/local_info.h" +#include "envoy/registry/registry.h" #include "envoy/secret/secret_manager.h" #include "envoy/service/discovery/v3/discovery.pb.h" #include "envoy/stats/scope.h" @@ -118,5 +119,7 @@ class EdsClusterFactory : public ClusterFactoryImplBase { ClusterFactoryContext& context) override; }; +DECLARE_FACTORY(EdsClusterFactory); + } // namespace Upstream } // namespace Envoy From a9ec898d6dfdb4875a5b3684a6ee84afd4bb9663 Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Mon, 5 Jun 2023 21:23:10 +0300 Subject: [PATCH 460/740] application_logs: add bootstrap option to write logs in JSON format (#27278) Commit Message: application_logs: add bootstrap option to write logs in JSON format Additional Description: Adds an option in bootstrap config to write application logs in JSON format, while supporting all the log-format flags as defined in the CLI --log-format option. Related to #25959 - this is the first step in the implementation for supporting custom JSON properties, while printing the application logs output in JSON format. Risk Level: Low (all new code paths are only enabled by config option) Testing: Unit tests Docs Changes: API, Application logs docs Release Notes: None Platform Specific Features: None Signed-off-by: ohadvano Signed-off-by: ohadvano <49730675+ohadvano@users.noreply.github.com> --- api/envoy/config/bootstrap/v3/bootstrap.proto | 23 ++- changelogs/current.yaml | 5 + .../observability/application_logging.rst | 22 +++ envoy/server/options.h | 5 + .../integration/client_integration_test.cc | 2 +- source/common/common/BUILD | 1 + source/common/common/logger.cc | 25 ++++ source/common/common/logger.h | 6 + source/server/BUILD | 2 + source/server/config_validation/BUILD | 1 + source/server/config_validation/server.cc | 6 + source/server/options_impl.cc | 1 + source/server/options_impl.h | 7 +- source/server/server.cc | 5 + source/server/utils.cc | 24 ++++ source/server/utils.h | 9 ++ test/common/common/logger_test.cc | 136 ++++++++++++++++-- test/mocks/common.h | 12 ++ test/mocks/server/options.h | 1 + test/server/BUILD | 1 + test/server/config_validation/server_test.cc | 130 ++++++++++++++++- .../test_data/json_application_logs.yaml | 11 ++ ...json_application_logs_forbidden_flag_.yaml | 11 ++ ...json_application_logs_forbidden_flagv.yaml | 11 ++ test/server/options_impl_test.cc | 5 + test/server/server_test.cc | 36 +++++ .../server/json_application_log.yaml | 11 ++ .../json_application_log_forbidden_flag_.yaml | 11 ++ .../json_application_log_forbidden_flagv.yaml | 11 ++ test/server/utils_test.cc | 73 ++++++++++ tools/code_format/config.yaml | 1 + 31 files changed, 585 insertions(+), 20 deletions(-) create mode 100644 test/server/config_validation/test_data/json_application_logs.yaml create mode 100644 test/server/config_validation/test_data/json_application_logs_forbidden_flag_.yaml create mode 100644 test/server/config_validation/test_data/json_application_logs_forbidden_flagv.yaml create mode 100644 test/server/test_data/server/json_application_log.yaml create mode 100644 test/server/test_data/server/json_application_log_forbidden_flag_.yaml create mode 100644 test/server/test_data/server/json_application_log_forbidden_flagv.yaml diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index 6b230536cd86..f171068aaeed 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -41,7 +41,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // ` for more detail. // Bootstrap :ref:`configuration overview `. -// [#next-free-field: 38] +// [#next-free-field: 39] message Bootstrap { option (udpa.annotations.versioning).previous_message_type = "envoy.config.bootstrap.v2.Bootstrap"; @@ -101,6 +101,24 @@ message Bootstrap { core.v3.ApiConfigSource ads_config = 3; } + message ApplicationLogConfig { + message LogFormat { + oneof log_format { + option (validate.required) = true; + + // Flush application logs in JSON format. The configured JSON struct can + // support all the format flags specified in the :option:`--log-format` + // command line options section, except for the ``%v`` and ``%_`` flags. + google.protobuf.Struct json_format = 1; + } + } + + // Optional field to set the application logs format. If this field is set, it will override + // the default log format. Setting both this field and :option:`--log-format` command line + // option is not allowed, and will cause a bootstrap error. + LogFormat log_format = 1; + } + reserved 10, 11; reserved "runtime"; @@ -360,6 +378,9 @@ message Bootstrap { // Envoy only supports ListenerManager for this field and Envoy Mobile // supports ApiListenerManager. core.v3.TypedExtensionConfig listener_manager = 37; + + // Optional application log configuration. + ApplicationLogConfig application_log_config = 38; } // Administration interface :ref:`operations documentation diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 932d78b1e19f..123aa4ab1f55 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -287,6 +287,11 @@ new_features: change: | added new field ``envoy.extensions.filters.http.fault.v3.HTTPFault.filter_metadata`` to aid in logging. Metadata will be stored in StreamInfo dynamic metadata under a namespace corresponding to the name of the fault filter. +- area: application_logs + change: | + Added bootstrap option + :ref:`application_log_format ` + to enable setting application log format as JSON structure. - area: ext_proc change: | added new field ``filter_metadata `: * The ``--log-level`` flag can be set to control the log severity logged to Stackdriver. `Reference documentation `_ for Stackdriver on GKE. + +Printing logs in JSON format +---------------------------- + +It is possible to use the bootstrap config :ref:`json_format ` +to print the logs in custom JSON format. The json format struct can support all the format flags that are specified in :ref:`command line options `, +except for the ``%v`` and ``%_`` flags, as they may break the JSON structure log. Instead, use the ``%j`` flag. Example: + +.. code-block:: yaml + + application_log_config: + log_format: + json_format: + Timestamp: "%Y-%m-%dT%T.%F" + ThreadId: "%t" + SourceLine: "%s:%#" + Level: "%l" + Message: "%j" + FixedValue: "SomeFixedValue" + +.. note:: + Setting both ``application_log_config.log_format`` and CLI option ``--log-format`` is not allowed, and will cause a bootstrap error. diff --git a/envoy/server/options.h b/envoy/server/options.h index ab90efdd1f0e..327da389107a 100644 --- a/envoy/server/options.h +++ b/envoy/server/options.h @@ -169,6 +169,11 @@ class Options { */ virtual const std::string& logFormat() const PURE; + /** + * @return whether or not a log format was set by CLI option. + */ + virtual bool logFormatSet() const PURE; + /** * @return const bool indicating whether to escape c-style escape sequences in logs. */ diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index ac06e467d150..13b2e3997e89 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -97,7 +97,7 @@ void ClientIntegrationTest::trickleTest() { stream_prototype_->setOnData([this](envoy_data c_data, bool) { if (explicit_flow_control_) { - // Allow reading up to 100 bytes + // Allow reading up to 100 bytes. stream_->readData(100); } cc_.on_data_calls++; diff --git a/source/common/common/BUILD b/source/common/common/BUILD index 2f45fc1c671c..7c00df446757 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -211,6 +211,7 @@ envoy_cc_library( ":lock_guard_lib", ":macros", ":non_copyable", + "//source/common/protobuf:protobuf", ] + select({ "//bazel:android_logger": ["logger_impl_lib_android"], "//conditions:default": ["logger_impl_lib_standard"], diff --git a/source/common/common/logger.cc b/source/common/common/logger.cc index 9b02e6749660..476f2e06a9f9 100644 --- a/source/common/common/logger.cc +++ b/source/common/common/logger.cc @@ -254,6 +254,31 @@ void Registry::setLogFormat(const std::string& log_format) { } } +absl::Status Registry::setJsonLogFormat(const Protobuf::Message& log_format_struct) { + Protobuf::util::JsonPrintOptions json_options; + json_options.preserve_proto_field_names = true; + json_options.always_print_primitive_fields = true; + + std::string format_as_json; + const auto status = + Protobuf::util::MessageToJsonString(log_format_struct, &format_as_json, json_options); + + if (!status.ok()) { + return absl::InvalidArgumentError("Provided struct cannot be serialized as JSON string"); + } + + if (format_as_json.find("%v") != std::string::npos) { + return absl::InvalidArgumentError("Usage of %v is unavailable for JSON log formats"); + } + + if (format_as_json.find("%_") != std::string::npos) { + return absl::InvalidArgumentError("Usage of %_ is unavailable for JSON log formats"); + } + + setLogFormat(format_as_json); + return absl::OkStatus(); +} + Logger* Registry::logger(const std::string& log_name) { Logger* logger_to_return = nullptr; for (Logger& logger : loggers()) { diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 7d7b86f29f2c..28aecaf86640 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -15,6 +15,7 @@ #include "source/common/common/logger_impl.h" #include "source/common/common/macros.h" #include "source/common/common/non_copyable.h" +#include "source/common/protobuf/protobuf.h" #include "absl/container/flat_hash_map.h" #include "absl/strings/string_view.h" @@ -353,6 +354,11 @@ class Registry { */ static void setLogFormat(const std::string& log_format); + /** + * Sets the log format from a struct as a JSON string. + */ + static absl::Status setJsonLogFormat(const Protobuf::Message& log_format_struct); + /** * @return std::vector& the installed loggers. */ diff --git a/source/server/BUILD b/source/server/BUILD index a5709a05c6e1..fc81eafc00f3 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -513,7 +513,9 @@ envoy_cc_library( hdrs = ["utils.h"], deps = [ "//envoy/init:manager_interface", + "//envoy/server:options_interface", "//source/common/common:assert_lib", "@envoy_api//envoy/admin/v3:pkg_cc_proto", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ], ) diff --git a/source/server/config_validation/BUILD b/source/server/config_validation/BUILD index 5d79fefce13a..f5e8bd30b1d9 100644 --- a/source/server/config_validation/BUILD +++ b/source/server/config_validation/BUILD @@ -101,6 +101,7 @@ envoy_cc_library( "//source/common/version:version_lib", "//source/server:configuration_lib", "//source/server:server_lib", + "//source/server:utils_lib", "//source/server/admin:admin_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/source/server/config_validation/server.cc b/source/server/config_validation/server.cc index c836e4cd1d1a..dc80c3e7145b 100644 --- a/source/server/config_validation/server.cc +++ b/source/server/config_validation/server.cc @@ -15,6 +15,7 @@ #include "source/server/listener_manager_factory.h" #include "source/server/regex_engine.h" #include "source/server/ssl_context_manager.h" +#include "source/server/utils.h" namespace Envoy { namespace Server { @@ -85,6 +86,11 @@ void ValidationInstance::initialize(const Options& options, InstanceUtil::loadBootstrapConfig(bootstrap_, options, messageValidationContext().staticValidationVisitor(), *api_); + if (bootstrap_.has_application_log_config()) { + Utility::assertExclusiveLogFormatMethod(options_, bootstrap_.application_log_config()); + Utility::maybeSetApplicationLogFormat(bootstrap_.application_log_config()); + } + // Inject regex engine to singleton. Regex::EnginePtr regex_engine = createRegexEngine( bootstrap_, messageValidationContext().staticValidationVisitor(), serverFactoryContext()); diff --git a/source/server/options_impl.cc b/source/server/options_impl.cc index eae78c65d513..6345c789e897 100644 --- a/source/server/options_impl.cc +++ b/source/server/options_impl.cc @@ -194,6 +194,7 @@ OptionsImpl::OptionsImpl(std::vector args, } log_format_ = log_format.getValue(); + log_format_set_ = log_format.isSet(); log_format_escaped_ = log_format_escaped.getValue(); enable_fine_grain_logging_ = enable_fine_grain_logging.getValue(); diff --git a/source/server/options_impl.h b/source/server/options_impl.h index 6528e0aeb8a1..d02f37dce5bc 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -78,7 +78,10 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable> component_log_levels_; std::string component_log_level_str_; std::string log_format_{Logger::Logger::DEFAULT_LOG_FORMAT}; + bool log_format_set_{false}; bool log_format_escaped_{false}; std::string log_path_; uint64_t restart_epoch_{0}; diff --git a/source/server/server.cc b/source/server/server.cc index f5c2b062ea8b..8ed78eb8b139 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -422,6 +422,11 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add messageValidationContext().staticValidationVisitor(), *api_); bootstrap_config_update_time_ = time_source_.systemTime(); + if (bootstrap_.has_application_log_config()) { + Utility::assertExclusiveLogFormatMethod(options_, bootstrap_.application_log_config()); + Utility::maybeSetApplicationLogFormat(bootstrap_.application_log_config()); + } + #ifdef ENVOY_PERFETTO perfetto::TracingInitArgs args; // Include in-process events only. diff --git a/source/server/utils.cc b/source/server/utils.cc index fcb5e043ec99..7bb8957a59c6 100644 --- a/source/server/utils.cc +++ b/source/server/utils.cc @@ -1,5 +1,7 @@ #include "source/server/utils.h" +#include "envoy/common/exception.h" + #include "source/common/common/assert.h" namespace Envoy { @@ -21,6 +23,28 @@ envoy::admin::v3::ServerInfo::State serverState(Init::Manager::State state, return envoy::admin::v3::ServerInfo::PRE_INITIALIZING; } +void assertExclusiveLogFormatMethod( + const Options& options, + const envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig& application_log_config) { + if (options.logFormatSet() && application_log_config.has_log_format()) { + throw EnvoyException( + "Only one of ApplicationLogConfig.log_format or CLI option --log-format can be specified."); + } +} + +void maybeSetApplicationLogFormat( + const envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig& application_log_config) { + if (application_log_config.has_log_format() && + application_log_config.log_format().has_json_format()) { + const auto status = + Logger::Registry::setJsonLogFormat(application_log_config.log_format().json_format()); + + if (!status.ok()) { + throw EnvoyException(fmt::format("setJsonLogFormat error: {}", status.ToString())); + } + } +} + } // namespace Utility } // namespace Server } // namespace Envoy diff --git a/source/server/utils.h b/source/server/utils.h index 1ab4b51de275..2d3b981c2c87 100644 --- a/source/server/utils.h +++ b/source/server/utils.h @@ -1,7 +1,9 @@ #pragma once #include "envoy/admin/v3/server_info.pb.h" +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/init/manager.h" +#include "envoy/server/options.h" namespace Envoy { namespace Server { @@ -14,6 +16,13 @@ namespace Utility { envoy::admin::v3::ServerInfo::State serverState(Init::Manager::State state, bool health_check_failed); +void assertExclusiveLogFormatMethod( + const Options& options, + const envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig& application_log_config); + +void maybeSetApplicationLogFormat( + const envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig& application_log_config); + } // namespace Utility } // namespace Server } // namespace Envoy diff --git a/test/common/common/logger_test.cc b/test/common/common/logger_test.cc index b026eddc6419..e9998db311e3 100644 --- a/test/common/common/logger_test.cc +++ b/test/common/common/logger_test.cc @@ -4,6 +4,7 @@ #include "source/common/common/json_escape_string.h" #include "source/common/common/logger.h" +#include "test/mocks/common.h" #include "test/test_common/environment.h" #include "gmock/gmock.h" @@ -176,17 +177,6 @@ TEST_P(LoggerCustomFlagsTest, LogMessageAsJsonStringEscaped) { "StreamAggregatedResources gRPC config stream closed: 14, connection error: desc = " "\\\"transport: Error while dialing dial tcp [::1]:15012: connect: connection refused\\\""); } - -struct MockLogSink : SinkDelegate { - MockLogSink(DelegatingLogSinkSharedPtr log_sink) : SinkDelegate(log_sink) { setDelegate(); } - ~MockLogSink() override { restoreDelegate(); } - - MOCK_METHOD(void, log, (absl::string_view, const spdlog::details::log_msg&)); - MOCK_METHOD(void, logWithStableName, - (absl::string_view, absl::string_view, absl::string_view, absl::string_view)); - void flush() override {} -}; - class NamedLogTest : public Loggable, public testing::Test {}; TEST_F(NamedLogTest, NamedLogsAreSentToSink) { @@ -270,6 +260,130 @@ TEST(LoggerTest, LogWithLogDetails) { ENVOY_LOG_MISC(info, "hello"); } +TEST(LoggerTest, TestJsonFormatError) { + ProtobufWkt::Any log_struct; + log_struct.set_type_url("type.googleapis.com/bad.type.url"); + log_struct.set_value("asdf"); + + // This scenario shouldn't happen in production, the test is added mainly for coverage. + auto status = Envoy::Logger::Registry::setJsonLogFormat(log_struct); + EXPECT_FALSE(status.ok()); + EXPECT_EQ("INVALID_ARGUMENT: Provided struct cannot be serialized as JSON string", + status.ToString()); +} + +TEST(LoggerTest, TestJsonFormatNonEscapedThrows) { + Envoy::Logger::Registry::setLogLevel(spdlog::level::info); + + { + ProtobufWkt::Struct log_struct; + (*log_struct.mutable_fields())["Message"].set_string_value("%v"); + (*log_struct.mutable_fields())["NullField"].set_null_value(ProtobufWkt::NULL_VALUE); + + auto status = Envoy::Logger::Registry::setJsonLogFormat(log_struct); + EXPECT_FALSE(status.ok()); + EXPECT_EQ("INVALID_ARGUMENT: Usage of %v is unavailable for JSON log formats", + status.ToString()); + } + + { + ProtobufWkt::Struct log_struct; + (*log_struct.mutable_fields())["Message"].set_string_value("%_"); + (*log_struct.mutable_fields())["NullField"].set_null_value(ProtobufWkt::NULL_VALUE); + + auto status = Envoy::Logger::Registry::setJsonLogFormat(log_struct); + EXPECT_FALSE(status.ok()); + EXPECT_EQ("INVALID_ARGUMENT: Usage of %_ is unavailable for JSON log formats", + status.ToString()); + } +} + +TEST(LoggerTest, TestJsonFormatEmptyStruct) { + ProtobufWkt::Struct log_struct; + Envoy::Logger::Registry::setLogLevel(spdlog::level::info); + EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok()); + + MockLogSink sink(Envoy::Logger::Registry::getSink()); + EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto& log) { + EXPECT_THAT(msg, HasSubstr("{}")); + EXPECT_EQ(log.logger_name, "misc"); + })); + + ENVOY_LOG_MISC(info, "hello"); +} + +TEST(LoggerTest, TestJsonFormatNullAndFixedField) { + ProtobufWkt::Struct log_struct; + (*log_struct.mutable_fields())["Message"].set_string_value("%j"); + (*log_struct.mutable_fields())["FixedValue"].set_string_value("Fixed"); + (*log_struct.mutable_fields())["NullField"].set_null_value(ProtobufWkt::NULL_VALUE); + Envoy::Logger::Registry::setLogLevel(spdlog::level::info); + EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok()); + + MockLogSink sink(Envoy::Logger::Registry::getSink()); + EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto&) { + EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg))); + EXPECT_THAT(msg, HasSubstr("\"Message\":\"hello\"")); + EXPECT_THAT(msg, HasSubstr("\"FixedValue\":\"Fixed\"")); + EXPECT_THAT(msg, HasSubstr("\"NullField\":null")); + })); + + ENVOY_LOG_MISC(info, "hello"); +} + +TEST(LoggerTest, TestJsonFormat) { + ProtobufWkt::Struct log_struct; + (*log_struct.mutable_fields())["Level"].set_string_value("%l"); + (*log_struct.mutable_fields())["Message"].set_string_value("%j"); + Envoy::Logger::Registry::setLogLevel(spdlog::level::info); + EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok()); + + MockLogSink sink(Envoy::Logger::Registry::getSink()); + EXPECT_CALL(sink, log(_, _)) + .WillOnce(Invoke([](auto msg, auto& log) { + EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg))); + EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\"")); + EXPECT_THAT(msg, HasSubstr("\"Message\":\"hello\"")); + EXPECT_EQ(log.logger_name, "misc"); + })) + .WillOnce(Invoke([](auto msg, auto& log) { + EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg))); + EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\"")); + EXPECT_THAT(msg, HasSubstr("\"Message\":\"hel\\nlo\"")); + EXPECT_EQ(log.logger_name, "misc"); + })) + .WillOnce(Invoke([](auto msg, auto& log) { + EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg))); + EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\"")); + EXPECT_THAT(msg, HasSubstr("\"Message\":\"hel\\\"lo\"")); + EXPECT_EQ(log.logger_name, "misc"); + })); + + ENVOY_LOG_MISC(info, "hello"); + ENVOY_LOG_MISC(info, "hel\nlo"); + ENVOY_LOG_MISC(info, "hel\"lo"); +} + +TEST(LoggerTest, TestJsonFormatWithNestedJsonMessage) { + ProtobufWkt::Struct log_struct; + (*log_struct.mutable_fields())["Level"].set_string_value("%l"); + (*log_struct.mutable_fields())["Message"].set_string_value("%j"); + (*log_struct.mutable_fields())["FixedValue"].set_string_value("Fixed"); + Envoy::Logger::Registry::setLogLevel(spdlog::level::info); + EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok()); + + MockLogSink sink(Envoy::Logger::Registry::getSink()); + EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto& log) { + EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg))); + EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\"")); + EXPECT_THAT(msg, HasSubstr("\"Message\":\"{\\\"nested_message\\\":\\\"hello\\\"}\"")); + EXPECT_THAT(msg, HasSubstr("\"FixedValue\":\"Fixed\"")); + EXPECT_EQ(log.logger_name, "misc"); + })); + + ENVOY_LOG_MISC(info, "{\"nested_message\":\"hello\"}"); +} + } // namespace } // namespace Logger } // namespace Envoy diff --git a/test/mocks/common.h b/test/mocks/common.h index 058a7736af69..1e496030e4f9 100644 --- a/test/mocks/common.h +++ b/test/mocks/common.h @@ -143,4 +143,16 @@ class MockKeyValueStoreFactory : public KeyValueStoreFactory { std::string name() const override { return "mock_key_value_store_factory"; } }; +struct MockLogSink : Logger::SinkDelegate { + MockLogSink(Logger::DelegatingLogSinkSharedPtr log_sink) : Logger::SinkDelegate(log_sink) { + setDelegate(); + } + ~MockLogSink() override { restoreDelegate(); } + + MOCK_METHOD(void, log, (absl::string_view, const spdlog::details::log_msg&)); + MOCK_METHOD(void, logWithStableName, + (absl::string_view, absl::string_view, absl::string_view, absl::string_view)); + void flush() override {} +}; + } // namespace Envoy diff --git a/test/mocks/server/options.h b/test/mocks/server/options.h index fb64f4f1b664..0a0f7433124c 100644 --- a/test/mocks/server/options.h +++ b/test/mocks/server/options.h @@ -34,6 +34,7 @@ class MockOptions : public Options { MOCK_METHOD((const std::vector>&), componentLogLevels, (), (const)); MOCK_METHOD(const std::string&, logFormat, (), (const)); + MOCK_METHOD(bool, logFormatSet, (), (const)); MOCK_METHOD(bool, logFormatEscaped, (), (const)); MOCK_METHOD(bool, enableFineGrainLogging, (), (const)); MOCK_METHOD(const std::string&, logPath, (), (const)); diff --git a/test/server/BUILD b/test/server/BUILD index 74ae0259db08..e2c210dc293e 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -397,5 +397,6 @@ envoy_cc_test( srcs = envoy_select_admin_functionality(["utils_test.cc"]), deps = [ "//source/server:utils_lib", + "//test/mocks/server:options_mocks", ], ) diff --git a/test/server/config_validation/server_test.cc b/test/server/config_validation/server_test.cc index d52e83202b07..cda36802a3dd 100644 --- a/test/server/config_validation/server_test.cc +++ b/test/server/config_validation/server_test.cc @@ -6,6 +6,7 @@ #include "source/server/config_validation/server.h" #include "test/integration/server.h" +#include "test/mocks/common.h" #include "test/mocks/network/mocks.h" #include "test/mocks/server/options.h" #include "test/mocks/stats/mocks.h" @@ -14,6 +15,9 @@ #include "test/test_common/registry.h" #include "test/test_common/test_time.h" +using testing::HasSubstr; +using testing::Return; + namespace Envoy { namespace Server { namespace { @@ -73,14 +77,54 @@ class RuntimeFeatureValidationServerTest : public ValidationServerTest { static const std::vector getAllConfigFiles() { setupTestDirectory(); + return {"runtime_config.yaml"}; + } +}; - auto files = TestUtility::listFiles(ValidationServerTest::directory_, false); +class JsonApplicationLogsValidationServerTest : public ValidationServerTest { +public: + static void SetUpTestSuite() { // NOLINT(readability-identifier-naming) + setupTestDirectory(); + } - // Strip directory part. options_ adds it for each test. - for (auto& file : files) { - file = file.substr(directory_.length() + 1); - } - return files; + static void setupTestDirectory() { + directory_ = + TestEnvironment::runfilesDirectory("envoy/test/server/config_validation/test_data/"); + } + + static const std::vector getAllConfigFiles() { + setupTestDirectory(); + return {"json_application_logs.yaml"}; + } +}; + +class JsonApplicationLogsValidationServerForbiddenFlagvTest : public ValidationServerTest { +public: + static void SetUpTestSuite() { // NOLINT(readability-identifier-naming) + setupTestDirectory(); + } + static void setupTestDirectory() { + directory_ = + TestEnvironment::runfilesDirectory("envoy/test/server/config_validation/test_data/"); + } + static const std::vector getAllConfigFiles() { + setupTestDirectory(); + return {"json_application_logs_forbidden_flagv.yaml"}; + } +}; + +class JsonApplicationLogsValidationServerForbiddenFlag_Test : public ValidationServerTest { +public: + static void SetUpTestSuite() { // NOLINT(readability-identifier-naming) + setupTestDirectory(); + } + static void setupTestDirectory() { + directory_ = + TestEnvironment::runfilesDirectory("envoy/test/server/config_validation/test_data/"); + } + static const std::vector getAllConfigFiles() { + setupTestDirectory(); + return {"json_application_logs_forbidden_flag_.yaml"}; } }; @@ -206,6 +250,80 @@ INSTANTIATE_TEST_SUITE_P( AllConfigs, RuntimeFeatureValidationServerTest, ::testing::ValuesIn(RuntimeFeatureValidationServerTest::getAllConfigFiles())); +TEST_P(JsonApplicationLogsValidationServerTest, BootstrapApplicationLogsAndCLIThrows) { + Thread::MutexBasicLockable access_log_lock; + Stats::IsolatedStoreImpl stats_store; + DangerousDeprecatedTestTime time_system; + EXPECT_CALL(options_, logFormatSet()).WillRepeatedly(Return(true)); + EXPECT_THROW_WITH_MESSAGE( + ValidationInstance server(options_, time_system.timeSystem(), + Network::Address::InstanceConstSharedPtr(), stats_store, + access_log_lock, component_factory_, Thread::threadFactoryForTest(), + Filesystem::fileSystemForTest()), + EnvoyException, + "Only one of ApplicationLogConfig.log_format or CLI option --log-format can be specified."); +} + +TEST_P(JsonApplicationLogsValidationServerTest, JsonApplicationLogs) { + Thread::MutexBasicLockable access_log_lock; + Stats::IsolatedStoreImpl stats_store; + DangerousDeprecatedTestTime time_system; + ValidationInstance server(options_, time_system.timeSystem(), + Network::Address::InstanceConstSharedPtr(), stats_store, + access_log_lock, component_factory_, Thread::threadFactoryForTest(), + Filesystem::fileSystemForTest()); + + Envoy::Logger::Registry::setLogLevel(spdlog::level::info); + MockLogSink sink(Envoy::Logger::Registry::getSink()); + EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto& log) { + EXPECT_THAT(msg, HasSubstr("{\"MessageFromProto\":\"hello\"}")); + EXPECT_EQ(log.logger_name, "misc"); + })); + + ENVOY_LOG_MISC(info, "hello"); + server.shutdown(); +} + +INSTANTIATE_TEST_SUITE_P( + AllConfigs, JsonApplicationLogsValidationServerTest, + ::testing::ValuesIn(JsonApplicationLogsValidationServerTest::getAllConfigFiles())); + +TEST_P(JsonApplicationLogsValidationServerForbiddenFlagvTest, TestForbiddenFlag) { + Thread::MutexBasicLockable access_log_lock; + Stats::IsolatedStoreImpl stats_store; + DangerousDeprecatedTestTime time_system; + EXPECT_THROW_WITH_MESSAGE( + ValidationInstance server(options_, time_system.timeSystem(), + Network::Address::InstanceConstSharedPtr(), stats_store, + access_log_lock, component_factory_, Thread::threadFactoryForTest(), + Filesystem::fileSystemForTest()), + EnvoyException, + "setJsonLogFormat error: INVALID_ARGUMENT: Usage of %v is unavailable for JSON log formats"); +} + +INSTANTIATE_TEST_SUITE_P( + AllConfigs, JsonApplicationLogsValidationServerForbiddenFlagvTest, + ::testing::ValuesIn( + JsonApplicationLogsValidationServerForbiddenFlagvTest::getAllConfigFiles())); + +TEST_P(JsonApplicationLogsValidationServerForbiddenFlag_Test, TestForbiddenFlag) { + Thread::MutexBasicLockable access_log_lock; + Stats::IsolatedStoreImpl stats_store; + DangerousDeprecatedTestTime time_system; + EXPECT_THROW_WITH_MESSAGE( + ValidationInstance server(options_, time_system.timeSystem(), + Network::Address::InstanceConstSharedPtr(), stats_store, + access_log_lock, component_factory_, Thread::threadFactoryForTest(), + Filesystem::fileSystemForTest()), + EnvoyException, + "setJsonLogFormat error: INVALID_ARGUMENT: Usage of %_ is unavailable for JSON log formats"); +} + +INSTANTIATE_TEST_SUITE_P( + AllConfigs, JsonApplicationLogsValidationServerForbiddenFlag_Test, + ::testing::ValuesIn( + JsonApplicationLogsValidationServerForbiddenFlag_Test::getAllConfigFiles())); + } // namespace } // namespace Server } // namespace Envoy diff --git a/test/server/config_validation/test_data/json_application_logs.yaml b/test/server/config_validation/test_data/json_application_logs.yaml new file mode 100644 index 000000000000..2510444d9840 --- /dev/null +++ b/test/server/config_validation/test_data/json_application_logs.yaml @@ -0,0 +1,11 @@ +--- +application_log_config: + log_format: + json_format: + MessageFromProto: "%j" + +admin: + address: + socket_address: + address: 0.0.0.0 + port_value: 9000 diff --git a/test/server/config_validation/test_data/json_application_logs_forbidden_flag_.yaml b/test/server/config_validation/test_data/json_application_logs_forbidden_flag_.yaml new file mode 100644 index 000000000000..3b3b8164adae --- /dev/null +++ b/test/server/config_validation/test_data/json_application_logs_forbidden_flag_.yaml @@ -0,0 +1,11 @@ +--- +application_log_config: + log_format: + json_format: + MessageFromProto: "%_" + +admin: + address: + socket_address: + address: 0.0.0.0 + port_value: 9000 diff --git a/test/server/config_validation/test_data/json_application_logs_forbidden_flagv.yaml b/test/server/config_validation/test_data/json_application_logs_forbidden_flagv.yaml new file mode 100644 index 000000000000..16474cea24cd --- /dev/null +++ b/test/server/config_validation/test_data/json_application_logs_forbidden_flagv.yaml @@ -0,0 +1,11 @@ +--- +application_log_config: + log_format: + json_format: + MessageFromProto: "%v" + +admin: + address: + socket_address: + address: 0.0.0.0 + port_value: 9000 diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index 79136c60e55e..b09db966bcc0 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -113,6 +113,7 @@ TEST_F(OptionsImplTest, All) { EXPECT_EQ(spdlog::level::info, options->logLevel()); EXPECT_EQ(2, options->componentLogLevels().size()); EXPECT_EQ("[%v]", options->logFormat()); + EXPECT_TRUE(options->logFormatSet()); EXPECT_EQ("/foo/bar", options->logPath()); EXPECT_EQ(true, options->enableFineGrainLogging()); EXPECT_EQ("cluster", options->serviceClusterName()); @@ -209,6 +210,7 @@ TEST_F(OptionsImplTest, SetAll) { EXPECT_EQ(Server::DrainStrategy::Immediate, options->drainStrategy()); EXPECT_EQ(spdlog::level::trace, options->logLevel()); EXPECT_EQ("%L %n %v", options->logFormat()); + EXPECT_TRUE(options->logFormatSet()); EXPECT_EQ("/foo/bar", options->logPath()); EXPECT_EQ(std::chrono::seconds(43), options->parentShutdownTime()); EXPECT_EQ(44, options->restartEpoch()); @@ -528,18 +530,21 @@ TEST_F(OptionsImplTest, SetCpusetOnly) { TEST_F(OptionsImplTest, LogFormatDefault) { std::unique_ptr options = createOptionsImpl({"envoy", "-c", "hello"}); EXPECT_EQ(options->logFormat(), "[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"); + EXPECT_FALSE(options->logFormatSet()); } TEST_F(OptionsImplTest, LogFormatOverride) { std::unique_ptr options = createOptionsImpl({"envoy", "-c", "hello", "--log-format", "%%v %v %t %v"}); EXPECT_EQ(options->logFormat(), "%%v %v %t %v"); + EXPECT_TRUE(options->logFormatSet()); } TEST_F(OptionsImplTest, LogFormatOverrideNoPrefix) { std::unique_ptr options = createOptionsImpl({"envoy", "-c", "hello", "--log-format", "%%v %v %t %v"}); EXPECT_EQ(options->logFormat(), "%%v %v %t %v"); + EXPECT_TRUE(options->logFormatSet()); } // Test that --base-id and --restart-epoch with non-default values are accepted. diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 59cbd75f3644..c47df5d57c57 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -20,6 +20,7 @@ #include "test/common/stats/stat_test_utility.h" #include "test/config/v2_link_hacks.h" #include "test/integration/server.h" +#include "test/mocks/common.h" #include "test/mocks/server/bootstrap_extension_factory.h" #include "test/mocks/server/fatal_action_factory.h" #include "test/mocks/server/hot_restart.h" @@ -1624,6 +1625,41 @@ TEST_P(ServerInstanceImplTest, AdminAccessLogFilter) { EXPECT_NO_THROW(initialize("test/server/test_data/server/access_log_filter_bootstrap.yaml")); } +TEST_P(ServerInstanceImplTest, BootstrapApplicationLogsAndCLIThrows) { + EXPECT_CALL(options_, logFormatSet()).WillRepeatedly(Return(true)); + EXPECT_THROW_WITH_MESSAGE( + initialize("test/server/test_data/server/json_application_log.yaml"), EnvoyException, + "Only one of ApplicationLogConfig.log_format or CLI option --log-format can be specified."); +} + +TEST_P(ServerInstanceImplTest, JsonApplicationLog) { + EXPECT_NO_THROW(initialize("test/server/test_data/server/json_application_log.yaml")); + + Envoy::Logger::Registry::setLogLevel(spdlog::level::info); + MockLogSink sink(Envoy::Logger::Registry::getSink()); + EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto& log) { + EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg))); + EXPECT_THAT(msg, HasSubstr("{\"MessageFromProto\":\"hello\"}")); + EXPECT_EQ(log.logger_name, "misc"); + })); + + ENVOY_LOG_MISC(info, "hello"); +} + +TEST_P(ServerInstanceImplTest, JsonApplicationLogFailWithForbiddenFlagv) { + EXPECT_THROW_WITH_MESSAGE( + initialize("test/server/test_data/server/json_application_log_forbidden_flagv.yaml"), + EnvoyException, + "setJsonLogFormat error: INVALID_ARGUMENT: Usage of %v is unavailable for JSON log formats"); +} + +TEST_P(ServerInstanceImplTest, JsonApplicationLogFailWithForbiddenFlag_) { + EXPECT_THROW_WITH_MESSAGE( + initialize("test/server/test_data/server/json_application_log_forbidden_flag_.yaml"), + EnvoyException, + "setJsonLogFormat error: INVALID_ARGUMENT: Usage of %_ is unavailable for JSON log formats"); +} + } // namespace } // namespace Server } // namespace Envoy diff --git a/test/server/test_data/server/json_application_log.yaml b/test/server/test_data/server/json_application_log.yaml new file mode 100644 index 000000000000..2510444d9840 --- /dev/null +++ b/test/server/test_data/server/json_application_log.yaml @@ -0,0 +1,11 @@ +--- +application_log_config: + log_format: + json_format: + MessageFromProto: "%j" + +admin: + address: + socket_address: + address: 0.0.0.0 + port_value: 9000 diff --git a/test/server/test_data/server/json_application_log_forbidden_flag_.yaml b/test/server/test_data/server/json_application_log_forbidden_flag_.yaml new file mode 100644 index 000000000000..3b3b8164adae --- /dev/null +++ b/test/server/test_data/server/json_application_log_forbidden_flag_.yaml @@ -0,0 +1,11 @@ +--- +application_log_config: + log_format: + json_format: + MessageFromProto: "%_" + +admin: + address: + socket_address: + address: 0.0.0.0 + port_value: 9000 diff --git a/test/server/test_data/server/json_application_log_forbidden_flagv.yaml b/test/server/test_data/server/json_application_log_forbidden_flagv.yaml new file mode 100644 index 000000000000..16474cea24cd --- /dev/null +++ b/test/server/test_data/server/json_application_log_forbidden_flagv.yaml @@ -0,0 +1,11 @@ +--- +application_log_config: + log_format: + json_format: + MessageFromProto: "%v" + +admin: + address: + socket_address: + address: 0.0.0.0 + port_value: 9000 diff --git a/test/server/utils_test.cc b/test/server/utils_test.cc index 1e2ae258f5a0..7576e41522c1 100644 --- a/test/server/utils_test.cc +++ b/test/server/utils_test.cc @@ -1,9 +1,12 @@ #include "source/server/utils.h" +#include "test/mocks/server/options.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" +using testing::Return; + namespace Envoy { namespace Server { namespace Utility { @@ -15,6 +18,76 @@ TEST(UtilsTest, BadServerState) { EXPECT_ENVOY_BUG(Utility::serverState(static_cast(123), true), "unexpected server state"); } + +TEST(UtilsTest, AssertExclusiveLogFormatMethod) { + { + testing::NiceMock options; + envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config; + EXPECT_NO_THROW(Utility::assertExclusiveLogFormatMethod(options, log_config)); + } + + { + testing::NiceMock options; + envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config; + EXPECT_CALL(options, logFormatSet()).WillRepeatedly(Return(true)); + EXPECT_NO_THROW(Utility::assertExclusiveLogFormatMethod(options, log_config)); + } + + { + testing::NiceMock options; + envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config; + log_config.mutable_log_format(); + EXPECT_NO_THROW(Utility::assertExclusiveLogFormatMethod(options, log_config)); + } + + { + testing::NiceMock options; + envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config; + EXPECT_CALL(options, logFormatSet()).WillRepeatedly(Return(true)); + log_config.mutable_log_format(); + EXPECT_THROW_WITH_MESSAGE( + Utility::assertExclusiveLogFormatMethod(options, log_config), EnvoyException, + "Only one of ApplicationLogConfig.log_format or CLI option --log-format can be specified."); + } +} + +TEST(UtilsTest, MaybeSetApplicationLogFormat) { + { + envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config; + EXPECT_NO_THROW(Utility::maybeSetApplicationLogFormat(log_config)); + } + + { + envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config; + log_config.mutable_log_format(); + EXPECT_NO_THROW(Utility::maybeSetApplicationLogFormat(log_config)); + } + + { + envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config; + log_config.mutable_log_format()->mutable_json_format(); + EXPECT_NO_THROW(Utility::maybeSetApplicationLogFormat(log_config)); + } + + { + envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config; + auto* format = log_config.mutable_log_format()->mutable_json_format(); + format->mutable_fields()->operator[]("Message").set_string_value("%v"); + EXPECT_THROW_WITH_MESSAGE(Utility::maybeSetApplicationLogFormat(log_config), EnvoyException, + "setJsonLogFormat error: INVALID_ARGUMENT: Usage of %v is " + "unavailable for JSON log formats"); + } + + { + envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config; + auto* format = log_config.mutable_log_format()->mutable_json_format(); + format->mutable_fields()->operator[]("Message").set_string_value("%_"); + EXPECT_THROW_WITH_MESSAGE(Utility::maybeSetApplicationLogFormat(log_config), EnvoyException, + "setJsonLogFormat error: INVALID_ARGUMENT: Usage of %_ is " + "unavailable for JSON log formats"); + } +} + } // namespace Utility } // namespace Server } // namespace Envoy diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index fed59e924743..f9c7f231773c 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -208,6 +208,7 @@ paths: - source/server/config_validation/server.cc - source/server/admin/html/active_stats.js - source/server/server.cc + - source/server/utils.cc - source/server/configuration_impl.h - source/server/hot_restarting_base.cc - source/server/hot_restart_impl.cc From 068adb188b025c48442d440d7942828e258ed467 Mon Sep 17 00:00:00 2001 From: Paul Sohn Date: Mon, 5 Jun 2023 15:24:03 -0400 Subject: [PATCH 461/740] QUIC deferred logging: add retransmission rate (#27699) Commit Message: [QUIC only] Log retransmitted packets and bytes. Additional Description: Adds to the deferred logging implementation from #23648 and implements the existing OnPacketRetransmitted function. Risk Level: Low Testing: Integration test Docs Changes: N/A Release Notes: added Signed-off-by: Paul Sohn --- changelogs/current.yaml | 3 ++ .../observability/access_log/usage.rst | 24 +++++++++++++ envoy/stream_info/stream_info.h | 20 +++++++++++ .../formatter/substitution_formatter.cc | 16 +++++++++ source/common/quic/quic_stats_gatherer.cc | 7 ++++ source/common/quic/quic_stats_gatherer.h | 4 ++- source/common/stream_info/stream_info_impl.h | 16 +++++++++ .../formatter/substitution_formatter_test.cc | 24 +++++++++++++ .../stream_info/stream_info_impl_test.cc | 12 +++++-- test/integration/BUILD | 1 + .../integration/quic_http_integration_test.cc | 36 +++++++++++++++++++ test/mocks/stream_info/mocks.h | 4 +++ tools/spelling/spelling_dictionary.txt | 1 + 13 files changed, 165 insertions(+), 3 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 123aa4ab1f55..71e667638643 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -194,6 +194,9 @@ new_features: - area: access_log change: | added :ref:`CEL ` access log formatter to print CEL expression. +- area: access_log + change: | + (QUIC only) Added support for %BYTES_RETRANSMITTED% and %PACKETS_RETRANSMITTED%. - area: dynamic_forward_proxy change: | added :ref:`sub_clusters_config diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index 7b707de2bf53..344a36aa12dd 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -187,6 +187,30 @@ The following command operators are supported: Renders a numeric value in typed JSON logs. +%BYTES_RETRANSMITTED% + HTTP/3 (QUIC) + Body bytes retransmitted. + + HTTP/1 and HTTP/2 + Not implemented (0). + + TCP/UDP + Not implemented (0). + + Renders a numeric value in typed JSON logs. + +%PACKETS_RETRANSMITTED% + HTTP/3 (QUIC) + Number of packets retransmitted. + + HTTP/1 and HTTP/2 + Not implemented (0). + + TCP/UDP + Not implemented (0). + + Renders a numeric value in typed JSON logs. + %PROTOCOL% HTTP Protocol. Currently either *HTTP/1.1* *HTTP/2* or *HTTP/3*. diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h index 0b31d8aaf2f3..d7176c3dcaec 100644 --- a/envoy/stream_info/stream_info.h +++ b/envoy/stream_info/stream_info.h @@ -580,6 +580,26 @@ class StreamInfo { */ virtual uint64_t bytesReceived() const PURE; + /** + * @param bytes_retransmitted denotes number of bytes to add to total retransmitted bytes. + */ + virtual void addBytesRetransmitted(uint64_t bytes_retransmitted) PURE; + + /** + * @return the number of bytes retransmitted by the stream. + */ + virtual uint64_t bytesRetransmitted() const PURE; + + /** + * @param packets_retransmitted denotes number of packets to add to total retransmitted packets. + */ + virtual void addPacketsRetransmitted(uint64_t packets_retransmitted) PURE; + + /** + * @return the number of packets retransmitted by the stream. + */ + virtual uint64_t packetsRetransmitted() const PURE; + /** * @return the protocol of the request. */ diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index bf97b1df017c..6d2d50cdf922 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -969,6 +969,22 @@ const StreamInfoFormatter::FieldExtractorLookupTbl& StreamInfoFormatter::getKnow return stream_info.bytesReceived(); }); }}}, + {"BYTES_RETRANSMITTED", + {CommandSyntaxChecker::COMMAND_ONLY, + [](const std::string&, const absl::optional&) { + return std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) { + return stream_info.bytesRetransmitted(); + }); + }}}, + {"PACKETS_RETRANSMITTED", + {CommandSyntaxChecker::COMMAND_ONLY, + [](const std::string&, const absl::optional&) { + return std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) { + return stream_info.packetsRetransmitted(); + }); + }}}, {"UPSTREAM_WIRE_BYTES_RECEIVED", {CommandSyntaxChecker::COMMAND_ONLY, [](const std::string&, const absl::optional&) { diff --git a/source/common/quic/quic_stats_gatherer.cc b/source/common/quic/quic_stats_gatherer.cc index e632062c5c70..d7acb9ffc0e6 100644 --- a/source/common/quic/quic_stats_gatherer.cc +++ b/source/common/quic/quic_stats_gatherer.cc @@ -13,6 +13,11 @@ void QuicStatsGatherer::OnPacketAcked(int acked_bytes, } } +void QuicStatsGatherer::OnPacketRetransmitted(int retransmitted_bytes) { + retransmitted_packets_++; + retransmitted_bytes_ += retransmitted_bytes; +} + void QuicStatsGatherer::maybeDoDeferredLog(bool record_ack_timing) { logging_done_ = true; if (stream_info_ == nullptr) { @@ -21,6 +26,8 @@ void QuicStatsGatherer::maybeDoDeferredLog(bool record_ack_timing) { if (time_source_ != nullptr && record_ack_timing) { stream_info_->downstreamTiming().onLastDownstreamAckReceived(*time_source_); } + stream_info_->addBytesRetransmitted(retransmitted_bytes_); + stream_info_->addPacketsRetransmitted(retransmitted_packets_); const Http::RequestHeaderMap* request_headers = request_header_map_.get(); const Http::ResponseHeaderMap* response_headers = response_header_map_.get(); const Http::ResponseTrailerMap* response_trailers = response_trailer_map_.get(); diff --git a/source/common/quic/quic_stats_gatherer.h b/source/common/quic/quic_stats_gatherer.h index 9708ab07f5cc..f704eeebc238 100644 --- a/source/common/quic/quic_stats_gatherer.h +++ b/source/common/quic/quic_stats_gatherer.h @@ -20,7 +20,7 @@ class QuicStatsGatherer : public quic::QuicAckListenerInterface { // QuicAckListenerInterface void OnPacketAcked(int acked_bytes, quic::QuicTime::Delta delta_largest_observed) override; - void OnPacketRetransmitted(int /* retransmitted_bytes */) override {} + void OnPacketRetransmitted(int retransmitted_bytes) override; // Add bytes sent for this stream, for internal tracking of bytes acked. void addBytesSent(uint64_t bytes_sent, bool end_stream) { @@ -58,6 +58,8 @@ class QuicStatsGatherer : public quic::QuicAckListenerInterface { std::unique_ptr stream_info_; Envoy::TimeSource* time_source_ = nullptr; bool logging_done_ = false; + uint64_t retransmitted_packets_ = 0; + uint64_t retransmitted_bytes_ = 0; }; } // namespace Quic diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h index b71bad0a7b1e..af11fcc817d3 100644 --- a/source/common/stream_info/stream_info_impl.h +++ b/source/common/stream_info/stream_info_impl.h @@ -185,6 +185,18 @@ struct StreamInfoImpl : public StreamInfo { uint64_t bytesReceived() const override { return bytes_received_; } + void addBytesRetransmitted(uint64_t bytes_retransmitted) override { + bytes_retransmitted_ += bytes_retransmitted; + } + + uint64_t bytesRetransmitted() const override { return bytes_retransmitted_; } + + void addPacketsRetransmitted(uint64_t packets_retransmitted) override { + packets_retransmitted_ += packets_retransmitted; + } + + uint64_t packetsRetransmitted() const override { return packets_retransmitted_; } + absl::optional protocol() const override { return protocol_; } void protocol(Http::Protocol protocol) override { protocol_ = protocol; } @@ -344,6 +356,8 @@ struct StreamInfoImpl : public StreamInfo { start_time_ = info.startTime(); start_time_monotonic_ = info.startTimeMonotonic(); downstream_transport_failure_reason_ = std::string(info.downstreamTransportFailureReason()); + bytes_retransmitted_ = info.bytesRetransmitted(); + packets_retransmitted_ = info.packetsRetransmitted(); } // This function is used to copy over every field exposed in the StreamInfo interface, with a @@ -435,6 +449,8 @@ struct StreamInfoImpl : public StreamInfo { std::shared_ptr upstream_info_; uint64_t bytes_received_{}; + uint64_t bytes_retransmitted_{}; + uint64_t packets_retransmitted_{}; uint64_t bytes_sent_{}; const Network::ConnectionInfoProviderSharedPtr downstream_connection_info_provider_; const Http::RequestHeaderMap* request_headers_{}; diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index c5211b117cc7..94ce4f716bce 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -402,6 +402,30 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { ProtoEq(ValueUtil::numberValue(25.0))); } + { + StreamInfoFormatter bytes_retransmitted_format("BYTES_RETRANSMITTED"); + EXPECT_CALL(stream_info, bytesRetransmitted()).WillRepeatedly(Return(1)); + EXPECT_EQ("1", bytes_retransmitted_format.format(request_headers, response_headers, + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(bytes_retransmitted_format.formatValue(request_headers, response_headers, + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(1.0))); + } + + { + StreamInfoFormatter packets_retransmitted_format("PACKETS_RETRANSMITTED"); + EXPECT_CALL(stream_info, packetsRetransmitted()).WillRepeatedly(Return(1)); + EXPECT_EQ("1", packets_retransmitted_format.format(request_headers, response_headers, + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet)); + EXPECT_THAT(packets_retransmitted_format.formatValue(request_headers, response_headers, + response_trailers, stream_info, body, + AccessLog::AccessLogType::NotSet), + ProtoEq(ValueUtil::numberValue(1.0))); + } + { StreamInfoFormatter bytes_received_format("BYTES_RECEIVED"); EXPECT_CALL(stream_info, bytesReceived()).WillRepeatedly(Return(1)); diff --git a/test/common/stream_info/stream_info_impl_test.cc b/test/common/stream_info/stream_info_impl_test.cc index 9c6cad4abccd..149e2dc04e29 100644 --- a/test/common/stream_info/stream_info_impl_test.cc +++ b/test/common/stream_info/stream_info_impl_test.cc @@ -37,8 +37,8 @@ std::chrono::nanoseconds checkDuration(std::chrono::nanoseconds last, class StreamInfoImplTest : public testing::Test { protected: void assertStreamInfoSize(StreamInfoImpl stream_info) { - ASSERT_TRUE(sizeof(stream_info) == 824 || sizeof(stream_info) == 840 || - sizeof(stream_info) == 872) + ASSERT_TRUE(sizeof(stream_info) == 840 || sizeof(stream_info) == 856 || + sizeof(stream_info) == 888) << "If adding fields to StreamInfoImpl, please check to see if you " "need to add them to setFromForRecreateStream or setFrom! Current size " << sizeof(stream_info); @@ -245,6 +245,8 @@ TEST_F(StreamInfoImplTest, SetFromForRecreateStream) { s1.addBytesReceived(1); s1.downstreamTiming().onLastDownstreamRxByteReceived(test_time_.timeSystem()); + s1.addBytesRetransmitted(1); + s1.addPacketsRetransmitted(1); #ifdef __clang__ #if defined(__linux__) @@ -264,6 +266,8 @@ TEST_F(StreamInfoImplTest, SetFromForRecreateStream) { EXPECT_EQ(s1.bytesReceived(), s2.bytesReceived()); EXPECT_EQ(s1.getDownstreamBytesMeter(), s2.getDownstreamBytesMeter()); EXPECT_EQ(s1.downstreamTransportFailureReason(), s2.downstreamTransportFailureReason()); + EXPECT_EQ(s1.bytesRetransmitted(), s2.bytesRetransmitted()); + EXPECT_EQ(s1.packetsRetransmitted(), s2.packetsRetransmitted()); } TEST_F(StreamInfoImplTest, SetFrom) { @@ -272,6 +276,8 @@ TEST_F(StreamInfoImplTest, SetFrom) { // setFromForRecreateStream s1.addBytesReceived(1); s1.downstreamTiming().onLastDownstreamRxByteReceived(test_time_.timeSystem()); + s1.addBytesRetransmitted(1); + s1.addPacketsRetransmitted(1); // setFrom s1.setRouteName("foo"); @@ -322,6 +328,8 @@ TEST_F(StreamInfoImplTest, SetFrom) { EXPECT_EQ(s1.bytesReceived(), s2.bytesReceived()); EXPECT_EQ(s1.getDownstreamBytesMeter(), s2.getDownstreamBytesMeter()); EXPECT_EQ(s1.downstreamTransportFailureReason(), s2.downstreamTransportFailureReason()); + EXPECT_EQ(s1.bytesRetransmitted(), s2.bytesRetransmitted()); + EXPECT_EQ(s1.packetsRetransmitted(), s2.packetsRetransmitted()); // Copied by setFrom EXPECT_EQ(s1.getRouteName(), s2.getRouteName()); diff --git a/test/integration/BUILD b/test/integration/BUILD index c918f09fadf1..2f53109884c5 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -2275,6 +2275,7 @@ envoy_cc_test( "//test/common/config:dummy_config_proto_cc_proto", "//test/extensions/transport_sockets/tls/cert_validator:timed_cert_validator", ":http_integration_lib", + ":socket_interface_swap_lib", "//source/common/quic:client_connection_factory_lib", "//source/common/quic:envoy_quic_client_connection_lib", "//source/common/quic:envoy_quic_client_session_lib", diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 97f93cda361e..64dc3b724b47 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -28,6 +28,7 @@ #include "test/config/utility.h" #include "test/extensions/transport_sockets/tls/cert_validator/timed_cert_validator.h" #include "test/integration/http_integration.h" +#include "test/integration/socket_interface_swap.h" #include "test/integration/ssl_utility.h" #include "test/test_common/registry.h" #include "test/test_common/test_runtime.h" @@ -1380,6 +1381,41 @@ TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithInternalRedirect) { EXPECT_EQ(/* RESP(test-header) */ metrics.at(21), "-"); } +TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithRetransmission) { + useAccessLog("%BYTES_RETRANSMITTED%,%PACKETS_RETRANSMITTED%"); + initialize(); + + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + IntegrationStreamDecoderPtr response = + codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(0, TestUtility::DefaultTimeout); + + // Temporarily prevent server from writing packets (i.e. to respond to downstream) + // to simulate packet loss and trigger retransmissions. + SocketInterfaceSwap socket_swap(upstreamProtocol() == Http::CodecType::HTTP3 + ? Network::Socket::Type::Datagram + : Network::Socket::Type::Stream); + Network::IoSocketError* ebadf = Network::IoSocketError::getIoSocketEbadfInstance(); + socket_swap.write_matcher_->setDestinationPort(lookupPort("http")); + socket_swap.write_matcher_->setWriteOverride(ebadf); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + absl::SleepFor(absl::Milliseconds(500 * TSAN_TIMEOUT_FACTOR)); + // Allow the response to be sent downstream again. + socket_swap.write_matcher_->setWriteOverride(nullptr); + + ASSERT_TRUE(response->waitForEndStream()); + codec_client_->close(); + ASSERT_TRUE(response->complete()); + + // Confirm that retransmissions are logged. + std::string log = waitForAccessLog(access_log_name_); + std::vector metrics = absl::StrSplit(log, ','); + ASSERT_EQ(metrics.size(), 2); + EXPECT_GT(/* BYTES_RETRANSMITTED */ std::stoi(metrics.at(0)), 0); + EXPECT_GT(/* PACKETS_RETRANSMITTED */ std::stoi(metrics.at(1)), 0); + EXPECT_GE(std::stoi(metrics.at(0)), std::stoi(metrics.at(1))); +} + TEST_P(QuicHttpIntegrationTest, InvalidTrailer) { initialize(); // Empty string in trailer key is invalid. diff --git a/test/mocks/stream_info/mocks.h b/test/mocks/stream_info/mocks.h index f965a739b9f6..7ad098443b98 100644 --- a/test/mocks/stream_info/mocks.h +++ b/test/mocks/stream_info/mocks.h @@ -85,6 +85,10 @@ class MockStreamInfo : public StreamInfo { MOCK_METHOD(OptRef, downstreamTiming, (), (const)); MOCK_METHOD(void, addBytesReceived, (uint64_t)); MOCK_METHOD(uint64_t, bytesReceived, (), (const)); + MOCK_METHOD(void, addBytesRetransmitted, (uint64_t)); + MOCK_METHOD(uint64_t, bytesRetransmitted, (), (const)); + MOCK_METHOD(void, addPacketsRetransmitted, (uint64_t)); + MOCK_METHOD(uint64_t, packetsRetransmitted, (), (const)); MOCK_METHOD(void, addWireBytesReceived, (uint64_t)); MOCK_METHOD(uint64_t, wireBytesReceived, (), (const)); MOCK_METHOD(void, setRouteName, (absl::string_view route_name)); diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index ef68770145af..dd3785342239 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -1160,6 +1160,7 @@ responder restarter resync ret +retransmissions retransmitted retransmitting retriable From 92e7b1c8a2373ce1193398e9105c14813abfecc6 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 5 Jun 2023 16:51:55 -0400 Subject: [PATCH 462/740] mobile: better filter docs (#27810) Signed-off-by: Alyssa Wilk --- mobile/docs/root/api/starting_envoy.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mobile/docs/root/api/starting_envoy.rst b/mobile/docs/root/api/starting_envoy.rst index f40d8244122e..67b68a5a101a 100644 --- a/mobile/docs/root/api/starting_envoy.rst +++ b/mobile/docs/root/api/starting_envoy.rst @@ -226,23 +226,26 @@ This information is sent as metadata when flushing stats. builder.addAppId("com.mydomain.myapp) ~~~~~~~~~~~~~~~~~~~~~~~~~ -``enableAdminInterface`` +``addNativeFilter`` ~~~~~~~~~~~~~~~~~~~~~~~~~ -Enable admin interface on 127.0.0.1:9901 address. +Add a C++ filter to the Envoy Mobile filter chain .. attention:: - Admin interface is intended to be used for development/debugging purposes only. - Enabling it in production may open your app to security vulnerabilities. + This will only work if the C++ filter specified is linked into your Envoy Mobile build. + For C++ and Android testing, calling addNativeFilter and linking the Envoy library by adding the + library to ``envoy_build_config/extensions_build_config.bzl`` is sufficient. + For iOS, due to enthusiastic garbage collection, and for upstream CI, to catch bugs, you will + also need to forceRegister the filter in ``envoy_build_config/extension_registry.cc`` **Example**:: // Kotlin - builder.enableAdminInterface() + builder.addNativeFilter("envoy.filters.http.buffer", "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}") // Swift - builder.enableAdminInterface() + builder.addNativeFilter("envoy.filters.http.buffer", "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}") ~~~~~~~~~~~~~~~~~~~~~~ ``setOnEngineRunning`` From 3b7328f253c9cdd82dc81b736a85aeaa622262dd Mon Sep 17 00:00:00 2001 From: ohadvano <49730675+ohadvano@users.noreply.github.com> Date: Tue, 6 Jun 2023 09:56:14 +0300 Subject: [PATCH 463/740] Fix post-submit clang tidy (#27815) Signed-off-by: ohadvano <49730675+ohadvano@users.noreply.github.com> --- test/server/config_validation/server_test.cc | 8 ++++---- test/server/server_test.cc | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/server/config_validation/server_test.cc b/test/server/config_validation/server_test.cc index cda36802a3dd..f03ec5f21719 100644 --- a/test/server/config_validation/server_test.cc +++ b/test/server/config_validation/server_test.cc @@ -113,7 +113,7 @@ class JsonApplicationLogsValidationServerForbiddenFlagvTest : public ValidationS } }; -class JsonApplicationLogsValidationServerForbiddenFlag_Test : public ValidationServerTest { +class JsonApplicationLogsValidationServerForbiddenFlagUnderscoreTest : public ValidationServerTest { public: static void SetUpTestSuite() { // NOLINT(readability-identifier-naming) setupTestDirectory(); @@ -306,7 +306,7 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn( JsonApplicationLogsValidationServerForbiddenFlagvTest::getAllConfigFiles())); -TEST_P(JsonApplicationLogsValidationServerForbiddenFlag_Test, TestForbiddenFlag) { +TEST_P(JsonApplicationLogsValidationServerForbiddenFlagUnderscoreTest, TestForbiddenFlag) { Thread::MutexBasicLockable access_log_lock; Stats::IsolatedStoreImpl stats_store; DangerousDeprecatedTestTime time_system; @@ -320,9 +320,9 @@ TEST_P(JsonApplicationLogsValidationServerForbiddenFlag_Test, TestForbiddenFlag) } INSTANTIATE_TEST_SUITE_P( - AllConfigs, JsonApplicationLogsValidationServerForbiddenFlag_Test, + AllConfigs, JsonApplicationLogsValidationServerForbiddenFlagUnderscoreTest, ::testing::ValuesIn( - JsonApplicationLogsValidationServerForbiddenFlag_Test::getAllConfigFiles())); + JsonApplicationLogsValidationServerForbiddenFlagUnderscoreTest::getAllConfigFiles())); } // namespace } // namespace Server diff --git a/test/server/server_test.cc b/test/server/server_test.cc index c47df5d57c57..abbc3690e0f5 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -1653,7 +1653,7 @@ TEST_P(ServerInstanceImplTest, JsonApplicationLogFailWithForbiddenFlagv) { "setJsonLogFormat error: INVALID_ARGUMENT: Usage of %v is unavailable for JSON log formats"); } -TEST_P(ServerInstanceImplTest, JsonApplicationLogFailWithForbiddenFlag_) { +TEST_P(ServerInstanceImplTest, JsonApplicationLogFailWithForbiddenFlagUnderscore) { EXPECT_THROW_WITH_MESSAGE( initialize("test/server/test_data/server/json_application_log_forbidden_flag_.yaml"), EnvoyException, From bbcc855c1bb1a9882ada62859de97b618abf0ea5 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 6 Jun 2023 09:24:35 -0400 Subject: [PATCH 464/740] mobile: running compile time options build on all Envoy PRs (#27707) the no-YAML build is easy to break and we've move this off the constrained machines. Signed-off-by: Alyssa Wilk --- .github/workflows/mobile-compile_time_options.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index c6c246432436..3c9693112247 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -17,7 +17,6 @@ jobs: secrets: inherit cc_test: - if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} needs: env name: cc_test runs-on: ubuntu-20.04 From 1c25c51275ad99fe411a6ea73bc634fcedcd0289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20B=C3=A9ky?= Date: Tue, 6 Jun 2023 10:29:16 -0400 Subject: [PATCH 465/740] [balsa] Disallow extended ASCII in header names. (#27738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [balsa] Disallow extended ASCII in header names. This is for parity with http-parser. Note that BalsaParser already disallows control characters, space, and certain separator characters [1, 2], but not characters above 127. [1] https://github.com/google/quiche/blob/ef9e527e14440794cc18813ef079ebdb3650f8ad/quiche/balsa/header_properties.h#L21-L30 [2] https://github.com/google/quiche/blob/ef9e527e14440794cc18813ef079ebdb3650f8ad/quiche/balsa/balsa_frame.cc#L414 Signed-off-by: Bence Béky --- source/common/http/http1/balsa_parser.cc | 36 ++++-- test/common/http/http1/codec_impl_test.cc | 149 ++++++++++++++++++++++ tools/spelling/spelling_dictionary.txt | 1 + 3 files changed, 172 insertions(+), 14 deletions(-) diff --git a/source/common/http/http1/balsa_parser.cc b/source/common/http/http1/balsa_parser.cc index 8826994d7bc3..7a354a8afe8c 100644 --- a/source/common/http/http1/balsa_parser.cc +++ b/source/common/http/http1/balsa_parser.cc @@ -23,6 +23,14 @@ constexpr absl::string_view kColonSlashSlash = "://"; // Response must start with "HTTP". constexpr char kResponseFirstByte = 'H'; +// Allowed characters for field names according to Section 5.1 +// and for methods according to Section 9.1 of RFC 9110: +// https://www.rfc-editor.org/rfc/rfc9110.html +constexpr absl::string_view kValidCharacters = + "!#$%&'*+-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ^_`abcdefghijklmnopqrstuvwxyz|~"; +constexpr absl::string_view::iterator kValidCharactersBegin = kValidCharacters.begin(); +constexpr absl::string_view::iterator kValidCharactersEnd = kValidCharacters.end(); + bool isFirstCharacterOfValidMethod(char c) { static constexpr char kValidFirstCharacters[] = {'A', 'B', 'C', 'D', 'G', 'H', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U'}; @@ -36,20 +44,9 @@ bool isFirstCharacterOfValidMethod(char c) { // enabled. bool isMethodValid(absl::string_view method, bool allow_custom_methods) { if (allow_custom_methods) { - // Allowed characters in method according to RFC 9110, - // https://www.rfc-editor.org/rfc/rfc9110.html#section-5.1. - static constexpr char kValidCharacters[] = { - '!', '#', '$', '%', '&', '\'', '*', '+', '-', '.', '0', '1', '2', '3', '4', '5', - '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', - 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '^', '_', - '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '|', '~'}; - const auto* begin = &kValidCharacters[0]; - const auto* end = &kValidCharacters[ABSL_ARRAYSIZE(kValidCharacters) - 1] + 1; - return !method.empty() && - std::all_of(method.begin(), method.end(), [begin, end](absl::string_view::value_type c) { - return std::binary_search(begin, end, c); + std::all_of(method.begin(), method.end(), [](absl::string_view::value_type c) { + return std::binary_search(kValidCharactersBegin, kValidCharactersEnd, c); }); } @@ -138,6 +135,12 @@ bool isVersionValid(absl::string_view version_input) { return regex->match(version_input); } +bool isHeaderNameValid(absl::string_view name) { + return std::all_of(name.begin(), name.end(), [](absl::string_view::value_type c) { + return std::binary_search(kValidCharactersBegin, kValidCharactersEnd, c); + }); +} + } // anonymous namespace BalsaParser::BalsaParser(MessageType type, ParserCallbacks* connection, size_t max_header_length, @@ -394,8 +397,13 @@ void BalsaParser::processHeadersOrTrailersImpl(const quiche::BalsaHeaders& heade } absl::string_view key = key_value.first; - status_ = convertResult(connection_->onHeaderField(key.data(), key.length())); + if (!isHeaderNameValid(key)) { + status_ = ParserStatus::Error; + error_message_ = "HPE_INVALID_HEADER_TOKEN"; + return; + } + status_ = convertResult(connection_->onHeaderField(key.data(), key.length())); if (status_ == ParserStatus::Error) { return; } diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 54bfa2561372..c269f4513533 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -4599,5 +4599,154 @@ TEST_P(Http1ServerConnectionImplTest, Http10Rejected) { EXPECT_THAT(status.message(), StartsWith("Upgrade required")); } +TEST_P(Http1ClientConnectionImplTest, SeparatorInHeaderName) { + initialize(); + + NiceMock response_decoder; + Http::RequestEncoder& request_encoder = codec_->newStream(response_decoder); + TestRequestHeaderMapImpl headers{{":method", "GET"}, {":path", "/"}, {":authority", "host"}}; + EXPECT_TRUE(request_encoder.encodeHeaders(headers, true).ok()); + + Buffer::OwnedImpl response("HTTP/1.1 200 OK\r\n" + "fo[o: bar\r\n" + "\r\n"); + auto status = codec_->dispatch(response); + + EXPECT_FALSE(status.ok()); + if (parser_impl_ == Http1ParserImpl::BalsaParser) { + EXPECT_EQ(status.message(), "http/1.1 protocol error: INVALID_HEADER_NAME_CHARACTER"); + } else { + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_HEADER_TOKEN"); + } +} + +TEST_P(Http1ServerConnectionImplTest, SeparatorInHeaderName) { + initialize(); + + StrictMock decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + EXPECT_CALL(decoder, + sendLocalReply(Http::Code::BadRequest, "Bad Request", _, _, "http1.codec_error")); + + Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\n" + "fo[o: bar\r\n" + "\r\n"); + auto status = codec_->dispatch(buffer); + + EXPECT_FALSE(status.ok()); + if (parser_impl_ == Http1ParserImpl::BalsaParser) { + EXPECT_EQ(status.message(), "http/1.1 protocol error: INVALID_HEADER_NAME_CHARACTER"); + } else { + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_HEADER_TOKEN"); + } +} + +// BalsaParser always rejects a header name with space. HttpParser only rejects +// it in strict mode, which is disabled when ENVOY_ENABLE_UHV is defined. +TEST_P(Http1ClientConnectionImplTest, SpaceInHeaderName) { + bool accept = parser_impl_ == Http1ParserImpl::HttpParser; +#ifndef ENVOY_ENABLE_UHV + accept = false; +#endif + + initialize(); + + NiceMock response_decoder; + Http::RequestEncoder& request_encoder = codec_->newStream(response_decoder); + TestRequestHeaderMapImpl headers{{":method", "GET"}, {":path", "/"}, {":authority", "host"}}; + EXPECT_TRUE(request_encoder.encodeHeaders(headers, true).ok()); + + if (accept) { + EXPECT_CALL(response_decoder, decodeHeaders_(_, false)); + } + + Buffer::OwnedImpl response("HTTP/1.1 200 OK\r\n" + "fo o: bar\r\n" + "\r\n"); + auto status = codec_->dispatch(response); + + if (accept) { + EXPECT_TRUE(status.ok()); + } else { + EXPECT_FALSE(status.ok()); + if (parser_impl_ == Http1ParserImpl::BalsaParser) { + EXPECT_EQ(status.message(), "http/1.1 protocol error: INVALID_HEADER_NAME_CHARACTER"); + } else { + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_HEADER_TOKEN"); + } + } +} + +TEST_P(Http1ServerConnectionImplTest, SpaceInHeaderName) { + bool accept = parser_impl_ == Http1ParserImpl::HttpParser; +#ifndef ENVOY_ENABLE_UHV + accept = false; +#endif + + initialize(); + + StrictMock decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + + if (accept) { + EXPECT_CALL(decoder, decodeHeaders_(_, true)); + } else { + EXPECT_CALL(decoder, + sendLocalReply(Http::Code::BadRequest, "Bad Request", _, _, "http1.codec_error")); + } + + Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\n" + "fo o: bar\r\n" + "\r\n"); + auto status = codec_->dispatch(buffer); + + if (accept) { + EXPECT_TRUE(status.ok()); + } else { + EXPECT_FALSE(status.ok()); + if (parser_impl_ == Http1ParserImpl::BalsaParser) { + EXPECT_EQ(status.message(), "http/1.1 protocol error: INVALID_HEADER_NAME_CHARACTER"); + } else { + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_HEADER_TOKEN"); + } + } +} + +TEST_P(Http1ClientConnectionImplTest, ExtendedAsciiInHeaderName) { + initialize(); + + NiceMock response_decoder; + Http::RequestEncoder& request_encoder = codec_->newStream(response_decoder); + TestRequestHeaderMapImpl headers{{":method", "GET"}, {":path", "/"}, {":authority", "host"}}; + EXPECT_TRUE(request_encoder.encodeHeaders(headers, true).ok()); + + // SPELLCHECKER(off) + Buffer::OwnedImpl response("HTTP/1.1 200 OK\r\n" + "föö: bar\r\n" + "\r\n"); + // SPELLCHECKER(on) + auto status = codec_->dispatch(response); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_HEADER_TOKEN"); +} + +TEST_P(Http1ServerConnectionImplTest, ExtendedAsciiInHeaderName) { + initialize(); + + StrictMock decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + EXPECT_CALL(decoder, + sendLocalReply(Http::Code::BadRequest, "Bad Request", _, _, "http1.codec_error")); + + // SPELLCHECKER(off) + Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\n" + "föö: bar\r\n" + "\r\n"); + // SPELLCHECKER(on) + auto status = codec_->dispatch(buffer); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_HEADER_TOKEN"); +} + } // namespace Http } // namespace Envoy diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index dd3785342239..429f6a848cd3 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -815,6 +815,7 @@ idx ifdef iff ified +ifndef impl implementors impls From 845be282e9968884dd2beef386a1946fac54df56 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Tue, 6 Jun 2023 12:09:10 -0400 Subject: [PATCH 466/740] cel_matcher: match on xds attributes. (#27790) With PR #25818, xds attributes (like upstream cluster metadata) are provided via streamInfo and are available to CEL library. This PR enables them and tests upstream cluster metadata specifically. --- .../matching/http/cel_input/cel_input.h | 9 +-- .../cel_matcher/cel_matcher_test.cc | 42 +++++++++++-- .../cel_matcher/cel_matcher_test.h | 60 +++++++++++++++++++ test/mocks/stream_info/mocks.cc | 5 ++ test/mocks/stream_info/mocks.h | 1 + 5 files changed, 105 insertions(+), 12 deletions(-) diff --git a/source/extensions/matching/http/cel_input/cel_input.h b/source/extensions/matching/http/cel_input/cel_input.h index 0e5707c4e2e4..324d2fd264cd 100644 --- a/source/extensions/matching/http/cel_input/cel_input.h +++ b/source/extensions/matching/http/cel_input/cel_input.h @@ -39,13 +39,8 @@ class HttpCelDataInput : public Matcher::DataInput activation = Extensions::Filters::Common::Expr::createActivation( data.streamInfo(), maybe_request_headers.ptr(), maybe_response_headers.ptr(), diff --git a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc index f1d3378a800d..0a46dfdc1133 100644 --- a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc +++ b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc @@ -40,10 +40,15 @@ enum class ExpressionType { NoExpression = 2, }; +constexpr absl::string_view kFilterNamespace = "cel_matcher"; +constexpr absl::string_view kMetadataKey = "service_name"; +constexpr absl::string_view kMetadataValue = "test_service"; + class CelMatcherTest : public ::testing::Test { public: CelMatcherTest() : inject_action_(action_factory_), + cluster_info_(std::make_shared>()), data_(Envoy::Http::Matching::HttpMatchingDataImpl(stream_info_)) {} void buildCustomHeader(const absl::flat_hash_map& custom_value_pairs, @@ -112,11 +117,21 @@ class CelMatcherTest : public ::testing::Test { return match_tree(); } + void setUpstreamClusterMetadata(const std::string& namespace_str, const std::string& metadata_key, + const std::string& metadata_value) { + Envoy::Config::Metadata::mutableMetadataValue(metadata_, namespace_str, metadata_key) + .set_string_value(metadata_value); + EXPECT_CALL(*cluster_info_, metadata()).WillRepeatedly(testing::ReturnPointee(&metadata_)); + stream_info_.setUpstreamClusterInfo(cluster_info_); + } + Matcher::StringActionFactory action_factory_; Registry::InjectFactory> inject_action_; + std::shared_ptr> cluster_info_; testing::NiceMock stream_info_; absl::string_view context_ = ""; testing::NiceMock factory_context_; + envoy::config::core::v3::Metadata metadata_; TestRequestHeaderMapImpl default_headers_{ {":method", "GET"}, {":scheme", "http"}, {":authority", "host"}}; @@ -130,7 +145,6 @@ TEST_F(CelMatcherTest, CelMatcherRequestHeaderMatched) { TestRequestHeaderMapImpl request_headers = default_headers_; buildCustomHeader({{"authenticated_user", "staging"}}, request_headers); data_.onRequestHeaders(request_headers); - // data.onRequestHeaders(request_headers); const auto result = matcher_tree->match(data_); // The match was complete, match found. @@ -164,13 +178,31 @@ TEST_F(CelMatcherTest, CelMatcherRequestHeaderNotMatched) { EXPECT_EQ(result_2.on_match_, absl::nullopt); } -TEST_F(CelMatcherTest, CelMatcherNoRequestAttributes) { - auto matcher_tree = buildMatcherTree(RequestHeaderCelExprString); +TEST_F(CelMatcherTest, CelMatcherClusterMetadataMatched) { + setUpstreamClusterMetadata(std::string(kFilterNamespace), std::string(kMetadataKey), + std::string(kMetadataValue)); + Envoy::Http::Matching::HttpMatchingDataImpl data = + Envoy::Http::Matching::HttpMatchingDataImpl(stream_info_); + auto matcher_tree = buildMatcherTree(absl::StrFormat( + UpstreamClusterMetadataCelString, kFilterNamespace, kMetadataKey, kMetadataValue)); + const auto result = matcher_tree->match(data_); + // The match was complete, match found. + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); + EXPECT_TRUE(result.on_match_.has_value()); + EXPECT_NE(result.on_match_->action_cb_, nullptr); +} + +TEST_F(CelMatcherTest, CelMatcherClusterMetadataNotMatched) { + setUpstreamClusterMetadata(std::string(kFilterNamespace), std::string(kMetadataKey), + "wrong_service"); + Envoy::Http::Matching::HttpMatchingDataImpl data = + Envoy::Http::Matching::HttpMatchingDataImpl(stream_info_); + auto matcher_tree = buildMatcherTree(absl::StrFormat( + UpstreamClusterMetadataCelString, kFilterNamespace, kMetadataKey, kMetadataValue)); - // No request attributes added to matching data. const auto result = matcher_tree->match(data_); // The match was completed, no match found. - EXPECT_EQ(result.match_state_, Matcher::MatchState::UnableToMatch); + EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete); EXPECT_EQ(result.on_match_, absl::nullopt); } diff --git a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h index efadd414fe34..65d6b0626f13 100644 --- a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h +++ b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h @@ -1,5 +1,7 @@ #pragma once +#include "absl/strings/string_view.h" + namespace Envoy { namespace Extensions { namespace Matching { @@ -547,6 +549,64 @@ inline constexpr char RequestAndResponseCelString[] = R"pb( } )pb"; +// xds.cluster_metadata.filter_metadata['cel_matcher']['service_name'] == 'test_service' +inline constexpr absl::string_view UpstreamClusterMetadataCelString = R"pb( + expr { + id: 8 + call_expr { + function: "_==_" + args { + id: 6 + call_expr { + function: "_[_]" + args { + id: 4 + call_expr { + function: "_[_]" + args { + id: 3 + select_expr { + operand { + id: 2 + select_expr { + operand { + id: 1 + ident_expr { + name: "xds" + } + } + field: "cluster_metadata" + } + } + field: "filter_metadata" + } + } + args { + id: 5 + const_expr { + string_value: "%s" + } + } + } + } + args { + id: 7 + const_expr { + string_value: "%s" + } + } + } + } + args { + id: 9 + const_expr { + string_value: "%s" + } + } + } + } +)pb"; + } // namespace CelMatcher } // namespace InputMatchers } // namespace Matching diff --git a/test/mocks/stream_info/mocks.cc b/test/mocks/stream_info/mocks.cc index d62ebea53694..141512314d4b 100644 --- a/test/mocks/stream_info/mocks.cc +++ b/test/mocks/stream_info/mocks.cc @@ -190,6 +190,11 @@ MockStreamInfo::MockStreamInfo() })); ON_CALL(*this, downstreamTransportFailureReason()) .WillByDefault(ReturnPointee(&downstream_transport_failure_reason_)); + ON_CALL(*this, setUpstreamClusterInfo(_)) + .WillByDefault(Invoke([this](const Upstream::ClusterInfoConstSharedPtr& cluster_info) { + upstream_cluster_info_ = std::move(cluster_info); + })); + ON_CALL(*this, upstreamClusterInfo()).WillByDefault(ReturnPointee(&upstream_cluster_info_)); } MockStreamInfo::~MockStreamInfo() = default; diff --git a/test/mocks/stream_info/mocks.h b/test/mocks/stream_info/mocks.h index 7ad098443b98..469169332a5b 100644 --- a/test/mocks/stream_info/mocks.h +++ b/test/mocks/stream_info/mocks.h @@ -150,6 +150,7 @@ class MockStreamInfo : public StreamInfo { absl::optional response_code_; absl::optional response_code_details_; absl::optional connection_termination_details_; + absl::optional upstream_cluster_info_; std::shared_ptr upstream_info_; uint64_t response_flags_{}; envoy::config::core::v3::Metadata metadata_; From 64e334de55386060e3124a0e9471e5b60804a4c0 Mon Sep 17 00:00:00 2001 From: ericorth <57926619+ericorth@users.noreply.github.com> Date: Tue, 6 Jun 2023 12:23:45 -0400 Subject: [PATCH 467/740] QUIC: remove all usage of quic::kRVCM (#27774) Commit Message: remove all usage of quic::kRVCM Additional Description: This QUIC connection option no longer does anything and is no longer needed after some recent (~1 month ago) changes in QUICHE. Now, QUICHE is removing the definition of quic::kRVCM. Removing all usage from Envoy to unblock QUICHE rolls to Envoy. Risk Level: Low Testing: Covered by existing unit tests Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a Signed-off-by: Eric Orth --- source/common/quic/client_connection_factory_impl.cc | 7 ------- test/common/quic/client_connection_factory_impl_test.cc | 3 --- test/integration/quic_http_integration_test.cc | 4 ++-- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/source/common/quic/client_connection_factory_impl.cc b/source/common/quic/client_connection_factory_impl.cc index 67f4fc103731..269dbbdf078f 100644 --- a/source/common/quic/client_connection_factory_impl.cc +++ b/source/common/quic/client_connection_factory_impl.cc @@ -23,13 +23,6 @@ createPersistentQuicInfoForCluster(Event::Dispatcher& dispatcher, quic_info->quic_config_.max_idle_time_before_crypto_handshake()) { quic_info->quic_config_.set_max_idle_time_before_crypto_handshake(crypto_timeout); } - // Default enable RVCM connection option so that port migration is enabled. - quic::QuicTagVector connection_options; - if (quic_info->quic_config_.HasSendConnectionOptions()) { - connection_options = quic_info->quic_config_.SendConnectionOptions(); - } - connection_options.push_back(quic::kRVCM); - quic_info->quic_config_.SetConnectionOptionsToSend(connection_options); return quic_info; } diff --git a/test/common/quic/client_connection_factory_impl_test.cc b/test/common/quic/client_connection_factory_impl_test.cc index 9a27e6cd33f0..f93fcce3482f 100644 --- a/test/common/quic/client_connection_factory_impl_test.cc +++ b/test/common/quic/client_connection_factory_impl_test.cc @@ -41,9 +41,6 @@ class QuicNetworkConnectionTest : public Event::TestUsingSimulatedTime, protocol_options->max_concurrent_streams().value()); EXPECT_EQ(quic_info_->quic_config_.GetInitialMaxStreamDataBytesIncomingBidirectionalToSend(), protocol_options->initial_stream_window_size().value()); - ASSERT_TRUE(quic_info_->quic_config_.HasSendConnectionOptions()); - EXPECT_TRUE( - quic::ContainsQuicTag(quic_info_->quic_config_.SendConnectionOptions(), quic::kRVCM)); test_address_ = Network::Utility::resolveUrl( absl::StrCat("tcp://", Network::Test::getLoopbackAddressUrlString(GetParam()), ":30")); diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 64dc3b724b47..c11b91987849 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -1704,7 +1704,7 @@ TEST_P(QuicHttpIntegrationTest, UsesPreferredAddress) { }); initialize(); - quic::QuicTagVector connection_options{quic::kRVCM, quic::kSPAD}; + quic::QuicTagVector connection_options{quic::kSPAD}; dynamic_cast(*quic_connection_persistent_info_) .quic_config_.SetConnectionOptionsToSend(connection_options); codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); @@ -1767,7 +1767,7 @@ TEST_P(QuicHttpIntegrationTest, UsesPreferredAddressDualStack) { }); initialize(); - quic::QuicTagVector connection_options{quic::kRVCM, quic::kSPAD}; + quic::QuicTagVector connection_options{quic::kSPAD}; dynamic_cast(*quic_connection_persistent_info_) .quic_config_.SetConnectionOptionsToSend(connection_options); codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); From e7bb2feaf8299239ce9f3cc591886965149bb263 Mon Sep 17 00:00:00 2001 From: DiazAlan <109677874+DiazAlan@users.noreply.github.com> Date: Tue, 6 Jun 2023 15:48:28 -0400 Subject: [PATCH 468/740] Add hidden functionality to stats_request (#27643) Commit Message: Implement hidden functionality of gauges to stats_request Additional Description: Risk Level: Low Testing: Added three tests to stats_requests_test Signed-off-by: AlanDiaz --- changelogs/current.yaml | 4 ++ docs/root/operations/admin.rst | 12 ++++ envoy/stats/sink.h | 2 +- envoy/stats/stats.h | 7 +-- source/common/stats/allocator_impl.cc | 6 +- source/server/admin/prometheus_stats.cc | 15 ++++- source/server/admin/stats_params.cc | 14 +++++ source/server/admin/stats_params.h | 7 +++ source/server/admin/stats_request.cc | 8 +++ test/common/stats/allocator_impl_test.cc | 59 +++++++++++++++++++ .../open_telemetry_impl_test.cc | 8 +-- test/server/admin/prometheus_stats_test.cc | 59 +++++++++++++++++-- test/server/admin/stats_handler_test.cc | 56 ++++++++++++++++++ test/server/admin/stats_params_test.cc | 17 ++++++ test/server/admin/stats_request_test.cc | 30 ++++++++++ 15 files changed, 287 insertions(+), 17 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 71e667638643..00ce45b7b76b 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -90,6 +90,10 @@ minor_behavior_changes: change: | Filter metadata containing ext proc stats has been moved from ext-proc-logging-info to a namespace corresponding to the name of the ext_proc filter. +- area: stats + change: | + Added new type of gauge with type hidden. These stats are hidden from admin/stats-sinks but can shown with a + query-parameter of ``/stats?hidden=include`` or ``/stats?hidden=showonly``. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index d0957b906108..bcc938406ce6 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -450,6 +450,18 @@ modify different aspects of the server: Outputs statistics that Envoy has updated (counters incremented at least once, gauges changed at least once, and histograms added to at least once). + .. http::get:: /stats?hidden=showonly + + Only outputs statistics that are internally marked as hidden. + + .. http::get:: /stats?hidden=include + + Hidden stats will be shown along side non-hidden stats. + + .. http::get:: /stats?hidden=exclude + + Hidden stats will be excluded from the output. This is the default behavior. + .. http:get:: /stats?filter=regex Filters the returned stats to those with names matching the regular diff --git a/envoy/stats/sink.h b/envoy/stats/sink.h index e833c44c554c..b83a06241b65 100644 --- a/envoy/stats/sink.h +++ b/envoy/stats/sink.h @@ -61,7 +61,7 @@ class SinkPredicates { virtual bool includeCounter(const Counter& counter) PURE; /** - * @return true if @param gague needs to be flushed to sinks. + * @return true if @param gauge needs to be flushed to sinks. */ virtual bool includeGauge(const Gauge& gauge) PURE; diff --git a/envoy/stats/stats.h b/envoy/stats/stats.h index cdc1ba896c9b..b2768144c60f 100644 --- a/envoy/stats/stats.h +++ b/envoy/stats/stats.h @@ -137,10 +137,9 @@ class Gauge : public Metric { public: // TODO(diazalan): Rename ImportMode to more generic name enum class ImportMode { - Uninitialized, // Gauge was discovered during hot-restart transfer. - NeverImport, // On hot-restart, each process starts with gauge at 0. - Accumulate, // Transfers gauge state on hot-restart. - // TODO(Diazalan): Add functionality for Hidden to be ignored by admin/stats-sink + Uninitialized, // Gauge was discovered during hot-restart transfer. + NeverImport, // On hot-restart, each process starts with gauge at 0. + Accumulate, // Transfers gauge state on hot-restart. HiddenAccumulate, // Will be transferred on hot-restart and ignored by admin/stats-sink }; diff --git a/source/common/stats/allocator_impl.cc b/source/common/stats/allocator_impl.cc index a8c9745461c2..c8c6905f4f48 100644 --- a/source/common/stats/allocator_impl.cc +++ b/source/common/stats/allocator_impl.cc @@ -419,7 +419,11 @@ void AllocatorImpl::forEachSinkedGauge(SizeFn f_size, StatFn f_stat) cons f_stat(*gauge); } } else { - forEachGauge(f_size, f_stat); + forEachGauge(f_size, [&f_stat](Gauge& gauge) { + if (!gauge.hidden()) { + f_stat(gauge); + } + }); } } diff --git a/source/server/admin/prometheus_stats.cc b/source/server/admin/prometheus_stats.cc index d65891f1c29a..195f047a4c6b 100644 --- a/source/server/admin/prometheus_stats.cc +++ b/source/server/admin/prometheus_stats.cc @@ -43,9 +43,12 @@ std::string sanitizeValue(const absl::string_view value) { }); } -/* - * Determine whether a metric has never been emitted and choose to - * not show it if we only wanted used metrics. +/** + * Determines whether a metric should be shown based on the specified query-parameters. This covers: + * ``usedonly``, hidden, and filter. + * + * @param metric the metric to test + * @param params captures query parameters indicating which metrics should be included. */ template static bool shouldShowMetric(const StatType& metric, const StatsParams& params) { @@ -55,6 +58,12 @@ static bool shouldShowMetric(const StatType& metric, const StatsParams& params) if (params.used_only_ && !metric.used()) { return false; } + if (params.hidden_ == HiddenFlag::ShowOnly && !metric.hidden()) { + return false; + } + if (params.hidden_ == HiddenFlag::Exclude && metric.hidden()) { + return false; + } if (params.re2_filter_ != nullptr && !re2::RE2::PartialMatch(metric.name(), *params.re2_filter_)) { return false; diff --git a/source/server/admin/stats_params.cc b/source/server/admin/stats_params.cc index 4c143a238dcf..3c5c95eb6e54 100644 --- a/source/server/admin/stats_params.cc +++ b/source/server/admin/stats_params.cc @@ -50,6 +50,20 @@ Http::Code StatsParams::parse(absl::string_view url, Buffer::Instance& response) return Http::Code::BadRequest; } + const absl::optional hidden_value = Utility::queryParam(query_, "hidden"); + if (hidden_value.has_value()) { + if (hidden_value.value() == "include") { + hidden_ = HiddenFlag::Include; + } else if (hidden_value.value() == "only") { + hidden_ = HiddenFlag::ShowOnly; + } else if (hidden_value.value() == "exclude") { + hidden_ = HiddenFlag::Exclude; + } else { + response.add("usage: /stats?hidden=(include|only|exclude)\n\n"); + return Http::Code::BadRequest; + } + } + const absl::optional format_value = Utility::formatParam(query_); if (format_value.has_value()) { if (format_value.value() == "prometheus") { diff --git a/source/server/admin/stats_params.h b/source/server/admin/stats_params.h index 208bcfa7162e..09b82a96ce04 100644 --- a/source/server/admin/stats_params.h +++ b/source/server/admin/stats_params.h @@ -40,6 +40,12 @@ enum class StatsType { All, }; +enum class HiddenFlag { + Include, // Will include hidden stats along side non-hidden stats + ShowOnly, // Will only show hidden stats and exclude hidden stats + Exclude, // Default behavior. Will exclude all hidden stats +}; + struct StatsParams { /** * Parses the URL's query parameter, populating this. @@ -59,6 +65,7 @@ struct StatsParams { bool prometheus_text_readouts_{false}; bool pretty_{false}; StatsFormat format_{StatsFormat::Text}; + HiddenFlag hidden_{HiddenFlag::Exclude}; std::string filter_string_; std::shared_ptr re2_filter_; Utility::HistogramBucketsMode histogram_buckets_mode_{Utility::HistogramBucketsMode::NoBuckets}; diff --git a/source/server/admin/stats_request.cc b/source/server/admin/stats_request.cc index 954eb1127dbf..181094ff836e 100644 --- a/source/server/admin/stats_request.cc +++ b/source/server/admin/stats_request.cc @@ -183,6 +183,14 @@ template void StatsRequest::populateStatsFromScopes(const Scope return true; } + if (params_.hidden_ == HiddenFlag::Exclude && stat->hidden()) { + return true; + } + + if (params_.hidden_ == HiddenFlag::ShowOnly && !stat->hidden()) { + return true; + } + // Capture the name if we did not early-exit due to used_only -- we'll use // the name for both filtering and for capturing the stat in the map. // stat->name() takes a symbol table lock and builds a string, so we only diff --git a/test/common/stats/allocator_impl_test.cc b/test/common/stats/allocator_impl_test.cc index eea975dcb3ea..38fcb17466b2 100644 --- a/test/common/stats/allocator_impl_test.cc +++ b/test/common/stats/allocator_impl_test.cc @@ -524,6 +524,65 @@ TEST_F(AllocatorImplTest, ForEachSinkedGauge) { EXPECT_EQ(num_iterations, 0); } +TEST_F(AllocatorImplTest, ForEachSinkedGaugeHidden) { + GaugeSharedPtr unhidden_gauge; + GaugeSharedPtr hidden_gauge; + + auto unhidden_stat_name = makeStat(absl::StrCat("unhidden.gauge")); + auto hidden_stat_name = makeStat(absl::StrCat("hidden.gauge")); + + size_t num_gauges = 0; + size_t num_iterations = 0; + + unhidden_gauge = + alloc_.makeGauge(unhidden_stat_name, StatName(), {}, Gauge::ImportMode::Accumulate); + + hidden_gauge = + alloc_.makeGauge(hidden_stat_name, StatName(), {}, Gauge::ImportMode::HiddenAccumulate); + + alloc_.forEachSinkedGauge([&num_gauges](std::size_t size) { num_gauges = size; }, + [&num_iterations, unhidden_stat_name](Gauge& gauge) { + EXPECT_EQ(unhidden_stat_name, gauge.statName()); + num_iterations++; + }); + EXPECT_EQ(num_gauges, 2); + EXPECT_EQ(num_iterations, 1); +} + +TEST_F(AllocatorImplTest, ForEachSinkedGaugeHiddenPredicate) { + std::unique_ptr moved_sink_predicates = + std::make_unique(); + TestUtil::TestSinkPredicates* sink_predicates = moved_sink_predicates.get(); + GaugeSharedPtr unhidden_gauge; + GaugeSharedPtr hidden_gauge; + + alloc_.setSinkPredicates(std::move(moved_sink_predicates)); + + auto unhidden_stat_name = makeStat(absl::StrCat("unhidden.gauge")); + auto hidden_stat_name = makeStat(absl::StrCat("hidden.gauge")); + + sink_predicates->add(unhidden_stat_name); + sink_predicates->add(hidden_stat_name); + + size_t num_gauges = 0; + size_t num_iterations = 0; + + unhidden_gauge = + alloc_.makeGauge(unhidden_stat_name, StatName(), {}, Gauge::ImportMode::Accumulate); + + hidden_gauge = + alloc_.makeGauge(hidden_stat_name, StatName(), {}, Gauge::ImportMode::HiddenAccumulate); + + alloc_.forEachSinkedGauge([&num_gauges](std::size_t size) { num_gauges = size; }, + [&num_iterations, &sink_predicates](Gauge& gauge) { + ++num_iterations; + EXPECT_TRUE(sink_predicates->has(gauge.statName())); + }); + + EXPECT_EQ(num_gauges, 2); + EXPECT_EQ(num_iterations, 2); +} + TEST_F(AllocatorImplTest, ForEachSinkedTextReadout) { std::unique_ptr moved_sink_predicates = std::make_unique(); diff --git a/test/extensions/stats_sinks/open_telemetry/open_telemetry_impl_test.cc b/test/extensions/stats_sinks/open_telemetry/open_telemetry_impl_test.cc index 8d4007efc6fe..5f5b253b63bc 100644 --- a/test/extensions/stats_sinks/open_telemetry/open_telemetry_impl_test.cc +++ b/test/extensions/stats_sinks/open_telemetry/open_telemetry_impl_test.cc @@ -289,13 +289,13 @@ TEST_F(OtlpMetricsFlusherTests, MetricsWithNoAttributes) { TEST_F(OtlpMetricsFlusherTests, GaugeMetric) { OtlpMetricsFlusherImpl flusher(otlpOptions()); - addGaugeToSnapshot("test_guage1", 1); - addGaugeToSnapshot("test_guage2", 2); + addGaugeToSnapshot("test_gauge1", 1); + addGaugeToSnapshot("test_gauge2", 2); MetricsExportRequestSharedPtr metrics = flusher.flush(snapshot_); expectMetricsCount(metrics, 2); - expectGauge(metricAt(0, metrics), getTagExtractedName("test_guage1"), 1); - expectGauge(metricAt(1, metrics), getTagExtractedName("test_guage2"), 2); + expectGauge(metricAt(0, metrics), getTagExtractedName("test_gauge1"), 1); + expectGauge(metricAt(1, metrics), getTagExtractedName("test_gauge2"), 2); } TEST_F(OtlpMetricsFlusherTests, CumulativeCounterMetric) { diff --git a/test/server/admin/prometheus_stats_test.cc b/test/server/admin/prometheus_stats_test.cc index dcf9ab510bd2..18a5b3ab83e2 100644 --- a/test/server/admin/prometheus_stats_test.cc +++ b/test/server/admin/prometheus_stats_test.cc @@ -53,12 +53,12 @@ class PrometheusStatsFormatterTest : public testing::Test { tag_extracted_name_storage.statName(), cluster_tags)); } - void addGauge(const std::string& name, Stats::StatNameTagVector cluster_tags) { + void addGauge(const std::string& name, Stats::StatNameTagVector cluster_tags, + Stats::Gauge::ImportMode import_mode = Stats::Gauge::ImportMode::Accumulate) { Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_); Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_); - gauges_.push_back(alloc_.makeGauge(name_storage.statName(), - tag_extracted_name_storage.statName(), cluster_tags, - Stats::Gauge::ImportMode::Accumulate)); + gauges_.push_back(alloc_.makeGauge( + name_storage.statName(), tag_extracted_name_storage.statName(), cluster_tags, import_mode)); } void addTextReadout(const std::string& name, const std::string& value, @@ -81,6 +81,7 @@ class PrometheusStatsFormatterTest : public testing::Test { histogram->setTagExtractedName(name); histogram->setTags(cluster_tags); histogram->used_ = true; + histogram->hidden_ = false; return histogram; } @@ -779,6 +780,56 @@ envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 EXPECT_EQ(expected_output, response.toString()); } +TEST_F(PrometheusStatsFormatterTest, OutputWithHiddenGauge) { + Stats::CustomStatNamespacesImpl custom_namespaces; + + addGauge("cluster.test_cluster_2.upstream_cx_total", + {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); + addGauge("cluster.test_cluster_2.upstream_cx_total", + {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}, + Stats::Gauge::ImportMode::HiddenAccumulate); + + StatsParams params; + + { + Buffer::OwnedImpl response; + params.hidden_ = HiddenFlag::Exclude; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + const std::string expected_output = + R"EOF(# TYPE envoy_cluster_test_cluster_2_upstream_cx_total gauge +envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 +)EOF"; + EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(1UL, size); + } + { + Buffer::OwnedImpl response; + params.hidden_ = HiddenFlag::ShowOnly; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + const std::string expected_output = + R"EOF(# TYPE envoy_cluster_test_cluster_2_upstream_cx_total gauge +envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0 +)EOF"; + EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(1UL, size); + } + { + Buffer::OwnedImpl response; + params.hidden_ = HiddenFlag::Include; + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + const std::string expected_output = + R"EOF(# TYPE envoy_cluster_test_cluster_2_upstream_cx_total gauge +envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 +envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0 +)EOF"; + EXPECT_EQ(expected_output, response.toString()); + EXPECT_EQ(1UL, size); + } +} + TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { Stats::CustomStatNamespacesImpl custom_namespaces; const std::vector h1_values = {}; diff --git a/test/server/admin/stats_handler_test.cc b/test/server/admin/stats_handler_test.cc index 529f0f0993e3..225525e0b31c 100644 --- a/test/server/admin/stats_handler_test.cc +++ b/test/server/admin/stats_handler_test.cc @@ -416,6 +416,62 @@ TEST_F(AdminStatsFilterTest, HandlerStatsJsonHistogramBucketsCumulative) { EXPECT_THAT(expected_json_used_and_filter, JsonStringEq(code_response.second)); } +TEST_F(AdminStatsFilterTest, HandlerStatsJsonHiddenGauges) { + // Test that hidden=include shows all hidden and non hidden values. + const std::string url_hidden_include = "/stats?gauges&format=json&hidden=include"; + + Stats::Gauge& g1 = + store_->gaugeFromString("hiddenG1", Stats::Gauge::ImportMode::HiddenAccumulate); + g1.inc(); + + Stats::Gauge& g2 = store_->gaugeFromString("nonHiddenG2", Stats::Gauge::ImportMode::Accumulate); + g2.add(2); + + CodeResponse code_response = handlerStats(url_hidden_include); + EXPECT_EQ(Http::Code::OK, code_response.first); + const std::string expected_json_hidden_include = R"EOF({ + "stats": [ + {"name":"hiddenG1", "value":1}, + {"name":"nonHiddenG2", "value":2}, + ] +})EOF"; + EXPECT_THAT(expected_json_hidden_include, JsonStringEq(code_response.second)); + + // Test that hidden-only will not show non-hidden gauges. + const std::string url_hidden_show_only = "/stats?gauges&format=json&hidden=only"; + + code_response = handlerStats(url_hidden_show_only); + EXPECT_EQ(Http::Code::OK, code_response.first); + const std::string expected_json_hidden_show_only = R"EOF({ + "stats": [ + {"name":"hiddenG1", "value":1}, + ] +})EOF"; + EXPECT_THAT(expected_json_hidden_show_only, JsonStringEq(code_response.second)); + + // Test that hidden=exclude will not show hidden gauges. + const std::string url_hidden_exclude = "/stats?gauges&format=json&hidden=exclude"; + + code_response = handlerStats(url_hidden_exclude); + EXPECT_EQ(Http::Code::OK, code_response.first); + const std::string expected_json_hidden_exclude = R"EOF({ + "stats": [ + {"name":"nonHiddenG2", "value":2}, + ] +})EOF"; + + EXPECT_THAT(expected_json_hidden_exclude, JsonStringEq(code_response.second)); +} + +TEST_F(AdminStatsFilterTest, HandlerStatsHiddenInvalid) { + // Test that hidden=(bad inputs) returns error. + const std::string url_hidden_bad_input = "/stats?gauges&format=json&hidden=foo"; + + CodeResponse code_response = handlerStats(url_hidden_bad_input); + EXPECT_EQ(Http::Code::BadRequest, code_response.first); + EXPECT_EQ("usage: /stats?hidden=(include|only|exclude)\n\n", code_response.second); +} + TEST_F(AdminStatsFilterTest, HandlerStatsJsonHistogramBucketsDisjoint) { const std::string url = "/stats?histogram_buckets=disjoint&format=json"; // Set h as prefix to match both histograms. diff --git a/test/server/admin/stats_params_test.cc b/test/server/admin/stats_params_test.cc index cc6d7e787ef7..505893f61559 100644 --- a/test/server/admin/stats_params_test.cc +++ b/test/server/admin/stats_params_test.cc @@ -53,6 +53,23 @@ TEST(StatsParamsTest, ParseParamsFormat) { EXPECT_EQ(Http::Code::BadRequest, params.parse("?format=bogus", response)); } +TEST(StatsParamsTest, ParseParamsHidden) { + Buffer::OwnedImpl response; + StatsParams params; + + EXPECT_EQ(HiddenFlag::Exclude, params.hidden_); + + ASSERT_EQ(Http::Code::OK, params.parse("?hidden=include", response)); + EXPECT_EQ(HiddenFlag::Include, params.hidden_); + ASSERT_EQ(Http::Code::OK, params.parse("?hidden=only", response)); + EXPECT_EQ(HiddenFlag::ShowOnly, params.hidden_); + ASSERT_EQ(Http::Code::OK, params.parse("?hidden=exclude", response)); + EXPECT_EQ(HiddenFlag::Exclude, params.hidden_); + ASSERT_EQ(Http::Code::BadRequest, params.parse("?hidden=foo", response)); + ASSERT_EQ(Http::Code::OK, params.parse("?hidden", response)); + EXPECT_EQ(HiddenFlag::Exclude, params.hidden_); +} + TEST(StatsParamsTest, ParseParamsFilter) { Buffer::OwnedImpl response; diff --git a/test/server/admin/stats_request_test.cc b/test/server/admin/stats_request_test.cc index 23bd46a4e2c1..bf8012662381 100644 --- a/test/server/admin/stats_request_test.cc +++ b/test/server/admin/stats_request_test.cc @@ -38,6 +38,15 @@ class StatsRequestTest : public testing::Test { return std::make_unique(store_, params); } + std::unique_ptr makeHiddenRequest(HiddenFlag hidden, StatsFormat format, + StatsType type) { + StatsParams params; + params.hidden_ = hidden; + params.type_ = type; + params.format_ = format; + return std::make_unique(store_, params); + } + // Executes a request, counting the chunks that were generated. uint32_t iterateChunks(StatsRequest& request, bool drain = true, Http::Code expect_code = Http::Code::OK) { @@ -147,6 +156,27 @@ TEST_F(StatsRequestTest, OneStatUsedOnly) { EXPECT_EQ(0, iterateChunks(*makeRequest(true, StatsFormat::Text, StatsType::All))); } +TEST_F(StatsRequestTest, OneStatHiddenExclude) { + store_.rootScope()->gaugeFromStatName(makeStatName("foo"), + Stats::Gauge::ImportMode::HiddenAccumulate); + EXPECT_EQ( + 0, iterateChunks(*makeHiddenRequest(HiddenFlag::Exclude, StatsFormat::Text, StatsType::All))); +} + +TEST_F(StatsRequestTest, OneStatHiddenShowOnly) { + store_.rootScope()->gaugeFromStatName(makeStatName("foo"), + Stats::Gauge::ImportMode::HiddenAccumulate); + EXPECT_EQ(1, iterateChunks( + *makeHiddenRequest(HiddenFlag::ShowOnly, StatsFormat::Text, StatsType::All))); +} + +TEST_F(StatsRequestTest, OneStatHiddenInclude) { + store_.rootScope()->gaugeFromStatName(makeStatName("foo"), + Stats::Gauge::ImportMode::HiddenAccumulate); + EXPECT_EQ( + 1, iterateChunks(*makeHiddenRequest(HiddenFlag::Include, StatsFormat::Text, StatsType::All))); +} + TEST_F(StatsRequestTest, OneStatJson) { store_.rootScope()->counterFromStatName(makeStatName("foo")); EXPECT_THAT(response(*makeRequest(false, StatsFormat::Json, StatsType::All)), StartsWith("{")); From 2904bdb496a9ed6a7848ec26630b0f44abaaa82a Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Tue, 6 Jun 2023 16:19:58 -0400 Subject: [PATCH 469/740] TLS Inspector: Capture bytes analyzed distribution. (#27742) * Add a histogram for the tls inspector to capture the amount of data the filter parsed. Signed-off-by: Kevin Baichoo --- changelogs/current.yaml | 7 ++ .../listener_filters/tls_inspector.rst | 53 ++++++++++-- .../listener/tls_inspector/tls_inspector.cc | 83 ++++++++++++------- .../listener/tls_inspector/tls_inspector.h | 10 ++- test/common/stats/stat_test_utility.cc | 4 + test/common/stats/stat_test_utility.h | 3 + .../filters/listener/tls_inspector/BUILD | 1 + .../tls_inspector_integration_test.cc | 10 +++ .../tls_inspector/tls_inspector_test.cc | 45 +++++++++- test/test_common/utility.cc | 15 ++++ test/test_common/utility.h | 8 ++ 11 files changed, 197 insertions(+), 42 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 00ce45b7b76b..39444ae95ea7 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -241,6 +241,13 @@ new_features: added new configuration field :ref:`domain ` to allow for setting rate limit domains on a per-route basis. +- area: tls_inspector + change: | + added histogram ``bytes_processed`` which records the number of bytes of + the tls_inspector processed while analyzing for tls usage. In cases where + the connection uses tls this records the tls client hello size. In cases + where the connection doesn't use tls this records the amount of bytes the + tls_inspector processed until it realized the connection was not using tls. - area: access_log change: | added access log filter :ref:`log_type_filter ` diff --git a/docs/root/configuration/listeners/listener_filters/tls_inspector.rst b/docs/root/configuration/listeners/listener_filters/tls_inspector.rst index b5fdadf4b924..d2a29785eaf9 100644 --- a/docs/root/configuration/listeners/listener_filters/tls_inspector.rst +++ b/docs/root/configuration/listeners/listener_filters/tls_inspector.rst @@ -35,15 +35,50 @@ Statistics This filter has a statistics tree rooted at *tls_inspector* with the following statistics: -.. csv-table:: - :header: Name, Type, Description +.. list-table:: + :header-rows: 1 :widths: 1, 1, 2 - client_hello_too_large, Counter, Total unreasonably large Client Hello received - tls_found, Counter, Total number of times TLS was found - tls_not_found, Counter, Total number of times TLS was not found - alpn_found, Counter, Total number of times `Application-Layer Protocol Negotiation `_ was successful - alpn_not_found, Counter, Total number of times `Application-Layer Protocol Negotiation `_ has failed - sni_found, Counter, Total number of times `Server Name Indication `_ was found - sni_not_found, Counter, Total number of times `Server Name Indication `_ was not found + * - Name + - Type + - Description + + * - client_hello_too_large + - counter + - total unreasonably large client hello received + + * - tls_found + - Counter + - Total number of times TLS was found + + * - tls_not_found + - Counter + - Total number of times TLS was not found + + * - alpn_found + - Counter + - Total number of times `Application-Layer Protocol Negotiation `_ was successful + + * - alpn_not_found + - Counter + - Total number of times `Application-Layer Protocol Negotiation `_ has failed + + * - sni_found + - Counter + - Total number of times `Server Name Indication `_ was found + + * - sni_not_found + - Counter + - Total number of times `Server Name Indication `_ was not found + + * - bytes_processed + - Histogram + - Records sizes which records the number of bytes the tls_inspector processed while analyzing for tls usage. + If the connection uses TLS: this is the size of client hello. If the client hello is too large, then the + recorded value will be 64KiB which is the maximum client hello size. + If the connection does not use TLS: it is the number of bytes processed + until the inspector determined the connection was not using TLS. + If the connection terminates early nothing is recorded if we didn't have + sufficient bytes for either of the cases above. + diff --git a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc index fb78a4fcafd1..31f849087866 100644 --- a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc +++ b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc @@ -26,6 +26,20 @@ namespace Envoy { namespace Extensions { namespace ListenerFilters { namespace TlsInspector { +namespace { + +uint64_t computeClientHelloSize(const BIO* bio, uint64_t prior_bytes_read, + size_t original_bio_length) { + const uint8_t* remaining_buffer; + size_t remaining_bytes; + const int rc = BIO_mem_contents(bio, &remaining_buffer, &remaining_bytes); + ASSERT(rc == 1); + ASSERT(original_bio_length >= remaining_bytes); + const size_t processed_bio_bytes = original_bio_length - remaining_bytes; + return processed_bio_bytes + prior_bytes_read; +} + +} // namespace // Min/max TLS version recognized by the underlying TLS/SSL library. const unsigned Config::TLS_MIN_SUPPORTED_VERSION = TLS1_VERSION; @@ -35,7 +49,8 @@ Config::Config( Stats::Scope& scope, const envoy::extensions::filters::listener::tls_inspector::v3::TlsInspector& proto_config, uint32_t max_client_hello_size) - : stats_{ALL_TLS_INSPECTOR_STATS(POOL_COUNTER_PREFIX(scope, "tls_inspector."))}, + : stats_{ALL_TLS_INSPECTOR_STATS(POOL_COUNTER_PREFIX(scope, "tls_inspector."), + POOL_HISTOGRAM_PREFIX(scope, "tls_inspector."))}, ssl_ctx_(SSL_CTX_new(TLS_with_buffers_method())), enable_ja3_fingerprinting_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config, enable_ja3_fingerprinting, false)), @@ -130,8 +145,9 @@ Network::FilterStatus Filter::onData(Network::ListenerFilterBuffer& buffer) { if (static_cast(raw_slice.len_) > read_) { const uint8_t* data = static_cast(raw_slice.mem_) + read_; const size_t len = raw_slice.len_ - read_; + const uint64_t bytes_already_processed = read_; read_ = raw_slice.len_; - ParseState parse_state = parseClientHello(data, len); + ParseState parse_state = parseClientHello(data, len, bytes_already_processed); switch (parse_state) { case ParseState::Error: cb_->socket().ioHandle().close(); @@ -148,46 +164,57 @@ Network::FilterStatus Filter::onData(Network::ListenerFilterBuffer& buffer) { return Network::FilterStatus::StopIteration; } -ParseState Filter::parseClientHello(const void* data, size_t len) { - // Ownership is passed to ssl_ in SSL_set_bio() +ParseState Filter::parseClientHello(const void* data, size_t len, + uint64_t bytes_already_processed) { + // Ownership remains here though we pass a reference to it in `SSL_set0_rbio()`. bssl::UniquePtr bio(BIO_new_mem_buf(data, len)); // Make the mem-BIO return that there is more data - // available beyond it's end + // available beyond it's end. BIO_set_mem_eof_return(bio.get(), -1); - SSL_set_bio(ssl_.get(), bio.get(), bio.get()); - bio.release(); + // We only do reading as we abort the handshake early. + SSL_set0_rbio(ssl_.get(), bssl::UpRef(bio).release()); int ret = SSL_do_handshake(ssl_.get()); // This should never succeed because an error is always returned from the SNI callback. ASSERT(ret <= 0); - switch (SSL_get_error(ssl_.get(), ret)) { - case SSL_ERROR_WANT_READ: - if (read_ == config_->maxClientHelloSize()) { - // We've hit the specified size limit. This is an unreasonably large ClientHello; - // indicate failure. - config_->stats().client_hello_too_large_.inc(); - return ParseState::Error; - } - return ParseState::Continue; - case SSL_ERROR_SSL: - if (clienthello_success_) { - config_->stats().tls_found_.inc(); - if (alpn_found_) { - config_->stats().alpn_found_.inc(); + ParseState state = [this, ret]() { + switch (SSL_get_error(ssl_.get(), ret)) { + case SSL_ERROR_WANT_READ: + if (read_ == config_->maxClientHelloSize()) { + // We've hit the specified size limit. This is an unreasonably large ClientHello; + // indicate failure. + config_->stats().client_hello_too_large_.inc(); + return ParseState::Error; + } + return ParseState::Continue; + case SSL_ERROR_SSL: + if (clienthello_success_) { + config_->stats().tls_found_.inc(); + if (alpn_found_) { + config_->stats().alpn_found_.inc(); + } else { + config_->stats().alpn_not_found_.inc(); + } + cb_->socket().setDetectedTransportProtocol("tls"); } else { - config_->stats().alpn_not_found_.inc(); + config_->stats().tls_not_found_.inc(); } - cb_->socket().setDetectedTransportProtocol("tls"); - } else { - config_->stats().tls_not_found_.inc(); + return ParseState::Done; + default: + return ParseState::Error; } - return ParseState::Done; - default: - return ParseState::Error; + }(); + + if (state != ParseState::Continue) { + // Record bytes analyzed as we're done processing. + config_->stats().bytes_processed_.recordValue( + computeClientHelloSize(bio.get(), bytes_already_processed, len)); } + + return state; } // Google GREASE values (https://datatracker.ietf.org/doc/html/rfc8701) diff --git a/source/extensions/filters/listener/tls_inspector/tls_inspector.h b/source/extensions/filters/listener/tls_inspector/tls_inspector.h index ad837c8b7efc..fd463f0234cb 100644 --- a/source/extensions/filters/listener/tls_inspector/tls_inspector.h +++ b/source/extensions/filters/listener/tls_inspector/tls_inspector.h @@ -4,6 +4,7 @@ #include "envoy/event/timer.h" #include "envoy/extensions/filters/listener/tls_inspector/v3/tls_inspector.pb.h" #include "envoy/network/filter.h" +#include "envoy/stats/histogram.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" @@ -19,20 +20,21 @@ namespace TlsInspector { /** * All stats for the TLS inspector. @see stats_macros.h */ -#define ALL_TLS_INSPECTOR_STATS(COUNTER) \ +#define ALL_TLS_INSPECTOR_STATS(COUNTER, HISTOGRAM) \ COUNTER(client_hello_too_large) \ COUNTER(tls_found) \ COUNTER(tls_not_found) \ COUNTER(alpn_found) \ COUNTER(alpn_not_found) \ COUNTER(sni_found) \ - COUNTER(sni_not_found) + COUNTER(sni_not_found) \ + HISTOGRAM(bytes_processed, Bytes) /** * Definition of all stats for the TLS inspector. @see stats_macros.h */ struct TlsInspectorStats { - ALL_TLS_INSPECTOR_STATS(GENERATE_COUNTER_STRUCT) + ALL_TLS_INSPECTOR_STATS(GENERATE_COUNTER_STRUCT, GENERATE_HISTOGRAM_STRUCT) }; enum class ParseState { @@ -83,7 +85,7 @@ class Filter : public Network::ListenerFilter, Logger::LoggablemaxClientHelloSize(); } private: - ParseState parseClientHello(const void* data, size_t len); + ParseState parseClientHello(const void* data, size_t len, uint64_t bytes_already_processed); ParseState onRead(); void onALPN(const unsigned char* data, unsigned int len); void onServername(absl::string_view name); diff --git a/test/common/stats/stat_test_utility.cc b/test/common/stats/stat_test_utility.cc index 1454a8a74a70..73621b20d11c 100644 --- a/test/common/stats/stat_test_utility.cc +++ b/test/common/stats/stat_test_utility.cc @@ -294,6 +294,10 @@ std::vector TestStore::histogramValues(const std::string& name, bool c return copy; } +bool TestStore::histogramRecordedValues(const std::string& name) const { + return histogram_values_map_.contains(name); +} + // TODO(jmarantz): this utility is intended to be used both for unit tests // and fuzz tests. But those have different checking macros, e.g. EXPECT_EQ vs // FUZZ_ASSERT. diff --git a/test/common/stats/stat_test_utility.h b/test/common/stats/stat_test_utility.h index 9112c039754c..5ea1edb97441 100644 --- a/test/common/stats/stat_test_utility.h +++ b/test/common/stats/stat_test_utility.h @@ -149,6 +149,9 @@ class TestStore : public SymbolTableProvider, public IsolatedStoreImpl { GaugeOptConstRef findGaugeByString(const std::string& name) const; HistogramOptConstRef findHistogramByString(const std::string& name) const; std::vector histogramValues(const std::string& name, bool clear); + // Returns whether the given histogram has recorded any value since it was + // created. + bool histogramRecordedValues(const std::string& name) const; protected: ScopeSharedPtr makeScope(StatName name) override; diff --git a/test/extensions/filters/listener/tls_inspector/BUILD b/test/extensions/filters/listener/tls_inspector/BUILD index dfad8b7ec3af..8c2ef09bdddf 100644 --- a/test/extensions/filters/listener/tls_inspector/BUILD +++ b/test/extensions/filters/listener/tls_inspector/BUILD @@ -26,6 +26,7 @@ envoy_cc_test( "//source/common/network:listener_filter_buffer_lib", "//source/extensions/filters/listener/tls_inspector:config", "//source/extensions/filters/listener/tls_inspector:tls_inspector_lib", + "//test/common/stats:stat_test_utility_lib", "//test/mocks/api:api_mocks", "//test/mocks/network:network_mocks", "//test/mocks/stats:stats_mocks", diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_integration_test.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_integration_test.cc index adf7513e1ded..51a252af2bb2 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_integration_test.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_integration_test.cc @@ -188,8 +188,18 @@ TEST_P(TlsInspectorIntegrationTest, JA3FingerprintIsSet) { /*ssl_options=*/ssl_options, /*curves_list=*/"P-256", /*enable_`ja3`_fingerprinting=*/true); client_->close(Network::ConnectionCloseType::NoFlush); + EXPECT_THAT(waitForAccessLog(listener_access_log_name_), testing::Eq("71d1f47d1125ac53c3c6a4863c087cfe")); + + test_server_->waitUntilHistogramHasSamples("tls_inspector.bytes_processed"); + auto bytes_processed_histogram = test_server_->histogram("tls_inspector.bytes_processed"); + EXPECT_EQ( + TestUtility::readSampleCount(test_server_->server().dispatcher(), *bytes_processed_histogram), + 1); + EXPECT_EQ(static_cast(TestUtility::readSampleSum(test_server_->server().dispatcher(), + *bytes_processed_histogram)), + 115); } INSTANTIATE_TEST_SUITE_P(IpVersions, TlsInspectorIntegrationTest, diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc index e7e423c54300..e9ae319ab3d3 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc @@ -4,6 +4,7 @@ #include "source/common/network/listener_filter_buffer_impl.h" #include "source/extensions/filters/listener/tls_inspector/tls_inspector.h" +#include "test/common/stats/stat_test_utility.h" #include "test/extensions/filters/listener/tls_inspector/tls_utility.h" #include "test/mocks/api/mocks.h" #include "test/mocks/network/mocks.h" @@ -83,7 +84,7 @@ class TlsInspectorTest : public testing::TestWithParam os_sys_calls_; TestThreadsafeSingletonInjector os_calls_{&os_sys_calls_}; - Stats::IsolatedStoreImpl store_; + Stats::TestUtil::TestStore store_; ConfigSharedPtr cfg_; std::unique_ptr filter_; Network::MockListenerFilterCallbacks cb_; @@ -145,10 +146,15 @@ TEST_P(TlsInspectorTest, AlpnRegistered) { // trigger the event to copy the client hello message into buffer file_event_callback_(Event::FileReadyType::Read); auto state = filter_->onData(*buffer_); + EXPECT_EQ(Network::FilterStatus::Continue, state); EXPECT_EQ(1, cfg_->stats().tls_found_.value()); EXPECT_EQ(1, cfg_->stats().sni_not_found_.value()); EXPECT_EQ(1, cfg_->stats().alpn_found_.value()); + const std::vector bytes_processed = + store_.histogramValues("tls_inspector.bytes_processed", false); + ASSERT_EQ(1, bytes_processed.size()); + EXPECT_EQ(client_hello.size(), bytes_processed[0]); } // Test with the ClientHello spread over multiple socket reads. @@ -208,6 +214,10 @@ TEST_P(TlsInspectorTest, MultipleReads) { EXPECT_EQ(1, cfg_->stats().tls_found_.value()); EXPECT_EQ(1, cfg_->stats().sni_found_.value()); EXPECT_EQ(1, cfg_->stats().alpn_found_.value()); + const std::vector bytes_processed = + store_.histogramValues("tls_inspector.bytes_processed", false); + ASSERT_EQ(1, bytes_processed.size()); + EXPECT_EQ(client_hello.size(), bytes_processed[0]); } // Test that the filter correctly handles a ClientHello with no extensions present. @@ -276,6 +286,10 @@ TEST_P(TlsInspectorTest, ClientHelloTooBig) { auto state = filter_->onData(*buffer_); EXPECT_EQ(Network::FilterStatus::StopIteration, state); EXPECT_EQ(1, cfg_->stats().client_hello_too_large_.value()); + const std::vector bytes_processed = + store_.histogramValues("tls_inspector.bytes_processed", false); + ASSERT_EQ(1, bytes_processed.size()); + EXPECT_EQ(max_size, bytes_processed[0]); } // Test that the filter sets the `JA3` hash @@ -397,6 +411,35 @@ TEST_P(TlsInspectorTest, NotSsl) { auto state = filter_->onData(*buffer_); EXPECT_EQ(Network::FilterStatus::Continue, state); EXPECT_EQ(1, cfg_->stats().tls_not_found_.value()); + const std::vector bytes_processed = + store_.histogramValues("tls_inspector.bytes_processed", false); + ASSERT_EQ(1, bytes_processed.size()); + EXPECT_EQ(5, bytes_processed[0]); +} + +TEST_P(TlsInspectorTest, EarlyTerminationShouldNotRecordBytesProcessed) { + envoy::extensions::filters::listener::tls_inspector::v3::TlsInspector proto_config; + cfg_ = std::make_shared(*store_.rootScope(), proto_config); + std::vector client_hello = Tls::Test::generateClientHello( + std::get<0>(GetParam()), std::get<1>(GetParam()), "example.com", ""); + + // Clobber half of client_hello so we don't have sufficient bytes to finish inspection. + client_hello.resize(client_hello.size() / 2); + + init(); + mockSysCallForPeek(client_hello); + EXPECT_CALL(socket_, setRequestedServerName(_)).Times(0); + EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); + EXPECT_CALL(socket_, detectedTransportProtocol()).Times(::testing::AnyNumber()); + // Trigger the event to copy the client hello message into buffer + file_event_callback_(Event::FileReadyType::Read); + auto state = filter_->onData(*buffer_); + EXPECT_EQ(Network::FilterStatus::StopIteration, state); + + // Terminate early. + filter_.reset(); + + ASSERT_FALSE(store_.histogramRecordedValues("tls_inspector.bytes_processed")); } } // namespace diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index f090c095bad5..593745501702 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -285,6 +285,21 @@ uint64_t TestUtility::readSampleCount(Event::Dispatcher& main_dispatcher, return sample_count; } +double TestUtility::readSampleSum(Event::Dispatcher& main_dispatcher, + const Stats::ParentHistogram& histogram) { + // Note: we need to read the sample count from the main thread, to avoid data races. + double sample_sum = 0; + absl::Notification notification; + + main_dispatcher.post([&] { + sample_sum = histogram.cumulativeStatistics().sampleSum(); + notification.Notify(); + }); + notification.WaitForNotification(); + + return sample_sum; +} + std::list TestUtility::makeDnsResponse(const std::list& addresses, std::chrono::seconds ttl) { std::list ret; diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 2995e0d7d843..e5cc0c5cfcd5 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -328,6 +328,14 @@ class TestUtility { */ static uint64_t readSampleCount(Event::Dispatcher& main_dispatcher, const Stats::ParentHistogram& histogram); + /** + * Read a histogram's sum from the main thread. + * @param store supplies the stats store. + * @param name histogram name. + * @return double the sample sum. + */ + static double readSampleSum(Event::Dispatcher& main_dispatcher, + const Stats::ParentHistogram& histogram); /** * Find a readout in a stats store. From 878d26f3b5ab7bfe3321f6f8d48b51763e9630d6 Mon Sep 17 00:00:00 2001 From: Kuat Date: Tue, 6 Jun 2023 13:21:58 -0700 Subject: [PATCH 470/740] tests: remove additional no_extension_lookup_by_name runtime override in tests (#27642) Signed-off-by: Kuat Yessenov --- test/common/config/BUILD | 1 - .../custom_config_validators_impl_test.cc | 26 ++++----- .../tracing/tracer_manager_impl_test.cc | 16 ++---- .../upstream/cluster_manager_impl_test.cc | 10 ++-- test/common/upstream/upstream_impl_test.cc | 33 ++++------- test/config/utility.cc | 2 +- test/extensions/access_loggers/grpc/BUILD | 1 + .../tcp_grpc_access_log_integration_test.cc | 5 +- test/extensions/clusters/eds/BUILD | 1 - test/extensions/clusters/eds/eds_test.cc | 6 +- .../aws_metadata_fetcher_integration_test.cc | 2 + .../common/dynamic_forward_proxy/BUILD | 1 - .../dns_cache_impl_test.cc | 17 +++--- test/extensions/filters/http/geoip/BUILD | 11 +++- .../filters/http/geoip/config_test.cc | 17 +++--- .../extensions/filters/http/geoip/dummy.proto | 6 ++ .../filters/http/geoip/geoip_filter_test.cc | 4 +- test/extensions/filters/http/geoip/mocks.h | 5 +- .../tls_inspector_integration_test.cc | 2 + .../network/echo/echo_integration_test.cc | 4 ++ .../filters/network/rbac/integration_test.cc | 2 + .../udp_proxy/udp_proxy_integration_test.cc | 2 + .../listener_managers/listener_manager/BUILD | 7 +++ .../listener_manager/config.proto | 15 +++++ .../listener_manager_impl_test.cc | 56 ++++++++----------- .../transport_sockets/proxy_protocol/BUILD | 2 + .../proxy_protocol_integration_test.cc | 54 +++++++++--------- .../transport_sockets/tcp_stats/BUILD | 2 + .../tcp_stats/tcp_stats_integration_test.cc | 4 ++ test/integration/base_integration_test.cc | 3 - .../filter_manager_integration_test.cc | 12 +++- test/integration/http_integration.cc | 3 + .../listener_lds_integration_test.cc | 5 +- .../multiplexed_upstream_integration_test.cc | 6 +- .../socket_interface_integration_test.cc | 2 + .../tcp_conn_pool_integration_test.cc | 1 + test/server/server_test.cc | 12 ---- .../callbacks_stats_sink_bootstrap.yaml | 2 + .../server/stats_sink_bootstrap.yaml | 2 + .../stats_sink_manual_flush_bootstrap.yaml | 2 + 40 files changed, 204 insertions(+), 160 deletions(-) create mode 100644 test/extensions/filters/http/geoip/dummy.proto create mode 100644 test/extensions/listener_managers/listener_manager/config.proto diff --git a/test/common/config/BUILD b/test/common/config/BUILD index dd5035c98579..13aed4619e3d 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -248,6 +248,5 @@ envoy_cc_test( "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/server:instance_mocks", "//test/test_common:registry_lib", - "//test/test_common:test_runtime_lib", ], ) diff --git a/test/common/config/custom_config_validators_impl_test.cc b/test/common/config/custom_config_validators_impl_test.cc index 8b119603d1d9..f907fedaf238 100644 --- a/test/common/config/custom_config_validators_impl_test.cc +++ b/test/common/config/custom_config_validators_impl_test.cc @@ -3,7 +3,6 @@ #include "test/mocks/protobuf/mocks.h" #include "test/mocks/server/instance.h" #include "test/test_common/registry.h" -#include "test/test_common/test_runtime.h" #include "gtest/gtest.h" @@ -44,11 +43,10 @@ class FakeConfigValidatorFactory : public ConfigValidatorFactory { Envoy::ProtobufTypes::MessagePtr createEmptyConfigProto() override { // Using Struct instead of a custom empty config proto. This is only allowed in tests. - return ProtobufTypes::MessagePtr{new Envoy::ProtobufWkt::Struct()}; + return should_reject_ ? ProtobufTypes::MessagePtr{new Envoy::ProtobufWkt::Struct()} + : ProtobufTypes::MessagePtr{new Envoy::ProtobufWkt::Value()}; } - std::set configTypes() override { return {}; } - std::string name() const override { return absl::StrCat(category(), ".fake_config_validator_", should_reject_ ? "reject" : "accept"); @@ -65,10 +63,7 @@ class CustomConfigValidatorsImplTest : public testing::Test { public: CustomConfigValidatorsImplTest() : factory_accept_(false), factory_reject_(true), register_factory_accept_(factory_accept_), - register_factory_reject_(factory_reject_) { - scoped_runtime_.mergeValues( - {{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - } + register_factory_reject_(factory_reject_) {} static envoy::config::core::v3::TypedExtensionConfig parseConfig(const std::string& config) { envoy::config::core::v3::TypedExtensionConfig proto; @@ -83,12 +78,17 @@ class CustomConfigValidatorsImplTest : public testing::Test { testing::NiceMock validation_visitor_; const testing::NiceMock server_; const std::string type_url_{Envoy::Config::getTypeUrl()}; - TestScopedRuntime scoped_runtime_; - static constexpr char AcceptValidatorConfig[] = - "name: envoy.config.validators.fake_config_validator_accept"; - static constexpr char RejectValidatorConfig[] = - "name: envoy.config.validators.fake_config_validator_reject"; + static constexpr char AcceptValidatorConfig[] = R"EOF( + name: envoy.config.validators.fake_config_validator_accept + typed_config: + "@type": type.googleapis.com/google.protobuf.Value + )EOF"; + static constexpr char RejectValidatorConfig[] = R"EOF( + name: envoy.config.validators.fake_config_validator_reject + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + )EOF"; }; // Validates that empty config that has no validators is always accepted. diff --git a/test/common/tracing/tracer_manager_impl_test.cc b/test/common/tracing/tracer_manager_impl_test.cc index 75864ff587fb..e9eeb4ec6da2 100644 --- a/test/common/tracing/tracer_manager_impl_test.cc +++ b/test/common/tracing/tracer_manager_impl_test.cc @@ -64,10 +64,9 @@ TEST_F(TracerManagerImplTest, ShouldReturnWhenNoTracingProviderHasBeenConfigured } TEST_F(TracerManagerImplTest, ShouldUseProperTracerFactory) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); envoy::config::trace::v3::Tracing_Http tracing_config; tracing_config.set_name("envoy.tracers.sample"); + tracing_config.mutable_typed_config()->PackFrom(ProtobufWkt::Struct()); auto tracer = tracer_manager_.getOrCreateTracer(&tracing_config); @@ -119,19 +118,16 @@ TEST_F(TracerManagerImplTest, ShouldCacheTracersBasedOnFullConfig) { } TEST_F(TracerManagerImplTest, ShouldFailIfTracerProviderIsUnknown) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); envoy::config::trace::v3::Tracing_Http tracing_config; tracing_config.set_name("invalid"); + tracing_config.mutable_typed_config()->PackFrom(ProtobufWkt::Value()); EXPECT_THROW_WITH_MESSAGE(tracer_manager_.getOrCreateTracer(&tracing_config), EnvoyException, - "Didn't find a registered implementation for name: 'invalid'"); + "Didn't find a registered implementation for 'invalid' " + "with type URL: 'google.protobuf.Value'"); } TEST_F(TracerManagerImplTest, ShouldFailIfProviderSpecificConfigIsNotValid) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - envoy::config::trace::v3::Tracing_Http tracing_config; tracing_config.set_name("envoy.tracers.sample"); tracing_config.mutable_typed_config()->PackFrom(ValueUtil::stringValue("value")); @@ -139,8 +135,8 @@ TEST_F(TracerManagerImplTest, ShouldFailIfProviderSpecificConfigIsNotValid) { ProtobufWkt::Any expected_any_proto; expected_any_proto.PackFrom(ValueUtil::stringValue("value")); EXPECT_THROW_WITH_MESSAGE(tracer_manager_.getOrCreateTracer(&tracing_config), EnvoyException, - fmt::format("Unable to unpack as google.protobuf.Struct: {}", - expected_any_proto.DebugString())); + "Didn't find a registered implementation for 'envoy.tracers.sample' " + "with type URL: 'google.protobuf.Value'"); } class TracerManagerImplCacheTest : public testing::Test { diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 97a7283a3d00..ce084f9c4a65 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -1028,9 +1028,6 @@ TEST_F(ClusterManagerImplTest, ClusterProvidedLbNotConfigured) { // Verify that multiple load balancing policies can be specified, and Envoy selects the first // policy that it has a factory for. TEST_F(ClusterManagerImplTest, LbPolicyConfig) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - // envoy.load_balancers.custom_lb is registered by linking in // //test/integration/load_balancers:custom_lb_policy. const std::string yaml = fmt::format(R"EOF( @@ -1046,6 +1043,8 @@ TEST_F(ClusterManagerImplTest, LbPolicyConfig) { name: envoy.load_balancers.unknown_lb - typed_extension_config: name: envoy.load_balancers.custom_lb + typed_config: + "@type": type.googleapis.com/test.integration.custom_lb.CustomLbConfig load_assignment: cluster_name: cluster_1 endpoints: @@ -4433,9 +4432,6 @@ class TestUpstreamNetworkFilterConfigFactory // Verify that configured upstream filters are added to client connections. TEST_F(ClusterManagerImplTest, AddUpstreamFilters) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - TestUpstreamNetworkFilterConfigFactory factory; Registry::InjectFactory registry( factory); @@ -4457,6 +4453,8 @@ TEST_F(ClusterManagerImplTest, AddUpstreamFilters) { port_value: 11001 filters: - name: envoy.test.filter + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct )EOF"; create(parseBootstrapFromV3Yaml(yaml)); diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 49c105eea8b3..b6d23ac41c2e 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -2348,9 +2348,6 @@ TEST_F(StaticClusterImplTest, UnsupportedLBType) { // load_balancing_policy should be used when lb_policy is set to LOAD_BALANCING_POLICY_CONFIG. TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithLbPolicy) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - const std::string yaml = R"EOF( name: staticcluster connect_timeout: 0.25s @@ -2360,6 +2357,8 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithLbPolicy) { policies: - typed_extension_config: name: custom_lb + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct load_assignment: endpoints: - lb_endpoints: @@ -2392,9 +2391,6 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithLbPolicy) { // lb_policy is set to LOAD_BALANCING_POLICY_CONFIG and has no load_balancing_policy. TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithoutConfiguration) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - const std::string yaml = R"EOF( name: staticcluster connect_timeout: 0.25s @@ -2426,9 +2422,6 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithoutConfiguration) { // load_balancing_policy is set and common_lb_config is set. TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithCommonLbConfig) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - const std::string yaml = R"EOF( name: staticcluster connect_timeout: 0.25s @@ -2437,6 +2430,8 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithCommonLbConfig) { policies: - typed_extension_config: name: custom_lb + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct common_lb_config: update_merge_window: 1s load_assignment: @@ -2464,9 +2459,6 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithCommonLbConfig) { // load_balancing_policy is set and some fields in common_lb_config are set. TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithCommonLbConfigAndSpecificFields) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - const std::string yaml = R"EOF( name: staticcluster connect_timeout: 0.25s @@ -2475,6 +2467,8 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithCommonLbConfigAndSpecificFi policies: - typed_extension_config: name: custom_lb + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct common_lb_config: locality_weighted_lb_config: {} load_assignment: @@ -2506,9 +2500,6 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithCommonLbConfigAndSpecificFi // load_balancing_policy is set and lb_subset_config is set. TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithLbSubsetConfig) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - const std::string yaml = R"EOF( name: staticcluster connect_timeout: 0.25s @@ -2517,6 +2508,8 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithLbSubsetConfig) { policies: - typed_extension_config: name: custom_lb + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct lb_subset_config: fallback_policy: ANY_ENDPOINT subset_selectors: @@ -2590,9 +2583,6 @@ TEST_F(StaticClusterImplTest, LbPolicyConfigThrowsExceptionIfNoLbPoliciesFound) // load_balancing_policy should also be used when lb_policy is set to something else besides // LOAD_BALANCING_POLICY_CONFIG. TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithOtherLbPolicy) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - const std::string yaml = R"EOF( name: staticcluster connect_timeout: 0.25s @@ -2602,6 +2592,8 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithOtherLbPolicy) { policies: - typed_extension_config: name: custom_lb + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct load_assignment: endpoints: - lb_endpoints: @@ -2632,9 +2624,6 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithOtherLbPolicy) { // load_balancing_policy should also be used when lb_policy is omitted. TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithoutLbPolicy) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - const std::string yaml = R"EOF( name: staticcluster connect_timeout: 0.25s @@ -2643,6 +2632,8 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithoutLbPolicy) { policies: - typed_extension_config: name: custom_lb + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct load_assignment: endpoints: - lb_endpoints: diff --git a/test/config/utility.cc b/test/config/utility.cc index b9b15c102ad3..f55bcbf0b362 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -207,7 +207,7 @@ std::string ConfigHelper::testInspectorFilter() { return R"EOF( name: "envoy.filters.listener.test" typed_config: - "@type": type.googleapis.com/google.protobuf.Struct + "@type": type.googleapis.com/test.integration.filters.TestInspectorFilterConfig )EOF"; } diff --git a/test/extensions/access_loggers/grpc/BUILD b/test/extensions/access_loggers/grpc/BUILD index 38747fdfc134..423c8734457f 100644 --- a/test/extensions/access_loggers/grpc/BUILD +++ b/test/extensions/access_loggers/grpc/BUILD @@ -130,6 +130,7 @@ envoy_extension_cc_test( "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/access_loggers/grpc/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/echo/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/rbac/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", "@envoy_api//envoy/service/accesslog/v3:pkg_cc_proto", diff --git a/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc index f84a0549a5c8..c962d56706ce 100644 --- a/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc @@ -1,6 +1,8 @@ #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/config/core/v3/address.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" +#include "envoy/extensions/filters/network/echo/v3/echo.pb.h" +#include "envoy/extensions/filters/network/echo/v3/echo.pb.validate.h" #include "envoy/extensions/filters/network/rbac/v3/rbac.pb.h" #include "envoy/extensions/filters/network/rbac/v3/rbac.pb.validate.h" #include "envoy/extensions/filters/network/tcp_proxy/v3/tcp_proxy.pb.h" @@ -125,8 +127,9 @@ class TcpGrpcAccessLogIntegrationTest : public Grpc::GrpcClientIntegrationParamT auto* alpn = filter_chain->mutable_filter_chain_match()->add_application_protocols(); *alpn = "envoyalpn"; auto* filter = filter_chain->mutable_filters(0); + envoy::extensions::filters::network::echo::v3::Echo echo; filter->set_name("envoy.filters.network.echo"); - filter->clear_typed_config(); + filter->mutable_typed_config()->PackFrom(echo); }); if (ssl_terminate) { config_helper_.addSslConfig(); diff --git a/test/extensions/clusters/eds/BUILD b/test/extensions/clusters/eds/BUILD index fe0af3d42039..b38d03fe9077 100644 --- a/test/extensions/clusters/eds/BUILD +++ b/test/extensions/clusters/eds/BUILD @@ -30,7 +30,6 @@ envoy_cc_test( "//test/mocks/ssl:ssl_mocks", "//test/mocks/upstream:cluster_manager_mocks", "//test/mocks/upstream:health_checker_mocks", - "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/test/extensions/clusters/eds/eds_test.cc b/test/extensions/clusters/eds/eds_test.cc index a308a65a64a6..3457970f8ae9 100644 --- a/test/extensions/clusters/eds/eds_test.cc +++ b/test/extensions/clusters/eds/eds_test.cc @@ -22,7 +22,6 @@ #include "test/mocks/ssl/mocks.h" #include "test/mocks/upstream/cluster_manager.h" #include "test/mocks/upstream/health_checker.h" -#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -1781,9 +1780,6 @@ TEST_F(EdsLocalityWeightsTest, WeightsPresentWithLocalityWeightedConfig) { // Validate that onConfigUpdate() propagates locality weights to the host set when the cluster uses // load balancing policy extensions. TEST_F(EdsLocalityWeightsTest, WeightsPresentWithLoadBalancingPolicyConfig) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - // envoy.load_balancers.custom_lb is registered by linking in // //test/integration/load_balancers:custom_lb_policy. expectLocalityWeightsPresentForClusterConfig(R"EOF( @@ -1795,6 +1791,8 @@ TEST_F(EdsLocalityWeightsTest, WeightsPresentWithLoadBalancingPolicyConfig) { policies: - typed_extension_config: name: envoy.load_balancers.custom_lb + typed_config: + "@type": type.googleapis.com/test.integration.custom_lb.CustomLbConfig eds_cluster_config: service_name: fare eds_config: diff --git a/test/extensions/common/aws/aws_metadata_fetcher_integration_test.cc b/test/extensions/common/aws/aws_metadata_fetcher_integration_test.cc index 4422cc7d3ac6..6e71984ee8a7 100644 --- a/test/extensions/common/aws/aws_metadata_fetcher_integration_test.cc +++ b/test/extensions/common/aws/aws_metadata_fetcher_integration_test.cc @@ -36,6 +36,8 @@ class AwsMetadataIntegrationTestBase : public ::testing::Test, public BaseIntegr numerator: 100 denominator: HUNDRED - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router codec_type: HTTP1 route_config: virtual_hosts: diff --git a/test/extensions/common/dynamic_forward_proxy/BUILD b/test/extensions/common/dynamic_forward_proxy/BUILD index 5bbcf74c2823..d57b03e6b71d 100644 --- a/test/extensions/common/dynamic_forward_proxy/BUILD +++ b/test/extensions/common/dynamic_forward_proxy/BUILD @@ -28,7 +28,6 @@ envoy_cc_test( "//test/mocks/thread_local:thread_local_mocks", "//test/test_common:registry_lib", "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/common/dynamic_forward_proxy/v3:pkg_cc_proto", diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index b33e2ed770b4..212e38eceeb5 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -18,7 +18,6 @@ #include "test/mocks/thread_local/mocks.h" #include "test/test_common/registry.h" #include "test/test_common/simulated_time_system.h" -#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" using testing::AtLeast; @@ -1337,9 +1336,6 @@ TEST(UtilityTest, PrepareDnsRefreshStrategy) { } TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - auto* time_source = new NiceMock(); context_.dispatcher_.time_system_.reset(time_source); @@ -1361,7 +1357,10 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { })); Registry::InjectFactory injector(factory); - config_.mutable_key_value_config()->mutable_config()->set_name("mock_key_value_store_factory"); + auto* key_value_config = config_.mutable_key_value_config()->mutable_config(); + key_value_config->set_name("mock_key_value_store_factory"); + key_value_config->mutable_typed_config()->PackFrom( + envoy::extensions::key_value::file_based::v3::FileBasedKeyValueStoreConfig()); initialize(); InSequence s; @@ -1471,9 +1470,6 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { } TEST_F(DnsCacheImplTest, CacheLoad) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - auto* time_source = new NiceMock(); context_.dispatcher_.time_system_.reset(time_source); @@ -1501,7 +1497,10 @@ TEST_F(DnsCacheImplTest, CacheLoad) { return ret; })); Registry::InjectFactory injector(factory); - config_.mutable_key_value_config()->mutable_config()->set_name("mock_key_value_store_factory"); + auto* key_value_config = config_.mutable_key_value_config()->mutable_config(); + key_value_config->set_name("mock_key_value_store_factory"); + key_value_config->mutable_typed_config()->PackFrom( + envoy::extensions::key_value::file_based::v3::FileBasedKeyValueStoreConfig()); initialize(); ASSERT(store != nullptr); diff --git a/test/extensions/filters/http/geoip/BUILD b/test/extensions/filters/http/geoip/BUILD index 98b555a13e5d..03ce9172a8ff 100644 --- a/test/extensions/filters/http/geoip/BUILD +++ b/test/extensions/filters/http/geoip/BUILD @@ -1,6 +1,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_package", + "envoy_proto_library", ) load( "//test/extensions:extensions_build_system.bzl", @@ -13,6 +14,11 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_proto_library( + name = "dummy", + srcs = ["dummy.proto"], +) + envoy_extension_cc_test( name = "config_test", size = "small", @@ -51,7 +57,10 @@ envoy_extension_cc_mock( name = "geoip_mocks", hdrs = ["mocks.h"], extension_names = ["envoy.filters.http.geoip"], - deps = ["//source/extensions/filters/http/geoip:provider_config"], + deps = [ + ":dummy_cc_proto", + "//source/extensions/filters/http/geoip:provider_config", + ], ) envoy_extension_cc_test_library( diff --git a/test/extensions/filters/http/geoip/config_test.cc b/test/extensions/filters/http/geoip/config_test.cc index 6b0471db7460..7c27734d1b91 100644 --- a/test/extensions/filters/http/geoip/config_test.cc +++ b/test/extensions/filters/http/geoip/config_test.cc @@ -94,7 +94,6 @@ MATCHER_P(HasAnonGeoHeadersSize, expected_size, "") { TEST(GeoipFilterConfigTest, GeoipFilterDefaultValues) { TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); DummyGeoipProviderFactory dummy_factory; Registry::InjectFactory registered(dummy_factory); std::string filter_config_yaml = R"EOF( @@ -102,7 +101,8 @@ TEST(GeoipFilterConfigTest, GeoipFilterDefaultValues) { city: "x-geo-city" provider: name: "envoy.geoip_providers.dummy" - typed_config: {} + typed_config: + "@type": type.googleapis.com/test.extensions.filters.http.geoip.DummyProvider )EOF"; GeoipFilterConfig filter_config; TestUtility::loadFromYaml(filter_config_yaml, filter_config); @@ -120,7 +120,6 @@ TEST(GeoipFilterConfigTest, GeoipFilterDefaultValues) { TEST(GeoipFilterConfigTest, GeoipFilterConfigWithCorrectProto) { TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); DummyGeoipProviderFactory dummy_factory; Registry::InjectFactory registered(dummy_factory); std::string filter_config_yaml = R"EOF( @@ -132,7 +131,8 @@ TEST(GeoipFilterConfigTest, GeoipFilterConfigWithCorrectProto) { anon_vpn: "x-anon-vpn" provider: name: "envoy.geoip_providers.dummy" - typed_config: {} + typed_config: + "@type": type.googleapis.com/test.extensions.filters.http.geoip.DummyProvider )EOF"; GeoipFilterConfig filter_config; TestUtility::loadFromYaml(filter_config_yaml, filter_config); @@ -151,14 +151,14 @@ TEST(GeoipFilterConfigTest, GeoipFilterConfigWithCorrectProto) { TEST(GeoipFilterConfigTest, GeoipFilterConfigMissingGeoHeaders) { TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); DummyGeoipProviderFactory dummy_factory; Registry::InjectFactory registered(dummy_factory); std::string filter_config_yaml = R"EOF( xff_config: xff_num_trusted_hops: 0 provider: - name: "envoy.geoip_providers.dummy" + typed_config: + "@type": type.googleapis.com/test.extensions.filters.http.geoip.DummyProvider )EOF"; GeoipFilterConfig filter_config; @@ -173,7 +173,6 @@ TEST(GeoipFilterConfigTest, GeoipFilterConfigMissingGeoHeaders) { TEST(GeoipFilterConfigTest, GeoipFilterConfigMissingProvider) { TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); DummyGeoipProviderFactory dummy_factory; Registry::InjectFactory registered(dummy_factory); std::string filter_config_yaml = R"EOF( @@ -194,7 +193,6 @@ TEST(GeoipFilterConfigTest, GeoipFilterConfigMissingProvider) { TEST(GeoipFilterConfigTest, GeoipFilterConfigUnknownProvider) { TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); DummyGeoipProviderFactory dummy_factory; Registry::InjectFactory registered(dummy_factory); std::string filter_config_yaml = R"EOF( @@ -213,7 +211,8 @@ TEST(GeoipFilterConfigTest, GeoipFilterConfigUnknownProvider) { EXPECT_THROW_WITH_MESSAGE( factory.createFilterFactoryFromProtoTyped(filter_config, "geoip", context), Envoy::EnvoyException, - "Didn't find a registered implementation for name: 'envoy.geoip_providers.unknown'"); + "Didn't find a registered implementation for 'envoy.geoip_providers.unknown' with type URL: " + "''"); } } // namespace diff --git a/test/extensions/filters/http/geoip/dummy.proto b/test/extensions/filters/http/geoip/dummy.proto new file mode 100644 index 000000000000..ed0e15d29275 --- /dev/null +++ b/test/extensions/filters/http/geoip/dummy.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +package test.extensions.filters.http.geoip; + +message DummyProvider { +} diff --git a/test/extensions/filters/http/geoip/geoip_filter_test.cc b/test/extensions/filters/http/geoip/geoip_filter_test.cc index 2dfa2697077e..035827901bc6 100644 --- a/test/extensions/filters/http/geoip/geoip_filter_test.cc +++ b/test/extensions/filters/http/geoip/geoip_filter_test.cc @@ -59,8 +59,6 @@ class GeoipFilterTest : public testing::Test { void initializeProviderFactory() { TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); Registry::InjectFactory registered(*dummy_factory_); } @@ -90,6 +88,8 @@ TEST_F(GeoipFilterTest, NoXffSuccessfulLookup) { city: "x-geo-city" provider: name: "envoy.geoip_providers.dummy" + typed_config: + "@type": type.googleapis.com/test.extensions.filters.http.geoip.DummyProvider )EOF"; initializeFilter(external_request_yaml); Http::TestRequestHeaderMapImpl request_headers; diff --git a/test/extensions/filters/http/geoip/mocks.h b/test/extensions/filters/http/geoip/mocks.h index 7fb49f398f34..ffc2c361d9ec 100644 --- a/test/extensions/filters/http/geoip/mocks.h +++ b/test/extensions/filters/http/geoip/mocks.h @@ -1,5 +1,8 @@ #include "source/extensions/filters/http/geoip/geoip_provider_config.h" +#include "test/extensions/filters/http/geoip/dummy.pb.h" +#include "test/extensions/filters/http/geoip/dummy.pb.validate.h" + #include "gmock/gmock.h" namespace Envoy { @@ -27,7 +30,7 @@ class DummyGeoipProviderFactory : public GeoipProviderFactory { std::string name() const override { return "envoy.geoip_providers.dummy"; } ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return std::make_unique(); + return std::make_unique(); } private: diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_integration_test.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_integration_test.cc index 51a252af2bb2..d36447a068ba 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_integration_test.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_integration_test.cc @@ -30,6 +30,8 @@ class TlsInspectorIntegrationTest : public testing::TestWithParam(); + return std::make_unique< + test::extensions::listener_managers::listener_manager::NonTerminalFilter>(); } std::string name() const override { return "non_terminal"; } }; TEST_P(ListenerManagerImplWithRealFiltersTest, TerminalNotLast) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - NonTerminalFilterFactory filter; Registry::InjectFactory registered(filter); @@ -621,6 +619,8 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, TerminalNotLast) { filter_chains: - filters: - name: non_terminal + typed_config: + "@type": type.googleapis.com/test.extensions.listener_managers.listener_manager.NonTerminalFilter name: foo )EOF"; @@ -680,9 +680,8 @@ class TestStatsConfigFactory : public Configuration::NamedNetworkFilterConfigFac } ProtobufTypes::MessagePtr createEmptyConfigProto() override { - // Using Struct instead of a custom per-filter empty config proto - // This is only allowed in tests. - return std::make_unique(); + return std::make_unique< + test::extensions::listener_managers::listener_manager::TestStatsFilter>(); } std::string name() const override { return "stats_test"; } @@ -700,9 +699,6 @@ class TestStatsConfigFactory : public Configuration::NamedNetworkFilterConfigFac }; TEST_P(ListenerManagerImplWithRealFiltersTest, StatsScopeTest) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - TestStatsConfigFactory filter; Registry::InjectFactory registered(filter); @@ -715,6 +711,8 @@ bind_to_port: false filter_chains: - filters: - name: stats_test + typed_config: + "@type": type.googleapis.com/test.extensions.listener_managers.listener_manager.TestStatsFilter name: foo )EOF"; @@ -728,9 +726,6 @@ bind_to_port: false } TEST_P(ListenerManagerImplWithRealFiltersTest, UsingAddressAsStatsPrefixForMultipleAddresses) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - TestStatsConfigFactory filter; Registry::InjectFactory registered(filter); @@ -748,6 +743,8 @@ bind_to_port: false filter_chains: - filters: - name: stats_test + typed_config: + "@type": type.googleapis.com/test.extensions.listener_managers.listener_manager.TestStatsFilter name: foo )EOF"; @@ -5919,18 +5916,14 @@ class OriginalDstTestConfigFactory : public Configuration::NamedListenerFilterCo } ProtobufTypes::MessagePtr createEmptyConfigProto() override { - // Using Struct instead of a custom per-filter empty config proto - // This is only allowed in tests. - return std::make_unique(); + return std::make_unique< + test::extensions::listener_managers::listener_manager::OriginalDstTestFilter>(); } std::string name() const override { return "test.listener.original_dst"; } }; TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterOutbound) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - #ifdef SOL_IP OriginalDstTestConfigFactory factory; Registry::InjectFactory registration(factory); @@ -5944,6 +5937,8 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterOutbound) { traffic_direction: OUTBOUND listener_filters: - name: "test.listener.original_dst" + typed_config: + "@type": type.googleapis.com/test.extensions.listener_managers.listener_manager.OriginalDstTestFilter )EOF", Network::Address::IpVersion::v4); @@ -5988,9 +5983,6 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterOutbound) { } TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstFilterStopsIteration) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - #if defined(WIN32) && defined(SOL_IP) OriginalDstTestConfigFactory factory; Registry::InjectFactory registration(factory); @@ -6004,6 +5996,8 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstFilterStopsIteration) traffic_direction: OUTBOUND listener_filters: - name: "test.listener.original_dst" + typed_config: + "@type": type.googleapis.com/test.extensions.listener_managers.listener_manager.OriginalDstTestFilter )EOF", Network::Address::IpVersion::v4); @@ -6043,9 +6037,6 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstFilterStopsIteration) } TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterInbound) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - #ifdef SOL_IP OriginalDstTestConfigFactory factory; Registry::InjectFactory registration(factory); @@ -6059,6 +6050,8 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterInbound) { traffic_direction: INBOUND listener_filters: - name: "test.listener.original_dst" + typed_config: + "@type": type.googleapis.com/test.extensions.listener_managers.listener_manager.OriginalDstTestFilter )EOF", Network::Address::IpVersion::v4); @@ -6123,16 +6116,13 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterIPv6) { } ProtobufTypes::MessagePtr createEmptyConfigProto() override { - // Using Struct instead of a custom per-filter empty config proto - // This is only allowed in tests. - return std::make_unique(); + return std::make_unique< + test::extensions::listener_managers::listener_manager::OriginalDstTestIPv6Filter>(); } std::string name() const override { return "test.listener.original_dstipv6"; } }; - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); OriginalDstTestConfigFactory factory; Registry::InjectFactory registration(factory); @@ -6144,6 +6134,8 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterIPv6) { name: foo listener_filters: - name: "test.listener.original_dstipv6" + typed_config: + "@type": type.googleapis.com/test.extensions.listener_managers.listener_manager.OriginalDstTestIPv6Filter )EOF", Network::Address::IpVersion::v6); diff --git a/test/extensions/transport_sockets/proxy_protocol/BUILD b/test/extensions/transport_sockets/proxy_protocol/BUILD index d615d59ce61c..7311e817b101 100644 --- a/test/extensions/transport_sockets/proxy_protocol/BUILD +++ b/test/extensions/transport_sockets/proxy_protocol/BUILD @@ -38,10 +38,12 @@ envoy_extension_cc_test( "//source/extensions/health_checkers/http:health_checker_lib", "//source/extensions/health_checkers/tcp:health_checker_lib", "//source/extensions/transport_sockets/proxy_protocol:upstream_config", + "//source/extensions/transport_sockets/raw_buffer:config", "//test/integration:http_integration_lib", "//test/integration:integration_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/listener/proxy_protocol/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/proxy_protocol/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/transport_sockets/raw_buffer/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/transport_sockets/proxy_protocol/proxy_protocol_integration_test.cc b/test/extensions/transport_sockets/proxy_protocol/proxy_protocol_integration_test.cc index af6675d5f3fe..f7c11c24359c 100644 --- a/test/extensions/transport_sockets/proxy_protocol/proxy_protocol_integration_test.cc +++ b/test/extensions/transport_sockets/proxy_protocol/proxy_protocol_integration_test.cc @@ -3,6 +3,8 @@ #include "envoy/config/core/v3/proxy_protocol.pb.h" #include "envoy/extensions/filters/listener/proxy_protocol/v3/proxy_protocol.pb.h" #include "envoy/extensions/transport_sockets/proxy_protocol/v3/upstream_proxy_protocol.pb.h" +#include "envoy/extensions/transport_sockets/raw_buffer/v3/raw_buffer.pb.h" +#include "envoy/extensions/transport_sockets/raw_buffer/v3/raw_buffer.pb.validate.h" #include "test/integration/http_integration.h" #include "test/integration/integration.h" @@ -24,10 +26,10 @@ class ProxyProtocolTcpIntegrationTest : public testing::TestWithParammutable_clusters(0)->mutable_transport_socket(); transport_socket->set_name("envoy.transport_sockets.upstream_proxy_protocol"); envoy::config::core::v3::TransportSocket inner_socket; - inner_socket.set_name(inner_socket_); + if (inner_tls_) { + envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls; + inner_socket.set_name("tls"); + inner_socket.mutable_typed_config()->PackFrom(tls); + } else { + envoy::extensions::transport_sockets::raw_buffer::v3::RawBuffer raw_buffer; + inner_socket.set_name("raw"); + inner_socket.mutable_typed_config()->PackFrom(raw_buffer); + } envoy::config::core::v3::ProxyProtocolConfig proxy_proto_config; proxy_proto_config.set_version(version_); envoy::extensions::transport_sockets::proxy_protocol::v3::ProxyProtocolUpstreamTransport @@ -70,7 +80,7 @@ class ProxyProtocolTcpIntegrationTest : public testing::TestWithParam void { std::string observed_data; @@ -187,8 +194,7 @@ TEST_P(ProxyProtocolTcpIntegrationTest, TestProxyProtocolHealthCheck) { // Test sending proxy protocol v2 TEST_P(ProxyProtocolTcpIntegrationTest, TestV2ProxyProtocol) { - setup(envoy::config::core::v3::ProxyProtocolConfig::V2, false, - "envoy.transport_sockets.raw_buffer"); + setup(envoy::config::core::v3::ProxyProtocolConfig::V2, false, false); initialize(); auto listener_port = lookupPort("listener_0"); @@ -246,11 +252,9 @@ class ProxyProtocolHttpIntegrationTest : public testing::TestWithParammutable_clusters(0)->mutable_transport_socket(); transport_socket->set_name("envoy.transport_sockets.upstream_proxy_protocol"); envoy::config::core::v3::TransportSocket inner_socket; - inner_socket.set_name(inner_socket_); + envoy::extensions::transport_sockets::raw_buffer::v3::RawBuffer raw_buffer; + inner_socket.set_name("raw"); + inner_socket.mutable_typed_config()->PackFrom(raw_buffer); envoy::config::core::v3::ProxyProtocolConfig proxy_proto_config; proxy_proto_config.set_version(version_); envoy::extensions::transport_sockets::proxy_protocol::v3::ProxyProtocolUpstreamTransport @@ -289,7 +295,6 @@ class ProxyProtocolHttpIntegrationTest : public testing::TestWithParam void { std::string observed_data; @@ -443,7 +445,9 @@ class ProxyProtocolTLVsIntegrationTest : public testing::TestWithParammutable_clusters(0)->mutable_transport_socket(); transport_socket->set_name("envoy.transport_sockets.upstream_proxy_protocol"); envoy::config::core::v3::TransportSocket inner_socket; - inner_socket.set_name("envoy.transport_sockets.raw_buffer"); + envoy::extensions::transport_sockets::raw_buffer::v3::RawBuffer raw_buffer; + inner_socket.set_name("raw"); + inner_socket.mutable_typed_config()->PackFrom(raw_buffer); envoy::config::core::v3::ProxyProtocolConfig proxy_protocol; proxy_protocol.set_version(envoy::config::core::v3::ProxyProtocolConfig::V2); diff --git a/test/extensions/transport_sockets/tcp_stats/BUILD b/test/extensions/transport_sockets/tcp_stats/BUILD index bbabcbb0efb2..67f79849bf86 100644 --- a/test/extensions/transport_sockets/tcp_stats/BUILD +++ b/test/extensions/transport_sockets/tcp_stats/BUILD @@ -36,9 +36,11 @@ envoy_extension_cc_test( extension_names = ["envoy.transport_sockets.tcp_stats"], deps = [ "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tcp_stats:config", "//test/integration:http_integration_lib", "//test/integration:integration_lib", + "@envoy_api//envoy/extensions/transport_sockets/raw_buffer/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/tcp_stats/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/transport_sockets/tcp_stats/tcp_stats_integration_test.cc b/test/extensions/transport_sockets/tcp_stats/tcp_stats_integration_test.cc index 2b8d37ee37e2..47f8b59c1608 100644 --- a/test/extensions/transport_sockets/tcp_stats/tcp_stats_integration_test.cc +++ b/test/extensions/transport_sockets/tcp_stats/tcp_stats_integration_test.cc @@ -1,4 +1,6 @@ #if defined(__linux__) +#include "envoy/extensions/transport_sockets/raw_buffer/v3/raw_buffer.pb.h" +#include "envoy/extensions/transport_sockets/raw_buffer/v3/raw_buffer.pb.validate.h" #include "envoy/extensions/transport_sockets/tcp_stats/v3/tcp_stats.pb.h" #include "test/integration/integration.h" @@ -18,6 +20,8 @@ class TcpStatsSocketIntegrationTest : public testing::TestWithParamPackFrom(raw_buffer); envoy::extensions::transport_sockets::tcp_stats::v3::Config proto_config; proto_config.mutable_transport_socket()->MergeFrom(inner_socket); diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 5a2bbfa161fa..0e767fb33833 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -74,9 +74,6 @@ BaseIntegrationTest::BaseIntegrationTest(const InstanceConstSharedPtrFn& upstrea })); ON_CALL(factory_context_.server_context_, api()).WillByDefault(ReturnRef(*api_)); ON_CALL(factory_context_, statsScope()).WillByDefault(ReturnRef(*stats_store_.rootScope())); - // Allow extension lookup by name in the integration tests. - config_helper_.addRuntimeOverride("envoy.reloadable_features.no_extension_lookup_by_name", - "false"); #ifndef ENVOY_ADMIN_FUNCTIONALITY config_helper_.addConfigModifier( diff --git a/test/integration/filter_manager_integration_test.cc b/test/integration/filter_manager_integration_test.cc index cfa5cdccd74c..8d758b288751 100644 --- a/test/integration/filter_manager_integration_test.cc +++ b/test/integration/filter_manager_integration_test.cc @@ -420,7 +420,10 @@ class InjectDataToFilterChainIntegrationTest tick_interval_ms: 1 max_chunk_length: 5 )EOF" - : ""; + : R"EOF( + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + )EOF"; } }; @@ -434,6 +437,8 @@ class InjectDataWithEchoFilterIntegrationTest : public InjectDataToFilterChainIn filter_chains: filters: - name: envoy.filters.network.echo + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.echo.v3.Echo )EOF"); } @@ -617,7 +622,10 @@ class InjectDataWithHttpConnectionManagerIntegrationTest tick_interval_ms: 1 max_chunk_length: 10 )EOF" - : ""; + : R"EOF( + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + )EOF"; } }; diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 7dbb1dd7e1be..802a4a0391be 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -341,6 +341,9 @@ HttpIntegrationTest::HttpIntegrationTest(Http::CodecType downstream_protocol, // lookupPort calls. config_helper_.renameListener("http"); config_helper_.setClientCodec(typeToCodecType(downstream_protocol_)); + // Allow extension lookup by name in the integration tests. + config_helper_.addRuntimeOverride("envoy.reloadable_features.no_extension_lookup_by_name", + "false"); } void HttpIntegrationTest::useAccessLog( diff --git a/test/integration/listener_lds_integration_test.cc b/test/integration/listener_lds_integration_test.cc index d72316ad2fcc..215dd438d70f 100644 --- a/test/integration/listener_lds_integration_test.cc +++ b/test/integration/listener_lds_integration_test.cc @@ -1652,8 +1652,9 @@ class RebalancerTest : public testing::TestWithParamset_value(true); // Note that the below original_dst is replaced by FakeOriginalDstListenerFilter at the // link time. - src_listener_config.add_listener_filters()->set_name( - "envoy.filters.listener.original_dst"); + auto& filter = *src_listener_config.add_listener_filters(); + filter.set_name("envoy.filters.listener.original_dst"); + filter.mutable_typed_config()->PackFrom(ProtobufWkt::Struct()); auto& virtual_listener_config = *bootstrap.mutable_static_resources()->add_listeners(); virtual_listener_config = src_listener_config; virtual_listener_config.mutable_use_original_dst()->set_value(false); diff --git a/test/integration/multiplexed_upstream_integration_test.cc b/test/integration/multiplexed_upstream_integration_test.cc index cd532484a67d..1c21629e5982 100644 --- a/test/integration/multiplexed_upstream_integration_test.cc +++ b/test/integration/multiplexed_upstream_integration_test.cc @@ -888,7 +888,6 @@ TEST_P(MultiplexedUpstreamIntegrationTest, UpstreamDisconnectDuringEarlyData) { if (upstreamProtocol() != Http::CodecType::HTTP3) { return; } - Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.no_extension_lookup_by_name", false); // Register and config this factory to control upstream QUIC handshakes. QuicFailHandshakeCryptoServerStreamFactory crypto_stream_factory; @@ -897,8 +896,9 @@ TEST_P(MultiplexedUpstreamIntegrationTest, UpstreamDisconnectDuringEarlyData) { crypto_stream_factory.setFailHandshake(false); envoy::config::listener::v3::QuicProtocolOptions options; - options.mutable_crypto_stream_config()->set_name( - "envoy.quic.crypto_stream.server.fail_handshake"); + auto* crypto_stream_config = options.mutable_crypto_stream_config(); + crypto_stream_config->set_name("envoy.quic.crypto_stream.server.fail_handshake"); + crypto_stream_config->mutable_typed_config()->PackFrom(ProtobufWkt::Struct()); mergeOptions(options); initialize(); diff --git a/test/integration/socket_interface_integration_test.cc b/test/integration/socket_interface_integration_test.cc index 9c19b388bbd2..d3c6ebda926a 100644 --- a/test/integration/socket_interface_integration_test.cc +++ b/test/integration/socket_interface_integration_test.cc @@ -40,6 +40,8 @@ default_socket_interface: "envoy.extensions.network.socket_interface.default_soc descriptors: [{"key": "foo", "value": "bar"}] filters: name: envoy.filters.network.echo + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.echo.v3.Echo )EOF"); } }; diff --git a/test/integration/tcp_conn_pool_integration_test.cc b/test/integration/tcp_conn_pool_integration_test.cc index e353244f2203..dc1ee5ed0b86 100644 --- a/test/integration/tcp_conn_pool_integration_test.cc +++ b/test/integration/tcp_conn_pool_integration_test.cc @@ -122,6 +122,7 @@ class TcpConnPoolIntegrationTest : public testing::TestWithParam registered(factory); @@ -790,9 +787,6 @@ TEST_P(ServerStatsTest, FlushStats) { } TEST_P(ServerInstanceImplTest, FlushStatsOnAdmin) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - CustomStatsSinkFactory factory; Registry::InjectFactory registered(factory); auto server_thread = @@ -819,9 +813,6 @@ TEST_P(ServerInstanceImplTest, FlushStatsOnAdmin) { } TEST_P(ServerInstanceImplTest, ConcurrentFlushes) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - CustomStatsSinkFactory factory; Registry::InjectFactory registered(factory); @@ -1580,9 +1571,6 @@ class CallbacksStatsSinkFactory : public Server::Configuration::StatsSinkFactory // lifecycle callback is also used to ensure that the cluster update callback is freed during // Server::Instance's destruction. See issue #9292 for more details. TEST_P(ServerInstanceImplTest, CallbacksStatsSinkTest) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); - CallbacksStatsSinkFactory factory; Registry::InjectFactory registered(factory); diff --git a/test/server/test_data/server/callbacks_stats_sink_bootstrap.yaml b/test/server/test_data/server/callbacks_stats_sink_bootstrap.yaml index 03fe2ae33da4..e11dcf8ea72a 100644 --- a/test/server/test_data/server/callbacks_stats_sink_bootstrap.yaml +++ b/test/server/test_data/server/callbacks_stats_sink_bootstrap.yaml @@ -16,4 +16,6 @@ admin: port_value: 0 stats_sinks: - name: envoy.callbacks_stats_sink + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct stats_flush_interval: 1s diff --git a/test/server/test_data/server/stats_sink_bootstrap.yaml b/test/server/test_data/server/stats_sink_bootstrap.yaml index 9e94fa854637..5ddf93b46e39 100644 --- a/test/server/test_data/server/stats_sink_bootstrap.yaml +++ b/test/server/test_data/server/stats_sink_bootstrap.yaml @@ -16,4 +16,6 @@ admin: port_value: 0 stats_sinks: - name: envoy.custom_stats_sink + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct stats_flush_interval: 1s diff --git a/test/server/test_data/server/stats_sink_manual_flush_bootstrap.yaml b/test/server/test_data/server/stats_sink_manual_flush_bootstrap.yaml index 47d41309d00f..517aeade1a61 100644 --- a/test/server/test_data/server/stats_sink_manual_flush_bootstrap.yaml +++ b/test/server/test_data/server/stats_sink_manual_flush_bootstrap.yaml @@ -16,5 +16,7 @@ admin: port_value: 0 stats_sinks: - name: envoy.custom_stats_sink + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct stats_flush_on_admin: true enable_dispatcher_stats: true From c127fe34cba8f8228c7a3d21da628e02622db6c4 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Tue, 6 Jun 2023 17:03:51 -0400 Subject: [PATCH 471/740] admin: add histogram rendering for html mode. (#27468) Commit Message: Renders histograms in the stats HTML view. Screenshot: Screenshot 2023-05-29 at 11 39 15 PM This is the next phase in https://github.com/envoyproxy/envoy/pull/26472 -- remaining phases are further refactors, as well as sorting histograms by change-count for active-html mode, and make web navigation work better. This includes unit tests for the new histogram rendering, as well as a mechanism to render histograms.js on its own. Additional Description: Risk Level: low Testing: //test/..., test/integration/admin_html/web_test.sh for browser-based tests on FF, and manually ran on Chrome as well. Docs Changes: n/a -- the histograms were previously rendered as text with format=html; now they are rendered graphically, but detail about this is not in the docs currently. Release Notes: Platform Specific Features: Signed-off-by: Joshua Marantz --- source/server/admin/admin_html_util.cc | 4 +- source/server/admin/html/BUILD | 4 +- source/server/admin/html/active_stats.js | 18 +- source/server/admin/html/admin.css | 116 ++++ .../server/admin/html/generate_admin_html.sh | 16 + source/server/admin/html/histograms.js | 628 ++++++++++++++++++ source/server/admin/stats_handler.cc | 46 +- source/server/admin/stats_html_render.cc | 30 +- source/server/admin/stats_html_render.h | 15 +- test/integration/admin_html/BUILD | 1 + .../admin_html/active_stats_test.js | 4 +- .../admin_html/histograms_test.html | 9 + .../integration/admin_html/histograms_test.js | 154 +++++ test/integration/admin_html/test_server.cc | 13 +- .../admin_html/test_server_test.sh | 1 + test/integration/admin_html/web_test.html | 2 + test/integration/admin_html/web_test.js | 27 + test/server/admin/stats_html_render_test.cc | 6 +- 18 files changed, 1047 insertions(+), 47 deletions(-) create mode 100644 source/server/admin/html/histograms.js create mode 100644 test/integration/admin_html/histograms_test.html create mode 100644 test/integration/admin_html/histograms_test.js diff --git a/source/server/admin/admin_html_util.cc b/source/server/admin/admin_html_util.cc index a7ba82c11d13..410e429e6d74 100644 --- a/source/server/admin/admin_html_util.cc +++ b/source/server/admin/admin_html_util.cc @@ -54,10 +54,11 @@ namespace Server { namespace { class BuiltinResourceProvider : public AdminHtmlUtil::ResourceProvider { public: - BuiltinResourceProvider() { + BuiltinResourceProvider() : histogram_js_(absl::StrCat(HistogramsJs1, HistogramsJs2)) { map_["admin_head_start.html"] = AdminHtmlStart; map_["admin.css"] = AdminCss; map_["active_stats.js"] = AdminActiveStatsJs; + map_["histograms.js"] = histogram_js_; map_["active_params.html"] = AdminActiveParamsHtml; } @@ -66,6 +67,7 @@ class BuiltinResourceProvider : public AdminHtmlUtil::ResourceProvider { } private: + const std::string histogram_js_; absl::flat_hash_map map_; }; diff --git a/source/server/admin/html/BUILD b/source/server/admin/html/BUILD index ec58ce1775a0..1e8dc1521644 100644 --- a/source/server/admin/html/BUILD +++ b/source/server/admin/html/BUILD @@ -17,11 +17,12 @@ genrule( "admin.css", "active_params.html", "active_stats.js", + "histograms.js", ], outs = ["admin_html_gen.h"], cmd = "./$(location :generate_admin_html.sh) \ $(location admin_head_start.html) $(location admin.css) $(location active_stats.js) \ - $(location active_params.html) > $@", + $(location active_params.html) $(location histograms.js) > $@", tools = [":generate_admin_html.sh"], ) @@ -34,5 +35,6 @@ filegroup( "active_stats.js", "admin.css", "admin_head_start.html", + "histograms.js", ], ) diff --git a/source/server/admin/html/active_stats.js b/source/server/admin/html/active_stats.js index 0d02bb798b5e..ca6bf6723981 100644 --- a/source/server/admin/html/active_stats.js +++ b/source/server/admin/html/active_stats.js @@ -28,6 +28,11 @@ const nameStatsMap = new Map(); */ let activeStatsPreElement = null; +/** + * A div into which we render histograms. + */ +let activeStatsHistogramsDiv = null; + /** * A small div for displaying status and error messages. */ @@ -65,8 +70,10 @@ function initHook() { statusDiv.className = 'error-status-line'; activeStatsPreElement = document.createElement('pre'); activeStatsPreElement.id = 'active-content-pre'; + activeStatsHistogramsDiv = document.createElement('div'); document.body.appendChild(statusDiv); document.body.appendChild(activeStatsPreElement); + document.body.appendChild(activeStatsHistogramsDiv); loadStats(); } @@ -76,18 +83,18 @@ function initHook() { async function loadStats() { const makeQueryParam = (name) => name + '=' + encodeURIComponent( document.getElementById(paramIdPrefix + name).value); - const params = ['filter', 'type', 'histogram_buckets']; + const params = ['filter', 'type']; const href = window.location.href; // Compute the fetch URL prefix based on the current URL, so that the admin // site can be hosted underneath a site-specific URL structure. - const stats_pos = href.indexOf('/stats?'); - if (stats_pos == -1) { + const statsPos = href.indexOf('/stats?'); + if (statsPos == -1) { statusDiv.textContent = 'Cannot find /stats? in ' + href; return; } - const prefix = href.substring(0, stats_pos); - const url = prefix + '/stats?format=json&usedonly&' + + const prefix = href.substring(0, statsPos); + const url = prefix + '/stats?format=json&usedonly&histogram_buckets=detailed&' + params.map(makeQueryParam).join('&'); try { @@ -175,6 +182,7 @@ function renderStats(data) { } sortedStats.push(statRecord); } + renderHistograms(activeStatsHistogramsDiv, data); // Sorts all the stats. This is inefficient; we should just pick the top N // based on field "active-max-display-count" and sort those. The best diff --git a/source/server/admin/html/admin.css b/source/server/admin/html/admin.css index e8f99cc2cc5e..bb1af5f0cbd5 100644 --- a/source/server/admin/html/admin.css +++ b/source/server/admin/html/admin.css @@ -46,3 +46,119 @@ font-size: medium; color: #c00; } + +.histogram-graphics { + height: 40px; + position: relative; + margin-top: 15px; + /* + * TODO(jmarantz): enable resizing; to finish this we need to add a ResizeObserver + * to keep a map of the resized histograms for use when refreshing them. + * + * resize: vertical; + * overflow: auto; + */ +} + +.histogram-annotations { + height: 40px; + position: relative; + margin-bottom: 10px; +} + +.histogram-labels { + background-color: #ddd; + position: relative; + height: 15px; +} + +.histogram-name { + font-family: sans-serif; + font-size: 14px; + font-weight: bold; + margin-top: 10px; + margin-bottom: 8px; +} + +.histogram-no-data { + font-family: sans-serif; + font-size: 14px; + font-style: italic; + margin-top: 10px; + margin-bottom: 8px; + margin-left: 10px; +} + +.histogram-bucket { + bottom: 0; + background-color: #e6d7ff; + position: absolute; + border-color: purple; + border-width: 1px; + border-style: solid; + width: 2%; + z-index: 1; +} + +.histogram-popup { + position: absolute; + background-color: yellow; + border-width: 1px; + border-style: solid; + border-color: black; + visibility: hidden; + z-index: 3; + top: 70px; + padding: 3px; + /*Imargin-left: -1%;*/ +} + +.histogram-popup div { + font-family: sans-serif; + font-size: 14px; + overflow: hidden; + text-align: center; +} + +.histogram-percentile { + width: 5px; + background-color: #306844; + height: 30%; + width: 4px; + top: 0; + position: absolute; +} + +.histogram-interval { + width: 5px; + background-color: #e06844; + height: 30%; + width: 4px; + margin-left: 0.5%; + top: 0; + position: absolute; +} + +.bucket-label { + position: absolute; + bottom: 1%; + font-family: sans-serif; + font-size: 12px; + margin-bottom: 1px; + text-align: center; +} + +.percentile-label { + position: absolute; + font-family: sans-serif; + font-size: 12px; + z-index: 2; +} + +.histogram-labels span { + position: absolute; + bottom: 0; + font-family: sans-serif; + font-size: 12px; + overflow-x: visible; +} diff --git a/source/server/admin/html/generate_admin_html.sh b/source/server/admin/html/generate_admin_html.sh index 51352f0b8d0d..36041855afae 100755 --- a/source/server/admin/html/generate_admin_html.sh +++ b/source/server/admin/html/generate_admin_html.sh @@ -12,3 +12,19 @@ echo ')EOF";' echo 'constexpr absl::string_view AdminActiveParamsHtml = R"EOF(' cat "$4" echo ')EOF";' + +# The Windows compiler complains histograms.js is too big to fit into a string +# constant, so generate the histograms implementation in two chunks. These will +# be combined in source/server/admin/admin_html_util.cc when constructing +# BuiltinResourceProvider. We do this even when not on Windows for consistency. +lines=$(wc -l < "$5") +first_lines=$((lines / 2)) +next_lines=$((first_lines + 1)) + +echo 'constexpr absl::string_view HistogramsJs1 = R"EOF(' +head --lines="$first_lines" "$5" +echo ')EOF";' + +echo 'constexpr absl::string_view HistogramsJs2 = R"EOF(' +tail --lines=+"$next_lines" "$5" +echo ')EOF";' diff --git a/source/server/admin/html/histograms.js b/source/server/admin/html/histograms.js new file mode 100644 index 000000000000..f60357999204 --- /dev/null +++ b/source/server/admin/html/histograms.js @@ -0,0 +1,628 @@ +// Functions to render a JSON histogram representation into an HTML/CSS. +// There are several ways to do this: +// 1. lay out in CSS with flex-boxes +// 2. draw using SVG +// 3. render as `divs` with pixel positions +// 4. render as `divs` with percentage positions +// This implements option #4. There are pros/cons to each of these. The benefits of #4: +// 1. The user can get a clearer picture by making the window bigger, without having +// to re-layout the graphics. +// 2. The `divs` can be made sensitive to mouse enter/leave to pop up +// more detail +// There are also drawbacks: +// 1. Need to write graphics layout code, and think about coordinate systems, resulting +// in several hundred lines of JavaScript. +// 2. Some risk of having the text look garbled in some scenarios. This appears to +// to be mitigated with heuristics and use of popup elements when more than one +// percentile or interval-data falls within a bucket. + +const constants = { + // Horizontal spacing between buckets, expressed in `VPX` (Virtual Pixels) in a + // coordinate system invented to make arithmetic and debugging easier. + marginWidthVpx: 20, + + // Horizontal spacing between the edge of the window and the histogram buckets, + // expressed as a fraction. + outerMarginFraction: 0.01, + + // The minimum height of a bucket bar, expressed as a percentage of the + // configured height of a configured bar. By giving a histogram with count=1 + // a minimum height we make it easier for the mouse to hover over it, in + // order to pop up more detail. + baseHeightFraction: 0.03, + + // If there are too many buckets, per-bucket label text gets too dense and the + // text becomes illegible, so skip some if needed. We always put the range in + // the popup, and when skipped, we'll put the count in the popup as well, in + // addition to any percentiles or interval ranges. + maxBucketsWithText: 20, +}; + + +const globalState = { + // Holds a function to be called when the mouse leaves a popup. This is + // null when there is no visible popup. + pendingLeave: null, + + // Holds the timer object for the currently-visible popup. This is + // maintained so we can cancel the timeout if the user moves to a new + // bucket, in which case we'll immediately hide the popup so we can + // display the new one. + pendingTimeout: null, + + // Holds the histogram bucket object that is currently highlighted, + // which makes it possible to erase the highlight after the mouse moves + // out of it + highlightedBucket: null, +}; + + +/** + * Formula to compute bucket width based on number of buckets. This formula + * was derived from trial and error, to improve the appearance of the display. + * Consider this table of values: + * numBuckets `bucketWidthVpx` Ratio vs `marginWidthVpx` + * 1 2 1:10 + * 2 21 1:2 + * 3 27 2:3 + * 10 36 1.8:1 + * 20 38 1.9:1 + * These ratios appear to look good across a wide range of numBuckets. + * + * @param {number} numBuckets the number of buckets in the histogram. + * @return {number} The bucket width in virtual pixels. + */ +function computeBucketWidthVpx(numBuckets) { + return 40 - 38/numBuckets; +} + + +/** + * Top-level entry point to render all the detailed histograms found in a JSON + * stats response. + * + * @param {!Element} histogramDiv the element in which to render the histograms. + * @param {!Object} data the stats JSON structure obtained from the server /stats method. + */ +function renderHistograms(histogramDiv, data) { // eslint-disable-line no-unused-vars + histogramDiv.replaceChildren(); + for (stat of data.stats) { + const histograms = stat.histograms; + if (histograms) { + if (histograms.supported_percentiles && histograms.details) { + for (histogram of histograms.details) { + renderHistogram(histogramDiv, histograms.supported_percentiles, histogram); + } + } + } + } +} + + +/** + * Formats a number using up to 2 decimal places. See + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/NumberFormat/NumberFormat + */ +const format = Intl.NumberFormat('en', { // eslint-disable-line new-cap + notation: 'compact', + maximumFractionDigits: 2, +}).format; + + +// Formats a percentage using up to 2 decimal places. These are +// used for writing CSS percents. +const formatPercent = Intl.NumberFormat('en', { // eslint-disable-line new-cap + style: 'percent', + maximumFractionDigits: 2, +}).format; + + +/** + * Formats a range for a histogram bucket, using inclusive/beginning and + * exclusive end, computed from the width. + * + * @param {number} lowerBound the lower bound of the range. + * @param {number} width the range. + * @return {string} A formatted string of the form "[lowerBound, upperBound)". + */ +function formatRange(lowerBound, width) { + return '[' + format(lowerBound) + ', ' + format(lowerBound + width) + ')'; +} + + +/** + * Generates a function to render histogram information, capturing the + * variables passed in. This is used to establish a mouse-enter list on + * the histogram bucket, but without pulling in the entire drawing context. + * + * @param {!Element} detailPopup the popup div for this histogram. + * @param {string} bucketPosPercent The bucket position, expressed as a string percentage. + * @param {!Object} bucket the bucket record from JSON, augmented with an annotations list + * @param {!Element} bucketSpan a span element for the bucket; used to color it yellow. + * @param {boolean} showingCount whether the the count is shown on the histogram itself, + * which dictates whether we will also show it in the popup. + * @return {!Function} a function to call when the mouse enters the span. + */ + function showPopupFn(detailPopup, bucketPosPercent, bucket, bucketSpan, showingCount) { + return (event) => { + if (globalState.pendingTimeout) { + window.clearTimeout(globalState.pendingTimeout); + globalState.pendingLeave(); + } + detailPopup.style.left = bucketPosPercent; + detailPopup.style.visibility = 'visible'; + bucketSpan.style.backgroundColor = 'yellow'; + highlightedBucket = bucketSpan; + + detailPopup.replaceChildren(); + appendNewElement(detailPopup, 'div').textContent = formatRange(bucket.lower_bound, bucket.width); + if (!showingCount) { + appendNewElement(detailPopup, 'div').textContent = 'count=' + bucket.count; + } + if (bucket.annotations) { + for (annotation of bucket.annotations) { + const span = appendNewElement(detailPopup, 'div'); + span.textContent = annotation.detail(); + } + } + }; +} + + +/** + * Generates a timeout function for the provided popup div. + * + * @param {!Element} detailPopup the popup div. + * @return {!Function} a function to be called when the mouse leaves the span. + */ +function timeoutFn(detailPopup) { + return (event) => { + globalState.pendingLeave = leaveHandlerFn(detailPopup); + globalState.pendingTimeout = window.setTimeout(globalState.pendingLeave, 2000); + }; +} + + +/** + * Generates a handler function to be called when the mouse leaves + * a popup. + * + * @param {!Element} detailPopup the popup div. + * @return {!Function} a function to hide the popup. + */ +function leaveHandlerFn(detailPopup) { + return (event) => { + globalState.pendingTimeout = null; + globalState.pendingLeave = null; + detailPopup.style.visibility = 'hidden'; + if (highlightedBucket) { + highlightedBucket.style.backgroundColor = '#e6d7ff'; + highlightedBucket = null; + } + }; +} + + +/** + * When the user moves the mouse into the popup, we cancel the timer + * to hide the popup, held in globalState.pendingTimeout. However the + * globalState.pendingLeave function is left in place so a timeout can + * be re-established when the mouse leaves the popup. + */ +function enterPopup() { + // If the mouse enters the popup then cancel the timer that would + // erase it. This should make it easier to cut&paste the contents + // of the popup. + if (globalState.pendingTimeout) { + window.clearTimeout(globalState.pendingTimeout); + globalState.pendingTimeout = null; + // The 'leave' handler needs to remain -- we'll call that 2 seconds + // after the mouse leaves the popup. + } +} + + +/** + * When the user moves the mouse out of the popup, we re-enable the + * timeout that was (a) previously established when the mouse entered + * the bucket span, and (b) disabled when the mouse entered the popup. + */ +function leavePopup() { + globalState.pendingTimeout = window.setTimeout(globalState.pendingLeave, 2000); +} + + +/** + * Helper function to create an element, append it to a parent, and optionally + * install a CSS class. + * + * @param {!Element} parent the parent element. + * @param {string} type the HTML element type, e.g. 'div'. + * @param {?string} className optional CSS class name. + * @return {!Element} the new element. + */ +function appendNewElement(parent, type, className) { + const element = document.createElement(type); + if (className) { + element.className = className; + } + parent.appendChild(element); + return element; +} + + +/** + * Assigns percentiles and intervals to buckets. We do not expect percentiles or + * intervals. If any occur, they will be assigned to the first or last bucket. + * + * We will only consider the lower_bound of interval-buckets and will ignore the + * width of those buckets. + * + * We will assign intervals and percentiles to the bucket with the highest + * lower-bound that is not greater than the interval lower_bound or percentile + * value. + * + * @param {!Object} histogram JSON structure for a histogram. + * @param {!Array} supportedPercentiles Array of supported histograms. + * @return {number} the maximum count across all buckets. + */ +function assignPercentilesAndIntervalsToBuckets(histogram, supportedPercentiles) { + let maxCount = 0; + let percentileIndex = 0; + let intervalIndex = 0; + const percentileValues = histogram.percentiles; + let nextBucket = histogram.totals[0]; + + for (let i = 0; i < histogram.totals.length; ++i) { + const bucket = nextBucket; + if (i < histogram.totals.length - 1) { + nextBucket = histogram.totals[i + 1]; + } else { + nextBucket = null; + } + maxCount = Math.max(maxCount, bucket.count); + bucket.annotations = []; + + // Attach percentile records with values between the previous bucket and + // this one. We will drop percentiles before the first bucket. Thus each + // bucket starting with the second one will have a 'percentiles' property + // with pairs of [percentile, value], which can then be used while + // rendering the bucket. + for (; percentileIndex < percentileValues.length; ++percentileIndex) { + const percentileValue = percentileValues[percentileIndex].cumulative; + if (nextBucket && percentileValue >= nextBucket.lower_bound) { + break; // do not increment index; re-consider percentile for next bucket. + } + bucket.annotations.push(new Percentile( + percentileValue, supportedPercentiles[percentileIndex])); + } + + for (; intervalIndex < histogram.intervals.length; ++intervalIndex) { + const interval = histogram.intervals[intervalIndex]; + if (nextBucket && interval.lower_bound >= nextBucket.lower_bound) { + break; // do not increment index; re-consider interval for next bucket. + } + bucket.annotations.push(new Interval(interval.lower_bound, interval.width, interval.count)); + + // It's unlikely that an interval bucket value will increase the overall maxCount + // but just to be sure we'll scan through them. + maxCount = Math.max(maxCount, interval.count); + } + + if (bucket.annotations.length > 0) { + bucket.annotations.sort((a, b) => { + if (a.value == b.value) { + if (a.cssClass() == b.cssClass()) { + return 0; + } + if (a.cssClass() == 'histogram-percentile') { + return -1; + } + return 1; + } + return a.value - b.value; + }); + let aa = 0; + for (a of bucket.annotations) { + console.log('[' + aa++ + ']: ' + a.value + ' (' + a.cssClass() + ')'); + } + } + } + return maxCount; +} + +/** + * Represents an annotation, which can be a percentile or interval-bucket. + */ +class Annotation { + /** + * @param {number} value The numeric value of the annotation. + */ + constructor(value) { + this.value = value; + } + + /** + * Returns the CSS class name used for rendering this annotation + */ + cssClass() { + throw new Error('pure virtual function cssClass'); + } + + /** + * Returns formats the annotation as an abbreviated string. + */ + toString() { + throw new Error('pure virtual function toString'); + } + + /** + * Returns formats the annotation with greater detail, suitable for a popup. + */ + detail() { + throw new Error('pure virtual function detail'); + } +} + +/** + * Represents a percentile. + */ +class Percentile extends Annotation { + /** + * @param {number} value The numeric value of the annotation. + * @param {number} percentile The percentile number. + */ + constructor(value, percentile) { + super(value); + this.percentile = percentile; + } + + /** + * @return {string} the css class name. + */ + cssClass() { + return 'histogram-percentile'; + } + + /** + * @return {string} brief format for graphical annotation. + */ + toString() { + return 'P' + this.percentile; + } + + /** + * @return {string} detailed for popup. + */ + detail() { + return `P${this.percentile}: ${format(this.value)}`; + } +} + +/** + * Represents a interval bucket. + */ +class Interval extends Annotation { + /** + * @param {number} value The lower bound of the bucket. + * @param {number} width The width of the bucket. + * @param {number} count The height of the bucket. + */ + constructor(value, width, count) { + super(value); + this.width = width; + this.count = count; + } + + /** + * @return {string} the css class name. + */ + cssClass() { + return 'histogram-interval'; + } + + /** + * @return {string} brief format for graphical annotation. + */ + toString() { + return 'i:' + this.count; + } + + /** + * @return {string} detailed for popup. + */ + detail() { + return 'Interval ' + formatRange(this.value, this.width) + ': ' + this.count; + } +} + +/** + * Captures context needed to lay out the histogram graphically. + */ +class Painter { + /** + * @param {!Element} div the HTML element into which to draw the histogram + * @param {number} numBuckets the number of buckets. + * @param {number} maxCount the maximum count for all buckets. + */ + constructor(div, numBuckets, maxCount) { + this.numBuckets = numBuckets; + this.maxCount = maxCount; + this.leftVpx = constants.marginWidthVpx; + this.bucketWidthVpx = computeBucketWidthVpx(numBuckets); + this.widthVpx = (numBuckets * this.bucketWidthVpx + + ((numBuckets + 1) * constants.marginWidthVpx) * + (1 + 2*constants.outerMarginFraction)); + this.textIntervalIndex = 0; + this.bucketWidthPercent = formatPercent(this.vpxToWidth(this.bucketWidthVpx)); + + this.graphics = appendNewElement(div, 'div', 'histogram-graphics'); + this.labels = appendNewElement(div, 'div', 'histogram-labels'); + this.annotationsDiv = appendNewElement(div, 'div', 'histogram-annotations'); + + // We have business logic to ensure only be one popup div is visible at a + // time. However, we need a separate popup div for each histogram + // so that they can be positioned relative to the histogram's graphics. + this.detailPopup = appendNewElement(this.graphics, 'div', 'histogram-popup'); + this.detailPopup.addEventListener('mouseenter', enterPopup); + this.detailPopup.addEventListener('mouseleave', leavePopup); + + this.textInterval = Math.ceil(numBuckets / constants.maxBucketsWithText); + } + + /** + * @param {!Object} bucket the JSON info for the current bucket. + * @param {?Object} nextBucket the JSON info for next bucket, or null if + * bucket is the last one. + * @param {!Annotation} annotation an annotation to draw. + */ + drawAnnotation(bucket, nextBucket, annotation) { + // Find the ideal place to draw the percentile bar, by linearly + // interpolating between the current bucket and the previous bucket. + // We know that the next bucket does not come into play because + // the percentiles held underneath a bucket are based on a value that + // is at most as large as the current bucket. + let percentileVpx = this.leftVpx; + const bucketDelta = nextBucket ? (nextBucket.lower_bound - bucket.lower_bound) : bucket.width; + if (bucketDelta > 0) { + let widthVpx = this.bucketWidthVpx; + if (nextBucket) { + widthVpx += constants.marginWidthVpx; + } + const nextVpx = this.leftVpx + widthVpx; + const weight = (bucket.lower_bound + bucketDelta - annotation.value) / bucketDelta; + percentileVpx = weight * this.leftVpx + (1 - weight) * nextVpx; + } + + // We always put the marker proportionally between this bucket and + // the next one. + const span = appendNewElement(this.annotationsDiv, 'span', annotation.cssClass()); + const percentilePercent = formatPercent(this.vpxToPosition(percentileVpx)); + span.style.left = percentilePercent; + + // Don't draw textual labels for the percentiles and intervals if there are + // more than one: they'll just get garbled. The user can over over the + // bucket to see the detail. + if (bucket.annotations.length == 1) { + const percentilePLabel = appendNewElement(this.annotationsDiv, 'span', 'percentile-label'); + percentilePLabel.style.bottom = 0; + percentilePLabel.textContent = annotation.toString(); + percentilePLabel.style.left = percentilePercent; + + const percentileVLabel = appendNewElement(this.annotationsDiv, 'span', 'percentile-label'); + percentileVLabel.style.bottom = '30%'; + percentileVLabel.textContent = format(annotation.value); + percentileVLabel.style.left = percentilePercent; + } + } + + /** + * Draws a bucket. + * + * @param {!Object} bucket the bucket to draw. + */ + drawBucket(bucket) { + this.leftPercent = formatPercent(this.vpxToPosition(this.leftVpx)); + + const bucketSpan = appendNewElement(this.graphics, 'span', 'histogram-bucket'); + const heightPercent = this.maxCount == 0 ? 0 : + formatPercent(constants.baseHeightFraction + (bucket.count / this.maxCount) * + (1 - constants.baseHeightFraction)); + bucketSpan.style.height = heightPercent; + bucketSpan.style.width = this.bucketWidthPercent; + bucketSpan.style.left = this.leftPercent; + + let showingCount = false; + if (++this.textIntervalIndex == this.textInterval) { + showingCount = true; + this.textIntervalIndex = 0; + this.drawBucketLabels(bucket, heightPercent); + } + + bucketSpan.addEventListener('mouseenter', showPopupFn( + this.detailPopup, formatPercent(this.vpxToPosition(this.leftVpx)), bucket, + bucketSpan, showingCount)); + + bucketSpan.addEventListener('mouseleave', timeoutFn(this.detailPopup)); + + this.leftVpx += this.bucketWidthVpx + constants.marginWidthVpx; + } + + /** + * Draws the labels for a bucket. + * + * @param {!Object} bucket the bucket to draw. + * @param {string} heightPercent The height of the bucket, expressed as a percent. + */ + drawBucketLabels(bucket, heightPercent) { + const lowerLabel = appendNewElement(this.labels, 'span'); + lowerLabel.textContent = format(bucket.lower_bound); + lowerLabel.style.left = this.leftPercent; + lowerLabel.style.width = this.bucketWidthPercent; + + const bucketLabel = appendNewElement(this.graphics, 'span', 'bucket-label'); + bucketLabel.textContent = format(bucket.count); + bucketLabel.style.left = this.leftPercent; + bucketLabel.style.width = this.bucketWidthPercent; + bucketLabel.style.bottom = heightPercent; + } + + /** + * @param {number} virtualPixels the number of virtual pixels. + * @return {number} the x-position as a percent, including an offset. + */ + vpxToPosition(virtualPixels) { + return this.vpxToWidth(virtualPixels) + constants.outerMarginFraction; + } + + /** + * @param {number} virtualPixels the number of virtual pixels. + * @return {number} the x-position as a numeric percent. + */ + vpxToWidth(virtualPixels) { + return virtualPixels / this.widthVpx; + } +} + +/** + * @param {!Element} histogramDiv the element in which to render the histograms. + * @param {!Array} supportedPercentiles Array of supported histograms. + * @param {!Object} histogram the stats JSON structure obtained from the server /stats method. + * @param {?number} changeCount the number of times this histogram has changed value. + */ +function renderHistogram(histogramDiv, supportedPercentiles, histogram, changeCount) { + const div = appendNewElement(histogramDiv, 'div'); + const label = appendNewElement(div, 'span', 'histogram-name'); + label.textContent = histogram.name + (changeCount == null ? '' : ' (' + changeCount + ')'); + + const numBuckets = histogram.totals.length; + if (numBuckets == 0) { + appendNewElement(div, 'span', 'histogram-no-data').textContent = 'No recorded values'; + return; + } + + const maxCount = assignPercentilesAndIntervalsToBuckets(histogram, supportedPercentiles); + + // Lay out the buckets evenly, independent of the bucket values. It's up + // to the `circlhist` library to space out the buckets in a shape tuned to + // the data. + // + // We will not draw percentile lines outside of the bucket values. E.g. we + // may skip drawing outer percentiles like P0 and P100 etc. + // + // We lay out horizontally based on CSS percentage so users can see the + // graphics better if they make the window wider. We do this by inventing + // arbitrary "virtual pixels" (variables with `Vpx` suffix) during the + // computation in JS and converting them to percentages for writing element + // style. + const painter = new Painter(div, numBuckets, maxCount); + + for (let i = 0; i < numBuckets; ++i) { + const bucket = histogram.totals[i]; + const nextBucket = (i < histogram.totals.length - 1) ? histogram.totals[i + 1] : null; + for (annotation of bucket.annotations) { + painter.drawAnnotation(bucket, nextBucket, annotation); + } + painter.drawBucket(bucket); + } +} diff --git a/source/server/admin/stats_handler.cc b/source/server/admin/stats_handler.cc index fb8dc1801f93..662e6a59c7f1 100644 --- a/source/server/admin/stats_handler.cc +++ b/source/server/admin/stats_handler.cc @@ -171,29 +171,33 @@ Http::Code StatsHandler::handlerContention(Http::ResponseHeaderMap& response_hea } Admin::UrlHandler StatsHandler::statsHandler(bool active_mode) { - const Admin::ParamDescriptorVec common_params{ - {Admin::ParamDescriptor::Type::String, "filter", - "Regular expression (Google re2) for filtering stats"}, - {Admin::ParamDescriptor::Type::Enum, - "format", - "Format to use", - {"html", "active-html", "text", "json"}}, - {Admin::ParamDescriptor::Type::Enum, - "type", - "Stat types to include.", - {StatLabels::All, StatLabels::Counters, StatLabels::Histograms, StatLabels::Gauges, - StatLabels::TextReadouts}}, - {Admin::ParamDescriptor::Type::Enum, - "histogram_buckets", - "Histogram bucket display mode", - {"cumulative", "disjoint", "detailed", "none"}}}; - - Admin::ParamDescriptorVec params; + Admin::ParamDescriptor usedonly{ + Admin::ParamDescriptor::Type::Boolean, "usedonly", + "Only include stats that have been written by system since restart"}; + Admin::ParamDescriptor histogram_buckets{Admin::ParamDescriptor::Type::Enum, + "histogram_buckets", + "Histogram bucket display mode", + {"cumulative", "disjoint", "detailed", "none"}}; + Admin::ParamDescriptor format{Admin::ParamDescriptor::Type::Enum, + "format", + "Format to use", + {"html", "active-html", "text", "json"}}; + Admin::ParamDescriptor filter{Admin::ParamDescriptor::Type::String, "filter", + "Regular expression (Google re2) for filtering stats"}; + Admin::ParamDescriptor type{Admin::ParamDescriptor::Type::Enum, + "type", + "Stat types to include.", + {StatLabels::All, StatLabels::Counters, StatLabels::Histograms, + StatLabels::Gauges, StatLabels::TextReadouts}}; + + Admin::ParamDescriptorVec params{usedonly, filter}; if (!active_mode) { - params.push_back({Admin::ParamDescriptor::Type::Boolean, "usedonly", - "Only include stats that have been written by system since restart"}); + params.push_back(format); + } + params.push_back(type); + if (!active_mode) { + params.push_back(histogram_buckets); } - params.insert(params.end(), common_params.begin(), common_params.end()); return { "/stats", diff --git a/source/server/admin/stats_html_render.cc b/source/server/admin/stats_html_render.cc index d170843e7003..52bbbbfa1108 100644 --- a/source/server/admin/stats_html_render.cc +++ b/source/server/admin/stats_html_render.cc @@ -23,9 +23,27 @@ StatsHtmlRender::StatsHtmlRender(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, const StatsParams& params) : StatsTextRender(params), active_(params.format_ == StatsFormat::ActiveHtml) { AdminHtmlUtil::renderHead(response_headers, response); + if (!active_) { + StatsParams json_params(params); + json_params.histogram_buckets_mode_ = Utility::HistogramBucketsMode::Detailed; + json_response_headers_ = Http::ResponseHeaderMapImpl::create(); + histogram_json_render_ = + std::make_unique(*json_response_headers_, json_data_, json_params); + } } -void StatsHtmlRender::finalize(Buffer::Instance& response) { AdminHtmlUtil::finalize(response); } +void StatsHtmlRender::finalize(Buffer::Instance& response) { + // Render all the histograms here using the JSON data we've accumulated + // for them. + if (!active_) { + histogram_json_render_->finalize(json_data_); + response.add("\n
\n\n"}); + std::string buf2; + response.addFragments({"\n"}); } else { - response.add("
\n");
+    response.addFragments(
+        {"\n
\n"});
   }
 }
 
diff --git a/source/server/admin/stats_html_render.h b/source/server/admin/stats_html_render.h
index 229e004a17cb..c86c105a88d6 100644
--- a/source/server/admin/stats_html_render.h
+++ b/source/server/admin/stats_html_render.h
@@ -35,16 +35,23 @@ class StatsHtmlRender : public StatsTextRender {
     StatsTextRender::generate(response, name, value);
   }
 
-  // generate() call directly calls parent method, which is needed to allow gcc
-  // to compile, otherwise it warns about hidden overrides.
-  void generate(Buffer::Instance& response, const std::string& name,
+  // We override the generate method for HTML to trigger some JS that will
+  // render the histogram graphically. We will render that from JavaScript
+  // and convey the histogram data to the JS via JSON, so we can delegate
+  // to an instantiated JSON `sub-renderer` that will write into json_data_.
+  void generate(Buffer::Instance&, const std::string& name,
                 const Stats::ParentHistogram& histogram) override {
-    StatsTextRender::generate(response, name, histogram);
+    histogram_json_render_->generate(json_data_, name, histogram);
   }
+
   void finalize(Buffer::Instance&) override;
 
 private:
   const bool active_{false};
+  Buffer::OwnedImpl json_data_;
+  std::unique_ptr histogram_json_render_;
+  Http::ResponseHeaderMapPtr json_response_headers_; // ignored.
+  std::unique_ptr json_headers_;
 };
 
 } // namespace Server
diff --git a/test/integration/admin_html/BUILD b/test/integration/admin_html/BUILD
index ed7e028cbc82..1215673070f8 100644
--- a/test/integration/admin_html/BUILD
+++ b/test/integration/admin_html/BUILD
@@ -52,6 +52,7 @@ filegroup(
     name = "test_server_files",
     srcs = [
         "active_stats_test.js",
+        "histograms_test.js",
         "web_test.html",
         "web_test.js",
         "web_test.yaml",
diff --git a/test/integration/admin_html/active_stats_test.js b/test/integration/admin_html/active_stats_test.js
index 0b83006dd779..25de79d3ba9b 100644
--- a/test/integration/admin_html/active_stats_test.js
+++ b/test/integration/admin_html/active_stats_test.js
@@ -1,6 +1,4 @@
-// This file contains helper functions for writing JavaScript unit tests, in the
-// style of Google's Closure Compiler, but without taking on a large dependency
-// or build infrastructure.
+// This file uses unit-test helper functions defined in web_test.js.
 //
 // See source/server/admin/javascript.md for background info.
 
diff --git a/test/integration/admin_html/histograms_test.html b/test/integration/admin_html/histograms_test.html
new file mode 100644
index 000000000000..e10ca5b79799
--- /dev/null
+++ b/test/integration/admin_html/histograms_test.html
@@ -0,0 +1,9 @@
+
+  
+    
+    
+  
+  
+    

+  
+
diff --git a/test/integration/admin_html/histograms_test.js b/test/integration/admin_html/histograms_test.js
new file mode 100644
index 000000000000..c9df555e0db7
--- /dev/null
+++ b/test/integration/admin_html/histograms_test.js
@@ -0,0 +1,154 @@
+/**
+ * Makes a histogram with the specified values so we can test that all the
+ * features of this are rendered graphically.
+ *
+ * @param {!Array} totals pre-populated json 'totals' field.
+ * @return {!Object} json stats object containing one histogram.
+ */
+function makeHistogramJson(totals) {
+  return {'stats': [{
+    'histograms': {
+      'supported_percentiles': [0, 25, 50, 75, 90, 95, 99, 99.5, 99.9, 100],
+      'details': [{
+        'name': 'h1',
+        'percentiles': [
+          {'cumulative': 200, 'interval': 200},
+          {'cumulative': 207.5, 'interval': 207.5},
+          {'cumulative': 302.5, 'interval': 302.5},
+          {'cumulative': 306.25, 'interval': 306.25},
+          {'cumulative': 308.5, 'interval': 308.5},
+          {'cumulative': 309.25, 'interval': 309.25},
+          {'cumulative': 309.85, 'interval': 309.85},
+          {'cumulative': 309.925, 'interval': 309.925},
+          {'cumulative': 309.985, 'interval': 309.985},
+          {'cumulative': 310, 'interval': 310},
+        ],
+        'totals': totals,
+        'intervals': [
+          {'lower_bound': 200, 'width': 10, 'count': 1},
+          {'lower_bound': 300, 'width': 10, 'count': 2}]}]}}]};
+}
+
+/**
+ * Tests the rendering of histograms.
+ *
+ * @param {!Element} iframe the iframe we can use for rendering.
+ */
+async function testRenderHistogram(iframe) {
+  const idoc = iframe.contentWindow.document;
+  renderHistograms(idoc.body, makeHistogramJson([
+    {'lower_bound': 200, 'width': 10, 'count': 1},
+    {'lower_bound': 300, 'width': 10, 'count': 2}]));
+  const buckets = idoc.getElementsByClassName('histogram-bucket');
+  assertEq(2, buckets.length);
+
+  // The first bucket is to the left of the second bucket;
+  assertLt(parseFloat(buckets[0].style.left), parseFloat(buckets[1].style.left));
+
+  // The first bucket has a height between 25% and 75%.
+  assertLt(25, parseFloat(buckets[0].style.height));
+  assertLt(parseFloat(buckets[0].style.height), 75);
+
+  // The second bucket as a 100% height.
+  assertEq('100%', buckets[1].style.height);
+
+  // There is one popup div and it is not visible initially.
+  const popups = idoc.getElementsByClassName('histogram-popup');
+  assertEq(1, popups.length);
+  const popup = popups[0];
+  assertEq('hidden', getComputedStyle(popup).visibility);
+
+  // When the mouse enters the first bucket it renders a visible popup with
+  // associated annotations.
+  buckets[0].dispatchEvent(new Event('mouseenter'));
+  assertEq('visible', getComputedStyle(popup).visibility);
+  assertEq(4, popup.children.length);
+  assertEq('[200, 210)', popup.children[0].textContent);
+  assertEq('P0: 200', popup.children[1].textContent);
+  assertEq('Interval [200, 210): 1', popup.children[2].textContent);
+  assertEq('P25: 207.5', popup.children[3].textContent);
+
+  // 2 seconds after the mouse leaves, that area, the popup will be made invisible.
+  buckets[0].dispatchEvent(new Event('mouseleave'));
+  await asyncTimeout(3000);
+  assertEq('hidden', getComputedStyle(popup).visibility);
+
+  // Now enter the other bucket. Just check the 1st 2 of 10 buckets.
+  buckets[1].dispatchEvent(new Event('mouseenter'));
+  assertEq('visible', getComputedStyle(popup).visibility);
+  assertEq(10, popup.children.length);
+  assertEq('[300, 310)', popup.children[0].textContent);
+  assertEq('Interval [300, 310): 2', popup.children[1].textContent);
+  buckets[1].dispatchEvent(new Event('mouseleave'));
+
+  // Re-enter the first bucket. The popup will immediately move to that with no delay.
+  buckets[0].dispatchEvent(new Event('mouseenter'));
+  assertEq('visible', getComputedStyle(popup).visibility);
+  assertEq(4, popup.children.length);
+  assertEq('[200, 210)', popup.children[0].textContent);
+  buckets[1].dispatchEvent(new Event('mouseleave'));
+
+  // There's exactly one annotations bucket.
+  assertEq(1, idoc.getElementsByClassName('histogram-annotations').length);
+
+  // There are 10 percentiles rendered each one to the right of the previous one.
+  const percentiles = idoc.getElementsByClassName('histogram-percentile');
+  assertEq(10, percentiles.length);
+  let prevPercent = 0;
+  for (percentile of percentiles) {
+    const left = parseFloat(percentile.style.left);
+    assertLt(prevPercent, left);
+    prevPercent = left;
+  }
+
+  // There are 2 intervals rendered each one to the right of the previous one.
+  const intervals = idoc.getElementsByClassName('histogram-interval');
+  assertEq(2, intervals.length);
+  assertLt(parseFloat(intervals[0].style.left), parseFloat(intervals[1].style.left));
+}
+
+
+/**
+ * Renders a histogram, returning the number of text entries.
+ *
+ * @param {!Element} iframe the iframe we can use for rendering.
+ * @param {number} numBuckets the number of histogram buckets to render.
+ * @return {number} the number of text entries found.
+ */
+async function renderManyBucketsCountingLabels(iframe, numBuckets) {
+  const idoc = iframe.contentWindow.document;
+  idoc.body.replaceChildren();
+
+  let totals = [];
+  for (let i = 0; i < numBuckets; ++i) {
+    totals.push({'lower_bound': i*100, 'width': 10, 'count': 1});
+  }
+
+  renderHistograms(idoc.body, makeHistogramJson(totals));
+  await asyncTimeout(200);
+  const labels = idoc.getElementsByClassName('histogram-labels')[0];
+  return labels.children.length;
+}
+
+
+/**
+ * Tests the rendering of histograms with a large number of buckets.
+ *
+ * @param {!Element} iframe the iframe we can use for rendering.
+ */
+async function testManyBuckets(iframe) {
+  assertEq(15, await renderManyBucketsCountingLabels(iframe, 15));
+  assertEq(16, await renderManyBucketsCountingLabels(iframe, 16));
+  assertEq(17, await renderManyBucketsCountingLabels(iframe, 17));
+  assertEq(18, await renderManyBucketsCountingLabels(iframe, 18));
+  assertEq(19, await renderManyBucketsCountingLabels(iframe, 19));
+  assertEq(20, await renderManyBucketsCountingLabels(iframe, 100));
+  assertEq(20, await renderManyBucketsCountingLabels(iframe, 200));
+  assertEq(19, await renderManyBucketsCountingLabels(iframe, 250));
+  assertEq(20, await renderManyBucketsCountingLabels(iframe, 400));
+  assertEq(20, await renderManyBucketsCountingLabels(iframe, 500));
+  assertEq(20, await renderManyBucketsCountingLabels(iframe, 1000));
+}
+
+addTest('?file=histograms_test.html', 'renderHistogram', testRenderHistogram);
+addTest('?file=histograms_test.html', 'manyBuckets', testManyBuckets);
diff --git a/test/integration/admin_html/test_server.cc b/test/integration/admin_html/test_server.cc
index 944cdc04a62b..2a9b2107d0fc 100644
--- a/test/integration/admin_html/test_server.cc
+++ b/test/integration/admin_html/test_server.cc
@@ -20,8 +20,13 @@ Http::Code testCallback(Http::ResponseHeaderMap& response_headers, Buffer::Insta
                         Server::AdminStream& admin_stream) {
   Http::Utility::QueryParams query_params = admin_stream.queryParams();
   auto iter = query_params.find("file");
-  if (iter == query_params.end()) {
-    response.add("query param 'file' missing");
+  std::string prefix;
+  if (iter != query_params.end()) {
+    prefix = "test/integration/admin_html/";
+  } else if (iter = query_params.find("src"); iter != query_params.end()) {
+    prefix = "source/server/admin/html/";
+  } else {
+    response.add("query param 'file' or 'src' missing");
     return Http::Code::BadRequest;
   }
   absl::string_view leaf = iter->second;
@@ -33,7 +38,7 @@ Http::Code testCallback(Http::ResponseHeaderMap& response_headers, Buffer::Insta
   }
 
   Filesystem::InstanceImpl file_system;
-  std::string path = absl::StrCat("test/integration/admin_html/", iter->second);
+  std::string path = absl::StrCat(prefix, leaf);
   TRY_ASSERT_MAIN_THREAD { response.add(file_system.fileReadToEnd(path)); }
   END_TRY
   catch (EnvoyException& e) {
@@ -44,6 +49,8 @@ Http::Code testCallback(Http::ResponseHeaderMap& response_headers, Buffer::Insta
     response_headers.setReferenceContentType(Http::Headers::get().ContentTypeValues.Html);
   } else if (absl::EndsWith(path, ".js")) {
     response_headers.setReferenceContentType("text/javascript");
+  } else if (absl::EndsWith(path, ".css")) {
+    response_headers.setReferenceContentType("text/css");
   }
   return Http::Code::OK;
 }
diff --git a/test/integration/admin_html/test_server_test.sh b/test/integration/admin_html/test_server_test.sh
index cd6d4cb8b81d..6e284a5bd025 100755
--- a/test/integration/admin_html/test_server_test.sh
+++ b/test/integration/admin_html/test_server_test.sh
@@ -90,4 +90,5 @@ check_debug_log() {
 check_debug_log active_stats.js
 check_debug_log admin_head_start.html
 check_debug_log admin.css
+check_debug_log histograms.js
 
diff --git a/test/integration/admin_html/web_test.html b/test/integration/admin_html/web_test.html
index a96b0aa15603..e03a5f4474a3 100644
--- a/test/integration/admin_html/web_test.html
+++ b/test/integration/admin_html/web_test.html
@@ -2,6 +2,8 @@
 
   
   
+  
+  
 
 
   

diff --git a/test/integration/admin_html/web_test.js b/test/integration/admin_html/web_test.js
index abd667757fcf..ed720ce0d91d 100644
--- a/test/integration/admin_html/web_test.js
+++ b/test/integration/admin_html/web_test.js
@@ -6,6 +6,8 @@
 //
 // Linted with https://validatejavascript.com/, 'Google' settings, long-line
 // checking disabled, indent-checking disabled.
+//
+// See source/server/admin/javascript.md for background info.
 
 
 /**
@@ -28,6 +30,7 @@ function addTest(url, name, testFunction) { // eslint-disable-line no-unused-var
   testList.push({'url': url, 'name': name, 'testFunction': testFunction});
 }
 
+
 /**
  * Provides an async version of the `onload` event.
  *
@@ -42,6 +45,7 @@ function waitForOnload(iframe) {
   });
 }
 
+
 /**
  * Renders a URL, and after 3 seconds delay, runs the 'tester' function.
  * Log whether that function failed (threw exception) or passed. Either
@@ -96,6 +100,28 @@ function assertEq(expected, actual) { // eslint-disable-line no-unused-vars
 }
 
 
+/**
+ * Checks for less-than, throwing an exception with a comment if it fails.
+ *
+ * @param {Object} a
+ * @param {Object} b
+ */
+function assertLt(a, b) { // eslint-disable-line no-unused-vars
+  assertTrue(a < b, 'assertLt mismatch: ' + a + ' < ' + b);
+}
+
+
+/**
+ * Async version of windows.setTimeout.
+ *
+ * @param {!number} timeoutMs Timeout in milliseconds.
+ * @return {!Promise} a promise that will be resolved when the timeout follows.
+ */
+function asyncTimeout(timeoutMs) { // eslint-disable-line no-unused-vars
+  return new Promise((resolve) => window.setTimeout(resolve, timeoutMs));
+}
+
+
 /**
  * Runs all tests added via addTest() above.
  */
@@ -113,5 +139,6 @@ function runAllTests() {
   next(0);
 }
 
+
 // Trigger the tests once all JS is loaded.
 window.addEventListener('DOMContentLoaded', runAllTests);
diff --git a/test/server/admin/stats_html_render_test.cc b/test/server/admin/stats_html_render_test.cc
index 4dfaf94f4eb6..81b0a8c8ce06 100644
--- a/test/server/admin/stats_html_render_test.cc
+++ b/test/server/admin/stats_html_render_test.cc
@@ -37,10 +37,8 @@ TEST_F(StatsHtmlRenderTest, String) {
 }
 
 TEST_F(StatsHtmlRenderTest, HistogramNoBuckets) {
-  constexpr absl::string_view expected =
-      "h1: P0(200,200) P25(207.5,207.5) P50(302.5,302.5) P75(306.25,306.25) "
-      "P90(308.5,308.5) P95(309.25,309.25) P99(309.85,309.85) P99.5(309.925,309.925) "
-      "P99.9(309.985,309.985) P100(310,310)\n";
+  // The goal of this test is to show that we have embedded the histogram as a json fragment.
+  constexpr absl::string_view expected = "const json = \n{\"stats\":[{\"histograms\":";
   EXPECT_THAT(render<>(renderer_, "h1", populateHistogram("h1", {200, 300, 300})),
               HasSubstr(expected));
 }

From 5858232fe38ca71aaefbe026c3e1ff7ccd4f8e91 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Tue, 6 Jun 2023 17:15:23 -0400
Subject: [PATCH 472/740] coverate: ratcheting (#27822)

Signed-off-by: Alyssa Wilk 
---
 test/per_file_coverage.sh | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh
index 76f798634a85..c96fc8feb579 100755
--- a/test/per_file_coverage.sh
+++ b/test/per_file_coverage.sh
@@ -4,9 +4,8 @@
 # for existing directories with low coverage.
 declare -a KNOWN_LOW_COVERAGE=(
 "source/common:96.2"
-"source/common/api:82.6"
+"source/common/api:84.5"
 "source/common/api/posix:81.3"
-"source/common/common/posix:92.7"
 "source/common/config:94.8"
 "source/common/crypto:88.1"
 "source/common/event:95.1" # Emulated edge events guards don't report LCOV
@@ -33,7 +32,6 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/common/tap:94.2"
 "source/extensions/common/wasm:87.5" # flaky: be careful adjusting
 "source/extensions/common/wasm/ext:92.0"
-"source/extensions/filters/common:96.5"
 "source/extensions/filters/common/fault:94.5"
 "source/extensions/filters/common/rbac:90.5"
 "source/extensions/filters/http/cache:93.4"
@@ -58,12 +56,12 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/stat_sinks/statsd:84.6"
 "source/extensions/tracers:95.8"
 "source/extensions/tracers/common:73.8"
-"source/extensions/tracers/common/ot:71.7"
+"source/extensions/tracers/common/ot:71.8"
 "source/extensions/tracers/opencensus:93.2"
 "source/extensions/tracers/zipkin:95.8"
-"source/extensions/transport_sockets:95.6"
-"source/extensions/transport_sockets/tls:94.9"
-"source/extensions/transport_sockets/tls/cert_validator:95.1"
+"source/extensions/transport_sockets:95.8"
+"source/extensions/transport_sockets/tls:95.0"
+"source/extensions/transport_sockets/tls/cert_validator:95.2"
 "source/extensions/transport_sockets/tls/private_key:88.9"
 "source/extensions/wasm_runtime/wamr:0.0" # Not enabled in coverage build
 "source/extensions/wasm_runtime/wasmtime:0.0" # Not enabled in coverage build
@@ -73,16 +71,14 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/watchdog/profile_action:83.3"
 "source/server:93.8" # flaky: be careful adjusting. See https://github.com/envoyproxy/envoy/issues/15239
 "source/server/admin:profiler-lib:83"
-"source/extensions/load_balancing_policies/common:94" # Death tests don't report LCOV
-"source/server/config_validation:88.2"
+"source/extensions/load_balancing_policies/common:94.1" # Death tests don't report LCOV
+"source/server/config_validation:88.4"
 "source/extensions/health_checkers:95.9"
 "source/extensions/health_checkers/http:93.8"
 "source/extensions/health_checkers/grpc:92.0"
 "source/extensions/load_balancing_policies:95.5"
 "source/extensions/load_balancing_policies/subset:94.3"
 "source/extensions/config_subscription/rest:94.3"
-"source/extensions/config_subscription:94.8"
-"source/extensions/config_subscription/grpc:94.0"
 "source/extensions/matching/input_matchers/cel_matcher:90.7" #Death tests don't report LCOV
 )
 

From d6727d1a1fa43ff7250929786906529d19d1c4c7 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Tue, 6 Jun 2023 17:34:12 -0400
Subject: [PATCH 473/740] tls: making CertificateValidationContextConfigImpl
 exception free (#27754)

Risk Level: low
Testing: existing tests
Docs Changes: n/a
Release Notes: n/a
Part of envoyproxy/envoy-mobile#176

Signed-off-by: Alyssa Wilk 
---
 ...tificate_validation_context_config_impl.cc | 29 ++++++++++---
 ...rtificate_validation_context_config_impl.h | 16 ++++++--
 .../tls/context_config_impl.cc                | 37 ++++++++++-------
 test/common/secret/sds_api_test.cc            | 29 ++++++-------
 .../common/secret/secret_manager_impl_test.cc | 41 +++++++++++--------
 tools/code_format/config.yaml                 |  1 -
 6 files changed, 97 insertions(+), 56 deletions(-)

diff --git a/source/common/ssl/certificate_validation_context_config_impl.cc b/source/common/ssl/certificate_validation_context_config_impl.cc
index 034409ad38f5..b1249969f8cf 100644
--- a/source/common/ssl/certificate_validation_context_config_impl.cc
+++ b/source/common/ssl/certificate_validation_context_config_impl.cc
@@ -42,20 +42,37 @@ CertificateValidationContextConfigImpl::CertificateValidationContextConfigImpl(
       api_(api), only_verify_leaf_cert_crl_(config.only_verify_leaf_cert_crl()),
       max_verify_depth_(config.has_max_verify_depth()
                             ? absl::optional(config.max_verify_depth().value())
-                            : absl::nullopt) {
+                            : absl::nullopt) {}
+
+absl::StatusOr>
+CertificateValidationContextConfigImpl::create(
+    const envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext& context,
+    Api::Api& api) {
+  auto config = std::unique_ptr(
+      new CertificateValidationContextConfigImpl(context, api));
+  absl::Status status = config->initialize();
+  if (status.ok()) {
+    return config;
+  }
+  return status;
+}
+
+absl::Status CertificateValidationContextConfigImpl::initialize() {
   if (ca_cert_.empty() && custom_validator_config_ == absl::nullopt) {
     if (!certificate_revocation_list_.empty()) {
-      throw EnvoyException(fmt::format("Failed to load CRL from {} without trusted CA",
-                                       certificateRevocationListPath()));
+      return absl::InvalidArgumentError(fmt::format("Failed to load CRL from {} without trusted CA",
+                                                    certificateRevocationListPath()));
     }
     if (!subject_alt_name_matchers_.empty()) {
-      throw EnvoyException("SAN-based verification of peer certificates without "
-                           "trusted CA is insecure and not allowed");
+      return absl::InvalidArgumentError("SAN-based verification of peer certificates without "
+                                        "trusted CA is insecure and not allowed");
     }
     if (allow_expired_certificate_) {
-      throw EnvoyException("Certificate validity period is always ignored without trusted CA");
+      return absl::InvalidArgumentError(
+          "Certificate validity period is always ignored without trusted CA");
     }
   }
+  return absl::OkStatus();
 }
 
 std::vector
diff --git a/source/common/ssl/certificate_validation_context_config_impl.h b/source/common/ssl/certificate_validation_context_config_impl.h
index b93d67fe6d3b..b8386f13d366 100644
--- a/source/common/ssl/certificate_validation_context_config_impl.h
+++ b/source/common/ssl/certificate_validation_context_config_impl.h
@@ -8,14 +8,19 @@
 #include "envoy/ssl/certificate_validation_context_config.h"
 #include "envoy/type/matcher/v3/string.pb.h"
 
+#include "absl/status/statusor.h"
+
 namespace Envoy {
 namespace Ssl {
 
 class CertificateValidationContextConfigImpl : public CertificateValidationContextConfig {
 public:
-  CertificateValidationContextConfigImpl(
-      const envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext& config,
-      Api::Api& api);
+  // Create a CertificateValidationContextConfigImpl or return an error status.
+  static absl::StatusOr>
+  create(const envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext& context,
+         Api::Api& api);
+
+  absl::Status initialize();
 
   const std::string& caCert() const override { return ca_cert_; }
   const std::string& caCertPath() const override { return ca_cert_path_; }
@@ -53,6 +58,11 @@ class CertificateValidationContextConfigImpl : public CertificateValidationConte
 
   absl::optional maxVerifyDepth() const override { return max_verify_depth_; }
 
+protected:
+  CertificateValidationContextConfigImpl(
+      const envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext& config,
+      Api::Api& api);
+
 private:
   static std::vector
   getSubjectAltNameMatchers(
diff --git a/source/extensions/transport_sockets/tls/context_config_impl.cc b/source/extensions/transport_sockets/tls/context_config_impl.cc
index 021c1d881fea..e19dc26c9d3d 100644
--- a/source/extensions/transport_sockets/tls/context_config_impl.cc
+++ b/source/extensions/transport_sockets/tls/context_config_impl.cc
@@ -90,17 +90,12 @@ getCertificateValidationContextConfigProvider(
         default_cvc) {
   switch (config.validation_context_type_case()) {
   case envoy::extensions::transport_sockets::tls::v3::CommonTlsContext::ValidationContextTypeCase::
-      kValidationContext: {
-    auto secret_provider =
-        factory_context.secretManager().createInlineCertificateValidationContextProvider(
-            config.validation_context());
-    return secret_provider;
-  }
+      kValidationContext:
+    return factory_context.secretManager().createInlineCertificateValidationContextProvider(
+        config.validation_context());
   case envoy::extensions::transport_sockets::tls::v3::CommonTlsContext::ValidationContextTypeCase::
-      kValidationContextSdsSecretConfig: {
-    const auto& sds_secret_config = config.validation_context_sds_secret_config();
-    return getProviderFromSds(factory_context, sds_secret_config);
-  }
+      kValidationContextSdsSecretConfig:
+    return getProviderFromSds(factory_context, config.validation_context_sds_secret_config());
   case envoy::extensions::transport_sockets::tls::v3::CommonTlsContext::ValidationContextTypeCase::
       kCombinedValidationContext: {
     *default_cvc = std::make_unique<
@@ -213,8 +208,12 @@ ContextConfigImpl::ContextConfigImpl(
         validation_context_config_ =
             getCombinedValidationContextConfig(*certificate_validation_context_provider_->secret());
       } else {
-        validation_context_config_ = std::make_unique(
+        auto config_or_status = Envoy::Ssl::CertificateValidationContextConfigImpl::create(
             *certificate_validation_context_provider_->secret(), api_);
+        if (!config_or_status.status().ok()) {
+          throw EnvoyException(std::string(config_or_status.status().message()));
+        }
+        validation_context_config_ = std::move(config_or_status.value());
       }
     }
   }
@@ -255,7 +254,12 @@ Ssl::CertificateValidationContextConfigPtr ContextConfigImpl::getCombinedValidat
   envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext combined_cvc =
       *default_cvc_;
   combined_cvc.MergeFrom(dynamic_cvc);
-  return std::make_unique(combined_cvc, api_);
+  auto config_or_status =
+      Envoy::Ssl::CertificateValidationContextConfigImpl::create(combined_cvc, api_);
+  if (!config_or_status.status().ok()) {
+    throw EnvoyException(std::string(config_or_status.status().message()));
+  }
+  return std::move(config_or_status.value());
 }
 
 void ContextConfigImpl::setSecretUpdateCallback(std::function callback) {
@@ -291,9 +295,12 @@ void ContextConfigImpl::setSecretUpdateCallback(std::function callback)
       // ContextConfigImpl::validation_context_config_ with new secret.
       cvc_update_callback_handle_ =
           certificate_validation_context_provider_->addUpdateCallback([this, callback]() {
-            validation_context_config_ =
-                std::make_unique(
-                    *certificate_validation_context_provider_->secret(), api_);
+            auto config_or_status = Envoy::Ssl::CertificateValidationContextConfigImpl::create(
+                *certificate_validation_context_provider_->secret(), api_);
+            if (!config_or_status.status().ok()) {
+              throw EnvoyException(std::string(config_or_status.status().message()));
+            }
+            validation_context_config_ = std::move(config_or_status.value());
             callback();
           });
     }
diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc
index 6cb640f23464..e0fb476c3155 100644
--- a/test/common/secret/sds_api_test.cc
+++ b/test/common/secret/sds_api_test.cc
@@ -629,11 +629,12 @@ TEST_F(SdsApiTest, DynamicCertificateValidationContextUpdateSuccess) {
   initialize();
   subscription_factory_.callbacks_->onConfigUpdate(decoded_resources.refvec_, "");
 
-  Ssl::CertificateValidationContextConfigImpl cvc_config(*sds_api.secret(), *api_);
+  auto cvc_config =
+      Ssl::CertificateValidationContextConfigImpl::create(*sds_api.secret(), *api_).value();
   const std::string ca_cert =
       "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem";
   EXPECT_EQ(TestEnvironment::readFileToStringForTest(TestEnvironment::substitute(ca_cert)),
-            cvc_config.caCert());
+            cvc_config->caCert());
 }
 
 class CvcValidationCallback {
@@ -705,31 +706,31 @@ TEST_F(SdsApiTest, DefaultCertificateValidationContextTest) {
   envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext merged_cvc =
       default_cvc;
   merged_cvc.MergeFrom(*sds_api.secret());
-  Ssl::CertificateValidationContextConfigImpl cvc_config(merged_cvc, *api_);
+  auto cvc_config = Ssl::CertificateValidationContextConfigImpl::create(merged_cvc, *api_).value();
   // Verify that merging CertificateValidationContext applies logical OR to bool
   // field.
-  EXPECT_TRUE(cvc_config.allowExpiredCertificate());
+  EXPECT_TRUE(cvc_config->allowExpiredCertificate());
   // Verify that singular fields are overwritten.
   const std::string ca_cert =
       "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem";
   EXPECT_EQ(TestEnvironment::readFileToStringForTest(TestEnvironment::substitute(ca_cert)),
-            cvc_config.caCert());
+            cvc_config->caCert());
   // Verify that repeated fields are concatenated.
-  EXPECT_EQ(2, cvc_config.subjectAltNameMatchers().size());
-  EXPECT_EQ("first san", cvc_config.subjectAltNameMatchers()[0].matcher().exact());
+  EXPECT_EQ(2, cvc_config->subjectAltNameMatchers().size());
+  EXPECT_EQ("first san", cvc_config->subjectAltNameMatchers()[0].matcher().exact());
   EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS,
-            cvc_config.subjectAltNameMatchers()[0].san_type());
-  EXPECT_EQ("second san", cvc_config.subjectAltNameMatchers()[1].matcher().exact());
+            cvc_config->subjectAltNameMatchers()[0].san_type());
+  EXPECT_EQ("second san", cvc_config->subjectAltNameMatchers()[1].matcher().exact());
   EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS,
-            cvc_config.subjectAltNameMatchers()[1].san_type());
+            cvc_config->subjectAltNameMatchers()[1].san_type());
   // Verify that if dynamic CertificateValidationContext does not set certificate hash list, the new
   // secret contains hash list from default CertificateValidationContext.
-  EXPECT_EQ(1, cvc_config.verifyCertificateHashList().size());
-  EXPECT_EQ(default_verify_certificate_hash, cvc_config.verifyCertificateHashList()[0]);
+  EXPECT_EQ(1, cvc_config->verifyCertificateHashList().size());
+  EXPECT_EQ(default_verify_certificate_hash, cvc_config->verifyCertificateHashList()[0]);
   // Verify that if default CertificateValidationContext does not set certificate SPKI list, the new
   // secret contains SPKI list from dynamic CertificateValidationContext.
-  EXPECT_EQ(1, cvc_config.verifyCertificateSpkiList().size());
-  EXPECT_EQ(dynamic_verify_certificate_spki, cvc_config.verifyCertificateSpkiList()[0]);
+  EXPECT_EQ(1, cvc_config->verifyCertificateSpkiList().size());
+  EXPECT_EQ(dynamic_verify_certificate_spki, cvc_config->verifyCertificateSpkiList()[0]);
 }
 
 class GenericSecretValidationCallback {
diff --git a/test/common/secret/secret_manager_impl_test.cc b/test/common/secret/secret_manager_impl_test.cc
index ead2895c984b..87a706c9ec4b 100644
--- a/test/common/secret/secret_manager_impl_test.cc
+++ b/test/common/secret/secret_manager_impl_test.cc
@@ -134,12 +134,15 @@ TEST_F(SecretManagerImplTest, CertificateValidationContextSecretLoadSuccess) {
 
   ASSERT_EQ(secret_manager->findStaticCertificateValidationContextProvider("undefined"), nullptr);
   ASSERT_NE(secret_manager->findStaticCertificateValidationContextProvider("abc.com"), nullptr);
-  Ssl::CertificateValidationContextConfigImpl cvc_config(
-      *secret_manager->findStaticCertificateValidationContextProvider("abc.com")->secret(), *api_);
+  auto cvc_config =
+      Ssl::CertificateValidationContextConfigImpl::create(
+          *secret_manager->findStaticCertificateValidationContextProvider("abc.com")->secret(),
+          *api_)
+          .value();
   const std::string cert_pem =
       "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem";
   EXPECT_EQ(TestEnvironment::readFileToStringForTest(TestEnvironment::substitute(cert_pem)),
-            cvc_config.caCert());
+            cvc_config->caCert());
 }
 
 // Validate that secret manager throws an exception when adding duplicated static certificate
@@ -526,9 +529,10 @@ name: "abc.com.validation"
   init_target_handle->initialize(init_watcher);
   secret_context.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(
       decoded_resources_2.refvec_, "validation-context-v1");
-  Ssl::CertificateValidationContextConfigImpl cert_validation_context(
-      *context_secret_provider->secret(), *api_);
-  EXPECT_EQ("DUMMY_INLINE_STRING_TRUSTED_CA", cert_validation_context.caCert());
+  auto cert_validation_context =
+      Ssl::CertificateValidationContextConfigImpl::create(*context_secret_provider->secret(), *api_)
+          .value();
+  EXPECT_EQ("DUMMY_INLINE_STRING_TRUSTED_CA", cert_validation_context->caCert());
   const std::string updated_config_dump = R"EOF(
 dynamic_active_secrets:
 - name: "abc.com"
@@ -1140,21 +1144,24 @@ TEST_F(SecretManagerImplTest, DeprecatedSanMatcher) {
 
   ASSERT_EQ(secret_manager->findStaticCertificateValidationContextProvider("undefined"), nullptr);
   ASSERT_NE(secret_manager->findStaticCertificateValidationContextProvider("abc.com"), nullptr);
-  Ssl::CertificateValidationContextConfigImpl cvc_config(
-      *secret_manager->findStaticCertificateValidationContextProvider("abc.com")->secret(), *api_);
-  EXPECT_EQ(cvc_config.subjectAltNameMatchers().size(), 4);
-  EXPECT_EQ("example.foo", cvc_config.subjectAltNameMatchers()[0].matcher().exact());
+  auto cvc_config =
+      Ssl::CertificateValidationContextConfigImpl::create(
+          *secret_manager->findStaticCertificateValidationContextProvider("abc.com")->secret(),
+          *api_)
+          .value();
+  EXPECT_EQ(cvc_config->subjectAltNameMatchers().size(), 4);
+  EXPECT_EQ("example.foo", cvc_config->subjectAltNameMatchers()[0].matcher().exact());
   EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS,
-            cvc_config.subjectAltNameMatchers()[0].san_type());
-  EXPECT_EQ("example.foo", cvc_config.subjectAltNameMatchers()[1].matcher().exact());
+            cvc_config->subjectAltNameMatchers()[0].san_type());
+  EXPECT_EQ("example.foo", cvc_config->subjectAltNameMatchers()[1].matcher().exact());
   EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::URI,
-            cvc_config.subjectAltNameMatchers()[1].san_type());
-  EXPECT_EQ("example.foo", cvc_config.subjectAltNameMatchers()[2].matcher().exact());
+            cvc_config->subjectAltNameMatchers()[1].san_type());
+  EXPECT_EQ("example.foo", cvc_config->subjectAltNameMatchers()[2].matcher().exact());
   EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::EMAIL,
-            cvc_config.subjectAltNameMatchers()[2].san_type());
-  EXPECT_EQ("example.foo", cvc_config.subjectAltNameMatchers()[3].matcher().exact());
+            cvc_config->subjectAltNameMatchers()[2].san_type());
+  EXPECT_EQ("example.foo", cvc_config->subjectAltNameMatchers()[3].matcher().exact());
   EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::IP_ADDRESS,
-            cvc_config.subjectAltNameMatchers()[3].san_type());
+            cvc_config->subjectAltNameMatchers()[3].san_type());
 }
 
 } // namespace
diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml
index f9c7f231773c..c189b2c994c1 100644
--- a/tools/code_format/config.yaml
+++ b/tools/code_format/config.yaml
@@ -117,7 +117,6 @@ paths:
     - source/common/network/lc_trie.h
     - source/common/network/socket_impl.cc
     - source/common/ssl/tls_certificate_config_impl.cc
-    - source/common/ssl/certificate_validation_context_config_impl.cc
     - source/common/formatter/substitution_formatter.cc
     - source/common/formatter/substitution_format_string.cc
     - source/common/stats/tag_extractor_impl.cc

From 6b99feb17872146adeb4fc3d5a9c26b9ab4ef996 Mon Sep 17 00:00:00 2001
From: Rama Chavali 
Date: Wed, 7 Jun 2023 06:22:37 +0530
Subject: [PATCH 474/740] add support for additional cookie attributes (#27529)

Signed-off-by: Rama Chavali 
---
 .../config/route/v3/route_components.proto    | 15 ++++++
 changelogs/current.yaml                       |  5 +-
 envoy/http/hash_policy.h                      | 20 +++++++-
 source/common/http/hash_policy.cc             | 18 +++++--
 source/common/http/utility.cc                 | 12 ++++-
 source/common/http/utility.h                  |  2 +-
 source/common/router/router.h                 | 11 ++--
 .../http/stateful_session/cookie/BUILD        |  1 +
 .../http/stateful_session/cookie/cookie.h     |  4 +-
 test/common/http/utility_fuzz_test.cc         |  3 +-
 test/common/http/utility_test.cc              | 37 +++++++++++---
 test/common/router/config_impl_test.cc        | 51 +++++++++++--------
 test/common/router/router_test.cc             | 12 +++--
 .../stateful_session_integration_test.cc      | 16 +++---
 .../stateful_session/cookie/cookie_test.cc    |  7 +--
 15 files changed, 156 insertions(+), 58 deletions(-)

diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto
index 56686a096050..7153a100772a 100644
--- a/api/envoy/config/route/v3/route_components.proto
+++ b/api/envoy/config/route/v3/route_components.proto
@@ -833,6 +833,18 @@ message RouteAction {
       type.matcher.v3.RegexMatchAndSubstitute regex_rewrite = 2;
     }
 
+    // CookieAttribute defines an API for adding additional attributes for a HTTP cookie.
+    message CookieAttribute {
+      // The name of the cookie attribute.
+      string name = 1
+          [(validate.rules).string =
+               {min_len: 1 max_bytes: 16384 well_known_regex: HTTP_HEADER_NAME strict: false}];
+
+      // The optional value of the cookie attribute.
+      string value = 2 [(validate.rules).string =
+                            {max_bytes: 16384 well_known_regex: HTTP_HEADER_VALUE strict: false}];
+    }
+
     // Envoy supports two types of cookie affinity:
     //
     // 1. Passive. Envoy takes a cookie that's present in the cookies header and
@@ -864,6 +876,9 @@ message RouteAction {
       // The name of the path for the cookie. If no path is specified here, no path
       // will be set for the cookie.
       string path = 3;
+
+      // Additional attributes for the cookie. They will be used when generating a new cookie.
+      repeated CookieAttribute attributes = 4;
     }
 
     message ConnectionProperties {
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 39444ae95ea7..841fea503761 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -309,7 +309,7 @@ new_features:
 - area: ext_proc
   change: |
     added new field ``filter_metadata `
@@ -320,6 +320,9 @@ new_features:
     This works with dynamic secrets when
     :ref:`CertificateValidationContext `
     is delivered via SDS.
+- area: http
+  change: |
+    added support for configuring additional :ref:`cookie attributes `.
 
 deprecated:
 - area: access_log
diff --git a/envoy/http/hash_policy.h b/envoy/http/hash_policy.h
index de04a8eecfa2..3a8e17ffc80c 100644
--- a/envoy/http/hash_policy.h
+++ b/envoy/http/hash_policy.h
@@ -9,6 +9,23 @@
 namespace Envoy {
 namespace Http {
 
+/**
+ * CookieAttribute that stores the name and value of a cookie.
+ */
+class CookieAttribute {
+public:
+  CookieAttribute(const std::string& name, const std::string& value) : name_(name), value_(value) {}
+
+  std::string name() const { return name_; }
+  std::string value() const { return value_; }
+
+private:
+  std::string name_;
+  std::string value_;
+};
+
+using CookieAttributeRefVector = std::vector>;
+
 /**
  * Request hash policy. I.e., if using a hashing load balancer, how a request should be hashed onto
  * an upstream host.
@@ -25,7 +42,8 @@ class HashPolicy {
    * @return std::string the opaque value of the cookie that will be set
    */
   using AddCookieCallback = std::function;
+      const std::string& key, const std::string& path, std::chrono::seconds ttl,
+      const CookieAttributeRefVector attributes)>;
 
   /**
    * @param downstream_address is the address of the connected client host, or nullptr if the
diff --git a/source/common/http/hash_policy.cc b/source/common/http/hash_policy.cc
index a12d5730a085..ca97775f318e 100644
--- a/source/common/http/hash_policy.cc
+++ b/source/common/http/hash_policy.cc
@@ -80,8 +80,9 @@ class HeaderHashMethod : public HashMethodImplBase {
 class CookieHashMethod : public HashMethodImplBase {
 public:
   CookieHashMethod(const std::string& key, const std::string& path,
-                   const absl::optional& ttl, bool terminal)
-      : HashMethodImplBase(terminal), key_(key), path_(path), ttl_(ttl) {}
+                   const absl::optional& ttl, bool terminal,
+                   const CookieAttributeRefVector attributes)
+      : HashMethodImplBase(terminal), key_(key), path_(path), ttl_(ttl), attributes_(attributes) {}
 
   absl::optional evaluate(const Network::Address::Instance*,
                                     const RequestHeaderMap& headers,
@@ -90,7 +91,7 @@ class CookieHashMethod : public HashMethodImplBase {
     absl::optional hash;
     std::string value = Utility::parseCookieValue(headers, key_);
     if (value.empty() && ttl_.has_value()) {
-      value = add_cookie(key_, path_, ttl_.value());
+      value = add_cookie(key_, path_, ttl_.value(), attributes_);
       hash = HashUtil::xxHash64(value);
 
     } else if (!value.empty()) {
@@ -103,6 +104,7 @@ class CookieHashMethod : public HashMethodImplBase {
   const std::string key_;
   const std::string path_;
   const absl::optional ttl_;
+  const CookieAttributeRefVector attributes_;
 };
 
 class IpHashMethod : public HashMethodImplBase {
@@ -188,9 +190,17 @@ HashPolicyImpl::HashPolicyImpl(
       if (hash_policy->cookie().has_ttl()) {
         ttl = std::chrono::seconds(hash_policy->cookie().ttl().seconds());
       }
+      std::vector attributes;
+      for (const auto& attribute : hash_policy->cookie().attributes()) {
+        attributes.push_back({attribute.name(), attribute.value()});
+      }
+      CookieAttributeRefVector ref_attributes;
+      for (const auto& attribute : attributes) {
+        ref_attributes.push_back(attribute);
+      }
       hash_impls_.emplace_back(new CookieHashMethod(hash_policy->cookie().name(),
                                                     hash_policy->cookie().path(), ttl,
-                                                    hash_policy->terminal()));
+                                                    hash_policy->terminal(), ref_attributes));
       break;
     }
     case envoy::config::route::v3::RouteAction::HashPolicy::PolicySpecifierCase::
diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc
index b805297d5e14..cfc300b18d22 100644
--- a/source/common/http/utility.cc
+++ b/source/common/http/utility.cc
@@ -588,7 +588,8 @@ std::string Utility::parseSetCookieValue(const Http::HeaderMap& headers, const s
 
 std::string Utility::makeSetCookieValue(const std::string& key, const std::string& value,
                                         const std::string& path, const std::chrono::seconds max_age,
-                                        bool httponly) {
+                                        bool httponly,
+                                        const Http::CookieAttributeRefVector attributes) {
   std::string cookie_value;
   // Best effort attempt to avoid numerous string copies.
   cookie_value.reserve(value.size() + path.size() + 30);
@@ -600,6 +601,15 @@ std::string Utility::makeSetCookieValue(const std::string& key, const std::strin
   if (!path.empty()) {
     absl::StrAppend(&cookie_value, "; Path=", path);
   }
+
+  for (auto const& attribute : attributes) {
+    if (attribute.get().value().empty()) {
+      absl::StrAppend(&cookie_value, "; ", attribute.get().name());
+    } else {
+      absl::StrAppend(&cookie_value, "; ", attribute.get().name(), "=", attribute.get().value());
+    }
+  }
+
   if (httponly) {
     absl::StrAppend(&cookie_value, "; HttpOnly");
   }
diff --git a/source/common/http/utility.h b/source/common/http/utility.h
index 76a708b773ca..87ec1cd02398 100644
--- a/source/common/http/utility.h
+++ b/source/common/http/utility.h
@@ -364,7 +364,7 @@ std::string parseSetCookieValue(const HeaderMap& headers, const std::string& key
  */
 std::string makeSetCookieValue(const std::string& key, const std::string& value,
                                const std::string& path, const std::chrono::seconds max_age,
-                               bool httponly);
+                               bool httponly, const Http::CookieAttributeRefVector attributes);
 
 /**
  * Get the response status from the response headers.
diff --git a/source/common/router/router.h b/source/common/router/router.h
index c3c3b61c8180..ad75603b1740 100644
--- a/source/common/router/router.h
+++ b/source/common/router/router.h
@@ -13,6 +13,7 @@
 #include "envoy/http/codes.h"
 #include "envoy/http/filter.h"
 #include "envoy/http/filter_factory.h"
+#include "envoy/http/hash_policy.h"
 #include "envoy/http/stateful_session.h"
 #include "envoy/local_info/local_info.h"
 #include "envoy/router/shadow_writer.h"
@@ -408,8 +409,9 @@ class Filter : Logger::Loggable,
         return hash_policy->generateHash(
             callbacks_->streamInfo().downstreamAddressProvider().remoteAddress().get(),
             *downstream_headers_,
-            [this](const std::string& key, const std::string& path, std::chrono::seconds max_age) {
-              return addDownstreamSetCookie(key, path, max_age);
+            [this](const std::string& key, const std::string& path, std::chrono::seconds max_age,
+                   Http::CookieAttributeRefVector attributes) {
+              return addDownstreamSetCookie(key, path, max_age, attributes);
             },
             callbacks_->streamInfo().filterState());
       }
@@ -500,7 +502,8 @@ class Filter : Logger::Loggable,
    * @return std::string the value of the new cookie
    */
   std::string addDownstreamSetCookie(const std::string& key, const std::string& path,
-                                     std::chrono::seconds max_age) {
+                                     std::chrono::seconds max_age,
+                                     Http::CookieAttributeRefVector attributes) {
     // The cookie value should be the same per connection so that if multiple
     // streams race on the same path, they all receive the same cookie.
     // Since the downstream port is part of the hashed value, multiple HTTP1
@@ -513,7 +516,7 @@ class Filter : Logger::Loggable,
 
     const std::string cookie_value = Hex::uint64ToHex(HashUtil::xxHash64(value));
     downstream_set_cookies_.emplace_back(
-        Http::Utility::makeSetCookieValue(key, cookie_value, path, max_age, true));
+        Http::Utility::makeSetCookieValue(key, cookie_value, path, max_age, true, attributes));
     return cookie_value;
   }
 
diff --git a/source/extensions/http/stateful_session/cookie/BUILD b/source/extensions/http/stateful_session/cookie/BUILD
index 0f267e7cc74c..12229ff205a3 100644
--- a/source/extensions/http/stateful_session/cookie/BUILD
+++ b/source/extensions/http/stateful_session/cookie/BUILD
@@ -16,6 +16,7 @@ envoy_cc_library(
     hdrs = ["cookie.h"],
     deps = [
         ":cookie_encoding_cc_proto",
+        "//envoy/http:hash_policy_interface",
         "//envoy/http:stateful_session_interface",
         "//source/common/common:base64_lib",
         "//source/common/http:headers_lib",
diff --git a/source/extensions/http/stateful_session/cookie/cookie.h b/source/extensions/http/stateful_session/cookie/cookie.h
index a03223c5bc8d..dc580395ae00 100644
--- a/source/extensions/http/stateful_session/cookie/cookie.h
+++ b/source/extensions/http/stateful_session/cookie/cookie.h
@@ -5,6 +5,7 @@
 #include 
 
 #include "envoy/extensions/http/stateful_session/cookie/v3/cookie.pb.h"
+#include "envoy/http/hash_policy.h"
 #include "envoy/http/stateful_session.h"
 
 #include "source/common/common/base64.h"
@@ -90,12 +91,13 @@ class CookieBasedSessionStateFactory : public Envoy::Http::SessionStateFactory {
   }
 
   std::string makeSetCookie(const std::string& address) const {
-    return Envoy::Http::Utility::makeSetCookieValue(name_, address, path_, ttl_, true);
+    return Envoy::Http::Utility::makeSetCookieValue(name_, address, path_, ttl_, true, attributes_);
   }
 
   const std::string name_;
   const std::chrono::seconds ttl_;
   const std::string path_;
+  const Envoy::Http::CookieAttributeRefVector attributes_;
   TimeSource& time_source_;
 
   std::function path_matcher_;
diff --git a/test/common/http/utility_fuzz_test.cc b/test/common/http/utility_fuzz_test.cc
index ca362680fb9f..dc311df0bf07 100644
--- a/test/common/http/utility_fuzz_test.cc
+++ b/test/common/http/utility_fuzz_test.cc
@@ -71,8 +71,9 @@ DEFINE_PROTO_FUZZER(const test::common::http::UtilityTestCase& input) {
   case test::common::http::UtilityTestCase::kMakeSetCookieValue: {
     const auto& cookie_value = input.make_set_cookie_value();
     std::chrono::seconds max_age(cookie_value.max_age());
+    Http::CookieAttributeRefVector cookie_attributes;
     Http::Utility::makeSetCookieValue(cookie_value.key(), cookie_value.value(), cookie_value.path(),
-                                      max_age, cookie_value.httponly());
+                                      max_age, cookie_value.httponly(), cookie_attributes);
     break;
   }
   case test::common::http::UtilityTestCase::kParseAuthorityString: {
diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc
index 43f8bb41e6af..63beeea63e0e 100644
--- a/test/common/http/utility_test.cc
+++ b/test/common/http/utility_test.cc
@@ -912,23 +912,44 @@ TEST(HttpUtility, TestParseSetCookieWithQuotes) {
 }
 
 TEST(HttpUtility, TestMakeSetCookieValue) {
+  CookieAttributeRefVector ref_attributes;
   EXPECT_EQ("name=\"value\"; Max-Age=10",
-            Utility::makeSetCookieValue("name", "value", "", std::chrono::seconds(10), false));
+            Utility::makeSetCookieValue("name", "value", "", std::chrono::seconds(10), false,
+                                        ref_attributes));
   EXPECT_EQ("name=\"value\"",
-            Utility::makeSetCookieValue("name", "value", "", std::chrono::seconds::zero(), false));
+            Utility::makeSetCookieValue("name", "value", "", std::chrono::seconds::zero(), false,
+                                        ref_attributes));
   EXPECT_EQ("name=\"value\"; Max-Age=10; HttpOnly",
-            Utility::makeSetCookieValue("name", "value", "", std::chrono::seconds(10), true));
+            Utility::makeSetCookieValue("name", "value", "", std::chrono::seconds(10), true,
+                                        ref_attributes));
   EXPECT_EQ("name=\"value\"; HttpOnly",
-            Utility::makeSetCookieValue("name", "value", "", std::chrono::seconds::zero(), true));
+            Utility::makeSetCookieValue("name", "value", "", std::chrono::seconds::zero(), true,
+                                        ref_attributes));
 
   EXPECT_EQ("name=\"value\"; Max-Age=10; Path=/",
-            Utility::makeSetCookieValue("name", "value", "/", std::chrono::seconds(10), false));
+            Utility::makeSetCookieValue("name", "value", "/", std::chrono::seconds(10), false,
+                                        ref_attributes));
   EXPECT_EQ("name=\"value\"; Path=/",
-            Utility::makeSetCookieValue("name", "value", "/", std::chrono::seconds::zero(), false));
+            Utility::makeSetCookieValue("name", "value", "/", std::chrono::seconds::zero(), false,
+                                        ref_attributes));
   EXPECT_EQ("name=\"value\"; Max-Age=10; Path=/; HttpOnly",
-            Utility::makeSetCookieValue("name", "value", "/", std::chrono::seconds(10), true));
+            Utility::makeSetCookieValue("name", "value", "/", std::chrono::seconds(10), true,
+                                        ref_attributes));
   EXPECT_EQ("name=\"value\"; Path=/; HttpOnly",
-            Utility::makeSetCookieValue("name", "value", "/", std::chrono::seconds::zero(), true));
+            Utility::makeSetCookieValue("name", "value", "/", std::chrono::seconds::zero(), true,
+                                        ref_attributes));
+
+  std::vector attributes;
+  attributes.push_back({"SameSite", "None"});
+  attributes.push_back({"Secure", ""});
+  attributes.push_back({"Partitioned", ""});
+  for (const auto& attribute : attributes) {
+    ref_attributes.push_back(attribute);
+  }
+
+  EXPECT_EQ("name=\"value\"; Path=/; SameSite=None; Secure; Partitioned; HttpOnly",
+            Utility::makeSetCookieValue("name", "value", "/", std::chrono::seconds::zero(), true,
+                                        ref_attributes));
 }
 
 TEST(HttpUtility, SendLocalReply) {
diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc
index dae94a11bb91..1ad203edca0b 100644
--- a/test/common/router/config_impl_test.cc
+++ b/test/common/router/config_impl_test.cc
@@ -2814,8 +2814,8 @@ TEST_F(RouteMatcherTest, DynamicMetadataMatchedRouting) {
 class RouterMatcherHashPolicyTest : public testing::Test, public ConfigImplTestBase {
 protected:
   RouterMatcherHashPolicyTest()
-      : add_cookie_nop_(
-            [](const std::string&, const std::string&, std::chrono::seconds) { return ""; }) {
+      : add_cookie_nop_([](const std::string&, const std::string&, std::chrono::seconds,
+                           const Http::CookieAttributeRefVector) { return ""; }) {
     const std::string yaml = R"EOF(
 virtual_hosts:
 - name: local_service
@@ -2977,16 +2977,19 @@ TEST_F(RouterMatcherCookieHashPolicyTest, DifferentCookies) {
 TEST_F(RouterMatcherCookieHashPolicyTest, TtlSet) {
   firstRouteHashPolicy()->mutable_cookie()->mutable_ttl()->set_seconds(42);
 
-  MockFunction mock_cookie_cb;
-  auto add_cookie = [&mock_cookie_cb](const std::string& name, const std::string& path,
-                                      std::chrono::seconds ttl) -> std::string {
-    return mock_cookie_cb.Call(name, path, ttl.count());
+  MockFunction
+      mock_cookie_cb;
+  auto add_cookie =
+      [&mock_cookie_cb](const std::string& name, const std::string& path, std::chrono::seconds ttl,
+                        const Http::CookieAttributeRefVector& attributes) -> std::string {
+    return mock_cookie_cb.Call(name, path, ttl.count(), attributes);
   };
 
   {
     Http::TestRequestHeaderMapImpl headers = genHeaders("www.lyft.com", "/foo", "GET");
     Router::RouteConstSharedPtr route = config().route(headers, 0);
-    EXPECT_CALL(mock_cookie_cb, Call("hash", "", 42));
+    EXPECT_CALL(mock_cookie_cb, Call("hash", "", 42, _));
     EXPECT_TRUE(
         route->routeEntry()->hashPolicy()->generateHash(nullptr, headers, add_cookie, nullptr));
   }
@@ -2994,7 +2997,7 @@ TEST_F(RouterMatcherCookieHashPolicyTest, TtlSet) {
     Http::TestRequestHeaderMapImpl headers = genHeaders("www.lyft.com", "/foo", "GET");
     headers.addCopy("Cookie", "choco=late; su=gar");
     Router::RouteConstSharedPtr route = config().route(headers, 0);
-    EXPECT_CALL(mock_cookie_cb, Call("hash", "", 42));
+    EXPECT_CALL(mock_cookie_cb, Call("hash", "", 42, _));
     EXPECT_TRUE(
         route->routeEntry()->hashPolicy()->generateHash(nullptr, headers, add_cookie, nullptr));
   }
@@ -3010,7 +3013,7 @@ TEST_F(RouterMatcherCookieHashPolicyTest, TtlSet) {
     {
       Http::TestRequestHeaderMapImpl headers = genHeaders("www.lyft.com", "/foo", "GET");
       Router::RouteConstSharedPtr route = config().route(headers, 0);
-      EXPECT_CALL(mock_cookie_cb, Call("hash", "", 42)).WillOnce(Return("AAAAAAA"));
+      EXPECT_CALL(mock_cookie_cb, Call("hash", "", 42, _)).WillOnce(Return("AAAAAAA"));
       hash_1 = route->routeEntry()
                    ->hashPolicy()
                    ->generateHash(nullptr, headers, add_cookie, nullptr)
@@ -3019,7 +3022,7 @@ TEST_F(RouterMatcherCookieHashPolicyTest, TtlSet) {
     {
       Http::TestRequestHeaderMapImpl headers = genHeaders("www.lyft.com", "/foo", "GET");
       Router::RouteConstSharedPtr route = config().route(headers, 0);
-      EXPECT_CALL(mock_cookie_cb, Call("hash", "", 42)).WillOnce(Return("BBBBBBB"));
+      EXPECT_CALL(mock_cookie_cb, Call("hash", "", 42, _)).WillOnce(Return("BBBBBBB"));
       hash_2 = route->routeEntry()
                    ->hashPolicy()
                    ->generateHash(nullptr, headers, add_cookie, nullptr)
@@ -3036,17 +3039,19 @@ TEST_F(RouterMatcherCookieHashPolicyTest, TtlSet) {
 
 TEST_F(RouterMatcherCookieHashPolicyTest, SetSessionCookie) {
   firstRouteHashPolicy()->mutable_cookie()->mutable_ttl()->set_seconds(0);
-
-  MockFunction mock_cookie_cb;
-  auto add_cookie = [&mock_cookie_cb](const std::string& name, const std::string& path,
-                                      std::chrono::seconds ttl) -> std::string {
-    return mock_cookie_cb.Call(name, path, ttl.count());
+  MockFunction
+      mock_cookie_cb;
+  auto add_cookie =
+      [&mock_cookie_cb](const std::string& name, const std::string& path, std::chrono::seconds ttl,
+                        const Http::CookieAttributeRefVector attributes) -> std::string {
+    return mock_cookie_cb.Call(name, path, ttl.count(), attributes);
   };
 
   {
     Http::TestRequestHeaderMapImpl headers = genHeaders("www.lyft.com", "/foo", "GET");
     Router::RouteConstSharedPtr route = config().route(headers, 0);
-    EXPECT_CALL(mock_cookie_cb, Call("hash", "", 0));
+    EXPECT_CALL(mock_cookie_cb, Call("hash", "", 0, _));
     EXPECT_TRUE(
         route->routeEntry()->hashPolicy()->generateHash(nullptr, headers, add_cookie, nullptr));
   }
@@ -3055,17 +3060,19 @@ TEST_F(RouterMatcherCookieHashPolicyTest, SetSessionCookie) {
 TEST_F(RouterMatcherCookieHashPolicyTest, SetCookiePath) {
   firstRouteHashPolicy()->mutable_cookie()->mutable_ttl()->set_seconds(0);
   firstRouteHashPolicy()->mutable_cookie()->set_path("/");
-
-  MockFunction mock_cookie_cb;
-  auto add_cookie = [&mock_cookie_cb](const std::string& name, const std::string& path,
-                                      std::chrono::seconds ttl) -> std::string {
-    return mock_cookie_cb.Call(name, path, ttl.count());
+  MockFunction
+      mock_cookie_cb;
+  auto add_cookie =
+      [&mock_cookie_cb](const std::string& name, const std::string& path, std::chrono::seconds ttl,
+                        const Http::CookieAttributeRefVector attributes) -> std::string {
+    return mock_cookie_cb.Call(name, path, ttl.count(), attributes);
   };
 
   {
     Http::TestRequestHeaderMapImpl headers = genHeaders("www.lyft.com", "/foo", "GET");
     Router::RouteConstSharedPtr route = config().route(headers, 0);
-    EXPECT_CALL(mock_cookie_cb, Call("hash", "/", 0));
+    EXPECT_CALL(mock_cookie_cb, Call("hash", "/", 0, _));
     EXPECT_TRUE(
         route->routeEntry()->hashPolicy()->generateHash(nullptr, headers, add_cookie, nullptr));
   }
diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc
index 82f7711cb50c..bada575c09e5 100644
--- a/test/common/router/router_test.cc
+++ b/test/common/router/router_test.cc
@@ -549,11 +549,12 @@ TEST_F(RouterTest, AddCookie) {
       }));
 
   std::string cookie_value;
+  Http::CookieAttributeRefVector cookie_attributes;
   EXPECT_CALL(callbacks_.route_->route_entry_.hash_policy_, generateHash(_, _, _, _))
       .WillOnce(Invoke([&](const Network::Address::Instance*, const Http::HeaderMap&,
                            const Http::HashPolicy::AddCookieCallback add_cookie,
                            const StreamInfo::FilterStateSharedPtr) {
-        cookie_value = add_cookie("foo", "", std::chrono::seconds(1337));
+        cookie_value = add_cookie("foo", "", std::chrono::seconds(1337), cookie_attributes);
         return absl::optional(10);
       }));
 
@@ -590,13 +591,13 @@ TEST_F(RouterTest, AddCookieNoDuplicate) {
         EXPECT_EQ(10UL, context->computeHashKey().value());
         return Upstream::HttpPoolData([]() {}, &cm_.thread_local_cluster_.conn_pool_);
       }));
-
+  Http::CookieAttributeRefVector cookie_attributes;
   EXPECT_CALL(callbacks_.route_->route_entry_.hash_policy_, generateHash(_, _, _, _))
       .WillOnce(Invoke([&](const Network::Address::Instance*, const Http::HeaderMap&,
                            const Http::HashPolicy::AddCookieCallback add_cookie,
                            const StreamInfo::FilterStateSharedPtr) {
         // this should be ignored
-        add_cookie("foo", "", std::chrono::seconds(1337));
+        add_cookie("foo", "", std::chrono::seconds(1337), cookie_attributes);
         return absl::optional(10);
       }));
 
@@ -634,12 +635,13 @@ TEST_F(RouterTest, AddMultipleCookies) {
       }));
 
   std::string choco_c, foo_c;
+  Http::CookieAttributeRefVector cookie_attributes;
   EXPECT_CALL(callbacks_.route_->route_entry_.hash_policy_, generateHash(_, _, _, _))
       .WillOnce(Invoke([&](const Network::Address::Instance*, const Http::HeaderMap&,
                            const Http::HashPolicy::AddCookieCallback add_cookie,
                            const StreamInfo::FilterStateSharedPtr) {
-        choco_c = add_cookie("choco", "", std::chrono::seconds(15));
-        foo_c = add_cookie("foo", "/path", std::chrono::seconds(1337));
+        choco_c = add_cookie("choco", "", std::chrono::seconds(15), cookie_attributes);
+        foo_c = add_cookie("foo", "/path", std::chrono::seconds(1337), cookie_attributes);
         return absl::optional(10);
       }));
 
diff --git a/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc b/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc
index 6aed8fffc6f3..a35a9a59d642 100644
--- a/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc
+++ b/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc
@@ -222,9 +222,11 @@ TEST_F(StatefulSessionIntegrationTest, NormalStatefulSession) {
     } else {
       const std::string encoded_address =
           Envoy::Base64::encode(address_string.data(), address_string.size());
+      Http::CookieAttributeRefVector cookie_attributes;
       EXPECT_EQ(
           Envoy::Http::Utility::makeSetCookieValue("global-session-cookie", encoded_address,
-                                                   "/test", std::chrono::seconds(120), true),
+                                                   "/test", std::chrono::seconds(120), true,
+                                                   cookie_attributes),
           response->headers().get(Http::LowerCaseString("set-cookie"))[0]->value().getStringView());
     }
     cleanupUpstreamAndDownstream();
@@ -421,10 +423,11 @@ TEST_F(StatefulSessionIntegrationTest, DownstreamRequestWithStatefulSessionCooki
                   ProtoCookieObject(address_string, 120, "/test", "HttpOnly"));
       } else {
         encoded_address = Envoy::Base64::encode(address_string.data(), address_string.size());
+        Http::CookieAttributeRefVector cookie_attributes;
         // The selected upstream server address would be selected to the response headers.
         EXPECT_EQ(Envoy::Http::Utility::makeSetCookieValue("global-session-cookie", encoded_address,
-                                                           "/test", std::chrono::seconds(120),
-                                                           true),
+                                                           "/test", std::chrono::seconds(120), true,
+                                                           cookie_attributes),
                   response->headers()
                       .get(Http::LowerCaseString("set-cookie"))[0]
                       ->value()
@@ -761,9 +764,10 @@ TEST_F(StatefulSessionIntegrationTest, CookieStatefulSessionOverriddenByRoute) {
       } else {
         const std::string route_encoded_address =
             Envoy::Base64::encode(route_address_string.data(), route_address_string.size());
-        EXPECT_EQ(Envoy::Http::Utility::makeSetCookieValue("route-session-cookie",
-                                                           route_encoded_address, "/test",
-                                                           std::chrono::seconds(120), true),
+        Http::CookieAttributeRefVector cookie_attributes;
+        EXPECT_EQ(Envoy::Http::Utility::makeSetCookieValue(
+                      "route-session-cookie", route_encoded_address, "/test",
+                      std::chrono::seconds(120), true, cookie_attributes),
                   response->headers()
                       .get(Http::LowerCaseString("set-cookie"))[0]
                       ->value()
diff --git a/test/extensions/http/stateful_session/cookie/cookie_test.cc b/test/extensions/http/stateful_session/cookie/cookie_test.cc
index 4420b98bb856..866edd4fd84a 100644
--- a/test/extensions/http/stateful_session/cookie/cookie_test.cc
+++ b/test/extensions/http/stateful_session/cookie/cookie_test.cc
@@ -63,12 +63,12 @@ TEST(CookieBasedSessionStateFactoryTest, SessionStateTest) {
       Envoy::Http::TestResponseHeaderMapImpl response_headers;
       // Check the format of the cookie sent back to client.
       session_state->onUpdate(mock_host, response_headers);
-
+      Envoy::Http::CookieAttributeRefVector cookie_attributes;
       EXPECT_EQ(response_headers.get_("set-cookie"),
                 Envoy::Http::Utility::makeSetCookieValue(
                     "override_host",
                     Envoy::Base64::encode(cookie_content.c_str(), cookie_content.length()), "",
-                    std::chrono::seconds(0), true));
+                    std::chrono::seconds(0), true, cookie_attributes));
     }
   }
 
@@ -128,11 +128,12 @@ TEST(CookieBasedSessionStateFactoryTest, SessionStateTest) {
       } else {
         cookie_content = "2.3.4.5:80";
       }
+      Envoy::Http::CookieAttributeRefVector cookie_attributes;
       EXPECT_EQ(response_headers.get_("set-cookie"),
                 Envoy::Http::Utility::makeSetCookieValue(
                     "override_host",
                     Envoy::Base64::encode(cookie_content.c_str(), cookie_content.length()), "/path",
-                    std::chrono::seconds(5), true));
+                    std::chrono::seconds(5), true, cookie_attributes));
     }
   }
   {

From 91a95cf1651bbac8c629b0903313e924e6ec6b03 Mon Sep 17 00:00:00 2001
From: code 
Date: Wed, 7 Jun 2023 09:39:54 +0800
Subject: [PATCH 475/740] upstream: the subset load balancing policy could be
 configured as extension (#27593)

* upstream: update interface of TypedLoadBalancerFactory to support hierarchical lb

Signed-off-by: wbpcode 

* subset lb: split child lb creation out to separated creator

Signed-off-by: wbpcode 

* subset lb: complete the development of lb

Signed-off-by: wbpcode 

* complete test of subset

Signed-off-by: wbpcode 

* fix docs

Signed-off-by: wbpcode 

* resolve conflict after merge main

Signed-off-by: wbpcode 

* fix type url in extension file

Signed-off-by: wbpcode 

* remove unnecessary comment

Signed-off-by: wbpcode 

* add more test for coverage

Signed-off-by: wbpcode 

---------

Signed-off-by: wbpcode 
---
 .../subset/v3/subset.proto                    |  17 +-
 .../upstream/load_balancing/subsets.rst       |   1 -
 envoy/upstream/load_balancer.h                |  30 ++-
 envoy/upstream/upstream.h                     |   7 +-
 .../common/upstream/cluster_manager_impl.cc   |  11 +-
 source/common/upstream/cluster_manager_impl.h |   7 +-
 .../upstream/load_balancer_factory_base.h     |  22 +-
 source/common/upstream/upstream_impl.cc       |   8 +-
 source/common/upstream/upstream_impl.h        |   6 +-
 source/extensions/extensions_metadata.yaml    |   2 +
 .../cluster_provided/config.cc                |   3 +-
 .../cluster_provided/config.h                 |   3 +-
 .../load_balancing_policies/common/BUILD      |   1 +
 .../common/factory_base.h                     |  38 ++--
 .../least_request/config.cc                   |  18 +-
 .../least_request/config.h                    |  16 +-
 .../load_balancing_policies/maglev/config.cc  |  23 ++-
 .../load_balancing_policies/maglev/config.h   |   8 +-
 .../load_balancing_policies/random/config.cc  |  18 +-
 .../load_balancing_policies/random/config.h   |  13 +-
 .../ring_hash/config.cc                       |  23 ++-
 .../ring_hash/config.h                        |   8 +-
 .../round_robin/config.cc                     |  18 +-
 .../round_robin/config.h                      |  15 +-
 .../load_balancing_policies/subset/BUILD      |   2 +
 .../load_balancing_policies/subset/config.cc  | 161 ++++++++++++++-
 .../load_balancing_policies/subset/config.h   |  17 ++
 .../subset/subset_lb.cc                       | 151 +++++++-------
 .../subset/subset_lb.h                        | 189 +++++++++++++-----
 .../upstream/cluster_manager_impl_test.cc     |   2 +-
 .../upstream/load_balancer_benchmark.cc       |  10 +-
 test/common/upstream/subset_lb_test.cc        | 111 ++++++----
 test/common/upstream/upstream_impl_test.cc    |   2 +-
 .../cluster_provided/config_test.cc           |   2 +-
 .../least_request/config_test.cc              |   7 +-
 .../maglev/config_test.cc                     |  17 +-
 .../random/config_test.cc                     |   7 +-
 .../ring_hash/config_test.cc                  |  13 +-
 .../round_robin/config_test.cc                |   6 +-
 .../load_balancing_policies/subset/BUILD      |  59 ++++++
 .../subset/config_test.cc                     | 114 +++++++++++
 .../subset/integration_test.cc                | 151 ++++++++++++++
 .../subset/subset_test.cc                     |  58 ++++++
 .../load_balancers/custom_lb_policy.h         |   3 +-
 test/mocks/upstream/BUILD                     |   1 +
 test/mocks/upstream/cluster_info.cc           |   3 +-
 test/mocks/upstream/cluster_info.h            |   3 +-
 .../upstream/typed_load_balancer_factory.h    |  12 +-
 48 files changed, 1085 insertions(+), 332 deletions(-)
 create mode 100644 test/extensions/load_balancing_policies/subset/BUILD
 create mode 100644 test/extensions/load_balancing_policies/subset/config_test.cc
 create mode 100644 test/extensions/load_balancing_policies/subset/integration_test.cc
 create mode 100644 test/extensions/load_balancing_policies/subset/subset_test.cc

diff --git a/api/envoy/extensions/load_balancing_policies/subset/v3/subset.proto b/api/envoy/extensions/load_balancing_policies/subset/v3/subset.proto
index 27610c053e13..9e24a6f1d476 100644
--- a/api/envoy/extensions/load_balancing_policies/subset/v3/subset.proto
+++ b/api/envoy/extensions/load_balancing_policies/subset/v3/subset.proto
@@ -18,7 +18,6 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 
 // [#protodoc-title: Subset Load Balancing Policy]
 // [#extension: envoy.load_balancing_policies.subset]
-// [#not-implemented-hide:]
 
 // Optionally divide the endpoints in this cluster into subsets defined by
 // endpoint metadata and selected by route and weighted cluster metadata.
@@ -104,7 +103,7 @@ message Subset {
 
       // If KEYS_SUBSET is selected, subset selector matching is performed again with metadata
       // keys reduced to
-      // :ref:`fallback_keys_subset`.
+      // :ref:`fallback_keys_subset`.
       // It allows for a fallback to a different, less specific selector if some of the keys of
       // the selector are considered optional.
       KEYS_SUBSET = 4;
@@ -130,30 +129,30 @@ message Subset {
         [(validate.rules).enum = {defined_only: true}];
 
     // Subset of
-    // :ref:`keys` used by
-    // :ref:`KEYS_SUBSET`
+    // :ref:`keys` used by
+    // :ref:`KEYS_SUBSET`
     // fallback policy.
     // It has to be a non empty list if KEYS_SUBSET fallback policy is selected.
     // For any other fallback policy the parameter is not used and should not be set.
     // Only values also present in
-    // :ref:`keys` are allowed, but
+    // :ref:`keys` are allowed, but
     // ``fallback_keys_subset`` cannot be equal to ``keys``.
     repeated string fallback_keys_subset = 3;
   }
 
   // The behavior used when no endpoint subset matches the selected route's
   // metadata. The value defaults to
-  // :ref:`NO_FALLBACK`.
+  // :ref:`NO_FALLBACK`.
   LbSubsetFallbackPolicy fallback_policy = 1 [(validate.rules).enum = {defined_only: true}];
 
   // Specifies the default subset of endpoints used during fallback if
   // fallback_policy is
-  // :ref:`DEFAULT_SUBSET`.
+  // :ref:`DEFAULT_SUBSET`.
   // Each field in default_subset is
   // compared to the matching LbEndpoint.Metadata under the ``envoy.lb``
   // namespace. It is valid for no hosts to match, in which case the behavior
   // is the same as a fallback_policy of
-  // :ref:`NO_FALLBACK`.
+  // :ref:`NO_FALLBACK`.
   google.protobuf.Struct default_subset = 2;
 
   // For each entry, LbEndpoint.Metadata's
@@ -210,7 +209,7 @@ message Subset {
   //
   // The value defaults to
   // :ref:`METADATA_NO_FALLBACK
-  // `.
+  // `.
   LbSubsetMetadataFallbackPolicy metadata_fallback_policy = 8
       [(validate.rules).enum = {defined_only: true}];
 
diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst b/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst
index f30ebf03c051..fdef94da57d6 100644
--- a/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst
+++ b/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst
@@ -1,5 +1,4 @@
 .. _arch_overview_load_balancer_subsets:
-.. _extension_envoy.load_balancing_policies.subset:
 
 Load Balancer Subsets
 ---------------------
diff --git a/envoy/upstream/load_balancer.h b/envoy/upstream/load_balancer.h
index 0d18614c8e46..c1e4cd48cefe 100644
--- a/envoy/upstream/load_balancer.h
+++ b/envoy/upstream/load_balancer.h
@@ -228,6 +228,15 @@ class ThreadAwareLoadBalancer {
 
 using ThreadAwareLoadBalancerPtr = std::unique_ptr;
 
+/*
+ * Parsed load balancer configuration that will be used to create load balancer.
+ */
+class LoadBalancerConfig {
+public:
+  virtual ~LoadBalancerConfig() = default;
+};
+using LoadBalancerConfigPtr = std::unique_ptr;
+
 /**
  * Factory config for load balancers. To support a load balancing policy of
  * LOAD_BALANCING_POLICY_CONFIG, at least one load balancer factory corresponding to a policy in
@@ -241,15 +250,30 @@ class TypedLoadBalancerFactory : public Config::TypedFactory {
   /**
    * @return ThreadAwareLoadBalancerPtr a new thread-aware load balancer.
    *
+   * @param lb_config supplies the parsed config of the load balancer.
    * @param cluster_info supplies the cluster info.
-   * @param priority_set supplies the priority set.
+   * @param priority_set supplies the priority set on the main thread.
    * @param runtime supplies the runtime loader.
    * @param random supplies the random generator.
    * @param time_source supplies the time source.
    */
   virtual ThreadAwareLoadBalancerPtr
-  create(const ClusterInfo& cluster_info, const PrioritySet& priority_set, Runtime::Loader& runtime,
-         Random::RandomGenerator& random, TimeSource& time_source) PURE;
+  create(OptRef lb_config, const ClusterInfo& cluster_info,
+         const PrioritySet& priority_set, Runtime::Loader& runtime, Random::RandomGenerator& random,
+         TimeSource& time_source) PURE;
+
+  /**
+   * This method is used to validate and create load balancer config from typed proto config.
+   *
+   * @return LoadBalancerConfigPtr a new load balancer config.
+   *
+   * @param config supplies the typed proto config of the load balancer. A dynamic_cast could
+   *        be performed on the config to the expected proto type.
+   * @param visitor supplies the validation visitor that will be used to validate the embedded
+   *        Any proto message.
+   */
+  virtual LoadBalancerConfigPtr loadConfig(ProtobufTypes::MessagePtr config,
+                                           ProtobufMessage::ValidationVisitor& visitor) PURE;
 
   std::string category() const override { return "envoy.load_balancing_policies"; }
 };
diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h
index 18460c6b6083..32bd2bddda55 100644
--- a/envoy/upstream/upstream.h
+++ b/envoy/upstream/upstream.h
@@ -823,6 +823,7 @@ using ProtocolOptionsConfigConstSharedPtr = std::shared_ptr the validated load balancing policy configuration to
+   * use for this cluster.
    */
-  virtual const ProtobufTypes::MessagePtr& loadBalancingPolicy() const PURE;
+  virtual OptRef loadBalancerConfig() const PURE;
 
   /**
    * @return the load balancer factory for this cluster if the load balancing type is
diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc
index a273a397b791..3a892ed69db1 100644
--- a/source/common/upstream/cluster_manager_impl.cc
+++ b/source/common/upstream/cluster_manager_impl.cc
@@ -909,22 +909,23 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust
       auto& factory = Config::Utility::getAndCheckFactoryByName(
           "envoy.load_balancing_policies.ring_hash");
       cluster_entry_it->second->thread_aware_lb_ = factory.create(
-          *cluster_info, cluster_reference.prioritySet(), runtime_, random_, time_source_);
+          {}, *cluster_info, cluster_reference.prioritySet(), runtime_, random_, time_source_);
     }
   } else if (cluster_info->lbType() == LoadBalancerType::Maglev) {
     if (!cluster_info->lbSubsetInfo().isEnabled()) {
       auto& factory = Config::Utility::getAndCheckFactoryByName(
           "envoy.load_balancing_policies.maglev");
       cluster_entry_it->second->thread_aware_lb_ = factory.create(
-          *cluster_info, cluster_reference.prioritySet(), runtime_, random_, time_source_);
+          {}, *cluster_info, cluster_reference.prioritySet(), runtime_, random_, time_source_);
     }
   } else if (cluster_provided_lb) {
     cluster_entry_it->second->thread_aware_lb_ = std::move(lb);
   } else if (cluster_info->lbType() == LoadBalancerType::LoadBalancingPolicyConfig) {
     TypedLoadBalancerFactory* typed_lb_factory = cluster_info->loadBalancerFactory();
     RELEASE_ASSERT(typed_lb_factory != nullptr, "ClusterInfo should contain a valid factory");
-    cluster_entry_it->second->thread_aware_lb_ = typed_lb_factory->create(
-        *cluster_info, cluster_reference.prioritySet(), runtime_, random_, time_source_);
+    cluster_entry_it->second->thread_aware_lb_ =
+        typed_lb_factory->create(cluster_info->loadBalancerConfig(), *cluster_info,
+                                 cluster_reference.prioritySet(), runtime_, random_, time_source_);
   }
 
   updateClusterCounts();
@@ -1543,7 +1544,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::addClusterUpdateCallbacks(
 ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::ClusterEntry(
     ThreadLocalClusterManagerImpl& parent, ClusterInfoConstSharedPtr cluster,
     const LoadBalancerFactorySharedPtr& lb_factory)
-    : parent_(parent), lb_factory_(lb_factory), cluster_info_(cluster),
+    : parent_(parent), cluster_info_(cluster), lb_factory_(lb_factory),
       override_host_statuses_(HostUtility::createOverrideHostStatus(cluster_info_->lbConfig())) {
   priority_set_.getOrCreateHostSet(0);
 
diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h
index 11c44a7365b1..440554a7ea7f 100644
--- a/source/common/upstream/cluster_manager_impl.h
+++ b/source/common/upstream/cluster_manager_impl.h
@@ -560,13 +560,16 @@ class ClusterManagerImpl : public ClusterManager,
 
       ThreadLocalClusterManagerImpl& parent_;
       PrioritySetImpl priority_set_;
+
+      // Don't change the order of cluster_info_ and lb_factory_/lb_ as the the lb_factory_/lb_
+      // may keep a reference to the cluster_info_.
+      ClusterInfoConstSharedPtr cluster_info_;
       // LB factory if applicable. Not all load balancer types have a factory. LB types that have
       // a factory will create a new LB on every membership update. LB types that don't have a
       // factory will create an LB on construction and use it forever.
       LoadBalancerFactorySharedPtr lb_factory_;
       // Current active LB.
       LoadBalancerPtr lb_;
-      ClusterInfoConstSharedPtr cluster_info_;
       Http::AsyncClientPtr lazy_http_async_client_;
       // Stores QUICHE specific objects which live through out the life time of the cluster and can
       // be shared across its hosts.
@@ -677,6 +680,8 @@ class ClusterManagerImpl : public ClusterManager,
     const envoy::config::cluster::v3::Cluster cluster_config_;
     const uint64_t config_hash_;
     const std::string version_info_;
+    // Don't change the order of cluster_ and thread_aware_lb_ as the thread_aware_lb_ may
+    // keep a reference to the cluster_.
     ClusterSharedPtr cluster_;
     // Optional thread aware LB depending on the LB type. Not all clusters have one.
     ThreadAwareLoadBalancerPtr thread_aware_lb_;
diff --git a/source/common/upstream/load_balancer_factory_base.h b/source/common/upstream/load_balancer_factory_base.h
index 6eaa658db555..ae9c066a550b 100644
--- a/source/common/upstream/load_balancer_factory_base.h
+++ b/source/common/upstream/load_balancer_factory_base.h
@@ -5,6 +5,19 @@
 namespace Envoy {
 namespace Upstream {
 
+class LoadBalancerConfigWrapper : public LoadBalancerConfig {
+public:
+  LoadBalancerConfigWrapper(ProtobufTypes::MessagePtr config) : config_(std::move(config)) {}
+
+  template  OptRef typedProtoConfig() const {
+    auto* typed_config = dynamic_cast(config_.get());
+    return makeOptRefFromPtr(typed_config);
+  }
+
+private:
+  ProtobufTypes::MessagePtr config_;
+};
+
 /**
  * Base class for cluster provided load balancers and load balancers specified by load balancing
  * policy config. This class should be extended directly if the load balancing policy specifies a
@@ -18,7 +31,14 @@ template  class TypedLoadBalancerFactoryBase : public TypedLoadBala
   // Upstream::TypedLoadBalancerFactory
   std::string name() const override { return name_; }
 
-  ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique(); }
+  ProtobufTypes::MessagePtr createEmptyConfigProto() override {
+    return ProtobufTypes::MessagePtr{new Proto()};
+  }
+
+  LoadBalancerConfigPtr loadConfig(ProtobufTypes::MessagePtr config,
+                                   ProtobufMessage::ValidationVisitor&) override {
+    return std::make_unique(std::move(config));
+  }
 
 protected:
   TypedLoadBalancerFactoryBase(const std::string& name) : name_(name) {}
diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc
index 5bda8581a491..0a36b908f407 100644
--- a/source/common/upstream/upstream_impl.cc
+++ b/source/common/upstream/upstream_impl.cc
@@ -1274,10 +1274,12 @@ void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Clus
             policy.typed_extension_config(), /*is_optional=*/true);
     if (factory != nullptr) {
       // Load and validate the configuration.
-      load_balancing_policy_ = factory->createEmptyConfigProto();
+      auto proto_message = factory->createEmptyConfigProto();
       Config::Utility::translateOpaqueConfig(policy.typed_extension_config().typed_config(),
-                                             context.messageValidationVisitor(),
-                                             *load_balancing_policy_);
+                                             context.messageValidationVisitor(), *proto_message);
+
+      load_balancer_config_ =
+          factory->loadConfig(std::move(proto_message), context.messageValidationVisitor());
 
       load_balancer_factory_ = factory;
       break;
diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h
index ad6f069c86c5..e030299da786 100644
--- a/source/common/upstream/upstream_impl.h
+++ b/source/common/upstream/upstream_impl.h
@@ -790,8 +790,8 @@ class ClusterInfoImpl : public ClusterInfo,
 
   // Upstream::ClusterInfo
   bool addedViaApi() const override { return added_via_api_; }
-  const ProtobufTypes::MessagePtr& loadBalancingPolicy() const override {
-    return load_balancing_policy_;
+  OptRef loadBalancerConfig() const override {
+    return makeOptRefFromPtr(load_balancer_config_.get());
   }
   TypedLoadBalancerFactory* loadBalancerFactory() const override { return load_balancer_factory_; }
   const envoy::config::cluster::v3::Cluster::CommonLbConfig& lbConfig() const override {
@@ -1057,7 +1057,7 @@ class ClusterInfoImpl : public ClusterInfo,
   std::unique_ptr lb_subset_;
   std::unique_ptr metadata_;
   std::unique_ptr typed_metadata_;
-  ProtobufTypes::MessagePtr load_balancing_policy_;
+  LoadBalancerConfigPtr load_balancer_config_;
   TypedLoadBalancerFactory* load_balancer_factory_ = nullptr;
   const std::shared_ptr
       common_lb_config_;
diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml
index 0d8f55d9fe26..9dc8d4f64040 100644
--- a/source/extensions/extensions_metadata.yaml
+++ b/source/extensions/extensions_metadata.yaml
@@ -1512,6 +1512,8 @@ envoy.load_balancing_policies.subset:
   - envoy.load_balancing_policies
   security_posture: robust_to_untrusted_downstream_and_upstream
   status: stable
+  type_urls:
+  - envoy.extensions.load_balancing_policies.subset.v3.Subset
 envoy.load_balancing_policies.cluster_provided:
   categories:
   - envoy.load_balancing_policies
diff --git a/source/extensions/load_balancing_policies/cluster_provided/config.cc b/source/extensions/load_balancing_policies/cluster_provided/config.cc
index 33245f9b7907..a1154fcb7344 100644
--- a/source/extensions/load_balancing_policies/cluster_provided/config.cc
+++ b/source/extensions/load_balancing_policies/cluster_provided/config.cc
@@ -5,7 +5,8 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace ClusterProvided {
 
-Upstream::ThreadAwareLoadBalancerPtr Factory::create(const Upstream::ClusterInfo&,
+Upstream::ThreadAwareLoadBalancerPtr Factory::create(OptRef,
+                                                     const Upstream::ClusterInfo&,
                                                      const Upstream::PrioritySet&, Runtime::Loader&,
                                                      Random::RandomGenerator&, TimeSource&) {
   // Cluster provided load balancer has empty implementation. Because it is a special case to
diff --git a/source/extensions/load_balancing_policies/cluster_provided/config.h b/source/extensions/load_balancing_policies/cluster_provided/config.h
index ef1d73da6969..df59f760498f 100644
--- a/source/extensions/load_balancing_policies/cluster_provided/config.h
+++ b/source/extensions/load_balancing_policies/cluster_provided/config.h
@@ -18,7 +18,8 @@ class Factory
 public:
   Factory() : TypedLoadBalancerFactoryBase("envoy.load_balancing_policies.cluster_provided") {}
 
-  Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo& cluster_info,
+  Upstream::ThreadAwareLoadBalancerPtr create(OptRef lb_config,
+                                              const Upstream::ClusterInfo& cluster_info,
                                               const Upstream::PrioritySet& priority_set,
                                               Runtime::Loader& runtime,
                                               Random::RandomGenerator& random,
diff --git a/source/extensions/load_balancing_policies/common/BUILD b/source/extensions/load_balancing_policies/common/BUILD
index 4c045e2ec9de..7eaecf268345 100644
--- a/source/extensions/load_balancing_policies/common/BUILD
+++ b/source/extensions/load_balancing_policies/common/BUILD
@@ -13,5 +13,6 @@ envoy_cc_library(
     hdrs = ["factory_base.h"],
     deps = [
         "//envoy/upstream:load_balancer_interface",
+        "//source/common/upstream:load_balancer_factory_base_lib",
     ],
 )
diff --git a/source/extensions/load_balancing_policies/common/factory_base.h b/source/extensions/load_balancing_policies/common/factory_base.h
index d532ec3d439f..c8d46e4481c2 100644
--- a/source/extensions/load_balancing_policies/common/factory_base.h
+++ b/source/extensions/load_balancing_policies/common/factory_base.h
@@ -4,46 +4,52 @@
 
 #include "envoy/upstream/load_balancer.h"
 
+#include "source/common/upstream/load_balancer_factory_base.h"
+
 namespace Envoy {
 namespace Extensions {
 namespace LoadBalancingPolices {
 namespace Common {
 
 template 
-class FactoryBase : public Upstream::TypedLoadBalancerFactory {
+class FactoryBase : public Upstream::TypedLoadBalancerFactoryBase {
 public:
-  FactoryBase(absl::string_view name) : name_(name) {}
+  FactoryBase(const std::string& name) : Upstream::TypedLoadBalancerFactoryBase(name) {}
 
-  Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo& cluster_info,
+  Upstream::ThreadAwareLoadBalancerPtr create(OptRef lb_config,
+                                              const Upstream::ClusterInfo& cluster_info,
                                               const Upstream::PrioritySet& priority_set,
                                               Runtime::Loader& runtime,
                                               Envoy::Random::RandomGenerator& random,
                                               TimeSource& time_source) override {
-    return std::make_unique(
-        std::make_shared(cluster_info, priority_set, runtime, random, time_source));
-  }
 
-  ProtobufTypes::MessagePtr createEmptyConfigProto() override {
-    return std::make_unique();
-  }
+    const auto* typed_lb_config =
+        dynamic_cast(lb_config.ptr());
+    auto typed_proto_config = typed_lb_config == nullptr
+                                  ? OptRef{}
+                                  : typed_lb_config->typedProtoConfig();
 
-  std::string name() const override { return name_; }
+    return std::make_unique(std::make_shared(
+        typed_proto_config, cluster_info, priority_set, runtime, random, time_source));
+  }
 
 private:
   class LbFactory : public Upstream::LoadBalancerFactory {
   public:
-    LbFactory(const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set,
-              Runtime::Loader& runtime, Envoy::Random::RandomGenerator& random,
-              TimeSource& time_source)
-        : cluster_info_(cluster_info), priority_set_(priority_set), runtime_(runtime),
-          random_(random), time_source_(time_source) {}
+    LbFactory(OptRef proto_config, const Upstream::ClusterInfo& cluster_info,
+              const Upstream::PrioritySet& priority_set, Runtime::Loader& runtime,
+              Envoy::Random::RandomGenerator& random, TimeSource& time_source)
+        : proto_config_(proto_config), cluster_info_(cluster_info), priority_set_(priority_set),
+          runtime_(runtime), random_(random), time_source_(time_source) {}
 
     Upstream::LoadBalancerPtr create() override { PANIC("not implemented"); }
     Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams params) override {
-      return Impl()(params, cluster_info_, priority_set_, runtime_, random_, time_source_);
+      return Impl()(params, proto_config_, cluster_info_, priority_set_, runtime_, random_,
+                    time_source_);
     }
 
   public:
+    OptRef proto_config_;
     const Upstream::ClusterInfo& cluster_info_;
     const Upstream::PrioritySet& priority_set_;
     Runtime::Loader& runtime_;
diff --git a/source/extensions/load_balancing_policies/least_request/config.cc b/source/extensions/load_balancing_policies/least_request/config.cc
index d2dab339ec1f..224aae7e2f6d 100644
--- a/source/extensions/load_balancing_policies/least_request/config.cc
+++ b/source/extensions/load_balancing_policies/least_request/config.cc
@@ -9,27 +9,21 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace LeastRequest {
 
-Upstream::LoadBalancerPtr LeastRequestCreator::operator()(Upstream::LoadBalancerParams params,
-                                                          const Upstream::ClusterInfo& cluster_info,
-                                                          const Upstream::PrioritySet&,
-                                                          Runtime::Loader& runtime,
-                                                          Random::RandomGenerator& random,
-                                                          TimeSource& time_source) {
-
-  const auto* typed_config = dynamic_cast<
-      const envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest*>(
-      cluster_info.loadBalancingPolicy().get());
+Upstream::LoadBalancerPtr LeastRequestCreator::operator()(
+    Upstream::LoadBalancerParams params, OptRef lb_config,
+    const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet&,
+    Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) {
 
   // The load balancing policy configuration will be loaded and validated in the main thread when we
   // load the cluster configuration. So we can assume the configuration is valid here.
-  ASSERT(typed_config != nullptr,
+  ASSERT(lb_config.has_value(),
          "Invalid load balancing policy configuration for least request load balancer");
 
   return std::make_unique(
       params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random,
       PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(),
                                                      healthy_panic_threshold, 100, 50),
-      *typed_config, time_source);
+      lb_config.value(), time_source);
 }
 
 /**
diff --git a/source/extensions/load_balancing_policies/least_request/config.h b/source/extensions/load_balancing_policies/least_request/config.h
index e2e787936c28..ffd9456d2d87 100644
--- a/source/extensions/load_balancing_policies/least_request/config.h
+++ b/source/extensions/load_balancing_policies/least_request/config.h
@@ -12,17 +12,17 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace LeastRequest {
 
+using LeastRequestLbProto =
+    envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest;
+
 struct LeastRequestCreator : public Logger::Loggable {
-  Upstream::LoadBalancerPtr operator()(Upstream::LoadBalancerParams params,
-                                       const Upstream::ClusterInfo& cluster_info,
-                                       const Upstream::PrioritySet& priority_set,
-                                       Runtime::Loader& runtime, Random::RandomGenerator& random,
-                                       TimeSource& time_source);
+  Upstream::LoadBalancerPtr
+  operator()(Upstream::LoadBalancerParams params, OptRef lb_config,
+             const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set,
+             Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source);
 };
 
-class Factory : public Common::FactoryBase<
-                    envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest,
-                    LeastRequestCreator> {
+class Factory : public Common::FactoryBase {
 public:
   Factory() : FactoryBase("envoy.load_balancing_policies.least_request") {}
 };
diff --git a/source/extensions/load_balancing_policies/maglev/config.cc b/source/extensions/load_balancing_policies/maglev/config.cc
index 0b1a4e89105f..76db3e002227 100644
--- a/source/extensions/load_balancing_policies/maglev/config.cc
+++ b/source/extensions/load_balancing_policies/maglev/config.cc
@@ -9,17 +9,20 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace Maglev {
 
-Upstream::ThreadAwareLoadBalancerPtr Factory::create(const Upstream::ClusterInfo& cluster_info,
-                                                     const Upstream::PrioritySet& priority_set,
-                                                     Runtime::Loader& runtime,
-                                                     Random::RandomGenerator& random, TimeSource&) {
-
-  const auto* typed_config =
-      dynamic_cast(
-          cluster_info.loadBalancingPolicy().get());
+Upstream::ThreadAwareLoadBalancerPtr
+Factory::create(OptRef lb_config,
+                const Upstream::ClusterInfo& cluster_info,
+                const Upstream::PrioritySet& priority_set, Runtime::Loader& runtime,
+                Random::RandomGenerator& random, TimeSource&) {
+
+  const auto* typed_lb_config =
+      dynamic_cast(lb_config.ptr());
+  auto typed_proto_config = typed_lb_config == nullptr
+                                ? OptRef{}
+                                : typed_lb_config->typedProtoConfig();
 
   // Assume legacy config.
-  if (!typed_config) {
+  if (!typed_proto_config.has_value()) {
     return std::make_unique(
         priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random,
         cluster_info.lbMaglevConfig(), cluster_info.lbConfig());
@@ -29,7 +32,7 @@ Upstream::ThreadAwareLoadBalancerPtr Factory::create(const Upstream::ClusterInfo
       priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random,
       static_cast(PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(
           cluster_info.lbConfig(), healthy_panic_threshold, 100, 50)),
-      *typed_config);
+      typed_proto_config.value());
 }
 
 /**
diff --git a/source/extensions/load_balancing_policies/maglev/config.h b/source/extensions/load_balancing_policies/maglev/config.h
index 2c072367f31e..c4eb711f2c4d 100644
--- a/source/extensions/load_balancing_policies/maglev/config.h
+++ b/source/extensions/load_balancing_policies/maglev/config.h
@@ -13,12 +13,14 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace Maglev {
 
-class Factory : public Upstream::TypedLoadBalancerFactoryBase<
-                    envoy::extensions::load_balancing_policies::maglev::v3::Maglev> {
+using MaglevLbProto = envoy::extensions::load_balancing_policies::maglev::v3::Maglev;
+
+class Factory : public Upstream::TypedLoadBalancerFactoryBase {
 public:
   Factory() : TypedLoadBalancerFactoryBase("envoy.load_balancing_policies.maglev") {}
 
-  Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo& cluster_info,
+  Upstream::ThreadAwareLoadBalancerPtr create(OptRef lb_config,
+                                              const Upstream::ClusterInfo& cluster_info,
                                               const Upstream::PrioritySet& priority_set,
                                               Runtime::Loader& runtime,
                                               Random::RandomGenerator& random,
diff --git a/source/extensions/load_balancing_policies/random/config.cc b/source/extensions/load_balancing_policies/random/config.cc
index 86d8394f32f7..0bad06c2bea7 100644
--- a/source/extensions/load_balancing_policies/random/config.cc
+++ b/source/extensions/load_balancing_policies/random/config.cc
@@ -9,27 +9,21 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace Random {
 
-Upstream::LoadBalancerPtr RandomCreator::operator()(Upstream::LoadBalancerParams params,
-                                                    const Upstream::ClusterInfo& cluster_info,
-                                                    const Upstream::PrioritySet&,
-                                                    Runtime::Loader& runtime,
-                                                    Envoy::Random::RandomGenerator& random,
-                                                    TimeSource&) {
-
-  const auto* typed_config =
-      dynamic_cast(
-          cluster_info.loadBalancingPolicy().get());
+Upstream::LoadBalancerPtr RandomCreator::operator()(
+    Upstream::LoadBalancerParams params, OptRef lb_config,
+    const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet&,
+    Runtime::Loader& runtime, Envoy::Random::RandomGenerator& random, TimeSource&) {
 
   // The load balancing policy configuration will be loaded and validated in the main thread when we
   // load the cluster configuration. So we can assume the configuration is valid here.
-  ASSERT(typed_config != nullptr,
+  ASSERT(lb_config.has_value(),
          "Invalid load balancing policy configuration for random load balancer");
 
   return std::make_unique(
       params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random,
       PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(),
                                                      healthy_panic_threshold, 100, 50),
-      *typed_config);
+      lb_config.value());
 }
 
 /**
diff --git a/source/extensions/load_balancing_policies/random/config.h b/source/extensions/load_balancing_policies/random/config.h
index b40754536a67..0d637262dfc3 100644
--- a/source/extensions/load_balancing_policies/random/config.h
+++ b/source/extensions/load_balancing_policies/random/config.h
@@ -12,16 +12,17 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace Random {
 
+using RandomLbProto = envoy::extensions::load_balancing_policies::random::v3::Random;
+
 struct RandomCreator : public Logger::Loggable {
   Upstream::LoadBalancerPtr
-  operator()(Upstream::LoadBalancerParams params, const Upstream::ClusterInfo& cluster_info,
-             const Upstream::PrioritySet& priority_set, Runtime::Loader& runtime,
-             Envoy::Random::RandomGenerator& random, TimeSource& time_source);
+  operator()(Upstream::LoadBalancerParams params, OptRef lb_config,
+             const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set,
+             Runtime::Loader& runtime, Envoy::Random::RandomGenerator& random,
+             TimeSource& time_source);
 };
 
-class Factory
-    : public Common::FactoryBase {
+class Factory : public Common::FactoryBase {
 public:
   Factory() : FactoryBase("envoy.load_balancing_policies.random") {}
 };
diff --git a/source/extensions/load_balancing_policies/ring_hash/config.cc b/source/extensions/load_balancing_policies/ring_hash/config.cc
index 41972d1d5252..05dc3ab98099 100644
--- a/source/extensions/load_balancing_policies/ring_hash/config.cc
+++ b/source/extensions/load_balancing_policies/ring_hash/config.cc
@@ -7,17 +7,20 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace RingHash {
 
-Upstream::ThreadAwareLoadBalancerPtr Factory::create(const Upstream::ClusterInfo& cluster_info,
-                                                     const Upstream::PrioritySet& priority_set,
-                                                     Runtime::Loader& runtime,
-                                                     Random::RandomGenerator& random, TimeSource&) {
-
-  const auto* typed_config =
-      dynamic_cast(
-          cluster_info.loadBalancingPolicy().get());
+Upstream::ThreadAwareLoadBalancerPtr
+Factory::create(OptRef lb_config,
+                const Upstream::ClusterInfo& cluster_info,
+                const Upstream::PrioritySet& priority_set, Runtime::Loader& runtime,
+                Random::RandomGenerator& random, TimeSource&) {
+
+  const auto* typed_lb_config =
+      dynamic_cast(lb_config.ptr());
+  auto typed_proto_config = typed_lb_config == nullptr
+                                ? OptRef{}
+                                : typed_lb_config->typedProtoConfig();
 
   // Assume legacy config.
-  if (!typed_config) {
+  if (!typed_proto_config.has_value()) {
     return std::make_unique(
         priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random,
         cluster_info.lbRingHashConfig(), cluster_info.lbConfig());
@@ -27,7 +30,7 @@ Upstream::ThreadAwareLoadBalancerPtr Factory::create(const Upstream::ClusterInfo
       priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random,
       PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(),
                                                      healthy_panic_threshold, 100, 50),
-      *typed_config);
+      typed_proto_config.value());
 }
 
 /**
diff --git a/source/extensions/load_balancing_policies/ring_hash/config.h b/source/extensions/load_balancing_policies/ring_hash/config.h
index 4e4a8c0e61cd..fdbac8d590cc 100644
--- a/source/extensions/load_balancing_policies/ring_hash/config.h
+++ b/source/extensions/load_balancing_policies/ring_hash/config.h
@@ -13,12 +13,14 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace RingHash {
 
-class Factory : public Upstream::TypedLoadBalancerFactoryBase<
-                    envoy::extensions::load_balancing_policies::ring_hash::v3::RingHash> {
+using RingHashLbProto = envoy::extensions::load_balancing_policies::ring_hash::v3::RingHash;
+
+class Factory : public Upstream::TypedLoadBalancerFactoryBase {
 public:
   Factory() : TypedLoadBalancerFactoryBase("envoy.load_balancing_policies.ring_hash") {}
 
-  Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo& cluster_info,
+  Upstream::ThreadAwareLoadBalancerPtr create(OptRef lb_config,
+                                              const Upstream::ClusterInfo& cluster_info,
                                               const Upstream::PrioritySet& priority_set,
                                               Runtime::Loader& runtime,
                                               Random::RandomGenerator& random,
diff --git a/source/extensions/load_balancing_policies/round_robin/config.cc b/source/extensions/load_balancing_policies/round_robin/config.cc
index a55d47fa9c68..fff5fb6483bc 100644
--- a/source/extensions/load_balancing_policies/round_robin/config.cc
+++ b/source/extensions/load_balancing_policies/round_robin/config.cc
@@ -9,27 +9,21 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace RoundRobin {
 
-Upstream::LoadBalancerPtr RoundRobinCreator::operator()(Upstream::LoadBalancerParams params,
-                                                        const Upstream::ClusterInfo& cluster_info,
-                                                        const Upstream::PrioritySet&,
-                                                        Runtime::Loader& runtime,
-                                                        Random::RandomGenerator& random,
-                                                        TimeSource& time_source) {
-
-  const auto* typed_config =
-      dynamic_cast(
-          cluster_info.loadBalancingPolicy().get());
+Upstream::LoadBalancerPtr RoundRobinCreator::operator()(
+    Upstream::LoadBalancerParams params, OptRef lb_config,
+    const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet&,
+    Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) {
 
   // The load balancing policy configuration will be loaded and validated in the main thread when we
   // load the cluster configuration. So we can assume the configuration is valid here.
-  ASSERT(typed_config != nullptr,
+  ASSERT(lb_config.has_value(),
          "Invalid load balancing policy configuration for least request load balancer");
 
   return std::make_unique(
       params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random,
       PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(),
                                                      healthy_panic_threshold, 100, 50),
-      *typed_config, time_source);
+      lb_config.value(), time_source);
 }
 
 /**
diff --git a/source/extensions/load_balancing_policies/round_robin/config.h b/source/extensions/load_balancing_policies/round_robin/config.h
index b0138b86dc01..4e06b5b48de1 100644
--- a/source/extensions/load_balancing_policies/round_robin/config.h
+++ b/source/extensions/load_balancing_policies/round_robin/config.h
@@ -12,17 +12,16 @@ namespace Extensions {
 namespace LoadBalancingPolices {
 namespace RoundRobin {
 
+using RoundRobinLbProto = envoy::extensions::load_balancing_policies::round_robin::v3::RoundRobin;
+
 struct RoundRobinCreator : public Logger::Loggable {
-  Upstream::LoadBalancerPtr operator()(Upstream::LoadBalancerParams params,
-                                       const Upstream::ClusterInfo& cluster_info,
-                                       const Upstream::PrioritySet& priority_set,
-                                       Runtime::Loader& runtime, Random::RandomGenerator& random,
-                                       TimeSource& time_source);
+  Upstream::LoadBalancerPtr
+  operator()(Upstream::LoadBalancerParams params, OptRef lb_config,
+             const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set,
+             Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source);
 };
 
-class Factory : public Common::FactoryBase<
-                    envoy::extensions::load_balancing_policies::round_robin::v3::RoundRobin,
-                    RoundRobinCreator> {
+class Factory : public Common::FactoryBase {
 public:
   Factory() : FactoryBase("envoy.load_balancing_policies.round_robin") {}
 };
diff --git a/source/extensions/load_balancing_policies/subset/BUILD b/source/extensions/load_balancing_policies/subset/BUILD
index ec66a84e18a7..7e001486b3a8 100644
--- a/source/extensions/load_balancing_policies/subset/BUILD
+++ b/source/extensions/load_balancing_policies/subset/BUILD
@@ -23,10 +23,12 @@ envoy_cc_library(
         "//source/common/protobuf:utility_lib",
         "//source/common/upstream:load_balancer_lib",
         "//source/common/upstream:upstream_lib",
+        "//source/extensions/load_balancing_policies/common:factory_base",
         "//source/extensions/load_balancing_policies/maglev:maglev_lb_lib",
         "//source/extensions/load_balancing_policies/ring_hash:ring_hash_lb_lib",
         "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto",
     ],
 )
 
diff --git a/source/extensions/load_balancing_policies/subset/config.cc b/source/extensions/load_balancing_policies/subset/config.cc
index 8944814252d8..486d3c77b733 100644
--- a/source/extensions/load_balancing_policies/subset/config.cc
+++ b/source/extensions/load_balancing_policies/subset/config.cc
@@ -12,11 +12,13 @@ Upstream::LoadBalancerPtr Factory::create(const Upstream::ClusterInfo& cluster,
                                           const Upstream::PrioritySet* local_priority_set,
                                           Runtime::Loader& runtime, Random::RandomGenerator& random,
                                           TimeSource& time_source) {
+  auto child_lb_creator = std::make_unique(
+      cluster.lbType(), cluster.lbRingHashConfig(), cluster.lbMaglevConfig(),
+      cluster.lbRoundRobinConfig(), cluster.lbLeastRequestConfig(), cluster.lbConfig());
+
   return std::make_unique(
-      cluster.lbType(), priority_set, local_priority_set, cluster.lbStats(), cluster.statsScope(),
-      runtime, random, cluster.lbSubsetInfo(), cluster.lbRingHashConfig(), cluster.lbMaglevConfig(),
-      cluster.lbRoundRobinConfig(), cluster.lbLeastRequestConfig(), cluster.lbConfig(),
-      time_source);
+      cluster.lbSubsetInfo(), std::move(child_lb_creator), priority_set, local_priority_set,
+      cluster.lbStats(), cluster.statsScope(), runtime, random, time_source);
 }
 
 /**
@@ -24,6 +26,157 @@ Upstream::LoadBalancerPtr Factory::create(const Upstream::ClusterInfo& cluster,
  */
 REGISTER_FACTORY(Factory, Upstream::NonThreadAwareLoadBalancerFactory);
 
+using LoadBalancerSubsetInfoImpl =
+    Upstream::LoadBalancerSubsetInfoImplBase;
+
+class SubsetLoadBalancerConfig : public Upstream::LoadBalancerConfig {
+public:
+  SubsetLoadBalancerConfig(const Upstream::SubsetLoadbalancingPolicyProto& subset_config,
+                           ProtobufMessage::ValidationVisitor& visitor)
+      : subset_info_(subset_config) {
+
+    absl::InlinedVector missing_policies;
+
+    for (const auto& policy : subset_config.subset_lb_policy().policies()) {
+      auto* factory = Config::Utility::getAndCheckFactory(
+          policy.typed_extension_config(), /*is_optional=*/true);
+
+      if (factory != nullptr) {
+        // Load and validate the configuration.
+        auto sub_lb_proto_message = factory->createEmptyConfigProto();
+        Config::Utility::translateOpaqueConfig(policy.typed_extension_config().typed_config(),
+                                               visitor, *sub_lb_proto_message);
+
+        sub_load_balancer_config_ = factory->loadConfig(std::move(sub_lb_proto_message), visitor);
+        sub_load_balancer_factory_ = factory;
+        break;
+      }
+
+      missing_policies.push_back(policy.typed_extension_config().name());
+    }
+
+    if (sub_load_balancer_factory_ == nullptr) {
+      throw EnvoyException(fmt::format("cluster: didn't find a registered load balancer factory "
+                                       "implementation for subset lb with names from [{}]",
+                                       absl::StrJoin(missing_policies, ", ")));
+    }
+  }
+
+  Upstream::ThreadAwareLoadBalancerPtr
+  createLoadBalancer(const Upstream::ClusterInfo& cluster_info,
+                     const Upstream::PrioritySet& child_priority_set, Runtime::Loader& runtime,
+                     Random::RandomGenerator& random, TimeSource& time_source) const {
+    return sub_load_balancer_factory_->create(*sub_load_balancer_config_, cluster_info,
+                                              child_priority_set, runtime, random, time_source);
+  }
+
+  const Upstream::LoadBalancerSubsetInfo& subsetInfo() const { return subset_info_; }
+
+private:
+  LoadBalancerSubsetInfoImpl subset_info_;
+
+  Upstream::LoadBalancerConfigPtr sub_load_balancer_config_;
+  Upstream::TypedLoadBalancerFactory* sub_load_balancer_factory_{};
+};
+
+class ChildLoadBalancerCreatorImpl : public Upstream::ChildLoadBalancerCreator {
+public:
+  ChildLoadBalancerCreatorImpl(const SubsetLoadBalancerConfig& subset_config,
+                               const Upstream::ClusterInfo& cluster_info)
+      : subset_config_(subset_config), cluster_info_(cluster_info) {}
+
+  std::pair
+  createLoadBalancer(const Upstream::PrioritySet& child_priority_set, const Upstream::PrioritySet*,
+                     Upstream::ClusterLbStats&, Stats::Scope&, Runtime::Loader& runtime,
+                     Random::RandomGenerator& random, TimeSource& time_source) override {
+    return {subset_config_.createLoadBalancer(cluster_info_, child_priority_set, runtime, random,
+                                              time_source),
+            nullptr};
+  }
+
+private:
+  const SubsetLoadBalancerConfig& subset_config_;
+  const Upstream::ClusterInfo& cluster_info_;
+};
+
+class LbFactory : public Upstream::LoadBalancerFactory {
+public:
+  LbFactory(const SubsetLoadBalancerConfig& subset_config,
+            const Upstream::ClusterInfo& cluster_info, Runtime::Loader& runtime,
+            Random::RandomGenerator& random, TimeSource& time_source)
+      : subset_config_(subset_config), cluster_info_(cluster_info), runtime_(runtime),
+        random_(random), time_source_(time_source) {}
+
+  Upstream::LoadBalancerPtr create() override { PANIC("not implemented"); };
+
+  Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams params) override {
+    auto child_lb_creator =
+        std::make_unique(subset_config_, cluster_info_);
+
+    return std::make_unique(
+        subset_config_.subsetInfo(), std::move(child_lb_creator), params.priority_set,
+        params.local_priority_set, cluster_info_.lbStats(), cluster_info_.statsScope(), runtime_,
+        random_, time_source_);
+  }
+
+private:
+  const SubsetLoadBalancerConfig& subset_config_;
+  const Upstream::ClusterInfo& cluster_info_;
+
+  Runtime::Loader& runtime_;
+  Random::RandomGenerator& random_;
+  TimeSource& time_source_;
+};
+
+class ThreadAwareLb : public Upstream::ThreadAwareLoadBalancer {
+public:
+  ThreadAwareLb(Upstream::LoadBalancerFactorySharedPtr factory) : factory_(std::move(factory)) {}
+
+  Upstream::LoadBalancerFactorySharedPtr factory() override { return factory_; }
+  void initialize() override {}
+
+private:
+  Upstream::LoadBalancerFactorySharedPtr factory_;
+};
+
+Upstream::ThreadAwareLoadBalancerPtr
+SubsetLbFactory::create(OptRef lb_config,
+                        const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet&,
+                        Runtime::Loader& runtime, Random::RandomGenerator& random,
+                        TimeSource& time_source) {
+
+  const auto* typed_config = dynamic_cast(lb_config.ptr());
+  // The load balancing policy configuration will be loaded and validated in the main thread when we
+  // load the cluster configuration. So we can assume the configuration is valid here.
+  ASSERT(typed_config != nullptr,
+         "Invalid load balancing policy configuration for subset load balancer");
+
+  // Create the load balancer factory that will be used to create the load balancer in the workers.
+  auto lb_factory =
+      std::make_shared(*typed_config, cluster_info, runtime, random, time_source);
+
+  // Move and store the load balancer factory in the thread aware load balancer. This thread aware
+  // load balancer is simply a wrapper of the load balancer factory for subset lb and no actual
+  // logic is implemented.
+  return std::make_unique(std::move(lb_factory));
+}
+
+Upstream::LoadBalancerConfigPtr
+SubsetLbFactory::loadConfig(ProtobufTypes::MessagePtr config,
+                            ProtobufMessage::ValidationVisitor& visitor) {
+  ASSERT(config != nullptr);
+  auto* proto_config = dynamic_cast(config.get());
+
+  // Load the subset load balancer configuration. This will contains child load balancer
+  // config and child load balancer factory.
+  return std::make_unique(*proto_config, visitor);
+}
+
+/**
+ * Static registration for the Factory. @see RegisterFactory.
+ */
+REGISTER_FACTORY(SubsetLbFactory, Upstream::TypedLoadBalancerFactory);
+
 } // namespace Subset
 } // namespace LoadBalancingPolices
 } // namespace Extensions
diff --git a/source/extensions/load_balancing_policies/subset/config.h b/source/extensions/load_balancing_policies/subset/config.h
index d3ed76b688d5..8476062ab0b5 100644
--- a/source/extensions/load_balancing_policies/subset/config.h
+++ b/source/extensions/load_balancing_policies/subset/config.h
@@ -5,6 +5,7 @@
 #include "envoy/upstream/load_balancer.h"
 
 #include "source/common/upstream/load_balancer_factory_base.h"
+#include "source/extensions/load_balancing_policies/subset/subset_lb.h"
 
 namespace Envoy {
 namespace Extensions {
@@ -22,6 +23,22 @@ class Factory : public Upstream::NonThreadAwareLoadBalancerFactory {
                                    TimeSource& time_source) override;
 };
 
+class SubsetLbFactory
+    : public Upstream::TypedLoadBalancerFactoryBase {
+public:
+  SubsetLbFactory() : TypedLoadBalancerFactoryBase("envoy.load_balancing_policies.subset") {}
+
+  Upstream::ThreadAwareLoadBalancerPtr create(OptRef lb_config,
+                                              const Upstream::ClusterInfo& cluster_info,
+                                              const Upstream::PrioritySet& priority_set,
+                                              Runtime::Loader& runtime,
+                                              Random::RandomGenerator& random,
+                                              TimeSource& time_source) override;
+
+  Upstream::LoadBalancerConfigPtr loadConfig(ProtobufTypes::MessagePtr config,
+                                             ProtobufMessage::ValidationVisitor& visitor) override;
+};
+
 } // namespace Subset
 } // namespace LoadBalancingPolices
 } // namespace Extensions
diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.cc b/source/extensions/load_balancing_policies/subset/subset_lb.cc
index 532f56693af2..f9e6e8d863c3 100644
--- a/source/extensions/load_balancing_policies/subset/subset_lb.cc
+++ b/source/extensions/load_balancing_policies/subset/subset_lb.cc
@@ -22,18 +22,15 @@ namespace Upstream {
 
 using HostPredicate = std::function;
 
-SubsetLoadBalancer::SubsetLoadBalancer(
-    LoadBalancerType lb_type, const PrioritySet& priority_set,
-    const PrioritySet* local_priority_set, ClusterLbStats& stats, Stats::Scope& scope,
-    Runtime::Loader& runtime, Random::RandomGenerator& random,
-    const LoadBalancerSubsetInfo& subsets,
+LegacyChildLoadBalancerCreatorImpl::LegacyChildLoadBalancerCreatorImpl(
+    LoadBalancerType lb_type,
     OptRef lb_ring_hash_config,
     OptRef lb_maglev_config,
     OptRef round_robin_config,
     OptRef least_request_config,
-    const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config,
-    TimeSource& time_source)
-    : lb_ring_hash_config_(
+    const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config)
+    : lb_type_(lb_type),
+      lb_ring_hash_config_(
           lb_ring_hash_config.has_value()
               ? std::make_unique(
                     lb_ring_hash_config.ref())
@@ -53,14 +50,71 @@ SubsetLoadBalancer::SubsetLoadBalancer(
               ? std::make_unique(
                     least_request_config.ref())
               : nullptr),
-      common_config_(common_config), stats_(stats), scope_(scope), runtime_(runtime),
-      random_(random), fallback_policy_(subsets.fallbackPolicy()),
+      common_config_(common_config) {}
+
+std::pair
+LegacyChildLoadBalancerCreatorImpl::createLoadBalancer(
+    const Upstream::PrioritySet& child_priority_set,
+    const Upstream::PrioritySet* local_priority_set, ClusterLbStats& stats, Stats::Scope& scope,
+    Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) {
+  switch (lb_type_) {
+  case Upstream::LoadBalancerType::LeastRequest: {
+    Upstream::LoadBalancerPtr lb = std::make_unique(
+        child_priority_set, local_priority_set, stats, runtime, random, common_config_,
+        lbLeastRequestConfig(), time_source);
+    return {nullptr, std::move(lb)};
+  }
+  case Upstream::LoadBalancerType::Random: {
+    Upstream::LoadBalancerPtr lb = std::make_unique(
+        child_priority_set, local_priority_set, stats, runtime, random, common_config_);
+    return {nullptr, std::move(lb)};
+  }
+  case Upstream::LoadBalancerType::RoundRobin: {
+    Upstream::LoadBalancerPtr lb = std::make_unique(
+        child_priority_set, local_priority_set, stats, runtime, random, common_config_,
+        lbRoundRobinConfig(), time_source);
+    return {nullptr, std::move(lb)};
+  }
+  case Upstream::LoadBalancerType::RingHash: {
+    // TODO(mattklein123): The ring hash LB is thread aware, but currently the subset LB is not.
+    // We should make the subset LB thread aware since the calculations are costly, and then we
+    // can also use a thread aware sub-LB properly. The following works fine but is not optimal.
+    Upstream::ThreadAwareLoadBalancerPtr lb = std::make_unique(
+        child_priority_set, stats, scope, runtime, random, lbRingHashConfig(), common_config_);
+    return {std::move(lb), nullptr};
+  }
+  case Upstream::LoadBalancerType::Maglev: {
+    // TODO(mattklein123): The Maglev LB is thread aware, but currently the subset LB is not.
+    // We should make the subset LB thread aware since the calculations are costly, and then we
+    // can also use a thread aware sub-LB properly. The following works fine but is not optimal.
+
+    Upstream::ThreadAwareLoadBalancerPtr lb = std::make_unique(
+        child_priority_set, stats, scope, runtime, random, lbMaglevConfig(), common_config_);
+    return {std::move(lb), nullptr};
+  }
+  case Upstream::LoadBalancerType::OriginalDst:
+  case Upstream::LoadBalancerType::ClusterProvided:
+  case Upstream::LoadBalancerType::LoadBalancingPolicyConfig:
+    // These load balancer types can only be created when there is no subset configuration.
+    PANIC("not implemented");
+  }
+  return {nullptr, nullptr};
+}
+
+SubsetLoadBalancer::SubsetLoadBalancer(const LoadBalancerSubsetInfo& subsets,
+                                       ChildLoadBalancerCreatorPtr child_lb,
+                                       const PrioritySet& priority_set,
+                                       const PrioritySet* local_priority_set, ClusterLbStats& stats,
+                                       Stats::Scope& scope, Runtime::Loader& runtime,
+                                       Random::RandomGenerator& random, TimeSource& time_source)
+    : stats_(stats), scope_(scope), runtime_(runtime), random_(random), time_source_(time_source),
+      fallback_policy_(subsets.fallbackPolicy()),
       metadata_fallback_policy_(subsets.metadataFallbackPolicy()),
       default_subset_metadata_(subsets.defaultSubset().fields().begin(),
                                subsets.defaultSubset().fields().end()),
       subset_selectors_(subsets.subsetSelectors()), original_priority_set_(priority_set),
-      original_local_priority_set_(local_priority_set), time_source_(time_source),
-      lb_type_(lb_type), locality_weight_aware_(subsets.localityWeightAware()),
+      original_local_priority_set_(local_priority_set), child_lb_creator_(std::move(child_lb)),
+      locality_weight_aware_(subsets.localityWeightAware()),
       scale_locality_weight_(subsets.scaleLocalityWeight()), list_as_any_(subsets.listAsAny()) {
   ASSERT(subsets.isEnabled());
 
@@ -342,8 +396,8 @@ HostConstSharedPtr SubsetLoadBalancer::chooseHostForSelectorFallbackPolicy(
 
 // Find a host from the subsets. Sets host_chosen to false and returns nullptr if the context has
 // no metadata match criteria, if there is no matching subset, or if the matching subset contains
-// no hosts (ignoring health). Otherwise, host_chosen is true and the returns HostConstSharedPtr is
-// from the subset's load balancer (technically, it may still be nullptr).
+// no hosts (ignoring health). Otherwise, host_chosen is true and the returns HostConstSharedPtr
+// is from the subset's load balancer (technically, it may still be nullptr).
 HostConstSharedPtr SubsetLoadBalancer::tryChooseHostFromContext(LoadBalancerContext* context,
                                                                 bool& host_chosen) {
   host_chosen = false;
@@ -364,8 +418,8 @@ HostConstSharedPtr SubsetLoadBalancer::tryChooseHostFromContext(LoadBalancerCont
   return entry->lb_subset_->chooseHost(context);
 }
 
-// Iterates over the given metadata match criteria (which must be lexically sorted by key) and find
-// a matching LbSubsetEntryPtr, if any.
+// Iterates over the given metadata match criteria (which must be lexically sorted by key) and
+// find a matching LbSubsetEntryPtr, if any.
 SubsetLoadBalancer::LbSubsetEntryPtr SubsetLoadBalancer::findSubset(
     const std::vector& match_criteria) {
   const LbSubsetMap* subsets = &subsets_;
@@ -374,8 +428,8 @@ SubsetLoadBalancer::LbSubsetEntryPtr SubsetLoadBalancer::findSubset(
   // same order, we can iterate over the criteria and perform a lookup for each key and value,
   // starting with the root LbSubsetMap and using the previous iteration's LbSubsetMap thereafter
   // (tracked in subsets). If ever a criterion's key or value is not found, there is no subset for
-  // this criteria. If we reach the last criterion, we've found the LbSubsetEntry for the criteria,
-  // which may or may not have a subset attached to it.
+  // this criteria. If we reach the last criterion, we've found the LbSubsetEntry for the
+  // criteria, which may or may not have a subset attached to it.
   for (uint32_t i = 0; i < match_criteria.size(); i++) {
     const Router::MetadataMatchCriterion& match_criterion = *match_criteria[i];
     const auto& subset_it = subsets->find(match_criterion.name());
@@ -483,8 +537,8 @@ void SubsetLoadBalancer::processSubsets(uint32_t priority, const HostVector& all
   }
 
   // This stat isn't added to `ClusterTrafficStats` because it wouldn't be used for nearly all
-  // clusters, and is only set during configuration updates, not in the data path, so performance of
-  // looking up the stat isn't critical.
+  // clusters, and is only set during configuration updates, not in the data path, so performance
+  // of looking up the stat isn't critical.
   if (single_duplicate_stat_ == nullptr) {
     Stats::StatNameManagedStorage name_storage("lb_subsets_single_host_per_subset_duplicate",
                                                scope_.symbolTable());
@@ -692,58 +746,21 @@ SubsetLoadBalancer::PrioritySubsetImpl::PrioritySubsetImpl(const SubsetLoadBalan
                                                            bool locality_weight_aware,
                                                            bool scale_locality_weight)
     : original_priority_set_(subset_lb.original_priority_set_),
+      original_local_priority_set_(subset_lb.original_local_priority_set_),
       locality_weight_aware_(locality_weight_aware), scale_locality_weight_(scale_locality_weight) {
   // Create at least one host set.
   getOrCreateHostSet(0);
 
-  switch (subset_lb.lb_type_) {
-  case LoadBalancerType::LeastRequest:
-    lb_ = std::make_unique(
-        *this, subset_lb.original_local_priority_set_, subset_lb.stats_, subset_lb.runtime_,
-        subset_lb.random_, subset_lb.common_config_, subset_lb.lbLeastRequestConfig(),
-        subset_lb.time_source_);
-    break;
-
-  case LoadBalancerType::Random:
-    lb_ = std::make_unique(*this, subset_lb.original_local_priority_set_,
-                                               subset_lb.stats_, subset_lb.runtime_,
-                                               subset_lb.random_, subset_lb.common_config_);
-    break;
-
-  case LoadBalancerType::RoundRobin:
-    lb_ = std::make_unique(
-        *this, subset_lb.original_local_priority_set_, subset_lb.stats_, subset_lb.runtime_,
-        subset_lb.random_, subset_lb.common_config_, subset_lb.lbRoundRobinConfig(),
-        subset_lb.time_source_);
-    break;
-
-  case LoadBalancerType::RingHash:
-    // TODO(mattklein123): The ring hash LB is thread aware, but currently the subset LB is not.
-    // We should make the subset LB thread aware since the calculations are costly, and then we
-    // can also use a thread aware sub-LB properly. The following works fine but is not optimal.
-    thread_aware_lb_ = std::make_unique(
-        *this, subset_lb.stats_, subset_lb.scope_, subset_lb.runtime_, subset_lb.random_,
-        subset_lb.lbRingHashConfig(), subset_lb.common_config_);
-    thread_aware_lb_->initialize();
-    lb_ = thread_aware_lb_->factory()->create();
-    break;
+  auto lb_pair = subset_lb.child_lb_creator_->createLoadBalancer(
+      *this, original_local_priority_set_, subset_lb.stats_, subset_lb.scope_, subset_lb.runtime_,
+      subset_lb.random_, subset_lb.time_source_);
 
-  case LoadBalancerType::Maglev:
-    // TODO(mattklein123): The Maglev LB is thread aware, but currently the subset LB is not.
-    // We should make the subset LB thread aware since the calculations are costly, and then we
-    // can also use a thread aware sub-LB properly. The following works fine but is not optimal.
-    thread_aware_lb_ = std::make_unique(
-        *this, subset_lb.stats_, subset_lb.scope_, subset_lb.runtime_, subset_lb.random_,
-        subset_lb.lbMaglevConfig(), subset_lb.common_config_);
+  if (lb_pair.first != nullptr) {
+    thread_aware_lb_ = std::move(lb_pair.first);
     thread_aware_lb_->initialize();
-    lb_ = thread_aware_lb_->factory()->create();
-    break;
-
-  case LoadBalancerType::OriginalDst:
-  case LoadBalancerType::ClusterProvided:
-  case LoadBalancerType::LoadBalancingPolicyConfig:
-    // These load balancer types can only be created when there is no subset configuration.
-    PANIC("not implemented");
+    lb_ = thread_aware_lb_->factory()->create({*this, original_local_priority_set_});
+  } else {
+    lb_ = std::move(lb_pair.second);
   }
 
   triggerCallbacks();
@@ -883,7 +900,7 @@ void SubsetLoadBalancer::PrioritySubsetImpl::update(uint32_t priority,
   // TODO(mattklein123): See the PrioritySubsetImpl constructor for additional comments on how
   // we can do better here.
   if (thread_aware_lb_ != nullptr) {
-    lb_ = thread_aware_lb_->factory()->create();
+    lb_ = thread_aware_lb_->factory()->create({*this, original_local_priority_set_});
   }
 }
 
diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.h b/source/extensions/load_balancing_policies/subset/subset_lb.h
index 0f5ec5ccada6..0072dcbe8006 100644
--- a/source/extensions/load_balancing_policies/subset/subset_lb.h
+++ b/source/extensions/load_balancing_policies/subset/subset_lb.h
@@ -9,6 +9,8 @@
 
 #include "envoy/common/optref.h"
 #include "envoy/config/cluster/v3/cluster.pb.h"
+#include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.h"
+#include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.validate.h"
 #include "envoy/runtime/runtime.h"
 #include "envoy/stats/scope.h"
 #include "envoy/upstream/load_balancer.h"
@@ -27,19 +29,143 @@ namespace Upstream {
 
 using HostHashSet = absl::flat_hash_set;
 
-class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable {
+using SubsetLoadbalancingPolicyProto =
+    envoy::extensions::load_balancing_policies::subset::v3::Subset;
+
+class ChildLoadBalancerCreator {
+public:
+  virtual ~ChildLoadBalancerCreator() = default;
+
+  virtual std::pair
+  createLoadBalancer(const PrioritySet& child_priority_set, const PrioritySet* local_priority_set,
+                     ClusterLbStats& stats, Stats::Scope& scope, Runtime::Loader& runtime,
+                     Random::RandomGenerator& random, TimeSource& time_source) PURE;
+};
+using ChildLoadBalancerCreatorPtr = std::unique_ptr;
+
+class LegacyChildLoadBalancerCreatorImpl : public Upstream::ChildLoadBalancerCreator {
 public:
-  SubsetLoadBalancer(
-      LoadBalancerType lb_type, const PrioritySet& priority_set,
-      const PrioritySet* local_priority_set, ClusterLbStats& stats, Stats::Scope& scope,
-      Runtime::Loader& runtime, Random::RandomGenerator& random,
-      const LoadBalancerSubsetInfo& subsets,
+  LegacyChildLoadBalancerCreatorImpl(
+      LoadBalancerType lb_type,
       OptRef lb_ring_hash_config,
       OptRef lb_maglev_config,
       OptRef round_robin_config,
       OptRef least_request_config,
-      const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config,
-      TimeSource& time_source);
+      const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config);
+
+  std::pair
+  createLoadBalancer(const Upstream::PrioritySet& child_priority_set,
+                     const Upstream::PrioritySet* local_priority_set, ClusterLbStats& stats,
+                     Stats::Scope& scope, Runtime::Loader& runtime, Random::RandomGenerator& random,
+                     TimeSource& time_source) override;
+
+  OptRef lbRoundRobinConfig() const {
+    if (round_robin_config_ != nullptr) {
+      return *round_robin_config_;
+    }
+    return absl::nullopt;
+  }
+
+  LoadBalancerType lbType() const { return lb_type_; }
+
+  OptRef
+  lbLeastRequestConfig() const {
+    if (least_request_config_ != nullptr) {
+      return *least_request_config_;
+    }
+    return absl::nullopt;
+  }
+
+  OptRef lbMaglevConfig() const {
+    if (lb_maglev_config_ != nullptr) {
+      return *lb_maglev_config_;
+    }
+    return absl::nullopt;
+  }
+
+  OptRef lbRingHashConfig() const {
+    if (lb_ring_hash_config_ != nullptr) {
+      return *lb_ring_hash_config_;
+    }
+    return absl::nullopt;
+  }
+
+private:
+  const LoadBalancerType lb_type_;
+  const std::unique_ptr
+      lb_ring_hash_config_;
+  const std::unique_ptr
+      lb_maglev_config_;
+  const std::unique_ptr
+      round_robin_config_;
+  const std::unique_ptr
+      least_request_config_;
+  const envoy::config::cluster::v3::Cluster::CommonLbConfig common_config_;
+};
+
+template 
+class LoadBalancerSubsetInfoImplBase : public Upstream::LoadBalancerSubsetInfo {
+public:
+  // TODO(wbpcode): use legacy enum for backward compatibility for now.
+  using FallbackPolicy =
+      envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy;
+  using MetadataFallbackPolicy =
+      envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetMetadataFallbackPolicy;
+  using SubsetFallbackPolicy = envoy::config::cluster::v3::Cluster::LbSubsetConfig::
+      LbSubsetSelector::LbSubsetSelectorFallbackPolicy;
+
+  LoadBalancerSubsetInfoImplBase(const ProtoType& subset_config)
+      : default_subset_(subset_config.default_subset()),
+        fallback_policy_(static_cast(subset_config.fallback_policy())),
+        metadata_fallback_policy_(
+            static_cast(subset_config.metadata_fallback_policy())),
+        enabled_(!subset_config.subset_selectors().empty()),
+        locality_weight_aware_(subset_config.locality_weight_aware()),
+        scale_locality_weight_(subset_config.scale_locality_weight()),
+        panic_mode_any_(subset_config.panic_mode_any()), list_as_any_(subset_config.list_as_any()) {
+    for (const auto& subset : subset_config.subset_selectors()) {
+      if (!subset.keys().empty()) {
+        subset_selectors_.emplace_back(std::make_shared(
+            subset.keys(), static_cast(subset.fallback_policy()),
+            subset.fallback_keys_subset(), subset.single_host_per_subset()));
+      }
+    }
+  }
+
+  // Upstream::LoadBalancerSubsetInfo
+  bool isEnabled() const override { return enabled_; }
+  FallbackPolicy fallbackPolicy() const override { return fallback_policy_; }
+  MetadataFallbackPolicy metadataFallbackPolicy() const override {
+    return metadata_fallback_policy_;
+  }
+  const ProtobufWkt::Struct& defaultSubset() const override { return default_subset_; }
+  const std::vector& subsetSelectors() const override {
+    return subset_selectors_;
+  }
+  bool localityWeightAware() const override { return locality_weight_aware_; }
+  bool scaleLocalityWeight() const override { return scale_locality_weight_; }
+  bool panicModeAny() const override { return panic_mode_any_; }
+  bool listAsAny() const override { return list_as_any_; }
+
+private:
+  const ProtobufWkt::Struct default_subset_;
+  std::vector subset_selectors_;
+  // Keep small members (bools and enums) at the end of class, to reduce alignment overhead.
+  const FallbackPolicy fallback_policy_;
+  const MetadataFallbackPolicy metadata_fallback_policy_;
+  const bool enabled_ : 1;
+  const bool locality_weight_aware_ : 1;
+  const bool scale_locality_weight_ : 1;
+  const bool panic_mode_any_ : 1;
+  const bool list_as_any_ : 1;
+};
+
+class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable {
+public:
+  SubsetLoadBalancer(const LoadBalancerSubsetInfo& subsets, ChildLoadBalancerCreatorPtr child_lb,
+                     const PrioritySet& priority_set, const PrioritySet* local_priority_set,
+                     ClusterLbStats& stats, Stats::Scope& scope, Runtime::Loader& runtime,
+                     Random::RandomGenerator& random, TimeSource& time_source);
   ~SubsetLoadBalancer() override;
 
   // Upstream::LoadBalancer
@@ -128,6 +254,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable;
   using MetadataFallbacks = ProtobufWkt::RepeatedPtrField;
 
+public:
   class LoadBalancerContextWrapper : public LoadBalancerContext {
   public:
     LoadBalancerContextWrapper(LoadBalancerContext* wrapped,
@@ -194,6 +322,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable lbRoundRobinConfig() const {
-    if (round_robin_config_ != nullptr) {
-      return *round_robin_config_;
-    }
-    return absl::nullopt;
-  }
-
-  OptRef
-  lbLeastRequestConfig() const {
-    if (least_request_config_ != nullptr) {
-      return *least_request_config_;
-    }
-    return absl::nullopt;
-  }
-
-  OptRef lbMaglevConfig() const {
-    if (lb_maglev_config_ != nullptr) {
-      return *lb_maglev_config_;
-    }
-    return absl::nullopt;
-  }
-
-  OptRef lbRingHashConfig() const {
-    if (lb_ring_hash_config_ != nullptr) {
-      return *lb_ring_hash_config_;
-    }
-    return absl::nullopt;
-  }
-
-  const std::unique_ptr
-      lb_ring_hash_config_;
-  const std::unique_ptr
-      lb_maglev_config_;
-  const std::unique_ptr
-      round_robin_config_;
-  const std::unique_ptr
-      least_request_config_;
-  const envoy::config::cluster::v3::Cluster::CommonLbConfig common_config_;
   ClusterLbStats& stats_;
   Stats::Scope& scope_;
   Runtime::Loader& runtime_;
   Random::RandomGenerator& random_;
+  TimeSource& time_source_;
 
   const envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy
       fallback_policy_;
@@ -411,6 +503,8 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggableclusters().getCluster("cluster_1");
   EXPECT_NE(cluster, absl::nullopt);
-  EXPECT_NE(cluster->get().info()->loadBalancingPolicy(), nullptr);
+  EXPECT_TRUE(cluster->get().info()->loadBalancerConfig().has_value());
 }
 
 class ClusterManagerImplThreadAwareLbTest : public ClusterManagerImplTest {
diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc
index 89b001f2b7fc..4f9a22e94b77 100644
--- a/test/common/upstream/load_balancer_benchmark.cc
+++ b/test/common/upstream/load_balancer_benchmark.cc
@@ -554,10 +554,12 @@ class SubsetLbTester : public BaseTester {
     *selector->mutable_keys()->Add() = std::string(metadata_key);
 
     subset_info_ = std::make_unique(subset_config);
-    lb_ = std::make_unique(
-        LoadBalancerType::Random, priority_set_, &local_priority_set_, stats_, stats_scope_,
-        runtime_, random_, *subset_info_, absl::nullopt, absl::nullopt, absl::nullopt,
-        absl::nullopt, common_config_, simTime());
+    auto child_lb_creator = std::make_unique(
+        LoadBalancerType::Random, absl::nullopt, absl::nullopt, absl::nullopt, absl::nullopt,
+        common_config_);
+    lb_ = std::make_unique(*subset_info_, std::move(child_lb_creator),
+                                               priority_set_, &local_priority_set_, stats_,
+                                               stats_scope_, runtime_, random_, simTime());
 
     const HostVector& hosts = priority_set_.getOrCreateHostSet(0).hosts();
     ASSERT(hosts.size() == num_hosts);
diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc
index f030eaf2b53d..0902264c537c 100644
--- a/test/common/upstream/subset_lb_test.cc
+++ b/test/common/upstream/subset_lb_test.cc
@@ -48,16 +48,24 @@ class SubsetLoadBalancerInternalStateTester {
     EXPECT_EQ(expected, lb_.get()->describeMetadata(subset_metadata));
   }
 
-  void validateLbTypeConfigs(LoadBalancerType lb_type) const {
+  void validateLbTypeConfigs() const {
+    const auto* legacy_child_lb_creator =
+        dynamic_cast(lb_->child_lb_creator_.get());
+
+    if (legacy_child_lb_creator == nullptr) {
+      return;
+    }
+
     // Each of these expects that an lb_type is set to that type iff the
     // returned config for that type exists.
-    EXPECT_EQ(lb_type == LoadBalancerType::RingHash,
-              lb_.get()->lbRingHashConfig() != absl::nullopt);
-    EXPECT_EQ(lb_type == LoadBalancerType::Maglev, lb_.get()->lbMaglevConfig() != absl::nullopt);
-    EXPECT_EQ(lb_type == LoadBalancerType::RoundRobin,
-              lb_.get()->lbRoundRobinConfig() != absl::nullopt);
-    EXPECT_EQ(lb_type == LoadBalancerType::LeastRequest,
-              lb_.get()->lbLeastRequestConfig() != absl::nullopt);
+    EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::RingHash,
+              legacy_child_lb_creator->lbRingHashConfig() != absl::nullopt);
+    EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::Maglev,
+              legacy_child_lb_creator->lbMaglevConfig() != absl::nullopt);
+    EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::RoundRobin,
+              legacy_child_lb_creator->lbRoundRobinConfig() != absl::nullopt);
+    EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::LeastRequest,
+              legacy_child_lb_creator->lbLeastRequestConfig() != absl::nullopt);
   }
 
 private:
@@ -250,8 +258,8 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime,
       configureHostSet(failover_host_metadata, *priority_set_.getMockHostSet(1));
     }
 
-    lb_ = std::make_shared(
-        lb_type_, priority_set_, nullptr, stats_, *scope_, runtime_, random_, subset_info_,
+    auto child_lb_creator = std::make_unique(
+        lb_type_,
         lb_type_ == LoadBalancerType::RingHash
             ? makeOptRef(
                   ring_hash_lb_config_)
@@ -268,7 +276,11 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime,
             ? makeOptRef(
                   least_request_lb_config_)
             : absl::nullopt,
-        common_config_, simTime());
+        common_config_);
+
+    lb_ = std::make_shared(subset_info_, std::move(child_lb_creator),
+                                               priority_set_, nullptr, stats_, *scope_, runtime_,
+                                               random_, simTime());
   }
 
   void zoneAwareInit(const std::vector& host_metadata_per_locality,
@@ -315,10 +327,13 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime,
             std::make_shared(), HostsPerLocalityImpl::empty()),
         {}, {}, {}, absl::nullopt);
 
-    lb_ = std::make_shared(
-        lb_type_, priority_set_, &local_priority_set_, stats_, *scope_, runtime_, random_,
-        subset_info_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
-        least_request_lb_config_, common_config_, simTime());
+    auto child_lb_creator = std::make_unique(
+        lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
+        least_request_lb_config_, common_config_);
+
+    lb_ = std::make_shared(subset_info_, std::move(child_lb_creator),
+                                               priority_set_, &local_priority_set_, stats_, *scope_,
+                                               runtime_, random_, simTime());
   }
 
   HostSharedPtr makeHost(const std::string& url, const HostMetadata& metadata) {
@@ -1516,10 +1531,12 @@ TEST_F(SubsetLoadBalancerTest, IgnoresHostsWithoutMetadata) {
   host_set_.healthy_hosts_ = host_set_.hosts_;
   host_set_.healthy_hosts_per_locality_ = host_set_.hosts_per_locality_;
 
+  auto child_lb_creator = std::make_unique(
+      lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
+      least_request_lb_config_, common_config_);
   lb_ = std::make_shared(
-      lb_type_, priority_set_, nullptr, stats_, *stats_store_.rootScope(), runtime_, random_,
-      subset_info_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
-      least_request_lb_config_, common_config_, simTime());
+      subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_,
+      *stats_store_.rootScope(), runtime_, random_, simTime());
 
   TestLoadBalancerContext context_version({{"version", "1.0"}});
 
@@ -1533,31 +1550,31 @@ TEST_F(SubsetLoadBalancerTest, IgnoresHostsWithoutMetadata) {
 TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRoundRobin) {
   doLbTypeTest(LoadBalancerType::RoundRobin);
   auto tester = SubsetLoadBalancerInternalStateTester(lb_);
-  tester.validateLbTypeConfigs(LoadBalancerType::RoundRobin);
+  tester.validateLbTypeConfigs();
 }
 
 TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesLeastRequest) {
   doLbTypeTest(LoadBalancerType::LeastRequest);
   auto tester = SubsetLoadBalancerInternalStateTester(lb_);
-  tester.validateLbTypeConfigs(LoadBalancerType::LeastRequest);
+  tester.validateLbTypeConfigs();
 }
 
 TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRandom) {
   doLbTypeTest(LoadBalancerType::Random);
   auto tester = SubsetLoadBalancerInternalStateTester(lb_);
-  tester.validateLbTypeConfigs(LoadBalancerType::Random);
+  tester.validateLbTypeConfigs();
 }
 
 TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRingHash) {
   doLbTypeTest(LoadBalancerType::RingHash);
   auto tester = SubsetLoadBalancerInternalStateTester(lb_);
-  tester.validateLbTypeConfigs(LoadBalancerType::RingHash);
+  tester.validateLbTypeConfigs();
 }
 
 TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesMaglev) {
   doLbTypeTest(LoadBalancerType::Maglev);
   auto tester = SubsetLoadBalancerInternalStateTester(lb_);
-  tester.validateLbTypeConfigs(LoadBalancerType::Maglev);
+  tester.validateLbTypeConfigs();
 }
 
 TEST_F(SubsetLoadBalancerTest, ZoneAwareFallback) {
@@ -1950,10 +1967,12 @@ TEST_F(SubsetLoadBalancerTest, DisabledLocalityWeightAwareness) {
       },
       host_set_, {1, 100});
 
+  auto child_lb_creator = std::make_unique(
+      lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
+      least_request_lb_config_, common_config_);
   lb_ = std::make_shared(
-      lb_type_, priority_set_, nullptr, stats_, *stats_store_.rootScope(), runtime_, random_,
-      subset_info_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
-      least_request_lb_config_, common_config_, simTime());
+      subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_,
+      *stats_store_.rootScope(), runtime_, random_, simTime());
 
   TestLoadBalancerContext context({{"version", "1.1"}});
 
@@ -1974,10 +1993,12 @@ TEST_F(SubsetLoadBalancerTest, DoesNotCheckHostHealth) {
 
   EXPECT_CALL(*mock_host, weight()).WillRepeatedly(Return(1));
 
+  auto child_lb_creator = std::make_unique(
+      lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
+      least_request_lb_config_, common_config_);
   lb_ = std::make_shared(
-      lb_type_, priority_set_, nullptr, stats_, *stats_store_.rootScope(), runtime_, random_,
-      subset_info_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
-      least_request_lb_config_, common_config_, simTime());
+      subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_,
+      *stats_store_.rootScope(), runtime_, random_, simTime());
 }
 
 TEST_F(SubsetLoadBalancerTest, EnabledLocalityWeightAwareness) {
@@ -1999,10 +2020,12 @@ TEST_F(SubsetLoadBalancerTest, EnabledLocalityWeightAwareness) {
       host_set_, {1, 100});
 
   common_config_.mutable_locality_weighted_lb_config();
+  auto child_lb_creator = std::make_unique(
+      lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
+      least_request_lb_config_, common_config_);
   lb_ = std::make_shared(
-      lb_type_, priority_set_, nullptr, stats_, *stats_store_.rootScope(), runtime_, random_,
-      subset_info_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
-      least_request_lb_config_, common_config_, simTime());
+      subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_,
+      *stats_store_.rootScope(), runtime_, random_, simTime());
 
   TestLoadBalancerContext context({{"version", "1.1"}});
 
@@ -2036,10 +2059,12 @@ TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeights) {
       host_set_, {50, 50});
 
   common_config_.mutable_locality_weighted_lb_config();
+  auto child_lb_creator = std::make_unique(
+      lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
+      least_request_lb_config_, common_config_);
   lb_ = std::make_shared(
-      lb_type_, priority_set_, nullptr, stats_, *stats_store_.rootScope(), runtime_, random_,
-      subset_info_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
-      least_request_lb_config_, common_config_, simTime());
+      subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_,
+      *stats_store_.rootScope(), runtime_, random_, simTime());
   TestLoadBalancerContext context({{"version", "1.1"}});
 
   // Since we scale the locality weights by number of hosts removed, we expect to see the second
@@ -2083,10 +2108,12 @@ TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeightsRounding) {
       host_set_, {2, 2});
 
   common_config_.mutable_locality_weighted_lb_config();
+  auto child_lb_creator = std::make_unique(
+      lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
+      least_request_lb_config_, common_config_);
   lb_ = std::make_shared(
-      lb_type_, priority_set_, nullptr, stats_, *stats_store_.rootScope(), runtime_, random_,
-      subset_info_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
-      least_request_lb_config_, common_config_, simTime());
+      subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_,
+      *stats_store_.rootScope(), runtime_, random_, simTime());
   TestLoadBalancerContext context({{"version", "1.0"}});
 
   // We expect to see a 33/66 split because 2 * 1 / 2 = 1 and 2 * 3 / 4 = 1.5 -> 2
@@ -2116,10 +2143,12 @@ TEST_F(SubsetLoadBalancerTest, ScaleLocalityWeightsWithNoLocalityWeights) {
       },
       host_set_);
 
+  auto child_lb_creator = std::make_unique(
+      lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
+      least_request_lb_config_, common_config_);
   lb_ = std::make_shared(
-      lb_type_, priority_set_, nullptr, stats_, *stats_store_.rootScope(), runtime_, random_,
-      subset_info_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_,
-      least_request_lb_config_, common_config_, simTime());
+      subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_,
+      *stats_store_.rootScope(), runtime_, random_, simTime());
 }
 
 TEST_P(SubsetLoadBalancerTest, GaugesUpdatedOnDestroy) {
diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc
index b6d23ac41c2e..76c6320909d6 100644
--- a/test/common/upstream/upstream_impl_test.cc
+++ b/test/common/upstream/upstream_impl_test.cc
@@ -2385,7 +2385,7 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithLbPolicy) {
   EXPECT_EQ(1UL, cluster.prioritySet().hostSetsPerPriority()[0]->healthyHosts().size());
   EXPECT_EQ(LoadBalancerType::LoadBalancingPolicyConfig, cluster.info()->lbType());
   EXPECT_TRUE(cluster.info()->addedViaApi());
-  EXPECT_NE(nullptr, cluster.info()->loadBalancingPolicy());
+  EXPECT_TRUE(cluster.info()->loadBalancerConfig().has_value());
   EXPECT_NE(nullptr, cluster.info()->loadBalancerFactory());
 }
 
diff --git a/test/extensions/load_balancing_policies/cluster_provided/config_test.cc b/test/extensions/load_balancing_policies/cluster_provided/config_test.cc
index e0966208e7aa..3bec7e1bbbd8 100644
--- a/test/extensions/load_balancing_policies/cluster_provided/config_test.cc
+++ b/test/extensions/load_balancing_policies/cluster_provided/config_test.cc
@@ -29,7 +29,7 @@ TEST(ClusterProvidedConfigTest, ClusterProvidedConfigTest) {
   EXPECT_EQ("envoy.load_balancing_policies.cluster_provided", factory.name());
 
   auto thread_aware_lb =
-      factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_,
+      factory.create({}, cluster_info, main_thread_priority_set, context.runtime_loader_,
                      context.api_.random_, context.time_system_);
   EXPECT_EQ(nullptr, thread_aware_lb);
 }
diff --git a/test/extensions/load_balancing_policies/least_request/config_test.cc b/test/extensions/load_balancing_policies/least_request/config_test.cc
index a2c4d2da2dc3..82e8405d8f00 100644
--- a/test/extensions/load_balancing_policies/least_request/config_test.cc
+++ b/test/extensions/load_balancing_policies/least_request/config_test.cc
@@ -28,11 +28,10 @@ TEST(LeastRequestConfigTest, ValidateFail) {
   auto& factory = Config::Utility::getAndCheckFactory(config);
   EXPECT_EQ("envoy.load_balancing_policies.least_request", factory.name());
 
-  auto message_ptr = factory.createEmptyConfigProto();
-  EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(message_ptr));
-
+  auto lb_config =
+      factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor());
   auto thread_aware_lb =
-      factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_,
+      factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_,
                      context.api_.random_, context.time_system_);
   EXPECT_NE(nullptr, thread_aware_lb);
 
diff --git a/test/extensions/load_balancing_policies/maglev/config_test.cc b/test/extensions/load_balancing_policies/maglev/config_test.cc
index 733ed14a3b4b..9b103239d2c6 100644
--- a/test/extensions/load_balancing_policies/maglev/config_test.cc
+++ b/test/extensions/load_balancing_policies/maglev/config_test.cc
@@ -30,12 +30,10 @@ TEST(MaglevConfigTest, Validate) {
     auto& factory = Config::Utility::getAndCheckFactory(config);
     EXPECT_EQ("envoy.load_balancing_policies.maglev", factory.name());
 
-    auto messsage_ptr = factory.createEmptyConfigProto();
-
-    EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(messsage_ptr));
-
+    auto lb_config =
+        factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor());
     auto thread_aware_lb =
-        factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_,
+        factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_,
                        context.api_.random_, context.time_system_);
     EXPECT_NE(nullptr, thread_aware_lb);
 
@@ -61,12 +59,11 @@ TEST(MaglevConfigTest, Validate) {
     auto& factory = Config::Utility::getAndCheckFactory(config);
     EXPECT_EQ("envoy.load_balancing_policies.maglev", factory.name());
 
-    auto messsage_ptr = factory.createEmptyConfigProto();
-    messsage_ptr->MergeFrom(config_msg);
-
-    EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(messsage_ptr));
+    auto message_ptr = factory.createEmptyConfigProto();
+    message_ptr->MergeFrom(config_msg);
+    auto lb_config = factory.loadConfig(std::move(message_ptr), context.messageValidationVisitor());
 
-    EXPECT_THROW_WITH_MESSAGE(factory.create(cluster_info, main_thread_priority_set,
+    EXPECT_THROW_WITH_MESSAGE(factory.create(*lb_config, cluster_info, main_thread_priority_set,
                                              context.runtime_loader_, context.api_.random_,
                                              context.time_system_),
                               EnvoyException, "The table size of maglev must be prime number");
diff --git a/test/extensions/load_balancing_policies/random/config_test.cc b/test/extensions/load_balancing_policies/random/config_test.cc
index 54aa165bf087..869d7ef64d07 100644
--- a/test/extensions/load_balancing_policies/random/config_test.cc
+++ b/test/extensions/load_balancing_policies/random/config_test.cc
@@ -28,11 +28,10 @@ TEST(RandomConfigTest, ValidateFail) {
   auto& factory = Config::Utility::getAndCheckFactory(config);
   EXPECT_EQ("envoy.load_balancing_policies.random", factory.name());
 
-  auto message_ptr = factory.createEmptyConfigProto();
-  EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(message_ptr));
-
+  auto lb_config =
+      factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor());
   auto thread_aware_lb =
-      factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_,
+      factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_,
                      context.api_.random_, context.time_system_);
   EXPECT_NE(nullptr, thread_aware_lb);
 
diff --git a/test/extensions/load_balancing_policies/ring_hash/config_test.cc b/test/extensions/load_balancing_policies/ring_hash/config_test.cc
index 7f2034f76fd3..24eb9c267795 100644
--- a/test/extensions/load_balancing_policies/ring_hash/config_test.cc
+++ b/test/extensions/load_balancing_policies/ring_hash/config_test.cc
@@ -30,11 +30,10 @@ TEST(RingHashConfigTest, Validate) {
     auto& factory = Config::Utility::getAndCheckFactory(config);
     EXPECT_EQ("envoy.load_balancing_policies.ring_hash", factory.name());
 
-    auto messsage_ptr = factory.createEmptyConfigProto();
-    EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(messsage_ptr));
-
+    auto lb_config =
+        factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor());
     auto thread_aware_lb =
-        factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_,
+        factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_,
                        context.api_.random_, context.time_system_);
     EXPECT_NE(nullptr, thread_aware_lb);
 
@@ -64,10 +63,12 @@ TEST(RingHashConfigTest, Validate) {
     auto messsage_ptr = factory.createEmptyConfigProto();
     messsage_ptr->MergeFrom(config_msg);
 
-    EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(messsage_ptr));
+    auto message_ptr = factory.createEmptyConfigProto();
+    message_ptr->MergeFrom(config_msg);
+    auto lb_config = factory.loadConfig(std::move(message_ptr), context.messageValidationVisitor());
 
     EXPECT_THROW_WITH_MESSAGE(
-        factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_,
+        factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_,
                        context.api_.random_, context.time_system_),
         EnvoyException, "ring hash: minimum_ring_size (4) > maximum_ring_size (2)");
   }
diff --git a/test/extensions/load_balancing_policies/round_robin/config_test.cc b/test/extensions/load_balancing_policies/round_robin/config_test.cc
index dd8541746144..a6aa8aacd5e4 100644
--- a/test/extensions/load_balancing_policies/round_robin/config_test.cc
+++ b/test/extensions/load_balancing_policies/round_robin/config_test.cc
@@ -28,11 +28,11 @@ TEST(RoundRobinConfigTest, ValidateFail) {
   auto& factory = Config::Utility::getAndCheckFactory(config);
   EXPECT_EQ("envoy.load_balancing_policies.round_robin", factory.name());
 
-  auto message_ptr = factory.createEmptyConfigProto();
-  EXPECT_CALL(cluster_info, loadBalancingPolicy()).WillOnce(testing::ReturnRef(message_ptr));
+  auto lb_config =
+      factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor());
 
   auto thread_aware_lb =
-      factory.create(cluster_info, main_thread_priority_set, context.runtime_loader_,
+      factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_,
                      context.api_.random_, context.time_system_);
   EXPECT_NE(nullptr, thread_aware_lb);
 
diff --git a/test/extensions/load_balancing_policies/subset/BUILD b/test/extensions/load_balancing_policies/subset/BUILD
new file mode 100644
index 000000000000..417a9affab11
--- /dev/null
+++ b/test/extensions/load_balancing_policies/subset/BUILD
@@ -0,0 +1,59 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_package",
+)
+load(
+    "//test/extensions:extensions_build_system.bzl",
+    "envoy_extension_cc_test",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+envoy_extension_cc_test(
+    name = "config_test",
+    srcs = ["config_test.cc"],
+    extension_names = ["envoy.load_balancing_policies.subset"],
+    deps = [
+        "//source/extensions/load_balancing_policies/random:config",
+        "//source/extensions/load_balancing_policies/subset:config",
+        "//test/mocks/server:factory_context_mocks",
+        "//test/mocks/upstream:cluster_info_mocks",
+        "//test/mocks/upstream:priority_set_mocks",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_extension_cc_test(
+    name = "integration_test",
+    size = "large",
+    srcs = ["integration_test.cc"],
+    extension_names = ["envoy.load_balancing_policies.subset"],
+    deps = [
+        "//source/common/protobuf",
+        "//source/extensions/filters/http/header_to_metadata:config",
+        "//source/extensions/load_balancing_policies/random:config",
+        "//source/extensions/load_balancing_policies/subset:config",
+        "//test/integration:http_integration_lib",
+        "//test/test_common:utility_lib",
+        "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_extension_cc_test(
+    name = "subset_test",
+    srcs = ["subset_test.cc"],
+    extension_names = ["envoy.load_balancing_policies.subset"],
+    deps = [
+        "//source/extensions/load_balancing_policies/random:config",
+        "//source/extensions/load_balancing_policies/subset:config",
+        "//test/mocks/server:factory_context_mocks",
+        "//test/mocks/upstream:cluster_info_mocks",
+        "//test/mocks/upstream:load_balancer_context_mock",
+        "//test/mocks/upstream:priority_set_mocks",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto",
+    ],
+)
diff --git a/test/extensions/load_balancing_policies/subset/config_test.cc b/test/extensions/load_balancing_policies/subset/config_test.cc
new file mode 100644
index 000000000000..b5d77b70384c
--- /dev/null
+++ b/test/extensions/load_balancing_policies/subset/config_test.cc
@@ -0,0 +1,114 @@
+#include "envoy/config/core/v3/extension.pb.h"
+#include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.h"
+
+#include "source/extensions/load_balancing_policies/subset/config.h"
+
+#include "test/mocks/server/factory_context.h"
+#include "test/mocks/upstream/cluster_info.h"
+#include "test/mocks/upstream/priority_set.h"
+
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace LoadBalancingPolices {
+namespace Subset {
+namespace {
+
+TEST(SubsetConfigTest, SubsetConfigTest) {
+  NiceMock context;
+  NiceMock cluster_info;
+  NiceMock main_thread_priority_set;
+  NiceMock thread_local_priority_set;
+
+  envoy::config::core::v3::TypedExtensionConfig config;
+  config.set_name("envoy.load_balancing_policies.subset");
+  auto config_msg =
+      std::make_unique();
+
+  const std::string config_yaml = R"EOF(
+    fallback_policy: ANY_ENDPOINT
+    subset_selectors:
+      - keys:
+          - "version"
+          - "stage"
+        fallback_policy: NO_FALLBACK
+      - keys:
+          - "version"
+        fallback_policy: ANY_ENDPOINT
+    list_as_any: true
+    subset_lb_policy:
+      policies:
+        - typed_extension_config:
+            name: envoy.load_balancing_policies.random
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.random.v3.Random
+    )EOF";
+  TestUtility::loadFromYaml(config_yaml, *config_msg);
+  config.mutable_typed_config()->PackFrom(*config_msg);
+
+  auto& factory = Config::Utility::getAndCheckFactory(config);
+  EXPECT_EQ("envoy.load_balancing_policies.subset", factory.name());
+
+  auto lb_config = factory.loadConfig(std::move(config_msg), context.messageValidationVisitor());
+
+  auto thread_aware_lb =
+      factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_,
+                     context.api_.random_, context.time_system_);
+  EXPECT_NE(nullptr, thread_aware_lb);
+
+  thread_aware_lb->initialize();
+
+  auto thread_local_lb_factory = thread_aware_lb->factory();
+  EXPECT_NE(nullptr, thread_local_lb_factory);
+
+  auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr});
+  EXPECT_NE(nullptr, thread_local_lb);
+
+  EXPECT_DEATH(thread_local_lb_factory->create(), "not implemented");
+}
+
+TEST(SubsetConfigTest, SubsetConfigTestWithUnknownSubsetLoadBalancingPolicy) {
+  NiceMock context;
+  NiceMock cluster_info;
+  NiceMock main_thread_priority_set;
+  NiceMock thread_local_priority_set;
+
+  envoy::config::core::v3::TypedExtensionConfig config;
+  config.set_name("envoy.load_balancing_policies.subset");
+  auto config_msg =
+      std::make_unique();
+
+  const std::string config_yaml = R"EOF(
+    fallback_policy: ANY_ENDPOINT
+    subset_selectors:
+      - keys:
+          - "version"
+          - "stage"
+        fallback_policy: NO_FALLBACK
+      - keys:
+          - "version"
+        fallback_policy: ANY_ENDPOINT
+    list_as_any: true
+    subset_lb_policy:
+      policies:
+        - typed_extension_config:
+            name: envoy.load_balancing_policies.unknown
+    )EOF";
+  TestUtility::loadFromYaml(config_yaml, *config_msg);
+  config.mutable_typed_config()->PackFrom(*config_msg);
+
+  auto& factory = Config::Utility::getAndCheckFactory(config);
+  EXPECT_EQ("envoy.load_balancing_policies.subset", factory.name());
+
+  EXPECT_THROW_WITH_MESSAGE(
+      factory.loadConfig(std::move(config_msg), context.messageValidationVisitor()), EnvoyException,
+      "cluster: didn't find a registered load balancer factory implementation for subset lb with "
+      "names from [envoy.load_balancing_policies.unknown]");
+}
+
+} // namespace
+} // namespace Subset
+} // namespace LoadBalancingPolices
+} // namespace Extensions
+} // namespace Envoy
diff --git a/test/extensions/load_balancing_policies/subset/integration_test.cc b/test/extensions/load_balancing_policies/subset/integration_test.cc
new file mode 100644
index 000000000000..e995d26ee6c8
--- /dev/null
+++ b/test/extensions/load_balancing_policies/subset/integration_test.cc
@@ -0,0 +1,151 @@
+#include 
+#include 
+
+#include "envoy/config/endpoint/v3/endpoint_components.pb.h"
+
+#include "source/common/common/base64.h"
+#include "source/common/http/utility.h"
+#include "source/common/protobuf/protobuf.h"
+#include "source/extensions/load_balancing_policies/random/config.h"
+
+#include "test/integration/http_integration.h"
+
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace LoadBalancingPolices {
+namespace Subset {
+namespace {
+
+class SubsetIntegrationTest : public testing::TestWithParam,
+                              public HttpIntegrationTest {
+public:
+  SubsetIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {
+    setUpstreamCount(3);
+
+    // Add the header to metadata filter to help set the metadata for subset load balancer.
+    config_helper_.prependFilter(R"EOF(
+    name: envoy.filters.http.header_to_metadata
+    typed_config:
+      "@type": type.googleapis.com/envoy.extensions.filters.http.header_to_metadata.v3.Config
+      request_rules:
+      - header: "version"
+        on_header_present:
+          metadata_namespace: "envoy.lb"
+          key: "version"
+    )EOF");
+
+    // Update endpoints of default cluster `cluster_0` to 3 different fake upstreams.
+    config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+      auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0);
+      ASSERT(cluster_0->name() == "cluster_0");
+      auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0);
+
+      constexpr absl::string_view endpoints_yaml = R"EOF(
+        lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: {}
+                port_value: 0
+          metadata:
+            filter_metadata:
+              envoy.lb:
+                version: v1
+                stage: canary
+        - endpoint:
+            address:
+              socket_address:
+                address: {}
+                port_value: 0
+          metadata:
+            filter_metadata:
+              envoy.lb:
+                version: v2
+                stage: canary
+        - endpoint:
+            address:
+              socket_address:
+                address: {}
+                port_value: 0
+          metadata:
+            filter_metadata:
+              envoy.lb:
+                version: v3
+      )EOF";
+
+      const std::string local_address = Network::Test::getLoopbackAddressString(GetParam());
+      TestUtility::loadFromYaml(
+          fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint);
+
+      auto* policy = cluster_0->mutable_load_balancing_policy();
+
+      const std::string policy_yaml = R"EOF(
+        policies:
+        - typed_extension_config:
+            name: envoy.load_balancing_policies.subset
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.subset.v3.Subset
+              fallback_policy: ANY_ENDPOINT
+              subset_selectors:
+              - keys:
+                - "version"
+                - "stage"
+                fallback_policy: NO_FALLBACK
+              - keys:
+                - "version"
+                fallback_policy: ANY_ENDPOINT
+              list_as_any: true
+              subset_lb_policy:
+                policies:
+                - typed_extension_config:
+                    name: envoy.load_balancing_policies.random
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.random.v3.Random
+       )EOF";
+
+      TestUtility::loadFromYaml(policy_yaml, *policy);
+    });
+  }
+};
+
+INSTANTIATE_TEST_SUITE_P(IpVersions, SubsetIntegrationTest,
+                         testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
+                         TestUtility::ipTestParamsToString);
+
+// Test the case where the subset load balancer is configured by the load balancing
+// policy API and it works as expected.
+TEST_P(SubsetIntegrationTest, NormalLoadBalancing) {
+  initialize();
+
+  for (uint64_t i = 1; i <= 3; i++) {
+
+    codec_client_ = makeHttpConnection(lookupPort("http"));
+
+    Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"},
+                                                   {":path", "/"},
+                                                   {":scheme", "http"},
+                                                   {":authority", "example.com"},
+                                                   {"version", fmt::format("v{}", i)}};
+
+    auto response = codec_client_->makeRequestWithBody(request_headers, 0);
+
+    waitForNextUpstreamRequest(i - 1);
+
+    upstream_request_->encodeHeaders(default_response_headers_, true);
+
+    ASSERT_TRUE(response->waitForEndStream());
+
+    EXPECT_TRUE(upstream_request_->complete());
+    EXPECT_TRUE(response->complete());
+
+    cleanupUpstreamAndDownstream();
+  }
+}
+
+} // namespace
+} // namespace Subset
+} // namespace LoadBalancingPolices
+} // namespace Extensions
+} // namespace Envoy
diff --git a/test/extensions/load_balancing_policies/subset/subset_test.cc b/test/extensions/load_balancing_policies/subset/subset_test.cc
new file mode 100644
index 000000000000..c91df65bb5ca
--- /dev/null
+++ b/test/extensions/load_balancing_policies/subset/subset_test.cc
@@ -0,0 +1,58 @@
+#include "envoy/config/core/v3/extension.pb.h"
+#include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.h"
+
+#include "source/common/router/metadatamatchcriteria_impl.h"
+#include "source/extensions/load_balancing_policies/subset/config.h"
+
+#include "test/mocks/server/factory_context.h"
+#include "test/mocks/upstream/cluster_info.h"
+#include "test/mocks/upstream/load_balancer_context.h"
+#include "test/mocks/upstream/priority_set.h"
+
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace LoadBalancingPolices {
+namespace Subset {
+namespace {
+
+// Test to improve coverage of the SubsetLoadBalancerFactory.
+TEST(LoadBalancerContextWrapperTest, LoadBalancingContextWrapperTest) {
+  testing::NiceMock mock_context;
+
+  ProtobufWkt::Struct empty_struct;
+  Router::MetadataMatchCriteriaImpl match_criteria(empty_struct);
+  ON_CALL(mock_context, metadataMatchCriteria()).WillByDefault(testing::Return(&match_criteria));
+
+  Upstream::SubsetLoadBalancer::LoadBalancerContextWrapper wrapper(&mock_context,
+                                                                   std::set{});
+
+  EXPECT_CALL(mock_context, computeHashKey());
+  wrapper.computeHashKey();
+
+  EXPECT_CALL(mock_context, downstreamConnection());
+  wrapper.downstreamConnection();
+
+  EXPECT_CALL(mock_context, downstreamHeaders());
+  wrapper.downstreamHeaders();
+
+  EXPECT_CALL(mock_context, hostSelectionRetryCount());
+
+  wrapper.hostSelectionRetryCount();
+
+  EXPECT_CALL(mock_context, upstreamSocketOptions());
+  wrapper.upstreamSocketOptions();
+
+  EXPECT_CALL(mock_context, upstreamTransportSocketOptions());
+  wrapper.upstreamTransportSocketOptions();
+
+  EXPECT_CALL(mock_context, overrideHostToSelect());
+  wrapper.overrideHostToSelect();
+}
+
+} // namespace
+} // namespace Subset
+} // namespace LoadBalancingPolices
+} // namespace Extensions
+} // namespace Envoy
diff --git a/test/integration/load_balancers/custom_lb_policy.h b/test/integration/load_balancers/custom_lb_policy.h
index a6d4ed239174..8147dcd6b2f6 100644
--- a/test/integration/load_balancers/custom_lb_policy.h
+++ b/test/integration/load_balancers/custom_lb_policy.h
@@ -60,7 +60,8 @@ class CustomLbFactory : public Upstream::TypedLoadBalancerFactoryBase<
 public:
   CustomLbFactory() : TypedLoadBalancerFactoryBase("envoy.load_balancers.custom_lb") {}
 
-  Upstream::ThreadAwareLoadBalancerPtr create(const Upstream::ClusterInfo&,
+  Upstream::ThreadAwareLoadBalancerPtr create(OptRef,
+                                              const Upstream::ClusterInfo&,
                                               const Upstream::PrioritySet&, Runtime::Loader&,
                                               Random::RandomGenerator&, TimeSource&) override {
     return std::make_unique();
diff --git a/test/mocks/upstream/BUILD b/test/mocks/upstream/BUILD
index ec4625080fb1..8b4258c4f668 100644
--- a/test/mocks/upstream/BUILD
+++ b/test/mocks/upstream/BUILD
@@ -216,6 +216,7 @@ envoy_cc_mock(
     hdrs = ["typed_load_balancer_factory.h"],
     deps = [
         "//envoy/upstream:load_balancer_interface",
+        "//source/common/upstream:load_balancer_factory_base_lib",
     ],
 )
 
diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc
index 6fd33414a641..b24f9069f58e 100644
--- a/test/mocks/upstream/cluster_info.cc
+++ b/test/mocks/upstream/cluster_info.cc
@@ -198,7 +198,8 @@ MockClusterInfo::MockClusterInfo()
             manager.applyFilterFactoryCb({}, factory_cb);
             return true;
           }));
-  ON_CALL(*this, loadBalancingPolicy).WillByDefault(ReturnRef(load_balancing_policy_));
+  ON_CALL(*this, loadBalancerConfig())
+      .WillByDefault(Return(makeOptRefFromPtr(nullptr)));
   ON_CALL(*this, makeHeaderValidator(_)).WillByDefault(Invoke([&](Http::Protocol protocol) {
     return header_validator_factory_ ? header_validator_factory_->createClientHeaderValidator(
                                            protocol, codecStats(protocol))
diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h
index 3c92b980557d..d81f84b94265 100644
--- a/test/mocks/upstream/cluster_info.h
+++ b/test/mocks/upstream/cluster_info.h
@@ -126,7 +126,7 @@ class MockClusterInfo : public ClusterInfo {
               (const));
   MOCK_METHOD(ProtocolOptionsConfigConstSharedPtr, extensionProtocolOptions, (const std::string&),
               (const));
-  MOCK_METHOD(const ProtobufTypes::MessagePtr&, loadBalancingPolicy, (), (const));
+  MOCK_METHOD(OptRef, loadBalancerConfig, (), (const));
   MOCK_METHOD(TypedLoadBalancerFactory*, loadBalancerFactory, (), (const));
   MOCK_METHOD(const envoy::config::cluster::v3::Cluster::CommonLbConfig&, lbConfig, (), (const));
   MOCK_METHOD(LoadBalancerType, lbType, (), (const));
@@ -253,7 +253,6 @@ class MockClusterInfo : public ClusterInfo {
   mutable Http::Http1::CodecStats::AtomicPtr http1_codec_stats_;
   mutable Http::Http2::CodecStats::AtomicPtr http2_codec_stats_;
   mutable Http::Http3::CodecStats::AtomicPtr http3_codec_stats_;
-  ProtobufTypes::MessagePtr load_balancing_policy_;
   Http::HeaderValidatorFactoryPtr header_validator_factory_;
 };
 
diff --git a/test/mocks/upstream/typed_load_balancer_factory.h b/test/mocks/upstream/typed_load_balancer_factory.h
index eb8b0cb8d4f3..a2ec9a45e18d 100644
--- a/test/mocks/upstream/typed_load_balancer_factory.h
+++ b/test/mocks/upstream/typed_load_balancer_factory.h
@@ -2,6 +2,8 @@
 
 #include "envoy/upstream/load_balancer.h"
 
+#include "source/common/upstream/load_balancer_factory_base.h"
+
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
@@ -15,8 +17,14 @@ class MockTypedLoadBalancerFactory : public TypedLoadBalancerFactory {
   // Upstream::TypedLoadBalancerFactory
   MOCK_METHOD(std::string, name, (), (const));
   MOCK_METHOD(ThreadAwareLoadBalancerPtr, create,
-              (const ClusterInfo& cluster_info, const PrioritySet& priority_set,
-               Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source));
+              (OptRef lb_config, const ClusterInfo& cluster_info,
+               const PrioritySet& priority_set, Runtime::Loader& runtime,
+               Random::RandomGenerator& random, TimeSource& time_source));
+
+  LoadBalancerConfigPtr loadConfig(ProtobufTypes::MessagePtr config,
+                                   ProtobufMessage::ValidationVisitor&) override {
+    return std::make_unique(std::move(config));
+  }
 
   ProtobufTypes::MessagePtr createEmptyConfigProto() override {
     // Using Struct instead of a custom per-filter empty config proto

From a216acd3cfac2e14932e40e14e35198400d30864 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Wed, 7 Jun 2023 08:31:53 -0400
Subject: [PATCH 476/740] build: adding an disable_exceptions (which does not
 yet work) (#27811)

The new build option simply compiles out all try/catch code, while leaving in the exceptions. This can not yet be successfully used as it turns up fno-exceptions which chokes on throw statements. This is by design as if we compiled out throw as well, config failures would be fatal instead of gracefully handled.

Risk Level: low
Testing: manual testing
Docs Changes: n/a
Release Notes: n/a
Part of #27412

Signed-off-by: Alyssa Wilk 
---
 bazel/BUILD                                   |  7 +++
 bazel/envoy_build_system.bzl                  |  2 +
 bazel/envoy_internal.bzl                      |  3 +-
 bazel/envoy_mobile_defines.bzl                |  3 +-
 bazel/envoy_select.bzl                        |  7 +++
 mobile/library/common/engine.cc               | 10 ++--
 source/common/common/thread.h                 | 31 ++++++++++-
 source/common/filter/config_discovery_impl.cc |  5 +-
 source/common/router/header_parser_utils.cc   |  5 +-
 source/common/runtime/runtime_impl.cc         |  4 +-
 source/common/secret/sds_api.cc               |  4 +-
 source/common/stats/tag_extractor_impl.cc     |  5 +-
 source/common/upstream/cds_api_helper.cc      |  6 +--
 .../upstream/health_discovery_service.cc      |  4 +-
 source/common/upstream/upstream_impl.cc       |  4 +-
 source/exe/stripped_main_base.cc              |  4 +-
 .../file_based_metadata/config.cc             |  3 +-
 .../network/dns_resolver/cares/dns_impl.cc    |  1 +
 source/server/options_impl.cc                 | 25 +++++----
 source/server/server.cc                       | 54 +++++++++----------
 20 files changed, 114 insertions(+), 73 deletions(-)

diff --git a/bazel/BUILD b/bazel/BUILD
index ca8db723eb3e..1ad963a26072 100644
--- a/bazel/BUILD
+++ b/bazel/BUILD
@@ -375,6 +375,13 @@ config_setting(
     values = {"define": "envoy_yaml=disabled"},
 )
 
+# The goal here is to allow Envoy to build with this option but it is not yet
+# complete.  See https://github.com/envoyproxy/envoy/issues/27412
+config_setting(
+    name = "disable_exceptions",
+    values = {"define": "envoy_exceptions=disabled"},
+)
+
 config_setting(
     name = "disable_envoy_mobile_listener",
     values = {"define": "envoy_mobile_listener=disabled"},
diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl
index da025768f034..8f05c9fcb8ff 100644
--- a/bazel/envoy_build_system.bzl
+++ b/bazel/envoy_build_system.bzl
@@ -22,6 +22,7 @@ load(
     _envoy_select_admin_html = "envoy_select_admin_html",
     _envoy_select_admin_no_html = "envoy_select_admin_no_html",
     _envoy_select_boringssl = "envoy_select_boringssl",
+    _envoy_select_disable_exceptions = "envoy_select_disable_exceptions",
     _envoy_select_disable_logging = "envoy_select_disable_logging",
     _envoy_select_enable_http3 = "envoy_select_enable_http3",
     _envoy_select_enable_http_datagrams = "envoy_select_enable_http_datagrams",
@@ -242,6 +243,7 @@ envoy_select_disable_logging = _envoy_select_disable_logging
 envoy_select_google_grpc = _envoy_select_google_grpc
 envoy_select_enable_http3 = _envoy_select_enable_http3
 envoy_select_enable_yaml = _envoy_select_enable_yaml
+envoy_select_disable_exceptions = _envoy_select_disable_exceptions
 envoy_select_hot_restart = _envoy_select_hot_restart
 envoy_select_enable_http_datagrams = _envoy_select_enable_http_datagrams
 envoy_select_signal_trace = _envoy_select_signal_trace
diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl
index 472dfd2fa483..a1f8f1dc6e50 100644
--- a/bazel/envoy_internal.bzl
+++ b/bazel/envoy_internal.bzl
@@ -1,6 +1,6 @@
 # DO NOT LOAD THIS FILE. Targets from this file should be considered private
 # and not used outside of the @envoy//bazel package.
-load(":envoy_select.bzl", "envoy_select_admin_html", "envoy_select_disable_logging", "envoy_select_google_grpc", "envoy_select_hot_restart", "envoy_select_signal_trace", "envoy_select_static_extension_registration")
+load(":envoy_select.bzl", "envoy_select_admin_html", "envoy_select_disable_exceptions", "envoy_select_disable_logging", "envoy_select_google_grpc", "envoy_select_hot_restart", "envoy_select_signal_trace", "envoy_select_static_extension_registration")
 
 # Compute the final copts based on various options.
 def envoy_copts(repository, test = False):
@@ -119,6 +119,7 @@ def envoy_copts(repository, test = False):
                repository + "//bazel:uhv_enabled": ["-DENVOY_ENABLE_UHV"],
                "//conditions:default": [],
            }) + envoy_select_hot_restart(["-DENVOY_HOT_RESTART"], repository) + \
+           envoy_select_disable_exceptions(["-fno-unwind-tables", "-fno-exceptions"], repository) + \
            envoy_select_admin_html(["-DENVOY_ADMIN_HTML"], repository) + \
            envoy_select_static_extension_registration(["-DENVOY_STATIC_EXTENSION_REGISTRATION"], repository) + \
            envoy_select_disable_logging(["-DENVOY_DISABLE_LOGGING"], repository) + \
diff --git a/bazel/envoy_mobile_defines.bzl b/bazel/envoy_mobile_defines.bzl
index df2fd7a7cdbf..5bce8ebef8bf 100644
--- a/bazel/envoy_mobile_defines.bzl
+++ b/bazel/envoy_mobile_defines.bzl
@@ -1,11 +1,12 @@
 # DO NOT LOAD THIS FILE. Load envoy_build_system.bzl instead.
-load(":envoy_select.bzl", "envoy_select_admin_functionality", "envoy_select_enable_http3", "envoy_select_enable_http_datagrams", "envoy_select_enable_yaml", "envoy_select_envoy_mobile_listener", "envoy_select_envoy_mobile_request_compression", "envoy_select_envoy_mobile_stats_reporting", "envoy_select_google_grpc")
+load(":envoy_select.bzl", "envoy_select_admin_functionality", "envoy_select_disable_exceptions", "envoy_select_enable_http3", "envoy_select_enable_http_datagrams", "envoy_select_enable_yaml", "envoy_select_envoy_mobile_listener", "envoy_select_envoy_mobile_request_compression", "envoy_select_envoy_mobile_stats_reporting", "envoy_select_google_grpc")
 
 # Compute the defines needed for Envoy Mobile libraries that don't use Envoy's main library wrappers.
 def envoy_mobile_defines(repository):
     return envoy_select_admin_functionality(["ENVOY_ADMIN_FUNCTIONALITY"], repository) + \
            envoy_select_enable_http3(["ENVOY_ENABLE_QUIC"], repository) + \
            envoy_select_enable_yaml(["ENVOY_ENABLE_YAML"], repository) + \
+           envoy_select_disable_exceptions(["ENVOY_DISABLE_EXCEPTIONS"], repository) + \
            envoy_select_enable_http_datagrams(["ENVOY_ENABLE_HTTP_DATAGRAMS"], repository) + \
            envoy_select_envoy_mobile_listener(["ENVOY_MOBILE_ENABLE_LISTENER"], repository) + \
            envoy_select_envoy_mobile_stats_reporting(["ENVOY_MOBILE_STATS_REPORTING"], repository) + \
diff --git a/bazel/envoy_select.bzl b/bazel/envoy_select.bzl
index 7fee86b3eeab..7cd774bd460e 100644
--- a/bazel/envoy_select.bzl
+++ b/bazel/envoy_select.bzl
@@ -94,6 +94,13 @@ def envoy_select_enable_yaml(xs, repository = ""):
         "//conditions:default": xs,
     })
 
+# Selects the given values if exceptions are disabled in the current build.
+def envoy_select_disable_exceptions(xs, repository = ""):
+    return select({
+        repository + "//bazel:disable_exceptions": xs,
+        "//conditions:default": [],
+    })
+
 # Selects the given values if HTTP datagram support is enabled in the current build.
 def envoy_select_enable_http_datagrams(xs, repository = ""):
     return select({
diff --git a/mobile/library/common/engine.cc b/mobile/library/common/engine.cc
index a21c37c80ac5..125b443728fe 100644
--- a/mobile/library/common/engine.cc
+++ b/mobile/library/common/engine.cc
@@ -49,7 +49,7 @@ envoy_status_t Engine::main(std::unique_ptr&& options) {
   std::unique_ptr main_common;
   {
     Thread::LockGuard lock(mutex_);
-    try {
+    TRY_NEEDS_AUDIT {
       if (event_tracker_.track != nullptr) {
         assert_handler_registration_ =
             Assert::addDebugAssertionFailureRecordAction([this](const char* location) {
@@ -79,13 +79,9 @@ envoy_status_t Engine::main(std::unique_ptr&& options) {
       event_dispatcher_ = &server_->dispatcher();
 
       cv_.notifyAll();
-    } catch (const Envoy::NoServingException& e) {
-      PANIC(e.what());
-    } catch (const Envoy::MalformedArgvException& e) {
-      PANIC(e.what());
-    } catch (const Envoy::EnvoyException& e) {
-      PANIC(e.what());
     }
+    END_TRY
+    CATCH(const Envoy::EnvoyException& e, { PANIC(e.what()); });
 
     // Note: We're waiting longer than we might otherwise to drain to the main thread's dispatcher.
     // This is because we're not simply waiting for its availability and for it to have started, but
diff --git a/source/common/common/thread.h b/source/common/common/thread.h
index b3f2fa17652c..a1d651efe679 100644
--- a/source/common/common/thread.h
+++ b/source/common/common/thread.h
@@ -237,9 +237,34 @@ class MainThread {
 
 #define END_TRY }
 
+#ifdef ENVOY_DISABLE_EXCEPTIONS
+#define TRY_NEEDS_AUDIT {
+#else
 // TODO(chaoqinli-1123): Remove this macros after we have removed all the exceptions from data
 // plane.
-#define TRY_NEEDS_AUDIT try
+#define TRY_NEEDS_AUDIT try {
+#endif
+
+#ifdef ENVOY_DISABLE_EXCEPTIONS
+#define CATCH(ExceptionType, Handler)
+#else
+#define CATCH(Exception, Handler)                                                                  \
+  catch (Exception) {                                                                              \
+    Handler                                                                                        \
+  }
+#endif
+
+#ifdef ENVOY_DISABLE_EXCEPTIONS
+#define MULTI_CATCH(ExceptionType, Handler)
+#else
+#define MULTI_CATCH(Exception, Handler, Handler2)                                                  \
+  catch (Exception) {                                                                              \
+    Handler                                                                                        \
+  }                                                                                                \
+  catch (...) {                                                                                    \
+    Handler2                                                                                       \
+  }
+#endif
 
 // These convenience macros assert properties of the threading system, when
 // feasible. There is a platform-specific mechanism for determining whether the
@@ -277,6 +302,9 @@ class MainThread {
 
 #endif
 
+#ifdef ENVOY_DISABLE_EXCEPTIONS
+#define TRY_ASSERT_MAIN_THREAD {
+#else
 /**
  * To improve exception safety in data plane, we plan to forbid the use of raw
  * try in the core code base. This macros uses main thread assertion to make
@@ -285,6 +313,7 @@ class MainThread {
 #define TRY_ASSERT_MAIN_THREAD                                                                     \
   try {                                                                                            \
     ASSERT_IS_MAIN_OR_TEST_THREAD();
+#endif
 
 /**
  * RAII class to override thread assertions checks in the macros:
diff --git a/source/common/filter/config_discovery_impl.cc b/source/common/filter/config_discovery_impl.cc
index c08e0d146383..3f6c40cce4b4 100644
--- a/source/common/filter/config_discovery_impl.cc
+++ b/source/common/filter/config_discovery_impl.cc
@@ -220,11 +220,12 @@ void FilterConfigProviderManagerImplBase::applyLastOrDefaultConfig(
                                subscription->lastFactoryName());
       last_config_valid = true;
     }
-    END_TRY catch (const EnvoyException& e) {
+    END_TRY CATCH(const EnvoyException& e, {
       ENVOY_LOG(debug, "ECDS subscription {} is invalid in a listener context: {}.",
                 filter_config_name, e.what());
       subscription->incrementConflictCounter();
-    }
+    });
+
     if (last_config_valid) {
       provider.onConfigUpdate(*subscription->lastConfig(), subscription->lastVersionInfo(),
                               nullptr);
diff --git a/source/common/router/header_parser_utils.cc b/source/common/router/header_parser_utils.cc
index 39136cfce837..d278e04de445 100644
--- a/source/common/router/header_parser_utils.cc
+++ b/source/common/router/header_parser_utils.cc
@@ -51,10 +51,7 @@ std::string HeaderParser::translateMetadataFormat(const std::string& header_valu
       int subs = absl::StrReplaceAll({{matches[0].as_string(), new_format}}, &new_header_value);
       ASSERT(subs > 0);
     }
-    END_TRY
-    catch (Json::Exception& e) {
-      return header_value;
-    }
+    END_TRY CATCH(Json::Exception & e, { return header_value; });
   }
 
   return new_header_value;
diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc
index 423493601583..bce5ae8e16a7 100644
--- a/source/common/runtime/runtime_impl.cc
+++ b/source/common/runtime/runtime_impl.cc
@@ -690,13 +690,13 @@ SnapshotImplPtr LoaderImpl::createNewSnapshot() {
           ++disk_layers;
         }
         END_TRY
-        catch (EnvoyException& e) {
+        CATCH(EnvoyException & e, {
           // TODO(htuch): Consider latching here, rather than ignoring the
           // layer. This would be consistent with filesystem RTDS.
           ++error_layers;
           ENVOY_LOG(debug, "error loading runtime values for layer {} from disk: {}",
                     layer.DebugString(), e.what());
-        }
+        });
       }
       break;
     }
diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc
index e20ca282f22e..effcac5a1a62 100644
--- a/source/common/secret/sds_api.cc
+++ b/source/common/secret/sds_api.cc
@@ -73,10 +73,10 @@ void SdsApi::onWatchUpdate() {
     }
   }
   END_TRY
-  catch (const EnvoyException& e) {
+  CATCH(const EnvoyException& e, {
     ENVOY_LOG_MISC(warn, fmt::format("Failed to reload certificates: {}", e.what()));
     sds_api_stats_.key_rotation_failed_.inc();
-  }
+  });
 }
 
 void SdsApi::onConfigUpdate(const std::vector& resources,
diff --git a/source/common/stats/tag_extractor_impl.cc b/source/common/stats/tag_extractor_impl.cc
index 2ac20ee1648f..75efdcf73017 100644
--- a/source/common/stats/tag_extractor_impl.cc
+++ b/source/common/stats/tag_extractor_impl.cc
@@ -21,9 +21,8 @@ namespace {
 std::regex parseStdRegex(const std::string& regex) {
   TRY_ASSERT_MAIN_THREAD { return std::regex(regex, std::regex::optimize); }
   END_TRY
-  catch (const std::regex_error& e) {
-    throw EnvoyException(fmt::format("Invalid regex '{}': {}", regex, e.what()));
-  }
+  CATCH(const std::regex_error& e,
+        { throw EnvoyException(fmt::format("Invalid regex '{}': {}", regex, e.what())); });
 }
 } // namespace
 
diff --git a/source/common/upstream/cds_api_helper.cc b/source/common/upstream/cds_api_helper.cc
index 6bd1655d4a5e..306d6c0fe1e9 100644
--- a/source/common/upstream/cds_api_helper.cc
+++ b/source/common/upstream/cds_api_helper.cc
@@ -54,10 +54,10 @@ CdsApiHelper::onConfigUpdate(const std::vector& adde
       }
     }
     END_TRY
-    catch (const EnvoyException& e) {
-      exception_msgs.push_back(fmt::format("{}: {}", cluster.name(), e.what()));
-    }
+    CATCH(const EnvoyException& e,
+          { exception_msgs.push_back(fmt::format("{}: {}", cluster.name(), e.what())); });
   }
+
   for (const auto& resource_name : removed_resources) {
     if (cm_.removeCluster(resource_name)) {
       any_applied = true;
diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc
index 30f16ac78084..08815cb4acdd 100644
--- a/source/common/upstream/health_discovery_service.cc
+++ b/source/common/upstream/health_discovery_service.cc
@@ -289,14 +289,14 @@ void HdsDelegate::onReceiveMessage(
                           server_context_.messageValidationContext().dynamicValidationVisitor());
   }
   END_TRY
-  catch (const ProtoValidationException& ex) {
+  CATCH(const ProtoValidationException& ex, {
     // Increment error count
     stats_.errors_.inc();
     ENVOY_LOG(warn, "Unable to validate health check specifier: {}", ex.what());
 
     // Do not continue processing message
     return;
-  }
+  });
 
   // Set response
   auto server_response_ms = PROTOBUF_GET_MS_OR_DEFAULT(*message, interval, 1000);
diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc
index 0a36b908f407..718f2f68ec30 100644
--- a/source/common/upstream/upstream_impl.cc
+++ b/source/common/upstream/upstream_impl.cc
@@ -1625,7 +1625,7 @@ const Network::Address::InstanceConstSharedPtr
 ClusterImplBase::resolveProtoAddress(const envoy::config::core::v3::Address& address) {
   TRY_ASSERT_MAIN_THREAD { return Network::Address::resolveProtoAddress(address); }
   END_TRY
-  catch (EnvoyException& e) {
+  CATCH(EnvoyException & e, {
     if (info_->type() == envoy::config::cluster::v3::Cluster::STATIC ||
         info_->type() == envoy::config::cluster::v3::Cluster::EDS) {
       throw EnvoyException(fmt::format("{}. Consider setting resolver_name or setting cluster type "
@@ -1633,7 +1633,7 @@ ClusterImplBase::resolveProtoAddress(const envoy::config::core::v3::Address& add
                                        e.what()));
     }
     throw e;
-  }
+  });
 }
 
 void ClusterImplBase::validateEndpointsForZoneAwareRouting(
diff --git a/source/exe/stripped_main_base.cc b/source/exe/stripped_main_base.cc
index 9d532a993b90..878e72a75735 100644
--- a/source/exe/stripped_main_base.cc
+++ b/source/exe/stripped_main_base.cc
@@ -129,10 +129,10 @@ void StrippedMainBase::configureHotRestarter(Random::RandomGenerator& random_gen
                                                                options_.socketMode());
         }
         END_TRY
-        catch (Server::HotRestartDomainSocketInUseException& ex) {
+        CATCH(Server::HotRestartDomainSocketInUseException & ex, {
           // No luck, try again.
           ENVOY_LOG_MISC(debug, "dynamic base id: {}", ex.what());
-        }
+        });
       }
 
       if (restarter == nullptr) {
diff --git a/source/extensions/grpc_credentials/file_based_metadata/config.cc b/source/extensions/grpc_credentials/file_based_metadata/config.cc
index aebba3b17150..1fa300a2f711 100644
--- a/source/extensions/grpc_credentials/file_based_metadata/config.cc
+++ b/source/extensions/grpc_credentials/file_based_metadata/config.cc
@@ -75,8 +75,9 @@ FileBasedMetadataAuthenticator::GetMetadata(grpc::string_ref, grpc::string_ref,
     std::string header_value = Envoy::Config::DataSource::read(config_.secret_data(), true, api_);
     metadata->insert(std::make_pair(header_key, header_prefix + header_value));
   }
+  END_TRY
   catch (const EnvoyException& e) {
-    return grpc::Status(grpc::StatusCode::NOT_FOUND, e.what());
+    return {grpc::StatusCode::NOT_FOUND, e.what()};
   }
   return grpc::Status::OK;
 }
diff --git a/source/extensions/network/dns_resolver/cares/dns_impl.cc b/source/extensions/network/dns_resolver/cares/dns_impl.cc
index 1c4bd6fa6a51..12cf91e9c33f 100644
--- a/source/extensions/network/dns_resolver/cares/dns_impl.cc
+++ b/source/extensions/network/dns_resolver/cares/dns_impl.cc
@@ -287,6 +287,7 @@ void DnsResolverImpl::PendingResolution::finishResolve() {
     TRY_NEEDS_AUDIT {
       callback_(pending_response_.status_, std::move(pending_response_.address_list_));
     }
+    END_TRY
     catch (const EnvoyException& e) {
       ENVOY_LOG(critical, "EnvoyException in c-ares callback: {}", e.what());
       dispatcher_.post([s = std::string(e.what())] { throw EnvoyException(s); });
diff --git a/source/server/options_impl.cc b/source/server/options_impl.cc
index 6345c789e897..81d7490c095e 100644
--- a/source/server/options_impl.cc
+++ b/source/server/options_impl.cc
@@ -161,25 +161,24 @@ OptionsImpl::OptionsImpl(std::vector args,
       false, "string", cmd);
 
   cmd.setExceptionHandling(false);
-  TRY_ASSERT_MAIN_THREAD {
-    cmd.parse(args);
-    count_ = cmd.getArgList().size();
-  }
-  END_TRY
-  catch (TCLAP::ArgException& e) {
+
+  std::function failure_function = [&](TCLAP::ArgException& e) {
     TRY_ASSERT_MAIN_THREAD { cmd.getOutput()->failure(cmd, e); }
     END_TRY
-    catch (const TCLAP::ExitException&) {
+    CATCH(const TCLAP::ExitException&, {
       // failure() has already written an informative message to stderr, so all that's left to do
       // is throw our own exception with the original message.
       throw MalformedArgvException(e.what());
-    }
-  }
-  catch (const TCLAP::ExitException& e) {
-    // parse() throws an ExitException with status 0 after printing the output for --help and
-    // --version.
-    throw NoServingException();
+    });
+  };
+
+  TRY_ASSERT_MAIN_THREAD {
+    cmd.parse(args);
+    count_ = cmd.getArgList().size();
   }
+  END_TRY
+  MULTI_CATCH(
+      TCLAP::ArgException & e, { failure_function(e); }, { throw NoServingException(); });
 
   hot_restart_disabled_ = disable_hot_restart.getValue();
   mutex_tracing_enabled_ = enable_mutex_tracing.getValue();
diff --git a/source/server/server.cc b/source/server/server.cc
index 8ed78eb8b139..91c72e529197 100644
--- a/source/server/server.cc
+++ b/source/server/server.cc
@@ -105,41 +105,41 @@ InstanceImpl::InstanceImpl(
       router_context_(store.symbolTable()), process_context_(std::move(process_context)),
       hooks_(hooks), quic_stat_names_(store.symbolTable()), server_contexts_(*this),
       enable_reuse_port_default_(true), stats_flush_in_progress_(false) {
+  std::function set_up_logger = [&] {
+    TRY_ASSERT_MAIN_THREAD {
+      file_logger_ = std::make_unique(
+          options.logPath(), access_log_manager_, Logger::Registry::getSink());
+    }
+    END_TRY
+    CATCH(const EnvoyException& e, {
+      throw EnvoyException(
+          fmt::format("Failed to open log-file '{}'. e.what(): {}", options.logPath(), e.what()));
+    });
+  };
+
   TRY_ASSERT_MAIN_THREAD {
     if (!options.logPath().empty()) {
-      TRY_ASSERT_MAIN_THREAD {
-        file_logger_ = std::make_unique(
-            options.logPath(), access_log_manager_, Logger::Registry::getSink());
-      }
-      END_TRY
-      catch (const EnvoyException& e) {
-        throw EnvoyException(
-            fmt::format("Failed to open log-file '{}'. e.what(): {}", options.logPath(), e.what()));
-      }
+      set_up_logger();
     }
-
     restarter_.initialize(*dispatcher_, *this);
     drain_manager_ = component_factory.createDrainManager(*this);
     initialize(std::move(local_address), component_factory);
   }
   END_TRY
-  catch (const EnvoyException& e) {
-    ENVOY_LOG(critical, "error initializing config '{} {} {}': {}",
-              options.configProto().DebugString(), options.configYaml(), options.configPath(),
-              e.what());
-    terminate();
-    throw;
-  }
-  catch (const std::exception& e) {
-    ENVOY_LOG(critical, "error initializing due to unexpected exception: {}", e.what());
-    terminate();
-    throw;
-  }
-  catch (...) {
-    ENVOY_LOG(critical, "error initializing due to unknown exception");
-    terminate();
-    throw;
-  }
+  MULTI_CATCH(
+      const EnvoyException& e,
+      {
+        ENVOY_LOG(critical, "error initializing config '{} {} {}': {}",
+                  options.configProto().DebugString(), options.configYaml(), options.configPath(),
+                  e.what());
+        terminate();
+        throw;
+      },
+      {
+        ENVOY_LOG(critical, "error initializing due to unknown exception");
+        terminate();
+        throw;
+      });
 }
 
 InstanceImpl::~InstanceImpl() {

From a34e994f2720c2516d59b978f481643ef372d9b2 Mon Sep 17 00:00:00 2001
From: yanavlasov 
Date: Wed, 7 Jun 2023 10:05:52 -0400
Subject: [PATCH 477/740] UHV: Enable oghttp2 and UHV in HTTP/2 flood test
 (#27543)

Enable oghttp2 and UHV in HTTP/2 flood test

Signed-off-by: Yan Avlasov 
---
 source/common/http/codec_client.cc            |  1 -
 .../http2_flood_integration_test.cc           | 38 +++++++++++++------
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc
index 51ac7cffa591..61cad323553c 100644
--- a/source/common/http/codec_client.cc
+++ b/source/common/http/codec_client.cc
@@ -247,7 +247,6 @@ void CodecClient::ActiveRequest::decodeHeaders(ResponseHeaderMapPtr&& headers, b
                 .http3Options()
                 .override_stream_error_on_invalid_http_message()
                 .value())) {
-        parent_.host_->cluster().trafficStats()->upstream_cx_protocol_error_.inc();
         parent_.protocol_error_ = true;
         parent_.close();
       } else {
diff --git a/test/integration/http2_flood_integration_test.cc b/test/integration/http2_flood_integration_test.cc
index 6a08c69b12a8..6f508bf35bdf 100644
--- a/test/integration/http2_flood_integration_test.cc
+++ b/test/integration/http2_flood_integration_test.cc
@@ -31,17 +31,18 @@ namespace {
 const uint32_t ControlFrameFloodLimit = 100;
 const uint32_t AllFrameFloodLimit = 1000;
 
-bool deferredProcessing(std::tuple params) {
+bool deferredProcessing(std::tuple params) {
   return std::get<2>(params);
 }
 
 } // namespace
 
 std::string testParamsToString(
-    const ::testing::TestParamInfo> params) {
-  const bool http2_new_codec_wrapper = std::get<1>(params.param);
+    const ::testing::TestParamInfo>
+        params) {
+  const Http2Impl http2_codec_impl = std::get<1>(params.param);
   return absl::StrCat(TestUtility::ipVersionToString(std::get<0>(params.param)),
-                      http2_new_codec_wrapper ? "WrappedHttp2" : "BareHttp2",
+                      http2_codec_impl == Http2Impl::Oghttp2 ? "Oghttp2" : "Nghttp2",
                       deferredProcessing(params.param) ? "WithDeferredProcessing"
                                                        : "NoDeferredProcessing");
 }
@@ -53,7 +54,7 @@ std::string testParamsToString(
 // Http2FrameIntegrationTest destructor completes.
 class Http2FloodMitigationTest
     : public SocketInterfaceSwap,
-      public testing::TestWithParam>,
+      public testing::TestWithParam>,
       public Http2RawFrameIntegrationTest {
 public:
   Http2FloodMitigationTest()
@@ -67,9 +68,7 @@ class Http2FloodMitigationTest
         [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager&
                hcm) { hcm.mutable_delayed_close_timeout()->set_seconds(1); });
     config_helper_.addConfigModifier(configureProxyStatus());
-    const bool enable_new_wrapper = std::get<1>(GetParam());
-    config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_new_codec_wrapper",
-                                      enable_new_wrapper ? "true" : "false");
+    setupHttp2ImplOverrides(std::get<1>(GetParam()));
     config_helper_.addRuntimeOverride(Runtime::defer_processing_backedup_streams,
                                       deferredProcessing(GetParam()) ? "true" : "false");
   }
@@ -92,7 +91,8 @@ class Http2FloodMitigationTest
 
 INSTANTIATE_TEST_SUITE_P(
     IpVersions, Http2FloodMitigationTest,
-    testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), ::testing::Bool(),
+    testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
+                     testing::ValuesIn({Http2Impl::Nghttp2, Http2Impl::Oghttp2}),
                      ::testing::Bool()),
     testParamsToString);
 
@@ -680,6 +680,14 @@ TEST_P(Http2FloodMitigationTest, WindowUpdateOnLowWatermarkFlood) {
 
 // Verify that the server can detect flood of RST_STREAM frames.
 TEST_P(Http2FloodMitigationTest, RST_STREAM) {
+#ifdef ENVOY_ENABLE_UHV
+  // TODO(#27541): the invalid frame that used to cause Envoy to send only RST_STREAM now causes
+  // Envoy to also send 400 in UHV mode (it is allowed by RFC). The test needs to be fixed to make
+  // server only send RST_STREAM.
+  if (std::get<1>(GetParam()) == Http2Impl::Oghttp2) {
+    return;
+  }
+#endif
   // Use invalid HTTP headers to trigger sending RST_STREAM frames.
   config_helper_.addConfigModifier(
       [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager&
@@ -1120,13 +1128,21 @@ TEST_P(Http2FloodMitigationTest, ZerolenHeader) {
   EXPECT_EQ(1, test_server_->counter("http2.rx_messaging_error")->value());
   EXPECT_EQ(1,
             test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value());
-  EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("http2.invalid.header.field"));
+  EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("header"));
   // expect a downstream protocol error.
   EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("DPE"));
 }
 
 // Verify that only the offending stream is terminated upon receiving invalid HEADERS frame.
 TEST_P(Http2FloodMitigationTest, ZerolenHeaderAllowed) {
+#ifdef ENVOY_ENABLE_UHV
+  // TODO(#27541): the invalid frame that used to cause Envoy to send only RST_STREAM now causes
+  // Envoy to also send 400 in UHV mode (it is allowed by RFC). The test needs to be fixed to make
+  // server only send RST_STREAM.
+  if (std::get<1>(GetParam()) == Http2Impl::Oghttp2) {
+    return;
+  }
+#endif
   useAccessLog("%RESPONSE_FLAGS% %RESPONSE_CODE_DETAILS%");
   config_helper_.addConfigModifier(
       [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager&
@@ -1161,7 +1177,7 @@ TEST_P(Http2FloodMitigationTest, ZerolenHeaderAllowed) {
   EXPECT_EQ(0,
             test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value());
   // expect Downstream Protocol Error
-  EXPECT_THAT(waitForAccessLog(access_log_name_, 0, true), HasSubstr("http2.invalid.header.field"));
+  EXPECT_THAT(waitForAccessLog(access_log_name_, 0, true), HasSubstr("header"));
   EXPECT_THAT(waitForAccessLog(access_log_name_, 0, true), HasSubstr("DPE"));
 }
 

From ad13bbb65d30d6c342dc842de51347a71e512951 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 7 Jun 2023 07:26:56 -0700
Subject: [PATCH 478/740] build(deps): bump elasticsearch from 8.7.1 to 8.8.0
 in /examples/skywalking (#27838)

build(deps): bump elasticsearch in /examples/skywalking

Bumps elasticsearch from 8.7.1 to 8.8.0.

---
updated-dependencies:
- dependency-name: elasticsearch
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/skywalking/Dockerfile-elasticsearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/skywalking/Dockerfile-elasticsearch b/examples/skywalking/Dockerfile-elasticsearch
index faa8bcdab143..b721cc086bfa 100644
--- a/examples/skywalking/Dockerfile-elasticsearch
+++ b/examples/skywalking/Dockerfile-elasticsearch
@@ -1 +1 @@
-FROM elasticsearch:8.7.1@sha256:160814e5972521291420c29edf4c0348b8591ac9235156f0dbf34befcf362825
+FROM elasticsearch:8.8.0@sha256:5c28849be5e91610761fcd4a49c2561dfae72be9ac0a3e7b5c42c9576aa9157b

From 14ef459c3691bdf722b1434a0e955ff1986bada1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 7 Jun 2023 07:28:23 -0700
Subject: [PATCH 479/740] build(deps): bump golang from 1.20.4-bullseye to
 1.20.5-bullseye in /examples/shared/golang (#27837)

build(deps): bump golang in /examples/shared/golang

Bumps golang from 1.20.4-bullseye to 1.20.5-bullseye.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/golang/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile
index 17bff24cf5a4..3e3ac40744eb 100644
--- a/examples/shared/golang/Dockerfile
+++ b/examples/shared/golang/Dockerfile
@@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache
 
 
-FROM golang:1.20.4-bullseye@sha256:918857f4064db0fff49799ce5e7c4d43e394f452111cd89cca9af539c18a76a8 as golang-base
+FROM golang:1.20.5-bullseye@sha256:419bc8954c0e08c539830c8669ccd116a063303481c748fabd09d8fd6d4e2c5f as golang-base
 
 
 FROM golang-base as golang-control-plane-builder

From 29780678dba79df1088f89ac71b50b97ab2eac35 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 7 Jun 2023 07:30:59 -0700
Subject: [PATCH 480/740] build(deps): bump openpolicyagent/opa from
 0.53.0-istio to 0.53.1-istio in /examples/ext_authz (#27835)

build(deps): bump openpolicyagent/opa in /examples/ext_authz

Bumps openpolicyagent/opa from 0.53.0-istio to 0.53.1-istio.

---
updated-dependencies:
- dependency-name: openpolicyagent/opa
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/ext_authz/Dockerfile-opa | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/ext_authz/Dockerfile-opa b/examples/ext_authz/Dockerfile-opa
index 70847d7d5ae2..f3ed85f6e594 100644
--- a/examples/ext_authz/Dockerfile-opa
+++ b/examples/ext_authz/Dockerfile-opa
@@ -1 +1 @@
-FROM openpolicyagent/opa:0.53.0-istio@sha256:dcf28d23b3d1919b6314ef34342687c558cec64fe4f93fa484e8c8e82824d56d
+FROM openpolicyagent/opa:0.53.1-istio@sha256:b4821d33fdbca4fe3d2ff85e3f4a18d292f2fb32fe7db995b9239d7242702bab

From 26ddee23f650f1d58bdf0327857ec3a31c54f004 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 7 Jun 2023 07:38:20 -0700
Subject: [PATCH 481/740] build(deps): bump otel/opentelemetry-collector from
 `32edac6` to `324e2c7` in /examples/opentelemetry (#27836)

build(deps): bump otel/opentelemetry-collector

Bumps otel/opentelemetry-collector from `32edac6` to `324e2c7`.

---
updated-dependencies:
- dependency-name: otel/opentelemetry-collector
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/opentelemetry/Dockerfile-opentelemetry | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry
index ad03819c59e4..38a00b8e466d 100644
--- a/examples/opentelemetry/Dockerfile-opentelemetry
+++ b/examples/opentelemetry/Dockerfile-opentelemetry
@@ -1,7 +1,7 @@
 FROM alpine:3.18@sha256:02bb6f428431fbc2809c5d1b41eab5a68350194fb508869a33cb1af4444c9b11 as otelc_curl
 RUN apk --update add curl
 
-FROM otel/opentelemetry-collector:latest@sha256:32edac6fe9479c4a33a60be226af0a3ee9f69a5b79d935835b37bf9b25e4b5ce
+FROM otel/opentelemetry-collector:latest@sha256:324e2c7bdd1ecd58c1a1347737174371b4917d71a05cc4403ce3fb83bcddf836
 
 COPY --from=otelc_curl / /
 

From 62d3974f5a9dd1291b9f9d96990e3e658c0a8240 Mon Sep 17 00:00:00 2001
From: Tony Han 
Date: Wed, 7 Jun 2023 22:59:11 +0800
Subject: [PATCH 482/740] docs: fix yaml syntax in scoped route example
 (#27805)

Signed-off-by: Bing Han 
---
 api/envoy/api/v2/scoped_route.proto          | 2 +-
 api/envoy/config/route/v3/scoped_route.proto | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/envoy/api/v2/scoped_route.proto b/api/envoy/api/v2/scoped_route.proto
index f3902d9d9e7e..99a39aedac5c 100644
--- a/api/envoy/api/v2/scoped_route.proto
+++ b/api/envoy/api/v2/scoped_route.proto
@@ -39,7 +39,7 @@ option (udpa.annotations.file_status).package_version_status = FROZEN;
 //       fragments:
 //         - header_value_extractor:
 //             name: X-Route-Selector
-//             element_separator: ,
+//             element_separator: ","
 //             element:
 //               separator: =
 //               key: vip
diff --git a/api/envoy/config/route/v3/scoped_route.proto b/api/envoy/config/route/v3/scoped_route.proto
index 27bcd617ae16..ff4cc689c898 100644
--- a/api/envoy/config/route/v3/scoped_route.proto
+++ b/api/envoy/config/route/v3/scoped_route.proto
@@ -44,7 +44,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 //       fragments:
 //         - header_value_extractor:
 //             name: X-Route-Selector
-//             element_separator: ,
+//             element_separator: ","
 //             element:
 //               separator: =
 //               key: vip

From 009d73924ca66c1ff9224c61ac403d4fe6496dad Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Wed, 7 Jun 2023 13:00:52 -0400
Subject: [PATCH 483/740] coverage: fixing a coverage flake by inlining an
 error condition only run in debug builds (#27845)

Signed-off-by: Alyssa Wilk 
---
 source/common/common/posix/thread_impl.cc | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/source/common/common/posix/thread_impl.cc b/source/common/common/posix/thread_impl.cc
index fae09194d8cc..5d78dd24a56e 100644
--- a/source/common/common/posix/thread_impl.cc
+++ b/source/common/common/posix/thread_impl.cc
@@ -101,12 +101,8 @@ class ThreadImplPosix : public Thread {
     // Verify that the name got written into the thread as expected.
     char buf[PTHREAD_MAX_THREADNAME_LEN_INCLUDING_NULL_BYTE];
     const int get_name_rc = pthread_getname_np(thread_handle_, buf, sizeof(buf));
-    if (get_name_rc != 0) {
-      ENVOY_LOG_MISC(trace, "Error {} getting name", get_name_rc);
-      return false;
-    }
     name = buf;
-    return true;
+    return get_name_rc == 0;
   }
 #endif
 

From 548d4771d8737fa18b568252d0e3ff833ed3bf64 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Wed, 7 Jun 2023 14:00:24 -0400
Subject: [PATCH 484/740] http: moving exceptions out of
 http_server_properties_cache_manager_impl (#27829)

Signed-off-by: Alyssa Wilk 
---
 .../http_server_properties_cache_manager_impl.cc   |  8 +-------
 .../http/alternate_protocols_cache/filter.cc       |  1 +
 source/extensions/upstreams/http/config.cc         | 14 +++++++++++---
 .../http_server_properties_cache_manager_test.cc   | 14 +++-----------
 test/extensions/upstreams/http/config_test.cc      | 12 ++++++++++++
 tools/code_format/config.yaml                      |  1 -
 6 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/source/common/http/http_server_properties_cache_manager_impl.cc b/source/common/http/http_server_properties_cache_manager_impl.cc
index e05694c30e04..307e8991eb55 100644
--- a/source/common/http/http_server_properties_cache_manager_impl.cc
+++ b/source/common/http/http_server_properties_cache_manager_impl.cc
@@ -24,16 +24,10 @@ HttpServerPropertiesCacheManagerImpl::HttpServerPropertiesCacheManagerImpl(
 HttpServerPropertiesCacheSharedPtr HttpServerPropertiesCacheManagerImpl::getCache(
     const envoy::config::core::v3::AlternateProtocolsCacheOptions& options,
     Event::Dispatcher& dispatcher) {
-  if (options.has_key_value_store_config() && data_.concurrency_ != 1) {
-    throw EnvoyException(
-        fmt::format("options has key value store but Envoy has concurrency = {} : {}",
-                    data_.concurrency_, options.DebugString()));
-  }
-
   const auto& existing_cache = (*slot_).caches_.find(options.name());
   if (existing_cache != (*slot_).caches_.end()) {
     if (!Protobuf::util::MessageDifferencer::Equivalent(options, existing_cache->second.options_)) {
-      throw EnvoyException(fmt::format(
+      IS_ENVOY_BUG(fmt::format(
           "options specified alternate protocols cache '{}' with different settings"
           " first '{}' second '{}'",
           options.name(), existing_cache->second.options_.DebugString(), options.DebugString()));
diff --git a/source/extensions/filters/http/alternate_protocols_cache/filter.cc b/source/extensions/filters/http/alternate_protocols_cache/filter.cc
index c3130a7ce175..5982ceceb3e6 100644
--- a/source/extensions/filters/http/alternate_protocols_cache/filter.cc
+++ b/source/extensions/filters/http/alternate_protocols_cache/filter.cc
@@ -38,6 +38,7 @@ Filter::Filter(const FilterConfigSharedPtr& config, Event::Dispatcher& dispatche
     : cache_(config->getAlternateProtocolCache(dispatcher)), time_source_(config->timeSource()) {}
 
 Http::FilterHeadersStatus Filter::encodeHeaders(Http::ResponseHeaderMap& headers, bool) {
+
   if (!cache_) {
     return Http::FilterHeadersStatus::Continue;
   }
diff --git a/source/extensions/upstreams/http/config.cc b/source/extensions/upstreams/http/config.cc
index 39ee82441ef4..41feefaaf5d9 100644
--- a/source/extensions/upstreams/http/config.cc
+++ b/source/extensions/upstreams/http/config.cc
@@ -84,13 +84,21 @@ bool useHttp3(const envoy::extensions::upstreams::http::v3::HttpProtocolOptions&
 
 absl::optional
 getAlternateProtocolsCacheOptions(
-    const envoy::extensions::upstreams::http::v3::HttpProtocolOptions& options) {
+    const envoy::extensions::upstreams::http::v3::HttpProtocolOptions& options,
+    Server::Configuration::ServerFactoryContext& server_context) {
   if (options.has_auto_config() && options.auto_config().has_http3_protocol_options()) {
     if (!options.auto_config().has_alternate_protocols_cache_options()) {
       throw EnvoyException(fmt::format("alternate protocols cache must be configured when HTTP/3 "
                                        "is enabled with auto_config"));
     }
-    return options.auto_config().alternate_protocols_cache_options();
+    auto cache_options = options.auto_config().alternate_protocols_cache_options();
+    if (cache_options.has_key_value_store_config() && server_context.options().concurrency() != 1) {
+      throw EnvoyException(
+          fmt::format("options has key value store but Envoy has concurrency = {} : {}",
+                      server_context.options().concurrency(), cache_options.DebugString()));
+    }
+
+    return cache_options;
   }
   return absl::nullopt;
 }
@@ -178,7 +186,7 @@ ProtocolOptionsConfigImpl::ProtocolOptionsConfigImpl(
                     options.upstream_http_protocol_options())
               : absl::nullopt),
       http_filters_(options.http_filters()),
-      alternate_protocol_cache_options_(getAlternateProtocolsCacheOptions(options)),
+      alternate_protocol_cache_options_(getAlternateProtocolsCacheOptions(options, server_context)),
       header_validator_factory_(createHeaderValidatorFactory(options, server_context)),
       use_downstream_protocol_(options.has_use_downstream_protocol_config()),
       use_http2_(useHttp2(options)), use_http3_(useHttp3(options)),
diff --git a/test/common/http/http_server_properties_cache_manager_test.cc b/test/common/http/http_server_properties_cache_manager_test.cc
index dd4f93dae443..5f6ccede15de 100644
--- a/test/common/http/http_server_properties_cache_manager_test.cc
+++ b/test/common/http/http_server_properties_cache_manager_test.cc
@@ -100,14 +100,6 @@ TEST_F(HttpServerPropertiesCacheManagerTest, GetCacheWithCanonicalEntry) {
   EXPECT_TRUE(cache->findAlternatives(origin).has_value());
 }
 
-TEST_F(HttpServerPropertiesCacheManagerTest, GetCacheWithFlushingAndConcurrency) {
-  EXPECT_CALL(context_.options_, concurrency()).WillOnce(Return(5));
-  options1_.mutable_key_value_store_config();
-  initialize();
-  EXPECT_THROW_WITH_REGEX(manager_->getCache(options1_, dispatcher_), EnvoyException,
-                          "options has key value store but Envoy has concurrency = 5");
-}
-
 TEST_F(HttpServerPropertiesCacheManagerTest, GetCacheForDifferentOptions) {
   initialize();
   HttpServerPropertiesCacheSharedPtr cache1 = manager_->getCache(options1_, dispatcher_);
@@ -120,9 +112,9 @@ TEST_F(HttpServerPropertiesCacheManagerTest, GetCacheForConflictingOptions) {
   initialize();
   HttpServerPropertiesCacheSharedPtr cache1 = manager_->getCache(options1_, dispatcher_);
   options2_.set_name(options1_.name());
-  EXPECT_THROW_WITH_REGEX(
-      manager_->getCache(options2_, dispatcher_), EnvoyException,
-      "options specified alternate protocols cache 'name1' with different settings.*");
+  EXPECT_ENVOY_BUG(manager_->getCache(options2_, dispatcher_),
+                   "options specified alternate protocols cache 'name1' with different settings "
+                   "first 'name: \"name1\"");
 }
 
 } // namespace
diff --git a/test/extensions/upstreams/http/config_test.cc b/test/extensions/upstreams/http/config_test.cc
index 60f5ae3b8a32..7b54807970be 100644
--- a/test/extensions/upstreams/http/config_test.cc
+++ b/test/extensions/upstreams/http/config_test.cc
@@ -84,6 +84,18 @@ TEST_F(ConfigTest, AutoHttp3NoCache) {
       "alternate protocols cache must be configured when HTTP/3 is enabled with auto_config");
 }
 
+TEST_F(ConfigTest, KvStoreConcurrencyFail) {
+  options_.mutable_auto_config();
+  options_.mutable_auto_config()->mutable_http3_protocol_options();
+  options_.mutable_auto_config()
+      ->mutable_alternate_protocols_cache_options()
+      ->mutable_key_value_store_config();
+  server_context_.options_.concurrency_ = 2;
+  EXPECT_THROW_WITH_MESSAGE(
+      ProtocolOptionsConfigImpl config(options_, server_context_), EnvoyException,
+      "options has key value store but Envoy has concurrency = 2 : key_value_store_config {\n}\n");
+}
+
 namespace {
 
 class TestHeaderValidatorFactoryConfig : public ::Envoy::Http::HeaderValidatorFactoryConfig {
diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml
index c189b2c994c1..5129d452f7d8 100644
--- a/tools/code_format/config.yaml
+++ b/tools/code_format/config.yaml
@@ -130,7 +130,6 @@ paths:
     - source/common/http/filter_chain_helper.h
     - source/common/http/conn_manager_utility.cc
     - source/common/http/match_delegate/config.cc
-    - source/common/http/http_server_properties_cache_manager_impl.cc
     - source/common/protobuf/yaml_utility.cc
     - source/common/protobuf/visitor_helper.h
     - source/common/protobuf/visitor.cc

From d1cc005b75e36f15d3c5af29ccaf2bf792550091 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Wed, 7 Jun 2023 15:32:48 -0400
Subject: [PATCH 485/740] Runtime: correctly outputting runtime on admin
 console (#27739)

Risk Level: low
Testing: new tests
Docs Changes: n/a
Release Notes: n/a
Fixes #27376

Signed-off-by: Alyssa Wilk 
---
 source/common/runtime/runtime_impl.cc         | 18 +++--
 .../circuit_breakers_integration_test.cc      | 71 ++++++++++++++++++-
 2 files changed, 83 insertions(+), 6 deletions(-)

diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc
index bce5ae8e16a7..20704683ac13 100644
--- a/source/common/runtime/runtime_impl.cc
+++ b/source/common/runtime/runtime_impl.cc
@@ -323,26 +323,35 @@ void parseEntryBooleanValue(Envoy::Runtime::Snapshot::Entry& entry) {
 SnapshotImpl::Entry SnapshotImpl::createEntry(const ProtobufWkt::Value& value,
                                               absl::string_view raw_string) {
   Entry entry;
+  entry.raw_string_value_ = value.string_value();
+  if (!raw_string.empty()) {
+    entry.raw_string_value_ = raw_string;
+  }
   switch (value.kind_case()) {
   case ProtobufWkt::Value::kNumberValue:
     setNumberValue(entry, value.number_value());
+    if (entry.raw_string_value_.empty()) {
+      entry.raw_string_value_ = absl::StrCat(value.number_value());
+    }
     break;
   case ProtobufWkt::Value::kBoolValue:
     entry.bool_value_ = value.bool_value();
+    if (entry.raw_string_value_.empty()) {
+      entry.raw_string_value_ = absl::StrCat(value.bool_value());
+    }
     break;
   case ProtobufWkt::Value::kStructValue:
+    if (entry.raw_string_value_.empty()) {
+      entry.raw_string_value_ = value.struct_value().DebugString();
+    }
     parseFractionValue(entry, value.struct_value());
     break;
   case ProtobufWkt::Value::kStringValue:
-    entry.raw_string_value_ = value.string_value();
     parseEntryDoubleValue(entry);
     // TODO(alyssawilk) after this PR lands and sticks, ENVOY_BUG these
     // functions and see if we can remove the special casing.
     parseEntryBooleanValue(entry);
     parseEntryFractionalPercentValue(entry);
-    if (!raw_string.empty()) {
-      entry.raw_string_value_ = raw_string;
-    }
   default:
     break;
   }
@@ -409,6 +418,7 @@ void DiskLayer::walkDirectory(const std::string& path, const std::string& prefix
       // Read the file and remove any comments. A comment is a line starting with a '#' character.
       // Comments are useful for placeholder files with no value.
       const std::string text_file{api.fileSystem().fileReadToEnd(full_path)};
+
       const auto lines = StringUtil::splitToken(text_file, "\n");
       for (const auto& line : lines) {
         if (!line.empty() && line.front() == '#') {
diff --git a/test/integration/circuit_breakers_integration_test.cc b/test/integration/circuit_breakers_integration_test.cc
index b0a7b5409d39..6d38b1077693 100644
--- a/test/integration/circuit_breakers_integration_test.cc
+++ b/test/integration/circuit_breakers_integration_test.cc
@@ -121,12 +121,79 @@ TEST_P(CircuitBreakersIntegrationTest, CircuitBreakerRuntime) {
   const std::string expected_json1 = R"EOF(
   "circuit_breakers.cluster_0.default.max_retries": {
 )EOF";
-  EXPECT_TRUE(absl::StrContains(response->body(), expected_json1));
+  EXPECT_TRUE(absl::StrContains(response->body(), expected_json1)) << response->body();
 
   const std::string expected_json2 = R"EOF("final_value": "1024")EOF";
-  EXPECT_TRUE(absl::StrContains(response->body(), expected_json2));
+  EXPECT_TRUE(absl::StrContains(response->body(), expected_json2)) << response->body();
 #endif
 }
 
+TEST_P(CircuitBreakersIntegrationTest, CircuitBreakerRuntimeProto) {
+  config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+    auto* static_resources = bootstrap.mutable_static_resources();
+    auto* cluster = static_resources->mutable_clusters(0);
+
+    auto* outlier_detection = cluster->mutable_outlier_detection();
+    outlier_detection->mutable_consecutive_gateway_failure()->set_value(1);
+    outlier_detection->mutable_consecutive_5xx()->set_value(1);
+    outlier_detection->mutable_consecutive_local_origin_failure()->set_value(1);
+
+    outlier_detection->mutable_enforcing_consecutive_gateway_failure()->set_value(100);
+    outlier_detection->mutable_enforcing_consecutive_5xx()->set_value(100);
+    outlier_detection->mutable_enforcing_consecutive_local_origin_failure()->set_value(100);
+
+    outlier_detection->set_split_external_local_origin_errors(true);
+
+    outlier_detection->mutable_max_ejection_percent()->set_value(100);
+    outlier_detection->mutable_interval()->set_nanos(1);
+    outlier_detection->mutable_base_ejection_time()->set_seconds(3600);
+    outlier_detection->mutable_max_ejection_time()->set_seconds(3600);
+
+    auto* layer = bootstrap.mutable_layered_runtime()->add_layers();
+    layer->set_name("enable layer");
+    ProtobufWkt::Struct& runtime = *layer->mutable_static_layer();
+
+    (*runtime.mutable_fields())["circuit_breakers.cluster_0.default.max_requests"].set_number_value(
+        0);
+    (*runtime.mutable_fields())["circuit_breakers.cluster_0.default.max_retries"].set_number_value(
+        1024);
+  });
+
+  initialize();
+
+  codec_client_ = makeHttpConnection(lookupPort("http"));
+
+  auto response = codec_client_->makeRequestWithBody(default_request_headers_, 1024);
+
+  test_server_->waitForGaugeEq("cluster.cluster_0.upstream_rq_active", 0);
+  test_server_->waitForGaugeEq("cluster.cluster_0.upstream_rq_pending_active", 0);
+
+  ASSERT_TRUE(response->waitForEndStream());
+
+  EXPECT_EQ("503", response->headers().getStatusValue());
+  test_server_->waitForCounterGe("cluster.cluster_0.upstream_rq_503", 1);
+
+  EXPECT_EQ(test_server_->counter("cluster.cluster_0.upstream_rq_pending_overflow")->value(), 1);
+
+  EXPECT_EQ(test_server_->counter("cluster.cluster_0.outlier_detection.ejections_enforced_total")
+                ->value(),
+            0);
+#ifdef ENVOY_ADMIN_FUNCTIONALITY
+  auto codec_client2 = makeHttpConnection(lookupPort("admin"));
+  default_request_headers_.setPath("/runtime");
+  response = codec_client2->makeHeaderOnlyRequest(default_request_headers_);
+  ASSERT_TRUE(response->waitForEndStream());
+  EXPECT_EQ("200", response->headers().getStatusValue());
+  codec_client2->close();
+
+  const std::string expected_json1 = R"EOF(
+  "circuit_breakers.cluster_0.default.max_retries": {
+)EOF";
+  EXPECT_TRUE(absl::StrContains(response->body(), expected_json1)) << response->body();
+
+  const std::string expected_json2 = R"EOF("final_value": "1024")EOF";
+  EXPECT_TRUE(absl::StrContains(response->body(), expected_json2)) << response->body();
+#endif
+}
 } // namespace
 } // namespace Envoy

From f3281aeb7339527fd7a473fe2c6af02aa058ee93 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Wed, 7 Jun 2023 19:35:53 -0400
Subject: [PATCH 486/740] disabling flake (#27843)

Signed-off-by: Alyssa Wilk 
---
 contrib/generic_proxy/filters/network/test/integration_test.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/contrib/generic_proxy/filters/network/test/integration_test.cc b/contrib/generic_proxy/filters/network/test/integration_test.cc
index d68f2d3b154e..552dc6afcc01 100644
--- a/contrib/generic_proxy/filters/network/test/integration_test.cc
+++ b/contrib/generic_proxy/filters/network/test/integration_test.cc
@@ -379,7 +379,8 @@ TEST_P(IntegrationTest, MultipleRequestsWithSameStreamId) {
   cleanup();
 }
 
-TEST_P(IntegrationTest, MultipleRequests) {
+// https://github.com/envoyproxy/envoy/issues/27842
+TEST_P(IntegrationTest, DISABLED_MultipleRequests) {
   FakeStreamCodecFactoryConfig codec_factory_config;
   codec_factory_config.protocol_options_ = ProtocolOptions{true};
   Registry::InjectFactory registration(codec_factory_config);

From a184a9e72ff2e491ceed3f2c36569b7fc6705eaa Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Wed, 7 Jun 2023 20:09:47 -0400
Subject: [PATCH 487/740] lb: adding subset lb tests (#27825)

Signed-off-by: Alyssa Wilk 
---
 .../subset/subset_lb.h                        |  1 -
 test/common/upstream/subset_lb_test.cc        | 41 ++++++++++++++++++-
 test/per_file_coverage.sh                     |  4 +-
 3 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.h b/source/extensions/load_balancing_policies/subset/subset_lb.h
index 0072dcbe8006..8d1c0523b464 100644
--- a/source/extensions/load_balancing_policies/subset/subset_lb.h
+++ b/source/extensions/load_balancing_policies/subset/subset_lb.h
@@ -239,7 +239,6 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable(host_sets_[priority].get())
           ->update(matching_hosts, hosts_added, hosts_removed);
-
       runUpdateCallbacks(hosts_added, hosts_removed);
     }
 
diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc
index 0902264c537c..aa2b44d3eb42 100644
--- a/test/common/upstream/subset_lb_test.cc
+++ b/test/common/upstream/subset_lb_test.cc
@@ -68,12 +68,12 @@ class SubsetLoadBalancerInternalStateTester {
               legacy_child_lb_creator->lbLeastRequestConfig() != absl::nullopt);
   }
 
+  static void testWrapper();
+
 private:
   std::shared_ptr lb_;
 };
 
-namespace SubsetLoadBalancerTest {
-
 class TestMetadataMatchCriterion : public Router::MetadataMatchCriterion {
 public:
   TestMetadataMatchCriterion(const std::string& name, const HashedValue& value)
@@ -161,6 +161,38 @@ class TestMetadataMatchCriteria : public Router::MetadataMatchCriteria {
   std::vector matches_;
 };
 
+void SubsetLoadBalancerInternalStateTester::testWrapper() {
+  MockLoadBalancerContext inner;
+  TestMetadataMatchCriteria test_criteria(std::map{});
+
+  EXPECT_CALL(inner, metadataMatchCriteria)
+      .Times(testing::AnyNumber())
+      .WillRepeatedly(Return(&test_criteria));
+  const std::set filtered_metadata_match_criteria_names;
+  SubsetLoadBalancer::LoadBalancerContextWrapper wrapper(&inner,
+                                                         filtered_metadata_match_criteria_names);
+
+  EXPECT_CALL(inner, computeHashKey());
+  wrapper.computeHashKey();
+
+  EXPECT_CALL(inner, downstreamConnection());
+  wrapper.downstreamConnection();
+
+  EXPECT_CALL(inner, downstreamHeaders());
+  wrapper.downstreamHeaders();
+
+  EXPECT_CALL(inner, upstreamSocketOptions());
+  wrapper.upstreamSocketOptions();
+
+  EXPECT_CALL(inner, upstreamTransportSocketOptions());
+  wrapper.upstreamTransportSocketOptions();
+
+  EXPECT_CALL(inner, overrideHostToSelect());
+  wrapper.overrideHostToSelect();
+}
+
+namespace SubsetLoadBalancerTest {
+
 class TestLoadBalancerContext : public LoadBalancerContextBase {
 public:
   TestLoadBalancerContext(
@@ -2860,6 +2892,11 @@ TEST_P(SubsetLoadBalancerSingleHostPerSubsetTest, Update) {
   EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_b)); // fallback
 }
 
+TEST(LoadBalancerContextWrapper, LoadBalancerContextWrapperTest) {
+  // Test private helper class via friend class.
+  SubsetLoadBalancerInternalStateTester::testWrapper();
+}
+
 INSTANTIATE_TEST_SUITE_P(UpdateOrderings, SubsetLoadBalancerSingleHostPerSubsetTest,
                          testing::ValuesIn({UpdateOrder::RemovesFirst, UpdateOrder::Simultaneous}));
 
diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh
index c96fc8feb579..6d350e692d40 100755
--- a/test/per_file_coverage.sh
+++ b/test/per_file_coverage.sh
@@ -76,8 +76,8 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/health_checkers:95.9"
 "source/extensions/health_checkers/http:93.8"
 "source/extensions/health_checkers/grpc:92.0"
-"source/extensions/load_balancing_policies:95.5"
-"source/extensions/load_balancing_policies/subset:94.3"
+"source/extensions/load_balancing_policies:96.4"
+"source/extensions/load_balancing_policies/subset:96.0"
 "source/extensions/config_subscription/rest:94.3"
 "source/extensions/matching/input_matchers/cel_matcher:90.7" #Death tests don't report LCOV
 )

From 72c2b962244585cb782161a732eea46f88cc6838 Mon Sep 17 00:00:00 2001
From: yanavlasov 
Date: Wed, 7 Jun 2023 22:50:31 -0400
Subject: [PATCH 488/740] Decommission
 http_strip_fragment_from_path_unsafe_if_disabled (#27847)

Signed-off-by: Yan Avlasov 
---
 changelogs/current.yaml                       |  3 ++
 source/common/http/conn_manager_utility.cc    |  6 +---
 source/common/runtime/runtime_features.cc     |  1 -
 test/common/http/conn_manager_utility_test.cc | 34 -------------------
 4 files changed, 4 insertions(+), 40 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 841fea503761..7fbafbe3002e 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -189,6 +189,9 @@ removed_config_or_runtime:
 - area: config
   change: |
     removed runtime key ``envoy.reloadable_features.delta_xds_subscription_state_tracking_fix`` and legacy code paths.
+- area: http
+  change: |
+    removed runtime key ``envoy.reloadable_features.http_strip_fragment_from_path_unsafe_if_disabled`` and legacy code paths.
 
 new_features:
 - area: access_log
diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc
index 39805b8de253..6d32df007dd5 100644
--- a/source/common/http/conn_manager_utility.cc
+++ b/source/common/http/conn_manager_utility.cc
@@ -564,11 +564,7 @@ ConnectionManagerUtility::maybeNormalizePath(RequestHeaderMap& request_headers,
       return NormalizePathAction::Reject;
     }
     // Check runtime override and throw away fragment from URI path
-    // TODO(yanavlasov): remove this override after deprecation period.
-    if (Runtime::runtimeFeatureEnabled(
-            "envoy.reloadable_features.http_strip_fragment_from_path_unsafe_if_disabled")) {
-      request_headers.setPath(request_headers.getPathValue().substr(0, fragment_pos));
-    }
+    request_headers.setPath(request_headers.getPathValue().substr(0, fragment_pos));
   }
 
   NormalizePathAction final_action = NormalizePathAction::Continue;
diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc
index c7ce0d53bc49..750b702ec48e 100644
--- a/source/common/runtime/runtime_features.cc
+++ b/source/common/runtime/runtime_features.cc
@@ -50,7 +50,6 @@ RUNTIME_GUARD(envoy_reloadable_features_http_allow_partial_urls_in_referer);
 RUNTIME_GUARD(envoy_reloadable_features_http_ext_auth_failure_mode_allow_header_add);
 RUNTIME_GUARD(envoy_reloadable_features_http_filter_avoid_reentrant_local_reply);
 RUNTIME_GUARD(envoy_reloadable_features_http_reject_path_with_fragment);
-RUNTIME_GUARD(envoy_reloadable_features_http_strip_fragment_from_path_unsafe_if_disabled);
 RUNTIME_GUARD(envoy_reloadable_features_ignore_optional_option_from_hcm_for_route_config);
 RUNTIME_GUARD(envoy_reloadable_features_initialize_upstream_filters);
 RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name);
diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc
index 9d6aadb157be..6633650dd204 100644
--- a/test/common/http/conn_manager_utility_test.cc
+++ b/test/common/http/conn_manager_utility_test.cc
@@ -2160,40 +2160,6 @@ TEST_F(ConnectionManagerUtilityTest, DropFragmentFromPathWithOverride) {
   EXPECT_EQ(header_map_with_fragment2.getPathValue(), "/baz/");
 }
 
-TEST_F(ConnectionManagerUtilityTest, KeepFragmentFromPathWithBothOverrides) {
-  TestScopedRuntime scoped_runtime;
-  scoped_runtime.mergeValues(
-      {{"envoy.reloadable_features.http_reject_path_with_fragment", "false"}});
-  scoped_runtime.mergeValues(
-      {{"envoy.reloadable_features.http_strip_fragment_from_path_unsafe_if_disabled", "false"}});
-
-  TestRequestHeaderMapImpl header_map{{":path", "/foo/bar#boom"}};
-  EXPECT_EQ(ConnectionManagerUtility::NormalizePathAction::Continue,
-            ConnectionManagerUtility::maybeNormalizePath(header_map, config_));
-  EXPECT_EQ(header_map.getPathValue(), "/foo/bar#boom");
-
-  TestRequestHeaderMapImpl header_map_just_fragment{{":path", "#"}};
-  EXPECT_EQ(ConnectionManagerUtility::NormalizePathAction::Continue,
-            ConnectionManagerUtility::maybeNormalizePath(header_map_just_fragment, config_));
-  EXPECT_EQ(header_map_just_fragment.getPathValue(), "#");
-
-  TestRequestHeaderMapImpl header_map_just_fragment2{{":path", "/#"}};
-  EXPECT_EQ(ConnectionManagerUtility::NormalizePathAction::Continue,
-            ConnectionManagerUtility::maybeNormalizePath(header_map_just_fragment2, config_));
-  EXPECT_EQ(header_map_just_fragment2.getPathValue(), "/#");
-
-  TestRequestHeaderMapImpl header_map_with_empty_fragment{{":path", "/foo/baz/#"}};
-  EXPECT_EQ(ConnectionManagerUtility::NormalizePathAction::Continue,
-            ConnectionManagerUtility::maybeNormalizePath(header_map_with_empty_fragment, config_));
-  EXPECT_EQ(header_map_with_empty_fragment.getPathValue(), "/foo/baz/#");
-
-  ON_CALL(config_, shouldNormalizePath()).WillByDefault(Return(true));
-  TestRequestHeaderMapImpl header_map_with_fragment2{{":path", "/foo/../baz/#fragment"}};
-  EXPECT_EQ(ConnectionManagerUtility::NormalizePathAction::Continue,
-            ConnectionManagerUtility::maybeNormalizePath(header_map_with_fragment2, config_));
-  EXPECT_EQ(header_map_with_fragment2.getPathValue(), "/baz/%23fragment");
-}
-
 // Verify when append_x_forwarded_port is turned on, the x-forwarded-port header should be appended.
 TEST_F(ConnectionManagerUtilityTest, AppendXForwardedPort) {
   ON_CALL(config_, appendXForwardedPort()).WillByDefault(Return(true));

From aa89bdf9f8f23c9a03c5c536949c93bfe9cb04d1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 8 Jun 2023 09:39:16 +0100
Subject: [PATCH 489/740] build(deps): bump python from 3.11.3-slim-bullseye to
 3.11.4-slim-bullseye in /examples/shared/python (#27867)

build(deps): bump python in /examples/shared/python

Bumps python from 3.11.3-slim-bullseye to 3.11.4-slim-bullseye.

---
updated-dependencies:
- dependency-name: python
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/python/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/python/Dockerfile b/examples/shared/python/Dockerfile
index d64811d72d2d..e152f2165b9d 100644
--- a/examples/shared/python/Dockerfile
+++ b/examples/shared/python/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11.3-slim-bullseye@sha256:eaee5f73efa9ae962d2077756292bc4878c04fcbc13dc168bb00cc365f35647e as python-base
+FROM python:3.11.4-slim-bullseye@sha256:1966141ab594e175852a033da2a38f0cb042b5b92896c22073f8477f96f43b06 as python-base
 RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache
 ARG PYTHON_REQUIREMENTS_FILE=aiohttp/requirements.txt

From 9b3d08985d70b7ff41812ab48816b062e4f1dbcc Mon Sep 17 00:00:00 2001
From: Paul Sohn 
Date: Thu, 8 Jun 2023 08:09:17 -0400
Subject: [PATCH 490/740] Fix flaky test
 QuicHttpIntegrationTest::DeferredLoggingWithRetransmission (#27850)

Fixes #27841:

Increase the sleep from 500->1000 ms.
Properly clean up the SocketInterfaceSwap.
Commit Message: Fix flaky test DeferredLoggingWithRetransmission
Risk Level: N/A, just testing

Signed-off-by: Paul Sohn 
---
 .../integration/quic_http_integration_test.cc | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc
index c11b91987849..41e7e6c92ce3 100644
--- a/test/integration/quic_http_integration_test.cc
+++ b/test/integration/quic_http_integration_test.cc
@@ -1392,16 +1392,16 @@ TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithRetransmission) {
 
   // Temporarily prevent server from writing packets (i.e. to respond to downstream)
   // to simulate packet loss and trigger retransmissions.
-  SocketInterfaceSwap socket_swap(upstreamProtocol() == Http::CodecType::HTTP3
-                                      ? Network::Socket::Type::Datagram
-                                      : Network::Socket::Type::Stream);
-  Network::IoSocketError* ebadf = Network::IoSocketError::getIoSocketEbadfInstance();
-  socket_swap.write_matcher_->setDestinationPort(lookupPort("http"));
-  socket_swap.write_matcher_->setWriteOverride(ebadf);
-  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true);
-  absl::SleepFor(absl::Milliseconds(500 * TSAN_TIMEOUT_FACTOR));
-  // Allow the response to be sent downstream again.
-  socket_swap.write_matcher_->setWriteOverride(nullptr);
+  {
+    SocketInterfaceSwap socket_swap(downstreamProtocol() == Http::CodecType::HTTP3
+                                        ? Network::Socket::Type::Datagram
+                                        : Network::Socket::Type::Stream);
+    Network::IoSocketError* ebadf = Network::IoSocketError::getIoSocketEbadfInstance();
+    socket_swap.write_matcher_->setDestinationPort(lookupPort("http"));
+    socket_swap.write_matcher_->setWriteOverride(ebadf);
+    upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true);
+    timeSystem().advanceTimeWait(std::chrono::seconds(TSAN_TIMEOUT_FACTOR));
+  }
 
   ASSERT_TRUE(response->waitForEndStream());
   codec_client_->close();

From a9571872dbf1c3c125ab0aaba086e5416e1048f9 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Thu, 8 Jun 2023 08:21:49 -0400
Subject: [PATCH 491/740] runtime: warning (ENVOY_BUG) on bad yaml (#27735)

Adding an ENVOY_BUG for badly formed runtime yaml. If no one complains, I'll runtime-guard remove support after the next release or two with the goal of giving it a year of runtime.

Also cleaning up tests with bad yaml or removed runtime guards which are now flagged by the cleaner code.

Risk Level: medium
Testing: updated tests
Docs Changes:
Release Notes:

Part of #27434

Signed-off-by: Alyssa Wilk 
---
 source/common/runtime/runtime_impl.cc         | 66 +++++++++++++------
 source/common/runtime/runtime_impl.h          |  5 +-
 test/common/runtime/runtime_impl_test.cc      | 30 +++++----
 .../runtime/test_data/root/envoy/file12       |  1 -
 test/config/utility.cc                        | 10 ++-
 .../proxy_filter_integration_test.cc          |  2 -
 .../ext_authz/ext_authz_integration_test.cc   | 11 +++-
 7 files changed, 86 insertions(+), 39 deletions(-)
 delete mode 100644 test/common/runtime/test_data/root/envoy/file12

diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc
index 20704683ac13..81bc35baaf29 100644
--- a/source/common/runtime/runtime_impl.cc
+++ b/source/common/runtime/runtime_impl.cc
@@ -265,39 +265,41 @@ void setNumberValue(Envoy::Runtime::Snapshot::Entry& entry, double value) {
 }
 
 // Handle corner cases in parsing: negatives and decimals aren't always parsed as doubles.
-void parseEntryDoubleValue(Envoy::Runtime::Snapshot::Entry& entry) {
+bool parseEntryDoubleValue(Envoy::Runtime::Snapshot::Entry& entry) {
   double converted_double;
   if (absl::SimpleAtod(entry.raw_string_value_, &converted_double)) {
     setNumberValue(entry, converted_double);
+    return true;
   }
+  return false;
 }
 
 // Handle an awful corner case where we explicitly shove a yaml percent in a proto string
 // value. Basically due to prior parsing logic we have to handle any combination
 // of numerator: #### [denominator Y] with quotes braces etc that could possibly be valid json.
 // E.g. "final_value": "{\"numerator\": 10000, \"denominator\": \"TEN_THOUSAND\"}",
-void parseEntryFractionalPercentValue(Envoy::Runtime::Snapshot::Entry& entry) {
+bool parseEntryFractionalPercentValue(Envoy::Runtime::Snapshot::Entry& entry) {
   if (!absl::StrContains(entry.raw_string_value_, "numerator")) {
-    return;
+    return false;
   }
 
   const re2::RE2 numerator_re(".*numerator[^\\d]+(\\d+)[^\\d]*");
 
   std::string match_string;
   if (!re2::RE2::FullMatch(entry.raw_string_value_.c_str(), numerator_re, &match_string)) {
-    return;
+    return false;
   }
 
   uint32_t numerator;
   if (!absl::SimpleAtoi(match_string, &numerator)) {
-    return;
+    return false;
   }
   envoy::type::v3::FractionalPercent converted_fractional_percent;
   converted_fractional_percent.set_numerator(numerator);
   entry.fractional_percent_value_ = converted_fractional_percent;
 
   if (!absl::StrContains(entry.raw_string_value_, "denominator")) {
-    return;
+    return true;
   }
   if (absl::StrContains(entry.raw_string_value_, "TEN_THOUSAND")) {
     entry.fractional_percent_value_->set_denominator(
@@ -306,22 +308,48 @@ void parseEntryFractionalPercentValue(Envoy::Runtime::Snapshot::Entry& entry) {
   if (absl::StrContains(entry.raw_string_value_, "MILLION")) {
     entry.fractional_percent_value_->set_denominator(envoy::type::v3::FractionalPercent::MILLION);
   }
+  return true;
 }
 
 // Handle corner cases in non-yaml parsing: mixed case strings aren't parsed as booleans.
-void parseEntryBooleanValue(Envoy::Runtime::Snapshot::Entry& entry) {
+bool parseEntryBooleanValue(Envoy::Runtime::Snapshot::Entry& entry) {
   absl::string_view stripped = entry.raw_string_value_;
   stripped = absl::StripAsciiWhitespace(stripped);
 
   if (absl::EqualsIgnoreCase(stripped, "true")) {
     entry.bool_value_ = true;
+    return true;
   } else if (absl::EqualsIgnoreCase(stripped, "false")) {
     entry.bool_value_ = false;
+    return true;
+  }
+  return false;
+}
+
+void SnapshotImpl::addEntry(Snapshot::EntryMap& values, const std::string& key,
+                            const ProtobufWkt::Value& value, absl::string_view raw_string) {
+  const char* error_message = nullptr;
+  values.emplace(key, SnapshotImpl::createEntry(value, raw_string, error_message));
+  if (error_message != nullptr) {
+    IS_ENVOY_BUG(
+        absl::StrCat(error_message, "\n[ key:", key, ", value: ", value.DebugString(), "]"));
   }
 }
 
+static const char* kBoolError =
+    "Runtime YAML appears to be setting booleans as strings. Support for this is planned "
+    "to removed in an upcoming release. If you can not fix your YAML and need this to continue "
+    "working "
+    "please ping on https://github.com/envoyproxy/envoy/issues/27434";
+static const char* kFractionError =
+    "Runtime YAML appears to be setting fractions as strings. Support for this is planned "
+    "to removed in an upcoming release. If you can not fix your YAML and need this to continue "
+    "working "
+    "please ping on https://github.com/envoyproxy/envoy/issues/27434";
+
 SnapshotImpl::Entry SnapshotImpl::createEntry(const ProtobufWkt::Value& value,
-                                              absl::string_view raw_string) {
+                                              absl::string_view raw_string,
+                                              const char*& error_message) {
   Entry entry;
   entry.raw_string_value_ = value.string_value();
   if (!raw_string.empty()) {
@@ -348,10 +376,12 @@ SnapshotImpl::Entry SnapshotImpl::createEntry(const ProtobufWkt::Value& value,
     break;
   case ProtobufWkt::Value::kStringValue:
     parseEntryDoubleValue(entry);
-    // TODO(alyssawilk) after this PR lands and sticks, ENVOY_BUG these
-    // functions and see if we can remove the special casing.
-    parseEntryBooleanValue(entry);
-    parseEntryFractionalPercentValue(entry);
+    if (parseEntryBooleanValue(entry)) {
+      error_message = kBoolError;
+    }
+    if (parseEntryFractionalPercentValue(entry)) {
+      error_message = kFractionError;
+    }
   default:
     break;
   }
@@ -364,8 +394,7 @@ void AdminLayer::mergeValues(const absl::node_hash_map
   for (const auto& kv : values) {
     values_.erase(kv.first);
     if (!kv.second.empty()) {
-      values_.emplace(kv.first,
-                      SnapshotImpl::createEntry(ValueUtil::loadFromYaml(kv.second), kv.second));
+      SnapshotImpl::addEntry(values_, kv.first, ValueUtil::loadFromYaml(kv.second), kv.second);
     }
   }
   stats_.admin_overrides_active_.set(values_.empty() ? 0 : 1);
@@ -435,8 +464,7 @@ void DiskLayer::walkDirectory(const std::string& path, const std::string& prefix
       // the use of the [] operator. Can leverage insert_or_assign in C++17 in the future.
       values_.erase(full_prefix);
 #ifdef ENVOY_ENABLE_YAML
-      values_.insert(
-          {full_prefix, SnapshotImpl::createEntry(ValueUtil::loadFromYaml(value), value)});
+      SnapshotImpl::addEntry(values_, full_prefix, ValueUtil::loadFromYaml(value), value);
 #else
       IS_ENVOY_BUG("Runtime admin reload requires YAML support");
       UNREFERENCED_PARAMETER(value);
@@ -461,7 +489,7 @@ void ProtoLayer::walkProtoValue(const ProtobufWkt::Value& v, const std::string&
     throw EnvoyException(absl::StrCat("Invalid runtime entry value for ", prefix));
     break;
   case ProtobufWkt::Value::kStringValue:
-    values_.emplace(prefix, SnapshotImpl::createEntry(v));
+    SnapshotImpl::addEntry(values_, prefix, v, "");
     break;
   case ProtobufWkt::Value::kNumberValue:
   case ProtobufWkt::Value::kBoolValue:
@@ -470,13 +498,13 @@ void ProtoLayer::walkProtoValue(const ProtobufWkt::Value& v, const std::string&
           "Using a removed guard ", prefix,
           ". In future version of Envoy this will be treated as invalid configuration"));
     }
-    values_.emplace(prefix, SnapshotImpl::createEntry(v));
+    SnapshotImpl::addEntry(values_, prefix, v, "");
     break;
   case ProtobufWkt::Value::kStructValue: {
     const ProtobufWkt::Struct& s = v.struct_value();
     if (s.fields().empty() || s.fields().find("numerator") != s.fields().end() ||
         s.fields().find("denominator") != s.fields().end()) {
-      values_.emplace(prefix, SnapshotImpl::createEntry(v));
+      SnapshotImpl::addEntry(values_, prefix, v, "");
       break;
     }
     for (const auto& f : s.fields()) {
diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h
index f8e98ec72aec..393228f3323c 100644
--- a/source/common/runtime/runtime_impl.h
+++ b/source/common/runtime/runtime_impl.h
@@ -87,7 +87,10 @@ class SnapshotImpl : public Snapshot, Logger::Loggable {
 
   const EntryMap& values() const;
 
-  static Entry createEntry(const ProtobufWkt::Value& value, absl::string_view raw_string = "");
+  static Entry createEntry(const ProtobufWkt::Value& value, absl::string_view raw_string,
+                           const char*& error_message);
+  static void addEntry(Snapshot::EntryMap& values, const std::string& key,
+                       const ProtobufWkt::Value& value, absl::string_view raw_string = "");
 
 private:
   const std::vector layers_;
diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc
index e4413c09413d..4992e9fb102c 100644
--- a/test/common/runtime/runtime_impl_test.cc
+++ b/test/common/runtime/runtime_impl_test.cc
@@ -180,9 +180,6 @@ TEST_F(DiskLoaderImplTest, All) {
   // Lower-case boolean specification.
   EXPECT_EQ(true, snapshot->getBoolean("file11", false));
   EXPECT_EQ(true, snapshot->getBoolean("file11", true));
-  // Mixed-case boolean specification.
-  EXPECT_EQ(false, snapshot->getBoolean("file12", true));
-  EXPECT_EQ(false, snapshot->getBoolean("file12", false));
   // Lower-case boolean specification with leading whitespace.
   EXPECT_EQ(true, snapshot->getBoolean("file13", true));
   EXPECT_EQ(true, snapshot->getBoolean("file13", false));
@@ -262,7 +259,7 @@ TEST_F(DiskLoaderImplTest, All) {
 
   EXPECT_EQ(0, store_.counter("runtime.load_error").value());
   EXPECT_EQ(1, store_.counter("runtime.load_success").value());
-  EXPECT_EQ(25, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value());
+  EXPECT_EQ(24, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value());
   EXPECT_EQ(4, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value());
 }
 
@@ -604,7 +601,6 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) {
       denominator: NONSENSE
     file10: 52
     file11: true
-    file12: FaLSe
     file13: false
     subdir:
       file: "hello"
@@ -648,9 +644,6 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) {
   EXPECT_EQ(true, snapshot->getBoolean("file11", true));
   EXPECT_EQ(true, snapshot->getBoolean("file11", false));
 
-  EXPECT_EQ(false, snapshot->getBoolean("file12", true));
-  EXPECT_EQ(false, snapshot->getBoolean("file12", false));
-
   EXPECT_EQ(false, snapshot->getBoolean("file13", true));
   EXPECT_EQ(false, snapshot->getBoolean("file13", false));
 
@@ -733,19 +726,20 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) {
 
   EXPECT_EQ(0, store_.counter("runtime.load_error").value());
   EXPECT_EQ(1, store_.counter("runtime.load_success").value());
-  EXPECT_EQ(23, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value());
+  EXPECT_EQ(22, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value());
   EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value());
 
+  const char* error = nullptr;
   // While null values are generally filtered out by walkProtoValue, test manually.
   ProtobufWkt::Value empty_value;
   const_cast(dynamic_cast(loader_->snapshot()))
-      .createEntry(empty_value);
+      .createEntry(empty_value, "", error);
 
   // Make sure the hacky fractional percent function works.
   ProtobufWkt::Value fractional_value;
   fractional_value.set_string_value(" numerator:  11 ");
   auto entry = const_cast(dynamic_cast(loader_->snapshot()))
-                   .createEntry(fractional_value);
+                   .createEntry(fractional_value, "", error);
   ASSERT_TRUE(entry.fractional_percent_value_.has_value());
   EXPECT_EQ(entry.fractional_percent_value_->denominator(),
             envoy::type::v3::FractionalPercent::HUNDRED);
@@ -754,7 +748,7 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) {
   // Make sure the hacky percent function works with numerator and denominator
   fractional_value.set_string_value("{\"numerator\": 10000, \"denominator\": \"TEN_THOUSAND\"}");
   entry = const_cast(dynamic_cast(loader_->snapshot()))
-              .createEntry(fractional_value);
+              .createEntry(fractional_value, "", error);
   ASSERT_TRUE(entry.fractional_percent_value_.has_value());
   EXPECT_EQ(entry.fractional_percent_value_->denominator(),
             envoy::type::v3::FractionalPercent::TEN_THOUSAND);
@@ -763,7 +757,7 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) {
   // Make sure the hacky fractional percent function works with million
   fractional_value.set_string_value("{\"numerator\": 10000, \"denominator\": \"MILLION\"}");
   entry = const_cast(dynamic_cast(loader_->snapshot()))
-              .createEntry(fractional_value);
+              .createEntry(fractional_value, "", error);
   ASSERT_TRUE(entry.fractional_percent_value_.has_value());
   EXPECT_EQ(entry.fractional_percent_value_->denominator(),
             envoy::type::v3::FractionalPercent::MILLION);
@@ -772,8 +766,16 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) {
   // Test atoi failure for the hacky fractional percent value function.
   fractional_value.set_string_value(" numerator:  1.1 ");
   entry = const_cast(dynamic_cast(loader_->snapshot()))
-              .createEntry(fractional_value);
+              .createEntry(fractional_value, "", error);
   ASSERT_FALSE(entry.fractional_percent_value_.has_value());
+
+  // Test legacy malformed boolean support
+  ProtobufWkt::Value boolean_value;
+  boolean_value.set_string_value("FaLsE");
+  entry = const_cast(dynamic_cast(loader_->snapshot()))
+              .createEntry(boolean_value, "", error);
+  ASSERT_TRUE(entry.bool_value_.has_value());
+  ASSERT_FALSE(entry.bool_value_.value());
 }
 
 TEST_F(StaticLoaderImplTest, InvalidNumerator) {
diff --git a/test/common/runtime/test_data/root/envoy/file12 b/test/common/runtime/test_data/root/envoy/file12
deleted file mode 100644
index 11bdefe7b29b..000000000000
--- a/test/common/runtime/test_data/root/envoy/file12
+++ /dev/null
@@ -1 +0,0 @@
-FaLSe
diff --git a/test/config/utility.cc b/test/config/utility.cc
index f55bcbf0b362..21781be3852e 100644
--- a/test/config/utility.cc
+++ b/test/config/utility.cc
@@ -928,7 +928,15 @@ void ConfigHelper::configureUpstreamTls(
 void ConfigHelper::addRuntimeOverride(absl::string_view key, absl::string_view value) {
   auto* static_layer =
       bootstrap_.mutable_layered_runtime()->mutable_layers(0)->mutable_static_layer();
-  (*static_layer->mutable_fields())[std::string(key)] = ValueUtil::stringValue(std::string(value));
+
+  if (value == "true") {
+    (*static_layer->mutable_fields())[std::string(key)] = ValueUtil::boolValue(true);
+  } else if (value == "false") {
+    (*static_layer->mutable_fields())[std::string(key)] = ValueUtil::boolValue(false);
+  } else {
+    (*static_layer->mutable_fields())[std::string(key)] =
+        ValueUtil::stringValue(std::string(value));
+  }
 }
 
 void ConfigHelper::setProtocolOptions(envoy::config::cluster::v3::Cluster& cluster,
diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc
index 093e2ef2b782..d4208cf070da 100644
--- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc
+++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc
@@ -639,8 +639,6 @@ TEST_P(ProxyFilterIntegrationTest, UseCacheFileAndTestHappyEyeballs) {
   upstream_tls_ = false; // upstream creation doesn't handle autonomous_upstream_
   autonomous_upstream_ = true;
 
-  config_helper_.addRuntimeOverride("envoy.reloadable_features.allow_multiple_dns_addresses",
-                                    "true");
   use_cache_file_ = true;
   // Prepend a bad address
   if (GetParam() == Network::Address::IpVersion::v4) {
diff --git a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc
index 3cd67502d74b..2fac3262c22a 100644
--- a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc
+++ b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc
@@ -87,7 +87,16 @@ class ExtAuthzGrpcIntegrationTest : public Grpc::GrpcClientIntegrationParamTest,
 
   void setDenyAtDisableRuntimeConfig(bool deny_at_disable, bool disable_with_metadata) {
     if (!disable_with_metadata) {
-      config_helper_.addRuntimeOverride("envoy.ext_authz.enable", "numerator: 0");
+      config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+        auto* layer = bootstrap.mutable_layered_runtime()->add_layers();
+        layer->set_name("enable layer");
+        ProtobufWkt::Struct& runtime = *layer->mutable_static_layer();
+        bootstrap.mutable_layered_runtime()->mutable_layers(0)->set_name("base layer");
+
+        ProtobufWkt::Struct& enable =
+            *(*runtime.mutable_fields())["envoy.ext_authz.enable"].mutable_struct_value();
+        (*enable.mutable_fields())["numerator"].set_number_value(0);
+      });
     }
     if (deny_at_disable) {
       config_helper_.addRuntimeOverride("envoy.ext_authz.deny_at_disable", "true");

From c5b204ce93f2557c9f1e5868ace160c5f703f89e Mon Sep 17 00:00:00 2001
From: Yousuk Seung 
Date: Thu, 8 Jun 2023 07:04:21 -0700
Subject: [PATCH 492/740] api: use application_utilization in WRR (#27807)

* api: use application_utilization in WRR

Signed-off-by: Yousuk Seung 

* Comment addressed

Signed-off-by: Yousuk Seung 

---------

Signed-off-by: Yousuk Seung 
---
 .../v3/client_side_weighted_round_robin.proto   | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.proto b/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.proto
index ae4a99b4517a..c70360a0946b 100644
--- a/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.proto
+++ b/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.proto
@@ -22,14 +22,15 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // This policy differs from the built-in ROUND_ROBIN policy in terms of
 // how the endpoint weights are determined. In the ROUND_ROBIN policy,
 // the endpoint weights are sent by the control plane via EDS. However,
-// in this policy, the endpoint weights are instead determined via
-// qps (queries per second), eps (errors per second), and CPU utilization
-// metrics sent by the endpoint using the Open Request Cost Aggregation (ORCA)
-// protocol. A query counts towards qps when successful, otherwise towards both
-// qps and eps. What counts as an error is up to the endpoint to define.
-// A config parameter error_utilization_penalty controls the penalty to adjust
-// endpoint weights using eps and qps. The weight of a given endpoint is
-// computed as: qps / (cpu_utilization + eps/qps * error_utilization_penalty)
+// in this policy, the endpoint weights are instead determined via qps (queries
+// per second), eps (errors per second), and utilization metrics sent by the
+// endpoint using the Open Request Cost Aggregation (ORCA) protocol. Utilization
+// is determined by using the ORCA application_utilization field, if set, or
+// else falling back to the cpu_utilization field. All queries count toward qps,
+// regardless of result. Only failed queries count toward eps. A config
+// parameter error_utilization_penalty controls the penalty to adjust endpoint
+// weights using eps and qps. The weight of a given endpoint is computed as:
+//   qps / (utilization + eps/qps * error_utilization_penalty)
 //
 // See the :ref:`load balancing architecture overview` for more information.
 //

From b6bf6fc63359bf551aa2e5fb15553475a9938e3b Mon Sep 17 00:00:00 2001
From: doujiang24 
Date: Thu, 8 Jun 2023 22:27:53 +0800
Subject: [PATCH 493/740] golang filter: parse plugin config in the golang side
 while initializing golang filter (#27821)

Signed-off-by: doujiang24 
---
 contrib/exe/BUILD                             |  1 +
 contrib/golang/common/dso/dso.h               | 57 ++++++++---
 contrib/golang/common/dso/test/dso_test.cc    | 21 ++--
 contrib/golang/filters/http/source/config.cc  |  7 +-
 .../filters/http/source/go/pkg/api/filter.go  |  3 -
 .../http/source/go/pkg/http/capi_impl.go      |  2 +-
 .../filters/http/source/go/pkg/http/config.go |  2 -
 .../filters/http/source/golang_filter.cc      | 96 ++++++++++++-------
 .../filters/http/source/golang_filter.h       | 13 ++-
 contrib/golang/filters/http/test/BUILD        |  1 +
 .../filters/http/test/golang_filter_test.cc   | 38 +++++---
 .../http/test/test_data/passthrough/filter.go |  2 +-
 .../http/test/test_data/routeconfig/config.go | 11 ++-
 .../source/golang_cluster_specifier.cc        | 10 +-
 test/config_test/BUILD                        |  5 +-
 15 files changed, 165 insertions(+), 104 deletions(-)

diff --git a/contrib/exe/BUILD b/contrib/exe/BUILD
index 4391b745912e..f6029dccc224 100644
--- a/contrib/exe/BUILD
+++ b/contrib/exe/BUILD
@@ -37,6 +37,7 @@ envoy_cc_test(
     env = {
         "EXAMPLE_CONFIGS_TAR_PATH": "envoy/configs/example_contrib_configs.tar",
         "DISABLE_TEST_MERGE": "true",
+        "GODEBUG": "cgocheck=0",
     },
     deps = [
         "//test/config_test:example_configs_test_lib",
diff --git a/contrib/golang/common/dso/dso.h b/contrib/golang/common/dso/dso.h
index 36dbedffbc79..4f3cba97688e 100644
--- a/contrib/golang/common/dso/dso.h
+++ b/contrib/golang/common/dso/dso.h
@@ -98,6 +98,11 @@ class ClusterSpecifierDsoImpl : public ClusterSpecifierDso {
 using HttpFilterDsoPtr = std::shared_ptr;
 using ClusterSpecifierDsoPtr = std::shared_ptr;
 
+/*
+ * We do not unload a dynamic library once it is loaded. This is because
+ * Go shared library could not be unload by dlclose yet, see:
+ * https://github.com/golang/go/issues/11100
+ */
 template  class DsoManager {
 
 public:
@@ -105,33 +110,54 @@ template  class DsoManager {
    * Load the go plugin dynamic library.
    * @param dso_id is unique ID for dynamic library.
    * @param dso_name used to specify the absolute path of the dynamic library.
-   * @return false if load are invalid. Otherwise, return true.
+   * @param plugin_name used to specify the unique plugin name.
+   * @return nullptr if load are invalid.
    */
-  static bool load(std::string dso_id, std::string dso_name) {
-    ENVOY_LOG_MISC(debug, "load {} {} dso instance.", dso_id, dso_name);
-    if (getDsoByID(dso_id) != nullptr) {
-      return true;
+  static std::shared_ptr load(std::string dso_id, std::string dso_name,
+                                 std::string plugin_name) {
+    auto dso = load(dso_id, dso_name);
+    if (dso != nullptr) {
+      DsoStoreType& dsoStore = getDsoStore();
+      absl::WriterMutexLock lock(&dsoStore.mutex_);
+      dsoStore.plugin_name_to_dso_[plugin_name] = dso;
     }
+    return dso;
+  };
+
+  /**
+   * Load the go plugin dynamic library.
+   * @param dso_id is unique ID for dynamic library.
+   * @param dso_name used to specify the absolute path of the dynamic library.
+   * @return nullptr if load are invalid.
+   */
+  static std::shared_ptr load(std::string dso_id, std::string dso_name) {
+    ENVOY_LOG_MISC(debug, "load {} {} dso instance.", dso_id, dso_name);
+
     DsoStoreType& dsoStore = getDsoStore();
     absl::WriterMutexLock lock(&dsoStore.mutex_);
+    auto it = dsoStore.id_to_dso_.find(dso_id);
+    if (it != dsoStore.id_to_dso_.end()) {
+      return it->second;
+    }
+
     auto dso = std::make_shared(dso_name);
     if (!dso->loaded()) {
-      return false;
+      return nullptr;
     }
-    dsoStore.map_[dso_id] = std::move(dso);
-    return true;
+    dsoStore.id_to_dso_[dso_id] = dso;
+    return dso;
   };
 
   /**
-   * Get the go plugin dynamic library.
-   * @param dso_id is unique ID for dynamic library.
+   * Get the go plugin dynamic library by plugin name.
+   * @param plugin_name is unique ID for a plugin, one DSO may contains multiple plugins.
    * @return nullptr if get failed. Otherwise, return the DSO instance.
    */
-  static std::shared_ptr getDsoByID(std::string dso_id) {
+  static std::shared_ptr getDsoByPluginName(std::string plugin_name) {
     DsoStoreType& dsoStore = getDsoStore();
     absl::ReaderMutexLock lock(&dsoStore.mutex_);
-    auto it = dsoStore.map_.find(dso_id);
-    if (it != dsoStore.map_.end()) {
+    auto it = dsoStore.plugin_name_to_dso_.find(plugin_name);
+    if (it != dsoStore.plugin_name_to_dso_.end()) {
       return it->second;
     }
     return nullptr;
@@ -140,7 +166,10 @@ template  class DsoManager {
 private:
   using DsoMapType = absl::flat_hash_map>;
   struct DsoStoreType {
-    DsoMapType map_ ABSL_GUARDED_BY(mutex_){{
+    DsoMapType id_to_dso_ ABSL_GUARDED_BY(mutex_){{
+        {"", nullptr},
+    }};
+    DsoMapType plugin_name_to_dso_ ABSL_GUARDED_BY(mutex_){{
         {"", nullptr},
     }};
     absl::Mutex mutex_;
diff --git a/contrib/golang/common/dso/test/dso_test.cc b/contrib/golang/common/dso/test/dso_test.cc
index a24d0cdd4d22..a3d0fd807497 100644
--- a/contrib/golang/common/dso/test/dso_test.cc
+++ b/contrib/golang/common/dso/test/dso_test.cc
@@ -26,32 +26,29 @@ TEST(DsoInstanceTest, SimpleAPI) {
 
 TEST(DsoManagerTest, Pub) {
   auto id = "simple.so";
+  auto plugin_name = "example";
   auto path = genSoPath(id);
 
   // get before load http filter dso
-  auto dso = DsoManager::getDsoByID(id);
+  auto dso = DsoManager::getDsoByPluginName(plugin_name);
   EXPECT_EQ(dso, nullptr);
 
   // first time load http filter dso
-  auto res = DsoManager::load(id, path);
-  EXPECT_EQ(res, true);
+  dso = DsoManager::load(id, path, plugin_name);
+  EXPECT_NE(dso, nullptr);
 
   // get after load http filter dso
-  dso = DsoManager::getDsoByID(id);
+  dso = DsoManager::getDsoByPluginName(plugin_name);
   EXPECT_NE(dso, nullptr);
   EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 0), 100);
 
   // second time load http filter dso
-  res = DsoManager::load(id, path);
-  EXPECT_EQ(res, true);
+  dso = DsoManager::load(id, path, plugin_name);
+  EXPECT_NE(dso, nullptr);
 
   // first time load cluster specifier dso
-  res = DsoManager::load(id, path);
-  EXPECT_EQ(res, true);
-
-  // get after load cluster specifier dso
-  auto cluster_dso = DsoManager::getDsoByID(id);
-  EXPECT_NE(cluster_dso, nullptr);
+  auto cluster_dso = DsoManager::load(id, path);
+  EXPECT_NE(dso, nullptr);
 
   EXPECT_EQ(cluster_dso->envoyGoClusterSpecifierNewPlugin(0, 0), 200);
 }
diff --git a/contrib/golang/filters/http/source/config.cc b/contrib/golang/filters/http/source/config.cc
index 8944c0e45c2f..3b27e08beaa0 100644
--- a/contrib/golang/filters/http/source/config.cc
+++ b/contrib/golang/filters/http/source/config.cc
@@ -22,14 +22,13 @@ Http::FilterFactoryCb GolangFilterConfig::createFilterFactoryFromProtoTyped(
   // loads DSO store a static map and a open handles leak will occur when the filter gets loaded and
   // unloaded.
   // TODO: unload DSO when filter updated.
-  auto res = Dso::DsoManager::load(proto_config.library_id(),
-                                                           proto_config.library_path());
-  if (!res) {
+  auto dso_lib = Dso::DsoManager::load(
+      proto_config.library_id(), proto_config.library_path(), proto_config.plugin_name());
+  if (dso_lib == nullptr) {
     throw EnvoyException(fmt::format("golang_filter: load library failed: {} {}",
                                      proto_config.library_id(), proto_config.library_path()));
   }
 
-  auto dso_lib = Dso::DsoManager::getDsoByID(proto_config.library_id());
   FilterConfigSharedPtr config = std::make_shared(
       proto_config, dso_lib, fmt::format("{}golang.", stats_prefix), context);
 
diff --git a/contrib/golang/filters/http/source/go/pkg/api/filter.go b/contrib/golang/filters/http/source/go/pkg/api/filter.go
index 1d3973a2f525..88d658c8f1b0 100644
--- a/contrib/golang/filters/http/source/go/pkg/api/filter.go
+++ b/contrib/golang/filters/http/source/go/pkg/api/filter.go
@@ -24,10 +24,8 @@ type StreamDecoderFilter interface {
 	DecodeHeaders(RequestHeaderMap, bool) StatusType
 	DecodeData(BufferInstance, bool) StatusType
 	DecodeTrailers(RequestTrailerMap) StatusType
-	// TODO add more for metadata
 }
 
-// TODO merge it to StreamFilterConfigFactory
 type StreamFilterConfigParser interface {
 	Parse(any *anypb.Any) (interface{}, error)
 	Merge(parentConfig interface{}, childConfig interface{}) interface{}
@@ -51,7 +49,6 @@ type StreamEncoderFilter interface {
 	EncodeHeaders(ResponseHeaderMap, bool) StatusType
 	EncodeData(BufferInstance, bool) StatusType
 	EncodeTrailers(ResponseTrailerMap) StatusType
-	// TODO add more for metadata
 }
 
 // stream info
diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
index 3df76682028e..ca5993d98814 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
@@ -58,7 +58,7 @@ const (
 type httpCApiImpl struct{}
 
 // Only CAPIOK is expected, otherwise, it means unexpected stage when invoke C API,
-// panic here and it will be recover in the Go entry function (TODO).
+// panic here and it will be recover in the Go entry function.
 func handleCApiStatus(status C.CAPIStatus) {
 	switch status {
 	case C.CAPIOK:
diff --git a/contrib/golang/filters/http/source/go/pkg/http/config.go b/contrib/golang/filters/http/source/go/pkg/http/config.go
index b58b2dfd9f0d..eb3dd284f625 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/config.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/config.go
@@ -67,8 +67,6 @@ func envoyGoFilterNewHttpPluginConfig(namePtr, nameLen, configPtr, configLen uin
 		parsedConfig, err := configParser.Parse(&any)
 		if err != nil {
 			cAPI.HttpLog(api.Error, fmt.Sprintf("failed to parse golang plugin config: %v", err))
-			// TODO: we should reject the config in the Envoy side when Go returning 0.
-			// https://github.com/envoyproxy/envoy/issues/25369
 			return 0
 		}
 		configCache.Store(configNum, parsedConfig)
diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc
index 82aa29bdd97f..674a58209378 100644
--- a/contrib/golang/filters/http/source/golang_filter.cc
+++ b/contrib/golang/filters/http/source/golang_filter.cc
@@ -1123,7 +1123,7 @@ uint64_t Filter::getMergedConfigId(ProcessorState& state) {
   auto id = config_->getConfigId();
   for (auto it : route_config_list) {
     auto route_config = *it;
-    id = route_config.getPluginConfigId(id, config_->pluginName(), config_->soId());
+    id = route_config.getPluginConfigId(id, config_->pluginName());
   }
 
   return id;
@@ -1138,33 +1138,29 @@ FilterConfig::FilterConfig(
     : plugin_name_(proto_config.plugin_name()), so_id_(proto_config.library_id()),
       so_path_(proto_config.library_path()), plugin_config_(proto_config.plugin_config()),
       stats_(GolangFilterStats::generateStats(stats_prefix, context.scope())), dso_lib_(dso_lib) {
-  ENVOY_LOG(debug, "initilizing golang filter config");
-  // NP: dso may not loaded yet, can not invoke envoyGoFilterNewHttpPluginConfig yet.
-};
-
-uint64_t FilterConfig::getConfigId() {
-  if (config_id_ != 0) {
-    return config_id_;
-  }
+  ENVOY_LOG(debug, "initializing golang filter config");
 
   std::string buf;
   auto res = plugin_config_.SerializeToString(&buf);
-  ASSERT(res, "SerializeToString is always successful");
+  ASSERT(res, "SerializeToString should always successful");
   auto buf_ptr = reinterpret_cast(buf.data());
   auto name_ptr = reinterpret_cast(plugin_name_.data());
   config_id_ = dso_lib_->envoyGoFilterNewHttpPluginConfig(name_ptr, plugin_name_.length(), buf_ptr,
                                                           buf.length());
-  ASSERT(config_id_, "config id is always grows");
+  if (config_id_ == 0) {
+    throw EnvoyException(fmt::format("golang filter failed to parse plugin config: {} {}",
+                                     proto_config.library_id(), proto_config.library_path()));
+  }
   ENVOY_LOG(debug, "golang filter new plugin config, id: {}", config_id_);
+};
 
-  return config_id_;
-}
+uint64_t FilterConfig::getConfigId() { return config_id_; }
 
 FilterConfigPerRoute::FilterConfigPerRoute(
     const envoy::extensions::filters::http::golang::v3alpha::ConfigsPerRoute& config,
     Server::Configuration::ServerFactoryContext&) {
   // NP: dso may not loaded yet, can not invoke envoyGoFilterNewHttpPluginConfig yet.
-  ENVOY_LOG(debug, "initilizing per route golang filter config");
+  ENVOY_LOG(debug, "initializing per route golang filter config");
 
   for (const auto& it : config.plugins_config()) {
     auto plugin_name = it.first;
@@ -1176,40 +1172,70 @@ FilterConfigPerRoute::FilterConfigPerRoute(
   }
 }
 
-uint64_t FilterConfigPerRoute::getPluginConfigId(uint64_t parent_id, std::string plugin_name,
-                                                 std::string so_id) const {
+uint64_t FilterConfigPerRoute::getPluginConfigId(uint64_t parent_id,
+                                                 std::string plugin_name) const {
   auto it = plugins_config_.find(plugin_name);
   if (it != plugins_config_.end()) {
-    return it->second->getMergedConfigId(parent_id, so_id);
+    return it->second->getMergedConfigId(parent_id);
   }
   ENVOY_LOG(debug, "golang filter not found plugin config: {}", plugin_name);
   // not found
   return parent_id;
 }
 
-uint64_t RoutePluginConfig::getMergedConfigId(uint64_t parent_id, std::string so_id) {
+RoutePluginConfig::RoutePluginConfig(
+    const std::string plugin_name,
+    const envoy::extensions::filters::http::golang::v3alpha::RouterPlugin& config)
+    : plugin_name_(plugin_name), plugin_config_(config.config()) {
+
+  ENVOY_LOG(debug, "initializing golang filter route plugin config, plugin_name: {}, type_url: {}",
+            plugin_name_, config.config().type_url());
+
+  dso_lib_ = Dso::DsoManager::getDsoByPluginName(plugin_name_);
+  if (dso_lib_ == nullptr) {
+    // RoutePluginConfig may be created before FilterConfig, so dso_lib_ may be null.
+    // i.e. per route config is used in LDS route_config.
+    return;
+  }
+
+  config_id_ = getConfigId();
+  if (config_id_ == 0) {
+    throw EnvoyException(
+        fmt::format("golang filter failed to parse plugin config: {}", plugin_name_));
+  }
+  ENVOY_LOG(debug, "golang filter new per route '{}' plugin config, id: {}", plugin_name_,
+            config_id_);
+};
+
+uint64_t RoutePluginConfig::getConfigId() {
+  if (config_id_ > 0) {
+    return config_id_;
+  }
+  if (dso_lib_ == nullptr) {
+    dso_lib_ = Dso::DsoManager::getDsoByPluginName(plugin_name_);
+    ASSERT(dso_lib_ != nullptr, "load at the request time, so it should not be null");
+  }
+
+  std::string buf;
+  auto res = plugin_config_.SerializeToString(&buf);
+  ASSERT(res, "SerializeToString is always successful");
+  auto buf_ptr = reinterpret_cast(buf.data());
+  auto name_ptr = reinterpret_cast(plugin_name_.data());
+  return dso_lib_->envoyGoFilterNewHttpPluginConfig(name_ptr, plugin_name_.length(), buf_ptr,
+                                                    buf.length());
+};
+
+uint64_t RoutePluginConfig::getMergedConfigId(uint64_t parent_id) {
   if (merged_config_id_ > 0) {
     return merged_config_id_;
   }
 
-  auto name_ptr = reinterpret_cast(plugin_name_.data());
-  auto dlib = Dso::DsoManager::getDsoByID(so_id);
-  ASSERT(dlib != nullptr, "load at the config parse phase, so it should not be null");
+  config_id_ = getConfigId();
+  RELEASE_ASSERT(config_id_, "TODO: terminate request or passthrough");
 
-  if (config_id_ == 0) {
-    std::string buf;
-    auto res = plugin_config_.SerializeToString(&buf);
-    ASSERT(res, "SerializeToString is always successful");
-    auto buf_ptr = reinterpret_cast(buf.data());
-    config_id_ = dlib->envoyGoFilterNewHttpPluginConfig(name_ptr, plugin_name_.length(), buf_ptr,
-                                                        buf.length());
-    ASSERT(config_id_, "config id is always grows");
-    ENVOY_LOG(debug, "golang filter new per route '{}' plugin config, id: {}", plugin_name_,
-              config_id_);
-  }
-
-  merged_config_id_ = dlib->envoyGoFilterMergeHttpPluginConfig(name_ptr, plugin_name_.length(),
-                                                               parent_id, config_id_);
+  auto name_ptr = reinterpret_cast(plugin_name_.data());
+  merged_config_id_ = dso_lib_->envoyGoFilterMergeHttpPluginConfig(name_ptr, plugin_name_.length(),
+                                                                   parent_id, config_id_);
   ASSERT(merged_config_id_, "config id is always grows");
   ENVOY_LOG(debug, "golang filter merge '{}' plugin config, from {} + {} to {}", plugin_name_,
             parent_id, config_id_, merged_config_id_);
diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h
index 7d175df45ed2..23f9e6c8b908 100644
--- a/contrib/golang/filters/http/source/golang_filter.h
+++ b/contrib/golang/filters/http/source/golang_filter.h
@@ -56,18 +56,17 @@ using FilterConfigSharedPtr = std::shared_ptr;
 class RoutePluginConfig : Logger::Loggable {
 public:
   RoutePluginConfig(const std::string plugin_name,
-                    const envoy::extensions::filters::http::golang::v3alpha::RouterPlugin& config)
-      : plugin_name_(plugin_name), plugin_config_(config.config()) {
-    ENVOY_LOG(debug, "initilizing golang filter route plugin config, type_url: {}",
-              config.config().type_url());
-  };
+                    const envoy::extensions::filters::http::golang::v3alpha::RouterPlugin& config);
   // TODO: delete plugin config in Go
   ~RoutePluginConfig() = default;
-  uint64_t getMergedConfigId(uint64_t parent_id, std::string so_id);
+  uint64_t getConfigId();
+  uint64_t getMergedConfigId(uint64_t parent_id);
 
 private:
   const std::string plugin_name_;
   const ProtobufWkt::Any plugin_config_;
+
+  Dso::HttpFilterDsoPtr dso_lib_;
   uint64_t config_id_{0};
   uint64_t merged_config_id_{0};
 };
@@ -82,7 +81,7 @@ class FilterConfigPerRoute : public Router::RouteSpecificFilterConfig,
 public:
   FilterConfigPerRoute(const envoy::extensions::filters::http::golang::v3alpha::ConfigsPerRoute&,
                        Server::Configuration::ServerFactoryContext&);
-  uint64_t getPluginConfigId(uint64_t parent_id, std::string plugin_name, std::string so_id) const;
+  uint64_t getPluginConfigId(uint64_t parent_id, std::string plugin_name) const;
 
   ~FilterConfigPerRoute() override { plugins_config_.clear(); }
 
diff --git a/contrib/golang/filters/http/test/BUILD b/contrib/golang/filters/http/test/BUILD
index 6721341093b2..040024a78082 100644
--- a/contrib/golang/filters/http/test/BUILD
+++ b/contrib/golang/filters/http/test/BUILD
@@ -29,6 +29,7 @@ envoy_cc_test(
     srcs = ["golang_filter_test.cc"],
     data = [
         "//contrib/golang/filters/http/test/test_data/passthrough:filter.so",
+        "//contrib/golang/filters/http/test/test_data/routeconfig:filter.so",
     ],
     env = {"GODEBUG": "cgocheck=0"},
     deps = [
diff --git a/contrib/golang/filters/http/test/golang_filter_test.cc b/contrib/golang/filters/http/test/golang_filter_test.cc
index 4af781d110f1..bf21fb7513bc 100644
--- a/contrib/golang/filters/http/test/golang_filter_test.cc
+++ b/contrib/golang/filters/http/test/golang_filter_test.cc
@@ -73,7 +73,11 @@ class GolangHttpFilterTest : public testing::Test {
     EXPECT_CALL(decoder_callbacks_, streamInfo()).Times(testing::AnyNumber());
   }
 
-  ~GolangHttpFilterTest() override { filter_->onDestroy(); }
+  ~GolangHttpFilterTest() override {
+    if (filter_ != nullptr) {
+      filter_->onDestroy();
+    }
+  }
 
   void setup(const std::string& lib_id, const std::string& lib_path,
              const std::string& plugin_name) {
@@ -81,13 +85,13 @@ class GolangHttpFilterTest : public testing::Test {
     library_id: %s
     library_path: %s
     plugin_name: %s
-    merge_policy: MERGE_VIRTUALHOST_ROUTER_FILTER
     plugin_config:
-      "@type": type.googleapis.com/udpa.type.v1.TypedStruct
+      "@type": type.googleapis.com/xds.type.v3.TypedStruct
       type_url: typexx
       value:
           key: value
           int: 10
+          invalid: "invalid"
     )EOF";
 
     auto yaml_string = absl::StrFormat(yaml_fmt, lib_id, lib_path, plugin_name);
@@ -95,9 +99,9 @@ class GolangHttpFilterTest : public testing::Test {
     TestUtility::loadFromYaml(yaml_string, proto_config);
 
     envoy::extensions::filters::http::golang::v3alpha::ConfigsPerRoute per_route_proto_config;
-    setupDso();
-    setupConfig(proto_config, per_route_proto_config);
-    setupFilter(lib_id);
+    setupDso(lib_id, lib_path, plugin_name);
+    setupConfig(proto_config, per_route_proto_config, plugin_name);
+    setupFilter(plugin_name);
   }
 
   std::string genSoPath(std::string name) {
@@ -105,29 +109,29 @@ class GolangHttpFilterTest : public testing::Test {
         "{{ test_rundir }}/contrib/golang/filters/http/test/test_data/" + name + "/filter.so");
   }
 
-  void setupDso() {
-    Dso::DsoManager::load(PASSTHROUGH, genSoPath(PASSTHROUGH));
+  void setupDso(std::string id, std::string path, std::string plugin_name) {
+    Dso::DsoManager::load(id, path, plugin_name);
   }
 
   void setupConfig(
       envoy::extensions::filters::http::golang::v3alpha::Config& proto_config,
-      envoy::extensions::filters::http::golang::v3alpha::ConfigsPerRoute& per_route_proto_config) {
+      envoy::extensions::filters::http::golang::v3alpha::ConfigsPerRoute& per_route_proto_config,
+      std::string plugin_name) {
     // Setup filter config for Golang filter.
     config_ = std::make_shared(
-        proto_config,
-        Dso::DsoManager::getDsoByID(proto_config.library_id()), "",
+        proto_config, Dso::DsoManager::getDsoByPluginName(plugin_name), "",
         context_);
     // Setup per route config for Golang filter.
     per_route_config_ =
         std::make_shared(per_route_proto_config, server_factory_context_);
   }
 
-  void setupFilter(const std::string& so_id) {
+  void setupFilter(const std::string& plugin_name) {
     Event::SimulatedTimeSystem test_time;
     test_time.setSystemTime(std::chrono::microseconds(1583879145572237));
 
     filter_ = std::make_unique(
-        config_, Dso::DsoManager::getDsoByID(so_id));
+        config_, Dso::DsoManager::getDsoByPluginName(plugin_name));
     filter_->setDecoderFilterCallbacks(decoder_callbacks_);
     filter_->setEncoderFilterCallbacks(encoder_callbacks_);
   }
@@ -155,6 +159,7 @@ class GolangHttpFilterTest : public testing::Test {
   Stats::TestUtil::TestStore stats_store_;
 
   const std::string PASSTHROUGH{"passthrough"};
+  const std::string ROUTECONFIG{"routeconfig"};
 };
 
 // request that is headers only.
@@ -175,6 +180,13 @@ TEST_F(GolangHttpFilterTest, SetHeaderAtWrongStage) {
   EXPECT_EQ(CAPINotInGo, filter_->setHeader("foo", "bar", HeaderSet));
 }
 
+// invalid config for routeconfig filter
+TEST_F(GolangHttpFilterTest, InvalidConfigForRouteConfigFilter) {
+  InSequence s;
+  EXPECT_THROW_WITH_REGEX(setup(ROUTECONFIG, genSoPath(ROUTECONFIG), ROUTECONFIG), EnvoyException,
+                          "golang filter failed to parse plugin config");
+}
+
 } // namespace
 } // namespace Golang
 } // namespace HttpFilters
diff --git a/contrib/golang/filters/http/test/test_data/passthrough/filter.go b/contrib/golang/filters/http/test/test_data/passthrough/filter.go
index c78dbee0f462..17b40b0e459e 100644
--- a/contrib/golang/filters/http/test/test_data/passthrough/filter.go
+++ b/contrib/golang/filters/http/test/test_data/passthrough/filter.go
@@ -5,7 +5,7 @@ import (
 )
 
 func init() {
-	http.RegisterHttpFilterConfigFactoryAndParser("", http.PassThroughFactory, nil)
+	http.RegisterHttpFilterConfigFactoryAndParser("passthrough", http.PassThroughFactory, nil)
 }
 
 func main() {
diff --git a/contrib/golang/filters/http/test/test_data/routeconfig/config.go b/contrib/golang/filters/http/test/test_data/routeconfig/config.go
index fe7067edb01c..5af08bf8a41e 100644
--- a/contrib/golang/filters/http/test/test_data/routeconfig/config.go
+++ b/contrib/golang/filters/http/test/test_data/routeconfig/config.go
@@ -1,6 +1,8 @@
 package main
 
 import (
+	"errors"
+
 	xds "github.com/cncf/xds/go/xds/type/v3"
 	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
 	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http"
@@ -40,12 +42,15 @@ func (p *parser) Parse(any *anypb.Any) (interface{}, error) {
 		return nil, err
 	}
 
-	v := configStruct.Value
 	conf := &config{}
-	if remove, ok := v.AsMap()["remove"].(string); ok {
+	m := configStruct.Value.AsMap()
+	if _, ok := m["invalid"].(string); ok {
+		return nil, errors.New("testing invalid config")
+	}
+	if remove, ok := m["remove"].(string); ok {
 		conf.removeHeader = remove
 	}
-	if set, ok := v.AsMap()["set"].(string); ok {
+	if set, ok := m["set"].(string); ok {
 		conf.setHeader = set
 	}
 	return conf, nil
diff --git a/contrib/golang/router/cluster_specifier/source/golang_cluster_specifier.cc b/contrib/golang/router/cluster_specifier/source/golang_cluster_specifier.cc
index 37cd2c0a1cb5..96f90930b2d4 100644
--- a/contrib/golang/router/cluster_specifier/source/golang_cluster_specifier.cc
+++ b/contrib/golang/router/cluster_specifier/source/golang_cluster_specifier.cc
@@ -21,15 +21,9 @@ ClusterConfig::ClusterConfig(const GolangClusterProto& config)
   // loads DSO store a static map and a open handles leak will occur when the filter gets loaded and
   // unloaded.
   // TODO: unload DSO when filter updated.
-  auto res = Envoy::Dso::DsoManager::load(so_id_, so_path_);
-  if (!res) {
-    throw EnvoyException(fmt::format("golang_cluster_specifier_plugin: load library failed: {} {}",
-                                     so_id_, so_path_));
-  }
-
-  dynamic_lib_ = Dso::DsoManager::getDsoByID(so_id_);
+  dynamic_lib_ = Envoy::Dso::DsoManager::load(so_id_, so_path_);
   if (dynamic_lib_ == nullptr) {
-    throw EnvoyException(fmt::format("golang_cluster_specifier_plugin: get library failed: {} {}",
+    throw EnvoyException(fmt::format("golang_cluster_specifier_plugin: load library failed: {} {}",
                                      so_id_, so_path_));
   }
 
diff --git a/test/config_test/BUILD b/test/config_test/BUILD
index 5ad16a344c68..3fa4a6234c2f 100644
--- a/test/config_test/BUILD
+++ b/test/config_test/BUILD
@@ -35,7 +35,10 @@ envoy_cc_test(
         "example_configs_test_setup.sh",
         "//configs:example_configs",
     ],
-    env = {"EXAMPLE_CONFIGS_TAR_PATH": "envoy/configs/example_configs.tar"},
+    env = {
+        "EXAMPLE_CONFIGS_TAR_PATH": "envoy/configs/example_configs.tar",
+        "GODEBUG": "cgocheck=0",
+    },
     deps = [
         ":example_configs_test_lib",
     ],

From 46c37678e5c9ffc7e06189fe060ec02e9296d58b Mon Sep 17 00:00:00 2001
From: botengyao 
Date: Thu, 8 Jun 2023 10:33:08 -0400
Subject: [PATCH 494/740] hc: add host related info to the health check event
 (#27833)

Signed-off-by: Boteng Yao 
---
 .../data/core/v3/health_check_event.proto     |  9 +++++-
 changelogs/current.yaml                       |  5 ++++
 .../upstream/health_checker_event_logger.cc   |  4 +++
 .../upstream/health_checker_impl_test.cc      | 28 +++++++++++++++++--
 4 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/api/envoy/data/core/v3/health_check_event.proto b/api/envoy/data/core/v3/health_check_event.proto
index a349fa31bc08..17e78ea5ecb1 100644
--- a/api/envoy/data/core/v3/health_check_event.proto
+++ b/api/envoy/data/core/v3/health_check_event.proto
@@ -3,6 +3,7 @@ syntax = "proto3";
 package envoy.data.core.v3;
 
 import "envoy/config/core/v3/address.proto";
+import "envoy/config/core/v3/base.proto";
 
 import "google/protobuf/timestamp.proto";
 
@@ -34,7 +35,7 @@ enum HealthCheckerType {
   THRIFT = 4;
 }
 
-// [#next-free-field: 10]
+// [#next-free-field: 12]
 message HealthCheckEvent {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.data.core.v2alpha.HealthCheckEvent";
@@ -66,6 +67,12 @@ message HealthCheckEvent {
 
   // Timestamp for event.
   google.protobuf.Timestamp timestamp = 6;
+
+  // Host metadata
+  config.core.v3.Metadata metadata = 10;
+
+  // Host locality
+  config.core.v3.Locality locality = 11;
 }
 
 message HealthCheckEjectUnhealthy {
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 7fbafbe3002e..a393fc0e6b39 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -326,6 +326,11 @@ new_features:
 - area: http
   change: |
     added support for configuring additional :ref:`cookie attributes `.
+- area: health_check
+  change: |
+    added host related information :ref:`metadata ` and
+    :ref:`locality ` to
+    the :ref:`health check event ` definition.
 
 deprecated:
 - area: access_log
diff --git a/source/common/upstream/health_checker_event_logger.cc b/source/common/upstream/health_checker_event_logger.cc
index a293bcaa3be3..c75d3a55e6db 100644
--- a/source/common/upstream/health_checker_event_logger.cc
+++ b/source/common/upstream/health_checker_event_logger.cc
@@ -61,6 +61,10 @@ void HealthCheckEventLoggerImpl::createHealthCheckEvent(
   envoy::config::core::v3::Address address;
   Network::Utility::addressToProtobufAddress(*host.address(), address);
   *event.mutable_host() = std::move(address);
+  if (host.metadata() != nullptr) {
+    *event.mutable_metadata() = *host.metadata();
+  }
+  *event.mutable_locality() = host.locality();
 
   TimestampUtil::systemClockToTimestamp(time_source_.systemTime(), *event.mutable_timestamp());
 
diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc
index 8e97475b2e73..8c44cd9b1f31 100644
--- a/test/common/upstream/health_checker_impl_test.cc
+++ b/test/common/upstream/health_checker_impl_test.cc
@@ -6404,7 +6404,9 @@ TEST(HealthCheckEventLoggerImplTest, All) {
 
   std::shared_ptr host(new NiceMock());
   NiceMock cluster_info;
+  MetadataConstSharedPtr metadata(new envoy::config::core::v3::Metadata());
   ON_CALL(*host, cluster()).WillByDefault(ReturnRef(cluster_info));
+  ON_CALL(*host, metadata()).WillByDefault(Return(metadata));
 
   HealthCheckerFactoryContextImpl context(cluster, runtime, dispatcher, validation_visitor, api,
                                           log_manager);
@@ -6420,6 +6422,8 @@ TEST(HealthCheckEventLoggerImplTest, All) {
                          "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
                          "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
                          "cluster\",\"eject_unhealthy_event\":{\"failure_type\":\"ACTIVE\"},"
+                         "\"metadata\":{\"filter_metadata\":{},\"typed_filter_metadata\":{}},"
+                         "\"locality\":{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},"
                          "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"}));
   event_logger.logEjectUnhealthy(envoy::data::core::v3::HTTP, host, envoy::data::core::v3::ACTIVE);
 
@@ -6427,7 +6431,9 @@ TEST(HealthCheckEventLoggerImplTest, All) {
                          "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{"
                          "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
                          "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
-                         "cluster\",\"add_healthy_event\":{\"first_check\":false},\"timestamp\":"
+                         "cluster\",\"add_healthy_event\":{\"first_check\":false},\"metadata\":"
+                         "{\"filter_metadata\":{},\"typed_filter_metadata\":{}},\"locality\":"
+                         "{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},\"timestamp\":"
                          "\"2009-02-13T23:31:31.234Z\"}\n"}));
   event_logger.logAddHealthy(envoy::data::core::v3::HTTP, host, false);
 
@@ -6437,6 +6443,8 @@ TEST(HealthCheckEventLoggerImplTest, All) {
                          "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
                          "cluster\",\"health_check_failure_event\":{\"failure_type\":\"ACTIVE\","
                          "\"first_check\":false},"
+                         "\"metadata\":{\"filter_metadata\":{},\"typed_filter_metadata\":{}},"
+                         "\"locality\":{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},"
                          "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"}));
   event_logger.logUnhealthy(envoy::data::core::v3::HTTP, host, envoy::data::core::v3::ACTIVE,
                             false);
@@ -6446,6 +6454,8 @@ TEST(HealthCheckEventLoggerImplTest, All) {
                          "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
                          "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
                          "cluster\",\"degraded_healthy_host\":{},"
+                         "\"metadata\":{\"filter_metadata\":{},\"typed_filter_metadata\":{}},"
+                         "\"locality\":{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},"
                          "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"}));
   event_logger.logDegraded(envoy::data::core::v3::HTTP, host);
 
@@ -6454,6 +6464,8 @@ TEST(HealthCheckEventLoggerImplTest, All) {
                          "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
                          "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
                          "cluster\",\"no_longer_degraded_host\":{},"
+                         "\"metadata\":{\"filter_metadata\":{},\"typed_filter_metadata\":{}},"
+                         "\"locality\":{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},"
                          "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"}));
   event_logger.logNoLongerDegraded(envoy::data::core::v3::HTTP, host);
 }
@@ -6477,7 +6489,9 @@ TEST(HealthCheckEventLoggerImplTest, OneEventLogger) {
 
   std::shared_ptr host(new NiceMock());
   NiceMock cluster_info;
+  MetadataConstSharedPtr metadata(new envoy::config::core::v3::Metadata());
   ON_CALL(*host, cluster()).WillByDefault(ReturnRef(cluster_info));
+  ON_CALL(*host, metadata()).WillByDefault(Return(metadata));
 
   HealthCheckerFactoryContextImpl context(cluster, runtime, dispatcher, validation_visitor, api,
                                           log_manager);
@@ -6493,13 +6507,17 @@ TEST(HealthCheckEventLoggerImplTest, OneEventLogger) {
                            "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
                            "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
                            "cluster\",\"eject_unhealthy_event\":{\"failure_type\":\"ACTIVE\"},"
+                           "\"metadata\":{\"filter_metadata\":{},\"typed_filter_metadata\":{}},"
+                           "\"locality\":{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},"
                            "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n");
 
   event_logger.logAddHealthy(envoy::data::core::v3::HTTP, host, false);
   EXPECT_EQ(file_log_data, "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{"
                            "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
                            "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
-                           "cluster\",\"add_healthy_event\":{\"first_check\":false},\"timestamp\":"
+                           "cluster\",\"add_healthy_event\":{\"first_check\":false},\"metadata\":"
+                           "{\"filter_metadata\":{},\"typed_filter_metadata\":{}},\"locality\":"
+                           "{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},\"timestamp\":"
                            "\"2009-02-13T23:31:31.234Z\"}\n");
 
   event_logger.logUnhealthy(envoy::data::core::v3::HTTP, host, envoy::data::core::v3::ACTIVE,
@@ -6510,6 +6528,8 @@ TEST(HealthCheckEventLoggerImplTest, OneEventLogger) {
             "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
             "cluster\",\"health_check_failure_event\":{\"failure_type\":\"ACTIVE\","
             "\"first_check\":false},"
+            "\"metadata\":{\"filter_metadata\":{},\"typed_filter_metadata\":{}},"
+            "\"locality\":{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},"
             "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n");
 
   event_logger.logDegraded(envoy::data::core::v3::HTTP, host);
@@ -6517,6 +6537,8 @@ TEST(HealthCheckEventLoggerImplTest, OneEventLogger) {
                            "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
                            "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
                            "cluster\",\"degraded_healthy_host\":{},"
+                           "\"metadata\":{\"filter_metadata\":{},\"typed_filter_metadata\":{}},"
+                           "\"locality\":{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},"
                            "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n");
 
   event_logger.logNoLongerDegraded(envoy::data::core::v3::HTTP, host);
@@ -6524,6 +6546,8 @@ TEST(HealthCheckEventLoggerImplTest, OneEventLogger) {
                            "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
                            "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
                            "cluster\",\"no_longer_degraded_host\":{},"
+                           "\"metadata\":{\"filter_metadata\":{},\"typed_filter_metadata\":{}},"
+                           "\"locality\":{\"region\":\"\",\"zone\":\"\",\"sub_zone\":\"\"},"
                            "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n");
 }
 

From 08dd6fedf0c433c341e74e689194beb23540932c Mon Sep 17 00:00:00 2001
From: ohadvano <49730675+ohadvano@users.noreply.github.com>
Date: Thu, 8 Jun 2023 17:34:03 +0300
Subject: [PATCH 495/740] application_logs: add bootstrap option to set log
 format (#27816)

add bootstrap option to set log format

Signed-off-by: ohadvano 
---
 api/envoy/config/bootstrap/v3/bootstrap.proto |  5 +++
 changelogs/current.yaml                       |  5 +++
 source/server/utils.cc                        |  9 +++-
 test/server/config_validation/server_test.cc  | 41 +++++++++++++++++++
 .../test_data/text_application_logs.yaml      | 10 +++++
 test/server/server_test.cc                    | 13 ++++++
 .../server/text_application_log.yaml          | 10 +++++
 test/server/utils_test.cc                     |  6 +++
 8 files changed, 97 insertions(+), 2 deletions(-)
 create mode 100644 test/server/config_validation/test_data/text_application_logs.yaml
 create mode 100644 test/server/test_data/server/text_application_log.yaml

diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto
index f171068aaeed..43e3a33a3f15 100644
--- a/api/envoy/config/bootstrap/v3/bootstrap.proto
+++ b/api/envoy/config/bootstrap/v3/bootstrap.proto
@@ -110,6 +110,11 @@ message Bootstrap {
         // support all the format flags specified in the :option:`--log-format`
         // command line options section, except for the ``%v`` and ``%_`` flags.
         google.protobuf.Struct json_format = 1;
+
+        // Flush application log in a format defined by a string. The text format
+        // can support all the format flags specified in the :option:`--log-format`
+        // command line option section.
+        string text_format = 2;
       }
     }
 
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index a393fc0e6b39..da7ca4343063 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -309,6 +309,11 @@ new_features:
     Added bootstrap option
     :ref:`application_log_format `
     to enable setting application log format as JSON structure.
+- area: application_logs
+  change: |
+    Added bootstrap option
+    :ref:`application_log_format `
+    to enable setting application log text format from config.
 - area: ext_proc
   change: |
     added new field ``filter_metadata  getAllConfigFiles() {
+    setupTestDirectory();
+    return {"text_application_logs.yaml"};
+  }
+};
+
 TEST_P(ValidationServerTest, Validate) {
   EXPECT_TRUE(validateConfig(options_, Network::Address::InstanceConstSharedPtr(),
                              component_factory_, Thread::threadFactoryForTest(),
@@ -324,6 +341,30 @@ INSTANTIATE_TEST_SUITE_P(
     ::testing::ValuesIn(
         JsonApplicationLogsValidationServerForbiddenFlagUnderscoreTest::getAllConfigFiles()));
 
+TEST_P(TextApplicationLogsValidationServerTest, TextApplicationLogs) {
+  Thread::MutexBasicLockable access_log_lock;
+  Stats::IsolatedStoreImpl stats_store;
+  DangerousDeprecatedTestTime time_system;
+  ValidationInstance server(options_, time_system.timeSystem(),
+                            Network::Address::InstanceConstSharedPtr(), stats_store,
+                            access_log_lock, component_factory_, Thread::threadFactoryForTest(),
+                            Filesystem::fileSystemForTest());
+
+  Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
+  MockLogSink sink(Envoy::Logger::Registry::getSink());
+  EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto& log) {
+    EXPECT_THAT(msg, HasSubstr("[lvl: info][msg: hello]"));
+    EXPECT_EQ(log.logger_name, "misc");
+  }));
+
+  ENVOY_LOG_MISC(info, "hello");
+  server.shutdown();
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    AllConfigs, TextApplicationLogsValidationServerTest,
+    ::testing::ValuesIn(TextApplicationLogsValidationServerTest::getAllConfigFiles()));
+
 } // namespace
 } // namespace Server
 } // namespace Envoy
diff --git a/test/server/config_validation/test_data/text_application_logs.yaml b/test/server/config_validation/test_data/text_application_logs.yaml
new file mode 100644
index 000000000000..f598e8ffcde5
--- /dev/null
+++ b/test/server/config_validation/test_data/text_application_logs.yaml
@@ -0,0 +1,10 @@
+---
+application_log_config:
+  log_format:
+    text_format: "[lvl: %l][msg: %v]"
+
+admin:
+  address:
+    socket_address:
+      address: 0.0.0.0
+      port_value: 9000
diff --git a/test/server/server_test.cc b/test/server/server_test.cc
index a98c6e9d2663..c732c88209ab 100644
--- a/test/server/server_test.cc
+++ b/test/server/server_test.cc
@@ -1648,6 +1648,19 @@ TEST_P(ServerInstanceImplTest, JsonApplicationLogFailWithForbiddenFlagUnderscore
       "setJsonLogFormat error: INVALID_ARGUMENT: Usage of %_ is unavailable for JSON log formats");
 }
 
+TEST_P(ServerInstanceImplTest, TextApplicationLog) {
+  EXPECT_NO_THROW(initialize("test/server/test_data/server/text_application_log.yaml"));
+
+  Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
+  MockLogSink sink(Envoy::Logger::Registry::getSink());
+  EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto& log) {
+    EXPECT_THAT(msg, HasSubstr("[lvl: info][msg: hello]"));
+    EXPECT_EQ(log.logger_name, "misc");
+  }));
+
+  ENVOY_LOG_MISC(info, "hello");
+}
+
 } // namespace
 } // namespace Server
 } // namespace Envoy
diff --git a/test/server/test_data/server/text_application_log.yaml b/test/server/test_data/server/text_application_log.yaml
new file mode 100644
index 000000000000..f598e8ffcde5
--- /dev/null
+++ b/test/server/test_data/server/text_application_log.yaml
@@ -0,0 +1,10 @@
+---
+application_log_config:
+  log_format:
+    text_format: "[lvl: %l][msg: %v]"
+
+admin:
+  address:
+    socket_address:
+      address: 0.0.0.0
+      port_value: 9000
diff --git a/test/server/utils_test.cc b/test/server/utils_test.cc
index 7576e41522c1..ea05b4836af2 100644
--- a/test/server/utils_test.cc
+++ b/test/server/utils_test.cc
@@ -69,6 +69,12 @@ TEST(UtilsTest, MaybeSetApplicationLogFormat) {
     EXPECT_NO_THROW(Utility::maybeSetApplicationLogFormat(log_config));
   }
 
+  {
+    envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config;
+    log_config.mutable_log_format()->mutable_text_format();
+    EXPECT_NO_THROW(Utility::maybeSetApplicationLogFormat(log_config));
+  }
+
   {
     envoy::config::bootstrap::v3::Bootstrap::ApplicationLogConfig log_config;
     auto* format = log_config.mutable_log_format()->mutable_json_format();

From 26f2fce16bb417f8615d929fe8b8b93486820cc3 Mon Sep 17 00:00:00 2001
From: realtimetodie 
Date: Thu, 8 Jun 2023 16:43:54 +0200
Subject: [PATCH 496/740] chore: update owner for http language filter contrib
 extension (#27873)

Signed-off-by: shooj4aegohbaivo 
---
 CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index 33b257f22307..c544bdb82b03 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -358,7 +358,7 @@ extensions/filters/http/oauth2 @derekargueta @snowp
 /contrib/cryptomb/ @giantcroc @soulxu
 /contrib/vcl/ @florincoras @KfreeZ
 /contrib/hyperscan/ @zhxie @soulxu
-/contrib/language/ @diceride @diceride
+/contrib/language/ @realtimetodie @realtimetodie
 /contrib/dlb/ @mattklein123 @daixiang0
 /contrib/qat/ @giantcroc @soulxu
 /contrib/generic_proxy/ @wbpcode @soulxu @zhaohuabing @rojkov @htuch

From d7e2fa27e5ca4633d8d64d44dcc56e9b661f5bd2 Mon Sep 17 00:00:00 2001
From: yanavlasov 
Date: Thu, 8 Jun 2023 10:54:06 -0400
Subject: [PATCH 497/740] Avoid copying garbage if pthread_getname_np fails
 (#27862)

Signed-off-by: Yan Avlasov 
---
 source/common/common/posix/thread_impl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/common/common/posix/thread_impl.cc b/source/common/common/posix/thread_impl.cc
index 5d78dd24a56e..e935ac53c11b 100644
--- a/source/common/common/posix/thread_impl.cc
+++ b/source/common/common/posix/thread_impl.cc
@@ -99,7 +99,7 @@ class ThreadImplPosix : public Thread {
   // may fail, if the thread exits prior to the system call.
   bool getNameFromOS(std::string& name) {
     // Verify that the name got written into the thread as expected.
-    char buf[PTHREAD_MAX_THREADNAME_LEN_INCLUDING_NULL_BYTE];
+    char buf[PTHREAD_MAX_THREADNAME_LEN_INCLUDING_NULL_BYTE] = {0};
     const int get_name_rc = pthread_getname_np(thread_handle_, buf, sizeof(buf));
     name = buf;
     return get_name_rc == 0;

From a757dc8d18f52680dec34485376687a7da0a2509 Mon Sep 17 00:00:00 2001
From: yanavlasov 
Date: Thu, 8 Jun 2023 10:58:19 -0400
Subject: [PATCH 498/740] Make Balsa validation of transfer-encoding consistent
 with http-parser and UHV (#27849)

Signed-off-by: Yan Avlasov 
---
 source/common/http/http1/balsa_parser.cc  |  3 ---
 test/common/http/http1/codec_impl_test.cc | 29 ++---------------------
 test/integration/integration_test.cc      |  9 -------
 3 files changed, 2 insertions(+), 39 deletions(-)

diff --git a/source/common/http/http1/balsa_parser.cc b/source/common/http/http1/balsa_parser.cc
index 7a354a8afe8c..33082e1b120f 100644
--- a/source/common/http/http1/balsa_parser.cc
+++ b/source/common/http/http1/balsa_parser.cc
@@ -153,10 +153,7 @@ BalsaParser::BalsaParser(MessageType type, ParserCallbacks* connection, size_t m
   http_validation_policy.require_header_colon = true;
   http_validation_policy.disallow_multiple_content_length = false;
   http_validation_policy.disallow_transfer_encoding_with_content_length = false;
-#ifdef ENVOY_ENABLE_UHV
-  // UHV - disable transfer-encoding validations in Balsa
   http_validation_policy.validate_transfer_encoding = false;
-#endif
   framer_.set_http_validation_policy(http_validation_policy);
 
   framer_.set_balsa_headers(&headers_);
diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc
index c269f4513533..94d848188028 100644
--- a/test/common/http/http1/codec_impl_test.cc
+++ b/test/common/http/http1/codec_impl_test.cc
@@ -549,18 +549,8 @@ TEST_P(Http1ServerConnectionImplTest, UnsupportedEncoding) {
   EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder));
 
   Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\nHost: host\r\ntransfer-encoding: gzip\r\n\r\n");
-#ifdef ENVOY_ENABLE_UHV
   EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, _, _, _,
                                       "http1.invalid_transfer_encoding"));
-#else
-  if (parser_impl_ == Http1ParserImpl::HttpParser) {
-    EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, _, _, _,
-                                        "http1.invalid_transfer_encoding"));
-  } else {
-    // TODO(#27375): Balsa codec produces invalid response in non UHV mode
-    EXPECT_CALL(decoder, sendLocalReply(Http::Code::BadRequest, _, _, _, "http1.codec_error"));
-  }
-#endif
   auto status = codec_->dispatch(buffer);
 #ifdef ENVOY_ENABLE_UHV
   EXPECT_TRUE(status.ok());
@@ -4542,18 +4532,8 @@ TEST_P(Http1ServerConnectionImplTest, MultipleTransferEncoding) {
         decoder.setResponseEncoder(&encoder);
         return decoder;
       }));
-#ifdef ENVOY_ENABLE_UHV
   EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, "Not Implemented", _, _,
                                       "http1.invalid_transfer_encoding"));
-#else
-  if (parser_impl_ == Http1ParserImpl::BalsaParser) {
-    EXPECT_CALL(decoder,
-                sendLocalReply(Http::Code::BadRequest, "Bad Request", _, _, "http1.codec_error"));
-  } else {
-    EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, "Not Implemented", _, _,
-                                        "http1.invalid_transfer_encoding"));
-  }
-#endif
   Buffer::OwnedImpl buffer("POST / HTTP/1.1\r\nHost: foo.bar\r\n"
                            "Transfer-Encoding: chunked\r\n"
                            "Transfer-Encoding: chunked\r\n"
@@ -4566,13 +4546,8 @@ TEST_P(Http1ServerConnectionImplTest, MultipleTransferEncoding) {
 #else
   EXPECT_TRUE(isCodecProtocolError(status));
 
-  if (parser_impl_ == Http1ParserImpl::BalsaParser) {
-    EXPECT_EQ("http1.codec_error", response_encoder->getStream().responseDetails());
-    EXPECT_EQ(status.message(), "http/1.1 protocol error: MULTIPLE_TRANSFER_ENCODING_KEYS");
-  } else {
-    EXPECT_EQ("http1.invalid_transfer_encoding", response_encoder->getStream().responseDetails());
-    EXPECT_EQ(status.message(), "http/1.1 protocol error: unsupported transfer encoding");
-  }
+  EXPECT_EQ("http1.invalid_transfer_encoding", response_encoder->getStream().responseDetails());
+  EXPECT_EQ(status.message(), "http/1.1 protocol error: unsupported transfer encoding");
 #endif
 }
 
diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc
index 7c30ed707159..bd5d97d44422 100644
--- a/test/integration/integration_test.cc
+++ b/test/integration/integration_test.cc
@@ -924,16 +924,7 @@ TEST_P(IntegrationTest, TestInvalidTransferEncoding) {
   const std::string request = "GET / HTTP/1.1\r\nHost: host\r\ntransfer-encoding: "
                               "identity\r\ntransfer-encoding: chunked \r\n\r\n";
   sendRawHttpAndWaitForResponse(lookupPort("http"), request.c_str(), &response, false);
-#ifdef ENVOY_ENABLE_UHV
   EXPECT_THAT(response, StartsWith("HTTP/1.1 501 Not Implemented\r\n"));
-#else
-  if (http1_implementation_ == Http1ParserImpl::BalsaParser) {
-    // TODO(#27375): Balsa codec produces invalid response in non UHV mode
-    EXPECT_THAT(response, StartsWith("HTTP/1.1 400 Bad Request\r\n"));
-  } else {
-    EXPECT_THAT(response, StartsWith("HTTP/1.1 501 Not Implemented\r\n"));
-  }
-#endif
 }
 
 TEST_P(IntegrationTest, TestPipelinedResponses) {

From df778241161ee9077fadde47bfe8e018ad6a685b Mon Sep 17 00:00:00 2001
From: phlax 
Date: Thu, 8 Jun 2023 16:04:57 +0100
Subject: [PATCH 499/740] ci: Fix cache mount (#27778)

Signed-off-by: Ryan Northey 
---
 .azure-pipelines/bazel.yml               |  4 ++++
 .azure-pipelines/cached.yml              |  5 ++++-
 .azure-pipelines/docker/load_cache.sh    | 10 +++++++---
 .azure-pipelines/docker/prepare_cache.sh |  8 ++++++--
 .azure-pipelines/docker/save_cache.sh    |  9 +++++++--
 .azure-pipelines/stage/publish.yml       |  3 ++-
 6 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml
index 37c7d4735fd0..f0fc48dde0f1 100644
--- a/.azure-pipelines/bazel.yml
+++ b/.azure-pipelines/bazel.yml
@@ -24,6 +24,9 @@ parameters:
 - name: cacheKeyDockerTmpDir
   type: string
   default: /mnt/docker_cache
+- name: cacheKeyDockerNoTmpfs
+  type: string
+  default: ''
 - name: cacheKey
   type: string
   default: $(cacheKeyBazelFiles)
@@ -151,6 +154,7 @@ steps:
     name: "${{ parameters.cacheKeyDockerName }}"
     path: "${{ parameters.cacheKeyDockerPath }}"
     tmpDirectory: "${{ parameters.cacheKeyDockerTmpDir }}"
+    tmpNoTmpfs: "${{ parameters.cacheKeyDockerNoTmpfs }}"
     arch: "${{ parameters.artifactSuffix }}"
 
 - ${{ each step in parameters.stepsPre }}:
diff --git a/.azure-pipelines/cached.yml b/.azure-pipelines/cached.yml
index 591807fb063d..d75ef8b5771e 100644
--- a/.azure-pipelines/cached.yml
+++ b/.azure-pipelines/cached.yml
@@ -15,6 +15,9 @@ parameters:
 - name: tmpDirectory
   type: string
   default: /mnt/docker_cache
+- name: tmpNoTmpfs
+  type: string
+  default:
 - name: path
   type: string
   default: /mnt/docker
@@ -27,7 +30,7 @@ parameters:
 
 
 steps:
-- script: sudo .azure-pipelines/docker/prepare_cache.sh "${{ parameters.tmpDirectory }}"
+- script: sudo .azure-pipelines/docker/prepare_cache.sh "${{ parameters.tmpDirectory }}" "${{ parameters.tmpNoTmpfs }}"
   displayName: "Cache/prepare (${{ parameters.name }})"
 - task: Cache@2
   env:
diff --git a/.azure-pipelines/docker/load_cache.sh b/.azure-pipelines/docker/load_cache.sh
index ae2c486f806a..78c6cd8e5d99 100755
--- a/.azure-pipelines/docker/load_cache.sh
+++ b/.azure-pipelines/docker/load_cache.sh
@@ -43,9 +43,13 @@ fi
 echo "Starting Docker daemon ..."
 systemctl start docker
 
-echo "Unmount cache tmp ${DOCKER_CACHE_PATH} ..."
-umount "${DOCKER_CACHE_PATH}"
-
+if mountpoint -q "${DOCKER_CACHE_PATH}"; then
+    echo "Unmount cache tmp ${DOCKER_CACHE_PATH} ..."
+    umount "${DOCKER_CACHE_PATH}"
+else
+    echo "Remove cache tmp ${DOCKER_CACHE_PATH} ..."
+    rm -rf "${DOCKER_CACHE_PATH}"
+fi
 docker images
 df -h
 
diff --git a/.azure-pipelines/docker/prepare_cache.sh b/.azure-pipelines/docker/prepare_cache.sh
index 0cb4f0e55772..fe417d5f5e41 100755
--- a/.azure-pipelines/docker/prepare_cache.sh
+++ b/.azure-pipelines/docker/prepare_cache.sh
@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
 DOCKER_CACHE_PATH="$1"
+NO_MOUNT_TMPFS="${2:-}"
 DOCKER_CACHE_OWNERSHIP="vsts:vsts"
 
 
@@ -13,7 +14,10 @@ if ! id -u vsts &> /dev/null; then
     DOCKER_CACHE_OWNERSHIP=azure-pipelines
 fi
 
-echo "Mounting tmpfs cache directory (${DOCKER_CACHE_PATH}) ..."
+echo "Creating cache directory (${DOCKER_CACHE_PATH}) ..."
 mkdir -p "${DOCKER_CACHE_PATH}"
-mount -t tmpfs none "${DOCKER_CACHE_PATH}"
+if [[ -z "$NO_MOUNT_TMPFS" ]]; then
+    echo "Mount tmpfs directory: ${DOCKER_CACHE_PATH}"
+    mount -t tmpfs none "$DOCKER_CACHE_PATH"
+fi
 chown -R "$DOCKER_CACHE_OWNERSHIP" "${DOCKER_CACHE_PATH}"
diff --git a/.azure-pipelines/docker/save_cache.sh b/.azure-pipelines/docker/save_cache.sh
index c21c6bad1db5..85f912cbad2d 100755
--- a/.azure-pipelines/docker/save_cache.sh
+++ b/.azure-pipelines/docker/save_cache.sh
@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
 DOCKER_CACHE_PATH="$1"
+NO_MOUNT_TMPFS="${2:-}"
 
 if [[ -z "$DOCKER_CACHE_PATH" ]]; then
     echo "prime_docker_cache called without path arg" >&2
@@ -19,9 +20,13 @@ docker images
 echo "Stopping Docker ..."
 systemctl stop docker
 
-echo "Creating tmpfs directory to save tarball: ${DOCKER_CACHE_PATH}"
+echo "Creating directory to save tarball: ${DOCKER_CACHE_PATH}"
 mkdir -p "$DOCKER_CACHE_PATH"
-mount -t tmpfs none "$DOCKER_CACHE_PATH"
+
+if [[ -z "$NO_MOUNT_TMPFS" ]]; then
+    echo "Mount tmpfs directory: ${DOCKER_CACHE_PATH}"
+    mount -t tmpfs none "$DOCKER_CACHE_PATH"
+fi
 
 echo "Creating tarball: /var/lib/docker -> ${DOCKER_CACHE_TARBALL}"
 tar cf - -C /var/lib/docker . | zstd - -T0 -o "$DOCKER_CACHE_TARBALL"
diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml
index beb715a6b46d..17101a5bc84f 100644
--- a/.azure-pipelines/stage/publish.yml
+++ b/.azure-pipelines/stage/publish.yml
@@ -122,6 +122,7 @@ jobs:
       cacheKeyDocker: "ci/Dockerfile-envoy | VERSION.txt| $(cacheKeyBazelFiles)"
       cacheKeyDockerName: publish_docker
       cacheKeyDockerTmpDir: /var/azpcache
+      cacheKeyDockerNoTmpfs: true
       cacheKeyDockerPath: ""
       cacheKeyDockerVersion: "$(cacheKeyDockerBuild)"
       env:
@@ -189,7 +190,7 @@ jobs:
           GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
           ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
           ENVOY_RBE: "1"
-      - script: sudo .azure-pipelines/docker/save_cache.sh /var/azpcache
+      - script: sudo .azure-pipelines/docker/save_cache.sh /var/azpcache true
         displayName: "Cache/save (publish_docker)"
 
 - job: package_x64

From d2adc854b6eabdc13844b5170478127969d16f24 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Thu, 8 Jun 2023 16:05:15 +0100
Subject: [PATCH 500/740] deps: Bump toolshed github actions version (#27800)

Signed-off-by: Ryan Northey 
---
 .github/workflows/commands.yml          | 2 +-
 .github/workflows/envoy-sync.yml        | 2 +-
 .github/workflows/workflow-complete.yml | 2 +-
 .github/workflows/workflow-start.yml    | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml
index 35da322861f1..63af84c63af5 100644
--- a/.github/workflows/commands.yml
+++ b/.github/workflows/commands.yml
@@ -22,6 +22,6 @@ jobs:
       pull-requests: write
       actions: write
     steps:
-    - uses: envoyproxy/toolshed/gh-actions/retest@56d5781416445ed530e075b71546dedee94cf054
+    - uses: envoyproxy/toolshed/gh-actions/retest@actions-v0.0.1
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/envoy-sync.yml b/.github/workflows/envoy-sync.yml
index da65ccc55984..ce191989bf3a 100644
--- a/.github/workflows/envoy-sync.yml
+++ b/.github/workflows/envoy-sync.yml
@@ -20,7 +20,7 @@ jobs:
         - envoy-filter-example
         - data-plane-api
     steps:
-    - uses: envoyproxy/toolshed/gh-actions/dispatch@1f1feae1e372dde41ecc6830028989bb6037c480
+    - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.0.1
       with:
         repository: "envoyproxy/${{ matrix.downstream }}"
         ref: main
diff --git a/.github/workflows/workflow-complete.yml b/.github/workflows/workflow-complete.yml
index 8b0d7f8c98d1..7b6050b9d75a 100644
--- a/.github/workflows/workflow-complete.yml
+++ b/.github/workflows/workflow-complete.yml
@@ -52,7 +52,7 @@ jobs:
         echo "state=${STATE}" >> "$GITHUB_OUTPUT"
       id: job
     - name: Complete status check
-      uses: envoyproxy/toolshed/gh-actions/status@a6e1c951217efae1ac6b2bf32c5a9729976442b8
+      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.1
       with:
         authToken: ${{ secrets.GITHUB_TOKEN }}
         context: ${{ github.event.workflow.name }}
diff --git a/.github/workflows/workflow-start.yml b/.github/workflows/workflow-start.yml
index 64c2999c1201..6b377a395e7a 100644
--- a/.github/workflows/workflow-start.yml
+++ b/.github/workflows/workflow-start.yml
@@ -19,7 +19,7 @@ jobs:
       statuses: write
     steps:
     - name: Start status check
-      uses: envoyproxy/toolshed/gh-actions/status@a6e1c951217efae1ac6b2bf32c5a9729976442b8
+      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.1
       with:
         authToken: ${{ secrets.GITHUB_TOKEN }}
         context: ${{ inputs.workflowName }}

From cf1af7660d910c275bee1d7b34218c32c6419198 Mon Sep 17 00:00:00 2001
From: "Vikas Choudhary (vikasc)" 
Date: Thu, 8 Jun 2023 20:40:35 +0530
Subject: [PATCH 501/740] Update curl command in the tunneling examples
 (#27777)

* Update curl command in the tunneling examples

Signed-off-by: Vikas Choudhary 

* fix typo

Signed-off-by: Vikas Choudhary 

* add host header

Signed-off-by: Vikas Choudhary 

* nits

Signed-off-by: Vikas Choudhary 

---------

Signed-off-by: Vikas Choudhary 
---
 configs/encapsulate_http_in_http2_connect.yaml | 1 +
 configs/encapsulate_in_http1_connect.yaml      | 2 +-
 configs/encapsulate_in_http2_connect.yaml      | 2 +-
 configs/encapsulate_in_http2_post.yaml         | 2 +-
 4 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/configs/encapsulate_http_in_http2_connect.yaml b/configs/encapsulate_http_in_http2_connect.yaml
index adbe66d1c3aa..d51724ae42f5 100644
--- a/configs/encapsulate_http_in_http2_connect.yaml
+++ b/configs/encapsulate_http_in_http2_connect.yaml
@@ -1,5 +1,6 @@
 # This configuration takes incoming HTTP requests on port 10000 and encapsulates it in a CONNECT
 # request which is sent upstream port 10001.
+# `curl -H 'Host: www.google.com' --resolve www.google.com:10000:127.0.0.1 https://www.google.com:10000`
 bootstrap_extensions:
 - name: envoy.bootstrap.internal_listener
   typed_config:
diff --git a/configs/encapsulate_in_http1_connect.yaml b/configs/encapsulate_in_http1_connect.yaml
index bc51e8c7cea1..4140cb6ccf9e 100644
--- a/configs/encapsulate_in_http1_connect.yaml
+++ b/configs/encapsulate_in_http1_connect.yaml
@@ -7,7 +7,7 @@
 # It can be used to test TCP tunneling as described in
 # https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/http/upgrades
 # and running `curl -x 127.0.0.1:10000 https://www.google.com`,
-# or running `curl -H "Host: the-hosted-domain-on-port-10003"  http://127.0.0.1:10002
+# or running `curl -H 'Host: www.google.com' --resolve www.google.com:10000:127.0.0.1 https://www.google.com:10000`
 
 admin:
   address:
diff --git a/configs/encapsulate_in_http2_connect.yaml b/configs/encapsulate_in_http2_connect.yaml
index 75a5d5fcd295..9346b75bb2f1 100644
--- a/configs/encapsulate_in_http2_connect.yaml
+++ b/configs/encapsulate_in_http2_connect.yaml
@@ -2,7 +2,7 @@
 # request which is sent upstream port 10001.
 # It can be used to test TCP tunneling as described in
 # https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/http/upgrades
-# and running `curl -x 127.0.0.1:10000 https://www.google.com`
+# and running `curl -H 'Host: www.google.com' --resolve www.google.com:10000:127.0.0.1 https://www.google.com:10000`
 
 admin:
   address:
diff --git a/configs/encapsulate_in_http2_post.yaml b/configs/encapsulate_in_http2_post.yaml
index d24737bbfdd4..55af299e1494 100644
--- a/configs/encapsulate_in_http2_post.yaml
+++ b/configs/encapsulate_in_http2_post.yaml
@@ -2,7 +2,7 @@
 # request which is sent upstream port 10001.
 # It can be used to test TCP tunneling as described in
 # https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/http/upgrades
-# and running `curl -x 127.0.0.1:10000 https://www.google.com`
+# and running `curl -H 'Host: www.google.com' --resolve www.google.com:10000:127.0.0.1 https://www.google.com:10000`
 
 admin:
   address:

From 75f7741c7af847cde4f54e31b9e612ca58cc4367 Mon Sep 17 00:00:00 2001
From: botengyao 
Date: Thu, 8 Jun 2023 12:49:39 -0400
Subject: [PATCH 502/740] hc: add more logs for tcp health checker (#27857)

add more logs for tcp health checker

Signed-off-by: Boteng Yao 
---
 .../health_checkers/tcp/health_checker_impl.cc      | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/source/extensions/health_checkers/tcp/health_checker_impl.cc b/source/extensions/health_checkers/tcp/health_checker_impl.cc
index 08a3fdde67fc..aefab127a1d2 100644
--- a/source/extensions/health_checkers/tcp/health_checker_impl.cc
+++ b/source/extensions/health_checkers/tcp/health_checker_impl.cc
@@ -69,12 +69,13 @@ void TcpHealthCheckerImpl::TcpActiveHealthCheckSession::onDeferredDelete() {
 }
 
 void TcpHealthCheckerImpl::TcpActiveHealthCheckSession::onData(Buffer::Instance& data) {
-  ENVOY_CONN_LOG(trace, "total pending buffer={}", *client_, data.length());
+  ENVOY_CONN_LOG(debug, "hc tcp total pending buffer={}", *client_, data.length());
   // TODO(lilika): The TCP health checker does generic pattern matching so we can't differentiate
   // between wrong data and not enough data. We could likely do better here and figure out cases in
   // which a match is not possible but that is not done now.
   if (PayloadMatcher::match(parent_.receive_bytes_, data)) {
-    ENVOY_CONN_LOG(trace, "healthcheck passed", *client_);
+    ENVOY_CONN_LOG(debug, "hc tcp healthcheck passed, health_check_address={}", *client_,
+                   host_->healthCheckAddress()->asString());
     data.drain(data.length());
     handleSuccess(false);
     if (!parent_.reuse_connection_) {
@@ -88,6 +89,8 @@ void TcpHealthCheckerImpl::TcpActiveHealthCheckSession::onEvent(Network::Connect
   if (event == Network::ConnectionEvent::RemoteClose ||
       event == Network::ConnectionEvent::LocalClose) {
     if (!expect_close_) {
+      ENVOY_CONN_LOG(debug, "hc tcp connection unexpected closed, health_check_address={}",
+                     *client_, host_->healthCheckAddress()->asString());
       handleFailure(envoy::data::core::v3::NETWORK);
     }
     parent_.dispatcher_.deferredDelete(std::move(client_));
@@ -108,7 +111,8 @@ void TcpHealthCheckerImpl::TcpActiveHealthCheckSession::onEvent(Network::Connect
     // TODO(mattklein123): In the case that a user configured bytes to write, they will not be
     // be written, since we currently have no way to know if the bytes actually get written via
     // the connection interface. We might want to figure out how to handle this better later.
-    ENVOY_CONN_LOG(trace, "healthcheck passed", *client_);
+    ENVOY_CONN_LOG(debug, "hc tcp healthcheck passed, health_check_address={}", *client_,
+                   host_->healthCheckAddress()->asString());
     expect_close_ = true;
     client_->close(Network::ConnectionCloseType::Abort);
     handleSuccess(false);
@@ -143,6 +147,9 @@ void TcpHealthCheckerImpl::TcpActiveHealthCheckSession::onInterval() {
 }
 
 void TcpHealthCheckerImpl::TcpActiveHealthCheckSession::onTimeout() {
+  ENVOY_CONN_LOG(debug, "hc tcp connection timeout, health_flags={}, health_check_address={}",
+                 *client_, HostUtility::healthFlagsToString(*host_),
+                 host_->healthCheckAddress()->asString());
   expect_close_ = true;
   client_->close(Network::ConnectionCloseType::Abort);
 }

From 285e0a4fc58ebd916745f02ce6a885a32f5f0754 Mon Sep 17 00:00:00 2001
From: Raven Black 
Date: Thu, 8 Jun 2023 12:24:07 -0700
Subject: [PATCH 503/740] Fix flaky race in file_system_http_cache_test
 (#27887)

Fix race in test

Signed-off-by: Raven Black 
---
 .../cache/file_system_http_cache/file_system_http_cache_test.cc  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc b/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc
index b334411bf390..43cbcb94f5b7 100644
--- a/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc
+++ b/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc
@@ -568,6 +568,7 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, FailedReadOfHeaderBlockInvalidatesT
   // unlink
   mock_async_file_manager_->nextActionCompletes(absl::OkStatus());
   EXPECT_EQ(result.cache_entry_status_, CacheEntryStatus::Unusable);
+  waitForEvictionThreadIdle();
   // Should have deducted the size of the file that got deleted. Since we started at 2 * 12345,
   // this should make the value 12345.
   EXPECT_EQ(cache_->stats().size_bytes_.value(), 12345);

From 2b0b1869a864abcc9e6f0309f914bdabf58b4455 Mon Sep 17 00:00:00 2001
From: realtimetodie 
Date: Thu, 8 Jun 2023 21:32:32 +0200
Subject: [PATCH 504/740] filter: clear route cache in language filter (#27872)

Signed-off-by: shooj4aegohbaivo 
---
 .../filters/http/language/v3alpha/language.proto    |  5 +++++
 contrib/language/filters/http/source/config.cc      |  3 ++-
 .../language/filters/http/source/language_filter.cc | 13 +++++++++++--
 .../language/filters/http/source/language_filter.h  |  9 ++++++++-
 4 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/api/contrib/envoy/extensions/filters/http/language/v3alpha/language.proto b/api/contrib/envoy/extensions/filters/http/language/v3alpha/language.proto
index 7f45d5e6be9a..47f296707b21 100644
--- a/api/contrib/envoy/extensions/filters/http/language/v3alpha/language.proto
+++ b/api/contrib/envoy/extensions/filters/http/language/v3alpha/language.proto
@@ -33,4 +33,9 @@ message Language {
     unique: true
     items {string {min_len: 2}}
   }];
+
+  // If the x-language header is altered, clear the route cache for the current request.
+  // This should be set if the route configuration may depend on the x-language header.
+  // Otherwise it should be unset to avoid the performance cost of route recalculation.
+  bool clear_route_cache = 3;
 }
diff --git a/contrib/language/filters/http/source/config.cc b/contrib/language/filters/http/source/config.cc
index 0130bf0ff771..b4bf1bd6b908 100644
--- a/contrib/language/filters/http/source/config.cc
+++ b/contrib/language/filters/http/source/config.cc
@@ -55,7 +55,8 @@ Http::FilterFactoryCb LanguageFilterFactory::createFilterFactoryFromProtoTyped(
   }
 
   auto config = std::make_shared(
-      std::make_shared(default_locale), locale_matcher, stats_prefix, context.scope());
+      std::make_shared(default_locale), locale_matcher,
+      proto_config.clear_route_cache(), stats_prefix, context.scope());
 
   return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void {
     auto filter = std::make_shared(config);
diff --git a/contrib/language/filters/http/source/language_filter.cc b/contrib/language/filters/http/source/language_filter.cc
index 5108dd28a960..647398a02ec2 100644
--- a/contrib/language/filters/http/source/language_filter.cc
+++ b/contrib/language/filters/http/source/language_filter.cc
@@ -31,9 +31,10 @@ bool validateHeaderValue(absl::string_view value_str) {
 
 LanguageFilterConfigImpl::LanguageFilterConfigImpl(
     const std::shared_ptr default_locale,
-    const std::shared_ptr locale_matcher, const std::string& stats_prefix,
-    Stats::Scope& scope)
+    const std::shared_ptr locale_matcher, const bool clear_route_cache,
+    const std::string& stats_prefix, Stats::Scope& scope)
     : default_locale_(std::move(default_locale)), locale_matcher_(std::move(locale_matcher)),
+      clear_route_cache_(clear_route_cache),
       stats_(LanguageFilter::generateStats(stats_prefix, scope)) {}
 
 const std::shared_ptr& LanguageFilterConfigImpl::defaultLocale() const {
@@ -44,6 +45,8 @@ const std::shared_ptr& LanguageFilterConfigImpl::localeMatch
   return locale_matcher_;
 }
 
+bool LanguageFilterConfigImpl::clearRouteCache() const { return clear_route_cache_; }
+
 LanguageStats& LanguageFilterConfigImpl::stats() { return stats_; }
 
 LanguageFilter::LanguageFilter(const LanguageFilterConfigSharedPtr config)
@@ -76,6 +79,12 @@ Http::FilterHeadersStatus LanguageFilter::decodeHeaders(Http::RequestHeaderMap&
         if (U_SUCCESS(errorCode)) {
           if (!language_tag.empty()) {
             request_headers.addCopy(Language, language_tag);
+
+            if (config_->clearRouteCache()) {
+              ENVOY_LOG(debug, "clearing route cache");
+              decoder_callbacks_->downstreamCallbacks()->clearRouteCache();
+            }
+
             config_->stats().header_.inc();
 
             return Http::FilterHeadersStatus::Continue;
diff --git a/contrib/language/filters/http/source/language_filter.h b/contrib/language/filters/http/source/language_filter.h
index 738ce9803bb5..b4cc1adbb478 100644
--- a/contrib/language/filters/http/source/language_filter.h
+++ b/contrib/language/filters/http/source/language_filter.h
@@ -41,6 +41,8 @@ class LanguageFilterConfig {
 
   virtual const std::shared_ptr& localeMatcher() const PURE;
 
+  virtual bool clearRouteCache() const PURE;
+
   virtual LanguageStats& stats() PURE;
 };
 
@@ -51,12 +53,15 @@ class LanguageFilterConfigImpl : public LanguageFilterConfig {
 public:
   LanguageFilterConfigImpl(const std::shared_ptr default_locale,
                            const std::shared_ptr locale_matcher,
-                           const std::string& stats_prefix, Stats::Scope& scope);
+                           const bool clear_route_cache, const std::string& stats_prefix,
+                           Stats::Scope& scope);
 
   const std::shared_ptr& defaultLocale() const override;
 
   const std::shared_ptr& localeMatcher() const override;
 
+  bool clearRouteCache() const override;
+
   LanguageStats& stats() override;
 
 private:
@@ -64,6 +69,8 @@ class LanguageFilterConfigImpl : public LanguageFilterConfig {
 
   const std::shared_ptr locale_matcher_;
 
+  const bool clear_route_cache_;
+
   LanguageStats stats_;
 };
 using LanguageFilterConfigSharedPtr = std::shared_ptr;

From 5e05a494c75aeb112661584975f4bd668cc923c6 Mon Sep 17 00:00:00 2001
From: Ali Beyad 
Date: Thu, 8 Jun 2023 16:17:30 -0400
Subject: [PATCH 505/740] test: Fix comment (#27884)

Signed-off-by: Ali Beyad 
---
 test/integration/base_integration_test.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/base_integration_test.h b/test/integration/base_integration_test.h
index fddd40fbff4e..0f750483b4a6 100644
--- a/test/integration/base_integration_test.h
+++ b/test/integration/base_integration_test.h
@@ -531,7 +531,7 @@ class BaseIntegrationTest : protected Logger::Loggable {
 private:
   // Configuration for the fake upstream.
   FakeUpstreamConfig upstream_config_{time_system_};
-  // True if initialized() has been called.
+  // True if initialize() has been called.
   bool initialized_{};
   // Optional factory that the proxy-under-test should use to create watermark buffers. If nullptr,
   // the proxy uses the default watermark buffer factory to create buffers.

From b8e112190ef14bced0509a0fb201b5ee49da46d7 Mon Sep 17 00:00:00 2001
From: code 
Date: Fri, 9 Jun 2023 05:55:47 +0800
Subject: [PATCH 506/740] generic proxy: add new request matcher to simplify
 route table (#27785)

A new custom matcher for generic proxy is added to simplify the route table. When simple AND semantic is used, the users needn't write complex configuration to combine different input/match.

Risk Level: low.
Testing: unit.

Signed-off-by: wbpcode 
---
 .../generic_proxy/matcher/v3/matcher.proto    |   2 -
 .../filters/network/source/match.cc           |  88 +++++++++
 .../filters/network/source/match.h            |  84 +++++++++
 .../filters/network/test/match_test.cc        | 174 ++++++++++++++++++
 4 files changed, 346 insertions(+), 2 deletions(-)

diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto
index 021d1f85346a..47dcc212f439 100644
--- a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto
+++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto
@@ -52,7 +52,6 @@ message PropertyMatchInput {
 message RequestMatchInput {
 }
 
-// [#not-implemented-hide:]
 // Used to match an arbitrary key-value pair for headers, trailers or properties.
 message KeyValueMatchEntry {
   // The key name to match on.
@@ -62,7 +61,6 @@ message KeyValueMatchEntry {
   type.matcher.v3.StringMatcher string_match = 2 [(validate.rules).message = {required: true}];
 }
 
-// [#not-implemented-hide:]
 // Custom matcher to match on the generic downstream request. This is used to match
 // multiple fields of the downstream request and avoid complex combinations of
 // HostMatchInput, PathMatchInput, MethodMatchInput and PropertyMatchInput.
diff --git a/contrib/generic_proxy/filters/network/source/match.cc b/contrib/generic_proxy/filters/network/source/match.cc
index 37ddc4bd15fa..83719c2b6c4d 100644
--- a/contrib/generic_proxy/filters/network/source/match.cc
+++ b/contrib/generic_proxy/filters/network/source/match.cc
@@ -15,6 +15,94 @@ REGISTER_FACTORY(MethodMatchDataInputFactory, Matcher::DataInputFactory
 
 REGISTER_FACTORY(PropertyMatchDataInputFactory, Matcher::DataInputFactory);
 
+REGISTER_FACTORY(RequestMatchDataInputFactory, Matcher::DataInputFactory);
+
+using StringMatcherImpl = Matchers::StringMatcherImpl;
+
+RequestMatchInputMatcher::RequestMatchInputMatcher(const RequestMatcherProto& proto_config) {
+
+  if (proto_config.has_host()) {
+    host_ = std::make_unique(proto_config.host());
+  }
+  if (proto_config.has_path()) {
+    path_ = std::make_unique(proto_config.path());
+  }
+  if (proto_config.has_method()) {
+    method_ = std::make_unique(proto_config.method());
+  }
+
+  for (const auto& property : proto_config.properties()) {
+    properties_.push_back(
+        {property.name(), std::make_unique(property.string_match())});
+  }
+}
+
+bool RequestMatchInputMatcher::match(const Matcher::MatchingDataType& input) {
+  if (!absl::holds_alternative>(input)) {
+    return false;
+  }
+
+  const auto* typed_data = dynamic_cast(
+      absl::get>(input).get());
+
+  if (typed_data == nullptr) {
+    return false;
+  }
+
+  return match(typed_data->request());
+}
+
+bool RequestMatchInputMatcher::match(const Request& request) {
+  // TODO(wbpcode): may add more debug log for request match?
+  if (host_ != nullptr) {
+    if (!host_->match(request.host())) {
+      // Host does not match.
+      return false;
+    }
+  }
+
+  if (path_ != nullptr) {
+    if (!path_->match(request.path())) {
+      // Path does not match.
+      return false;
+    }
+  }
+
+  if (method_ != nullptr) {
+    if (!method_->match(request.method())) {
+      // Method does not match.
+      return false;
+    }
+  }
+
+  for (const auto& property : properties_) {
+    if (auto val = request.getByKey(property.first); val.has_value()) {
+      if (!property.second->match(val.value())) {
+        // Property does not match.
+        return false;
+      }
+    } else {
+      // Property does not exist.
+      return false;
+    }
+  }
+
+  // All matchers passed.
+  return true;
+}
+
+Matcher::InputMatcherFactoryCb RequestMatchDataInputMatcherFactory::createInputMatcherFactoryCb(
+    const Protobuf::Message& config, Server::Configuration::ServerFactoryContext& factory_context) {
+  const auto& proto_config = MessageUtil::downcastAndValidate(
+      config, factory_context.messageValidationVisitor());
+
+  return [proto_config]() -> Matcher::InputMatcherPtr {
+    return std::make_unique(proto_config);
+  };
+}
+
+REGISTER_FACTORY(RequestMatchDataInputMatcherFactory, Matcher::InputMatcherFactory);
+
 } // namespace GenericProxy
 } // namespace NetworkFilters
 } // namespace Extensions
diff --git a/contrib/generic_proxy/filters/network/source/match.h b/contrib/generic_proxy/filters/network/source/match.h
index d339f4c54ef5..447e44a4a7c6 100644
--- a/contrib/generic_proxy/filters/network/source/match.h
+++ b/contrib/generic_proxy/filters/network/source/match.h
@@ -24,6 +24,16 @@ using MethodDataInputProto =
     envoy::extensions::filters::network::generic_proxy::matcher::v3::MethodMatchInput;
 using PropertyDataInputProto =
     envoy::extensions::filters::network::generic_proxy::matcher::v3::PropertyMatchInput;
+using RequestInputProto =
+    envoy::extensions::filters::network::generic_proxy::matcher::v3::RequestMatchInput;
+using RequestMatcherProto =
+    envoy::extensions::filters::network::generic_proxy::matcher::v3::RequestMatcher;
+using StringMatcherProto = envoy::type::matcher::v3::StringMatcher;
+
+// Fully qualified name of the generic proxy request match data type to avoid any possible
+// collision with other match data types.
+inline constexpr absl::string_view GenericRequestMatcheInputType =
+    "Envoy::Extensions::NetworkFilters::GenericProxy::RequestMatchData";
 
 class ServiceMatchDataInput : public Matcher::DataInput {
 public:
@@ -158,6 +168,80 @@ class PropertyMatchDataInputFactory : public Matcher::DataInputFactory
   std::string name() const override { return "envoy.matching.generic_proxy.input.property"; }
 };
 
+// RequestMatchData is a wrapper of Request to be used as the matching data type.
+class RequestMatchData : public Matcher::CustomMatchData {
+public:
+  RequestMatchData(const Request& data) : data_(data) {}
+
+  const Request& request() const { return data_; }
+
+private:
+  const Request& data_;
+};
+
+class RequestMatchDataInput : public Matcher::DataInput {
+public:
+  RequestMatchDataInput() = default;
+
+  Matcher::DataInputGetResult get(const Request& data) const override {
+    auto request = std::make_shared(data);
+    return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable,
+            Matcher::MatchingDataType{std::move(request)}};
+  }
+
+  absl::string_view dataInputType() const override { return GenericRequestMatcheInputType; }
+};
+
+class RequestMatchDataInputFactory : public Matcher::DataInputFactory {
+public:
+  RequestMatchDataInputFactory() = default;
+
+  ProtobufTypes::MessagePtr createEmptyConfigProto() override {
+    return std::make_unique();
+  }
+
+  Matcher::DataInputFactoryCb
+  createDataInputFactoryCb(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override {
+    return []() { return std::make_unique(); };
+  }
+
+  std::string name() const override { return "envoy.matching.generic_proxy.input.request"; }
+};
+
+class RequestMatchInputMatcher : public Matcher::InputMatcher {
+public:
+  RequestMatchInputMatcher(const RequestMatcherProto& config);
+
+  bool match(const Matcher::MatchingDataType& input) override;
+
+  bool match(const Request& request);
+
+  absl::flat_hash_set supportedDataInputTypes() const override {
+    return absl::flat_hash_set{std::string(GenericRequestMatcheInputType)};
+  }
+
+private:
+  Matchers::StringMatcherPtr host_;
+  Matchers::StringMatcherPtr path_;
+  Matchers::StringMatcherPtr method_;
+  std::vector> properties_;
+};
+
+class RequestMatchDataInputMatcherFactory : public Matcher::InputMatcherFactory {
+public:
+  Matcher::InputMatcherFactoryCb createInputMatcherFactoryCb(
+      const Protobuf::Message& config,
+      Server::Configuration::ServerFactoryContext& factory_context) override;
+
+  ProtobufTypes::MessagePtr createEmptyConfigProto() override {
+    return std::make_unique();
+  }
+
+  std::string name() const override {
+    return "envoy.matching.input_matchers.generic_request_matcher";
+  }
+};
+
 } // namespace GenericProxy
 } // namespace NetworkFilters
 } // namespace Extensions
diff --git a/contrib/generic_proxy/filters/network/test/match_test.cc b/contrib/generic_proxy/filters/network/test/match_test.cc
index 447810ad13dd..e2daf8a0d55d 100644
--- a/contrib/generic_proxy/filters/network/test/match_test.cc
+++ b/contrib/generic_proxy/filters/network/test/match_test.cc
@@ -1,3 +1,5 @@
+#include 
+
 #include "test/mocks/server/factory_context.h"
 
 #include "contrib/generic_proxy/filters/network/source/match.h"
@@ -97,6 +99,178 @@ TEST(PropertyMatchDataInputTest, PropertyMatchDataInputTest) {
   EXPECT_EQ("value_0", absl::get(input->get(request).data_));
 }
 
+TEST(RequestMatchDataInputTest, RequestMatchDataInputTest) {
+  NiceMock factory_context;
+  RequestMatchDataInputFactory factory;
+  auto proto_config = factory.createEmptyConfigProto();
+  auto input =
+      factory.createDataInputFactoryCb(*proto_config, factory_context.messageValidationVisitor())();
+
+  FakeStreamCodecFactory::FakeRequest request;
+
+  EXPECT_EQ(
+      &request,
+      &dynamic_cast(
+           absl::get>(input->get(request).data_).get())
+           ->request());
+}
+
+TEST(RequestMatchInputMatcherTest, RequestMatchInputMatcherTest) {
+  NiceMock factory_context;
+  RequestMatchDataInputMatcherFactory factory;
+  auto proto_config = factory.createEmptyConfigProto();
+  auto matcher = factory.createInputMatcherFactoryCb(*proto_config,
+                                                     factory_context.getServerFactoryContext())();
+
+  {
+    Matcher::MatchingDataType input;
+    EXPECT_FALSE(matcher->match(input));
+  }
+
+  {
+    Matcher::MatchingDataType input = std::string("fake_data");
+    EXPECT_FALSE(matcher->match(input));
+  }
+
+  {
+    FakeStreamCodecFactory::FakeRequest request;
+    Matcher::MatchingDataType input = std::make_shared(request);
+    EXPECT_TRUE(matcher->match(input));
+  }
+}
+
+TEST(RequestMatchInputMatcherTest, SpecificRequestMatchInputMatcherTest) {
+  // Empty matcher.
+  {
+    RequestMatcherProto matcher_proto;
+    RequestMatchInputMatcher matcher(matcher_proto);
+
+    FakeStreamCodecFactory::FakeRequest request;
+    EXPECT_TRUE(matcher.match(request));
+  }
+
+  // Host match failed.
+  {
+    RequestMatcherProto matcher_proto;
+
+    const std::string config_yaml = R"EOF(
+    host:
+      exact: fake_host
+    )EOF";
+
+    TestUtility::loadFromYaml(config_yaml, matcher_proto);
+
+    RequestMatchInputMatcher matcher(matcher_proto);
+
+    FakeStreamCodecFactory::FakeRequest request;
+    request.host_ = "another_fake_host";
+    EXPECT_FALSE(matcher.match(request));
+  }
+
+  // Path match failed.
+  {
+    RequestMatcherProto matcher_proto;
+
+    const std::string config_yaml = R"EOF(
+    host:
+      exact: fake_host
+    path:
+      exact: fake_path
+    )EOF";
+
+    TestUtility::loadFromYaml(config_yaml, matcher_proto);
+
+    RequestMatchInputMatcher matcher(matcher_proto);
+
+    FakeStreamCodecFactory::FakeRequest request;
+    request.host_ = "fake_host";
+    request.path_ = "another_fake_path";
+    EXPECT_FALSE(matcher.match(request));
+  }
+
+  // Method match failed.
+  {
+    RequestMatcherProto matcher_proto;
+
+    const std::string config_yaml = R"EOF(
+    host:
+      exact: fake_host
+    path:
+      exact: fake_path
+    method:
+      exact: fake_method
+    )EOF";
+
+    TestUtility::loadFromYaml(config_yaml, matcher_proto);
+
+    RequestMatchInputMatcher matcher(matcher_proto);
+
+    FakeStreamCodecFactory::FakeRequest request;
+    request.host_ = "fake_host";
+    request.path_ = "fake_path";
+    request.method_ = "another_fake_method";
+    EXPECT_FALSE(matcher.match(request));
+  }
+
+  // Property match failed.
+  {
+    RequestMatcherProto matcher_proto;
+
+    const std::string config_yaml = R"EOF(
+    host:
+      exact: fake_host
+    path:
+      exact: fake_path
+    method:
+      exact: fake_method
+    properties:
+      - name: key_0
+        string_match:
+          exact: value_0
+    )EOF";
+
+    TestUtility::loadFromYaml(config_yaml, matcher_proto);
+
+    RequestMatchInputMatcher matcher(matcher_proto);
+
+    FakeStreamCodecFactory::FakeRequest request;
+    request.host_ = "fake_host";
+    request.path_ = "fake_path";
+    request.method_ = "fake_method";
+    request.data_["key_0"] = "another_value_0";
+    EXPECT_FALSE(matcher.match(request));
+  }
+
+  // All match.
+  {
+    RequestMatcherProto matcher_proto;
+
+    const std::string config_yaml = R"EOF(
+    host:
+      exact: fake_host
+    path:
+      exact: fake_path
+    method:
+      exact: fake_method
+    properties:
+      - name: key_0
+        string_match:
+          exact: value_0
+    )EOF";
+
+    TestUtility::loadFromYaml(config_yaml, matcher_proto);
+
+    RequestMatchInputMatcher matcher(matcher_proto);
+
+    FakeStreamCodecFactory::FakeRequest request;
+    request.host_ = "fake_host";
+    request.path_ = "fake_path";
+    request.method_ = "fake_method";
+    request.data_["key_0"] = "value_0";
+    EXPECT_TRUE(matcher.match(request));
+  }
+}
+
 } // namespace
 } // namespace GenericProxy
 } // namespace NetworkFilters

From ddc50bb6b07269ba499a93af77d85e141d534882 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 9 Jun 2023 07:40:04 -0700
Subject: [PATCH 507/740] build(deps): bump urllib3 from 2.0.2 to 2.0.3 in
 /mobile/docs (#27893)

Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.0.2 to 2.0.3.
- [Release notes](https://github.com/urllib3/urllib3/releases)
- [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst)
- [Commits](https://github.com/urllib3/urllib3/compare/2.0.2...2.0.3)

---
updated-dependencies:
- dependency-name: urllib3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 mobile/docs/requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mobile/docs/requirements.txt b/mobile/docs/requirements.txt
index a5feef288926..e1cec53555c7 100644
--- a/mobile/docs/requirements.txt
+++ b/mobile/docs/requirements.txt
@@ -197,6 +197,6 @@ sphinxcontrib-qthelp==1.0.3 \
 sphinxcontrib-serializinghtml==1.1.5 \
     --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \
     --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952
-urllib3==2.0.2 \
-    --hash=sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc \
-    --hash=sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e
+urllib3==2.0.3 \
+    --hash=sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1 \
+    --hash=sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825

From 52cfd445ffe20110c5c75dbc8aa00f8edb82864d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 9 Jun 2023 07:40:47 -0700
Subject: [PATCH 508/740] build(deps): bump jaegertracing/all-in-one from
 `12e96c7` to `3703087` in /examples/shared/jaeger (#27894)

build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger

Bumps jaegertracing/all-in-one from `12e96c7` to `3703087`.

---
updated-dependencies:
- dependency-name: jaegertracing/all-in-one
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/jaeger/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile
index 5c671c6b6cad..27d9f94cafcf 100644
--- a/examples/shared/jaeger/Dockerfile
+++ b/examples/shared/jaeger/Dockerfile
@@ -1,4 +1,4 @@
-FROM jaegertracing/all-in-one@sha256:12e96c7396d758da7d300d162f2aab899c2f6cc51e037cc7bced8dbd6d3cfc5d
+FROM jaegertracing/all-in-one@sha256:37030876f8e4d42ed793c545b9420afebc46d9fb303affcec46c4b60b97e3c4a
 HEALTHCHECK \
     --interval=1s \
     --timeout=1s \

From de3ae4a6e996ef7e6356c62b44990a6cf01e3a1c Mon Sep 17 00:00:00 2001
From: jaychenatr <54647402+jaychenatr@users.noreply.github.com>
Date: Fri, 9 Jun 2023 12:43:45 -0500
Subject: [PATCH 509/740] api-csds: Propose a boolean field in
 ClientStatusRequest to indicate whether xds server should exclude detailed
 config(xds_config field in GenericXdsConfig) but keep the rest of fields in
 response (#27880)

Signed-off-by: jaychenatr 
---
 api/envoy/service/status/v3/csds.proto | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/api/envoy/service/status/v3/csds.proto b/api/envoy/service/status/v3/csds.proto
index 74b7005190d8..b764f1cc3a4f 100644
--- a/api/envoy/service/status/v3/csds.proto
+++ b/api/envoy/service/status/v3/csds.proto
@@ -85,6 +85,11 @@ message ClientStatusRequest {
 
   // The node making the csds request.
   config.core.v3.Node node = 2;
+
+  // If true, the server will not include the resource contents in the response
+  // (i.e., the generic_xds_configs.xds_config field will not be populated).
+  // [#not-implemented-hide:]
+  bool exclude_resource_contents = 3;
 }
 
 // Detailed config (per xDS) with status.

From 12823e7718f8d63d51f317c01ede0379c2dd0aa6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Jun 2023 07:45:47 +0000
Subject: [PATCH 510/740] build(deps): bump orjson from 3.9.0 to 3.9.1 in
 /tools/base (#27918)

Bumps [orjson](https://github.com/ijl/orjson) from 3.9.0 to 3.9.1.
- [Release notes](https://github.com/ijl/orjson/releases)
- [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ijl/orjson/compare/3.9.0...3.9.1)

---
updated-dependencies:
- dependency-name: orjson
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tools/base/requirements.txt | 94 ++++++++++++++++++-------------------
 1 file changed, 47 insertions(+), 47 deletions(-)

diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt
index b60e9117dd11..689b131a2d73 100644
--- a/tools/base/requirements.txt
+++ b/tools/base/requirements.txt
@@ -927,53 +927,53 @@ oauth2client==4.1.3 \
     # via
     #   gcs-oauth2-boto-plugin
     #   google-apitools
-orjson==3.9.0 \
-    --hash=sha256:04e61db09ff155846b69d07cf5aa21001f2010ea669ec3169c1fbad9c9e40cd5 \
-    --hash=sha256:08cb43569198c1f5c89ecafcbfc62414f6115d894ff908d8cf8e5e24801364e6 \
-    --hash=sha256:09522937479bd39d5bb32d11a5ecdf6926fda43ac2cbde21cc1a9508b4e4ea29 \
-    --hash=sha256:09ee828572fadcd58bf356d2c1bad99a95c7c9c1f182b407abbc7dec1810f542 \
-    --hash=sha256:0e7fe5d603ee9177ff2e45858b4fc47fea2da0688f23d9773654889d56dfbc82 \
-    --hash=sha256:108c58d2c7648c991f82f9b2217c50981ad7cf6aaee3efbfaa9d807e49cd69b8 \
-    --hash=sha256:128b1cd0f00a37ba64a12cceeba4e8070655d4400edd55a737513ee663c1ed5a \
-    --hash=sha256:1e3bde77c1e0061eb34bae6fea44818b2198e043ee10a16ad7b160921fee26ea \
-    --hash=sha256:21f6a6fdfbc13cd715c61e9fa9daeff732df6401ab7d6a2ebad0042313a40bd1 \
-    --hash=sha256:2536a7f30fd4d77532769ea9285cd20c69bd2b40acf980de94bbc79b1c6fad5a \
-    --hash=sha256:271b6f1018757fc6bca40ae72e6cdb6cf84584dde2d1e5eaac30e387a13d9e72 \
-    --hash=sha256:2af7dff1c7ddb0c83eb5773acf6566b153f8cd32e4ba782ae9ccd6d0f324efd3 \
-    --hash=sha256:3235c31d0fe674f6e3433e9ddfed212aa840c83a9b6ef5ae128950e2c808c303 \
-    --hash=sha256:3a208d0bca609de3152eb8320d5093ad9c52979332f626c13500d1645c66bf8d \
-    --hash=sha256:3f1193417b5a93deb41bcb8db27b61179b9b3e299b337b578c31f19159664da3 \
-    --hash=sha256:44fa74b497e608a8cdca1ee37fe3533a30f17163c7e2872ab1b854900cf0dfcf \
-    --hash=sha256:45df5bf6531ffda518331cc93cdcd4c84f4a4a0507d72af8fb698c7131a440a0 \
-    --hash=sha256:46c9733330b75c116438f555c0b971a2388b5f502e2dd4ec3bf6bacb96f82741 \
-    --hash=sha256:47d7e4a3effc0e9314bd5b06e7431f2490a5e64dcdcbbc4d60e713786fec327d \
-    --hash=sha256:5afd22847b07b63f2b8fcfddd5b7a6f47c5aaa25e19b97a3d6d39508b8fd465a \
-    --hash=sha256:6c50654e4870805e4b1a587c2c3c5ef2f36f3e67fc463a738339ff40d65f7db1 \
-    --hash=sha256:721d47dffedb7795ffea8a06f2de7d192de7b58e085cf357a99abf0eb931f2c3 \
-    --hash=sha256:748c1e8df0b0880c63d323e167ad17ab4db2e1178a40902c2fcb68cbe402d7c8 \
-    --hash=sha256:7a3693fde44b2eeb80074ecbe8c504b25baf71e66c080af2a574193a5ba81960 \
-    --hash=sha256:86da00836029b2a071229c8aecab998a2f316c1bc7de10ae020d7311de3a6d0d \
-    --hash=sha256:88626d898c408450c57664899831cf072787898af4847fa4466607ad2a83f454 \
-    --hash=sha256:8a1fcddcabe121e393f3c4a31ed6d3535214d42a4ece0f9dde2e250006d6a58d \
-    --hash=sha256:949698bdddb1daff986d73e6bbe6cd68833cd80c4adc6b69fafbd46634d4672c \
-    --hash=sha256:9de2129d40674007cb24164939e075b5b39fee768bf20801e08c0e3283bfb18e \
-    --hash=sha256:9ee5f1ba82146a50d61fb58d310a37c0f406eda898172f9c98673b5d6f9461c3 \
-    --hash=sha256:a901c432828c191332d75f358142736c433d4a192f7794123e1d30d68193de86 \
-    --hash=sha256:bd89d63707ac616462832bfc5d16fa0c12483f86add2432ce55c8710c9531c03 \
-    --hash=sha256:c41d1ef6ec308e9e3701764b3de889ed8c1c126eceaea881dd1027bffbed89fe \
-    --hash=sha256:c4949fc1304b702197c0840882e84b86d8d5ca33c3d945cc60727bc1786c2b20 \
-    --hash=sha256:c68af71b1110820c914f9df75842895b5528ff524d3286fde57097b2b5ed8f22 \
-    --hash=sha256:c7b241c3229084035b38cac9b5c96b43644da829da41d9d5be0fefb96fb116e1 \
-    --hash=sha256:d2fbf34667a8be48ec89d5ef479a00d4e7b3acda62d722c97377702da0c30ffd \
-    --hash=sha256:d414fd0678e949779104f5b307f0f9fac861728e19d3cdde66759af77f892da0 \
-    --hash=sha256:d4c2d31178e3027affd98eead033f1c406890df83a0ca2016604cc21f722a1d1 \
-    --hash=sha256:d4fcf598bd5a99a94caa7ec92ce657939f12491e4753ea7e4d6c03faf5f7912e \
-    --hash=sha256:e44ebe2129d43c5a48f3affa3fa59c6484ed16faf5b00486add1061a95384ab0 \
-    --hash=sha256:ebe372e9f4e4f0335b7b4ebfab991b3734371e3d5b7f989ca3baa5da25185f4a \
-    --hash=sha256:edd77183c154cbedaa6dac32fee9cb770b04e2a7f367a5864f444578554cc946 \
-    --hash=sha256:f6476e2487c0b7387187de15e5b8f6635c29b75934f2e689ca8cad6550439f3d \
-    --hash=sha256:f6ab80b60195f166a9d666b2eaf6d2c74202b6da2a1fb4b4d66b9cc0ce5c9957 \
-    --hash=sha256:f6dd27c71cd6e146795f876449a8eae74f67ae1e4e244dfc1203489103eb2d94
+orjson==3.9.1 \
+    --hash=sha256:06f6ab4697fab090517f295915318763a97a12ee8186054adf21c1e6f6abbd3d \
+    --hash=sha256:08927970365d2e1f3ce4894f9ff928a7b865d53f26768f1bbdd85dd4fee3e966 \
+    --hash=sha256:09faf14f74ed47e773fa56833be118e04aa534956f661eb491522970b7478e3b \
+    --hash=sha256:0b53b5f72cf536dd8aa4fc4c95e7e09a7adb119f8ff8ee6cc60f735d7740ad6a \
+    --hash=sha256:0b7ab18d55ecb1de543d452f0a5f8094b52282b916aa4097ac11a4c79f317b86 \
+    --hash=sha256:0fd828e0656615a711c4cc4da70f3cac142e66a6703ba876c20156a14e28e3fa \
+    --hash=sha256:103952c21575b9805803c98add2eaecd005580a1e746292ed2ec0d76dd3b9746 \
+    --hash=sha256:125f63e56d38393daa0a1a6dc6fedefca16c538614b66ea5997c3bd3af35ef26 \
+    --hash=sha256:15d28872fb055bf17ffca913826e618af61b2f689d2b170f72ecae1a86f80d52 \
+    --hash=sha256:19f70ba1f441e1c4bb1a581f0baa092e8b3e3ce5b2aac2e1e090f0ac097966da \
+    --hash=sha256:1e4d905338f9ef32c67566929dfbfbb23cc80287af8a2c38930fb0eda3d40b76 \
+    --hash=sha256:20f2804b5a1dbd3609c086041bd243519224d47716efd7429db6c03ed28b7cc3 \
+    --hash=sha256:24d4ddaa2876e657c0fd32902b5c451fd2afc35159d66a58da7837357044b8c2 \
+    --hash=sha256:2cb0121e6f2c9da3eddf049b99b95fef0adf8480ea7cb544ce858706cdf916eb \
+    --hash=sha256:31229f9d0b8dc2ef7ee7e4393f2e4433a28e16582d4b25afbfccc9d68dc768f8 \
+    --hash=sha256:375d65f002e686212aac42680aed044872c45ee4bc656cf63d4a215137a6124a \
+    --hash=sha256:393d0697d1dfa18d27d193e980c04fdfb672c87f7765b87952f550521e21b627 \
+    --hash=sha256:402f9d3edfec4560a98880224ec10eba4c5f7b4791e4bc0d4f4d8df5faf2a006 \
+    --hash=sha256:46b4facc32643b2689dfc292c0c463985dac4b6ab504799cf51fc3c6959ed668 \
+    --hash=sha256:4751cee4a7b1daeacb90a7f5adf2170ccab893c3ab7c5cea58b45a13f89b30b3 \
+    --hash=sha256:48a27da6c7306965846565cc385611d03382bbd84120008653aa2f6741e2105d \
+    --hash=sha256:49c0d78dcd34626e2e934f1192d7c052b94e0ecadc5f386fd2bda6d2e03dadf5 \
+    --hash=sha256:503eb86a8d53a187fe66aa80c69295a3ca35475804da89a9547e4fce5f803822 \
+    --hash=sha256:5d1dbf36db7240c61eec98c8d21545d671bce70be0730deb2c0d772e06b71af3 \
+    --hash=sha256:6d173d3921dd58a068c88ec22baea7dbc87a137411501618b1292a9d6252318e \
+    --hash=sha256:761b6efd33c49de20dd73ce64cc59da62c0dab10aa6015f582680e0663cc792c \
+    --hash=sha256:78d9a2a4b2302d5ebc3695498ebc305c3568e5ad4f3501eb30a6405a32d8af22 \
+    --hash=sha256:80a1e384626f76b66df615f7bb622a79a25c166d08c5d2151ffd41f24c4cc104 \
+    --hash=sha256:8515867713301fa065c58ec4c9053ba1a22c35113ab4acad555317b8fd802e50 \
+    --hash=sha256:9e20bca5e13041e31ceba7a09bf142e6d63c8a7467f5a9c974f8c13377c75af2 \
+    --hash=sha256:a4cc5d21e68af982d9a2528ac61e604f092c60eed27aef3324969c68f182ec7e \
+    --hash=sha256:ae47ef8c0fe89c4677db7e9e1fb2093ca6e66c3acbee5442d84d74e727edad5e \
+    --hash=sha256:c4434b7b786fdc394b95d029fb99949d7c2b05bbd4bf5cb5e3906be96ffeee3b \
+    --hash=sha256:d1c2b0b4246c992ce2529fc610a446b945f1429445ece1c1f826a234c829a918 \
+    --hash=sha256:d3a40b0fbe06ccd4d6a99e523d20b47985655bcada8d1eba485b1b32a43e4904 \
+    --hash=sha256:d4b68d01a506242316a07f1d2f29fb0a8b36cee30a7c35076f1ef59dce0890c1 \
+    --hash=sha256:d4edee78503016f4df30aeede0d999b3cb11fb56f47e9db0e487bce0aaca9285 \
+    --hash=sha256:d8ae0467d01eb1e4bcffef4486d964bfd1c2e608103e75f7074ed34be5df48cc \
+    --hash=sha256:d96747662d3666f79119e5d28c124e7d356c7dc195cd4b09faea4031c9079dc9 \
+    --hash=sha256:d9dd4abe6c6fd352f00f4246d85228f6a9847d0cc14f4d54ee553718c225388f \
+    --hash=sha256:db373a25ec4a4fccf8186f9a72a1b3442837e40807a736a815ab42481e83b7d0 \
+    --hash=sha256:db774344c39041f4801c7dfe03483df9203cbd6c84e601a65908e5552228dd25 \
+    --hash=sha256:e186ae76b0d97c505500664193ddf508c13c1e675d9b25f1f4414a7606100da6 \
+    --hash=sha256:ec53d648176f873203b9c700a0abacab33ca1ab595066e9d616f98cdc56f4434 \
+    --hash=sha256:ec7c8a0f1bf35da0d5fd14f8956f3b82a9a6918a3c6963d718dfd414d6d3b604 \
+    --hash=sha256:f9a744e212d4780ecd67f4b6b128b2e727bee1df03e7059cddb2dfe1083e7dc4
     # via
     #   -r requirements.in
     #   envoy-base-utils

From 89e0eb886e0cf24351dd3df8f2295f92abfda2e8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Jun 2023 07:45:56 +0000
Subject: [PATCH 511/740] build(deps): bump github/codeql-action from 2.3.6 to
 2.13.4 (#27916)

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.3.6 to 2.13.4.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/83f0fe6c4988d98a455712a27f0255212bba9bd4...cdcdbb579706841c47f7063dda365e292e5cad7a)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/codeql-daily.yml | 4 ++--
 .github/workflows/codeql-push.yml  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml
index 7c069b38723f..b0977506656c 100644
--- a/.github/workflows/codeql-daily.yml
+++ b/.github/workflows/codeql-daily.yml
@@ -33,7 +33,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@83f0fe6c4988d98a455712a27f0255212bba9bd4
+      uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a
       # Override language selection by uncommenting this and choosing your languages
       with:
         languages: cpp
@@ -64,4 +64,4 @@ jobs:
         git clean -xdf
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@83f0fe6c4988d98a455712a27f0255212bba9bd4
+      uses: github/codeql-action/analyze@cdcdbb579706841c47f7063dda365e292e5cad7a
diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml
index f303a369c068..d6461e1878bb 100644
--- a/.github/workflows/codeql-push.yml
+++ b/.github/workflows/codeql-push.yml
@@ -45,7 +45,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@83f0fe6c4988d98a455712a27f0255212bba9bd4
+      uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a
       # Override language selection by uncommenting this and choosing your languages
       with:
         languages: cpp
@@ -78,4 +78,4 @@ jobs:
 
     - name: Perform CodeQL Analysis
       if: env.BUILD_TARGETS != ''
-      uses: github/codeql-action/analyze@83f0fe6c4988d98a455712a27f0255212bba9bd4
+      uses: github/codeql-action/analyze@cdcdbb579706841c47f7063dda365e292e5cad7a

From c70aa7bf03f847e43695d8929adf8eff54b6d524 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Jun 2023 07:51:42 +0000
Subject: [PATCH 512/740] build(deps): bump envoyproxy/toolshed from
 actions-v0.0.1 to 0.0.2 (#27917)

Bumps [envoyproxy/toolshed](https://github.com/envoyproxy/toolshed) from actions-v0.0.1 to 0.0.2. This release includes the previously tagged commit.
- [Release notes](https://github.com/envoyproxy/toolshed/releases)
- [Commits](https://github.com/envoyproxy/toolshed/compare/actions-v0.0.1...actions-v0.0.2)

---
updated-dependencies:
- dependency-name: envoyproxy/toolshed
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/commands.yml          | 2 +-
 .github/workflows/envoy-sync.yml        | 2 +-
 .github/workflows/workflow-complete.yml | 2 +-
 .github/workflows/workflow-start.yml    | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml
index 63af84c63af5..5fc807e0c3c2 100644
--- a/.github/workflows/commands.yml
+++ b/.github/workflows/commands.yml
@@ -22,6 +22,6 @@ jobs:
       pull-requests: write
       actions: write
     steps:
-    - uses: envoyproxy/toolshed/gh-actions/retest@actions-v0.0.1
+    - uses: envoyproxy/toolshed/gh-actions/retest@actions-v0.0.2
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/envoy-sync.yml b/.github/workflows/envoy-sync.yml
index ce191989bf3a..8fcfc67e1c4b 100644
--- a/.github/workflows/envoy-sync.yml
+++ b/.github/workflows/envoy-sync.yml
@@ -20,7 +20,7 @@ jobs:
         - envoy-filter-example
         - data-plane-api
     steps:
-    - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.0.1
+    - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.0.2
       with:
         repository: "envoyproxy/${{ matrix.downstream }}"
         ref: main
diff --git a/.github/workflows/workflow-complete.yml b/.github/workflows/workflow-complete.yml
index 7b6050b9d75a..12a7363fe127 100644
--- a/.github/workflows/workflow-complete.yml
+++ b/.github/workflows/workflow-complete.yml
@@ -52,7 +52,7 @@ jobs:
         echo "state=${STATE}" >> "$GITHUB_OUTPUT"
       id: job
     - name: Complete status check
-      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.1
+      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.2
       with:
         authToken: ${{ secrets.GITHUB_TOKEN }}
         context: ${{ github.event.workflow.name }}
diff --git a/.github/workflows/workflow-start.yml b/.github/workflows/workflow-start.yml
index 6b377a395e7a..1e832b6261f8 100644
--- a/.github/workflows/workflow-start.yml
+++ b/.github/workflows/workflow-start.yml
@@ -19,7 +19,7 @@ jobs:
       statuses: write
     steps:
     - name: Start status check
-      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.1
+      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.2
       with:
         authToken: ${{ secrets.GITHUB_TOKEN }}
         context: ${{ inputs.workflowName }}

From 86ba3bfb923d3f94d8ea8659859abac7ab2cdb0d Mon Sep 17 00:00:00 2001
From: phlax 
Date: Mon, 12 Jun 2023 13:51:46 +0100
Subject: [PATCH 513/740] ci: Dont run sync wf in non-Envoy repos (#27904)

Signed-off-by: Ryan Northey 
---
 .github/workflows/envoy-sync.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/envoy-sync.yml b/.github/workflows/envoy-sync.yml
index 8fcfc67e1c4b..1b7f4fe27d4a 100644
--- a/.github/workflows/envoy-sync.yml
+++ b/.github/workflows/envoy-sync.yml
@@ -12,6 +12,7 @@ concurrency:
 jobs:
   sync:
     runs-on: ubuntu-20.04
+    if: github.repository == 'envoyproxy/envoy'
     strategy:
       fail-fast: false
       matrix:

From 94a981e63bb2ae3afbd329a8d531f802bfe20317 Mon Sep 17 00:00:00 2001
From: Pawan Bishnoi 
Date: Mon, 12 Jun 2023 19:13:52 +0530
Subject: [PATCH 514/740] Outlier detection: fix bug in max_ejection_percentage
 (#27624)

Signed-off-by: Pawan Bishnoi 
---
 .../common/upstream/outlier_detection_impl.cc |  5 +--
 .../upstream/outlier_detection_impl_test.cc   | 36 ++++++++++++++++++-
 2 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/source/common/upstream/outlier_detection_impl.cc b/source/common/upstream/outlier_detection_impl.cc
index 2b4b8a9b9eef..3c51d3a5e790 100644
--- a/source/common/upstream/outlier_detection_impl.cc
+++ b/source/common/upstream/outlier_detection_impl.cc
@@ -477,10 +477,11 @@ void DetectorImpl::ejectHost(HostSharedPtr host,
                              envoy::data::cluster::v3::OutlierEjectionType type) {
   uint64_t max_ejection_percent = std::min(
       100, runtime_.snapshot().getInteger(MaxEjectionPercentRuntime, config_.maxEjectionPercent()));
-  double ejected_percent = 100.0 * ejections_active_helper_.value() / host_monitors_.size();
+  double ejected_percent = 100.0 * (ejections_active_helper_.value() + 1) / host_monitors_.size();
   // Note this is not currently checked per-priority level, so it is possible
   // for outlier detection to eject all hosts at any given priority level.
-  if (ejected_percent < max_ejection_percent) {
+  // Note: at-least one host is ejected, we ignore max ejection percentage when ejecting first host.
+  if ((ejections_active_helper_.value() == 0) || (ejected_percent <= max_ejection_percent)) {
     if (type == envoy::data::cluster::v3::CONSECUTIVE_5XX ||
         type == envoy::data::cluster::v3::SUCCESS_RATE) {
       // Deprecated counter, preserving old behaviour until it's removed.
diff --git a/test/common/upstream/outlier_detection_impl_test.cc b/test/common/upstream/outlier_detection_impl_test.cc
index ad27b0ad6b8b..30847c388a5d 100644
--- a/test/common/upstream/outlier_detection_impl_test.cc
+++ b/test/common/upstream/outlier_detection_impl_test.cc
@@ -1747,6 +1747,41 @@ TEST_F(OutlierDetectorImplTest, CrossThreadFailRace) {
   EXPECT_EQ(1UL, outlier_detection_ejections_active_.value());
 }
 
+TEST_F(OutlierDetectorImplTest, MaxEjectionPercentage) {
+  // A 50% ejection limit should not eject more than 1 out of 3 pods.
+  const std::string yaml = R"EOF(
+max_ejection_percent: 50
+max_ejection_time_jitter: 13s
+  )EOF";
+  envoy::config::cluster::v3::OutlierDetection outlier_detection_;
+  TestUtility::loadFromYaml(yaml, outlier_detection_);
+  EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_));
+
+  // add 3 hosts.
+  addHosts({"tcp://127.0.0.2:80"});
+  addHosts({"tcp://127.0.0.3:80"});
+  addHosts({"tcp://127.0.0.4:80"});
+
+  EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _));
+  std::shared_ptr detector(DetectorImpl::create(
+      cluster_, outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_, random_));
+
+  detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); });
+
+  time_system_.setMonotonicTime(std::chrono::milliseconds(0));
+  // Expect only one ejection.
+  EXPECT_CALL(checker_, check(hosts_[0]));
+  EXPECT_CALL(*event_logger_, logEject(std::static_pointer_cast(hosts_[0]),
+                                       _, envoy::data::cluster::v3::CONSECUTIVE_5XX, true));
+
+  loadRq(hosts_[0], 5, 500);
+  loadRq(hosts_[1], 5, 500);
+  loadRq(hosts_[2], 5, 500);
+  EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK));
+  EXPECT_FALSE(hosts_[1]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK));
+  EXPECT_FALSE(hosts_[2]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK));
+}
+
 TEST_F(OutlierDetectorImplTest, Consecutive_5xxAlreadyEjected) {
   EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_));
   addHosts({"tcp://127.0.0.1:80"});
@@ -1755,7 +1790,6 @@ TEST_F(OutlierDetectorImplTest, Consecutive_5xxAlreadyEjected) {
                                                               dispatcher_, runtime_, time_system_,
                                                               event_logger_, random_));
   detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); });
-
   // Cause a consecutive 5xx error.
   loadRq(hosts_[0], 4, 500);
 

From 2608a46dcc8b7ef76cc5095ffd25a43a2055b4eb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Jun 2023 09:45:47 -0400
Subject: [PATCH 515/740] build(deps): bump jaegertracing/all-in-one from
 `3703087` to `4f85b75` in /examples/shared/jaeger (#27919)

Signed-off-by: dependabot[bot] 
---
 examples/shared/jaeger/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile
index 27d9f94cafcf..8fdd5df9af5c 100644
--- a/examples/shared/jaeger/Dockerfile
+++ b/examples/shared/jaeger/Dockerfile
@@ -1,4 +1,4 @@
-FROM jaegertracing/all-in-one@sha256:37030876f8e4d42ed793c545b9420afebc46d9fb303affcec46c4b60b97e3c4a
+FROM jaegertracing/all-in-one@sha256:4f85b75673c0863ef869f3e54d043f842935351ef0856d5ddad1709da4f17354
 HEALTHCHECK \
     --interval=1s \
     --timeout=1s \

From bb81ea33ed9c7a48c2017f6958050753dc9dff90 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Jun 2023 09:46:30 -0400
Subject: [PATCH 516/740] build(deps): bump node from 20.2-bullseye-slim to
 20.3-bullseye-slim in /examples/shared/node (#27920)

Signed-off-by: dependabot[bot] 
---
 examples/shared/node/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile
index eba7faacef82..f44be4a0ec27 100644
--- a/examples/shared/node/Dockerfile
+++ b/examples/shared/node/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:20.2-bullseye-slim@sha256:dc1906714d1993d291e1e7b5f236291236b0a0b6dfacdb164e4a9ea44d09c52e as node-base
+FROM node:20.3-bullseye-slim@sha256:f52e0eb0f31863051b56d76d191b283c2b49ac084762eddfeb1afb54791b250b as node-base
 
 
 FROM node-base as node-http-auth

From eec764f67f3af41e9e37d791281a611dbe8b70d0 Mon Sep 17 00:00:00 2001
From: code 
Date: Mon, 12 Jun 2023 22:54:59 +0800
Subject: [PATCH 517/740] http: route level filter `disabled` support (#27210)

* refactor and fix tests

Signed-off-by: wbpcode 

* fix test

Signed-off-by: wbpcode 

* more detailed comments

Signed-off-by: wbpcode 

* resolve conflict

Signed-off-by: wbpcode 

* fix test

Signed-off-by: wbpcode 

* change log and docs

Signed-off-by: wbpcode 

* Update changelogs/current.yaml

Co-authored-by: Matt Klein 
Signed-off-by: code 

* Update changelogs/current.yaml

Co-authored-by: Matt Klein 
Signed-off-by: code 

* Update changelogs/current.yaml

Co-authored-by: Matt Klein 
Signed-off-by: code 

* Update envoy/http/filter_factory.h

Co-authored-by: Matt Klein 
Signed-off-by: code 

* Update source/common/http/filter_chain_helper.cc

Co-authored-by: Matt Klein 
Signed-off-by: code 

* add new integration test

Signed-off-by: wbpcode 

---------

Signed-off-by: wbpcode 
Signed-off-by: code 
Co-authored-by: Matt Klein 
---
 .../config/route/v3/route_components.proto    |  2 -
 changelogs/current.yaml                       |  6 ++
 envoy/http/filter_factory.h                   | 31 +++++++-
 envoy/router/router.h                         |  8 ++
 source/common/http/async_client_impl.h        |  1 +
 source/common/http/filter_chain_helper.cc     |  9 ++-
 source/common/http/filter_chain_helper.h      |  1 +
 source/common/http/filter_manager.cc          |  6 +-
 source/common/http/filter_manager.h           | 12 +++
 source/common/router/config_impl.cc           | 49 ++++++++++--
 source/common/router/config_impl.h            | 30 ++++++-
 source/common/router/delegating_route_impl.h  |  3 +
 source/common/router/router.h                 |  7 +-
 source/common/upstream/upstream_impl.h        |  7 +-
 .../network/http_connection_manager/config.cc |  8 +-
 .../network/http_connection_manager/config.h  |  4 +-
 source/server/admin/admin.cc                  |  3 +-
 source/server/admin/admin.h                   |  3 +-
 test/common/http/BUILD                        | 12 +++
 test/common/http/filter_chain_helper_test.cc  | 59 ++++++++++++++
 test/common/router/config_impl_test.cc        | 78 +++++++++++++++++++
 test/common/router/upstream_request_test.cc   | 28 +++----
 .../header_mutation_integration_test.cc       | 62 ++++++++++++++-
 .../http/tcp/upstream_request_test.cc         |  7 +-
 test/mocks/http/mocks.h                       |  3 +-
 test/mocks/router/mocks.h                     |  1 +
 test/mocks/upstream/cluster_info.cc           | 26 +++----
 test/mocks/upstream/cluster_info.h            |  3 +-
 28 files changed, 411 insertions(+), 58 deletions(-)
 create mode 100644 test/common/http/filter_chain_helper_test.cc

diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto
index 7153a100772a..014bb0d9261a 100644
--- a/api/envoy/config/route/v3/route_components.proto
+++ b/api/envoy/config/route/v3/route_components.proto
@@ -2412,7 +2412,5 @@ message FilterConfig {
   //   created and it is too late to change the chain.
   //
   //   This field only make sense for the downstream HTTP filters for now.
-  //
-  // [#not-implemented-hide:]
   bool disabled = 3;
 }
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index da7ca4343063..44a909319a4e 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -331,6 +331,12 @@ new_features:
 - area: http
   change: |
     added support for configuring additional :ref:`cookie attributes `.
+- area: http
+  change: |
+    Added support for the route/virtual host level
+    :ref:`disabled ` field.
+    A route/virtual host level per filter config can be marked as disabled, which means that
+    the filter will be disabled in a specific route/virtual host.
 - area: health_check
   change: |
     added host related information :ref:`metadata ` and
diff --git a/envoy/http/filter_factory.h b/envoy/http/filter_factory.h
index eec92abd9600..71d6bc51f19f 100644
--- a/envoy/http/filter_factory.h
+++ b/envoy/http/filter_factory.h
@@ -35,6 +35,31 @@ struct FilterContext {
   std::string filter_name;
 };
 
+/**
+ * Additional options for creating HTTP filter chain.
+ * TODO(wbpcode): it is possible to add more options to customize HTTP filter chain creation.
+ * For example, we can add related options here to tell FilterChainFactory to create
+ * upgrade filter chain or not.
+ */
+class FilterChainOptions {
+public:
+  virtual ~FilterChainOptions() = default;
+
+  /**
+   * Skip filter creation if the filter is explicitly disabled after the filter chain is
+   * selected.
+   *
+   * @param config_name the config name of the filter.
+   * @return whether the filter should be disabled or enabled based on the config name.
+   */
+  virtual bool filterDisabled(absl::string_view config_name) const PURE;
+};
+
+class EmptyFilterChainOptions : public FilterChainOptions {
+public:
+  bool filterDisabled(absl::string_view) const override { return false; }
+};
+
 /**
  * The filter chain manager is provided by the connection manager to the filter chain factory.
  * The filter chain factory will post the filter factory context and filter factory to the
@@ -69,10 +94,12 @@ class FilterChainFactory {
    *                FilterChainManager.
    * @param only_create_if_configured if true, only creates filter chain if there is a non-default
    *                                  configured filter chain. Default false.
+   * @param options additional options for creating a filter chain.
    * @return whather a filter chain has been created.
    */
-  virtual bool createFilterChain(FilterChainManager& manager,
-                                 bool only_create_if_configured = false) const PURE;
+  virtual bool
+  createFilterChain(FilterChainManager& manager, bool only_create_if_configured = false,
+                    const FilterChainOptions& options = EmptyFilterChainOptions{}) const PURE;
 
   /**
    * Called when a new upgrade stream is created on the connection.
diff --git a/envoy/router/router.h b/envoy/router/router.h
index 9e24d76b8160..7cb57a8d02b9 100644
--- a/envoy/router/router.h
+++ b/envoy/router/router.h
@@ -1199,6 +1199,14 @@ class Route {
    */
   virtual const RouteTracing* tracingConfig() const PURE;
 
+  /**
+   * Check if the filter is disabled for this route.
+   * @param config_name supplies the name of the filter config in the HTTP filter chain. This name
+   * may be different from the filter extension qualified name.
+   * @return true if the filter is disabled for this route, false otherwise.
+   */
+  virtual bool filterDisabled(absl::string_view config_name) const PURE;
+
   /**
    * This is a helper to get the route's per-filter config if it exists, up along the config
    * hierarchy(Route --> VirtualHost --> RouteConfiguration). Or nullptr if none of them exist.
diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h
index 8a4dc0d67284..83cee970b40b 100644
--- a/source/common/http/async_client_impl.h
+++ b/source/common/http/async_client_impl.h
@@ -374,6 +374,7 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream,
         std::function) const override {}
     const envoy::config::core::v3::Metadata& metadata() const override { return metadata_; }
     const Envoy::Config::TypedMetadata& typedMetadata() const override { return typed_metadata_; }
+    bool filterDisabled(absl::string_view) const override { return false; }
 
     RouteEntryImpl route_entry_;
     const envoy::config::core::v3::Metadata metadata_;
diff --git a/source/common/http/filter_chain_helper.cc b/source/common/http/filter_chain_helper.cc
index a8be35ae1ace..80281d098ca8 100644
--- a/source/common/http/filter_chain_helper.cc
+++ b/source/common/http/filter_chain_helper.cc
@@ -32,10 +32,15 @@ static Http::FilterFactoryCb MissingConfigFilterFactory =
     };
 
 void FilterChainUtility::createFilterChainForFactories(
-    Http::FilterChainManager& manager,
-    const FilterChainUtility::FilterFactoriesList& filter_factories) {
+    Http::FilterChainManager& manager, const FilterChainOptions& options,
+    const FilterFactoriesList& filter_factories) {
   bool added_missing_config_filter = false;
   for (const auto& filter_config_provider : filter_factories) {
+    // If this filter is disabled explicitly, skip trying to create it.
+    if (options.filterDisabled(filter_config_provider->name())) {
+      continue;
+    }
+
     auto config = filter_config_provider->config();
     if (config.has_value()) {
       Filter::NamedHttpFilterFactoryCb& factory_cb = config.value().get();
diff --git a/source/common/http/filter_chain_helper.h b/source/common/http/filter_chain_helper.h
index 592d29793a2c..fe925ef30610 100644
--- a/source/common/http/filter_chain_helper.h
+++ b/source/common/http/filter_chain_helper.h
@@ -28,6 +28,7 @@ class FilterChainUtility : Logger::Loggable {
       envoy::extensions::filters::network::http_connection_manager::v3::HttpFilter>;
 
   static void createFilterChainForFactories(Http::FilterChainManager& manager,
+                                            const FilterChainOptions& options,
                                             const FilterFactoriesList& filter_factories);
 
   static std::shared_ptr
diff --git a/source/common/http/filter_manager.cc b/source/common/http/filter_manager.cc
index 916fe91dabae..5f71e1cfb5f4 100644
--- a/source/common/http/filter_manager.cc
+++ b/source/common/http/filter_manager.cc
@@ -1492,7 +1492,11 @@ bool FilterManager::createFilterChain() {
     }
   }
 
-  filter_chain_factory_.createFilterChain(*this);
+  // This filter chain options is only used for the downstream filter chains for now. So, try to
+  // set valid initial route only when the downstream callbacks is available.
+  FilterChainOptionsImpl options(
+      filter_manager_callbacks_.downstreamCallbacks().has_value() ? streamInfo().route() : nullptr);
+  filter_chain_factory_.createFilterChain(*this, false, options);
   return !upgrade_rejected;
 }
 
diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h
index 87be59dc76a0..610ca49fd9ea 100644
--- a/source/common/http/filter_manager.h
+++ b/source/common/http/filter_manager.h
@@ -908,6 +908,18 @@ class FilterManager : public ScopeTrackedObject,
     const Http::FilterContext& context_;
   };
 
+  class FilterChainOptionsImpl : public FilterChainOptions {
+  public:
+    FilterChainOptionsImpl(Router::RouteConstSharedPtr route) : route_(std::move(route)) {}
+
+    bool filterDisabled(absl::string_view config_name) const override {
+      return route_ != nullptr && route_->filterDisabled(config_name);
+    }
+
+  private:
+    const Router::RouteConstSharedPtr route_;
+  };
+
   // Indicates which filter to start the iteration with.
   enum class FilterIterationStartState { AlwaysStartFromNext, CanStartFromCurrent };
 
diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc
index e854bd3507b6..29d631dc1f37 100644
--- a/source/common/router/config_impl.cc
+++ b/source/common/router/config_impl.cc
@@ -1371,6 +1371,14 @@ void RouteEntryImplBase::validateClusters(
   }
 }
 
+bool RouteEntryImplBase::filterDisabled(absl::string_view config_name) const {
+  absl::optional result = per_filter_configs_.disabled(config_name);
+  if (result.has_value()) {
+    return result.value();
+  }
+  return vhost_->filterDisabled(config_name);
+}
+
 void RouteEntryImplBase::traversePerFilterConfig(
     const std::string& filter_name,
     std::function cb) const {
@@ -1725,6 +1733,14 @@ CommonVirtualHostImpl::VirtualClusterEntry::VirtualClusterEntry(
 
 const CommonConfig& CommonVirtualHostImpl::routeConfig() const { return *global_route_config_; }
 
+bool CommonVirtualHostImpl::filterDisabled(absl::string_view config_name) const {
+  absl::optional result = per_filter_configs_.disabled(config_name);
+  if (result.has_value()) {
+    return result.value();
+  }
+  return global_route_config_->filterDisabled(config_name);
+}
+
 const RouteSpecificFilterConfig*
 CommonVirtualHostImpl::mostSpecificPerFilterConfig(const std::string& name) const {
   auto* per_filter_config = per_filter_configs_.get(name);
@@ -2107,7 +2123,6 @@ RouteSpecificFilterConfigConstSharedPtr PerFilterConfigs::createRouteSpecificFil
     const std::string& name, const ProtobufWkt::Any& typed_config, bool is_optional,
     Server::Configuration::ServerFactoryContext& factory_context,
     ProtobufMessage::ValidationVisitor& validator) {
-
   Server::Configuration::NamedHttpFilterConfigFactory* factory =
       Envoy::Config::Utility::getFactoryByType(
           typed_config);
@@ -2178,11 +2193,25 @@ PerFilterConfigs::PerFilterConfigs(
       Envoy::Config::Utility::translateOpaqueConfig(per_filter_config.second, validator,
                                                     filter_config);
 
+      // The filter is marked as disabled explicitly and the config is ignored directly.
+      if (filter_config.disabled()) {
+        configs_.emplace(name, FilterConfig{nullptr, true});
+        continue;
+      }
+
+      // If the field `config` is not configured, we treat it as configuration error.
       if (!filter_config.has_config()) {
         throw EnvoyException(
             fmt::format("Empty route/virtual host per filter configuration for {} filter", name));
       }
 
+      // If the field `config` is configured but is empty, we treat the filter as disabled
+      // explicitly.
+      if (filter_config.config().type_url().empty()) {
+        configs_.emplace(name, FilterConfig{nullptr, false});
+        continue;
+      }
+
       config = createRouteSpecificFilterConfig(name, filter_config.config(),
                                                is_optional_by_hcm || filter_config.is_optional(),
                                                factory_context, validator);
@@ -2191,15 +2220,25 @@ PerFilterConfigs::PerFilterConfigs(
                                                factory_context, validator);
     }
 
-    if (config != nullptr) {
-      configs_[name] = std::move(config);
-    }
+    // If a filter is explicitly configured we treat it as enabled.
+    // The config may be nullptr because the filter could be optional.
+    configs_.emplace(name, FilterConfig{std::move(config), false});
   }
 }
 
 const RouteSpecificFilterConfig* PerFilterConfigs::get(const std::string& name) const {
   auto it = configs_.find(name);
-  return it == configs_.end() ? nullptr : it->second.get();
+  return it == configs_.end() ? nullptr : it->second.config_.get();
+}
+
+absl::optional PerFilterConfigs::disabled(absl::string_view name) const {
+  // Quick exit if there are no configs.
+  if (configs_.empty()) {
+    return absl::nullopt;
+  }
+
+  const auto it = configs_.find(name);
+  return it != configs_.end() ? absl::optional{it->second.disabled_} : absl::nullopt;
 }
 
 Matcher::ActionFactoryCb RouteMatchActionFactory::createActionFactoryCb(
diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h
index 3f31ce43a986..65cdfb86996d 100644
--- a/source/common/router/config_impl.h
+++ b/source/common/router/config_impl.h
@@ -81,6 +81,11 @@ using OptionalHttpFilters = absl::flat_hash_set;
 
 class PerFilterConfigs : public Logger::Loggable {
 public:
+  struct FilterConfig {
+    RouteSpecificFilterConfigConstSharedPtr config_;
+    bool disabled_{};
+  };
+
   PerFilterConfigs(const Protobuf::Map& typed_configs,
                    const OptionalHttpFilters& optional_http_filters,
                    Server::Configuration::ServerFactoryContext& factory_context,
@@ -88,13 +93,20 @@ class PerFilterConfigs : public Logger::Loggable {
 
   const RouteSpecificFilterConfig* get(const std::string& name) const;
 
+  /**
+   * @return true if the filter is explicitly disabled for this route or virtual host, false
+   * if the filter is explicitly enabled. If the filter is not explicitly enabled or disabled,
+   * returns absl::nullopt.
+   */
+  absl::optional disabled(absl::string_view name) const;
+
 private:
   RouteSpecificFilterConfigConstSharedPtr
   createRouteSpecificFilterConfig(const std::string& name, const ProtobufWkt::Any& typed_config,
                                   bool is_optional,
                                   Server::Configuration::ServerFactoryContext& factory_context,
                                   ProtobufMessage::ValidationVisitor& validator);
-  absl::node_hash_map configs_;
+  absl::flat_hash_map configs_;
 };
 
 class RouteEntryImplBase;
@@ -132,6 +144,7 @@ class SslRedirectRoute : public Route {
   const RouteSpecificFilterConfig* mostSpecificPerFilterConfig(const std::string&) const override {
     return nullptr;
   }
+  bool filterDisabled(absl::string_view) const override { return false; }
   void traversePerFilterConfig(
       const std::string&,
       std::function) const override {}
@@ -244,6 +257,7 @@ class CommonVirtualHostImpl : public VirtualHost, Logger::Loggabledecorator(); }
     const RouteTracing* tracingConfig() const override { return parent_->tracingConfig(); }
+    bool filterDisabled(absl::string_view config_name) const override {
+      return parent_->filterDisabled(config_name);
+    }
     const RouteSpecificFilterConfig*
     mostSpecificPerFilterConfig(const std::string& name) const override {
       return parent_->mostSpecificPerFilterConfig(name);
@@ -998,6 +1016,13 @@ class RouteEntryImplBase : public RouteEntryAndRoute,
     Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info,
                                                     bool do_formatting = true) const override;
 
+    bool filterDisabled(absl::string_view config_name) const override {
+      absl::optional result = per_filter_configs_.disabled(config_name);
+      if (result.has_value()) {
+        return result.value();
+      }
+      return DynamicRouteEntry::filterDisabled(config_name);
+    }
     const RouteSpecificFilterConfig*
     mostSpecificPerFilterConfig(const std::string& name) const override {
       auto* config = per_filter_configs_.get(name);
@@ -1536,6 +1561,9 @@ class CommonConfigImpl : public CommonConfig {
   const RouteSpecificFilterConfig* perFilterConfig(const std::string& name) const {
     return per_filter_configs_.get(name);
   }
+  bool filterDisabled(absl::string_view config_name) const {
+    return per_filter_configs_.disabled(config_name).value_or(false);
+  }
 
   // Router::CommonConfig
   const std::list& internalOnlyHeaders() const override {
diff --git a/source/common/router/delegating_route_impl.h b/source/common/router/delegating_route_impl.h
index 422f770bded6..fff2306dfcf7 100644
--- a/source/common/router/delegating_route_impl.h
+++ b/source/common/router/delegating_route_impl.h
@@ -43,6 +43,9 @@ class DelegatingRoute : public Router::Route {
   const Envoy::Config::TypedMetadata& typedMetadata() const override {
     return base_route_->typedMetadata();
   }
+  bool filterDisabled(absl::string_view name) const override {
+    return base_route_->filterDisabled(name);
+  }
 
 private:
   const Router::RouteConstSharedPtr base_route_;
diff --git a/source/common/router/router.h b/source/common/router/router.h
index ad75603b1740..7d86381d312d 100644
--- a/source/common/router/router.h
+++ b/source/common/router/router.h
@@ -271,15 +271,16 @@ class FilterConfig : Http::FilterChainFactory {
     }
   }
 
-  bool createFilterChain(Http::FilterChainManager& manager,
-                         bool only_create_if_configured = false) const override {
+  bool createFilterChain(
+      Http::FilterChainManager& manager, bool only_create_if_configured = false,
+      const Http::FilterChainOptions& options = Http::EmptyFilterChainOptions{}) const override {
     // Currently there is no default filter chain, so only_create_if_configured true doesn't make
     // sense.
     ASSERT(!only_create_if_configured);
     if (upstream_http_filter_factories_.empty()) {
       return false;
     }
-    Http::FilterChainUtility::createFilterChainForFactories(manager,
+    Http::FilterChainUtility::createFilterChainForFactories(manager, options,
                                                             upstream_http_filter_factories_);
     return true;
   }
diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h
index e030299da786..ee759e33f9cb 100644
--- a/source/common/upstream/upstream_impl.h
+++ b/source/common/upstream/upstream_impl.h
@@ -974,12 +974,13 @@ class ClusterInfoImpl : public ClusterInfo,
   upstreamHttpProtocol(absl::optional downstream_protocol) const override;
 
   // Http::FilterChainFactory
-  bool createFilterChain(Http::FilterChainManager& manager,
-                         bool only_create_if_configured) const override {
+  bool createFilterChain(Http::FilterChainManager& manager, bool only_create_if_configured,
+                         const Http::FilterChainOptions&) const override {
     if (!has_configured_http_filters_ && only_create_if_configured) {
       return false;
     }
-    Http::FilterChainUtility::createFilterChainForFactories(manager, http_filter_factories_);
+    Http::FilterChainUtility::createFilterChainForFactories(
+        manager, Http::EmptyFilterChainOptions{}, http_filter_factories_);
     return true;
   }
   bool createUpgradeFilterChain(absl::string_view, const UpgradeMap*,
diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc
index 7594c5f02247..46477c6d60d7 100644
--- a/source/extensions/filters/network/http_connection_manager/config.cc
+++ b/source/extensions/filters/network/http_connection_manager/config.cc
@@ -689,8 +689,9 @@ Http::ServerConnectionPtr HttpConnectionManagerConfig::createCodec(
   PANIC_DUE_TO_CORRUPT_ENUM;
 }
 
-bool HttpConnectionManagerConfig::createFilterChain(Http::FilterChainManager& manager, bool) const {
-  Http::FilterChainUtility::createFilterChainForFactories(manager, filter_factories_);
+bool HttpConnectionManagerConfig::createFilterChain(Http::FilterChainManager& manager, bool,
+                                                    const Http::FilterChainOptions& options) const {
+  Http::FilterChainUtility::createFilterChainForFactories(manager, options, filter_factories_);
   return true;
 }
 
@@ -722,7 +723,8 @@ bool HttpConnectionManagerConfig::createUpgradeFilterChain(
     filters_to_use = it->second.filter_factories.get();
   }
 
-  Http::FilterChainUtility::createFilterChainForFactories(callbacks, *filters_to_use);
+  Http::FilterChainUtility::createFilterChainForFactories(
+      callbacks, Http::EmptyFilterChainOptions{}, *filters_to_use);
   return true;
 }
 
diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h
index 09d76f31dcbd..d2c43c2fe142 100644
--- a/source/extensions/filters/network/http_connection_manager/config.h
+++ b/source/extensions/filters/network/http_connection_manager/config.h
@@ -136,7 +136,9 @@ class HttpConnectionManagerConfig : Logger::Loggable,
       FilterConfigProviderManager& filter_config_provider_manager);
 
   // Http::FilterChainFactory
-  bool createFilterChain(Http::FilterChainManager& manager, bool = false) const override;
+  bool createFilterChain(
+      Http::FilterChainManager& manager, bool = false,
+      const Http::FilterChainOptions& = Http::EmptyFilterChainOptions{}) const override;
   using FilterFactoriesList =
       std::list>;
   struct FilterConfig {
diff --git a/source/server/admin/admin.cc b/source/server/admin/admin.cc
index f71f71c5abae..c4479a047f86 100644
--- a/source/server/admin/admin.cc
+++ b/source/server/admin/admin.cc
@@ -262,7 +262,8 @@ bool AdminImpl::createNetworkFilterChain(Network::Connection& connection,
   return true;
 }
 
-bool AdminImpl::createFilterChain(Http::FilterChainManager& manager, bool) const {
+bool AdminImpl::createFilterChain(Http::FilterChainManager& manager, bool,
+                                  const Http::FilterChainOptions&) const {
   Http::FilterFactoryCb factory = [this](Http::FilterChainFactoryCallbacks& callbacks) {
     callbacks.addStreamFilter(std::make_shared(createRequestFunction()));
   };
diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h
index 240b970af7e0..fc5f3cf43ac3 100644
--- a/source/server/admin/admin.h
+++ b/source/server/admin/admin.h
@@ -112,7 +112,8 @@ class AdminImpl : public Admin,
                                     Network::UdpReadFilterCallbacks&) override {}
 
   // Http::FilterChainFactory
-  bool createFilterChain(Http::FilterChainManager& manager, bool) const override;
+  bool createFilterChain(Http::FilterChainManager& manager, bool,
+                         const Http::FilterChainOptions&) const override;
   bool createUpgradeFilterChain(absl::string_view, const Http::FilterChainFactory::UpgradeMap*,
                                 Http::FilterChainManager&) const override {
     return false;
diff --git a/test/common/http/BUILD b/test/common/http/BUILD
index 5e8e183e9c68..43f38fb16128 100644
--- a/test/common/http/BUILD
+++ b/test/common/http/BUILD
@@ -582,3 +582,15 @@ envoy_cc_test(
         "//test/test_common:utility_lib",
     ],
 )
+
+envoy_cc_test(
+    name = "filter_chain_helper_test",
+    srcs = ["filter_chain_helper_test.cc"],
+    deps = [
+        "//source/common/http:filter_chain_helper_lib",
+        "//test/mocks:common_lib",
+        "//test/mocks/http:http_mocks",
+        "//test/mocks/router:router_mocks",
+        "//test/test_common:utility_lib",
+    ],
+)
diff --git a/test/common/http/filter_chain_helper_test.cc b/test/common/http/filter_chain_helper_test.cc
new file mode 100644
index 000000000000..dfeefcde2907
--- /dev/null
+++ b/test/common/http/filter_chain_helper_test.cc
@@ -0,0 +1,59 @@
+#include "source/common/http/filter_chain_helper.h"
+
+#include "test/mocks/http/mocks.h"
+#include "test/mocks/router/mocks.h"
+#include "test/test_common/utility.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using testing::NiceMock;
+using testing::Return;
+
+namespace Envoy {
+namespace Http {
+namespace {
+
+class MockFilterChainOptions : public FilterChainOptions {
+public:
+  MockFilterChainOptions() = default;
+
+  MOCK_METHOD(bool, filterDisabled, (absl::string_view), (const));
+};
+
+TEST(FilterChainUtilityTest, CreateFilterChainForFactoriesWithRouteDisabled) {
+  NiceMock manager;
+  NiceMock options;
+  FilterChainUtility::FilterFactoriesList filter_factories;
+
+  for (const auto& name : {"filter_0", "filter_1", "filter_2"}) {
+    auto provider =
+        std::make_unique>(
+            Filter::NamedHttpFilterFactoryCb{"filter_type_name",
+                                             [](FilterChainFactoryCallbacks&) {}},
+            name);
+    filter_factories.push_back(std::move(provider));
+  }
+
+  {
+    // If empty filter chain options is provided, all filters should be added.
+    EXPECT_CALL(manager, applyFilterFactoryCb(_, _)).Times(3);
+    FilterChainUtility::createFilterChainForFactories(manager, Http::EmptyFilterChainOptions{},
+                                                      filter_factories);
+  }
+
+  {
+
+    EXPECT_CALL(options, filterDisabled("filter_0")).WillOnce(Return(true));
+    EXPECT_CALL(options, filterDisabled("filter_1")).WillOnce(Return(false));
+    EXPECT_CALL(options, filterDisabled("filter_2")).WillOnce(Return(true));
+
+    // Only filter_1 should be added.
+    EXPECT_CALL(manager, applyFilterFactoryCb(_, _));
+    FilterChainUtility::createFilterChainForFactories(manager, options, filter_factories);
+  }
+}
+
+} // namespace
+} // namespace Http
+} // namespace Envoy
diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc
index 1ad203edca0b..dd4219522581 100644
--- a/test/common/router/config_impl_test.cc
+++ b/test/common/router/config_impl_test.cc
@@ -10745,6 +10745,84 @@ TEST_F(PerFilterConfigsTest, RouteTypedConfigWithErrorFilterName) {
   checkEach(yaml, 123, expected_traveled_config, "filter.unknown");
 }
 
+TEST_F(PerFilterConfigsTest, RouteFilterDisabledTest) {
+  const std::string yaml = R"EOF(
+typed_per_filter_config:
+  test.filter:
+    "@type":  type.googleapis.com/envoy.config.route.v3.FilterConfig
+    disabled: true
+virtual_hosts:
+  - name: bar
+    domains: ["host1"]
+    routes:
+      - match: { prefix: "/route1" }
+        route: { cluster: baz }
+        # test.filter will be enabled for this route because this config
+        # will override virtual host level config.
+        typed_per_filter_config:
+          test.filter:
+            "@type": type.googleapis.com/google.protobuf.Timestamp
+            value:
+              seconds: 123
+      - match: { prefix: "/route2" }
+        route: { cluster: baz }
+        # test.filter will be disabled for this route.
+        typed_per_filter_config:
+          test.filter:
+            "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig
+            disabled: true
+    typed_per_filter_config:
+      test.filter:
+        "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig
+        disabled: true
+  - name: bar
+    domains: ["host2"]
+    routes:
+      # test.filter will be disabled for this route because the virtual host
+      # level config.
+      - match: { prefix: "/route3" }
+        route: { cluster: baz }
+      - match: { prefix: "/route4" }
+        route: { cluster: baz }
+        # test.filter will be enabled for this route but no valid route level config
+        # is provided in this route.
+        typed_per_filter_config:
+          test.filter:
+            "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig
+            # Provide an empty config to enable the filter.
+            config: {}
+    typed_per_filter_config:
+      test.filter:
+        "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig
+        disabled: true
+  - name: bar
+    domains: ["host3"]
+    routes:
+      # test.filter will be disabled for this route because the global route config.
+      - match: { prefix: "/route5" }
+        route: { cluster: baz }
+)EOF";
+
+  factory_context_.cluster_manager_.initializeClusters({"baz"}, {});
+
+  const TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true);
+
+  const auto route1 = config.route(genHeaders("host1", "/route1", "GET"), 0);
+  EXPECT_FALSE(route1->filterDisabled("test.filter"));
+
+  const auto route2 = config.route(genHeaders("host1", "/route2", "GET"), 0);
+  EXPECT_TRUE(route2->filterDisabled("test.filter"));
+
+  const auto route3 = config.route(genHeaders("host2", "/route3", "GET"), 0);
+  EXPECT_TRUE(route3->filterDisabled("test.filter"));
+
+  const auto route4 = config.route(genHeaders("host2", "/route4", "GET"), 0);
+  EXPECT_FALSE(route4->filterDisabled("test.filter"));
+
+  const auto route5 = config.route(genHeaders("host3", "/route5", "GET"), 0);
+  EXPECT_TRUE(route5->filterDisabled("test.filter"));
+}
+
 class RouteMatchOverrideTest : public testing::Test, public ConfigImplTestBase {};
 
 TEST_F(RouteMatchOverrideTest, VerifyAllMatchableRoutes) {
diff --git a/test/common/router/upstream_request_test.cc b/test/common/router/upstream_request_test.cc
index c9c8bc89b6ba..b658fb092ce8 100644
--- a/test/common/router/upstream_request_test.cc
+++ b/test/common/router/upstream_request_test.cc
@@ -113,20 +113,20 @@ TEST_F(UpstreamRequestTest, AcceptRouterHeaders) {
 
   EXPECT_CALL(*router_filter_interface_.cluster_info_, createFilterChain)
       .Times(2)
-      .WillRepeatedly(
-          Invoke([&](Http::FilterChainManager& manager, bool only_create_if_configured) -> bool {
-            if (only_create_if_configured) {
-              return false;
-            }
-            auto factory = createDecoderFilterFactoryCb(filter);
-            manager.applyFilterFactoryCb({}, factory);
-            Http::FilterFactoryCb factory_cb =
-                [](Http::FilterChainFactoryCallbacks& callbacks) -> void {
-              callbacks.addStreamDecoderFilter(std::make_shared());
-            };
-            manager.applyFilterFactoryCb({}, factory_cb);
-            return true;
-          }));
+      .WillRepeatedly(Invoke([&](Http::FilterChainManager& manager, bool only_create_if_configured,
+                                 const Http::FilterChainOptions&) -> bool {
+        if (only_create_if_configured) {
+          return false;
+        }
+        auto factory = createDecoderFilterFactoryCb(filter);
+        manager.applyFilterFactoryCb({}, factory);
+        Http::FilterFactoryCb factory_cb =
+            [](Http::FilterChainFactoryCallbacks& callbacks) -> void {
+          callbacks.addStreamDecoderFilter(std::make_shared());
+        };
+        manager.applyFilterFactoryCb({}, factory_cb);
+        return true;
+      }));
 
   initialize();
   ASSERT_TRUE(filter->callbacks_ != nullptr);
diff --git a/test/extensions/filters/http/header_mutation/header_mutation_integration_test.cc b/test/extensions/filters/http/header_mutation/header_mutation_integration_test.cc
index e0da5c540783..92e62124603c 100644
--- a/test/extensions/filters/http/header_mutation/header_mutation_integration_test.cc
+++ b/test/extensions/filters/http/header_mutation/header_mutation_integration_test.cc
@@ -70,6 +70,13 @@ name: upstream-header-mutation
                hcm) {
           auto* route = hcm.mutable_route_config()->mutable_virtual_hosts(0)->mutable_routes(0);
 
+          // Another route that disables downstream header mutation.
+          auto* another_route = hcm.mutable_route_config()->mutable_virtual_hosts(0)->add_routes();
+          *another_route = *route;
+
+          route->mutable_match()->set_path("/default/route");
+          another_route->mutable_match()->set_path("/disable/filter/route");
+
           // Per route header mutation for downstream.
           PerRouteProtoConfig header_mutation_1;
           auto response_mutation =
@@ -103,6 +110,18 @@ name: upstream-header-mutation
 
           route->mutable_typed_per_filter_config()->insert(
               {"upstream-header-mutation", per_route_config_2});
+
+          // Per route disable downstream header mutation.
+          envoy::config::route::v3::FilterConfig filter_config;
+          filter_config.mutable_config();
+          filter_config.set_disabled(true);
+          ProtobufWkt::Any per_route_config_3;
+          per_route_config_3.PackFrom(filter_config);
+          another_route->mutable_typed_per_filter_config()->insert(
+              {"donwstream-header-mutation", per_route_config_3});
+          // Try disable upstream header mutation but this is not supported and should not work.
+          another_route->mutable_typed_per_filter_config()->insert(
+              {"upstream-header-mutation", per_route_config_3});
         });
     HttpIntegrationTest::initialize();
   }
@@ -116,7 +135,7 @@ TEST_P(HeaderMutationIntegrationTest, TestHeaderMutation) {
   initializeFilter();
 
   codec_client_ = makeHttpConnection(lookupPort("http"));
-
+  default_request_headers_.setPath("/default/route");
   auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_);
   waitForNextUpstreamRequest();
 
@@ -163,6 +182,47 @@ TEST_P(HeaderMutationIntegrationTest, TestHeaderMutation) {
   codec_client_->close();
 }
 
+TEST_P(HeaderMutationIntegrationTest, TestDisableDownstreamHeaderMutation) {
+  initializeFilter();
+
+  codec_client_ = makeHttpConnection(lookupPort("http"));
+  default_request_headers_.setPath("/disable/filter/route");
+  auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_);
+  waitForNextUpstreamRequest();
+
+  EXPECT_EQ(0, upstream_request_->headers()
+                   .get(Http::LowerCaseString("downstream-request-global-flag-header"))
+                   .size());
+
+  EXPECT_EQ("upstream-request-global-flag-header-value",
+            upstream_request_->headers()
+                .get(Http::LowerCaseString("upstream-request-global-flag-header"))[0]
+                ->value()
+                .getStringView());
+
+  upstream_request_->encodeHeaders(default_response_headers_, true);
+
+  ASSERT_TRUE(response->waitForEndStream());
+  EXPECT_TRUE(response->complete());
+  EXPECT_EQ("200", response->headers().getStatusValue());
+
+  EXPECT_EQ(0,
+            response->headers().get(Http::LowerCaseString("downstream-global-flag-header")).size());
+  EXPECT_EQ(
+      0, response->headers().get(Http::LowerCaseString("downstream-per-route-flag-header")).size());
+
+  EXPECT_EQ("upstream-global-flag-header-value",
+            response->headers()
+                .get(Http::LowerCaseString("upstream-global-flag-header"))[0]
+                ->value()
+                .getStringView());
+  EXPECT_EQ("GET", response->headers()
+                       .get(Http::LowerCaseString("request-method-in-upstream-filter"))[0]
+                       ->value()
+                       .getStringView());
+  codec_client_->close();
+}
+
 } // namespace
 } // namespace HeaderMutation
 } // namespace HttpFilters
diff --git a/test/extensions/upstreams/http/tcp/upstream_request_test.cc b/test/extensions/upstreams/http/tcp/upstream_request_test.cc
index 02796da47bdf..aa51aa515cbc 100644
--- a/test/extensions/upstreams/http/tcp/upstream_request_test.cc
+++ b/test/extensions/upstreams/http/tcp/upstream_request_test.cc
@@ -94,9 +94,10 @@ TEST_F(TcpConnPoolTest, Cancel) {
 class TcpUpstreamTest : public ::testing::Test {
 public:
   TcpUpstreamTest() {
-    ON_CALL(*mock_router_filter_.cluster_info_, createFilterChain(_, _))
-        .WillByDefault(Invoke(
-            [&](Envoy::Http::FilterChainManager& manager, bool only_create_if_configured) -> bool {
+    ON_CALL(*mock_router_filter_.cluster_info_, createFilterChain(_, _, _))
+        .WillByDefault(
+            Invoke([&](Envoy::Http::FilterChainManager& manager, bool only_create_if_configured,
+                       const Envoy::Http::FilterChainOptions&) -> bool {
               if (only_create_if_configured) {
                 return false;
               }
diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h
index d05224a06b04..adac56934c96 100644
--- a/test/mocks/http/mocks.h
+++ b/test/mocks/http/mocks.h
@@ -201,7 +201,8 @@ class MockFilterChainFactory : public FilterChainFactory {
   ~MockFilterChainFactory() override;
 
   // Http::FilterChainFactory
-  bool createFilterChain(FilterChainManager& manager, bool) const override {
+  bool createFilterChain(FilterChainManager& manager, bool,
+                         const FilterChainOptions&) const override {
     return createFilterChain(manager);
   }
   MOCK_METHOD(bool, createFilterChain, (FilterChainManager & manager), (const));
diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h
index 8acad60e6201..c2769096ae10 100644
--- a/test/mocks/router/mocks.h
+++ b/test/mocks/router/mocks.h
@@ -507,6 +507,7 @@ class MockRoute : public Route {
   MOCK_METHOD(const RouteEntry*, routeEntry, (), (const));
   MOCK_METHOD(const Decorator*, decorator, (), (const));
   MOCK_METHOD(const RouteTracing*, tracingConfig, (), (const));
+  MOCK_METHOD(bool, filterDisabled, (absl::string_view), (const));
   MOCK_METHOD(const RouteSpecificFilterConfig*, perFilterConfig, (const std::string&), (const));
   MOCK_METHOD(const RouteSpecificFilterConfig*, mostSpecificPerFilterConfig, (const std::string&),
               (const));
diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc
index b24f9069f58e..108df1c3c0cb 100644
--- a/test/mocks/upstream/cluster_info.cc
+++ b/test/mocks/upstream/cluster_info.cc
@@ -185,19 +185,19 @@ MockClusterInfo::MockClusterInfo()
           }));
   ON_CALL(*this, upstreamHttpProtocol(_))
       .WillByDefault(Return(std::vector{Http::Protocol::Http11}));
-  ON_CALL(*this, createFilterChain(_, _))
-      .WillByDefault(
-          Invoke([&](Http::FilterChainManager& manager, bool only_create_if_configured) -> bool {
-            if (only_create_if_configured) {
-              return false;
-            }
-            Http::FilterFactoryCb factory_cb =
-                [](Http::FilterChainFactoryCallbacks& callbacks) -> void {
-              callbacks.addStreamDecoderFilter(std::make_shared());
-            };
-            manager.applyFilterFactoryCb({}, factory_cb);
-            return true;
-          }));
+  ON_CALL(*this, createFilterChain(_, _, _))
+      .WillByDefault(Invoke([&](Http::FilterChainManager& manager, bool only_create_if_configured,
+                                const Http::FilterChainOptions&) -> bool {
+        if (only_create_if_configured) {
+          return false;
+        }
+        Http::FilterFactoryCb factory_cb =
+            [](Http::FilterChainFactoryCallbacks& callbacks) -> void {
+          callbacks.addStreamDecoderFilter(std::make_shared());
+        };
+        manager.applyFilterFactoryCb({}, factory_cb);
+        return true;
+      }));
   ON_CALL(*this, loadBalancerConfig())
       .WillByDefault(Return(makeOptRefFromPtr(nullptr)));
   ON_CALL(*this, makeHeaderValidator(_)).WillByDefault(Invoke([&](Http::Protocol protocol) {
diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h
index d81f84b94265..1245ea7bdf3f 100644
--- a/test/mocks/upstream/cluster_info.h
+++ b/test/mocks/upstream/cluster_info.h
@@ -179,7 +179,8 @@ class MockClusterInfo : public ClusterInfo {
               (const));
 
   MOCK_METHOD(bool, createFilterChain,
-              (Http::FilterChainManager & manager, bool only_create_if_configured),
+              (Http::FilterChainManager & manager, bool only_create_if_configured,
+               const Http::FilterChainOptions& options),
               (const, override));
   MOCK_METHOD(bool, createUpgradeFilterChain,
               (absl::string_view upgrade_type,

From aa4cdd9264d4c02ba7a034cb83e9e0b4e1c36828 Mon Sep 17 00:00:00 2001
From: Matt Klein 
Date: Mon, 12 Jun 2023 15:40:21 -0400
Subject: [PATCH 518/740] owners: change email (#27924)

Signed-off-by: Matt Klein 
---
 OWNERS.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/OWNERS.md b/OWNERS.md
index 90f167fd9166..6cc1566b9105 100644
--- a/OWNERS.md
+++ b/OWNERS.md
@@ -7,7 +7,7 @@ routing PRs, questions, etc. to the right place.
 
 # Senior maintainers
 
-* Matt Klein ([mattklein123](https://github.com/mattklein123)) (mklein@lyft.com)
+* Matt Klein ([mattklein123](https://github.com/mattklein123)) (mattklein123@gmail.com)
   * Catch-all, "all the things", and generally trying to make himself obsolete as fast as
     possible.
 * Harvey Tuch ([htuch](https://github.com/htuch)) (htuch@google.com)

From 66f414d988952024bda32d61ebc76d0b025b9d42 Mon Sep 17 00:00:00 2001
From: Tianyu <72890320+tyxia@users.noreply.github.com>
Date: Mon, 12 Jun 2023 19:54:17 -0400
Subject: [PATCH 519/740] composite: Implements onStreamComplete (#27913)

* implements onStreamComplete

Signed-off-by: tyxia 
---
 source/extensions/filters/http/composite/filter.cc    | 10 ++++++++++
 source/extensions/filters/http/composite/filter.h     |  9 +++++++++
 test/extensions/filters/http/composite/filter_test.cc |  4 ++++
 3 files changed, 23 insertions(+)

diff --git a/source/extensions/filters/http/composite/filter.cc b/source/extensions/filters/http/composite/filter.cc
index e3c7ecfbf478..aace379d0f05 100644
--- a/source/extensions/filters/http/composite/filter.cc
+++ b/source/extensions/filters/http/composite/filter.cc
@@ -200,6 +200,16 @@ void Filter::StreamFilterWrapper::onDestroy() {
   }
 }
 
+void Filter::StreamFilterWrapper::onStreamComplete() {
+  if (decoder_filter_) {
+    decoder_filter_->onStreamComplete();
+  }
+
+  if (encoder_filter_) {
+    encoder_filter_->onStreamComplete();
+  }
+}
+
 } // namespace Composite
 } // namespace HttpFilters
 } // namespace Extensions
diff --git a/source/extensions/filters/http/composite/filter.h b/source/extensions/filters/http/composite/filter.h
index e76de9750b84..bd0e56c8571a 100644
--- a/source/extensions/filters/http/composite/filter.h
+++ b/source/extensions/filters/http/composite/filter.h
@@ -64,6 +64,14 @@ class Filter : public Http::StreamFilter,
     }
   }
 
+  void onStreamComplete() override {
+    if (delegated_filter_) {
+      // We need to explicitly specify which base class to the conversion via due
+      // to the diamond inheritance between StreamFilter and StreamFilterBase.
+      static_cast(*delegated_filter_).onStreamComplete();
+    }
+  }
+
   void onMatchCallback(const Matcher::Action& action) override;
 
   // AccessLog::Instance
@@ -118,6 +126,7 @@ class Filter : public Http::StreamFilter,
 
     // Http::StreamFilterBase
     void onDestroy() override;
+    void onStreamComplete() override;
 
   private:
     Http::StreamEncoderFilterSharedPtr encoder_filter_;
diff --git a/test/extensions/filters/http/composite/filter_test.cc b/test/extensions/filters/http/composite/filter_test.cc
index c7595e762198..4085800097ac 100644
--- a/test/extensions/filters/http/composite/filter_test.cc
+++ b/test/extensions/filters/http/composite/filter_test.cc
@@ -101,7 +101,9 @@ TEST_F(FilterTest, StreamEncoderFilterDelegation) {
   doAllDecodingCallbacks();
   expectDelegatedEncoding(*stream_filter);
   doAllEncodingCallbacks();
+  EXPECT_CALL(*stream_filter, onStreamComplete());
   EXPECT_CALL(*stream_filter, onDestroy());
+  filter_.onStreamComplete();
   filter_.onDestroy();
 }
 
@@ -120,7 +122,9 @@ TEST_F(FilterTest, StreamDecoderFilterDelegation) {
   expectDelegatedDecoding(*stream_filter);
   doAllDecodingCallbacks();
   doAllEncodingCallbacks();
+  EXPECT_CALL(*stream_filter, onStreamComplete());
   EXPECT_CALL(*stream_filter, onDestroy());
+  filter_.onStreamComplete();
   filter_.onDestroy();
 }
 

From ce456d54777392e4ef1a0b459a794fda9a960c53 Mon Sep 17 00:00:00 2001
From: yanjunxiang-google
 <78807980+yanjunxiang-google@users.noreply.github.com>
Date: Mon, 12 Jun 2023 19:56:21 -0400
Subject: [PATCH 520/740] ext_proc fuzzer test trigger ENVOY_BUG when clear
 route cache  (#27657)

* ext_proc fuzzer test trigger ENVOY_BUG when clear route cache for upstream response

Signed-off-by: Yanjun Xiang 
---
 .../filters/http/ext_proc/processor_state.cc  | 43 ++++++++--------
 .../filters/http/ext_proc/processor_state.h   |  5 +-
 ..._proc_grpc_fuzz_test-4756668218736640.test |  1 +
 .../ext_proc/ext_proc_integration_test.cc     | 51 +++++++++++++++++++
 .../filters/http/ext_proc/filter_test.cc      | 22 +++++---
 5 files changed, 91 insertions(+), 31 deletions(-)
 create mode 100644 test/extensions/filters/http/ext_proc/ext_proc_grpc_corpus/clusterfuzz-testcase-minimized-ext_proc_grpc_fuzz_test-4756668218736640.test

diff --git a/source/extensions/filters/http/ext_proc/processor_state.cc b/source/extensions/filters/http/ext_proc/processor_state.cc
index e4cd7e0b9175..4e1ea746ac59 100644
--- a/source/extensions/filters/http/ext_proc/processor_state.cc
+++ b/source/extensions/filters/http/ext_proc/processor_state.cc
@@ -93,10 +93,7 @@ absl::Status ProcessorState::handleHeadersResponse(const HeadersResponse& respon
       }
     }
 
-    if (common_response.clear_route_cache()) {
-      clearRouteCache(common_response);
-    }
-
+    clearRouteCache(common_response);
     onFinishProcessorCall(Grpc::Status::Ok);
 
     if (common_response.status() == CommonResponse::CONTINUE_AND_REPLACE) {
@@ -324,10 +321,7 @@ absl::Status ProcessorState::handleBodyResponse(const BodyResponse& response) {
       onFinishProcessorCall(Grpc::Status::FailedPrecondition);
     }
 
-    if (common_response.clear_route_cache()) {
-      clearRouteCache(common_response);
-    }
-
+    clearRouteCache(common_response);
     headers_ = nullptr;
 
     if (send_trailers_ && trailers_available_) {
@@ -365,20 +359,6 @@ absl::Status ProcessorState::handleTrailersResponse(const TrailersResponse& resp
   return absl::FailedPreconditionError("spurious message");
 }
 
-void ProcessorState::clearRouteCache(const CommonResponse& common_response) {
-  // Only clear the route cache if there is a mutation to the header and clearing is allowed.
-  if (filter_.config().disableClearRouteCache()) {
-    filter_.stats().clear_route_cache_disabled_.inc();
-    ENVOY_LOG(debug, "NOT clearing route cache, it is disabled in the config");
-  } else if (common_response.has_header_mutation()) {
-    ENVOY_LOG(debug, "clearing route cache");
-    filter_callbacks_->downstreamCallbacks()->clearRouteCache();
-  } else {
-    filter_.stats().clear_route_cache_ignored_.inc();
-    ENVOY_LOG(debug, "NOT clearing route cache, no header mutations detected");
-  }
-}
-
 void ProcessorState::enqueueStreamingChunk(Buffer::Instance& data, bool end_stream,
                                            bool delivered) {
   chunk_queue_.push(data, end_stream, delivered);
@@ -440,6 +420,25 @@ void DecodingProcessorState::clearWatermark() {
   }
 }
 
+void DecodingProcessorState::clearRouteCache(const CommonResponse& common_response) {
+  if (!common_response.clear_route_cache()) {
+    return;
+  }
+  // Only clear the route cache if there is a mutation to the header and clearing is allowed.
+  if (filter_.config().disableClearRouteCache()) {
+    filter_.stats().clear_route_cache_disabled_.inc();
+    ENVOY_LOG(debug, "NOT clearing route cache, it is disabled in the config");
+    return;
+  }
+  if (common_response.has_header_mutation()) {
+    ENVOY_LOG(debug, "clearing route cache");
+    decoder_callbacks_->downstreamCallbacks()->clearRouteCache();
+    return;
+  }
+  filter_.stats().clear_route_cache_ignored_.inc();
+  ENVOY_LOG(debug, "NOT clearing route cache, no header mutations detected");
+}
+
 void EncodingProcessorState::setProcessingModeInternal(const ProcessingMode& mode) {
   // Account for the different default behaviors of headers and trailers --
   // headers are sent by default and trailers are not.
diff --git a/source/extensions/filters/http/ext_proc/processor_state.h b/source/extensions/filters/http/ext_proc/processor_state.h
index 32bff6adabd0..c921cdb322c2 100644
--- a/source/extensions/filters/http/ext_proc/processor_state.h
+++ b/source/extensions/filters/http/ext_proc/processor_state.h
@@ -210,7 +210,7 @@ class ProcessorState : public Logger::Loggable {
   const envoy::config::core::v3::TrafficDirection traffic_direction_;
 
 private:
-  void clearRouteCache(const envoy::service::ext_proc::v3::CommonResponse& common_response);
+  virtual void clearRouteCache(const envoy::service::ext_proc::v3::CommonResponse&) {}
 };
 
 class DecodingProcessorState : public ProcessorState {
@@ -280,6 +280,9 @@ class DecodingProcessorState : public ProcessorState {
   void setProcessingModeInternal(
       const envoy::extensions::filters::http::ext_proc::v3::ProcessingMode& mode);
 
+  void
+  clearRouteCache(const envoy::service::ext_proc::v3::CommonResponse& common_response) override;
+
   Http::StreamDecoderFilterCallbacks* decoder_callbacks_{};
 };
 
diff --git a/test/extensions/filters/http/ext_proc/ext_proc_grpc_corpus/clusterfuzz-testcase-minimized-ext_proc_grpc_fuzz_test-4756668218736640.test b/test/extensions/filters/http/ext_proc/ext_proc_grpc_corpus/clusterfuzz-testcase-minimized-ext_proc_grpc_fuzz_test-4756668218736640.test
new file mode 100644
index 000000000000..d065d4bb9763
--- /dev/null
+++ b/test/extensions/filters/http/ext_proc/ext_proc_grpc_corpus/clusterfuzz-testcase-minimized-ext_proc_grpc_fuzz_test-4756668218736640.test
@@ -0,0 +1 @@
+ext_proc_data: "scterpc_cre:csp_yilooo0\n\000*!pV1:ae!FoFFF,F\n"
diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc
index 74f011c3a541..3fef3b403fa1 100644
--- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc
+++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc
@@ -2086,4 +2086,55 @@ TEST_P(ExtProcIntegrationTest, GetAndSetHeadersAndTrailersWithHeaderScrubbing) {
   verifyDownstreamResponse(*response, 200);
 }
 
+// Test clear route cache in both upstream and downstream header and body processing.
+TEST_P(ExtProcIntegrationTest, GetAndSetBodyOnBothWithClearRouteCache) {
+  proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::STREAMED);
+  proto_config_.mutable_processing_mode()->set_response_body_mode(ProcessingMode::STREAMED);
+  initializeConfig();
+  HttpIntegrationTest::initialize();
+
+  auto response = sendDownstreamRequestWithBody("Replace this!", absl::nullopt);
+  processRequestHeadersMessage(
+      *grpc_upstreams_[0], true, [](const HttpHeaders&, HeadersResponse& headers_resp) {
+        auto* content_length =
+            headers_resp.mutable_response()->mutable_header_mutation()->add_set_headers();
+        content_length->mutable_header()->set_key("content-length");
+        content_length->mutable_header()->set_value("13");
+        headers_resp.mutable_response()->set_clear_route_cache(true);
+        return true;
+      });
+  processRequestBodyMessage(
+      *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) {
+        EXPECT_TRUE(body.end_of_stream());
+        auto* body_mut = body_resp.mutable_response()->mutable_body_mutation();
+        body_mut->set_body("Hello, World!");
+        body_resp.mutable_response()->set_clear_route_cache(true);
+        return true;
+      });
+
+  ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_));
+  ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_));
+  ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_));
+  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false);
+
+  processResponseHeadersMessage(
+      *grpc_upstreams_[0], false, [](const HttpHeaders&, HeadersResponse& headers_resp) {
+        headers_resp.mutable_response()->mutable_header_mutation()->add_remove_headers(
+            "content-length");
+        headers_resp.mutable_response()->set_clear_route_cache(true);
+        return true;
+      });
+  upstream_request_->encodeData(100, true);
+  processResponseBodyMessage(
+      *grpc_upstreams_[0], false, [](const HttpBody&, BodyResponse& body_resp) {
+        auto* header_mut = body_resp.mutable_response()->mutable_header_mutation();
+        auto* header_add = header_mut->add_set_headers();
+        header_add->mutable_header()->set_key("x-testing-response-header");
+        header_add->mutable_header()->set_value("Yes");
+        body_resp.mutable_response()->set_clear_route_cache(true);
+        return true;
+      });
+  verifyDownstreamResponse(*response, 200);
+}
+
 } // namespace Envoy
diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc
index 3f54f761d261..3c77b8c99957 100644
--- a/test/extensions/filters/http/ext_proc/filter_test.cc
+++ b/test/extensions/filters/http/ext_proc/filter_test.cc
@@ -2064,7 +2064,8 @@ TEST_F(HttpFilterTest, ProcessingModeResponseHeadersOnlyWithoutCallingDecodeHead
 }
 
 // Using the default configuration, verify that the "clear_route_cache" flag makes the appropriate
-// callback on the filter when header modifications are also present.
+// callback on the filter for inbound traffic when header modifications are also present.
+// Also verify it does not make the callback for outbound traffic.
 TEST_F(HttpFilterTest, ClearRouteCacheHeaderMutation) {
   initialize(R"EOF(
   grpc_service:
@@ -2075,7 +2076,7 @@ TEST_F(HttpFilterTest, ClearRouteCacheHeaderMutation) {
   )EOF");
 
   EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, true));
-
+  // Call ClearRouteCache() for inbound traffic with header mutation.
   EXPECT_CALL(decoder_callbacks_.downstream_callbacks_, clearRouteCache());
   processRequestHeaders(false, [](const HttpHeaders&, ProcessingResponse&, HeadersResponse& resp) {
     auto* resp_headers_mut = resp.mutable_response()->mutable_header_mutation();
@@ -2092,9 +2093,8 @@ TEST_F(HttpFilterTest, ClearRouteCacheHeaderMutation) {
   Buffer::OwnedImpl buffered_response_data;
   setUpEncodingBuffering(buffered_response_data);
 
+  // There is no ClearRouteCache() call for outbound traffic.
   EXPECT_EQ(FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(resp_data, true));
-
-  EXPECT_CALL(encoder_callbacks_.downstream_callbacks_, clearRouteCache());
   processResponseBody([](const HttpBody&, ProcessingResponse&, BodyResponse& resp) {
     auto* resp_headers_mut = resp.mutable_response()->mutable_header_mutation();
     auto* resp_add = resp_headers_mut->add_set_headers();
@@ -2105,6 +2105,8 @@ TEST_F(HttpFilterTest, ClearRouteCacheHeaderMutation) {
 
   filter_->onDestroy();
 
+  EXPECT_EQ(config_->stats().clear_route_cache_disabled_.value(), 0);
+  EXPECT_EQ(config_->stats().clear_route_cache_ignored_.value(), 0);
   EXPECT_EQ(config_->stats().streams_started_.value(), 1);
   EXPECT_EQ(config_->stats().stream_msgs_sent_.value(), 3);
   EXPECT_EQ(config_->stats().stream_msgs_received_.value(), 3);
@@ -2112,7 +2114,7 @@ TEST_F(HttpFilterTest, ClearRouteCacheHeaderMutation) {
 }
 
 // Verify that the "disable_route_cache_clearing" setting prevents the "clear_route_cache" flag
-// from performing route clearing callbacks when enabled.
+// from performing route clearing callbacks for inbound traffic when enabled.
 TEST_F(HttpFilterTest, ClearRouteCacheDisabledHeaderMutation) {
   initialize(R"EOF(
   grpc_service:
@@ -2123,6 +2125,7 @@ TEST_F(HttpFilterTest, ClearRouteCacheDisabledHeaderMutation) {
   disable_clear_route_cache: true
   )EOF");
 
+  // The ClearRouteCache() call is disabled for inbound traffic.
   EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, true));
   processRequestHeaders(false, [](const HttpHeaders&, ProcessingResponse&, HeadersResponse& resp) {
     auto* resp_headers_mut = resp.mutable_response()->mutable_header_mutation();
@@ -2139,6 +2142,7 @@ TEST_F(HttpFilterTest, ClearRouteCacheDisabledHeaderMutation) {
   Buffer::OwnedImpl buffered_response_data;
   setUpEncodingBuffering(buffered_response_data);
 
+  // There is no ClearRouteCache() call for outbound traffic regardless.
   EXPECT_EQ(FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(resp_data, true));
   processResponseBody([](const HttpBody&, ProcessingResponse&, BodyResponse& resp) {
     auto* resp_headers_mut = resp.mutable_response()->mutable_header_mutation();
@@ -2150,8 +2154,8 @@ TEST_F(HttpFilterTest, ClearRouteCacheDisabledHeaderMutation) {
 
   filter_->onDestroy();
 
+  EXPECT_EQ(config_->stats().clear_route_cache_disabled_.value(), 1);
   EXPECT_EQ(config_->stats().clear_route_cache_ignored_.value(), 0);
-  EXPECT_EQ(config_->stats().clear_route_cache_disabled_.value(), 2);
   EXPECT_EQ(config_->stats().streams_started_.value(), 1);
   EXPECT_EQ(config_->stats().stream_msgs_sent_.value(), 3);
   EXPECT_EQ(config_->stats().stream_msgs_received_.value(), 3);
@@ -2159,7 +2163,7 @@ TEST_F(HttpFilterTest, ClearRouteCacheDisabledHeaderMutation) {
 }
 
 // Using the default configuration, verify that the "clear_route_cache" flag does not preform
-// route clearing callbacks on the filter when no header changes are present.
+// route clearing callbacks for inbound traffic when no header changes are present.
 TEST_F(HttpFilterTest, ClearRouteCacheUnchanged) {
   initialize(R"EOF(
   grpc_service:
@@ -2169,6 +2173,7 @@ TEST_F(HttpFilterTest, ClearRouteCacheUnchanged) {
     response_body_mode: "BUFFERED"
   )EOF");
 
+  // Do not call ClearRouteCache() for inbound traffic without header mutation.
   EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, true));
   processRequestHeaders(false, [](const HttpHeaders&, ProcessingResponse&, HeadersResponse& resp) {
     resp.mutable_response()->set_clear_route_cache(true);
@@ -2181,6 +2186,7 @@ TEST_F(HttpFilterTest, ClearRouteCacheUnchanged) {
   Buffer::OwnedImpl buffered_response_data;
   setUpEncodingBuffering(buffered_response_data);
 
+  // There is no ClearRouteCache() call for outbound traffic regardless.
   EXPECT_EQ(FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(resp_data, true));
   processResponseBody([](const HttpBody&, ProcessingResponse&, BodyResponse& resp) {
     resp.mutable_response()->set_clear_route_cache(true);
@@ -2188,7 +2194,7 @@ TEST_F(HttpFilterTest, ClearRouteCacheUnchanged) {
 
   filter_->onDestroy();
 
-  EXPECT_EQ(config_->stats().clear_route_cache_ignored_.value(), 2);
+  EXPECT_EQ(config_->stats().clear_route_cache_ignored_.value(), 1);
   EXPECT_EQ(config_->stats().clear_route_cache_disabled_.value(), 0);
   EXPECT_EQ(config_->stats().streams_started_.value(), 1);
   EXPECT_EQ(config_->stats().stream_msgs_sent_.value(), 3);

From 7d664bc26d30936bfaa990ed6e276617ce7b7812 Mon Sep 17 00:00:00 2001
From: Tianyu <72890320+tyxia@users.noreply.github.com>
Date: Mon, 12 Jun 2023 20:20:24 -0400
Subject: [PATCH 521/740] Create downstream filter from server factory context
 (#27621)

Signed-off-by: tyxia 
---
 envoy/server/filter_config.h                  |  18 +
 source/common/http/match_delegate/config.cc   |   4 +-
 source/common/http/matching/data_impl.h       |   3 +-
 .../filters/http/common/factory_base.h        |  17 +
 .../filters/http/composite/action.h           |  30 +-
 .../filters/http/ext_proc/config.cc           |  21 +
 .../extensions/filters/http/ext_proc/config.h |   5 +
 test/config/utility.h                         |   3 +-
 test/extensions/filters/http/composite/BUILD  |  16 +
 .../composite_filter_integration_test.cc      | 480 ++++++++++++++++++
 test/integration/filters/BUILD                |  25 +
 .../filters/server_factory_context_filter.cc  | 152 ++++++
 ...server_factory_context_filter_config.proto |   9 +
 13 files changed, 776 insertions(+), 7 deletions(-)
 create mode 100644 test/integration/filters/server_factory_context_filter.cc
 create mode 100644 test/integration/filters/server_factory_context_filter_config.proto

diff --git a/envoy/server/filter_config.h b/envoy/server/filter_config.h
index 98af7d3e8bde..ace1d61cd619 100644
--- a/envoy/server/filter_config.h
+++ b/envoy/server/filter_config.h
@@ -250,6 +250,24 @@ class NamedHttpFilterConfigFactory : public virtual HttpFilterConfigFactoryBase
   virtual Http::FilterFactoryCb
   createFilterFactoryFromProto(const Protobuf::Message& config, const std::string& stat_prefix,
                                Server::Configuration::FactoryContext& context) PURE;
+
+  /**
+   * Create a particular http filter factory implementation. If the implementation is unable to
+   * produce a factory with the provided parameters or this method is not supported, it should throw
+   * an EnvoyException. The returned callback should always be initialized.
+   * @param config supplies the general Protobuf message to be marshaled into a filter-specific
+   * configuration.
+   * @param stat_prefix prefix for stat logging
+   * @param context supplies the filter's ServerFactoryContext.
+   * @return Http::FilterFactoryCb the factory creation function.
+   */
+  virtual Http::FilterFactoryCb
+  createFilterServerFactoryFromProto(const Protobuf::Message&, const std::string&,
+                                     Server::Configuration::ServerFactoryContext&) {
+    ExceptionUtil::throwEnvoyException(
+        "Creating filter factory from server factory context is not supported");
+    return nullptr;
+  }
 };
 
 class UpstreamHttpFilterConfigFactory : public virtual HttpFilterConfigFactoryBase {
diff --git a/source/common/http/match_delegate/config.cc b/source/common/http/match_delegate/config.cc
index d4f4bee35584..587c8de71104 100644
--- a/source/common/http/match_delegate/config.cc
+++ b/source/common/http/match_delegate/config.cc
@@ -260,7 +260,9 @@ Envoy::Http::FilterFactoryCb MatchDelegateConfig::createFilterFactoryFromProtoTy
 
   Factory::MatchTreeValidationVisitor validation_visitor(*factory.matchingRequirements());
 
-  Envoy::Http::Matching::HttpFilterActionContext action_context{prefix, context};
+  Envoy::Http::Matching::HttpFilterActionContext action_context{prefix, context,
+                                                                context.getServerFactoryContext()};
+
   Matcher::MatchTreeFactory
       matcher_factory(action_context, context.getServerFactoryContext(), validation_visitor);
diff --git a/source/common/http/matching/data_impl.h b/source/common/http/matching/data_impl.h
index ded401d85553..7bcf9e251d12 100644
--- a/source/common/http/matching/data_impl.h
+++ b/source/common/http/matching/data_impl.h
@@ -68,7 +68,8 @@ using HttpMatchingDataImplSharedPtr = std::shared_ptr;
 
 struct HttpFilterActionContext {
   const std::string& stat_prefix_;
-  Server::Configuration::FactoryContext& factory_context_;
+  OptRef factory_context_;
+  OptRef server_factory_context_;
 };
 } // namespace Matching
 } // namespace Http
diff --git a/source/extensions/filters/http/common/factory_base.h b/source/extensions/filters/http/common/factory_base.h
index 285cf6b2c8c3..4e13c4b8bf1e 100644
--- a/source/extensions/filters/http/common/factory_base.h
+++ b/source/extensions/filters/http/common/factory_base.h
@@ -76,6 +76,23 @@ class FactoryBase : public CommonFactoryBase,
   createFilterFactoryFromProtoTyped(const ConfigProto& proto_config,
                                     const std::string& stats_prefix,
                                     Server::Configuration::FactoryContext& context) PURE;
+
+  Envoy::Http::FilterFactoryCb createFilterServerFactoryFromProto(
+      const Protobuf::Message& proto_config, const std::string& stats_prefix,
+      Server::Configuration::ServerFactoryContext& server_context) override {
+    return createFilterServerFactoryFromProtoTyped(
+        MessageUtil::downcastAndValidate(
+            proto_config, server_context.messageValidationVisitor()),
+        stats_prefix, server_context);
+  }
+
+  virtual Envoy::Http::FilterFactoryCb
+  createFilterServerFactoryFromProtoTyped(const ConfigProto&, const std::string&,
+                                          Server::Configuration::ServerFactoryContext&) {
+    ExceptionUtil::throwEnvoyException(
+        "Creating filter factory from server factory context is not supported");
+    return nullptr;
+  }
 };
 
 template 
diff --git a/source/extensions/filters/http/composite/action.h b/source/extensions/filters/http/composite/action.h
index 05b719cdd4cd..86b72de5ce7c 100644
--- a/source/extensions/filters/http/composite/action.h
+++ b/source/extensions/filters/http/composite/action.h
@@ -23,7 +23,8 @@ class ExecuteFilterAction
 };
 
 class ExecuteFilterActionFactory
-    : public Matcher::ActionFactory {
+    : public Logger::Loggable,
+      public Matcher::ActionFactory {
 public:
   std::string name() const override { return "composite-action"; }
   Matcher::ActionFactoryCb
@@ -39,8 +40,31 @@ class ExecuteFilterActionFactory
             composite_action.typed_config());
     ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig(
         composite_action.typed_config().typed_config(), validation_visitor, factory);
-    auto callback = factory.createFilterFactoryFromProto(*message, context.stat_prefix_,
-                                                         context.factory_context_);
+
+    Envoy::Http::FilterFactoryCb callback = nullptr;
+
+    // TODO(tyxia) Update the logic later to create the filter from the `factoryContext` first if it
+    // is present.
+    try {
+      if (context.server_factory_context_.has_value()) {
+        callback = factory.createFilterServerFactoryFromProto(
+            *message, context.stat_prefix_, context.server_factory_context_.value());
+      }
+    } catch (EnvoyException& e) {
+      // First, we try to create the delegated filter creation callback from server factory context.
+      // If it failed (i.e., the corresponding filter doesn't support this method), we log this
+      // message and fallback to creating the filter from factory context.
+      ENVOY_LOG(trace,
+                absl::StrCat(e.what(), ", fallback to creating the filter from factory context."));
+    }
+
+    if (callback == nullptr) {
+      RELEASE_ASSERT(context.factory_context_.has_value(),
+                     "The factory context must exist here to create the delegated filter");
+      callback = factory.createFilterFactoryFromProto(*message, context.stat_prefix_,
+                                                      context.factory_context_.value());
+    }
+
     return [cb = std::move(callback)]() -> Matcher::ActionPtr {
       return std::make_unique(cb);
     };
diff --git a/source/extensions/filters/http/ext_proc/config.cc b/source/extensions/filters/http/ext_proc/config.cc
index c063e0903bcd..5ccf5a51f1be 100644
--- a/source/extensions/filters/http/ext_proc/config.cc
+++ b/source/extensions/filters/http/ext_proc/config.cc
@@ -36,6 +36,27 @@ ExternalProcessingFilterConfig::createRouteSpecificFilterConfigTyped(
   return std::make_shared(proto_config);
 }
 
+Http::FilterFactoryCb ExternalProcessingFilterConfig::createFilterServerFactoryFromProtoTyped(
+    const envoy::extensions::filters::http::ext_proc::v3::ExternalProcessor& proto_config,
+    const std::string& stats_prefix, Server::Configuration::ServerFactoryContext& server_context) {
+  const uint32_t message_timeout_ms =
+      PROTOBUF_GET_MS_OR_DEFAULT(proto_config, message_timeout, DefaultMessageTimeoutMs);
+  const uint32_t max_message_timeout_ms =
+      PROTOBUF_GET_MS_OR_DEFAULT(proto_config, max_message_timeout, DefaultMaxMessageTimeoutMs);
+  const auto filter_config =
+      std::make_shared(proto_config, std::chrono::milliseconds(message_timeout_ms),
+                                     max_message_timeout_ms, server_context.scope(), stats_prefix);
+
+  return [filter_config, grpc_service = proto_config.grpc_service(),
+          &server_context](Http::FilterChainFactoryCallbacks& callbacks) {
+    auto client = std::make_unique(
+        server_context.clusterManager().grpcAsyncClientManager(), server_context.scope());
+
+    callbacks.addStreamFilter(Http::StreamFilterSharedPtr{
+        std::make_shared(filter_config, std::move(client), grpc_service)});
+  };
+}
+
 LEGACY_REGISTER_FACTORY(ExternalProcessingFilterConfig,
                         Server::Configuration::NamedHttpFilterConfigFactory, "envoy.ext_proc");
 
diff --git a/source/extensions/filters/http/ext_proc/config.h b/source/extensions/filters/http/ext_proc/config.h
index e6dc44c704a2..12a03d6710db 100644
--- a/source/extensions/filters/http/ext_proc/config.h
+++ b/source/extensions/filters/http/ext_proc/config.h
@@ -31,6 +31,11 @@ class ExternalProcessingFilterConfig
       const envoy::extensions::filters::http::ext_proc::v3::ExtProcPerRoute& proto_config,
       Server::Configuration::ServerFactoryContext& context,
       ProtobufMessage::ValidationVisitor& validator) override;
+
+  Http::FilterFactoryCb createFilterServerFactoryFromProtoTyped(
+      const envoy::extensions::filters::http::ext_proc::v3::ExternalProcessor& proto_config,
+      const std::string& stats_prefix,
+      Server::Configuration::ServerFactoryContext& server_context) override;
 };
 
 } // namespace ExternalProcessing
diff --git a/test/config/utility.h b/test/config/utility.h
index 9f8101b7c604..800446c20395 100644
--- a/test/config/utility.h
+++ b/test/config/utility.h
@@ -424,14 +424,13 @@ class ConfigHelper {
   static envoy::config::core::v3::Http3ProtocolOptions
   http2ToHttp3ProtocolOptions(const envoy::config::core::v3::Http2ProtocolOptions& http2_options,
                               size_t http3_max_stream_receive_window);
-
-private:
   // Load the first HCM struct from the first listener into a parsed proto.
   bool loadHttpConnectionManager(HttpConnectionManager& hcm);
   // Take the contents of the provided HCM proto and stuff them into the first HCM
   // struct of the first listener.
   void storeHttpConnectionManager(const HttpConnectionManager& hcm);
 
+private:
   // Load the first FilterType struct from the first listener into a parsed proto.
   template  bool loadFilter(const std::string& name, FilterType& filter) {
     RELEASE_ASSERT(!finalized_, "");
diff --git a/test/extensions/filters/http/composite/BUILD b/test/extensions/filters/http/composite/BUILD
index b69800fa21e8..229c6ba9a006 100644
--- a/test/extensions/filters/http/composite/BUILD
+++ b/test/extensions/filters/http/composite/BUILD
@@ -31,10 +31,26 @@ envoy_extension_cc_test(
     extension_names = ["envoy.filters.http.composite"],
     deps = [
         "//source/common/http:header_map_lib",
+        "//source/common/http/match_delegate:config",
         "//source/extensions/filters/http/composite:config",
         "//source/extensions/filters/http/composite:filter_lib",
+        "//source/extensions/filters/http/ext_proc:config",
+        "//source/extensions/matching/http/cel_input:cel_input_lib",
+        "//source/extensions/matching/input_matchers/cel_matcher:cel_matcher_lib",
+        "//source/extensions/matching/input_matchers/cel_matcher:config",
+        "//test/common/grpc:grpc_client_integration_lib",
+        "//test/common/http:common_lib",
+        "//test/extensions/filters/http/ext_proc:utils_lib",
         "//test/integration:http_integration_lib",
+        "//test/integration/filters:server_factory_context_filter_config_proto_cc_proto",
+        "//test/integration/filters:server_factory_context_filter_lib",
         "//test/integration/filters:set_response_code_filter_lib",
+        "//test/proto:helloworld_proto_cc_proto",
+        "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/filters/http/composite/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/filters/http/ext_proc/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto",
+        "@envoy_api//envoy/service/ext_proc/v3:pkg_cc_proto",
     ],
 )
diff --git a/test/extensions/filters/http/composite/composite_filter_integration_test.cc b/test/extensions/filters/http/composite/composite_filter_integration_test.cc
index 30311497e307..4eb61efb41e6 100644
--- a/test/extensions/filters/http/composite/composite_filter_integration_test.cc
+++ b/test/extensions/filters/http/composite/composite_filter_integration_test.cc
@@ -1,7 +1,21 @@
+#include "envoy/extensions/common/matching/v3/extension_matcher.pb.validate.h"
+#include "envoy/extensions/filters/http/composite/v3/composite.pb.h"
+#include "envoy/extensions/filters/http/ext_proc/v3/ext_proc.pb.h"
 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h"
+#include "envoy/network/address.h"
+#include "envoy/service/ext_proc/v3/external_processor.pb.h"
 
+#include "source/common/http/match_delegate/config.h"
+#include "source/common/http/matching/inputs.h"
+#include "source/extensions/filters/http/ext_proc/config.h"
+
+#include "test/common/grpc/grpc_client_integration.h"
+#include "test/common/http/common.h"
+#include "test/extensions/filters/http/ext_proc/utils.h"
+#include "test/integration/filters/server_factory_context_filter_config.pb.h"
 #include "test/integration/http_integration.h"
 #include "test/mocks/http/mocks.h"
+#include "test/proto/helloworld.pb.h"
 #include "test/test_common/utility.h"
 
 #include "gtest/gtest.h"
@@ -77,4 +91,470 @@ TEST_P(CompositeFilterIntegrationTest, TestBasic) {
   }
 }
 
+using envoy::extensions::filters::http::ext_proc::v3::ProcessingMode;
+using envoy::service::ext_proc::v3::HeadersResponse;
+using envoy::service::ext_proc::v3::HttpHeaders;
+using envoy::service::ext_proc::v3::ProcessingRequest;
+using envoy::service::ext_proc::v3::ProcessingResponse;
+using Extensions::HttpFilters::ExternalProcessing::HasNoHeader;
+using Extensions::HttpFilters::ExternalProcessing::HeaderProtosEqual;
+using Extensions::HttpFilters::ExternalProcessing::SingleHeaderValueIs;
+
+// Integration test that has ext_proc filter as the delegated filter.
+class CompositeFilterWithExtProcIntegrationTest
+    : public HttpIntegrationTest,
+      public Grpc::GrpcClientIntegrationParamTestWithDeferredProcessing {
+public:
+  CompositeFilterWithExtProcIntegrationTest()
+      : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {}
+
+  void createUpstreams() override {
+    HttpIntegrationTest::createUpstreams();
+    // Create separate "upstreams" for ExtProc gRPC servers
+    for (int i = 0; i < 2; ++i) {
+      grpc_upstreams_.push_back(&addFakeUpstream(Http::CodecType::HTTP2));
+    }
+  }
+
+  void TearDown() override {
+    if (processor_connection_) {
+      ASSERT_TRUE(processor_connection_->close());
+      ASSERT_TRUE(processor_connection_->waitForDisconnect());
+    }
+    cleanupUpstreamAndDownstream();
+  }
+
+  void initializeConfig() {
+    config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+      // Ensure "HTTP2 with no prior knowledge." Necessary for gRPC and for headers
+      ConfigHelper::setHttp2(
+          *(bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0)));
+
+      // Clusters for ExtProc gRPC servers, starting by copying an existing cluster
+      for (size_t i = 0; i < grpc_upstreams_.size(); ++i) {
+        auto* server_cluster = bootstrap.mutable_static_resources()->add_clusters();
+        server_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]);
+        std::string cluster_name = absl::StrCat("ext_proc_server_", i);
+        server_cluster->set_name(cluster_name);
+        server_cluster->mutable_load_assignment()->set_cluster_name(cluster_name);
+      }
+      // Load configuration of the server from YAML and use a helper to add a grpc_service
+      // stanza pointing to the cluster that we just made
+      setGrpcService(*proto_config_.mutable_grpc_service(), "ext_proc_server_0",
+                     grpc_upstreams_[0]->localAddress());
+
+      addFilter();
+
+      // Parameterize with defer processing to prevent bit rot as filter made
+      // assumptions of data flow, prior relying on eager processing.
+      config_helper_.addRuntimeOverride(Runtime::defer_processing_backedup_streams,
+                                        deferredProcessing() ? "true" : "false");
+    });
+
+    setUpstreamProtocol(Http::CodecType::HTTP2);
+    setDownstreamProtocol(Http::CodecType::HTTP2);
+  }
+
+  void addFilter() {
+    // Add the filter to the loaded hcm.
+    envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager
+        hcm_config;
+    config_helper_.loadHttpConnectionManager(hcm_config);
+    auto* match_delegate_filter = hcm_config.add_http_filters();
+    match_delegate_filter->set_name("envoy.filters.http.match_delegate");
+
+    // Build extension config with composite filter inside.
+    envoy::extensions::common::matching::v3::ExtensionWithMatcher extension_config;
+    extension_config.mutable_extension_config()->set_name("composite");
+    envoy::extensions::filters::http::composite::v3::Composite composite_config;
+    extension_config.mutable_extension_config()->mutable_typed_config()->PackFrom(composite_config);
+    auto* matcher_tree = extension_config.mutable_xds_matcher()->mutable_matcher_tree();
+
+    // Set up the match input.
+    auto* matcher_input = matcher_tree->mutable_input();
+    matcher_input->set_name("request-headers");
+    envoy::type::matcher::v3::HttpRequestHeaderMatchInput request_header_match_input;
+    request_header_match_input.set_header_name("match-header");
+    matcher_input->mutable_typed_config()->PackFrom(request_header_match_input);
+
+    // Set up the match action with ext_proc filter as the delegated filter.
+    auto* exact_match_map = matcher_tree->mutable_exact_match_map()->mutable_map();
+    envoy::extensions::filters::http::composite::v3::ExecuteFilterAction ext_proc_filter_action;
+    ext_proc_filter_action.mutable_typed_config()->set_name("envoy.filters.http.ext_proc");
+    // Set up ext_proc processing mode.
+    proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SEND);
+    proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SEND);
+    proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::BUFFERED);
+    proto_config_.mutable_processing_mode()->set_response_body_mode(ProcessingMode::NONE);
+    proto_config_.mutable_processing_mode()->set_request_trailer_mode(ProcessingMode::SKIP);
+    proto_config_.mutable_processing_mode()->set_response_trailer_mode(ProcessingMode::SKIP);
+    ext_proc_filter_action.mutable_typed_config()->mutable_typed_config()->PackFrom(proto_config_);
+
+    ::xds::type::matcher::v3::Matcher_OnMatch on_match;
+    auto* on_match_action = on_match.mutable_action();
+    on_match_action->set_name("composite-action");
+    on_match_action->mutable_typed_config()->PackFrom(ext_proc_filter_action);
+
+    (*exact_match_map)["match"] = on_match;
+
+    // Finish up the construction of match_delegate_filter.
+    match_delegate_filter->mutable_typed_config()->PackFrom(extension_config);
+
+    // Now move the built filter to the front.
+    for (int i = hcm_config.http_filters_size() - 1; i > 0; --i) {
+      hcm_config.mutable_http_filters()->SwapElements(i, i - 1);
+    }
+
+    // Store it to hcm.
+    config_helper_.storeHttpConnectionManager(hcm_config);
+  }
+
+  IntegrationStreamDecoderPtr sendDownstreamRequest(
+      absl::optional> modify_headers) {
+    auto conn = makeClientConnection(lookupPort("http"));
+    codec_client_ = makeHttpConnection(std::move(conn));
+    Http::TestRequestHeaderMapImpl headers;
+    HttpTestUtility::addDefaultHeaders(headers);
+    if (modify_headers) {
+      (*modify_headers)(headers);
+    }
+    return codec_client_->makeHeaderOnlyRequest(headers);
+  }
+
+  void waitForFirstMessage(FakeUpstream& grpc_upstream,
+                           envoy::service::ext_proc::v3::ProcessingRequest& request) {
+    ASSERT_TRUE(grpc_upstream.waitForHttpConnection(*dispatcher_, processor_connection_));
+    ASSERT_TRUE(processor_connection_->waitForNewStream(*dispatcher_, processor_stream_));
+    ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, request));
+  }
+
+  void handleUpstreamRequest() {
+    ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_));
+    ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_));
+    ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_));
+    upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false);
+    upstream_request_->encodeData(100, true);
+  }
+
+  void verifyDownstreamResponse(IntegrationStreamDecoder& response, int status_code) {
+    ASSERT_TRUE(response.waitForEndStream());
+    EXPECT_TRUE(response.complete());
+    EXPECT_EQ(std::to_string(status_code), response.headers().getStatusValue());
+  }
+
+  void processRequestHeadersMessage(
+      FakeUpstream& grpc_upstream, bool first_message,
+      absl::optional> cb) {
+    ProcessingRequest request;
+    if (first_message) {
+      ASSERT_TRUE(grpc_upstream.waitForHttpConnection(*dispatcher_, processor_connection_));
+      ASSERT_TRUE(processor_connection_->waitForNewStream(*dispatcher_, processor_stream_));
+    }
+    ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, request));
+    ASSERT_TRUE(request.has_request_headers());
+    if (first_message) {
+      processor_stream_->startGrpcStream();
+    }
+    ProcessingResponse response;
+    auto* headers = response.mutable_request_headers();
+    const bool sendReply = !cb || (*cb)(request.request_headers(), *headers);
+    if (sendReply) {
+      processor_stream_->sendGrpcMessage(response);
+    }
+  }
+
+  void processResponseHeadersMessage(
+      FakeUpstream& grpc_upstream, bool first_message,
+      absl::optional> cb) {
+    ProcessingRequest request;
+    if (first_message) {
+      ASSERT_TRUE(grpc_upstream.waitForHttpConnection(*dispatcher_, processor_connection_));
+      ASSERT_TRUE(processor_connection_->waitForNewStream(*dispatcher_, processor_stream_));
+    }
+    ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, request));
+    ASSERT_TRUE(request.has_response_headers());
+    if (first_message) {
+      processor_stream_->startGrpcStream();
+    }
+    ProcessingResponse response;
+    auto* headers = response.mutable_response_headers();
+    const bool sendReply = !cb || (*cb)(request.response_headers(), *headers);
+    if (sendReply) {
+      processor_stream_->sendGrpcMessage(response);
+    }
+  }
+
+  envoy::extensions::filters::http::ext_proc::v3::ExternalProcessor proto_config_{};
+  std::vector grpc_upstreams_;
+  FakeHttpConnectionPtr processor_connection_;
+  FakeStreamPtr processor_stream_;
+};
+
+INSTANTIATE_TEST_SUITE_P(
+    IpVersionsClientTypeDeferredProcessing, CompositeFilterWithExtProcIntegrationTest,
+    GRPC_CLIENT_INTEGRATION_DEFERRED_PROCESSING_PARAMS,
+    Grpc::GrpcClientIntegrationParamTestWithDeferredProcessing::protocolTestParamsToString);
+
+TEST_P(CompositeFilterWithExtProcIntegrationTest, GetAndCloseStream) {
+  initializeConfig();
+  HttpIntegrationTest::initialize();
+
+  auto conn = makeClientConnection(lookupPort("http"));
+  codec_client_ = makeHttpConnection(std::move(conn));
+  const Http::TestRequestHeaderMapImpl request_headers = {{":method", "GET"},
+                                                          {":path", "/somepath"},
+                                                          {":scheme", "http"},
+                                                          {"match-header", "match"},
+                                                          {":authority", "blah"}};
+  auto response = codec_client_->makeHeaderOnlyRequest(request_headers);
+
+  envoy::service::ext_proc::v3::ProcessingRequest request_headers_msg;
+  waitForFirstMessage(*grpc_upstreams_[0], request_headers_msg);
+  // Just close the stream without doing anything
+  processor_stream_->startGrpcStream();
+  processor_stream_->finishGrpcStream(Grpc::Status::Ok);
+
+  handleUpstreamRequest();
+  verifyDownstreamResponse(*response, 200);
+}
+
+// Test the filter using the default configuration by connecting to
+// an ext_proc server that responds to the request_headers message
+// successfully but closes the stream after response_headers.
+TEST_P(CompositeFilterWithExtProcIntegrationTest, GetAndCloseStreamOnResponse) {
+  initializeConfig();
+  HttpIntegrationTest::initialize();
+
+  auto conn = makeClientConnection(lookupPort("http"));
+  codec_client_ = makeHttpConnection(std::move(conn));
+  const Http::TestRequestHeaderMapImpl request_headers = {{":method", "GET"},
+                                                          {":path", "/somepath"},
+                                                          {":scheme", "http"},
+                                                          {"match-header", "match"},
+                                                          {":authority", "blah"}};
+  auto response = codec_client_->makeHeaderOnlyRequest(request_headers);
+
+  ProcessingRequest request_headers_msg;
+  waitForFirstMessage(*grpc_upstreams_[0], request_headers_msg);
+  processor_stream_->startGrpcStream();
+  ProcessingResponse resp1;
+  resp1.mutable_request_headers();
+  processor_stream_->sendGrpcMessage(resp1);
+
+  handleUpstreamRequest();
+
+  ProcessingRequest response_headers_msg;
+  ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, response_headers_msg));
+  processor_stream_->finishGrpcStream(Grpc::Status::Ok);
+
+  verifyDownstreamResponse(*response, 200);
+}
+
+TEST_P(CompositeFilterWithExtProcIntegrationTest, GetAndSetHeaders) {
+  initializeConfig();
+  HttpIntegrationTest::initialize();
+
+  auto conn = makeClientConnection(lookupPort("http"));
+  codec_client_ = makeHttpConnection(std::move(conn));
+  Http::TestRequestHeaderMapImpl request_headers = {{":method", "GET"},
+                                                    {":path", "/somepath"},
+                                                    {":scheme", "http"},
+                                                    {"match-header", "match"},
+                                                    {":authority", "blah"}};
+  request_headers.addCopy(Http::LowerCaseString("x-remove-this"), "yes");
+  auto response = codec_client_->makeHeaderOnlyRequest(request_headers);
+
+  processRequestHeadersMessage(
+      *grpc_upstreams_[0], true, [](const HttpHeaders& headers, HeadersResponse& headers_resp) {
+        Http::TestRequestHeaderMapImpl expected_request_headers{
+            {":method", "GET"},           {":path", "/somepath"}, {":scheme", "http"},
+            {"match-header", "match"},    {":authority", "blah"}, {"x-remove-this", "yes"},
+            {"x-forwarded-proto", "http"}};
+        EXPECT_THAT(headers.headers(), HeaderProtosEqual(expected_request_headers));
+
+        auto response_header_mutation = headers_resp.mutable_response()->mutable_header_mutation();
+        auto* mut1 = response_header_mutation->add_set_headers();
+        mut1->mutable_header()->set_key("x-new-header");
+        mut1->mutable_header()->set_value("new");
+        response_header_mutation->add_remove_headers("x-remove-this");
+        return true;
+      });
+
+  ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_));
+  ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_));
+  ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_));
+
+  EXPECT_THAT(upstream_request_->headers(), HasNoHeader("x-remove-this"));
+  EXPECT_THAT(upstream_request_->headers(), SingleHeaderValueIs("x-new-header", "new"));
+
+  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false);
+  upstream_request_->encodeData(100, true);
+
+  processResponseHeadersMessage(
+      *grpc_upstreams_[0], false, [](const HttpHeaders& headers, HeadersResponse&) {
+        Http::TestRequestHeaderMapImpl expected_response_headers{{":status", "200"}};
+        EXPECT_THAT(headers.headers(), HeaderProtosEqual(expected_response_headers));
+        return true;
+      });
+
+  verifyDownstreamResponse(*response, 200);
+}
+
+class CompositeFilterSeverContextIntegrationTest
+    : public HttpIntegrationTest,
+      public Grpc::GrpcClientIntegrationParamTestWithDeferredProcessing {
+public:
+  CompositeFilterSeverContextIntegrationTest()
+      : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {}
+
+  void createUpstreams() override {
+    HttpIntegrationTest::createUpstreams();
+    // Create separate "upstreams" for test gRPC servers
+    for (int i = 0; i < 2; ++i) {
+      grpc_upstreams_.push_back(&addFakeUpstream(Http::CodecType::HTTP2));
+    }
+  }
+
+  void TearDown() override {
+    if (connection_) {
+      ASSERT_TRUE(connection_->close());
+      ASSERT_TRUE(connection_->waitForDisconnect());
+    }
+    cleanupUpstreamAndDownstream();
+  }
+
+  void initializeConfig() {
+    config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+      // Ensure "HTTP2 with no prior knowledge." Necessary for gRPC and for headers
+      ConfigHelper::setHttp2(
+          *(bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0)));
+
+      // Clusters for test gRPC servers, starting by copying an existing cluster
+      for (size_t i = 0; i < grpc_upstreams_.size(); ++i) {
+        auto* server_cluster = bootstrap.mutable_static_resources()->add_clusters();
+        server_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]);
+        std::string cluster_name = absl::StrCat("test_server_", i);
+        server_cluster->set_name(cluster_name);
+        server_cluster->mutable_load_assignment()->set_cluster_name(cluster_name);
+      }
+      // Load configuration of the server from YAML and use a helper to add a grpc_service
+      // stanza pointing to the cluster that we just made
+      setGrpcService(*filter_config_.mutable_grpc_service(), "test_server_0",
+                     grpc_upstreams_[0]->localAddress());
+
+      addFilter();
+
+      // Parameterize with defer processing to prevent bit rot as filter made
+      // assumptions of data flow, prior relying on eager processing.
+      config_helper_.addRuntimeOverride(Runtime::defer_processing_backedup_streams,
+                                        deferredProcessing() ? "true" : "false");
+    });
+
+    setUpstreamProtocol(Http::CodecType::HTTP2);
+    setDownstreamProtocol(Http::CodecType::HTTP2);
+  }
+
+  void addFilter() {
+    // Add the filter to the loaded hcm.
+    envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager
+        hcm_config;
+    config_helper_.loadHttpConnectionManager(hcm_config);
+    auto* match_delegate_filter = hcm_config.add_http_filters();
+    match_delegate_filter->set_name("envoy.filters.http.match_delegate");
+
+    // Build extension config with composite filter inside.
+    envoy::extensions::common::matching::v3::ExtensionWithMatcher extension_config;
+    extension_config.mutable_extension_config()->set_name("composite");
+    envoy::extensions::filters::http::composite::v3::Composite composite_config;
+    extension_config.mutable_extension_config()->mutable_typed_config()->PackFrom(composite_config);
+    auto* matcher_tree = extension_config.mutable_xds_matcher()->mutable_matcher_tree();
+
+    // Set up the match input.
+    auto* matcher_input = matcher_tree->mutable_input();
+    matcher_input->set_name("request-headers");
+    envoy::type::matcher::v3::HttpRequestHeaderMatchInput request_header_match_input;
+    request_header_match_input.set_header_name("match-header");
+    matcher_input->mutable_typed_config()->PackFrom(request_header_match_input);
+
+    // Set up the match action with test filter as the delegated filter.
+    auto* exact_match_map = matcher_tree->mutable_exact_match_map()->mutable_map();
+    envoy::extensions::filters::http::composite::v3::ExecuteFilterAction test_filter_action;
+    test_filter_action.mutable_typed_config()->set_name("server-factory-context-filter");
+    test_filter_action.mutable_typed_config()->mutable_typed_config()->PackFrom(filter_config_);
+
+    ::xds::type::matcher::v3::Matcher_OnMatch on_match;
+    auto* on_match_action = on_match.mutable_action();
+    on_match_action->set_name("composite-action");
+    on_match_action->mutable_typed_config()->PackFrom(test_filter_action);
+
+    (*exact_match_map)["match"] = on_match;
+
+    // Finish up the construction of match_delegate_filter.
+    match_delegate_filter->mutable_typed_config()->PackFrom(extension_config);
+
+    // Now move the built filter to the front.
+    for (int i = hcm_config.http_filters_size() - 1; i > 0; --i) {
+      hcm_config.mutable_http_filters()->SwapElements(i, i - 1);
+    }
+
+    // Store it to hcm.
+    config_helper_.storeHttpConnectionManager(hcm_config);
+  }
+
+  test::integration::filters::ServerFactoryContextFilterConfig filter_config_;
+  std::vector grpc_upstreams_;
+  FakeHttpConnectionPtr connection_;
+  FakeStreamPtr stream_;
+};
+
+INSTANTIATE_TEST_SUITE_P(
+    IpVersionsClientTypeDeferredProcessing, CompositeFilterSeverContextIntegrationTest,
+    GRPC_CLIENT_INTEGRATION_DEFERRED_PROCESSING_PARAMS,
+    Grpc::GrpcClientIntegrationParamTestWithDeferredProcessing::protocolTestParamsToString);
+
+TEST_P(CompositeFilterSeverContextIntegrationTest, BasicFlow) {
+  initializeConfig();
+  HttpIntegrationTest::initialize();
+
+  auto conn = makeClientConnection(lookupPort("http"));
+  codec_client_ = makeHttpConnection(std::move(conn));
+  const Http::TestRequestHeaderMapImpl request_headers = {{":method", "GET"},
+                                                          {":path", "/somepath"},
+                                                          {":scheme", "http"},
+                                                          {"match-header", "match"},
+                                                          {":authority", "blah"}};
+  // Send request from downstream to upstream.
+  auto response = codec_client_->makeHeaderOnlyRequest(request_headers);
+
+  // Wait for side stream request.
+  helloworld::HelloRequest request;
+  request.set_name("hello");
+  ASSERT_TRUE(grpc_upstreams_[0]->waitForHttpConnection(*dispatcher_, connection_));
+  ASSERT_TRUE(connection_->waitForNewStream(*dispatcher_, stream_));
+  ASSERT_TRUE(stream_->waitForGrpcMessage(*dispatcher_, request));
+
+  // Start the grpc side stream.
+  stream_->startGrpcStream();
+
+  // Send the side stream response.
+  helloworld::HelloReply reply;
+  reply.set_message("ack");
+  stream_->sendGrpcMessage(reply);
+
+  // Handle the upstream request.
+  ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_));
+  ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_));
+  ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_));
+  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false);
+  upstream_request_->encodeData(100, true);
+
+  // Close the grpc stream.
+  stream_->finishGrpcStream(Grpc::Status::Ok);
+
+  // Verify the response from upstream to downstream.
+  ASSERT_TRUE(response->waitForEndStream());
+  EXPECT_TRUE(response->complete());
+  EXPECT_EQ(response->headers().getStatusValue(), "200");
+}
+
 } // namespace Envoy
diff --git a/test/integration/filters/BUILD b/test/integration/filters/BUILD
index 03f1d1f2a871..478b8972fe5d 100644
--- a/test/integration/filters/BUILD
+++ b/test/integration/filters/BUILD
@@ -509,6 +509,31 @@ envoy_proto_library(
     srcs = [":set_is_terminal_filter_config.proto"],
 )
 
+envoy_cc_test_library(
+    name = "server_factory_context_filter_lib",
+    srcs = [
+        "server_factory_context_filter.cc",
+    ],
+    deps = [
+        ":server_factory_context_filter_config_proto_cc_proto",
+        "//envoy/grpc:async_client_interface",
+        "//envoy/grpc:async_client_manager_interface",
+        "//envoy/http:filter_interface",
+        "//envoy/registry",
+        "//source/common/grpc:typed_async_client_lib",
+        "//source/extensions/filters/http/common:factory_base_lib",
+        "//source/extensions/filters/http/common:pass_through_filter_lib",
+        "//test/proto:helloworld_proto_cc_proto",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_proto_library(
+    name = "server_factory_context_filter_config_proto",
+    srcs = [":server_factory_context_filter_config.proto"],
+    deps = ["@envoy_api//envoy/config/core/v3:pkg"],
+)
+
 envoy_proto_library(
     name = "stop_and_continue_filter_config_proto",
     srcs = [":stop_and_continue_filter_config.proto"],
diff --git a/test/integration/filters/server_factory_context_filter.cc b/test/integration/filters/server_factory_context_filter.cc
new file mode 100644
index 000000000000..abc532d900fa
--- /dev/null
+++ b/test/integration/filters/server_factory_context_filter.cc
@@ -0,0 +1,152 @@
+#include 
+
+#include "envoy/config/core/v3/grpc_service.pb.h"
+#include "envoy/grpc/async_client_manager.h"
+#include "envoy/http/filter.h"
+#include "envoy/registry/registry.h"
+
+#include "source/common/grpc/typed_async_client.h"
+#include "source/extensions/filters/http/common/factory_base.h"
+#include "source/extensions/filters/http/common/pass_through_filter.h"
+
+#include "test/integration/filters/server_factory_context_filter_config.pb.h"
+#include "test/integration/filters/server_factory_context_filter_config.pb.validate.h"
+#include "test/proto/helloworld.pb.h"
+
+namespace Envoy {
+
+using ResponsePtr = std::unique_ptr;
+using FilterConfigSharedPtr =
+    std::shared_ptr;
+
+class FilterCallbacks {
+public:
+  virtual ~FilterCallbacks() = default;
+  virtual void onComplete() PURE;
+};
+
+class TestGrpcClient : public Grpc::AsyncStreamCallbacks {
+public:
+  TestGrpcClient(Server::Configuration::ServerFactoryContext& context,
+                 const envoy::config::core::v3::GrpcService& grpc_service)
+      : client_(context.clusterManager().grpcAsyncClientManager().getOrCreateRawAsyncClient(
+            grpc_service, context.scope(), true)),
+        method_descriptor_(helloworld::Greeter::descriptor()->FindMethodByName("SayHello")) {}
+
+  // AsyncStreamCallbacks
+  void onReceiveMessage(ResponsePtr&&) override { filter_callback_->onComplete(); }
+
+  // RawAsyncStreamCallbacks
+  void onCreateInitialMetadata(Http::RequestHeaderMap&) override {}
+  void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&&) override {}
+  void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {}
+  void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override {
+    stream_closed_ = true;
+    filter_callback_->onComplete();
+  }
+
+  void startStream() {
+    Http::AsyncClient::StreamOptions options;
+    stream_ = client_.start(*method_descriptor_, *this, options);
+  }
+
+  void sendMessage(FilterCallbacks& callbacks) {
+    filter_callback_ = &callbacks;
+    helloworld::HelloRequest request;
+    request.set_name("hello");
+    send(std::move(request), false);
+  }
+
+  bool isStreamClosed() { return stream_closed_; }
+
+  void close() {
+    if (stream_ != nullptr && !stream_closed_) {
+      stream_->closeStream();
+      stream_closed_ = true;
+      stream_->resetStream();
+    }
+  }
+
+private:
+  void send(helloworld::HelloRequest&& request, bool end_stream) {
+    stream_->sendMessage(std::move(request), end_stream);
+  }
+  Grpc::AsyncClient client_;
+  const Protobuf::MethodDescriptor* method_descriptor_;
+  Grpc::AsyncStream stream_;
+  bool stream_closed_ = false;
+  FilterCallbacks* filter_callback_;
+};
+
+// A test filter that is created from server factory context. This filter communicate with
+// external server via gRPC.
+class ServerFactoryContextFilter : public Http::PassThroughFilter, public FilterCallbacks {
+public:
+  ServerFactoryContextFilter(FilterConfigSharedPtr config,
+                             Server::Configuration::ServerFactoryContext& context)
+      : filter_config_(std::move(config)), context_(context),
+        test_client_(std::make_unique(context_, filter_config_->grpc_service())) {}
+
+  void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override {
+    decoder_callbacks_ = &callbacks;
+  }
+
+  Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap&, bool) override {
+    test_client_->startStream();
+    if (!test_client_->isStreamClosed()) {
+      test_client_->sendMessage(*this);
+    } else {
+      return Http::FilterHeadersStatus::Continue;
+    }
+    return Http::FilterHeadersStatus::StopIteration;
+  }
+
+  void onComplete() override {
+    if (!filter_chain_continued_) {
+      filter_chain_continued_ = true;
+      decoder_callbacks_->continueDecoding();
+    }
+  }
+
+  void onDestroy() override { test_client_->close(); }
+
+private:
+  FilterConfigSharedPtr filter_config_;
+  Server::Configuration::ServerFactoryContext& context_;
+  std::unique_ptr test_client_;
+  Http::StreamDecoderFilterCallbacks* decoder_callbacks_{};
+  bool filter_chain_continued_ = false;
+};
+
+class ServerFactoryContextFilterFactory
+    : public Extensions::HttpFilters::Common::FactoryBase<
+          test::integration::filters::ServerFactoryContextFilterConfig> {
+public:
+  ServerFactoryContextFilterFactory() : FactoryBase("server-factory-context-filter") {}
+
+private:
+  // Only the creation from serverFactoryContext is implemented, returns nullptr in
+  // `createFilterFactoryFromProtoTyped`
+  Http::FilterFactoryCb createFilterFactoryFromProtoTyped(
+      const test::integration::filters::ServerFactoryContextFilterConfig&, const std::string&,
+      Server::Configuration::FactoryContext&) override {
+    return nullptr;
+  }
+
+  Http::FilterFactoryCb createFilterServerFactoryFromProtoTyped(
+      const test::integration::filters::ServerFactoryContextFilterConfig& proto_config,
+      const std::string&, Server::Configuration::ServerFactoryContext& server_context) override {
+    FilterConfigSharedPtr filter_config =
+        std::make_shared(
+            proto_config);
+    return [&server_context, filter_config = std::move(filter_config)](
+               Http::FilterChainFactoryCallbacks& callbacks) -> void {
+      callbacks.addStreamFilter(
+          std::make_shared(filter_config, server_context));
+    };
+  }
+};
+
+REGISTER_FACTORY(ServerFactoryContextFilterFactory,
+                 Server::Configuration::NamedHttpFilterConfigFactory);
+} // namespace Envoy
diff --git a/test/integration/filters/server_factory_context_filter_config.proto b/test/integration/filters/server_factory_context_filter_config.proto
new file mode 100644
index 000000000000..c42c69333190
--- /dev/null
+++ b/test/integration/filters/server_factory_context_filter_config.proto
@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package test.integration.filters;
+
+import "envoy/config/core/v3/grpc_service.proto";
+
+message ServerFactoryContextFilterConfig {
+  envoy.config.core.v3.GrpcService grpc_service = 1;
+}

From 158b72649ad1b999906741bdd27c3c0a080c5e8a Mon Sep 17 00:00:00 2001
From: code 
Date: Tue, 13 Jun 2023 08:55:07 +0800
Subject: [PATCH 522/740] upstream code cleanup: remove unnecessary create()
 method and test (#27874)

* upstream: remove unnecessary create() method

Signed-off-by: wbpcode 

* remove unnecessary test

Signed-off-by: wbpcode 

* fix test

Signed-off-by: wbpcode 

* remove unnecessary low coverage

Signed-off-by: wbpcode 

* fix another some tests

Signed-off-by: wbpcode 

* fix test again

Signed-off-by: wbpcode 

* minor coverage improvement

Signed-off-by: wbpcode 

---------

Signed-off-by: wbpcode 
---
 envoy/upstream/load_balancer.h                |  6 --
 .../common/upstream/thread_aware_lb_impl.cc   |  2 +-
 source/common/upstream/thread_aware_lb_impl.h |  3 +-
 .../extensions/clusters/aggregate/cluster.h   |  3 +-
 .../clusters/dynamic_forward_proxy/cluster.h  |  5 +-
 .../original_dst/original_dst_cluster.h       |  5 +-
 .../clusters/redis/redis_cluster_lb.cc        |  2 +-
 .../clusters/redis/redis_cluster_lb.h         |  3 +-
 .../common/factory_base.h                     |  1 -
 .../load_balancing_policies/subset/config.cc  |  2 -
 .../upstream/load_balancer_benchmark.cc       | 20 +++---
 test/common/upstream/subset_lb_test.cc        | 49 +++------------
 test/extensions/clusters/aggregate/BUILD      |  1 +
 .../clusters/aggregate/cluster_test.cc        |  7 ++-
 .../clusters/dynamic_forward_proxy/BUILD      |  1 +
 .../dynamic_forward_proxy/cluster_test.cc     |  7 ++-
 test/extensions/clusters/redis/BUILD          |  1 +
 .../clusters/redis/redis_cluster_lb_test.cc   | 13 ++--
 .../least_request/config_test.cc              |  2 -
 .../maglev/config_test.cc                     |  2 -
 .../maglev/maglev_lb_test.cc                  | 29 +++++----
 .../random/config_test.cc                     |  2 -
 .../ring_hash/config_test.cc                  |  2 -
 .../ring_hash/ring_hash_lb_test.cc            | 63 ++++++++++---------
 .../round_robin/config_test.cc                |  2 -
 .../subset/config_test.cc                     |  2 -
 .../load_balancers/custom_lb_policy.h         |  5 +-
 test/per_file_coverage.sh                     |  3 -
 28 files changed, 111 insertions(+), 132 deletions(-)

diff --git a/envoy/upstream/load_balancer.h b/envoy/upstream/load_balancer.h
index c1e4cd48cefe..de07149356b5 100644
--- a/envoy/upstream/load_balancer.h
+++ b/envoy/upstream/load_balancer.h
@@ -167,12 +167,6 @@ class LoadBalancerFactory {
 public:
   virtual ~LoadBalancerFactory() = default;
 
-  /**
-   * @return LoadBalancerPtr a new worker local load balancer.
-   * TODO(wbpcode): remove this method in the future and used the new method below.
-   */
-  virtual LoadBalancerPtr create() PURE;
-
   /**
    * @return LoadBalancerPtr a new worker local load balancer.
    */
diff --git a/source/common/upstream/thread_aware_lb_impl.cc b/source/common/upstream/thread_aware_lb_impl.cc
index c3f1c07fc961..a787c466a533 100644
--- a/source/common/upstream/thread_aware_lb_impl.cc
+++ b/source/common/upstream/thread_aware_lb_impl.cc
@@ -175,7 +175,7 @@ ThreadAwareLoadBalancerBase::LoadBalancerImpl::chooseHost(LoadBalancerContext* c
   return host;
 }
 
-LoadBalancerPtr ThreadAwareLoadBalancerBase::LoadBalancerFactoryImpl::create() {
+LoadBalancerPtr ThreadAwareLoadBalancerBase::LoadBalancerFactoryImpl::create(LoadBalancerParams) {
   auto lb = std::make_unique(stats_, random_);
 
   // We must protect current_lb_ via a RW lock since it is accessed and written to by multiple
diff --git a/source/common/upstream/thread_aware_lb_impl.h b/source/common/upstream/thread_aware_lb_impl.h
index 094297c06d6d..31ad4dfd3946 100644
--- a/source/common/upstream/thread_aware_lb_impl.h
+++ b/source/common/upstream/thread_aware_lb_impl.h
@@ -150,9 +150,8 @@ class ThreadAwareLoadBalancerBase : public LoadBalancerBase, public ThreadAwareL
         : stats_(stats), random_(random) {}
 
     // Upstream::LoadBalancerFactory
-    LoadBalancerPtr create() override;
     // Ignore the params for the thread-aware LB.
-    LoadBalancerPtr create(LoadBalancerParams) override { return create(); }
+    LoadBalancerPtr create(LoadBalancerParams) override;
 
     ClusterLbStats& stats_;
     Random::RandomGenerator& random_;
diff --git a/source/extensions/clusters/aggregate/cluster.h b/source/extensions/clusters/aggregate/cluster.h
index 370a3ab1e415..6990b07e92e2 100644
--- a/source/extensions/clusters/aggregate/cluster.h
+++ b/source/extensions/clusters/aggregate/cluster.h
@@ -139,12 +139,11 @@ class AggregateClusterLoadBalancer : public Upstream::LoadBalancer,
 struct AggregateLoadBalancerFactory : public Upstream::LoadBalancerFactory {
   AggregateLoadBalancerFactory(const Cluster& cluster) : cluster_(cluster) {}
   // Upstream::LoadBalancerFactory
-  Upstream::LoadBalancerPtr create() override {
+  Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override {
     return std::make_unique(
         cluster_.info(), cluster_.cluster_manager_, cluster_.runtime_, cluster_.random_,
         cluster_.clusters_);
   }
-  Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); }
 
   const Cluster& cluster_;
 };
diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.h b/source/extensions/clusters/dynamic_forward_proxy/cluster.h
index 9a38eb362abe..4fa6daa4d26a 100644
--- a/source/extensions/clusters/dynamic_forward_proxy/cluster.h
+++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.h
@@ -130,8 +130,9 @@ class Cluster : public Upstream::BaseDynamicClusterImpl,
     LoadBalancerFactory(Cluster& cluster) : cluster_(cluster) {}
 
     // Upstream::LoadBalancerFactory
-    Upstream::LoadBalancerPtr create() override { return std::make_unique(cluster_); }
-    Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); }
+    Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override {
+      return std::make_unique(cluster_);
+    }
 
   private:
     Cluster& cluster_;
diff --git a/source/extensions/clusters/original_dst/original_dst_cluster.h b/source/extensions/clusters/original_dst/original_dst_cluster.h
index f10aa82595b1..47ab51199759 100644
--- a/source/extensions/clusters/original_dst/original_dst_cluster.h
+++ b/source/extensions/clusters/original_dst/original_dst_cluster.h
@@ -104,8 +104,9 @@ class OriginalDstCluster : public ClusterImplBase {
     LoadBalancerFactory(const std::shared_ptr& cluster) : cluster_(cluster) {}
 
     // Upstream::LoadBalancerFactory
-    Upstream::LoadBalancerPtr create() override { return std::make_unique(cluster_); }
-    Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); }
+    Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override {
+      return std::make_unique(cluster_);
+    }
 
     const std::shared_ptr cluster_;
   };
diff --git a/source/extensions/clusters/redis/redis_cluster_lb.cc b/source/extensions/clusters/redis/redis_cluster_lb.cc
index d6d6be61f1b4..441b5a2b276c 100644
--- a/source/extensions/clusters/redis/redis_cluster_lb.cc
+++ b/source/extensions/clusters/redis/redis_cluster_lb.cc
@@ -99,7 +99,7 @@ void RedisClusterLoadBalancerFactory::onHostHealthUpdate() {
   }
 }
 
-Upstream::LoadBalancerPtr RedisClusterLoadBalancerFactory::create() {
+Upstream::LoadBalancerPtr RedisClusterLoadBalancerFactory::create(Upstream::LoadBalancerParams) {
   absl::ReaderMutexLock lock(&mutex_);
   return std::make_unique(slot_array_, shard_vector_, random_);
 }
diff --git a/source/extensions/clusters/redis/redis_cluster_lb.h b/source/extensions/clusters/redis/redis_cluster_lb.h
index fe2870adb912..8f94b8aa033d 100644
--- a/source/extensions/clusters/redis/redis_cluster_lb.h
+++ b/source/extensions/clusters/redis/redis_cluster_lb.h
@@ -149,8 +149,7 @@ class RedisClusterLoadBalancerFactory : public ClusterSlotUpdateCallBack,
   void onHostHealthUpdate() override;
 
   // Upstream::LoadBalancerFactory
-  Upstream::LoadBalancerPtr create() override;
-  Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); }
+  Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override;
 
 private:
   class RedisShard {
diff --git a/source/extensions/load_balancing_policies/common/factory_base.h b/source/extensions/load_balancing_policies/common/factory_base.h
index c8d46e4481c2..04797d5ab953 100644
--- a/source/extensions/load_balancing_policies/common/factory_base.h
+++ b/source/extensions/load_balancing_policies/common/factory_base.h
@@ -42,7 +42,6 @@ class FactoryBase : public Upstream::TypedLoadBalancerFactoryBase {
         : proto_config_(proto_config), cluster_info_(cluster_info), priority_set_(priority_set),
           runtime_(runtime), random_(random), time_source_(time_source) {}
 
-    Upstream::LoadBalancerPtr create() override { PANIC("not implemented"); }
     Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams params) override {
       return Impl()(params, proto_config_, cluster_info_, priority_set_, runtime_, random_,
                     time_source_);
diff --git a/source/extensions/load_balancing_policies/subset/config.cc b/source/extensions/load_balancing_policies/subset/config.cc
index 486d3c77b733..f7e6facc2d78 100644
--- a/source/extensions/load_balancing_policies/subset/config.cc
+++ b/source/extensions/load_balancing_policies/subset/config.cc
@@ -107,8 +107,6 @@ class LbFactory : public Upstream::LoadBalancerFactory {
       : subset_config_(subset_config), cluster_info_(cluster_info), runtime_(runtime),
         random_(random), time_source_(time_source) {}
 
-  Upstream::LoadBalancerPtr create() override { PANIC("not implemented"); };
-
   Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams params) override {
     auto child_lb_creator =
         std::make_unique(subset_config_, cluster_info_);
diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc
index 4f9a22e94b77..c52d7b7a2c12 100644
--- a/test/common/upstream/load_balancer_benchmark.cc
+++ b/test/common/upstream/load_balancer_benchmark.cc
@@ -66,6 +66,10 @@ class BaseTester : public Event::TestUsingSimulatedTime {
 
   PrioritySetImpl priority_set_;
   PrioritySetImpl local_priority_set_;
+
+  // The following are needed to create a load balancer by the load balancer factory.
+  LoadBalancerParams lb_params_{priority_set_, &local_priority_set_};
+
   Stats::IsolatedStoreImpl stats_store_;
   Stats::Scope& stats_scope_{*stats_store_.rootScope()};
   ClusterLbStatNames stat_names_{stats_store_.symbolTable()};
@@ -321,7 +325,7 @@ void benchmarkRingHashLoadBalancerChooseHost(::benchmark::State& state) {
     const uint64_t keys_to_simulate = state.range(2);
     RingHashTester tester(num_hosts, min_ring_size);
     tester.ring_hash_lb_->initialize();
-    LoadBalancerPtr lb = tester.ring_hash_lb_->factory()->create();
+    LoadBalancerPtr lb = tester.ring_hash_lb_->factory()->create(tester.lb_params_);
     absl::node_hash_map hit_counter;
     TestLoadBalancerContext context;
     state.ResumeTiming();
@@ -359,7 +363,7 @@ void benchmarkMaglevLoadBalancerChooseHost(::benchmark::State& state) {
     const uint64_t keys_to_simulate = state.range(1);
     MaglevTester tester(num_hosts);
     tester.maglev_lb_->initialize();
-    LoadBalancerPtr lb = tester.maglev_lb_->factory()->create();
+    LoadBalancerPtr lb = tester.maglev_lb_->factory()->create(tester.lb_params_);
     absl::node_hash_map hit_counter;
     TestLoadBalancerContext context;
     state.ResumeTiming();
@@ -398,7 +402,7 @@ void benchmarkRingHashLoadBalancerHostLoss(::benchmark::State& state) {
   for (auto _ : state) { // NOLINT: Silences warning about dead store
     RingHashTester tester(num_hosts, min_ring_size);
     tester.ring_hash_lb_->initialize();
-    LoadBalancerPtr lb = tester.ring_hash_lb_->factory()->create();
+    LoadBalancerPtr lb = tester.ring_hash_lb_->factory()->create(tester.lb_params_);
     std::vector hosts;
     TestLoadBalancerContext context;
     for (uint64_t i = 0; i < keys_to_simulate; i++) {
@@ -408,7 +412,7 @@ void benchmarkRingHashLoadBalancerHostLoss(::benchmark::State& state) {
 
     RingHashTester tester2(num_hosts - hosts_to_lose, min_ring_size);
     tester2.ring_hash_lb_->initialize();
-    lb = tester2.ring_hash_lb_->factory()->create();
+    lb = tester2.ring_hash_lb_->factory()->create(tester2.lb_params_);
     std::vector hosts2;
     for (uint64_t i = 0; i < keys_to_simulate; i++) {
       context.hash_key_ = hashInt(i);
@@ -446,7 +450,7 @@ void benchmarkMaglevLoadBalancerHostLoss(::benchmark::State& state) {
 
     MaglevTester tester(num_hosts);
     tester.maglev_lb_->initialize();
-    LoadBalancerPtr lb = tester.maglev_lb_->factory()->create();
+    LoadBalancerPtr lb = tester.maglev_lb_->factory()->create(tester.lb_params_);
     std::vector hosts;
     TestLoadBalancerContext context;
     for (uint64_t i = 0; i < keys_to_simulate; i++) {
@@ -456,7 +460,7 @@ void benchmarkMaglevLoadBalancerHostLoss(::benchmark::State& state) {
 
     MaglevTester tester2(num_hosts - hosts_to_lose);
     tester2.maglev_lb_->initialize();
-    lb = tester2.maglev_lb_->factory()->create();
+    lb = tester2.maglev_lb_->factory()->create(tester2.lb_params_);
     std::vector hosts2;
     for (uint64_t i = 0; i < keys_to_simulate; i++) {
       context.hash_key_ = hashInt(i);
@@ -493,7 +497,7 @@ void benchmarkMaglevLoadBalancerWeighted(::benchmark::State& state) {
 
     MaglevTester tester(num_hosts, weighted_subset_percent, before_weight);
     tester.maglev_lb_->initialize();
-    LoadBalancerPtr lb = tester.maglev_lb_->factory()->create();
+    LoadBalancerPtr lb = tester.maglev_lb_->factory()->create(tester.lb_params_);
     std::vector hosts;
     TestLoadBalancerContext context;
     for (uint64_t i = 0; i < keys_to_simulate; i++) {
@@ -503,7 +507,7 @@ void benchmarkMaglevLoadBalancerWeighted(::benchmark::State& state) {
 
     MaglevTester tester2(num_hosts, weighted_subset_percent, after_weight);
     tester2.maglev_lb_->initialize();
-    lb = tester2.maglev_lb_->factory()->create();
+    lb = tester2.maglev_lb_->factory()->create(tester2.lb_params_);
     std::vector hosts2;
     for (uint64_t i = 0; i < keys_to_simulate; i++) {
       context.hash_key_ = hashInt(i);
diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc
index aa2b44d3eb42..56cd3ccc3d73 100644
--- a/test/common/upstream/subset_lb_test.cc
+++ b/test/common/upstream/subset_lb_test.cc
@@ -68,8 +68,6 @@ class SubsetLoadBalancerInternalStateTester {
               legacy_child_lb_creator->lbLeastRequestConfig() != absl::nullopt);
   }
 
-  static void testWrapper();
-
 private:
   std::shared_ptr lb_;
 };
@@ -161,36 +159,6 @@ class TestMetadataMatchCriteria : public Router::MetadataMatchCriteria {
   std::vector matches_;
 };
 
-void SubsetLoadBalancerInternalStateTester::testWrapper() {
-  MockLoadBalancerContext inner;
-  TestMetadataMatchCriteria test_criteria(std::map{});
-
-  EXPECT_CALL(inner, metadataMatchCriteria)
-      .Times(testing::AnyNumber())
-      .WillRepeatedly(Return(&test_criteria));
-  const std::set filtered_metadata_match_criteria_names;
-  SubsetLoadBalancer::LoadBalancerContextWrapper wrapper(&inner,
-                                                         filtered_metadata_match_criteria_names);
-
-  EXPECT_CALL(inner, computeHashKey());
-  wrapper.computeHashKey();
-
-  EXPECT_CALL(inner, downstreamConnection());
-  wrapper.downstreamConnection();
-
-  EXPECT_CALL(inner, downstreamHeaders());
-  wrapper.downstreamHeaders();
-
-  EXPECT_CALL(inner, upstreamSocketOptions());
-  wrapper.upstreamSocketOptions();
-
-  EXPECT_CALL(inner, upstreamTransportSocketOptions());
-  wrapper.upstreamTransportSocketOptions();
-
-  EXPECT_CALL(inner, overrideHostToSelect());
-  wrapper.overrideHostToSelect();
-}
-
 namespace SubsetLoadBalancerTest {
 
 class TestLoadBalancerContext : public LoadBalancerContextBase {
@@ -211,8 +179,7 @@ class TestLoadBalancerContext : public LoadBalancerContextBase {
   const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return matches_.get(); }
   const Http::RequestHeaderMap* downstreamHeaders() const override { return nullptr; }
 
-private:
-  const std::shared_ptr matches_;
+  std::shared_ptr matches_;
 };
 
 enum class UpdateOrder { RemovesFirst, Simultaneous };
@@ -2523,6 +2490,15 @@ TEST_P(SubsetLoadBalancerTest, MetadataFallbackList) {
   const auto version2_host = host_set_.hosts_[1];
   const auto version3_host = host_set_.hosts_[2];
 
+  // No context.
+  EXPECT_EQ(nullptr, lb_->chooseHost(nullptr));
+
+  TestLoadBalancerContext context_without_metadata({{"key", "value"}});
+  context_without_metadata.matches_ = nullptr;
+
+  // No metadata in context.
+  EXPECT_EQ(nullptr, lb_->chooseHost(&context_without_metadata));
+
   TestLoadBalancerContext context_with_fallback({{"fallback_list", valueFromJson(R""""(
     [
       {"version": "2.0"},
@@ -2892,11 +2868,6 @@ TEST_P(SubsetLoadBalancerSingleHostPerSubsetTest, Update) {
   EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_b)); // fallback
 }
 
-TEST(LoadBalancerContextWrapper, LoadBalancerContextWrapperTest) {
-  // Test private helper class via friend class.
-  SubsetLoadBalancerInternalStateTester::testWrapper();
-}
-
 INSTANTIATE_TEST_SUITE_P(UpdateOrderings, SubsetLoadBalancerSingleHostPerSubsetTest,
                          testing::ValuesIn({UpdateOrder::RemovesFirst, UpdateOrder::Simultaneous}));
 
diff --git a/test/extensions/clusters/aggregate/BUILD b/test/extensions/clusters/aggregate/BUILD
index 5697246cd78a..f0a927f83651 100644
--- a/test/extensions/clusters/aggregate/BUILD
+++ b/test/extensions/clusters/aggregate/BUILD
@@ -25,6 +25,7 @@ envoy_extension_cc_test(
         "//test/mocks/ssl:ssl_mocks",
         "//test/mocks/upstream:load_balancer_context_mock",
         "//test/mocks/upstream:load_balancer_mocks",
+        "//test/mocks/upstream:priority_set_mocks",
         "//test/test_common:environment_lib",
         "//test/test_common:simulated_time_system_lib",
         "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
diff --git a/test/extensions/clusters/aggregate/cluster_test.cc b/test/extensions/clusters/aggregate/cluster_test.cc
index 3876c2440aff..60e34275e32b 100644
--- a/test/extensions/clusters/aggregate/cluster_test.cc
+++ b/test/extensions/clusters/aggregate/cluster_test.cc
@@ -12,6 +12,7 @@
 #include "test/mocks/ssl/mocks.h"
 #include "test/mocks/upstream/load_balancer.h"
 #include "test/mocks/upstream/load_balancer_context.h"
+#include "test/mocks/upstream/priority_set.h"
 #include "test/test_common/environment.h"
 #include "test/test_common/simulated_time_system.h"
 
@@ -123,7 +124,7 @@ class AggregateClusterTest : public Event::TestUsingSimulatedTime, public testin
 
     thread_aware_lb_ = std::make_unique(*cluster_);
     lb_factory_ = thread_aware_lb_->factory();
-    lb_ = lb_factory_->create();
+    lb_ = lb_factory_->create(lb_params_);
   }
 
   NiceMock server_context_;
@@ -146,6 +147,10 @@ class AggregateClusterTest : public Event::TestUsingSimulatedTime, public testin
   Upstream::PrioritySetImpl primary_ps_, secondary_ps_;
   NiceMock primary_load_balancer_, secondary_load_balancer_;
 
+  // Just use this as parameters of create() method but thread aware load balancer will not use it.
+  NiceMock worker_priority_set_;
+  Upstream::LoadBalancerParams lb_params_{worker_priority_set_, {}};
+
   const std::string default_yaml_config_ = R"EOF(
     name: aggregate_cluster
     connect_timeout: 0.25s
diff --git a/test/extensions/clusters/dynamic_forward_proxy/BUILD b/test/extensions/clusters/dynamic_forward_proxy/BUILD
index cb8a884506ad..6dfc94cf3ce2 100644
--- a/test/extensions/clusters/dynamic_forward_proxy/BUILD
+++ b/test/extensions/clusters/dynamic_forward_proxy/BUILD
@@ -35,6 +35,7 @@ envoy_extension_cc_test(
         "//test/mocks/ssl:ssl_mocks",
         "//test/mocks/upstream:load_balancer_context_mock",
         "//test/mocks/upstream:load_balancer_mocks",
+        "//test/mocks/upstream:priority_set_mocks",
         "//test/test_common:environment_lib",
         "//test/test_common:test_runtime_lib",
         "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
diff --git a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc
index 472a301e0860..a8ffed2724fd 100644
--- a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc
+++ b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc
@@ -18,6 +18,7 @@
 #include "test/mocks/ssl/mocks.h"
 #include "test/mocks/upstream/load_balancer.h"
 #include "test/mocks/upstream/load_balancer_context.h"
+#include "test/mocks/upstream/priority_set.h"
 #include "test/test_common/environment.h"
 #include "test/test_common/test_runtime.h"
 
@@ -108,7 +109,7 @@ class ClusterTest : public testing::Test,
     host_map_[host]->address_ = Network::Utility::parseInternetAddress(address);
   }
 
-  void refreshLb() { lb_ = lb_factory_->create(); }
+  void refreshLb() { lb_ = lb_factory_->create(lb_params_); }
 
   Upstream::MockLoadBalancerContext* setHostAndReturnContext(const std::string& host) {
     downstream_headers_.remove(":authority");
@@ -170,6 +171,10 @@ class ClusterTest : public testing::Test,
   NiceMock stream_info_;
   NiceMock connection_;
 
+  // Just use this as parameters of create() method but thread aware load balancer will not use it.
+  NiceMock worker_priority_set_;
+  Upstream::LoadBalancerParams lb_params_{worker_priority_set_, {}};
+
   const std::string sub_cluster_yaml_config_ = R"EOF(
 name: name
 connect_timeout: 0.25s
diff --git a/test/extensions/clusters/redis/BUILD b/test/extensions/clusters/redis/BUILD
index c003bb6e7d5f..9a630a0860ad 100644
--- a/test/extensions/clusters/redis/BUILD
+++ b/test/extensions/clusters/redis/BUILD
@@ -44,6 +44,7 @@ envoy_extension_cc_test(
         "//test/mocks/upstream:cluster_manager_mocks",
         "//test/mocks/upstream:health_check_event_logger_mocks",
         "//test/mocks/upstream:health_checker_mocks",
+        "//test/mocks/upstream:priority_set_mocks",
         "//test/test_common:utility_lib",
         "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/clusters/redis/v3:pkg_cc_proto",
diff --git a/test/extensions/clusters/redis/redis_cluster_lb_test.cc b/test/extensions/clusters/redis/redis_cluster_lb_test.cc
index bcf65ba83376..eb3b005080ba 100644
--- a/test/extensions/clusters/redis/redis_cluster_lb_test.cc
+++ b/test/extensions/clusters/redis/redis_cluster_lb_test.cc
@@ -6,6 +6,7 @@
 #include "test/common/upstream/utility.h"
 #include "test/mocks/common.h"
 #include "test/mocks/upstream/cluster_info.h"
+#include "test/mocks/upstream/priority_set.h"
 #include "test/test_common/simulated_time_system.h"
 
 using testing::Return;
@@ -54,7 +55,7 @@ class RedisClusterLoadBalancerTest : public Event::TestUsingSimulatedTime, publi
                           NetworkFilters::Common::Redis::Client::ReadPolicy read_policy =
                               NetworkFilters::Common::Redis::Client::ReadPolicy::Primary) {
 
-    Upstream::LoadBalancerPtr lb = lb_->factory()->create();
+    Upstream::LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
     for (auto& assignment : expected_assignments) {
       TestLoadBalancerContext context(assignment.first, read_command, read_policy);
       auto host = lb->chooseHost(&context);
@@ -77,6 +78,10 @@ class RedisClusterLoadBalancerTest : public Event::TestUsingSimulatedTime, publi
   std::unique_ptr lb_;
   std::shared_ptr info_{new NiceMock()};
   NiceMock random_;
+
+  // Just use this as parameters of create() method but thread aware load balancer will not use it.
+  NiceMock worker_priority_set_;
+  Upstream::LoadBalancerParams lb_params_{worker_priority_set_, {}};
 };
 
 class RedisLoadBalancerContextImplTest : public testing::Test {
@@ -97,7 +102,7 @@ class RedisLoadBalancerContextImplTest : public testing::Test {
 // Works correctly without any hosts.
 TEST_F(RedisClusterLoadBalancerTest, NoHost) {
   init();
-  EXPECT_EQ(nullptr, lb_->factory()->create()->chooseHost(nullptr));
+  EXPECT_EQ(nullptr, lb_->factory()->create(lb_params_)->chooseHost(nullptr));
 };
 
 // Works correctly with empty context
@@ -119,7 +124,7 @@ TEST_F(RedisClusterLoadBalancerTest, NoHash) {
   init();
   factory_->onClusterSlotUpdate(std::move(slots), all_hosts);
   TestLoadBalancerContext context(absl::nullopt);
-  EXPECT_EQ(nullptr, lb_->factory()->create()->chooseHost(&context));
+  EXPECT_EQ(nullptr, lb_->factory()->create(lb_params_)->chooseHost(&context));
 };
 
 TEST_F(RedisClusterLoadBalancerTest, Basic) {
@@ -308,7 +313,7 @@ TEST_F(RedisClusterLoadBalancerTest, ReadStrategiesNoReplica) {
   validateAssignment(hosts, primary_assignments, true,
                      NetworkFilters::Common::Redis::Client::ReadPolicy::PreferReplica);
 
-  Upstream::LoadBalancerPtr lb = lb_->factory()->create();
+  Upstream::LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   TestLoadBalancerContext context(1100, true,
                                   NetworkFilters::Common::Redis::Client::ReadPolicy::Replica);
   auto host = lb->chooseHost(&context);
diff --git a/test/extensions/load_balancing_policies/least_request/config_test.cc b/test/extensions/load_balancing_policies/least_request/config_test.cc
index 82e8405d8f00..0f05f3887a9e 100644
--- a/test/extensions/load_balancing_policies/least_request/config_test.cc
+++ b/test/extensions/load_balancing_policies/least_request/config_test.cc
@@ -42,8 +42,6 @@ TEST(LeastRequestConfigTest, ValidateFail) {
 
   auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr});
   EXPECT_NE(nullptr, thread_local_lb);
-
-  EXPECT_DEATH(thread_local_lb_factory->create(), "not implemented");
 }
 
 } // namespace
diff --git a/test/extensions/load_balancing_policies/maglev/config_test.cc b/test/extensions/load_balancing_policies/maglev/config_test.cc
index 9b103239d2c6..636fd4c77794 100644
--- a/test/extensions/load_balancing_policies/maglev/config_test.cc
+++ b/test/extensions/load_balancing_policies/maglev/config_test.cc
@@ -44,8 +44,6 @@ TEST(MaglevConfigTest, Validate) {
 
     auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr});
     EXPECT_NE(nullptr, thread_local_lb);
-
-    EXPECT_NE(nullptr, thread_local_lb_factory->create());
   }
 
   {
diff --git a/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc
index 850a2fa00493..857e6ea9fd17 100644
--- a/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc
+++ b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc
@@ -76,6 +76,11 @@ class MaglevLoadBalancerTest : public Event::TestUsingSimulatedTime,
   }
 
   NiceMock priority_set_;
+
+  // Just use this as parameters of create() method but thread aware load balancer will not use it.
+  NiceMock worker_priority_set_;
+  LoadBalancerParams lb_params_{worker_priority_set_, {}};
+
   MockHostSet& host_set_ = *priority_set_.getMockHostSet(0);
   std::shared_ptr info_{new NiceMock()};
   Stats::IsolatedStoreImpl stats_store_;
@@ -94,7 +99,7 @@ INSTANTIATE_TEST_SUITE_P(MaglevTests, MaglevLoadBalancerTest, ::testing::Bool())
 // Works correctly without any hosts.
 TEST_P(MaglevLoadBalancerTest, NoHost) {
   init(7);
-  EXPECT_EQ(nullptr, lb_->factory()->create()->chooseHost(nullptr));
+  EXPECT_EQ(nullptr, lb_->factory()->create(lb_params_)->chooseHost(nullptr));
 };
 
 // Test for thread aware load balancer destructed before load balancer factory. After CDS removes a
@@ -107,7 +112,7 @@ TEST_P(MaglevLoadBalancerTest, LbDestructedBeforeFactory) {
   auto factory = lb_->factory();
   lb_.reset();
 
-  EXPECT_NE(nullptr, factory->create());
+  EXPECT_NE(nullptr, factory->create(lb_params_));
 }
 
 // Throws an exception if table size is not a prime number.
@@ -153,7 +158,7 @@ TEST_P(MaglevLoadBalancerTest, Basic) {
   // maglev: i=4 host=127.0.0.1:95
   // maglev: i=5 host=127.0.0.1:90
   // maglev: i=6 host=127.0.0.1:93
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   const std::vector expected_assignments{2, 4, 0, 1, 5, 0, 3};
   for (uint32_t i = 0; i < 3 * expected_assignments.size(); ++i) {
     TestLoadBalancerContext context(i);
@@ -188,7 +193,7 @@ TEST_P(MaglevLoadBalancerTest, BasicWithHostName) {
   // maglev: i=4 host=94
   // maglev: i=5 host=91
   // maglev: i=6 host=90
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   const std::vector expected_assignments{2, 5, 0, 3, 4, 1, 0};
   for (uint32_t i = 0; i < 3 * expected_assignments.size(); ++i) {
     TestLoadBalancerContext context(i);
@@ -223,7 +228,7 @@ TEST_P(MaglevLoadBalancerTest, BasicWithMetadataHashKey) {
   // maglev: i=4 host=94
   // maglev: i=5 host=91
   // maglev: i=6 host=90
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   const std::vector expected_assignments{2, 5, 0, 3, 4, 1, 0};
   for (uint32_t i = 0; i < 3 * expected_assignments.size(); ++i) {
     TestLoadBalancerContext context(i);
@@ -256,7 +261,7 @@ TEST_P(MaglevLoadBalancerTest, BasicWithRetryHostPredicate) {
   // maglev: i=4 host=127.0.0.1:95
   // maglev: i=5 host=127.0.0.1:90
   // maglev: i=6 host=127.0.0.1:93
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   {
     // Confirm that i=3 is selected by the hash.
     TestLoadBalancerContext context(10);
@@ -307,7 +312,7 @@ TEST_P(MaglevLoadBalancerTest, Weighted) {
   // maglev: i=14 host=127.0.0.1:91
   // maglev: i=15 host=127.0.0.1:90
   // maglev: i=16 host=127.0.0.1:91
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   const std::vector expected_assignments{1, 0, 0, 1, 0, 1, 1, 0, 1,
                                                    1, 1, 1, 1, 0, 1, 0, 1};
   for (uint32_t i = 0; i < 3 * expected_assignments.size(); ++i) {
@@ -350,7 +355,7 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedSameLocalityWeights) {
   // maglev: i=14 host=127.0.0.1:91
   // maglev: i=15 host=127.0.0.1:90
   // maglev: i=16 host=127.0.0.1:91
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   const std::vector expected_assignments{1, 0, 0, 1, 0, 1, 1, 0, 0,
                                                    1, 0, 1, 0, 0, 1, 0, 1};
   for (uint32_t i = 0; i < 3 * expected_assignments.size(); ++i) {
@@ -394,7 +399,7 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedDifferentLocalityWeights) {
   // maglev: i=14 host=127.0.0.1:90
   // maglev: i=15 host=127.0.0.1:90
   // maglev: i=16 host=127.0.0.1:90
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   const std::vector expected_assignments{1, 0, 0, 0, 0, 0, 1, 0, 0,
                                                    1, 0, 1, 0, 0, 0, 0, 0};
   for (uint32_t i = 0; i < 3 * expected_assignments.size(); ++i) {
@@ -414,7 +419,7 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedAllZeroLocalityWeights) {
   host_set_.locality_weights_ = locality_weights;
   host_set_.runCallbacks({}, {});
   init(17, true);
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   TestLoadBalancerContext context(0);
   EXPECT_EQ(nullptr, lb->chooseHost(&context));
 }
@@ -452,7 +457,7 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedGlobalPanic) {
   // maglev: i=14 host=127.0.0.1:91
   // maglev: i=15 host=127.0.0.1:90
   // maglev: i=16 host=127.0.0.1:91
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   const std::vector expected_assignments{1, 0, 0, 1, 0, 1, 1, 0, 0,
                                                    1, 0, 1, 0, 0, 1, 0, 1};
   for (uint32_t i = 0; i < 3 * expected_assignments.size(); ++i) {
@@ -481,7 +486,7 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedLopsided) {
   EXPECT_EQ(1, lb_->stats().min_entries_per_host_.value());
   EXPECT_EQ(MaglevTable::DefaultTableSize - 1023, lb_->stats().max_entries_per_host_.value());
 
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // Populate a histogram of the number of table entries for each host...
   uint32_t counts[1024] = {0};
diff --git a/test/extensions/load_balancing_policies/random/config_test.cc b/test/extensions/load_balancing_policies/random/config_test.cc
index 869d7ef64d07..4e1acac4f811 100644
--- a/test/extensions/load_balancing_policies/random/config_test.cc
+++ b/test/extensions/load_balancing_policies/random/config_test.cc
@@ -42,8 +42,6 @@ TEST(RandomConfigTest, ValidateFail) {
 
   auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr});
   EXPECT_NE(nullptr, thread_local_lb);
-
-  EXPECT_DEATH(thread_local_lb_factory->create(), "not implemented");
 }
 
 } // namespace
diff --git a/test/extensions/load_balancing_policies/ring_hash/config_test.cc b/test/extensions/load_balancing_policies/ring_hash/config_test.cc
index 24eb9c267795..24a59c6627d9 100644
--- a/test/extensions/load_balancing_policies/ring_hash/config_test.cc
+++ b/test/extensions/load_balancing_policies/ring_hash/config_test.cc
@@ -44,8 +44,6 @@ TEST(RingHashConfigTest, Validate) {
 
     auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr});
     EXPECT_NE(nullptr, thread_local_lb);
-
-    EXPECT_NE(nullptr, thread_local_lb_factory->create());
   }
 
   {
diff --git a/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc b/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc
index 8fb244f6af8a..911bbc51e2b7 100644
--- a/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc
+++ b/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc
@@ -81,6 +81,11 @@ class RingHashLoadBalancerTest : public Event::TestUsingSimulatedTime,
   MockHostSet& hostSet() { return GetParam() ? host_set_ : failover_host_set_; }
 
   NiceMock priority_set_;
+
+  // Just use this as parameters of create() method but thread aware load balancer will not use it.
+  NiceMock worker_priority_set_;
+  LoadBalancerParams lb_params_{worker_priority_set_, {}};
+
   MockHostSet& host_set_ = *priority_set_.getMockHostSet(0);
   MockHostSet& failover_host_set_ = *priority_set_.getMockHostSet(1);
   std::shared_ptr info_{new NiceMock()};
@@ -104,14 +109,14 @@ INSTANTIATE_TEST_SUITE_P(RingHashPrimaryOrFailover, RingHashFailoverTest, ::test
 // Given no hosts, expect chooseHost to return null.
 TEST_P(RingHashLoadBalancerTest, NoHost) {
   init();
-  EXPECT_EQ(nullptr, lb_->factory()->create()->chooseHost(nullptr));
+  EXPECT_EQ(nullptr, lb_->factory()->create(lb_params_)->chooseHost(nullptr));
 
-  EXPECT_EQ(nullptr, lb_->factory()->create()->peekAnotherHost(nullptr));
-  EXPECT_FALSE(lb_->factory()->create()->lifetimeCallbacks().has_value());
+  EXPECT_EQ(nullptr, lb_->factory()->create(lb_params_)->peekAnotherHost(nullptr));
+  EXPECT_FALSE(lb_->factory()->create(lb_params_)->lifetimeCallbacks().has_value());
   std::vector hash_key;
   auto mock_host = std::make_shared>();
   EXPECT_FALSE(lb_->factory()
-                   ->create()
+                   ->create(lb_params_)
                    ->selectExistingConnection(nullptr, *mock_host, hash_key)
                    .has_value());
 }
@@ -135,7 +140,7 @@ TEST_P(RingHashLoadBalancerTest, LbDestructedBeforeFactory) {
   auto factory = lb_->factory();
   lb_.reset();
 
-  EXPECT_NE(nullptr, factory->create());
+  EXPECT_NE(nullptr, factory->create(lb_params_));
 }
 
 // Given minimum_ring_size > maximum_ring_size, expect an exception.
@@ -184,7 +189,7 @@ TEST_P(RingHashLoadBalancerTest, Basic) {
   // :94  | 15516499411664133160
   // :90  | 16117243373044804889
 
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   {
     TestLoadBalancerContext context(0);
     EXPECT_EQ(hostSet().hosts_[4], lb->chooseHost(&context));
@@ -209,7 +214,7 @@ TEST_P(RingHashLoadBalancerTest, Basic) {
 
   hostSet().healthy_hosts_.clear();
   hostSet().runCallbacks({}, {});
-  lb = lb_->factory()->create();
+  lb = lb_->factory()->create(lb_params_);
   {
     TestLoadBalancerContext context(0);
     EXPECT_EQ(hostSet().hosts_[4], lb->chooseHost(&context));
@@ -230,19 +235,19 @@ TEST_P(RingHashFailoverTest, BasicFailover) {
   EXPECT_EQ(12, lb_->stats().min_hashes_per_host_.value());
   EXPECT_EQ(12, lb_->stats().max_hashes_per_host_.value());
 
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   EXPECT_EQ(failover_host_set_.healthy_hosts_[0], lb->chooseHost(nullptr));
 
   // Add a healthy host at P=0 and it will be chosen.
   host_set_.healthy_hosts_ = host_set_.hosts_;
   host_set_.runCallbacks({}, {});
-  lb = lb_->factory()->create();
+  lb = lb_->factory()->create(lb_params_);
   EXPECT_EQ(host_set_.healthy_hosts_[0], lb->chooseHost(nullptr));
 
   // Remove the healthy host and ensure we fail back over to the failover_host_set_
   host_set_.healthy_hosts_ = {};
   host_set_.runCallbacks({}, {});
-  lb = lb_->factory()->create();
+  lb = lb_->factory()->create(lb_params_);
   EXPECT_EQ(failover_host_set_.healthy_hosts_[0], lb->chooseHost(nullptr));
 
   // Set up so P=0 gets 70% of the load, and P=1 gets 30%.
@@ -250,7 +255,7 @@ TEST_P(RingHashFailoverTest, BasicFailover) {
                       makeTestHost(info_, "tcp://127.0.0.1:81", simTime())};
   host_set_.healthy_hosts_ = {host_set_.hosts_[0]};
   host_set_.runCallbacks({}, {});
-  lb = lb_->factory()->create();
+  lb = lb_->factory()->create(lb_params_);
   EXPECT_CALL(random_, random()).WillOnce(Return(69));
   EXPECT_EQ(host_set_.healthy_hosts_[0], lb->chooseHost(nullptr));
   EXPECT_CALL(random_, random()).WillOnce(Return(71));
@@ -290,7 +295,7 @@ TEST_P(RingHashLoadBalancerTest, BasicWithMurmur2) {
   // ring hash: host=127.0.0.1:80 hash=15427156902705414897
   // ring hash: host=127.0.0.1:85 hash=16375050414328759093
   // ring hash: host=127.0.0.1:80 hash=17613279263364193813
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   {
     TestLoadBalancerContext context(0);
     EXPECT_EQ(hostSet().hosts_[5], lb->chooseHost(&context));
@@ -356,7 +361,7 @@ TEST_P(RingHashLoadBalancerTest, BasicWithHostname) {
   // 91 | 14338313586354474791
   // 94 | 15364271037087512980
 
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   {
     TestLoadBalancerContext context(0);
     EXPECT_EQ(hostSet().hosts_[5], lb->chooseHost(&context));
@@ -378,7 +383,7 @@ TEST_P(RingHashLoadBalancerTest, BasicWithHostname) {
 
   hostSet().healthy_hosts_.clear();
   hostSet().runCallbacks({}, {});
-  lb = lb_->factory()->create();
+  lb = lb_->factory()->create(lb_params_);
   {
     TestLoadBalancerContext context(0);
     EXPECT_EQ(hostSet().hosts_[5], lb->chooseHost(&context));
@@ -428,7 +433,7 @@ TEST_P(RingHashLoadBalancerTest, BasicWithMetadataHashKey) {
   // 91 | 14338313586354474791
   // 94 | 15364271037087512980
 
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   {
     TestLoadBalancerContext context(0);
     EXPECT_EQ(hostSet().hosts_[5], lb->chooseHost(&context));
@@ -450,7 +455,7 @@ TEST_P(RingHashLoadBalancerTest, BasicWithMetadataHashKey) {
 
   hostSet().healthy_hosts_.clear();
   hostSet().runCallbacks({}, {});
-  lb = lb_->factory()->create();
+  lb = lb_->factory()->create(lb_params_);
   {
     TestLoadBalancerContext context(0);
     EXPECT_EQ(hostSet().hosts_[5], lb->chooseHost(&context));
@@ -496,7 +501,7 @@ TEST_P(RingHashLoadBalancerTest, BasicWithRetryHostPredicate) {
   // :94  | 15516499411664133160
   // :90  | 16117243373044804889
 
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   {
     // Proof that we know which host will be selected.
     TestLoadBalancerContext context(0);
@@ -547,7 +552,7 @@ TEST_P(RingHashLoadBalancerTest, UnevenHosts) {
   // :80  | 13838424394637650569
   // :81  | 16064866803292627174
 
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
   {
     TestLoadBalancerContext context(0);
     EXPECT_EQ(hostSet().hosts_[0], lb->chooseHost(&context));
@@ -566,7 +571,7 @@ TEST_P(RingHashLoadBalancerTest, UnevenHosts) {
   // :82  | 12882406409176325258
   // :81  | 16064866803292627174
 
-  lb = lb_->factory()->create();
+  lb = lb_->factory()->create(lb_params_);
   {
     TestLoadBalancerContext context(0);
     EXPECT_EQ(hostSet().hosts_[0], lb->chooseHost(&context));
@@ -590,7 +595,7 @@ TEST_P(RingHashLoadBalancerTest, HostWeightedTinyRing) {
   EXPECT_EQ(6, lb_->stats().size_.value());
   EXPECT_EQ(1, lb_->stats().min_hashes_per_host_.value());
   EXPECT_EQ(3, lb_->stats().max_hashes_per_host_.value());
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // :90 should appear once, :91 should appear twice and :92 should appear three times.
   absl::node_hash_map expected{
@@ -617,7 +622,7 @@ TEST_P(RingHashLoadBalancerTest, HostWeightedLargeRing) {
   EXPECT_EQ(6144, lb_->stats().size_.value());
   EXPECT_EQ(1024, lb_->stats().min_hashes_per_host_.value());
   EXPECT_EQ(3072, lb_->stats().max_hashes_per_host_.value());
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // Generate 6000 hashes around the ring and populate a histogram of which hosts they mapped to...
   uint32_t counts[3] = {0};
@@ -644,7 +649,7 @@ TEST_P(RingHashLoadBalancerTest, ZeroLocalityWeights) {
   hostSet().runCallbacks({}, {});
 
   init(true);
-  EXPECT_EQ(nullptr, lb_->factory()->create()->chooseHost(nullptr));
+  EXPECT_EQ(nullptr, lb_->factory()->create(lb_params_)->chooseHost(nullptr));
 }
 
 // Given localities with weights 1, 2, 3 and 0, and a ring size of exactly 6, expect the correct
@@ -669,7 +674,7 @@ TEST_P(RingHashLoadBalancerTest, LocalityWeightedTinyRing) {
   EXPECT_EQ(6, lb_->stats().size_.value());
   EXPECT_EQ(1, lb_->stats().min_hashes_per_host_.value());
   EXPECT_EQ(3, lb_->stats().max_hashes_per_host_.value());
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // :90 should appear once, :91 should appear twice, :92 should appear three times,
   // and :93 shouldn't appear at all.
@@ -702,7 +707,7 @@ TEST_P(RingHashLoadBalancerTest, LocalityWeightedLargeRing) {
   EXPECT_EQ(6144, lb_->stats().size_.value());
   EXPECT_EQ(1024, lb_->stats().min_hashes_per_host_.value());
   EXPECT_EQ(3072, lb_->stats().max_hashes_per_host_.value());
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // Generate 6000 hashes around the ring and populate a histogram of which hosts they mapped to...
   uint32_t counts[4] = {0};
@@ -741,7 +746,7 @@ TEST_P(RingHashLoadBalancerTest, HostAndLocalityWeightedTinyRing) {
   EXPECT_EQ(9, lb_->stats().size_.value());
   EXPECT_EQ(1, lb_->stats().min_hashes_per_host_.value());
   EXPECT_EQ(4, lb_->stats().max_hashes_per_host_.value());
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // :90 should appear once, :91 and :92 should each appear two times, and :93 should appear four
   // times, to get the correct overall proportions.
@@ -777,7 +782,7 @@ TEST_P(RingHashLoadBalancerTest, HostAndLocalityWeightedLargeRing) {
   EXPECT_EQ(9216, lb_->stats().size_.value());
   EXPECT_EQ(1024, lb_->stats().min_hashes_per_host_.value());
   EXPECT_EQ(4096, lb_->stats().max_hashes_per_host_.value());
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // Generate 9000 hashes around the ring and populate a histogram of which hosts they mapped to...
   uint32_t counts[4] = {0};
@@ -810,7 +815,7 @@ TEST_P(RingHashLoadBalancerTest, SmallFractionalScale) {
   EXPECT_EQ(2, lb_->stats().size_.value());
   EXPECT_EQ(0, lb_->stats().min_hashes_per_host_.value());
   EXPECT_EQ(1, lb_->stats().max_hashes_per_host_.value());
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // Generate some reasonable number of hashes around the ring and populate a histogram of which
   // hosts they mapped to. Here we don't care about the distribution (because the scale is
@@ -850,7 +855,7 @@ TEST_P(RingHashLoadBalancerTest, LargeFractionalScale) {
   EXPECT_EQ(1023, lb_->stats().size_.value());
   EXPECT_EQ(511, lb_->stats().min_hashes_per_host_.value());
   EXPECT_EQ(512, lb_->stats().max_hashes_per_host_.value());
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // Generate 1023 hashes around the ring and populate a histogram of which hosts they mapped to...
   uint32_t counts[2] = {0};
@@ -889,7 +894,7 @@ TEST_P(RingHashLoadBalancerTest, LopsidedWeightSmallScale) {
   // Host :0, from the heavy-but-sparse locality, should have 1016 out of the 1024 entries on the
   // ring, which gives us the right ratio of 127/128.
   EXPECT_EQ(1016, lb_->stats().max_hashes_per_host_.value());
-  LoadBalancerPtr lb = lb_->factory()->create();
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
 
   // Every 128th host in the light-but-dense locality should have an entry on the ring, for a total
   // of 8 entries. This gives us the right ratio of 1/128.
diff --git a/test/extensions/load_balancing_policies/round_robin/config_test.cc b/test/extensions/load_balancing_policies/round_robin/config_test.cc
index a6aa8aacd5e4..2e460ec81e34 100644
--- a/test/extensions/load_balancing_policies/round_robin/config_test.cc
+++ b/test/extensions/load_balancing_policies/round_robin/config_test.cc
@@ -43,8 +43,6 @@ TEST(RoundRobinConfigTest, ValidateFail) {
 
   auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr});
   EXPECT_NE(nullptr, thread_local_lb);
-
-  EXPECT_DEATH(thread_local_lb_factory->create(), "not implemented");
 }
 
 } // namespace
diff --git a/test/extensions/load_balancing_policies/subset/config_test.cc b/test/extensions/load_balancing_policies/subset/config_test.cc
index b5d77b70384c..f4bdd68267d5 100644
--- a/test/extensions/load_balancing_policies/subset/config_test.cc
+++ b/test/extensions/load_balancing_policies/subset/config_test.cc
@@ -64,8 +64,6 @@ TEST(SubsetConfigTest, SubsetConfigTest) {
 
   auto thread_local_lb = thread_local_lb_factory->create({thread_local_priority_set, nullptr});
   EXPECT_NE(nullptr, thread_local_lb);
-
-  EXPECT_DEATH(thread_local_lb_factory->create(), "not implemented");
 }
 
 TEST(SubsetConfigTest, SubsetConfigTestWithUnknownSubsetLoadBalancingPolicy) {
diff --git a/test/integration/load_balancers/custom_lb_policy.h b/test/integration/load_balancers/custom_lb_policy.h
index 8147dcd6b2f6..0fd5d87ba589 100644
--- a/test/integration/load_balancers/custom_lb_policy.h
+++ b/test/integration/load_balancers/custom_lb_policy.h
@@ -46,8 +46,9 @@ class ThreadAwareLbImpl : public Upstream::ThreadAwareLoadBalancer {
   public:
     LbFactory(const Upstream::HostSharedPtr& host) : host_(host) {}
 
-    Upstream::LoadBalancerPtr create() override { return std::make_unique(host_); }
-    Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override { return create(); }
+    Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams) override {
+      return std::make_unique(host_);
+    }
 
     const Upstream::HostSharedPtr host_;
   };
diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh
index 6d350e692d40..0fb812f5e751 100755
--- a/test/per_file_coverage.sh
+++ b/test/per_file_coverage.sh
@@ -71,13 +71,10 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/watchdog/profile_action:83.3"
 "source/server:93.8" # flaky: be careful adjusting. See https://github.com/envoyproxy/envoy/issues/15239
 "source/server/admin:profiler-lib:83"
-"source/extensions/load_balancing_policies/common:94.1" # Death tests don't report LCOV
 "source/server/config_validation:88.4"
 "source/extensions/health_checkers:95.9"
 "source/extensions/health_checkers/http:93.8"
 "source/extensions/health_checkers/grpc:92.0"
-"source/extensions/load_balancing_policies:96.4"
-"source/extensions/load_balancing_policies/subset:96.0"
 "source/extensions/config_subscription/rest:94.3"
 "source/extensions/matching/input_matchers/cel_matcher:90.7" #Death tests don't report LCOV
 )

From ee3a5459e6404981ce8bcfc8f4a51df7834a2449 Mon Sep 17 00:00:00 2001
From: StarryNight 
Date: Tue, 13 Jun 2023 10:40:55 +0800
Subject: [PATCH 523/740] go extension filter state get api (#27864)

* go extension filter state get api

Signed-off-by: wangkai19 

* fix some review comments

Signed-off-by: wangkai19 

* fix review comments

Signed-off-by: wangkai19 

* remove unnecessary lock

Signed-off-by: wangkai19 

---------

Signed-off-by: wangkai19 
---
 contrib/golang/common/dso/dso.cc              |  7 +++
 contrib/golang/common/dso/dso.h               |  3 ++
 contrib/golang/common/dso/libgolang.h         |  4 ++
 contrib/golang/common/dso/test/mocks.h        |  1 +
 .../common/dso/test/test_data/simple.go       |  4 ++
 contrib/golang/filters/http/source/cgo.cc     |  9 ++++
 .../filters/http/source/go/pkg/api/BUILD      |  1 -
 .../filters/http/source/go/pkg/api/api.h      |  2 +
 .../filters/http/source/go/pkg/api/filter.go  |  1 +
 .../filters/http/source/go/pkg/http/BUILD     |  1 +
 .../http/source/go/pkg/{api => http}/capi.go  | 17 ++++---
 .../http/source/go/pkg/http/capi_impl.go      | 20 ++++++++-
 .../filters/http/source/go/pkg/http/filter.go | 13 ++++--
 .../filters/http/source/go/pkg/http/shim.go   | 13 ++++++
 .../filters/http/source/golang_filter.cc      | 44 +++++++++++++++++++
 .../filters/http/source/golang_filter.h       |  1 +
 .../http/test/golang_integration_test.cc      |  4 ++
 .../http/test/test_data/basic/filter.go       |  3 ++
 18 files changed, 136 insertions(+), 12 deletions(-)
 rename contrib/golang/filters/http/source/go/pkg/{api => http}/capi.go (86%)

diff --git a/contrib/golang/common/dso/dso.cc b/contrib/golang/common/dso/dso.cc
index 3a7ec19fe5d9..71fb1c89e29c 100644
--- a/contrib/golang/common/dso/dso.cc
+++ b/contrib/golang/common/dso/dso.cc
@@ -56,6 +56,8 @@ HttpFilterDsoImpl::HttpFilterDsoImpl(const std::string dso_name) : HttpFilterDso
       envoy_go_filter_on_http_data_, handler_, dso_name, "envoyGoFilterOnHttpData");
   loaded_ &= dlsymInternal(
       envoy_go_filter_on_http_destroy_, handler_, dso_name, "envoyGoFilterOnHttpDestroy");
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_go_request_sema_dec_, handler_, dso_name, "envoyGoRequestSemaDec");
 }
 
 GoUint64 HttpFilterDsoImpl::envoyGoFilterNewHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2,
@@ -87,6 +89,11 @@ void HttpFilterDsoImpl::envoyGoFilterOnHttpDestroy(httpRequest* p0, int p1) {
   envoy_go_filter_on_http_destroy_(p0, GoUint64(p1));
 }
 
+void HttpFilterDsoImpl::envoyGoRequestSemaDec(httpRequest* p0) {
+  ASSERT(envoy_go_filter_go_request_sema_dec_ != nullptr);
+  envoy_go_filter_go_request_sema_dec_(p0);
+}
+
 ClusterSpecifierDsoImpl::ClusterSpecifierDsoImpl(const std::string dso_name)
     : ClusterSpecifierDso(dso_name) {
   loaded_ &= dlsymInternal(
diff --git a/contrib/golang/common/dso/dso.h b/contrib/golang/common/dso/dso.h
index 4f3cba97688e..9e687efe8dfe 100644
--- a/contrib/golang/common/dso/dso.h
+++ b/contrib/golang/common/dso/dso.h
@@ -39,6 +39,7 @@ class HttpFilterDso : public Dso {
   virtual GoUint64 envoyGoFilterOnHttpData(httpRequest* p0, GoUint64 p1, GoUint64 p2,
                                            GoUint64 p3) PURE;
   virtual void envoyGoFilterOnHttpDestroy(httpRequest* p0, int p1) PURE;
+  virtual void envoyGoRequestSemaDec(httpRequest* p0) PURE;
 };
 
 class HttpFilterDsoImpl : public HttpFilterDso {
@@ -54,6 +55,7 @@ class HttpFilterDsoImpl : public HttpFilterDso {
                                      GoUint64 p3) override;
   GoUint64 envoyGoFilterOnHttpData(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) override;
   void envoyGoFilterOnHttpDestroy(httpRequest* p0, int p1) override;
+  void envoyGoRequestSemaDec(httpRequest* p0) override;
 
 private:
   GoUint64 (*envoy_go_filter_new_http_plugin_config_)(GoUint64 p0, GoUint64 p1, GoUint64 p2,
@@ -65,6 +67,7 @@ class HttpFilterDsoImpl : public HttpFilterDso {
   GoUint64 (*envoy_go_filter_on_http_data_)(httpRequest* p0, GoUint64 p1, GoUint64 p2,
                                             GoUint64 p3) = {nullptr};
   void (*envoy_go_filter_on_http_destroy_)(httpRequest* p0, GoUint64 p1) = {nullptr};
+  void (*envoy_go_filter_go_request_sema_dec_)(httpRequest* p0) = {nullptr};
 };
 
 class ClusterSpecifierDso : public Dso {
diff --git a/contrib/golang/common/dso/libgolang.h b/contrib/golang/common/dso/libgolang.h
index 28e52133f0db..66a8d27ba43b 100644
--- a/contrib/golang/common/dso/libgolang.h
+++ b/contrib/golang/common/dso/libgolang.h
@@ -132,6 +132,10 @@ extern GoUint64 envoyGoFilterOnHttpData(httpRequest* r,
 // github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoFilterOnHttpDestroy
 extern void envoyGoFilterOnHttpDestroy(httpRequest* r, GoUint64 reason);
 
+// go:linkname envoyGoRequestSemaDec
+// github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoRequestSemaDec
+extern void envoyGoRequestSemaDec(httpRequest* r);
+
 // go:linkname envoyGoOnClusterSpecify
 // github.com/envoyproxy/envoy/contrib/golang/router/cluster_specifier/source/go/pkg/cluster_specifier.envoyGoOnClusterSpecify
 extern GoInt64 envoyGoOnClusterSpecify(GoUint64 pluginPtr,  // NOLINT(readability-identifier-naming)
diff --git a/contrib/golang/common/dso/test/mocks.h b/contrib/golang/common/dso/test/mocks.h
index 9894c40ec959..21287d6e9c82 100644
--- a/contrib/golang/common/dso/test/mocks.h
+++ b/contrib/golang/common/dso/test/mocks.h
@@ -20,6 +20,7 @@ class MockHttpFilterDsoImpl : public HttpFilterDso {
   MOCK_METHOD(GoUint64, envoyGoFilterOnHttpData,
               (httpRequest * p0, GoUint64 p1, GoUint64 p2, GoUint64 p3));
   MOCK_METHOD(void, envoyGoFilterOnHttpDestroy, (httpRequest * p0, int p1));
+  MOCK_METHOD(void, envoyGoRequestSemaDec, (httpRequest * p0));
 };
 
 } // namespace Dso
diff --git a/contrib/golang/common/dso/test/test_data/simple.go b/contrib/golang/common/dso/test/test_data/simple.go
index 4b4ca92f1e51..86c6ac0c1be8 100644
--- a/contrib/golang/common/dso/test/test_data/simple.go
+++ b/contrib/golang/common/dso/test/test_data/simple.go
@@ -45,5 +45,9 @@ func envoyGoOnClusterSpecify(pluginPtr uint64, headerPtr uint64, pluginId uint64
 	return 0
 }
 
+//export envoyGoRequestSemaDec
+func envoyGoRequestSemaDec(r *C.httpRequest) {
+}
+
 func main() {
 }
diff --git a/contrib/golang/filters/http/source/cgo.cc b/contrib/golang/filters/http/source/cgo.cc
index 74adb835d778..4a593dbe2016 100644
--- a/contrib/golang/filters/http/source/cgo.cc
+++ b/contrib/golang/filters/http/source/cgo.cc
@@ -215,6 +215,15 @@ CAPIStatus envoyGoFilterHttpSetStringFilterState(void* r, void* key, void* value
                                      });
 }
 
+CAPIStatus envoyGoFilterHttpGetStringFilterState(void* r, void* key, void* value) {
+  return envoyGoFilterHandlerWrapper(r,
+                                     [key, value](std::shared_ptr& filter) -> CAPIStatus {
+                                       auto key_str = referGoString(key);
+                                       auto value_str = reinterpret_cast(value);
+                                       return filter->getStringFilterState(key_str, value_str);
+                                     });
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/contrib/golang/filters/http/source/go/pkg/api/BUILD b/contrib/golang/filters/http/source/go/pkg/api/BUILD
index 392f1cab149d..bbce799a1516 100644
--- a/contrib/golang/filters/http/source/go/pkg/api/BUILD
+++ b/contrib/golang/filters/http/source/go/pkg/api/BUILD
@@ -5,7 +5,6 @@ licenses(["notice"])  # Apache 2
 go_library(
     name = "api",
     srcs = [
-        "capi.go",
         "cgocheck.go",
         "filter.go",
         "type.go",
diff --git a/contrib/golang/filters/http/source/go/pkg/api/api.h b/contrib/golang/filters/http/source/go/pkg/api/api.h
index 3620ff1dc454..34127323724f 100644
--- a/contrib/golang/filters/http/source/go/pkg/api/api.h
+++ b/contrib/golang/filters/http/source/go/pkg/api/api.h
@@ -38,6 +38,7 @@ typedef enum { // NOLINT(modernize-use-using)
   CAPINotInGo = -3,
   CAPIInvalidPhase = -4,
   CAPIValueNotFound = -5,
+  CAPIYield = -6,
 } CAPIStatus;
 
 CAPIStatus envoyGoFilterHttpContinue(void* r, int status);
@@ -71,6 +72,7 @@ void envoyGoFilterHttpFinalize(void* r, int reason);
 
 CAPIStatus envoyGoFilterHttpSetStringFilterState(void* r, void* key, void* value, int state_type,
                                                  int life_span, int stream_sharing);
+CAPIStatus envoyGoFilterHttpGetStringFilterState(void* r, void* key, void* value);
 
 #ifdef __cplusplus
 } // extern "C"
diff --git a/contrib/golang/filters/http/source/go/pkg/api/filter.go b/contrib/golang/filters/http/source/go/pkg/api/filter.go
index 88d658c8f1b0..2f4fa70d9829 100644
--- a/contrib/golang/filters/http/source/go/pkg/api/filter.go
+++ b/contrib/golang/filters/http/source/go/pkg/api/filter.go
@@ -129,4 +129,5 @@ const (
 
 type FilterState interface {
 	SetString(key, value string, stateType StateType, lifeSpan LifeSpan, streamSharing StreamSharing)
+	GetString(key string) string
 }
diff --git a/contrib/golang/filters/http/source/go/pkg/http/BUILD b/contrib/golang/filters/http/source/go/pkg/http/BUILD
index 54c5fb5af4f5..6cfca08915ce 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/BUILD
+++ b/contrib/golang/filters/http/source/go/pkg/http/BUILD
@@ -6,6 +6,7 @@ go_library(
     name = "http",
     srcs = [
         "api.h",
+        "capi.go",
         "capi_impl.go",
         "config.go",
         "filter.go",
diff --git a/contrib/golang/filters/http/source/go/pkg/api/capi.go b/contrib/golang/filters/http/source/go/pkg/http/capi.go
similarity index 86%
rename from contrib/golang/filters/http/source/go/pkg/api/capi.go
rename to contrib/golang/filters/http/source/go/pkg/http/capi.go
index 494bf72572be..1e35e3c8f85e 100644
--- a/contrib/golang/filters/http/source/go/pkg/api/capi.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/capi.go
@@ -15,9 +15,13 @@
  * limitations under the License.
  */
 
-package api
+package http
 
-import "unsafe"
+import (
+	"unsafe"
+
+	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+)
 
 type HttpCAPI interface {
 	HttpContinue(r unsafe.Pointer, status uint64)
@@ -33,7 +37,7 @@ type HttpCAPI interface {
 	HttpRemoveHeader(r unsafe.Pointer, key *string)
 
 	HttpGetBuffer(r unsafe.Pointer, bufferPtr uint64, value *string, length uint64)
-	HttpSetBufferHelper(r unsafe.Pointer, bufferPtr uint64, value string, action BufferAction)
+	HttpSetBufferHelper(r unsafe.Pointer, bufferPtr uint64, value string, action api.BufferAction)
 
 	HttpCopyTrailers(r unsafe.Pointer, num uint64, bytes uint64) map[string][]string
 	HttpSetTrailer(r unsafe.Pointer, key *string, value *string, add bool)
@@ -45,10 +49,11 @@ type HttpCAPI interface {
 	// TODO: HttpGetDynamicMetadata(r unsafe.Pointer, filterName string) map[string]interface{}
 	HttpSetDynamicMetadata(r unsafe.Pointer, filterName string, key string, value interface{})
 
-	HttpLog(level LogType, message string)
-	HttpLogLevel() LogType
+	HttpLog(level api.LogType, message string)
+	HttpLogLevel() api.LogType
 
 	HttpFinalize(r unsafe.Pointer, reason int)
 
-	HttpSetStringFilterState(r unsafe.Pointer, key string, value string, stateType StateType, lifeSpan LifeSpan, streamSharing StreamSharing)
+	HttpSetStringFilterState(r unsafe.Pointer, key string, value string, stateType api.StateType, lifeSpan api.LifeSpan, streamSharing api.StreamSharing)
+	HttpGetStringFilterState(r *httpRequest, key string) string
 }
diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
index ca5993d98814..1cb8a4e65939 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
@@ -35,6 +35,7 @@ import (
 	"reflect"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"unsafe"
 
 	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
@@ -263,10 +264,10 @@ func (c *httpCApiImpl) HttpFinalize(r unsafe.Pointer, reason int) {
 	C.envoyGoFilterHttpFinalize(r, C.int(reason))
 }
 
-var cAPI api.HttpCAPI = &httpCApiImpl{}
+var cAPI HttpCAPI = &httpCApiImpl{}
 
 // SetHttpCAPI for mock cAPI
-func SetHttpCAPI(api api.HttpCAPI) {
+func SetHttpCAPI(api HttpCAPI) {
 	cAPI = api
 }
 
@@ -274,3 +275,18 @@ func (c *httpCApiImpl) HttpSetStringFilterState(r unsafe.Pointer, key string, va
 	res := C.envoyGoFilterHttpSetStringFilterState(r, unsafe.Pointer(&key), unsafe.Pointer(&value), C.int(stateType), C.int(lifeSpan), C.int(streamSharing))
 	handleCApiStatus(res)
 }
+
+func (c *httpCApiImpl) HttpGetStringFilterState(r *httpRequest, key string) string {
+	var value string
+	r.sema.Add(1)
+	res := C.envoyGoFilterHttpGetStringFilterState(unsafe.Pointer(r.req), unsafe.Pointer(&key), unsafe.Pointer(&value))
+	if res == C.CAPIYield {
+		atomic.AddInt32(&r.waitingOnEnvoy, 1)
+		r.sema.Wait()
+	} else {
+		r.sema.Done()
+		handleCApiStatus(res)
+	}
+
+	return strings.Clone(value)
+}
diff --git a/contrib/golang/filters/http/source/go/pkg/http/filter.go b/contrib/golang/filters/http/source/go/pkg/http/filter.go
index b035ab7a5e81..01b43e47b129 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/filter.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/filter.go
@@ -33,6 +33,7 @@ package http
 import "C"
 import (
 	"fmt"
+	"sync"
 	"unsafe"
 
 	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
@@ -57,9 +58,11 @@ type panicInfo struct {
 	details string
 }
 type httpRequest struct {
-	req        *C.httpRequest
-	httpFilter api.StreamFilter
-	pInfo      panicInfo
+	req            *C.httpRequest
+	httpFilter     api.StreamFilter
+	pInfo          panicInfo
+	sema           sync.WaitGroup
+	waitingOnEnvoy int32
 }
 
 func (r *httpRequest) pluginName() string {
@@ -214,3 +217,7 @@ func (s *streamInfo) FilterState() api.FilterState {
 func (f *filterState) SetString(key, value string, stateType api.StateType, lifeSpan api.LifeSpan, streamSharing api.StreamSharing) {
 	cAPI.HttpSetStringFilterState(unsafe.Pointer(f.request.req), key, value, stateType, lifeSpan, streamSharing)
 }
+
+func (f *filterState) GetString(key string) string {
+	return cAPI.HttpGetStringFilterState(f.request, key)
+}
diff --git a/contrib/golang/filters/http/source/go/pkg/http/shim.go b/contrib/golang/filters/http/source/go/pkg/http/shim.go
index fe0cb3751792..53e3499ac5bd 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/shim.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/shim.go
@@ -37,6 +37,7 @@ import (
 	"fmt"
 	"runtime"
 	"sync"
+	"sync/atomic"
 
 	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
 )
@@ -208,6 +209,9 @@ func envoyGoFilterOnHttpDestroy(r *C.httpRequest, reason uint64) {
 	req := getRequest(r)
 	// do nothing even when req.panic is true, since filter is already destroying.
 	defer req.RecoverPanic()
+	if atomic.CompareAndSwapInt32(&req.waitingOnEnvoy, 1, 0) {
+		req.sema.Done()
+	}
 
 	v := api.DestroyReason(reason)
 
@@ -216,3 +220,12 @@ func envoyGoFilterOnHttpDestroy(r *C.httpRequest, reason uint64) {
 
 	Requests.DeleteReq(r)
 }
+
+//export envoyGoRequestSemaDec
+func envoyGoRequestSemaDec(r *C.httpRequest) {
+	req := getRequest(r)
+	defer req.RecoverPanic()
+	if atomic.CompareAndSwapInt32(&req.waitingOnEnvoy, 1, 0) {
+		req.sema.Done()
+	}
+}
diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc
index 674a58209378..4b9518a6cfa2 100644
--- a/contrib/golang/filters/http/source/golang_filter.cc
+++ b/contrib/golang/filters/http/source/golang_filter.cc
@@ -1106,6 +1106,50 @@ CAPIStatus Filter::setStringFilterState(absl::string_view key, absl::string_view
   return CAPIStatus::CAPIOK;
 }
 
+CAPIStatus Filter::getStringFilterState(absl::string_view key, GoString* value_str) {
+  // lock until this function return since it may running in a Go thread.
+  Thread::LockGuard lock(mutex_);
+  if (has_destroyed_) {
+    ENVOY_LOG(debug, "golang filter has been destroyed");
+    return CAPIStatus::CAPIFilterIsDestroy;
+  }
+
+  auto& state = getProcessorState();
+  if (!state.isProcessingInGo()) {
+    ENVOY_LOG(debug, "golang filter is not processing Go");
+    return CAPIStatus::CAPINotInGo;
+  }
+
+  if (state.isThreadSafe()) {
+    auto go_filter_state =
+        state.streamInfo().filterState()->getDataReadOnly(key);
+    if (go_filter_state) {
+      req_->strValue = go_filter_state->value();
+      value_str->p = req_->strValue.data();
+      value_str->n = req_->strValue.length();
+    }
+  } else {
+    auto key_str = std::string(key);
+    auto weak_ptr = weak_from_this();
+    state.getDispatcher().post([this, &state, weak_ptr, key_str, value_str] {
+      if (!weak_ptr.expired() && !hasDestroyed()) {
+        auto go_filter_state =
+            state.streamInfo().filterState()->getDataReadOnly(key_str);
+        if (go_filter_state) {
+          req_->strValue = go_filter_state->value();
+          value_str->p = req_->strValue.data();
+          value_str->n = req_->strValue.length();
+        }
+        dynamic_lib_->envoyGoRequestSemaDec(req_);
+      } else {
+        ENVOY_LOG(info, "golang filter has gone or destroyed in setStringFilterState");
+      }
+    });
+    return CAPIStatus::CAPIYield;
+  }
+  return CAPIStatus::CAPIOK;
+}
+
 /* ConfigId */
 
 uint64_t Filter::getMergedConfigId(ProcessorState& state) {
diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h
index 23f9e6c8b908..762c7f5043f7 100644
--- a/contrib/golang/filters/http/source/golang_filter.h
+++ b/contrib/golang/filters/http/source/golang_filter.h
@@ -182,6 +182,7 @@ class Filter : public Http::StreamFilter,
   CAPIStatus setDynamicMetadata(std::string filter_name, std::string key, absl::string_view buf);
   CAPIStatus setStringFilterState(absl::string_view key, absl::string_view value, int state_type,
                                   int life_span, int stream_sharing);
+  CAPIStatus getStringFilterState(absl::string_view key, GoString* value_str);
 
 private:
   bool hasDestroyed() {
diff --git a/contrib/golang/filters/http/test/golang_integration_test.cc b/contrib/golang/filters/http/test/golang_integration_test.cc
index 1e4908e9986e..f8083b99b9ce 100644
--- a/contrib/golang/filters/http/test/golang_integration_test.cc
+++ b/contrib/golang/filters/http/test/golang_integration_test.cc
@@ -247,6 +247,10 @@ name: golang
     codec_client_->sendTrailers(request_encoder, request_trailers);
 
     waitForNextUpstreamRequest();
+
+    EXPECT_EQ("go_state_test_value",
+              getHeader(upstream_request_->headers(), "go-state-test-header-key"));
+
     // original header: x-test-header-0
     EXPECT_EQ("foo", getHeader(upstream_request_->headers(), "x-test-header-0"));
 
diff --git a/contrib/golang/filters/http/test/test_data/basic/filter.go b/contrib/golang/filters/http/test/test_data/basic/filter.go
index b1838067c1c3..7181f36034be 100644
--- a/contrib/golang/filters/http/test/test_data/basic/filter.go
+++ b/contrib/golang/filters/http/test/test_data/basic/filter.go
@@ -114,6 +114,9 @@ func (f *filter) decodeHeaders(header api.RequestHeaderMap, endStream bool) api.
 	fs := f.callbacks.StreamInfo().FilterState()
 	fs.SetString("go_state_test_key", "go_state_test_value", api.StateTypeReadOnly, api.LifeSpanRequest, api.SharedWithUpstreamConnection)
 
+	val := fs.GetString("go_state_test_key")
+	header.Add("go-state-test-header-key", val)
+
 	if strings.Contains(f.localreplay, "decode-header") {
 		return f.sendLocalReply("decode-header")
 	}

From ba570d87414a7ae31c92939c571da7c8fa0b6f03 Mon Sep 17 00:00:00 2001
From: Rama Chavali 
Date: Tue, 13 Jun 2023 18:46:54 +0530
Subject: [PATCH 524/740] grpc: optimize getGrpcStatus implementation (#27896)

Signed-off-by: Rama Chavali 
---
 source/common/grpc/common.cc | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc
index fcbb413cba37..72abbbefcaa0 100644
--- a/source/common/grpc/common.cc
+++ b/source/common/grpc/common.cc
@@ -123,21 +123,18 @@ absl::optional Common::getGrpcStatus(const Http::ResponseTra
   //   1. trailers gRPC status, if it exists.
   //   2. headers gRPC status, if it exists.
   //   3. Inferred from info HTTP status, if it exists.
-  const std::array, 3> optional_statuses = {{
-      {Grpc::Common::getGrpcStatus(trailers, allow_user_defined)},
-      {Grpc::Common::getGrpcStatus(headers, allow_user_defined)},
-      {info.responseCode() ? absl::optional(
-                                 Grpc::Utility::httpToGrpcStatus(info.responseCode().value()))
-                           : absl::nullopt},
-  }};
-
-  for (const auto& optional_status : optional_statuses) {
-    if (optional_status.has_value()) {
-      return optional_status;
-    }
+  absl::optional optional_status;
+  optional_status = Grpc::Common::getGrpcStatus(trailers, allow_user_defined);
+  if (optional_status.has_value()) {
+    return optional_status;
   }
-
-  return absl::nullopt;
+  optional_status = Grpc::Common::getGrpcStatus(headers, allow_user_defined);
+  if (optional_status.has_value()) {
+    return optional_status;
+  }
+  return info.responseCode() ? absl::optional(
+                                   Grpc::Utility::httpToGrpcStatus(info.responseCode().value()))
+                             : absl::nullopt;
 }
 
 std::string Common::getGrpcMessage(const Http::ResponseHeaderOrTrailerMap& trailers) {

From 79bead722032b42359d7a1f08c13951db73b2e27 Mon Sep 17 00:00:00 2001
From: Tianyu <72890320+tyxia@users.noreply.github.com>
Date: Tue, 13 Jun 2023 10:07:36 -0400
Subject: [PATCH 525/740] Composite: Clean up test. (#27927)

Signed-off-by: tyxia 
---
 test/extensions/filters/http/composite/BUILD  |   6 +-
 .../composite_filter_integration_test.cc      | 313 ------------------
 2 files changed, 1 insertion(+), 318 deletions(-)

diff --git a/test/extensions/filters/http/composite/BUILD b/test/extensions/filters/http/composite/BUILD
index 229c6ba9a006..4c60b323301d 100644
--- a/test/extensions/filters/http/composite/BUILD
+++ b/test/extensions/filters/http/composite/BUILD
@@ -25,7 +25,7 @@ envoy_extension_cc_test(
 )
 
 envoy_extension_cc_test(
-    name = "composite_integration_test",
+    name = "composite_filter_integration_test",
     size = "large",
     srcs = ["composite_filter_integration_test.cc"],
     extension_names = ["envoy.filters.http.composite"],
@@ -34,13 +34,11 @@ envoy_extension_cc_test(
         "//source/common/http/match_delegate:config",
         "//source/extensions/filters/http/composite:config",
         "//source/extensions/filters/http/composite:filter_lib",
-        "//source/extensions/filters/http/ext_proc:config",
         "//source/extensions/matching/http/cel_input:cel_input_lib",
         "//source/extensions/matching/input_matchers/cel_matcher:cel_matcher_lib",
         "//source/extensions/matching/input_matchers/cel_matcher:config",
         "//test/common/grpc:grpc_client_integration_lib",
         "//test/common/http:common_lib",
-        "//test/extensions/filters/http/ext_proc:utils_lib",
         "//test/integration:http_integration_lib",
         "//test/integration/filters:server_factory_context_filter_config_proto_cc_proto",
         "//test/integration/filters:server_factory_context_filter_lib",
@@ -49,8 +47,6 @@ envoy_extension_cc_test(
         "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/http/composite/v3:pkg_cc_proto",
-        "@envoy_api//envoy/extensions/filters/http/ext_proc/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto",
-        "@envoy_api//envoy/service/ext_proc/v3:pkg_cc_proto",
     ],
 )
diff --git a/test/extensions/filters/http/composite/composite_filter_integration_test.cc b/test/extensions/filters/http/composite/composite_filter_integration_test.cc
index 4eb61efb41e6..aaad8465ed98 100644
--- a/test/extensions/filters/http/composite/composite_filter_integration_test.cc
+++ b/test/extensions/filters/http/composite/composite_filter_integration_test.cc
@@ -1,17 +1,13 @@
 #include "envoy/extensions/common/matching/v3/extension_matcher.pb.validate.h"
 #include "envoy/extensions/filters/http/composite/v3/composite.pb.h"
-#include "envoy/extensions/filters/http/ext_proc/v3/ext_proc.pb.h"
 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h"
 #include "envoy/network/address.h"
-#include "envoy/service/ext_proc/v3/external_processor.pb.h"
 
 #include "source/common/http/match_delegate/config.h"
 #include "source/common/http/matching/inputs.h"
-#include "source/extensions/filters/http/ext_proc/config.h"
 
 #include "test/common/grpc/grpc_client_integration.h"
 #include "test/common/http/common.h"
-#include "test/extensions/filters/http/ext_proc/utils.h"
 #include "test/integration/filters/server_factory_context_filter_config.pb.h"
 #include "test/integration/http_integration.h"
 #include "test/mocks/http/mocks.h"
@@ -91,315 +87,6 @@ TEST_P(CompositeFilterIntegrationTest, TestBasic) {
   }
 }
 
-using envoy::extensions::filters::http::ext_proc::v3::ProcessingMode;
-using envoy::service::ext_proc::v3::HeadersResponse;
-using envoy::service::ext_proc::v3::HttpHeaders;
-using envoy::service::ext_proc::v3::ProcessingRequest;
-using envoy::service::ext_proc::v3::ProcessingResponse;
-using Extensions::HttpFilters::ExternalProcessing::HasNoHeader;
-using Extensions::HttpFilters::ExternalProcessing::HeaderProtosEqual;
-using Extensions::HttpFilters::ExternalProcessing::SingleHeaderValueIs;
-
-// Integration test that has ext_proc filter as the delegated filter.
-class CompositeFilterWithExtProcIntegrationTest
-    : public HttpIntegrationTest,
-      public Grpc::GrpcClientIntegrationParamTestWithDeferredProcessing {
-public:
-  CompositeFilterWithExtProcIntegrationTest()
-      : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {}
-
-  void createUpstreams() override {
-    HttpIntegrationTest::createUpstreams();
-    // Create separate "upstreams" for ExtProc gRPC servers
-    for (int i = 0; i < 2; ++i) {
-      grpc_upstreams_.push_back(&addFakeUpstream(Http::CodecType::HTTP2));
-    }
-  }
-
-  void TearDown() override {
-    if (processor_connection_) {
-      ASSERT_TRUE(processor_connection_->close());
-      ASSERT_TRUE(processor_connection_->waitForDisconnect());
-    }
-    cleanupUpstreamAndDownstream();
-  }
-
-  void initializeConfig() {
-    config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
-      // Ensure "HTTP2 with no prior knowledge." Necessary for gRPC and for headers
-      ConfigHelper::setHttp2(
-          *(bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0)));
-
-      // Clusters for ExtProc gRPC servers, starting by copying an existing cluster
-      for (size_t i = 0; i < grpc_upstreams_.size(); ++i) {
-        auto* server_cluster = bootstrap.mutable_static_resources()->add_clusters();
-        server_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]);
-        std::string cluster_name = absl::StrCat("ext_proc_server_", i);
-        server_cluster->set_name(cluster_name);
-        server_cluster->mutable_load_assignment()->set_cluster_name(cluster_name);
-      }
-      // Load configuration of the server from YAML and use a helper to add a grpc_service
-      // stanza pointing to the cluster that we just made
-      setGrpcService(*proto_config_.mutable_grpc_service(), "ext_proc_server_0",
-                     grpc_upstreams_[0]->localAddress());
-
-      addFilter();
-
-      // Parameterize with defer processing to prevent bit rot as filter made
-      // assumptions of data flow, prior relying on eager processing.
-      config_helper_.addRuntimeOverride(Runtime::defer_processing_backedup_streams,
-                                        deferredProcessing() ? "true" : "false");
-    });
-
-    setUpstreamProtocol(Http::CodecType::HTTP2);
-    setDownstreamProtocol(Http::CodecType::HTTP2);
-  }
-
-  void addFilter() {
-    // Add the filter to the loaded hcm.
-    envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager
-        hcm_config;
-    config_helper_.loadHttpConnectionManager(hcm_config);
-    auto* match_delegate_filter = hcm_config.add_http_filters();
-    match_delegate_filter->set_name("envoy.filters.http.match_delegate");
-
-    // Build extension config with composite filter inside.
-    envoy::extensions::common::matching::v3::ExtensionWithMatcher extension_config;
-    extension_config.mutable_extension_config()->set_name("composite");
-    envoy::extensions::filters::http::composite::v3::Composite composite_config;
-    extension_config.mutable_extension_config()->mutable_typed_config()->PackFrom(composite_config);
-    auto* matcher_tree = extension_config.mutable_xds_matcher()->mutable_matcher_tree();
-
-    // Set up the match input.
-    auto* matcher_input = matcher_tree->mutable_input();
-    matcher_input->set_name("request-headers");
-    envoy::type::matcher::v3::HttpRequestHeaderMatchInput request_header_match_input;
-    request_header_match_input.set_header_name("match-header");
-    matcher_input->mutable_typed_config()->PackFrom(request_header_match_input);
-
-    // Set up the match action with ext_proc filter as the delegated filter.
-    auto* exact_match_map = matcher_tree->mutable_exact_match_map()->mutable_map();
-    envoy::extensions::filters::http::composite::v3::ExecuteFilterAction ext_proc_filter_action;
-    ext_proc_filter_action.mutable_typed_config()->set_name("envoy.filters.http.ext_proc");
-    // Set up ext_proc processing mode.
-    proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SEND);
-    proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SEND);
-    proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::BUFFERED);
-    proto_config_.mutable_processing_mode()->set_response_body_mode(ProcessingMode::NONE);
-    proto_config_.mutable_processing_mode()->set_request_trailer_mode(ProcessingMode::SKIP);
-    proto_config_.mutable_processing_mode()->set_response_trailer_mode(ProcessingMode::SKIP);
-    ext_proc_filter_action.mutable_typed_config()->mutable_typed_config()->PackFrom(proto_config_);
-
-    ::xds::type::matcher::v3::Matcher_OnMatch on_match;
-    auto* on_match_action = on_match.mutable_action();
-    on_match_action->set_name("composite-action");
-    on_match_action->mutable_typed_config()->PackFrom(ext_proc_filter_action);
-
-    (*exact_match_map)["match"] = on_match;
-
-    // Finish up the construction of match_delegate_filter.
-    match_delegate_filter->mutable_typed_config()->PackFrom(extension_config);
-
-    // Now move the built filter to the front.
-    for (int i = hcm_config.http_filters_size() - 1; i > 0; --i) {
-      hcm_config.mutable_http_filters()->SwapElements(i, i - 1);
-    }
-
-    // Store it to hcm.
-    config_helper_.storeHttpConnectionManager(hcm_config);
-  }
-
-  IntegrationStreamDecoderPtr sendDownstreamRequest(
-      absl::optional> modify_headers) {
-    auto conn = makeClientConnection(lookupPort("http"));
-    codec_client_ = makeHttpConnection(std::move(conn));
-    Http::TestRequestHeaderMapImpl headers;
-    HttpTestUtility::addDefaultHeaders(headers);
-    if (modify_headers) {
-      (*modify_headers)(headers);
-    }
-    return codec_client_->makeHeaderOnlyRequest(headers);
-  }
-
-  void waitForFirstMessage(FakeUpstream& grpc_upstream,
-                           envoy::service::ext_proc::v3::ProcessingRequest& request) {
-    ASSERT_TRUE(grpc_upstream.waitForHttpConnection(*dispatcher_, processor_connection_));
-    ASSERT_TRUE(processor_connection_->waitForNewStream(*dispatcher_, processor_stream_));
-    ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, request));
-  }
-
-  void handleUpstreamRequest() {
-    ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_));
-    ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_));
-    ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_));
-    upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false);
-    upstream_request_->encodeData(100, true);
-  }
-
-  void verifyDownstreamResponse(IntegrationStreamDecoder& response, int status_code) {
-    ASSERT_TRUE(response.waitForEndStream());
-    EXPECT_TRUE(response.complete());
-    EXPECT_EQ(std::to_string(status_code), response.headers().getStatusValue());
-  }
-
-  void processRequestHeadersMessage(
-      FakeUpstream& grpc_upstream, bool first_message,
-      absl::optional> cb) {
-    ProcessingRequest request;
-    if (first_message) {
-      ASSERT_TRUE(grpc_upstream.waitForHttpConnection(*dispatcher_, processor_connection_));
-      ASSERT_TRUE(processor_connection_->waitForNewStream(*dispatcher_, processor_stream_));
-    }
-    ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, request));
-    ASSERT_TRUE(request.has_request_headers());
-    if (first_message) {
-      processor_stream_->startGrpcStream();
-    }
-    ProcessingResponse response;
-    auto* headers = response.mutable_request_headers();
-    const bool sendReply = !cb || (*cb)(request.request_headers(), *headers);
-    if (sendReply) {
-      processor_stream_->sendGrpcMessage(response);
-    }
-  }
-
-  void processResponseHeadersMessage(
-      FakeUpstream& grpc_upstream, bool first_message,
-      absl::optional> cb) {
-    ProcessingRequest request;
-    if (first_message) {
-      ASSERT_TRUE(grpc_upstream.waitForHttpConnection(*dispatcher_, processor_connection_));
-      ASSERT_TRUE(processor_connection_->waitForNewStream(*dispatcher_, processor_stream_));
-    }
-    ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, request));
-    ASSERT_TRUE(request.has_response_headers());
-    if (first_message) {
-      processor_stream_->startGrpcStream();
-    }
-    ProcessingResponse response;
-    auto* headers = response.mutable_response_headers();
-    const bool sendReply = !cb || (*cb)(request.response_headers(), *headers);
-    if (sendReply) {
-      processor_stream_->sendGrpcMessage(response);
-    }
-  }
-
-  envoy::extensions::filters::http::ext_proc::v3::ExternalProcessor proto_config_{};
-  std::vector grpc_upstreams_;
-  FakeHttpConnectionPtr processor_connection_;
-  FakeStreamPtr processor_stream_;
-};
-
-INSTANTIATE_TEST_SUITE_P(
-    IpVersionsClientTypeDeferredProcessing, CompositeFilterWithExtProcIntegrationTest,
-    GRPC_CLIENT_INTEGRATION_DEFERRED_PROCESSING_PARAMS,
-    Grpc::GrpcClientIntegrationParamTestWithDeferredProcessing::protocolTestParamsToString);
-
-TEST_P(CompositeFilterWithExtProcIntegrationTest, GetAndCloseStream) {
-  initializeConfig();
-  HttpIntegrationTest::initialize();
-
-  auto conn = makeClientConnection(lookupPort("http"));
-  codec_client_ = makeHttpConnection(std::move(conn));
-  const Http::TestRequestHeaderMapImpl request_headers = {{":method", "GET"},
-                                                          {":path", "/somepath"},
-                                                          {":scheme", "http"},
-                                                          {"match-header", "match"},
-                                                          {":authority", "blah"}};
-  auto response = codec_client_->makeHeaderOnlyRequest(request_headers);
-
-  envoy::service::ext_proc::v3::ProcessingRequest request_headers_msg;
-  waitForFirstMessage(*grpc_upstreams_[0], request_headers_msg);
-  // Just close the stream without doing anything
-  processor_stream_->startGrpcStream();
-  processor_stream_->finishGrpcStream(Grpc::Status::Ok);
-
-  handleUpstreamRequest();
-  verifyDownstreamResponse(*response, 200);
-}
-
-// Test the filter using the default configuration by connecting to
-// an ext_proc server that responds to the request_headers message
-// successfully but closes the stream after response_headers.
-TEST_P(CompositeFilterWithExtProcIntegrationTest, GetAndCloseStreamOnResponse) {
-  initializeConfig();
-  HttpIntegrationTest::initialize();
-
-  auto conn = makeClientConnection(lookupPort("http"));
-  codec_client_ = makeHttpConnection(std::move(conn));
-  const Http::TestRequestHeaderMapImpl request_headers = {{":method", "GET"},
-                                                          {":path", "/somepath"},
-                                                          {":scheme", "http"},
-                                                          {"match-header", "match"},
-                                                          {":authority", "blah"}};
-  auto response = codec_client_->makeHeaderOnlyRequest(request_headers);
-
-  ProcessingRequest request_headers_msg;
-  waitForFirstMessage(*grpc_upstreams_[0], request_headers_msg);
-  processor_stream_->startGrpcStream();
-  ProcessingResponse resp1;
-  resp1.mutable_request_headers();
-  processor_stream_->sendGrpcMessage(resp1);
-
-  handleUpstreamRequest();
-
-  ProcessingRequest response_headers_msg;
-  ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, response_headers_msg));
-  processor_stream_->finishGrpcStream(Grpc::Status::Ok);
-
-  verifyDownstreamResponse(*response, 200);
-}
-
-TEST_P(CompositeFilterWithExtProcIntegrationTest, GetAndSetHeaders) {
-  initializeConfig();
-  HttpIntegrationTest::initialize();
-
-  auto conn = makeClientConnection(lookupPort("http"));
-  codec_client_ = makeHttpConnection(std::move(conn));
-  Http::TestRequestHeaderMapImpl request_headers = {{":method", "GET"},
-                                                    {":path", "/somepath"},
-                                                    {":scheme", "http"},
-                                                    {"match-header", "match"},
-                                                    {":authority", "blah"}};
-  request_headers.addCopy(Http::LowerCaseString("x-remove-this"), "yes");
-  auto response = codec_client_->makeHeaderOnlyRequest(request_headers);
-
-  processRequestHeadersMessage(
-      *grpc_upstreams_[0], true, [](const HttpHeaders& headers, HeadersResponse& headers_resp) {
-        Http::TestRequestHeaderMapImpl expected_request_headers{
-            {":method", "GET"},           {":path", "/somepath"}, {":scheme", "http"},
-            {"match-header", "match"},    {":authority", "blah"}, {"x-remove-this", "yes"},
-            {"x-forwarded-proto", "http"}};
-        EXPECT_THAT(headers.headers(), HeaderProtosEqual(expected_request_headers));
-
-        auto response_header_mutation = headers_resp.mutable_response()->mutable_header_mutation();
-        auto* mut1 = response_header_mutation->add_set_headers();
-        mut1->mutable_header()->set_key("x-new-header");
-        mut1->mutable_header()->set_value("new");
-        response_header_mutation->add_remove_headers("x-remove-this");
-        return true;
-      });
-
-  ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_));
-  ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_));
-  ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_));
-
-  EXPECT_THAT(upstream_request_->headers(), HasNoHeader("x-remove-this"));
-  EXPECT_THAT(upstream_request_->headers(), SingleHeaderValueIs("x-new-header", "new"));
-
-  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false);
-  upstream_request_->encodeData(100, true);
-
-  processResponseHeadersMessage(
-      *grpc_upstreams_[0], false, [](const HttpHeaders& headers, HeadersResponse&) {
-        Http::TestRequestHeaderMapImpl expected_response_headers{{":status", "200"}};
-        EXPECT_THAT(headers.headers(), HeaderProtosEqual(expected_response_headers));
-        return true;
-      });
-
-  verifyDownstreamResponse(*response, 200);
-}
-
 class CompositeFilterSeverContextIntegrationTest
     : public HttpIntegrationTest,
       public Grpc::GrpcClientIntegrationParamTestWithDeferredProcessing {

From a974ec7c626f19d2b7a8a2092b5b787a81500372 Mon Sep 17 00:00:00 2001
From: Zhewei Hu 
Date: Tue, 13 Jun 2023 07:37:12 -0700
Subject: [PATCH 526/740] [ZK filter] Add 'addWatch' opcode support (#27791)

* [ZK filter] Add 'addWatch' opcode support

Signed-off-by: Zhewei Hu 
---
 .../zookeeper_proxy/v3/zookeeper_proxy.proto       |  1 +
 changelogs/current.yaml                            |  3 +++
 .../network_filters/zookeeper_proxy_filter.rst     |  5 +++++
 .../filters/network/zookeeper_proxy/decoder.cc     | 12 ++++++++++++
 .../filters/network/zookeeper_proxy/decoder.h      |  7 ++++++-
 .../filters/network/zookeeper_proxy/filter.cc      |  7 +++++++
 .../filters/network/zookeeper_proxy/filter.h       |  8 +++++++-
 .../filters/network/zookeeper_proxy/config_test.cc |  2 +-
 .../filters/network/zookeeper_proxy/filter_test.cc | 14 ++++++++++++++
 9 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto
index 05422e9c3f8a..9fdcaaf9db2f 100644
--- a/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto
+++ b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto
@@ -89,6 +89,7 @@ message LatencyThresholdOverride {
     GetEphemerals = 23;
     GetAllChildrenNumber = 24;
     SetWatches2 = 25;
+    AddWatch = 26;
   }
 
   // The ZooKeeper opcodes. Can be found as part of the ZooKeeper source code:
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 44a909319a4e..747ec9c010a9 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -342,6 +342,9 @@ new_features:
     added host related information :ref:`metadata ` and
     :ref:`locality ` to
     the :ref:`health check event ` definition.
+- area: zookeeper
+  change: |
+    Added the "addWatch" opcode support to the ZooKeeper proxy filter.
 
 deprecated:
 - area: access_log
diff --git a/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst
index b3785d9ec37d..f62255909101 100644
--- a/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst
+++ b/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst
@@ -63,6 +63,7 @@ The following counters are available:
   reconfig_rq, Counter, Number of reconfig requests
   setwatches_rq, Counter, Number of setwatches requests
   setwatches2_rq, Counter, Number of setwatches2 requests
+  addwatch_rq, Counter, Number of addwatch requests
   checkwatches_rq, Counter, Number of checkwatches requests
   removewatches_rq, Counter, Number of removewatches requests
   getephemerals_rq, Counter, Number of getephemerals requests
@@ -91,6 +92,7 @@ The following counters are available:
   setauth_resp, Counter, Number of setauth responses
   setwatches_resp, Counter, Number of setwatches responses
   setwatches2_resp, Counter, Number of setwatches2 responses
+  addwatch_resp, Counter, Number of addwatch responses
   checkwatches_resp, Counter, Number of checkwatches responses
   removewatches_resp, Counter, Number of removewatches responses
   getephemerals_resp, Counter, Number of getephemerals responses
@@ -119,6 +121,7 @@ The following counters are available:
   setauth_resp_fast, Counter, Number of setauth responses faster than or equal to the threshold
   setwatches_resp_fast, Counter, Number of setwatches responses faster than or equal to the threshold
   setwatches2_resp_fast, Counter, Number of setwatches2 responses faster than or equal to the threshold
+  addwatch_resp_fast, Counter, Number of addwatch responses faster than or equal to the threshold
   checkwatches_resp_fast, Counter, Number of checkwatches responses faster than or equal to the threshold
   removewatches_resp_fast, Counter, Number of removewatches responses faster than or equal to the threshold
   getephemerals_resp_fast, Counter, Number of getephemerals responses faster than or equal to the threshold
@@ -146,6 +149,7 @@ The following counters are available:
   setauth_resp_slow, Counter, Number of setauth responses slower than the threshold
   setwatches_resp_slow, Counter, Number of setwatches responses slower than the threshold
   setwatches2_resp_slow, Counter, Number of setwatches2 responses slower than the threshold
+  addwatch_resp_slow, Counter, Number of addwatch responses slower than the threshold
   checkwatches_resp_slow, Counter, Number of checkwatches responses slower than the threshold
   removewatches_resp_slow, Counter, Number of removewatches responses slower than the threshold
   getephemerals_resp_slow, Counter, Number of getephemerals responses slower than the threshold
@@ -190,6 +194,7 @@ Latency stats are in milliseconds:
   setauth_resp_latency, Histogram, Opcode execution time in milliseconds
   setwatches_resp_latency, Histogram, Opcode execution time in milliseconds
   setwatches2_resp_latency, Histogram, Opcode execution time in milliseconds
+  addwatch_resp_latency, Histogram, Opcode execution time in milliseconds
   checkwatches_resp_latency, Histogram, Opcode execution time in milliseconds
   removewatches_resp_latency, Histogram, Opcode execution time in milliseconds
   check_resp_latency, Histogram, Opcode execution time in milliseconds
diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.cc b/source/extensions/filters/network/zookeeper_proxy/decoder.cc
index 18a9ee1129da..190d7ae518fc 100644
--- a/source/extensions/filters/network/zookeeper_proxy/decoder.cc
+++ b/source/extensions/filters/network/zookeeper_proxy/decoder.cc
@@ -149,6 +149,9 @@ void DecoderImpl::decodeOnData(Buffer::Instance& data, uint64_t& offset) {
   case OpCodes::SetWatches2:
     parseSetWatches2Request(data, offset, len);
     break;
+  case OpCodes::AddWatch:
+    parseAddWatchRequest(data, offset, len);
+    break;
   case OpCodes::CheckWatches:
     parseXWatchesRequest(data, offset, len, OpCodes::CheckWatches);
     break;
@@ -467,6 +470,15 @@ void DecoderImpl::parseSetWatches2Request(Buffer::Instance& data, uint64_t& offs
   callbacks_.onSetWatches2Request();
 }
 
+void DecoderImpl::parseAddWatchRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) {
+  ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH));
+
+  const std::string path = helper_.peekString(data, offset);
+  const int32_t mode = helper_.peekInt32(data, offset);
+
+  callbacks_.onAddWatchRequest(path, mode);
+}
+
 void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len,
                                        OpCodes opcode) {
   ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH));
diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.h b/source/extensions/filters/network/zookeeper_proxy/decoder.h
index 540cff9d8e23..36278932524c 100644
--- a/source/extensions/filters/network/zookeeper_proxy/decoder.h
+++ b/source/extensions/filters/network/zookeeper_proxy/decoder.h
@@ -51,11 +51,14 @@ enum class OpCodes {
   SetWatches = 101,
   GetEphemerals = 103,
   GetAllChildrenNumber = 104,
-  SetWatches2 = 105
+  SetWatches2 = 105,
+  AddWatch = 106,
 };
 
 enum class WatcherType { Children = 1, Data = 2, Any = 3 };
 
+enum class AddWatchMode { Persistent, PersistentRecursive };
+
 enum class CreateFlags {
   Persistent,
   PersistentSequential,
@@ -96,6 +99,7 @@ class DecoderCallbacks {
   virtual void onReconfigRequest() PURE;
   virtual void onSetWatchesRequest() PURE;
   virtual void onSetWatches2Request() PURE;
+  virtual void onAddWatchRequest(const std::string& path, const int32_t mode) PURE;
   virtual void onCheckWatchesRequest(const std::string& path, int32_t type) PURE;
   virtual void onRemoveWatchesRequest(const std::string& path, int32_t type) PURE;
   virtual void onCloseRequest() PURE;
@@ -167,6 +171,7 @@ class DecoderImpl : public Decoder, Logger::Loggable {
   void parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len);
   void parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len);
   void parseSetWatches2Request(Buffer::Instance& data, uint64_t& offset, uint32_t len);
+  void parseAddWatchRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len);
   void parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode);
   void skipString(Buffer::Instance& data, uint64_t& offset);
   void skipStrings(Buffer::Instance& data, uint64_t& offset);
diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.cc b/source/extensions/filters/network/zookeeper_proxy/filter.cc
index a6369e84444a..03d2202a7395 100644
--- a/source/extensions/filters/network/zookeeper_proxy/filter.cc
+++ b/source/extensions/filters/network/zookeeper_proxy/filter.cc
@@ -81,6 +81,8 @@ ZooKeeperFilterConfig::ZooKeeperFilterConfig(
              stats_.setwatches_resp_slow_, "setwatches_resp");
   initOpCode(OpCodes::SetWatches2, stats_.setwatches2_resp_, stats_.setwatches2_resp_fast_,
              stats_.setwatches2_resp_slow_, "setwatches2_resp");
+  initOpCode(OpCodes::AddWatch, stats_.addwatch_resp_, stats_.addwatch_resp_fast_,
+             stats_.addwatch_resp_slow_, "addwatch_resp");
   initOpCode(OpCodes::CheckWatches, stats_.checkwatches_resp_, stats_.checkwatches_resp_fast_,
              stats_.checkwatches_resp_slow_, "checkwatches_resp");
   initOpCode(OpCodes::RemoveWatches, stats_.removewatches_resp_, stats_.removewatches_resp_fast_,
@@ -351,6 +353,11 @@ void ZooKeeperFilter::onSetWatches2Request() {
   setDynamicMetadata("opname", "setwatches2");
 }
 
+void ZooKeeperFilter::onAddWatchRequest(const std::string& path, const int32_t mode) {
+  config_->stats_.addwatch_rq_.inc();
+  setDynamicMetadata({{"opname", "addwatch"}, {"path", path}, {"mode", std::to_string(mode)}});
+}
+
 void ZooKeeperFilter::onGetEphemeralsRequest(const std::string& path) {
   config_->stats_.getephemerals_rq_.inc();
   setDynamicMetadata({{"opname", "getephemerals"}, {"path", path}});
diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.h b/source/extensions/filters/network/zookeeper_proxy/filter.h
index 483c5653a59f..c152fc59f399 100644
--- a/source/extensions/filters/network/zookeeper_proxy/filter.h
+++ b/source/extensions/filters/network/zookeeper_proxy/filter.h
@@ -52,6 +52,7 @@ namespace ZooKeeperProxy {
   COUNTER(setauth_rq)                                                                              \
   COUNTER(setwatches_rq)                                                                           \
   COUNTER(setwatches2_rq)                                                                          \
+  COUNTER(addwatch_rq)                                                                             \
   COUNTER(checkwatches_rq)                                                                         \
   COUNTER(removewatches_rq)                                                                        \
   COUNTER(check_rq)                                                                                \
@@ -80,6 +81,7 @@ namespace ZooKeeperProxy {
   COUNTER(setauth_resp)                                                                            \
   COUNTER(setwatches_resp)                                                                         \
   COUNTER(setwatches2_resp)                                                                        \
+  COUNTER(addwatch_resp)                                                                           \
   COUNTER(checkwatches_resp)                                                                       \
   COUNTER(removewatches_resp)                                                                      \
   COUNTER(check_resp)                                                                              \
@@ -108,6 +110,7 @@ namespace ZooKeeperProxy {
   COUNTER(setauth_resp_fast)                                                                       \
   COUNTER(setwatches_resp_fast)                                                                    \
   COUNTER(setwatches2_resp_fast)                                                                   \
+  COUNTER(addwatch_resp_fast)                                                                      \
   COUNTER(checkwatches_resp_fast)                                                                  \
   COUNTER(removewatches_resp_fast)                                                                 \
   COUNTER(check_resp_fast)                                                                         \
@@ -135,6 +138,7 @@ namespace ZooKeeperProxy {
   COUNTER(setauth_resp_slow)                                                                       \
   COUNTER(setwatches_resp_slow)                                                                    \
   COUNTER(setwatches2_resp_slow)                                                                   \
+  COUNTER(addwatch_resp_slow)                                                                      \
   COUNTER(checkwatches_resp_slow)                                                                  \
   COUNTER(removewatches_resp_slow)                                                                 \
   COUNTER(check_resp_slow)
@@ -232,7 +236,8 @@ class ZooKeeperFilterConfig {
                                        {LatencyThresholdOverride::SetWatches, 101},
                                        {LatencyThresholdOverride::GetEphemerals, 103},
                                        {LatencyThresholdOverride::GetAllChildrenNumber, 104},
-                                       {LatencyThresholdOverride::SetWatches2, 105}});
+                                       {LatencyThresholdOverride::SetWatches2, 105},
+                                       {LatencyThresholdOverride::AddWatch, 106}});
   }
 
   int32_t getOpCodeIndex(LatencyThresholdOverride_Opcode opcode);
@@ -284,6 +289,7 @@ class ZooKeeperFilter : public Network::Filter,
   void onReconfigRequest() override;
   void onSetWatchesRequest() override;
   void onSetWatches2Request() override;
+  void onAddWatchRequest(const std::string& path, const int32_t mode) override;
   void onCheckWatchesRequest(const std::string& path, int32_t type) override;
   void onRemoveWatchesRequest(const std::string& path, int32_t type) override;
   void onGetEphemeralsRequest(const std::string& path) override;
diff --git a/test/extensions/filters/network/zookeeper_proxy/config_test.cc b/test/extensions/filters/network/zookeeper_proxy/config_test.cc
index be99a561eca1..2d0e24a0af3a 100644
--- a/test/extensions/filters/network/zookeeper_proxy/config_test.cc
+++ b/test/extensions/filters/network/zookeeper_proxy/config_test.cc
@@ -253,7 +253,7 @@ TEST_F(ZookeeperFilterConfigTest, FullConfig) {
   EXPECT_EQ(proto_config_.enable_latency_threshold_metrics(), true);
   EXPECT_EQ(proto_config_.default_latency_threshold(),
             ProtobufWkt::util::TimeUtil::MillisecondsToDuration(100));
-  EXPECT_EQ(proto_config_.latency_threshold_overrides_size(), 26);
+  EXPECT_EQ(proto_config_.latency_threshold_overrides_size(), 27);
 
   for (int i = 0; i < opcode_descriptor->value_count(); i++) {
     LatencyThresholdOverride threshold_override = proto_config_.latency_threshold_overrides().at(i);
diff --git a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc
index 707e33ae887d..b619800fc55b 100644
--- a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc
+++ b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc
@@ -1240,6 +1240,20 @@ TEST_F(ZooKeeperFilterTest, SetWatches2Request) {
       config_->stats().setwatches2_resp_slow_);
 }
 
+TEST_F(ZooKeeperFilterTest, AddWatchRequest) {
+  initialize();
+
+  Buffer::OwnedImpl data =
+      encodePathVersion("/foo", enumToSignedInt(AddWatchMode::PersistentRecursive),
+                        enumToSignedInt(OpCodes::AddWatch));
+
+  testRequest(data, {{{"opname", "addwatch"}, {"path", "/foo"}, {"mode", "1"}}, {{"bytes", "24"}}},
+              config_->stats().addwatch_rq_, 24);
+  testResponse({{{"opname", "addwatch_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}},
+               config_->stats().addwatch_resp_, config_->stats().addwatch_resp_fast_,
+               config_->stats().addwatch_resp_slow_);
+}
+
 TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) {
   initialize();
 

From 29a1c05c31e910eb25b86cff49c0408f84619fbc Mon Sep 17 00:00:00 2001
From: Snow Pettersen 
Date: Tue, 13 Jun 2023 10:51:51 -0400
Subject: [PATCH 527/740] owners: move snowp to emiritus (#27752)

Signed-off-by: Snow Pettersen 
---
 CODEOWNERS | 56 +++++++++++++++++++++++++++---------------------------
 OWNERS.md  |  3 +--
 2 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index c544bdb82b03..3a2176501fb6 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -13,22 +13,22 @@
 # csrf extension
 /*/extensions/filters/http/csrf @dschaller @mattklein123
 # original_src http filter extension
-/*/extensions/filters/http/original_src @snowp @klarose
+/*/extensions/filters/http/original_src @klarose @mattklein123
 # original_src listener filter extension
-/*/extensions/filters/listener/original_src @snowp @klarose
+/*/extensions/filters/listener/original_src @klarose @mattklein123
 # original_src common extension
-extensions/filters/common/original_src @snowp @klarose
+extensions/filters/common/original_src @klarose @mattklein123
 # dubbo_proxy extension
 /*/extensions/filters/network/dubbo_proxy @zyfjeff @lizan @wbpcode
 # cdn_loop extension
 /*/extensions/filters/http/cdn_loop @justin-mp @penguingao @alyssawilk
 # external processing filter
-/*/extensions/filters/http/ext_proc @gbrail @snowp @stevenzzzz @tyxia
-/*/extensions/filters/common/mutation_rules @gbrail @snowp @chaoqin-li1123 @tyxia
+/*/extensions/filters/http/ext_proc @gbrail @stevenzzzz @tyxia @mattklein123
+/*/extensions/filters/common/mutation_rules @gbrail @chaoqin-li1123 @tyxia @mattklein123
 # jwt_authn http filter extension
 /*/extensions/filters/http/jwt_authn @qiwzhang @lizan
 # grpc_http1_reverse_bridge http filter extension
-/*/extensions/filters/http/grpc_http1_reverse_bridge @snowp @zuercher
+/*/extensions/filters/http/grpc_http1_reverse_bridge @zuercher @mattklein123
 # alts transport socket extension
 /*/extensions/transport_sockets/alts @adisuissa @yangminzhu
 # tcp_stats transport socket extension
@@ -73,11 +73,11 @@ extensions/filters/common/original_src @snowp @klarose
 /*/extensions/clusters/dynamic_forward_proxy @mattklein123 @alyssawilk
 /*/extensions/common/dynamic_forward_proxy @mattklein123 @alyssawilk
 /*/extensions/filters/http/dynamic_forward_proxy @mattklein123 @alyssawilk
-/*/extensions/filters/http/composite @snowp @mattklein123
+/*/extensions/filters/http/composite @mattklein123 @mattklein123
 # omit_canary_hosts retry predicate
-/*/extensions/retry/host/omit_canary_hosts @sriduth @snowp
+/*/extensions/retry/host/omit_canary_hosts @sriduth @mattklein123
 # previous hosts
-/*/extensions/retry/host/previous_hosts @snowp @alyssawilk
+/*/extensions/retry/host/previous_hosts @alyssawilk @mattklein123
 # HTTP caching extension
 /*/extensions/filters/http/cache @toddmgreer @jmarantz @penguingao @mpwarres @capoferro
 /*/extensions/http/cache/simple_http_cache @toddmgreer @jmarantz @penguingao @mpwarres @capoferro
@@ -107,16 +107,16 @@ extensions/filters/common/original_src @snowp @klarose
 # common matcher
 /*/extensions/common/matcher @mattklein123 @yangminzhu
 /*/extensions/common/proxy_protocol @alyssawilk @wez470
-/*/extensions/filters/http/grpc_http1_bridge @snowp @jose
+/*/extensions/filters/http/grpc_http1_bridge @jose @mattklein123
 /*/extensions/filters/http/fault @rshriram @alyssawilk
 /*/extensions/filters/common/fault @rshriram @alyssawilk
 /*/extensions/filters/http/grpc_json_transcoder @qiwzhang @lizan
-/*/extensions/filters/http/router @alyssawilk @mattklein123 @snowp
+/*/extensions/filters/http/router @alyssawilk @mattklein123
 /*/extensions/filters/common/rbac/matchers @conqerAtapple @ggreenway @alyssawilk
 /*/extensions/filters/http/grpc_web @fengli79 @lizan
 /*/extensions/filters/http/grpc_stats @kyessenov @lizan
 /*/extensions/filters/http/connect_grpc_bridge @UNOWNED @UNOWNED
-/*/extensions/filters/common/original_src @klarose @snowp
+/*/extensions/filters/common/original_src @klarose @mattklein123
 /*/extensions/filters/listener/tls_inspector @ggreenway @KBaichoo
 /*/extensions/grpc_credentials/example @wozz @htuch
 /*/extensions/grpc_credentials/file_based_metadata @wozz @htuch
@@ -132,16 +132,16 @@ extensions/filters/common/original_src @snowp @klarose
 /*/extensions/resource_monitors/common @eziskind @htuch
 /*/extensions/resource_monitors/fixed_heap @eziskind @htuch
 /*/extensions/resource_monitors/downstream_connections @nezdolik @mattklein123
-/*/extensions/retry/priority @snowp @alyssawilk
-/*/extensions/retry/priority/previous_priorities @snowp @alyssawilk
-/*/extensions/retry/host @snowp @alyssawilk
+/*/extensions/retry/priority @alyssawilk @mattklein123
+/*/extensions/retry/priority/previous_priorities @alyssawilk @mattklein123
+/*/extensions/retry/host @alyssawilk @mattklein123
 /*/extensions/filters/network/http_connection_manager @alyssawilk @mattklein123
 /*/extensions/filters/network/tcp_proxy @alyssawilk @zuercher @ggreenway
 /*/extensions/filters/network/echo @htuch @alyssawilk
 /*/extensions/filters/udp/dns_filter @abaptiste @mattklein123 @yanjunxiang-google
 /*/extensions/filters/network/direct_response @kyessenov @zuercher
 /*/extensions/filters/udp/udp_proxy @mattklein123 @danzh2010
-/*/extensions/clusters/aggregate @yxue @snowp
+/*/extensions/clusters/aggregate @yxue @mattklein123
 # support for on-demand VHDS requests
 /*/extensions/filters/http/on_demand @dmitri-d @htuch @kyessenov
 /*/extensions/filters/network/connection_limit @mattklein123 @alyssawilk @delong-coder
@@ -152,10 +152,10 @@ extensions/filters/common/original_src @snowp @klarose
 # Watchdog Extensions
 /*/extensions/watchdog/profile_action @kbaichoo @alyssawilk
 # Core upstream code
-extensions/upstreams/http @alyssawilk @snowp @mattklein123
+extensions/upstreams/http @alyssawilk @mattklein123
 extensions/upstreams/tcp @alyssawilk @ggreenway @mattklein123
 # OAuth2
-extensions/filters/http/oauth2 @derekargueta @snowp
+extensions/filters/http/oauth2 @derekargueta @mattklein123
 # HTTP Local Rate Limit
 /*/extensions/filters/http/local_ratelimit @mattklein123 @wbpcode
 /*/extensions/filters/common/local_ratelimit @mattklein123 @wbpcode
@@ -164,13 +164,13 @@ extensions/filters/http/oauth2 @derekargueta @snowp
 # Rate limit expression descriptor
 /*/extensions/rate_limit_descriptors/expr @kyessenov @lizan
 # hash input matcher
-/*/extensions/matching/input_matchers/consistent_hashing @snowp @donyu
+/*/extensions/matching/input_matchers/consistent_hashing @donyu @mattklein123
 # runtime fraction input matcher
 /*/extensions/matching/input_matchers/runtime_fraction @ravenblackx @UNOWNED
 # CEL input matcher
 /*/extensions/matching/input_matchers/cel_matcher @tyxia @UNOWNED
 # environment generic input
-/*/extensions/matching/common_inputs/environment @snowp @donyu
+/*/extensions/matching/common_inputs/environment @donyu @mattklein123
 # format string matching
 /*/extensions/matching/actions/format_string @kyessenov @UNOWNED
 # CEL data input
@@ -194,12 +194,12 @@ extensions/filters/http/oauth2 @derekargueta @snowp
 /*/extensions/http/original_ip_detection/custom_header @alyssawilk @mattklein123
 /*/extensions/http/original_ip_detection/xff @alyssawilk @mattklein123
 # set_metadata extension
-/*/extensions/filters/http/set_metadata @aguinet @snowp
+/*/extensions/filters/http/set_metadata @aguinet @mattklein123
 # Formatters
 /*/extensions/formatter/metadata @cpakulski @lizan
 /*/extensions/formatter/cel @kyessenov @zirain
 # IP address input matcher
-/*/extensions/matching/input_matchers/ip @aguinet @snowp
+/*/extensions/matching/input_matchers/ip @aguinet @mattklein123
 # Key Value store
 /*/extensions/key_value @alyssawilk @ryantheoptimist
 # Config Validators
@@ -283,7 +283,7 @@ extensions/filters/http/oauth2 @derekargueta @snowp
 # Header to metadata
 /*/extensions/filters/http/header_to_metadata @zuercher @JuniorHsu
 # zookeeper
-/*/extensions/filters/network/zookeeper_proxy @snowp @JuniorHsu
+/*/extensions/filters/network/zookeeper_proxy @JuniorHsu @Winbobob @mattklein123
 # Custom response filter
 /*/extensions/filters/http/custom_response @pradeepcrao @yanavlasov
 /*/extensions/http/custom_response/redirect_policy @pradeepcrao @yanavlasov
@@ -306,19 +306,19 @@ extensions/filters/http/oauth2 @derekargueta @snowp
 # Early header mutation
 /*/extensions/http/early_header_mutation/header_mutation @wbpcode @UNOWNED
 # Network matching extensions
-/*/extensions/matching/network/ @kyessenov @snowp
+/*/extensions/matching/network/ @kyessenov @mattklein123
 # Header mutation
 /*/extensions/filters/http/header_mutation @wbpcode @htuch @soulxu
 # Health checkers
-/*/extensions/health_checkers/grpc @snowp @zuercher
-/*/extensions/health_checkers/http @snowp @zuercher
-/*/extensions/health_checkers/tcp @snowp @zuercher
+/*/extensions/health_checkers/grpc @zuercher @botengyao
+/*/extensions/health_checkers/http @zuercher @botengyao
+/*/extensions/health_checkers/tcp @zuercher @botengyao
 # Health check event sinks
 /*/extensions/health_check/event_sinks/file @botengyao @yanavlasov
 # IP Geolocation
 /*/extensions/filters/http/geoip @nezdolik @ravenblackx
 
-/*/extensions/health_checkers/common @snowp @zuercher
+/*/extensions/health_checkers/common @zuercher @botengyao
 
 # Intentionally exempt (treated as core code)
 /*/extensions/filters/common @UNOWNED @UNOWNED
diff --git a/OWNERS.md b/OWNERS.md
index 6cc1566b9105..b8c130874f51 100644
--- a/OWNERS.md
+++ b/OWNERS.md
@@ -21,8 +21,6 @@ routing PRs, questions, etc. to the right place.
 * Lizan Zhou ([lizan](https://github.com/lizan)) (lizan.j@gmail.com)
   * gRPC, gRPC/JSON transcoding, and core networking (transport socket abstractions), Bazel, build
     issues, and CI in general.
-* Snow Pettersen ([snowp](https://github.com/snowp)) (aickck@gmail.com)
-  * Upstream, host/priority sets, load balancing, and retry plugins.
 * Greg Greenway ([ggreenway](https://github.com/ggreenway)) (ggreenway@apple.com)
   * TLS, TCP proxy, listeners, and HTTP proxy/connection pooling.
 * Yan Avlasov ([yanavlasov](https://github.com/yanavlasov)) (yavlasov@google.com)
@@ -111,6 +109,7 @@ contributors to envoy-setec and relevant Slack channels from:
 * Antonio Vicente ([antoniovicente](https://github.com/antoniovicente)) (avd@google.com)
 * JP Simard ([jpsim](https://github.com/jpsim)) (jp@lyft.com)
 * Rafal Augustyniak ([Augustyniak](https://github.com/Augustyniak)) (raugustyniak@lyft.com)
+* Snow Pettersen ([snowp](https://github.com/snowp)) (aickck@gmail.com)
 
 # Friends of Envoy
 

From 417f0d343f5c215b289f731dbb4f64b4d7a67874 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Tue, 13 Jun 2023 11:54:15 -0400
Subject: [PATCH 528/740] api listener: removing exceptions (#27824)

Commit Message:
Additional Description:
Risk Level:
Testing:
Docs Changes:
Release Notes:

envoyproxy/envoy-mobile#176

Signed-off-by: Alyssa Wilk 
---
 envoy/server/listener_manager.h               | 10 +++--
 .../api_listener_manager.cc                   |  8 ++--
 .../api_listener_manager.h                    |  5 ++-
 .../listener_manager/lds_api.cc               |  7 +++-
 .../listener_manager/listener_manager_impl.cc | 13 +++----
 .../listener_manager/listener_manager_impl.h  |  5 ++-
 source/server/configuration_impl.cc           |  6 ++-
 .../network/echo/echo_integration_test.cc     |  7 +++-
 .../listener_manager_impl_test.cc             | 38 +++++++++----------
 .../listener_manager_impl_test.h              |  6 ++-
 test/mocks/server/listener_manager.cc         |  7 +++-
 test/mocks/server/listener_manager.h          |  2 +-
 12 files changed, 70 insertions(+), 44 deletions(-)

diff --git a/envoy/server/listener_manager.h b/envoy/server/listener_manager.h
index b59c2a1ba7d8..37d4daff291e 100644
--- a/envoy/server/listener_manager.h
+++ b/envoy/server/listener_manager.h
@@ -166,11 +166,13 @@ class ListenerManager {
    *        listener is not modifiable, future calls to this function or removeListener() on behalf
    *        of this listener will return false.
    * @return TRUE if a listener was added or FALSE if the listener was not updated because it is
-   *         a duplicate of the existing listener. This routine will throw an EnvoyException if
-   *         there is a fundamental error preventing the listener from being added or updated.
+   *         a duplicate of the existing listener. This routine will return
+   *         absl::InvalidArgumentError if there is a fundamental error preventing the listener
+   *         from being added or updated.
    */
-  virtual bool addOrUpdateListener(const envoy::config::listener::v3::Listener& config,
-                                   const std::string& version_info, bool modifiable) PURE;
+  virtual absl::StatusOr
+  addOrUpdateListener(const envoy::config::listener::v3::Listener& config,
+                      const std::string& version_info, bool modifiable) PURE;
 
   /**
    * Instruct the listener manager to create an LDS API provider. This is a separate operation
diff --git a/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.cc b/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.cc
index 2185b2dcca7a..e3170b9231be 100644
--- a/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.cc
+++ b/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.cc
@@ -12,8 +12,10 @@ namespace Envoy {
 namespace Server {
 
 ApiListenerManagerImpl::ApiListenerManagerImpl(Instance& server) : server_(server) {}
-bool ApiListenerManagerImpl::addOrUpdateListener(
-    const envoy::config::listener::v3::Listener& config, const std::string&, bool added_via_api) {
+
+absl::StatusOr
+ApiListenerManagerImpl::addOrUpdateListener(const envoy::config::listener::v3::Listener& config,
+                                            const std::string&, bool added_via_api) {
   ENVOY_LOG(debug, "Creating API listener manager");
   std::string name;
   if (!config.name().empty()) {
@@ -29,7 +31,7 @@ bool ApiListenerManagerImpl::addOrUpdateListener(
   // future allow multiple ApiListeners, and allow them to be created via LDS as well as bootstrap.
   if (config.has_api_listener()) {
     if (config.has_internal_listener()) {
-      throw EnvoyException(fmt::format(
+      return absl::InvalidArgumentError(fmt::format(
           "error adding listener named '{}': api_listener and internal_listener cannot be both set",
           name));
     }
diff --git a/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.h b/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.h
index 57028b7a8e9b..0687d544e1ae 100644
--- a/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.h
+++ b/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.h
@@ -19,8 +19,9 @@ class ApiListenerManagerImpl : public ListenerManager, Logger::Loggable addOrUpdateListener(const envoy::config::listener::v3::Listener& config,
+                                           const std::string& version_info,
+                                           bool added_via_api) override;
   void createLdsApi(const envoy::config::core::v3::ConfigSource&,
                     const xds::core::v3::ResourceLocator*) override {}
   std::vector> listeners(ListenerState) override {
diff --git a/source/extensions/listener_managers/listener_manager/lds_api.cc b/source/extensions/listener_managers/listener_manager/lds_api.cc
index 51f795e36d5b..8a1235317a6e 100644
--- a/source/extensions/listener_managers/listener_manager/lds_api.cc
+++ b/source/extensions/listener_managers/listener_manager/lds_api.cc
@@ -75,7 +75,12 @@ void LdsApiImpl::onConfigUpdate(const std::vector& a
         // applied.
         throw EnvoyException(fmt::format("duplicate listener {} found", listener.name()));
       }
-      if (listener_manager_.addOrUpdateListener(listener, resource.get().version(), true)) {
+      absl::StatusOr update_or_error =
+          listener_manager_.addOrUpdateListener(listener, resource.get().version(), true);
+      if (!update_or_error.status().ok()) {
+        throw EnvoyException(std::string(update_or_error.status().message()));
+      }
+      if (update_or_error.value()) {
         ENVOY_LOG(info, "lds: add/update listener '{}'", listener.name());
         any_applied = true;
       } else {
diff --git a/source/extensions/listener_managers/listener_manager/listener_manager_impl.cc b/source/extensions/listener_managers/listener_manager/listener_manager_impl.cc
index 648ac1a129f5..17502b66af69 100644
--- a/source/extensions/listener_managers/listener_manager/listener_manager_impl.cc
+++ b/source/extensions/listener_managers/listener_manager/listener_manager_impl.cc
@@ -377,8 +377,9 @@ ListenerManagerStats ListenerManagerImpl::generateStats(Stats::Scope& scope) {
   return {ALL_LISTENER_MANAGER_STATS(POOL_COUNTER(scope), POOL_GAUGE(scope))};
 }
 
-bool ListenerManagerImpl::addOrUpdateListener(const envoy::config::listener::v3::Listener& config,
-                                              const std::string& version_info, bool added_via_api) {
+absl::StatusOr
+ListenerManagerImpl::addOrUpdateListener(const envoy::config::listener::v3::Listener& config,
+                                         const std::string& version_info, bool added_via_api) {
   std::string name;
   if (!config.name().empty()) {
     name = config.name();
@@ -393,13 +394,11 @@ bool ListenerManagerImpl::addOrUpdateListener(const envoy::config::listener::v3:
   // future allow multiple ApiListeners, and allow them to be created via LDS as well as bootstrap.
   if (config.has_api_listener()) {
     if (config.has_internal_listener()) {
-      throw EnvoyException(fmt::format(
+      return absl::InvalidArgumentError(fmt::format(
           "error adding listener named '{}': api_listener and internal_listener cannot be both set",
           name));
     }
     if (!api_listener_ && !added_via_api) {
-      // TODO(junr03): dispatch to different concrete constructors when there are other
-      // ApiListenerImplBase derived classes.
       api_listener_ = std::make_unique(config, server_, config.name());
       return true;
     } else {
@@ -411,7 +410,7 @@ bool ListenerManagerImpl::addOrUpdateListener(const envoy::config::listener::v3:
 
   // Address field is not required for internal listeners.
   if (!config.has_internal_listener() && !config.has_address()) {
-    throw EnvoyException(
+    return absl::InvalidArgumentError(
         fmt::format("error adding listener named '{}': address is necessary", name));
   }
 
@@ -428,7 +427,7 @@ bool ListenerManagerImpl::addOrUpdateListener(const envoy::config::listener::v3:
                                           *(it->second->mutable_last_update_attempt()));
     it->second->set_details(e.what());
     it->second->mutable_failed_configuration()->PackFrom(config);
-    throw e;
+    return absl::InvalidArgumentError(e.what());
   }
   error_state_tracker_.erase(it);
   return false;
diff --git a/source/extensions/listener_managers/listener_manager/listener_manager_impl.h b/source/extensions/listener_managers/listener_manager/listener_manager_impl.h
index ef5483041aa5..2e6a943ac797 100644
--- a/source/extensions/listener_managers/listener_manager/listener_manager_impl.h
+++ b/source/extensions/listener_managers/listener_manager/listener_manager_impl.h
@@ -192,8 +192,9 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable addOrUpdateListener(const envoy::config::listener::v3::Listener& config,
+                                           const std::string& version_info,
+                                           bool added_via_api) override;
   void createLdsApi(const envoy::config::core::v3::ConfigSource& lds_config,
                     const xds::core::v3::ResourceLocator* lds_resources_locator) override {
     ASSERT(lds_api_ == nullptr);
diff --git a/source/server/configuration_impl.cc b/source/server/configuration_impl.cc
index 51e78065f718..86ce9a2bb461 100644
--- a/source/server/configuration_impl.cc
+++ b/source/server/configuration_impl.cc
@@ -101,7 +101,11 @@ void MainImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstr
   ENVOY_LOG(info, "loading {} listener(s)", listeners.size());
   for (ssize_t i = 0; i < listeners.size(); i++) {
     ENVOY_LOG(debug, "listener #{}:", i);
-    server.listenerManager().addOrUpdateListener(listeners[i], "", false);
+    absl::StatusOr update_or_error =
+        server.listenerManager().addOrUpdateListener(listeners[i], "", false);
+    if (!update_or_error.status().ok()) {
+      throw EnvoyException(std::string(update_or_error.status().message()));
+    }
   }
 
   initializeWatchdogs(bootstrap, server);
diff --git a/test/extensions/filters/network/echo/echo_integration_test.cc b/test/extensions/filters/network/echo/echo_integration_test.cc
index fdcef216fec4..eec6a59af6f4 100644
--- a/test/extensions/filters/network/echo/echo_integration_test.cc
+++ b/test/extensions/filters/network/echo/echo_integration_test.cc
@@ -70,8 +70,11 @@ name: new_listener
   test_server_->setOnWorkerListenerAddedCb(
       [&listener_added_by_worker]() -> void { listener_added_by_worker.setReady(); });
   test_server_->server().dispatcher().post([this, json, &listener_added_by_manager]() -> void {
-    EXPECT_TRUE(test_server_->server().listenerManager().addOrUpdateListener(
-        Server::parseListenerFromV3Yaml(json), "", true));
+    EXPECT_TRUE(test_server_->server()
+                    .listenerManager()
+                    .addOrUpdateListener(Server::parseListenerFromV3Yaml(json), "", true)
+                    .status()
+                    .ok());
     listener_added_by_manager.setReady();
   });
   listener_added_by_worker.waitReady();
diff --git a/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc b/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc
index a6ca81b9a548..df1bf46ccb26 100644
--- a/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc
+++ b/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc
@@ -781,10 +781,10 @@ TEST_P(ListenerManagerImplTest, MultipleSocketTypeSpecifiedInAddresses) {
     - filters: []
   )EOF";
 
-  EXPECT_THROW_WITH_MESSAGE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml), "", true),
-                            EnvoyException,
-                            "listener foo: has different socket type. The listener only "
-                            "support same socket type for all the addresses.");
+  EXPECT_EQ(
+      manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml), "", true).status().message(),
+      "listener foo: has different socket type. The listener only "
+      "support same socket type for all the addresses.");
 }
 
 TEST_P(ListenerManagerImplTest, RejectNoAddresses) {
@@ -794,9 +794,9 @@ TEST_P(ListenerManagerImplTest, RejectNoAddresses) {
       exact_balance: {}
   )EOF";
 
-  EXPECT_THROW_WITH_MESSAGE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml), "", true),
-                            EnvoyException,
-                            "error adding listener named 'foo': address is necessary");
+  EXPECT_EQ(
+      manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml), "", true).status().message(),
+      "error adding listener named 'foo': address is necessary");
 }
 
 TEST_P(ListenerManagerImplTest, RejectMutlipleInternalAddresses) {
@@ -816,10 +816,10 @@ TEST_P(ListenerManagerImplTest, RejectMutlipleInternalAddresses) {
     - filters: []
   )EOF";
 
-  EXPECT_THROW_WITH_MESSAGE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml), "", true),
-                            EnvoyException,
-                            "error adding listener named 'foo': use internal_listener field "
-                            "instead of address for internal listeners");
+  EXPECT_EQ(
+      manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml), "", true).status().message(),
+      "error adding listener named 'foo': use internal_listener field "
+      "instead of address for internal listeners");
 
   const std::string yaml2 = R"EOF(
     name: "foo"
@@ -839,10 +839,10 @@ TEST_P(ListenerManagerImplTest, RejectMutlipleInternalAddresses) {
     - filters: []
   )EOF";
 
-  EXPECT_THROW_WITH_MESSAGE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml2), "", true),
-                            EnvoyException,
-                            "error adding listener named 'foo': use internal_listener field "
-                            "instead of address for internal listeners");
+  EXPECT_EQ(
+      manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml2), "", true).status().message(),
+      "error adding listener named 'foo': use internal_listener field "
+      "instead of address for internal listeners");
 
   const std::string yaml3 = R"EOF(
     name: "foo"
@@ -859,10 +859,10 @@ TEST_P(ListenerManagerImplTest, RejectMutlipleInternalAddresses) {
     - filters: []
   )EOF";
 
-  EXPECT_THROW_WITH_MESSAGE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml3), "", true),
-                            EnvoyException,
-                            "error adding listener named 'foo': use internal_listener field "
-                            "instead of address for internal listeners");
+  EXPECT_EQ(
+      manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml3), "", true).status().message(),
+      "error adding listener named 'foo': use internal_listener field "
+      "instead of address for internal listeners");
 }
 
 TEST_P(ListenerManagerImplTest, RejectIpv4CompatOnIpv4Address) {
diff --git a/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h b/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h
index df3cbccc49b8..68e3c967ecde 100644
--- a/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h
+++ b/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h
@@ -348,7 +348,11 @@ class ListenerManagerImplTest : public testing::TestWithParam {
       )EOF";
       TestUtility::loadFromYaml(filter_chain_matcher, *listener.mutable_filter_chain_matcher());
     }
-    return manager_->addOrUpdateListener(listener, version_info, added_via_api);
+    auto status_or_error = manager_->addOrUpdateListener(listener, version_info, added_via_api);
+    if (status_or_error.status().ok()) {
+      return status_or_error.value();
+    }
+    throw EnvoyException(std::string(status_or_error.status().message()));
   }
 
   void testListenerUpdateWithSocketOptionsChange(const std::string& origin,
diff --git a/test/mocks/server/listener_manager.cc b/test/mocks/server/listener_manager.cc
index 0448ff4e7122..8c0069fcc611 100644
--- a/test/mocks/server/listener_manager.cc
+++ b/test/mocks/server/listener_manager.cc
@@ -5,10 +5,15 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+using testing::_;
+using testing::Return;
+
 namespace Envoy {
 namespace Server {
 
-MockListenerManager::MockListenerManager() = default;
+MockListenerManager::MockListenerManager() {
+  ON_CALL(*this, addOrUpdateListener(_, _, _)).WillByDefault(Return(false));
+}
 
 MockListenerManager::~MockListenerManager() = default;
 
diff --git a/test/mocks/server/listener_manager.h b/test/mocks/server/listener_manager.h
index c7c855508f6d..9fc49154ba7c 100644
--- a/test/mocks/server/listener_manager.h
+++ b/test/mocks/server/listener_manager.h
@@ -11,7 +11,7 @@ class MockListenerManager : public ListenerManager {
   MockListenerManager();
   ~MockListenerManager() override;
 
-  MOCK_METHOD(bool, addOrUpdateListener,
+  MOCK_METHOD(absl::StatusOr, addOrUpdateListener,
               (const envoy::config::listener::v3::Listener& config, const std::string& version_info,
                bool modifiable));
   MOCK_METHOD(void, createLdsApi,

From d7c6d6cce6db5086fced568a5bfd792fc9e7bde8 Mon Sep 17 00:00:00 2001
From: Ali Beyad 
Date: Tue, 13 Jun 2023 12:00:48 -0400
Subject: [PATCH 529/740] tests: Add a Traffic Director integration test
 (#27813)

This test initializes an Envoy Mobile engine that retrieves xDS
resources from a public Traffic Director instance.  The purpose of this
test is to have a regular running test on CI that validates Envoy Mobile
communicating with Traffic Director over the public internet.

Signed-off-by: Ali Beyad 
Co-authored-by: phlax 
---
 .github/workflows/mobile-traffic_director.yml |  50 ++
 mobile/test/non_hermetic/BUILD                |  49 ++
 .../gcp_traffic_director_integration_test.cc  | 172 ++++
 .../integration/certs/google_root_certs.pem   | 805 ++++++++++++++++++
 4 files changed, 1076 insertions(+)
 create mode 100644 .github/workflows/mobile-traffic_director.yml
 create mode 100644 mobile/test/non_hermetic/BUILD
 create mode 100644 mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc
 create mode 100644 test/config/integration/certs/google_root_certs.pem

diff --git a/.github/workflows/mobile-traffic_director.yml b/.github/workflows/mobile-traffic_director.yml
new file mode 100644
index 000000000000..51cb8bc96dc4
--- /dev/null
+++ b/.github/workflows/mobile-traffic_director.yml
@@ -0,0 +1,50 @@
+name: mobile_traffic_director
+
+on:
+  schedule:
+  # Once a day at midnight.
+  - cron: '0 0 * * *'
+  # Allows manual triggering in the UI. Makes it easier to test.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}-github.workflow
+  cancel-in-progress: true
+
+jobs:
+  cc_test:
+    if: |
+      ${{
+          github.repository == 'envoyproxy/envoy'
+          && (github.event.schedule
+              || !contains(github.actor, '[bot]'))
+      }}
+    name: cc_test
+    permissions:
+      packages: read
+    runs-on: ubuntu-20.04
+    timeout-minutes: 120
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 0
+    - name: Add safe directory
+      run: git config --global --add safe.directory /__w/envoy/envoy
+    - name: 'Run GcpTrafficDirectorIntegrationTest'
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GCP_JWT_PRIVATE_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_JWT_TOKEN }}
+        ENVOY_IP_TEST_VERSIONS: v4only
+      run: |
+        cd mobile
+        ./bazelw run
+            --config=remote-ci-linux \
+            --config=ci \
+            --test_output=all \
+            //test/non_hermetic:gcp_traffic_director_integration_test
diff --git a/mobile/test/non_hermetic/BUILD b/mobile/test/non_hermetic/BUILD
new file mode 100644
index 000000000000..1d1d8ad9942d
--- /dev/null
+++ b/mobile/test/non_hermetic/BUILD
@@ -0,0 +1,49 @@
+load(
+    "@envoy//bazel:envoy_build_system.bzl",
+    "envoy_cc_test_binary",
+    "envoy_package",
+    "envoy_select_google_grpc",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+# This is a envoy_cc_test_binary instead of an envoy_cc_test because we don't want it to be run
+# when `bazel test //test/...` is called.
+envoy_cc_test_binary(
+    name = "gcp_traffic_director_integration_test",
+    # The test relies on the Google gRPC library.
+    srcs = envoy_select_google_grpc(
+        ["gcp_traffic_director_integration_test.cc"],
+        "@envoy",
+    ),
+    data = [
+        "@envoy//test/config/integration/certs",
+    ],
+    external_deps = [
+        "abseil_strings",
+    ],
+    repository = "@envoy",
+    deps = [
+        "//library/common:envoy_main_interface_lib_no_stamp",
+        "//library/common/data:utility_lib",
+        "//library/common/types:c_types_lib",
+        "//test/common/integration:base_client_integration_test_lib",
+        "@envoy//source/common/config:api_version_lib",
+        "@envoy//source/common/grpc:google_grpc_creds_lib",
+        "@envoy//source/common/protobuf:utility_lib_header",
+        "@envoy//source/extensions/clusters/eds:eds_lib",
+        "@envoy//source/extensions/config_subscription/grpc:grpc_mux_lib",
+        "@envoy//source/extensions/config_subscription/grpc:grpc_subscription_lib",
+        "@envoy//source/extensions/config_subscription/grpc:new_grpc_mux_lib",
+        "@envoy//test/common/grpc:grpc_client_integration_lib",
+        "@envoy//test/integration:http_integration_lib",
+        "@envoy//test/test_common:environment_lib",
+        "@envoy//test/test_common:network_utility_lib",
+        "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
+        "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+        "@envoy_build_config//:test_extensions",
+    ],
+)
diff --git a/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc b/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc
new file mode 100644
index 000000000000..0c21ae35e020
--- /dev/null
+++ b/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc
@@ -0,0 +1,172 @@
+// This test is not meant to be run on the command line, because it depends on a GCP
+// authentication token provided as a GitHub encrypted secret through a GitHub actions workflow.
+
+#include 
+#include 
+#include 
+
+#include "envoy/config/bootstrap/v3/bootstrap.pb.h"
+#include "envoy/config/cluster/v3/cluster.pb.h"
+#include "envoy/config/core/v3/base.pb.h"
+#include "envoy/config/core/v3/config_source.pb.h"
+
+#include "source/common/grpc/google_grpc_creds_impl.h"
+#include "source/common/protobuf/utility.h"
+#include "source/extensions/clusters/eds/eds.h"
+#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h"
+#include "source/extensions/config_subscription/grpc/grpc_subscription_factory.h"
+#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h"
+
+#include "test/common/grpc/grpc_client_integration.h"
+#include "test/common/integration/base_client_integration_test.h"
+#include "test/test_common/environment.h"
+
+#include "absl/strings/substitute.h"
+#include "absl/synchronization/notification.h"
+#include "gtest/gtest.h"
+#include "library/common/data/utility.h"
+#include "library/common/engine_handle.h"
+#include "library/common/types/c_types.h"
+#include "tools/cpp/runfiles/runfiles.h"
+
+namespace Envoy {
+namespace {
+
+using ::Envoy::Grpc::SotwOrDelta;
+using ::Envoy::Network::Address::IpVersion;
+
+// The One-Platform API endpoint for Traffic Director.
+constexpr char TD_API_ENDPOINT[] = "trafficdirector.googleapis.com";
+// The name of the project in Google Cloud Console; copied from the project_id
+// field in the generated JWT token.
+constexpr char PROJECT_NAME[] = "td-testing-gfq";
+// The project number of the project, found on the main page of the project in
+// Google Cloud Console.
+constexpr char PROJECT_ID[] = "798832730858";
+// Copied from the "client_id" field in the generated JWT token.
+constexpr char CLIENT_ID[] = "102524055118681734203";
+// Copied from the "private_key_id" field in the generated JWT token.
+constexpr char PRIVATE_KEY_ID[] = "e07f02d49044a533cf4342d138eacecc6acdb6ed";
+
+// Using a JWT token to authenticate to Traffic Director.
+std::string jwtToken() {
+  const std::string email =
+      absl::Substitute("$0-compute@developer.gserviceaccount.com", PROJECT_ID);
+  const std::string cert_url = absl::Substitute("https://www.googleapis.com/robot/v1/metadata/x509/"
+                                                "$0-compute%40developer.gserviceaccount.com",
+                                                PROJECT_ID);
+
+  const char* private_key = std::getenv("GCP_JWT_PRIVATE_KEY");
+  RELEASE_ASSERT(private_key != nullptr, "GCP_JWT_PRIVATE_KEY environment variable not set.");
+
+  return absl::Substitute(
+      R"json({
+        "private_key": "$0",
+        "private_key_id": "$1",
+        "project_id": "$2",
+        "client_email": "$3",
+        "client_id": "$4",
+        "client_x509_cert_url": "$5",
+        "type": "service_account",
+        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+        "token_uri": "https://oauth2.googleapis.com/token",
+        "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs"
+      })json",
+      private_key, PRIVATE_KEY_ID, PROJECT_NAME, email, CLIENT_ID, cert_url);
+}
+
+// Tests that Envoy Mobile can connect to Traffic Director (an xDS management server offered by GCP)
+// via a test GCP project, and can pull down xDS config for the given project.
+class GcpTrafficDirectorIntegrationTest
+    : public BaseClientIntegrationTest,
+      public testing::TestWithParam> {
+public:
+  GcpTrafficDirectorIntegrationTest() : BaseClientIntegrationTest(ip_version()) {
+    // TODO(https://github.com/envoyproxy/envoy/issues/27848): remove these force registrations
+    // once the EngineBuilder APIs support conditional force registration.
+
+    // Force register the Google gRPC library.
+    Grpc::forceRegisterDefaultGoogleGrpcCredentialsFactory();
+    // Force register the gRPC mux implementations.
+    Config::forceRegisterGrpcMuxFactory();
+    Config::forceRegisterNewGrpcMuxFactory();
+    Config::forceRegisterAdsConfigSubscriptionFactory();
+    // Force register the cluster factories used by the test.
+    Upstream::forceRegisterEdsClusterFactory();
+
+    std::string root_certs(TestEnvironment::readFileToStringForTest(
+        TestEnvironment::runfilesPath("test/config/integration/certs/google_root_certs.pem")));
+
+    // TODO(abeyad): switch to using API key authentication instead of a JWT token.
+    builder_.addLogLevel(Platform::LogLevel::trace)
+        .setNodeId(absl::Substitute("projects/$0/networks/default/nodes/111222333444", PROJECT_ID))
+        .addCdsLayer()
+        .setAggregatedDiscoveryService(std::string(TD_API_ENDPOINT), /*port=*/443, jwtToken(),
+                                       Platform::DefaultJwtTokenLifetimeSeconds,
+                                       std::move(root_certs));
+
+    // Other test knobs.
+    skip_tag_extraction_rule_check_ = true;
+    // Envoy Mobile does not use LDS.
+    use_lds_ = false;
+    // We don't need a fake xDS upstream since we are using Traffic Director.
+    create_xds_upstream_ = false;
+    sotw_or_delta_ = api_type();
+
+    if (api_type() == SotwOrDelta::UnifiedSotw || api_type() == SotwOrDelta::UnifiedDelta) {
+      config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true");
+    }
+  }
+
+  void TearDown() override { BaseClientIntegrationTest::TearDown(); }
+
+  IpVersion ip_version() const { return std::get<0>(GetParam()); }
+  SotwOrDelta api_type() const { return std::get<1>(GetParam()); }
+};
+
+INSTANTIATE_TEST_SUITE_P(
+    GrpcOptions, GcpTrafficDirectorIntegrationTest,
+    testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
+                     testing::Values(SotwOrDelta::Sotw, SotwOrDelta::UnifiedSotw)));
+
+TEST_P(GcpTrafficDirectorIntegrationTest, AdsDynamicClusters) {
+  // Starts up Envoy and loads the bootstrap config, which will trigger fetching
+  // of the dynamic cluster resources from Traffic Director.
+  initialize();
+
+  // Wait for the xDS cluster resources to be retrieved and loaded.
+  //
+  // There are 5 total active clusters after the Envoy engine has finished initialization.
+  //
+  // (A) There are three dynamic clusters retrieved from Traffic Director:
+  //      1. cloud-internal-istio:cloud_mp_798832730858_1578897841695688881
+  //      2. cloud-internal-istio:cloud_mp_798832730858_523871542841416155
+  //      3. cloud-internal-istio:cloud_mp_798832730858_4497773746904456309
+  // (B) There are two static clusters added by the EngineBuilder by default:
+  //      4. base
+  //      5. base_clear
+  ASSERT_TRUE(waitForGaugeGe("cluster_manager.active_clusters", 5));
+
+  // TODO(abeyad): Once we have a Envoy Mobile stats/admin API, we can use it to check the
+  // actual cluster names.
+}
+
+} // namespace
+} // namespace Envoy
+
+int main(int argc, char** argv) {
+  Envoy::TestEnvironment::initializeOptions(argc, argv);
+  std::string error;
+  std::unique_ptr runfiles(
+      bazel::tools::cpp::runfiles::Runfiles::Create(argv[0], &error));
+  RELEASE_ASSERT(runfiles != nullptr, error);
+  Envoy::TestEnvironment::setRunfiles(runfiles.get());
+
+  Envoy::Thread::MutexBasicLockable lock;
+  Envoy::Logger::Context logging_context(spdlog::level::level_enum::trace,
+                                         Envoy::Logger::Logger::DEFAULT_LOG_FORMAT, lock, false);
+  Envoy::Event::Libevent::Global::initialize();
+
+  testing::InitGoogleTest();
+  return RUN_ALL_TESTS();
+}
diff --git a/test/config/integration/certs/google_root_certs.pem b/test/config/integration/certs/google_root_certs.pem
new file mode 100644
index 000000000000..a13000650e18
--- /dev/null
+++ b/test/config/integration/certs/google_root_certs.pem
@@ -0,0 +1,805 @@
+-----BEGIN CERTIFICATE-----
+MIIDdzCCAl+gAwIBAgIEAgAAuTANBgkqhkiG9w0BAQUFADBaMQswCQYDVQQGEwJJ
+RTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJlclRydXN0MSIwIAYD
+VQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTAwMDUxMjE4NDYwMFoX
+DTI1MDUxMjIzNTkwMFowWjELMAkGA1UEBhMCSUUxEjAQBgNVBAoTCUJhbHRpbW9y
+ZTETMBEGA1UECxMKQ3liZXJUcnVzdDEiMCAGA1UEAxMZQmFsdGltb3JlIEN5YmVy
+VHJ1c3QgUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKMEuyKr
+mD1X6CZymrV51Cni4eiVgLGw41uOKymaZN+hXe2wCQVt2yguzmKiYv60iNoS6zjr
+IZ3AQSsBUnuId9Mcj8e6uYi1agnnc+gRQKfRzMpijS3ljwumUNKoUMMo6vWrJYeK
+mpYcqWe4PwzV9/lSEy/CG9VwcPCPwBLKBsua4dnKM3p31vjsufFoREJIE9LAwqSu
+XmD+tqYF/LTdB1kC1FkYmGP1pWPgkAx9XbIGevOF6uvUA65ehD5f/xXtabz5OTZy
+dc93Uk3zyZAsuT3lySNTPx8kmCFcB5kpvcY67Oduhjprl3RjM71oGDHweI12v/ye
+jl0qhqdNkNwnGjkCAwEAAaNFMEMwHQYDVR0OBBYEFOWdWTCCR1jMrPoIVDaGezq1
+BE3wMBIGA1UdEwEB/wQIMAYBAf8CAQMwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3
+DQEBBQUAA4IBAQCFDF2O5G9RaEIFoN27TyclhAO992T9Ldcw46QQF+vaKSm2eT92
+9hkTI7gQCvlYpNRhcL0EYWoSihfVCr3FvDB81ukMJY2GQE/szKN+OMY3EU/t3Wgx
+jkzSswF07r51XgdIGn9w/xZchMB5hbgF/X++ZRGjD8ACtPhSNzkE1akxehi/oCr0
+Epn3o0WC4zxe9Z2etciefC7IpJ5OCBRLbf1wbWsaY71k5h+3zvDyny67G7fyUIhz
+ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS
+R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDtzCCAp+gAwIBAgIQDOfg5RfYRv6P5WD8G/AwOTANBgkqhkiG9w0BAQUFADBl
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv
+b3QgQ0EwHhcNMDYxMTEwMDAwMDAwWhcNMzExMTEwMDAwMDAwWjBlMQswCQYDVQQG
+EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl
+cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgQ0EwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtDhXO5EOAXLGH87dg+XESpa7c
+JpSIqvTO9SA5KFhgDPiA2qkVlTJhPLWxKISKityfCgyDF3qPkKyK53lTXDGEKvYP
+mDI2dsze3Tyoou9q+yHyUmHfnyDXH+Kx2f4YZNISW1/5WBg1vEfNoTb5a3/UsDg+
+wRvDjDPZ2C8Y/igPs6eD1sNuRMBhNZYW/lmci3Zt1/GiSw0r/wty2p5g0I6QNcZ4
+VYcgoc/lbQrISXwxmDNsIumH0DJaoroTghHtORedmTpyoeb6pNnVFzF1roV9Iq4/
+AUaG9ih5yLHa5FcXxH4cDrC0kqZWs72yl+2qp/C3xag/lRbQ/6GW6whfGHdPAgMB
+AAGjYzBhMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW
+BBRF66Kv9JLLgjEtUYunpyGd823IDzAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYun
+pyGd823IDzANBgkqhkiG9w0BAQUFAAOCAQEAog683+Lt8ONyc3pklL/3cmbYMuRC
+dWKuh+vy1dneVrOfzM4UKLkNl2BcEkxY5NM9g0lFWJc1aRqoR+pWxnmrEthngYTf
+fwk8lOa4JiwgvT2zKIn3X/8i4peEH+ll74fg38FnSbNd67IJKusm7Xi+fT8r87cm
+NW1fiQG2SVufAQWbqz0lwcy2f8Lxb4bG+mRo64EtlOtCt/qMHt1i8b5QZ7dsvfPx
+H2sMNgcWfzd8qVttevESRmCD1ycEvkvOl77DZypoEd+A5wwzZr8TDRRu838fYxAe
++o0bJW1sj6W3YQGx0qMmoRBxna3iw/nDmVG3KwcIzi7mULKn+gpFL6Lw8g==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDljCCAn6gAwIBAgIQC5McOtY5Z+pnI7/Dr5r0SzANBgkqhkiG9w0BAQsFADBl
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv
+b3QgRzIwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQG
+EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl
+cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzIwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZ5ygvUj82ckmIkzTz+GoeMVSA
+n61UQbVH35ao1K+ALbkKz3X9iaV9JPrjIgwrvJUXCzO/GU1BBpAAvQxNEP4Htecc
+biJVMWWXvdMX0h5i89vqbFCMP4QMls+3ywPgym2hFEwbid3tALBSfK+RbLE4E9Hp
+EgjAALAcKxHad3A2m67OeYfcgnDmCXRwVWmvo2ifv922ebPynXApVfSr/5Vh88lA
+bx3RvpO704gqu52/clpWcTs/1PPRCv4o76Pu2ZmvA9OPYLfykqGxvYmJHzDNw6Yu
+YjOuFgJ3RFrngQo8p0Quebg/BLxcoIfhG69Rjs3sLPr4/m3wOnyqi+RnlTGNAgMB
+AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQW
+BBTOw0q5mVXyuNtgv6l+vVa1lzan1jANBgkqhkiG9w0BAQsFAAOCAQEAyqVVjOPI
+QW5pJ6d1Ee88hjZv0p3GeDgdaZaikmkuOGybfQTUiaWxMTeKySHMq2zNixya1r9I
+0jJmwYrA8y8678Dj1JGG0VDjA9tzd29KOVPt3ibHtX2vK0LRdWLjSisCx1BL4Gni
+lmwORGYQRI+tBev4eaymG+g3NJ1TyWGqolKvSnAWhsI6yLETcDbYz+70CjTVW0z9
+B5yiutkBclzzTcHdDrEcDcRjvq30FPuJ7KJBDkzMyFdA0G4Dqs0MjomZmWzwPDCv
+ON9vvKO+KSAnq3T/EyJ43pdSVR6DtVQgA+6uwE9W3jfMw3+qBCe703e4YtsXfJwo
+IhNzbM8m9Yop5w==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw
+CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu
+ZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3Qg
+RzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQGEwJV
+UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu
+Y29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQBgcq
+hkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJf
+Zn4f5dwbRXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17Q
+RSAPWXYQ1qAk8C3eNvJsKTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/
+BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgFUaFNN6KDec6NHSrkhDAKBggqhkjOPQQD
+AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY
+JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv
+6pZjamVFkpUBtA==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD
+QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
+b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB
+CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97
+nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt
+43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P
+T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4
+gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO
+BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR
+TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw
+DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr
+hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg
+06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF
+PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls
+YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk
+CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
+b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI
+2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx
+1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ
+q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz
+tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ
+vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP
+BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV
+5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY
+1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4
+NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG
+Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91
+8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe
+pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl
+MrY=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw
+CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu
+ZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAe
+Fw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVTMRUw
+EwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20x
+IDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0CAQYF
+K4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FG
+fp4tn+6OYwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPO
+Z9wj/wMco+I+o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAd
+BgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNpYim8S8YwCgYIKoZIzj0EAwMDaAAwZQIx
+AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/
+oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8
+sycX
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j
+ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL
+MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3
+LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug
+RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm
++9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW
+PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM
+xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB
+Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3
+hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg
+EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF
+MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA
+FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec
+nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z
+eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF
+hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2
+Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe
+vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep
++OkuE6N36B9K
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg
+RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV
+UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu
+Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG
+SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y
+ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If
+xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV
+ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO
+DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ
+jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/
+CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi
+EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM
+fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY
+uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK
+chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t
+9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB
+hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD
+ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2
+SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd
++SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc
+fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa
+sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N
+cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N
+0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie
+4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI
+r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1
+/YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm
+gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEkTCCA3mgAwIBAgIERWtQVDANBgkqhkiG9w0BAQUFADCBsDELMAkGA1UEBhMC
+VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xOTA3BgNVBAsTMHd3dy5lbnRydXN0
+Lm5ldC9DUFMgaXMgaW5jb3Jwb3JhdGVkIGJ5IHJlZmVyZW5jZTEfMB0GA1UECxMW
+KGMpIDIwMDYgRW50cnVzdCwgSW5jLjEtMCsGA1UEAxMkRW50cnVzdCBSb290IENl
+cnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA2MTEyNzIwMjM0MloXDTI2MTEyNzIw
+NTM0MlowgbAxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMTkw
+NwYDVQQLEzB3d3cuZW50cnVzdC5uZXQvQ1BTIGlzIGluY29ycG9yYXRlZCBieSBy
+ZWZlcmVuY2UxHzAdBgNVBAsTFihjKSAyMDA2IEVudHJ1c3QsIEluYy4xLTArBgNV
+BAMTJEVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASIwDQYJ
+KoZIhvcNAQEBBQADggEPADCCAQoCggEBALaVtkNC+sZtKm9I35RMOVcF7sN5EUFo
+Nu3s/poBj6E4KPz3EEZmLk0eGrEaTsbRwJWIsMn/MYszA9u3g3s+IIRe7bJWKKf4
+4LlAcTfFy0cOlypowCKVYhXbR9n10Cv/gkvJrT7eTNuQgFA/CYqEAOwwCj0Yzfv9
+KlmaI5UXLEWeH25DeW0MXJj+SKfFI0dcXv1u5x609mhF0YaDW6KKjbHjKYD+JXGI
+rb68j6xSlkuqUY3kEzEZ6E5Nn9uss2rVvDlUccp6en+Q3X0dgNmBu1kmwhH+5pPi
+94DkZfs0Nw4pgHBNrziGLp5/V6+eF67rHMsoIV+2HNjnogQi+dPa2MsCAwEAAaOB
+sDCBrTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zArBgNVHRAEJDAi
+gA8yMDA2MTEyNzIwMjM0MlqBDzIwMjYxMTI3MjA1MzQyWjAfBgNVHSMEGDAWgBRo
+kORnpKZTgMeGZqTx90tD+4S9bTAdBgNVHQ4EFgQUaJDkZ6SmU4DHhmak8fdLQ/uE
+vW0wHQYJKoZIhvZ9B0EABBAwDhsIVjcuMTo0LjADAgSQMA0GCSqGSIb3DQEBBQUA
+A4IBAQCT1DCw1wMgKtD5Y+iRDAUgqV8ZyntyTtSx29CW+1RaGSwMCPeyvIWonX9t
+O1KzKtvn1ISMY/YPyyYBkVBs9F8U4pN0wBOeMDpQ47RgxRzwIkSNcUesyBrJ6Zua
+AGAT/3B+XxFNSRuzFVJ7yVTav52Vr2ua2J7p8eRDjeIRRDq/r72DQnNSi6q7pynP
+9WQcCk3RvKqsnyrQ/39/2n3qse0wJcGE2jTSW3iDVuycNsMm4hH2Z0kdkquM++v/
+eu6FSqdQgPCnXEqULl8FmTxSQeDNtGPPAUO6nIPcj2A781q0tHuu2guQOHXvgR1m
+0vdXcDazv/wor3ElhVsT/h5/WrQ8
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIC+TCCAoCgAwIBAgINAKaLeSkAAAAAUNCR+TAKBggqhkjOPQQDAzCBvzELMAkG
+A1UEBhMCVVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3
+d3cuZW50cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDEyIEVu
+dHJ1c3QsIEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEzMDEGA1UEAxMq
+RW50cnVzdCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRUMxMB4XDTEy
+MTIxODE1MjUzNloXDTM3MTIxODE1NTUzNlowgb8xCzAJBgNVBAYTAlVTMRYwFAYD
+VQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1c3QubmV0
+L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxMiBFbnRydXN0LCBJbmMuIC0g
+Zm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMzAxBgNVBAMTKkVudHJ1c3QgUm9vdCBD
+ZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEVDMTB2MBAGByqGSM49AgEGBSuBBAAi
+A2IABIQTydC6bUF74mzQ61VfZgIaJPRbiWlH47jCffHyAsWfoPZb1YsGGYZPUxBt
+ByQnoaD41UcZYUx9ypMn6nQM72+WCf5j7HBdNq1nd67JnXxVRDqiY1Ef9eNi1KlH
+Bz7MIKNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0O
+BBYEFLdj5xrdjekIplWDpOBqUEFlEUJJMAoGCCqGSM49BAMDA2cAMGQCMGF52OVC
+R98crlOZF7ZvHH3hvxGU0QOIdeSNiaSKd0bebWHvAvX7td/M/k7//qnmpwIwW5nX
+hTcGtXsI/esni0qU+eH6p44mCOh8kmhtc9hvJqwhAriZtyZBWyVgrtBIGu4G
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMC
+VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50
+cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3Qs
+IEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVz
+dCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwHhcNMDkwNzA3MTcy
+NTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUVu
+dHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwt
+dGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0
+aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmlj
+YXRpb24gQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP/vaCeb9zYQYKpSfYs1/T
+RU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXzHHfV1IWN
+cCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hW
+wcKUs/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1
+U1+cPvQXLOZprE4yTGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0
+jaWvYkxN4FisZDQSA/i2jZRjJKRxAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
+BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ60B7vfec7aVHUbI2fkBJmqzAN
+BgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5ZiXMRrEPR9RP/
+jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ
+Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v
+1fN2D807iDginWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4R
+nAuknZoh8/CbCzB428Hch0P+vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmH
+VHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xOe4pIb4tF9g==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEKjCCAxKgAwIBAgIEOGPe+DANBgkqhkiG9w0BAQUFADCBtDEUMBIGA1UEChML
+RW50cnVzdC5uZXQxQDA+BgNVBAsUN3d3dy5lbnRydXN0Lm5ldC9DUFNfMjA0OCBp
+bmNvcnAuIGJ5IHJlZi4gKGxpbWl0cyBsaWFiLikxJTAjBgNVBAsTHChjKSAxOTk5
+IEVudHJ1c3QubmV0IExpbWl0ZWQxMzAxBgNVBAMTKkVudHJ1c3QubmV0IENlcnRp
+ZmljYXRpb24gQXV0aG9yaXR5ICgyMDQ4KTAeFw05OTEyMjQxNzUwNTFaFw0yOTA3
+MjQxNDE1MTJaMIG0MRQwEgYDVQQKEwtFbnRydXN0Lm5ldDFAMD4GA1UECxQ3d3d3
+LmVudHJ1c3QubmV0L0NQU18yMDQ4IGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxp
+YWIuKTElMCMGA1UECxMcKGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDEzMDEG
+A1UEAxMqRW50cnVzdC5uZXQgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgKDIwNDgp
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArU1LqRKGsuqjIAcVFmQq
+K0vRvwtKTY7tgHalZ7d4QMBzQshowNtTK91euHaYNZOLGp18EzoOH1u3Hs/lJBQe
+sYGpjX24zGtLA/ECDNyrpUAkAH90lKGdCCmziAv1h3edVc3kw37XamSrhRSGlVuX
+MlBvPci6Zgzj/L24ScF2iUkZ/cCovYmjZy/Gn7xxGWC4LeksyZB2ZnuU4q941mVT
+XTzWnLLPKQP5L6RQstRIzgUyVYr9smRMDuSYB3Xbf9+5CFVghTAp+XtIpGmG4zU/
+HoZdenoVve8AjhUiVBcAkCaTvA5JaJG/+EfTnZVCwQ5N328mz8MYIWJmQ3DW1cAH
+4QIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV
+HQ4EFgQUVeSB0RGAvtiJuQijMfmhJAkWuXAwDQYJKoZIhvcNAQEFBQADggEBADub
+j1abMOdTmXx6eadNl9cZlZD7Bh/KM3xGY4+WZiT6QBshJ8rmcnPyT/4xmf3IDExo
+U8aAghOY+rat2l098c5u9hURlIIM7j+VrxGrD9cv3h8Dj1csHsm7mhpElesYT6Yf
+zX1XEC+bBAlahLVu2B064dae0Wx5XnkcFMXj0EyTO2U87d89vqbllRrDtRnDvV5b
+u/8j72gZyxKTJ1wDLW8w0B62GqzeWvfRqqgnpv55gcR5mTNXuhKwqeBCbJPKVt7+
+bYQLCIt+jerXmCHG8+c8eS9enNFMFY3h7CI3zJpDC5fcgJCNs2ebb0gIFVbPv/Er
+fF6adulZkMV8gzURZVE=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDTDCCAjSgAwIBAgIId3cGJyapsXwwDQYJKoZIhvcNAQELBQAwRDELMAkGA1UE
+BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz
+dCBDb21tZXJjaWFsMB4XDTEwMDEyOTE0MDYwNloXDTMwMTIzMTE0MDYwNlowRDEL
+MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp
+cm1UcnVzdCBDb21tZXJjaWFsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEA9htPZwcroRX1BiLLHwGy43NFBkRJLLtJJRTWzsO3qyxPxkEylFf6EqdbDuKP
+Hx6GGaeqtS25Xw2Kwq+FNXkyLbscYjfysVtKPcrNcV/pQr6U6Mje+SJIZMblq8Yr
+ba0F8PrVC8+a5fBQpIs7R6UjW3p6+DM/uO+Zl+MgwdYoic+U+7lF7eNAFxHUdPAL
+MeIrJmqbTFeurCA+ukV6BfO9m2kVrn1OIGPENXY6BwLJN/3HR+7o8XYdcxXyl6S1
+yHp52UKqK39c/s4mT6NmgTWvRLpUHhwwMmWd5jyTXlBOeuM61G7MGvv50jeuJCqr
+VwMiKA1JdX+3KNp1v47j3A55MQIDAQABo0IwQDAdBgNVHQ4EFgQUnZPGU4teyq8/
+nx4P5ZmVvCT2lI8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ
+KoZIhvcNAQELBQADggEBAFis9AQOzcAN/wr91LoWXym9e2iZWEnStB03TX8nfUYG
+XUPGhi4+c7ImfU+TqbbEKpqrIZcUsd6M06uJFdhrJNTxFq7YpFzUf1GO7RgBsZNj
+vbz4YYCanrHOQnDiqX0GJX0nof5v7LMeJNrjS1UaADs1tDvZ110w/YETifLCBivt
+Z8SOyUOyXGsViQK8YvxO8rUzqrJv0wqiUOP2O+guRMLbZjipM1ZI8W0bM40NjD9g
+N53Tym1+NH4Nn3J2ixufcv1SNUFFApYvHLKac0khsUlHRUe072o0EclNmsxZt9YC
+nlpOZbWUrhvfKbAW8b8Angc6F2S1BLUjIZkKlTuXfO8=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDTDCCAjSgAwIBAgIIfE8EORzUmS0wDQYJKoZIhvcNAQEFBQAwRDELMAkGA1UE
+BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz
+dCBOZXR3b3JraW5nMB4XDTEwMDEyOTE0MDgyNFoXDTMwMTIzMTE0MDgyNFowRDEL
+MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp
+cm1UcnVzdCBOZXR3b3JraW5nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAtITMMxcua5Rsa2FSoOujz3mUTOWUgJnLVWREZY9nZOIG41w3SfYvm4SEHi3y
+YJ0wTsyEheIszx6e/jarM3c1RNg1lho9Nuh6DtjVR6FqaYvZ/Ls6rnla1fTWcbua
+kCNrmreIdIcMHl+5ni36q1Mr3Lt2PpNMCAiMHqIjHNRqrSK6mQEubWXLviRmVSRL
+QESxG9fhwoXA3hA/Pe24/PHxI1Pcv2WXb9n5QHGNfb2V1M6+oF4nI979ptAmDgAp
+6zxG8D1gvz9Q0twmQVGeFDdCBKNwV6gbh+0t+nvujArjqWaJGctB+d1ENmHP4ndG
+yH329JKBNv3bNPFyfvMMFr20FQIDAQABo0IwQDAdBgNVHQ4EFgQUBx/S55zawm6i
+QLSwelAQUHTEyL0wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ
+KoZIhvcNAQEFBQADggEBAIlXshZ6qML91tmbmzTCnLQyFE2npN/svqe++EPbkTfO
+tDIuUFUaNU52Q3Eg75N3ThVwLofDwR1t3Mu1J9QsVtFSUzpE0nPIxBsFZVpikpzu
+QY0x2+c06lkh1QF612S4ZDnNye2v7UsDSKegmQGA3GWjNq5lWUhPgkvIZfFXHeVZ
+Lgo/bNjR9eUJtGxUAArgFU2HdW23WJZa3W3SAKD0m0i+wzekujbgfIeFlxoVot4u
+olu9rxj5kFDNcFn4J2dHy8egBzp90SxdbBk6ZrV9/ZFvgrG+CJPbFEfxojfHRZ48
+x3evZKiT3/Zpg4Jg8klCNO1aAFSFHBY2kgxc+qatv9s=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFRjCCAy6gAwIBAgIIbYwURrGmCu4wDQYJKoZIhvcNAQEMBQAwQTELMAkGA1UE
+BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1UcnVz
+dCBQcmVtaXVtMB4XDTEwMDEyOTE0MTAzNloXDTQwMTIzMTE0MTAzNlowQTELMAkG
+A1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1U
+cnVzdCBQcmVtaXVtMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxBLf
+qV/+Qd3d9Z+K4/as4Tx4mrzY8H96oDMq3I0gW64tb+eT2TZwamjPjlGjhVtnBKAQ
+JG9dKILBl1fYSCkTtuG+kU3fhQxTGJoeJKJPj/CihQvL9Cl/0qRY7iZNyaqoe5rZ
++jjeRFcV5fiMyNlI4g0WJx0eyIOFJbe6qlVBzAMiSy2RjYvmia9mx+n/K+k8rNrS
+s8PhaJyJ+HoAVt70VZVs+7pk3WKL3wt3MutizCaam7uqYoNMtAZ6MMgpv+0GTZe5
+HMQxK9VfvFMSF5yZVylmd2EhMQcuJUmdGPLu8ytxjLW6OQdJd/zvLpKQBY0tL3d7
+70O/Nbua2Plzpyzy0FfuKE4mX4+QaAkvuPjcBukumj5Rp9EixAqnOEhss/n/fauG
+V+O61oV4d7pD6kh/9ti+I20ev9E2bFhc8e6kGVQa9QPSdubhjL08s9NIS+LI+H+S
+qHZGnEJlPqQewQcDWkYtuJfzt9WyVSHvutxMAJf7FJUnM7/oQ0dG0giZFmA7mn7S
+5u046uwBHjxIVkkJx0w3AJ6IDsBz4W9m6XJHMD4Q5QsDyZpCAGzFlH5hxIrff4Ia
+C1nEWTJ3s7xgaVY5/bQGeyzWZDbZvUjthB9+pSKPKrhC9IK31FOQeE4tGv2Bb0TX
+OwF0lkLgAOIua+rF7nKsu7/+6qqo+Nz2snmKtmcCAwEAAaNCMEAwHQYDVR0OBBYE
+FJ3AZ6YMItkm9UWrpmVSESfYRaxjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/
+BAQDAgEGMA0GCSqGSIb3DQEBDAUAA4ICAQCzV00QYk465KzquByvMiPIs0laUZx2
+KI15qldGF9X1Uva3ROgIRL8YhNILgM3FEv0AVQVhh0HctSSePMTYyPtwni94loMg
+Nt58D2kTiKV1NpgIpsbfrM7jWNa3Pt668+s0QNiigfV4Py/VpfzZotReBA4Xrf5B
+8OWycvpEgjNC6C1Y91aMYj+6QrCcDFx+LmUmXFNPALJ4fqENmS2NuB2OosSw/WDQ
+MKSOyARiqcTtNd56l+0OOF6SL5Nwpamcb6d9Ex1+xghIsV5n61EIJenmJWtSKZGc
+0jlzCFfemQa0W50QBuHCAKi4HEoCChTQwUHK+4w1IX2COPKpVJEZNZOUbWo6xbLQ
+u4mGk+ibyQ86p3q4ofB4Rvr8Ny/lioTz3/4E2aFooC8k4gmVBtWVyuEklut89pMF
+u+1z6S3RdTnX5yTb2E5fQ4+e0BQ5v1VwSJlXMbSc7kqYA5YwH2AG7hsj/oFgIxpH
+YoWlzBk0gG+zrBrjn/B7SK3VAdlntqlyk+otZrWyuOQ9PLLvTIzq6we/qzWaVYa8
+GKa1qF60g2xraUDTn9zxw2lrueFtCfTxqlB2Cnp9ehehVZZCmTEJ3WARjQUwfuaO
+RtGdFNrHF+QFlozEJLUbzxQHskD4o55BhrwE0GuWyCqANP2/7waj3VjFhT0+j/6e
+KeC2uAloGRwYQw==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIB/jCCAYWgAwIBAgIIdJclisc/elQwCgYIKoZIzj0EAwMwRTELMAkGA1UEBhMC
+VVMxFDASBgNVBAoMC0FmZmlybVRydXN0MSAwHgYDVQQDDBdBZmZpcm1UcnVzdCBQ
+cmVtaXVtIEVDQzAeFw0xMDAxMjkxNDIwMjRaFw00MDEyMzExNDIwMjRaMEUxCzAJ
+BgNVBAYTAlVTMRQwEgYDVQQKDAtBZmZpcm1UcnVzdDEgMB4GA1UEAwwXQWZmaXJt
+VHJ1c3QgUHJlbWl1bSBFQ0MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQNMF4bFZ0D
+0KF5Nbc6PJJ6yhUczWLznCZcBz3lVPqj1swS6vQUX+iOGasvLkjmrBhDeKzQN8O9
+ss0s5kfiGuZjuD0uL3jET9v0D6RoTFVya5UdThhClXjMNzyR4ptlKymjQjBAMB0G
+A1UdDgQWBBSaryl6wBE1NSZRMADDav5A1a7WPDAPBgNVHRMBAf8EBTADAQH/MA4G
+A1UdDwEB/wQEAwIBBjAKBggqhkjOPQQDAwNnADBkAjAXCfOHiFBar8jAQr9HX/Vs
+aobgxCd05DhT1wV/GzTjxi+zygk8N53X57hG8f2h4nECMEJZh0PUUd+60wkyWs6I
+flc9nF9Ca/UHLbXwgpP5WW+uZPpY5Yse42O+tYHNbwKMeQ==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG
+A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv
+b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw
+MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i
+YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT
+aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ
+jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp
+xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp
+1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG
+snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ
+U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8
+9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E
+BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B
+AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz
+yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE
+38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP
+AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad
+DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME
+HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G
+A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp
+Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4
+MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG
+A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8
+RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT
+gHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm
+KPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd
+QQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ
+XriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw
+DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o
+LkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU
+RUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp
+jjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK
+6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX
+mcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs
+Mx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH
+WD9f
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICHjCCAaSgAwIBAgIRYFlJ4CYuu1X5CneKcflK2GwwCgYIKoZIzj0EAwMwUDEk
+MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI1MRMwEQYDVQQKEwpH
+bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX
+DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD
+QSAtIFI1MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu
+MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAER0UOlvt9Xb/pOdEh+J8LttV7HpI6SFkc
+8GIxLcB6KP4ap1yztsyX50XUWPrRd21DosCHZTQKH3rd6zwzocWdTaRvQZU4f8ke
+hOvRnkmSh5SHDDqFSmafnVmTTZdhBoZKo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD
+VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUPeYpSJvqB8ohREom3m7e0oPQn1kwCgYI
+KoZIzj0EAwMDaAAwZQIxAOVpEslu28YxuglB4Zf4+/2a4n0Sye18ZNPLBSWLVtmg
+515dTguDnFt2KaAJJiFqYgIwcdK1j1zqO+F4CYWodZI7yFz9SO8NdCKoCOJuxUnO
+xwy8p2Fp8fc74SrL+SvzZpA3
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg
+MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh
+bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx
+MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET
+MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ
+KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI
+xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k
+ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD
+aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw
+LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw
+1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX
+k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2
+SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h
+bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n
+WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY
+rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce
+MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD
+AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu
+bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN
+nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt
+Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61
+55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj
+vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf
+cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz
+oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp
+nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs
+pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v
+JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R
+8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4
+5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx
+EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT
+EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp
+ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAwMFoXDTM3MTIzMTIz
+NTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH
+EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8GA1UE
+AxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIw
+DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKD
+E6bFIEMBO4Tx5oVJnyfq9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH
+/PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD+qK+ihVqf94Lw7YZFAXK6sOoBJQ7Rnwy
+DfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutdfMh8+7ArU6SSYmlRJQVh
+GkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMlNAJWJwGR
+tDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEA
+AaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE
+FDqahQcQZyi27/a9BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmX
+WWcDYfF+OwYxdS2hII5PZYe096acvNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu
+9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r5N9ss4UXnT3ZJE95kTXWXwTr
+gIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYVN8Gb5DKj7Tjo
+2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO
+LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI
+4uJEvlz36hz1
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIID3TCCAsWgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjzELMAkGA1UEBhMCVVMx
+EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT
+HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAMTKVN0YXJmaWVs
+ZCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAw
+MFoXDTM3MTIzMTIzNTk1OVowgY8xCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6
+b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFyZmllbGQgVGVj
+aG5vbG9naWVzLCBJbmMuMTIwMAYDVQQDEylTdGFyZmllbGQgUm9vdCBDZXJ0aWZp
+Y2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBAL3twQP89o/8ArFvW59I2Z154qK3A2FWGMNHttfKPTUuiUP3oWmb3ooa/RMg
+nLRJdzIpVv257IzdIvpy3Cdhl+72WoTsbhm5iSzchFvVdPtrX8WJpRBSiUZV9Lh1
+HOZ/5FSuS/hVclcCGfgXcVnrHigHdMWdSL5stPSksPNkN3mSwOxGXn/hbVNMYq/N
+Hwtjuzqd+/x5AJhhdM8mgkBj87JyahkNmcrUDnXMN/uLicFZ8WJ/X7NfZTD4p7dN
+dloedl40wOiWVpmKs/B/pM293DIxfJHP4F8R+GuqSVzRmZTRouNjWwl2tVZi4Ut0
+HZbUJtQIBFnQmA4O5t78w+wfkPECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO
+BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFHwMMh+n2TB/xH1oo2Kooc6rB1snMA0G
+CSqGSIb3DQEBCwUAA4IBAQARWfolTwNvlJk7mh+ChTnUdgWUXuEok21iXQnCoKjU
+sHU48TRqneSfioYmUeYs0cYtbpUgSpIB7LiKZ3sx4mcujJUDJi5DnUox9g61DLu3
+4jd/IroAow57UvtruzvE03lRTs2Q9GcHGcg8RnoNAX3FWOdt5oUwF5okxBDgBPfg
+8n/Uqgr/Qh037ZTlZFkSIHc40zI+OIF1lnP6aI+xy84fxez6nH7PfrHxBy22/L/K
+pL/QlwVKvOoYKAKQvVR4CSFx09F9HdkWsKlhPdAKACL8x3vLCWRFCztAgfd9fDL1
+mMpYjn0q7pBZc2T5NnReJaH1ZgUufzkVqSr7UIuOhWn0
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEDzCCAvegAwIBAgIBADANBgkqhkiG9w0BAQUFADBoMQswCQYDVQQGEwJVUzEl
+MCMGA1UEChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMp
+U3RhcmZpZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQw
+NjI5MTczOTE2WhcNMzQwNjI5MTczOTE2WjBoMQswCQYDVQQGEwJVUzElMCMGA1UE
+ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMpU3RhcmZp
+ZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggEgMA0GCSqGSIb3
+DQEBAQUAA4IBDQAwggEIAoIBAQC3Msj+6XGmBIWtDBFk385N78gDGIc/oav7PKaf
+8MOh2tTYbitTkPskpD6E8J7oX+zlJ0T1KKY/e97gKvDIr1MvnsoFAZMej2YcOadN
++lq2cwQlZut3f+dZxkqZJRRU6ybH838Z1TBwj6+wRir/resp7defqgSHo9T5iaU0
+X9tDkYI22WY8sbi5gv2cOj4QyDvvBmVmepsZGD3/cVE8MC5fvj13c7JdBmzDI1aa
+K4UmkhynArPkPw2vCHmCuDY96pzTNbO8acr1zJ3o/WSNF4Azbl5KXZnJHoe0nRrA
+1W4TNSNe35tfPe/W93bC6j67eA0cQmdrBNj41tpvi/JEoAGrAgEDo4HFMIHCMB0G
+A1UdDgQWBBS/X7fRzt0fhvRbVazc1xDCDqmI5zCBkgYDVR0jBIGKMIGHgBS/X7fR
+zt0fhvRbVazc1xDCDqmI56FspGowaDELMAkGA1UEBhMCVVMxJTAjBgNVBAoTHFN0
+YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAsTKVN0YXJmaWVsZCBD
+bGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8w
+DQYJKoZIhvcNAQEFBQADggEBAAWdP4id0ckaVaGsafPzWdqbAYcaT1epoXkJKtv3
+L7IezMdeatiDh6GX70k1PncGQVhiv45YuApnP+yz3SFmH8lU+nLMPUxA2IGvd56D
+eruix/U0F47ZEUD0/CwqTRV/p2JdLiXTAAsgGh1o+Re49L2L7ShZ3U0WixeDyLJl
+xy16paq8U4Zt3VekyvggQQto8PT7dL5WXXp59fkdheMtlb71cZBDzI0fmgAKhynp
+VSJYACPq4xJDKVtHCN2MQWplBqjlIapBtJUhlbl90TSrE9atvNziPTnNvT51cKEY
+WQPJIrSPnNVeKtelttQKbfi3QBFGmh95DmK/D5fs4C8fF5Q=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh
+MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE
+YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3
+MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo
+ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg
+MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN
+ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA
+PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w
+wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi
+EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY
+avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+
+YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE
+sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h
+/t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5
+IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj
+YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD
+ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy
+OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P
+TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ
+HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER
+dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf
+ReYNnyicsbkqWletNw+vHX/bvZ8=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb
+MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow
+GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj
+YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL
+MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE
+BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM
+GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua
+BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe
+3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4
+YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR
+rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm
+ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU
+oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF
+MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v
+QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t
+b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF
+AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q
+GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz
+Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2
+G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi
+l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3
+smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEHTCCAwWgAwIBAgIQToEtioJl4AsC7j41AkblPTANBgkqhkiG9w0BAQUFADCB
+gTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G
+A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxJzAlBgNV
+BAMTHkNPTU9ETyBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjEyMDEwMDAw
+MDBaFw0yOTEyMzEyMzU5NTlaMIGBMQswCQYDVQQGEwJHQjEbMBkGA1UECBMSR3Jl
+YXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFDT01P
+RE8gQ0EgTGltaXRlZDEnMCUGA1UEAxMeQ09NT0RPIENlcnRpZmljYXRpb24gQXV0
+aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0ECLi3LjkRv3
+UcEbVASY06m/weaKXTuH+7uIzg3jLz8GlvCiKVCZrts7oVewdFFxze1CkU1B/qnI
+2GqGd0S7WWaXUF601CxwRM/aN5VCaTwwxHGzUvAhTaHYujl8HJ6jJJ3ygxaYqhZ8
+Q5sVW7euNJH+1GImGEaaP+vB+fGQV+useg2L23IwambV4EajcNxo2f8ESIl33rXp
++2dtQem8Ob0y2WIC8bGoPW43nOIv4tOiJovGuFVDiOEjPqXSJDlqR6sA1KGzqSX+
+DT+nHbrTUcELpNqsOO9VUCQFZUaTNE8tja3G1CEZ0o7KBWFxB3NH5YoZEr0ETc5O
+nKVIrLsm9wIDAQABo4GOMIGLMB0GA1UdDgQWBBQLWOWLxkwVN6RAqTCpIb5HNlpW
+/zAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zBJBgNVHR8EQjBAMD6g
+PKA6hjhodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9DT01PRE9DZXJ0aWZpY2F0aW9u
+QXV0aG9yaXR5LmNybDANBgkqhkiG9w0BAQUFAAOCAQEAPpiem/Yb6dc5t3iuHXIY
+SdOH5EOC6z/JqvWote9VfCFSZfnVDeFs9D6Mk3ORLgLETgdxb8CPOGEIqB6BCsAv
+IC9Bi5HcSEW88cbeunZrM8gALTFGTO3nnc+IlP8zwFboJIYmuNg4ON8qa90SzMc/
+RxdMosIGlgnW2/4/PEZB31jiVg88O8EckzXZOFKs7sjsLjBOlDW0JB9LeGna8gI4
+zJVSk/BwJVmcIGfE7vmLV2H0knZ9P4SNVbfo5azV8fUZVqZa+5Acr5Pr5RzUZ5dd
+BA6+C4OmF4O5MBKgxTMVBbkN+8cFduPYSo38NBejxiEovjBFMR7HeL5YYTisO+IB
+ZQ==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL
+MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE
+BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT
+IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw
+MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy
+ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N
+T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv
+biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR
+FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J
+cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW
+BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/
+BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm
+fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv
+GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB
+hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G
+A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV
+BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5
+MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT
+EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR
+Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh
+dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR
+6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X
+pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC
+9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV
+/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf
+Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z
++pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w
+qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah
+SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC
+u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf
+Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq
+crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E
+FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl
+wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM
+4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV
+2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna
+FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ
+CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK
+boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke
+jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL
+S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb
+QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl
+0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB
+NVOFBkpdn627G190
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICjzCCAhWgAwIBAgIQXIuZxVqUxdJxVt7NiYDMJjAKBggqhkjOPQQDAzCBiDEL
+MAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNl
+eSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMT
+JVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMjAx
+MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT
+Ck5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVUaGUg
+VVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBFQ0MgQ2VydGlm
+aWNhdGlvbiBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQarFRaqflo
+I+d61SRvU8Za2EurxtW20eZzca7dnNYMYf3boIkDuAUU7FfO7l0/4iGzzvfUinng
+o4N+LZfQYcTxmdwlkWOrfzCjtHDix6EznPO/LlxTsV+zfTJ/ijTjeXmjQjBAMB0G
+A1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1xmNjmjAOBgNVHQ8BAf8EBAMCAQYwDwYD
+VR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjA2Z6EWCNzklwBBHU6+4WMB
+zzuqQhFkoJ2UOQIReVx7Hfpkue4WQrO/isIJxOzksU0CMQDpKmFHjFJKS04YcPbW
+RNZu9YO6bVi9JNlWSOrvxKJGgYhqOkbRqZtNyWHa0V1Xahg=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB
+iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl
+cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV
+BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw
+MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV
+BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU
+aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy
+dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK
+AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B
+3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY
+tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/
+Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2
+VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT
+79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6
+c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT
+Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l
+c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee
+UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE
+Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd
+BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G
+A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF
+Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO
+VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3
+ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs
+8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR
+iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze
+Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ
+XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/
+qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB
+VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB
+L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG
+jjxDah2nGN59PRbxYvnKkKj9
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFVzCCAz+gAwIBAgINAgPlk28xsBNJiGuiFzANBgkqhkiG9w0BAQwFADBHMQsw
+CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU
+MBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw
+MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp
+Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEBAQUA
+A4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaMf/vo
+27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vXmX7w
+Cl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7zUjw
+TcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0Pfybl
+qAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtcvfaH
+szVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4Zor8
+Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUspzBmk
+MiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOORc92
+wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYWk70p
+aDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+DVrN
+VjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgFlQID
+AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E
+FgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBAJ+qQibb
+C5u+/x6Wki4+omVKapi6Ist9wTrYggoGxval3sBOh2Z5ofmmWJyq+bXmYOfg6LEe
+QkEzCzc9zolwFcq1JKjPa7XSQCGYzyI0zzvFIoTgxQ6KfF2I5DUkzps+GlQebtuy
+h6f88/qBVRRiClmpIgUxPoLW7ttXNLwzldMXG+gnoot7TiYaelpkttGsN/H9oPM4
+7HLwEXWdyzRSjeZ2axfG34arJ45JK3VmgRAhpuo+9K4l/3wV3s6MJT/KYnAK9y8J
+ZgfIPxz88NtFMN9iiMG1D53Dn0reWVlHxYciNuaCp+0KueIHoI17eko8cdLiA6Ef
+MgfdG+RCzgwARWGAtQsgWSl4vflVy2PFPEz0tv/bal8xa5meLMFrUKTX5hgUvYU/
+Z6tGn6D/Qqc6f1zLXbBwHSs09dR2CQzreExZBfMzQsNhFRAbd03OIozUhfJFfbdT
+6u9AWpQKXCBfTkBdYiJ23//OYb2MI3jSNwLgjt7RETeJ9r/tSQdirpLsQBqvFAnZ
+0E6yove+7u7Y/9waLd64NnHi/Hm3lCXRSHNboTXns5lndcEZOitHTtNCjv0xyBZm
+2tIMPNuzjsmhDYAPexZ3FL//2wmUspO8IFgV6dtxQ/PeEMMA3KgqlbbC1j+Qa3bb
+bP6MvPJwNQzcmRk13NfIRmPVNnGuV/u3gm3c
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFVzCCAz+gAwIBAgINAgPlrsWNBCUaqxElqjANBgkqhkiG9w0BAQwFADBHMQsw
+CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU
+MBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw
+MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp
+Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEBAQUA
+A4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3LvCvpt
+nfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3KgGjSY
+6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9BuXvAu
+MC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOdre7k
+RXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXuPuWg
+f9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1mKPV
++3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K8Yzo
+dDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqjx5RW
+Ir9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsRnTKa
+G73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0kzCq
+gc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9OktwID
+AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E
+FgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBAB/Kzt3H
+vqGf2SdMC9wXmBFqiN495nFWcrKeGk6c1SuYJF2ba3uwM4IJvd8lRuqYnrYb/oM8
+0mJhwQTtzuDFycgTE1XnqGOtjHsB/ncw4c5omwX4Eu55MaBBRTUoCnGkJE+M3DyC
+B19m3H0Q/gxhswWV7uGugQ+o+MePTagjAiZrHYNSVc61LwDKgEDg4XSsYPWHgJ2u
+NmSRXbBoGOqKYcl3qJfEycel/FVL8/B/uWU9J2jQzGv6U53hkRrJXRqWbTKH7QMg
+yALOWr7Z6v2yTcQvG99fevX4i8buMTolUVVnjWQye+mew4K6Ki3pHrTgSAai/Gev
+HyICc/sgCq+dVEuhzf9gR7A/Xe8bVr2XIZYtCtFenTgCR2y59PYjJbigapordwj6
+xLEokCZYCDzifqrXPW+6MYgKBesntaFJ7qBFVHvmJ2WZICGoo7z7GJa7Um8M7YNR
+TOlZ4iBgxcJlkoKM8xAfDoqXvneCbT+PHV28SSe9zE8P4c52hgQjxcCMElv924Sg
+JPFI/2R80L5cFtHvma3AH/vLrrw4IgYmZNralw4/KBVEqE8AyvCazM90arQ+POuV
+7LXTWtiBmelDGDfrs7vRWGJB82bSj6p4lVQgw1oudCvV0b4YacCs1aTPObpRhANl
+6WLAYv7YTVWW4tAR+kg0Eeye7QUd5MjWHYbL
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICCTCCAY6gAwIBAgINAgPluILrIPglJ209ZjAKBggqhkjOPQQDAzBHMQswCQYD
+VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG
+A1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw
+WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz
+IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQAIgNi
+AAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout736G
+jOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2ADDL2
+4CejQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW
+BBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEA9uEglRR7
+VKOQFhG/hMjqb2sXnh5GmCCbn9MN2azTL818+FsuVbu/3ZL3pAzcMeGiAjEA/Jdm
+ZuVDFhOD3cffL74UOO0BzrEXGhF16b0DjyZ+hOXJYKaV11RZt+cRLInUue4X
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICCTCCAY6gAwIBAgINAgPlwGjvYxqccpBQUjAKBggqhkjOPQQDAzBHMQswCQYD
+VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG
+A1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw
+WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz
+IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQAIgNi
+AATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzuhXyi
+QHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/lxKvR
+HYqjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW
+BBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNpADBmAjEA6ED/g94D
+9J+uHXqnLrmvT/aDHQ4thQEd0dlq7A/Cr8deVl5c1RxYIigL9zC2L7F8AjEA8GE8
+p/SgguMh1YQdc4acLa/KNJvxn7kjNuK8YAOdgLOaVsjh4rsUecrNIdSUtUlD
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIB3DCCAYOgAwIBAgINAgPlfvU/k/2lCSGypjAKBggqhkjOPQQDAjBQMSQwIgYD
+VQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0gUjQxEzARBgNVBAoTCkdsb2Jh
+bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTIxMTEzMDAwMDAwWhcNMzgw
+MTE5MDMxNDA3WjBQMSQwIgYDVQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0g
+UjQxEzARBgNVBAoTCkdsb2JhbFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wWTAT
+BgcqhkjOPQIBBggqhkjOPQMBBwNCAAS4xnnTj2wlDp8uORkcA6SumuU5BwkWymOx
+uYb4ilfBV85C+nOh92VC/x7BALJucw7/xyHlGKSq2XE/qNS5zowdo0IwQDAOBgNV
+HQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUVLB7rUW44kB/
++wpu+74zyTyjhNUwCgYIKoZIzj0EAwIDRwAwRAIgIk90crlgr/HmnKAWBVBfw147
+bmF0774BxL4YSFlhgjICICadVGNA3jdgUM/I2O2dgq43mLyjj0xMqTQrbO/7lZsm
+-----END CERTIFICATE-----

From 7c571ac7c61768d72929feda7ef9edeb5046b56f Mon Sep 17 00:00:00 2001
From: Ali Beyad 
Date: Tue, 13 Jun 2023 15:53:33 -0400
Subject: [PATCH 530/740] ci: Fix the Envoy Mobile Traffic Director YAML
 (#27945)

Signed-off-by: Ali Beyad 
---
 .github/workflows/mobile-traffic_director.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mobile-traffic_director.yml b/.github/workflows/mobile-traffic_director.yml
index 51cb8bc96dc4..10e6c2b9ed90 100644
--- a/.github/workflows/mobile-traffic_director.yml
+++ b/.github/workflows/mobile-traffic_director.yml
@@ -43,7 +43,7 @@ jobs:
         ENVOY_IP_TEST_VERSIONS: v4only
       run: |
         cd mobile
-        ./bazelw run
+        ./bazelw run \
             --config=remote-ci-linux \
             --config=ci \
             --test_output=all \

From 61af06091f9f913fc50daecf4481a1dc7de84e9a Mon Sep 17 00:00:00 2001
From: ohadvano <49730675+ohadvano@users.noreply.github.com>
Date: Wed, 14 Jun 2023 05:33:09 +0300
Subject: [PATCH 531/740] logging: Add ENVOY_TAGGED_LOG macro to enable tagged
 logging (#27882)

Signed-off-by: ohadvano 
---
 source/common/common/logger.cc        | 19 +++++++++++
 source/common/common/logger.h         | 24 ++++++++++++++
 test/common/common/log_macros_test.cc |  4 +++
 test/common/common/logger_test.cc     | 48 +++++++++++++++++++++++++++
 4 files changed, 95 insertions(+)

diff --git a/source/common/common/logger.cc b/source/common/common/logger.cc
index 476f2e06a9f9..74cee4d084e4 100644
--- a/source/common/common/logger.cc
+++ b/source/common/common/logger.cc
@@ -3,6 +3,7 @@
 #include  // use direct system-assert to avoid cyclic dependency.
 #include 
 #include 
+#include 
 #include 
 #include 
 
@@ -306,6 +307,24 @@ void setLogFormatForLogger(spdlog::logger& logger, const std::string& log_format
   logger.set_formatter(std::move(formatter));
 }
 
+std::string serializeLogTags(const std::map& tags) {
+  if (tags.empty()) {
+    return "";
+  }
+
+  std::stringstream tags_stream;
+  tags_stream << "[Tags: ";
+  for (const auto& tag : tags) {
+    tags_stream << "\"" << tag.first << "\":\"" << tag.second << "\",";
+  }
+
+  std::string serialized = tags_stream.str();
+  serialized.pop_back();
+  serialized += "] ";
+
+  return serialized;
+}
+
 } // namespace Utility
 
 namespace CustomFlagFormatter {
diff --git a/source/common/common/logger.h b/source/common/common/logger.h
index 28aecaf86640..720e9b1bb68d 100644
--- a/source/common/common/logger.h
+++ b/source/common/common/logger.h
@@ -401,6 +401,11 @@ namespace Utility {
  */
 void setLogFormatForLogger(spdlog::logger& logger, const std::string& log_format);
 
+/**
+ * Serializes custom log tags to a string that will be prepended to the log message.
+ */
+std::string serializeLogTags(const std::map& tags);
+
 } // namespace Utility
 
 // Contains custom flags to introduce user defined flags in log pattern. Reference:
@@ -517,6 +522,25 @@ class EscapeMessageJsonString : public spdlog::custom_flag_formatter {
  */
 #define ENVOY_LOG(LEVEL, ...) ENVOY_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, ##__VA_ARGS__)
 
+#define ENVOY_TAGGED_LOG_TO_LOGGER(LOGGER, LEVEL, TAGS, FORMAT, ...)                               \
+  do {                                                                                             \
+    if (ENVOY_LOG_COMP_LEVEL(LOGGER, LEVEL)) {                                                     \
+      ENVOY_LOG_TO_LOGGER(LOGGER, LEVEL,                                                           \
+                          ::Envoy::Logger::Utility::serializeLogTags(TAGS) + FORMAT,               \
+                          ##__VA_ARGS__);                                                          \
+    }                                                                                              \
+  } while (0)
+
+/**
+ * Log with tags which are a map of key and value strings. When ENVOY_TAGGED_LOG is used, the tags
+ * are serialized and prepended to the log message.
+ * For example, the map {{"key1","val1","key2","val2"}} would be serialized to:
+ * [Tags: "key1":"val1","key2":"val2"]. The serialization pattern is defined by
+ * Envoy::Logger::Utility::serializeLogTags function.
+ */
+#define ENVOY_TAGGED_LOG(LEVEL, TAGS, FORMAT, ...)                                                 \
+  ENVOY_TAGGED_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, TAGS, FORMAT, ##__VA_ARGS__)
+
 /**
  * Log with a stable event name. This allows emitting a log line with a stable name in addition to
  * the standard log line. The stable log line is passed to custom sinks that may want to intercept
diff --git a/test/common/common/log_macros_test.cc b/test/common/common/log_macros_test.cc
index 5058e032f35f..aaa9dce4154a 100644
--- a/test/common/common/log_macros_test.cc
+++ b/test/common/common/log_macros_test.cc
@@ -32,11 +32,15 @@ class TestFilterLog : public Logger::Loggable {
     ENVOY_STREAM_LOG(info, "fake message", stream_);
     ENVOY_CONN_LOG(error, "fake error", connection_);
     ENVOY_STREAM_LOG(error, "fake error", stream_);
+    ENVOY_TAGGED_LOG(info, tags_, "fake message {}", "val");
+    ENVOY_TAGGED_LOG(info, (std::map{{"key", "val"}}), "fake message {}",
+                     "val");
   }
 
   void logMessageEscapeSequences() { ENVOY_LOG_MISC(info, "line 1 \n line 2 \t tab \\r test"); }
 
 private:
+  std::map tags_{{"key", "val"}};
   NiceMock connection_;
   NiceMock stream_;
 };
diff --git a/test/common/common/logger_test.cc b/test/common/common/logger_test.cc
index e9998db311e3..d9e039a4d517 100644
--- a/test/common/common/logger_test.cc
+++ b/test/common/common/logger_test.cc
@@ -384,6 +384,54 @@ TEST(LoggerTest, TestJsonFormatWithNestedJsonMessage) {
   ENVOY_LOG_MISC(info, "{\"nested_message\":\"hello\"}");
 }
 
+TEST(LoggerUtilityTest, TestSerializeLogTags) {
+  // No entries
+  EXPECT_EQ("", Envoy::Logger::Utility::serializeLogTags({}));
+
+  // Empty key or value
+  EXPECT_EQ("[Tags: \"\":\"\"] ", Envoy::Logger::Utility::serializeLogTags({{"", ""}}));
+  EXPECT_EQ("[Tags: \"\":\"value\"] ", Envoy::Logger::Utility::serializeLogTags({{"", "value"}}));
+  EXPECT_EQ("[Tags: \"key\":\"\"] ", Envoy::Logger::Utility::serializeLogTags({{"key", ""}}));
+
+  // Single entry
+  EXPECT_EQ("[Tags: \"key\":\"value\"] ",
+            Envoy::Logger::Utility::serializeLogTags({{"key", "value"}}));
+
+  // Multiple entries
+  EXPECT_EQ("[Tags: \"key1\":\"value1\",\"key2\":\"value2\",\"key3\":\"value3\"] ",
+            Envoy::Logger::Utility::serializeLogTags(
+                {{"key1", "value1"}, {"key2", "value2"}, {"key3", "value3"}}));
+}
+
+class ClassForTaggedLog : public Envoy::Logger::Loggable {
+public:
+  void logMessageWithPreCreatedTags() { ENVOY_TAGGED_LOG(info, tags_, "fake message {}", "val"); }
+
+  void logMessageWithInlineTags() {
+    ENVOY_TAGGED_LOG(info, (std::map{{"key_inline", "val"}}),
+                     "fake message {}", "val");
+  }
+
+private:
+  std::map tags_{{"key", "val"}};
+};
+
+TEST(TaggedLogTest, TestTaggedLog) {
+  Envoy::Logger::Registry::setLogFormat("%v");
+  MockLogSink sink(Envoy::Logger::Registry::getSink());
+  EXPECT_CALL(sink, log(_, _))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_THAT(msg, HasSubstr("[Tags: \"key\":\"val\"] fake message val"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_THAT(msg, HasSubstr("[Tags: \"key_inline\":\"val\"] fake message val"));
+      }));
+
+  ClassForTaggedLog object;
+  object.logMessageWithPreCreatedTags();
+  object.logMessageWithInlineTags();
+}
+
 } // namespace
 } // namespace Logger
 } // namespace Envoy

From 2ee62ca6ae6cd89fe96e153a641d22fbb8fd4828 Mon Sep 17 00:00:00 2001
From: yanjunxiang-google
 <78807980+yanjunxiang-google@users.noreply.github.com>
Date: Wed, 14 Jun 2023 08:32:45 -0400
Subject: [PATCH 532/740] Docs: fixing a few issues in ext_proc documentation
 (#27851)

Signed-off-by: Yanjun Xiang 
---
 .../ext_proc/v3/external_processor.proto        | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/api/envoy/service/ext_proc/v3/external_processor.proto b/api/envoy/service/ext_proc/v3/external_processor.proto
index 9d09828e1c50..21ae8c032cd0 100644
--- a/api/envoy/service/ext_proc/v3/external_processor.proto
+++ b/api/envoy/service/ext_proc/v3/external_processor.proto
@@ -170,6 +170,10 @@ message ProcessingResponse {
   // for the duration of this particular request/response only. Servers
   // may use this to intelligently control how requests are processed
   // based on the headers and other metadata that they see.
+  // This field is ignored by Envoy when the ext_proc filter config
+  // :ref:`allow_mode_override
+  // `
+  // is set to false.
   envoy.extensions.filters.http.ext_proc.v3.ProcessingMode mode_override = 9;
 
   // When ext_proc server receives a request message, in case it needs more
@@ -275,8 +279,11 @@ message CommonResponse {
 
   // Replace the body of the last message sent to the remote server on this
   // stream. If responding to an HttpBody request, simply replace or clear
-  // the body chunk that was sent with that request. Body mutations only take
-  // effect in response to ``body`` messages and are ignored otherwise.
+  // the body chunk that was sent with that request. Body mutations may take
+  // effect in response either to ``header`` or ``body`` messages. When it is
+  // in response to ``header`` messages, it only take effect if the
+  // :ref:`status `
+  // is set to CONTINUE_AND_REPLACE.
   BodyMutation body_mutation = 3;
 
   // [#not-implemented-hide:]
@@ -285,9 +292,9 @@ message CommonResponse {
   // along with the CONTINUE_AND_REPLACE status.
   config.core.v3.HeaderMap trailers = 4;
 
-  // Clear the route cache for the current request.
-  // This is necessary if the remote server
-  // modified headers that are used to calculate the route.
+  // Clear the route cache for the current client request. This is necessary
+  // if the remote server modified headers that are used to calculate the route.
+  // This field is ignored in the response direction.
   bool clear_route_cache = 5;
 }
 

From 98ef5c5cacabdf545bbcc8a8a411be83584b8015 Mon Sep 17 00:00:00 2001
From: Robert Femmer <114982872+robertfemmer@users.noreply.github.com>
Date: Wed, 14 Jun 2023 15:37:49 +0200
Subject: [PATCH 533/740] fuzz: add quic + http/3 fuzz test (#26546)

Signed-off-by: Robert Femmer 
---
 test/common/quic/BUILD                        |  36 ++
 test/common/quic/envoy_quic_h3_fuzz.proto     | 232 +++++++++
 test/common/quic/envoy_quic_h3_fuzz_helper.cc | 452 ++++++++++++++++++
 test/common/quic/envoy_quic_h3_fuzz_helper.h  | 102 ++++
 test/common/quic/envoy_quic_h3_fuzz_test.cc   | 226 +++++++++
 .../envoy_quic_h3_fuzz_test_corpus/HDRS000    |  71 +++
 .../HDRS000_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS001    |  75 +++
 .../HDRS001_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS002    |  75 +++
 .../HDRS002_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS003    |  75 +++
 .../HDRS003_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS004    |  67 +++
 .../HDRS004_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS005    |  71 +++
 .../HDRS005_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS006    |  67 +++
 .../HDRS006_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS007    |  67 +++
 .../HDRS007_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS008    |  75 +++
 .../HDRS008_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS009    |  71 +++
 .../HDRS009_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS010    |  63 +++
 .../HDRS010_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS011    |  71 +++
 .../HDRS011_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS012    |  71 +++
 .../HDRS012_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS013    |  75 +++
 .../HDRS013_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS014    |  75 +++
 .../HDRS014_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS015    |  71 +++
 .../HDRS015_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS016    |  63 +++
 .../HDRS016_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS017    |  71 +++
 .../HDRS017_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS018    |  67 +++
 .../HDRS018_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS019    |  71 +++
 .../HDRS019_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS020    |  71 +++
 .../HDRS020_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS021    |  71 +++
 .../HDRS021_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS022    |  71 +++
 .../HDRS022_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS023    |  75 +++
 .../HDRS023_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS024    |  67 +++
 .../HDRS024_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS025    |  67 +++
 .../HDRS025_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS026    |  75 +++
 .../HDRS026_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS027    |  67 +++
 .../HDRS027_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS028    |  67 +++
 .../HDRS028_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS029    |  67 +++
 .../HDRS029_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS030    |  71 +++
 .../HDRS030_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS031    |  71 +++
 .../HDRS031_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS032    |  67 +++
 .../HDRS032_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS033    |  75 +++
 .../HDRS033_junk                              |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/HDRS034    |  75 +++
 .../HDRS034_junk                              |  12 +
 .../quic/envoy_quic_h3_fuzz_test_corpus/h3get |  43 ++
 .../envoy_quic_h3_fuzz_test_corpus/h3get_junk |  12 +
 .../envoy_quic_h3_fuzz_test_corpus/quic_ack   |  12 +
 .../quic_ack_freq                             |  10 +
 .../quic_ack_freq_junk                        |   6 +
 .../quic_ack_junk                             |   6 +
 .../quic_blocked                              |   9 +
 .../quic_blocked_junk                         |   6 +
 .../quic_connection_close                     |  10 +
 .../quic_connection_close_junk                |   6 +
 .../quic_crypto                               |   8 +
 .../quic_crypto_junk                          |   6 +
 .../quic_handshake_done                       |   7 +
 .../quic_handshake_done_junk                  |   6 +
 .../quic_max_streams                          |   9 +
 .../quic_max_streams_junk                     |   6 +
 .../quic_message                              |   8 +
 .../quic_message_junk                         |   6 +
 .../quic_mtu_discovery                        |   5 +
 .../quic_mtu_discovery_junk                   |   6 +
 .../quic_new_connection_id                    |  11 +
 .../quic_new_connection_id_junk               |   6 +
 .../quic_new_token                            |   8 +
 .../quic_new_token_junk                       |   6 +
 .../quic_padding                              |   7 +
 .../quic_padding_junk                         |   6 +
 .../quic_path_challenge                       |   8 +
 .../quic_path_challenge_junk                  |   6 +
 .../quic_path_response                        |   8 +
 .../quic_path_response_junk                   |   6 +
 .../envoy_quic_h3_fuzz_test_corpus/quic_ping  |   7 +
 .../quic_ping_junk                            |   6 +
 .../quic_rst_frame                            |  10 +
 .../quic_rst_frame_junk                       |   6 +
 .../quic_stop_sending                         |   9 +
 .../quic_stop_sending_junk                    |   6 +
 .../quic_stream                               |  12 +
 .../quic_stream_junk                          |   6 +
 .../quic_streams_blocked                      |   9 +
 .../quic_streams_blocked_junk                 |   6 +
 .../quic_window_update                        |   9 +
 .../quic_window_update_junk                   |   6 +
 117 files changed, 4288 insertions(+)
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz.proto
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_helper.cc
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_helper.h
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test.cc
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS000
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS000_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS001
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS001_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS002
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS002_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS003
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS003_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS004
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS004_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS005
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS005_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS006
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS006_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS007
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS007_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS008
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS008_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS009
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS009_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS010
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS010_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS011
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS011_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS012
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS012_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS013
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS013_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS014
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS014_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS015
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS015_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS016
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS016_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS017
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS017_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS018
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS018_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS019
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS019_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS020
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS020_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS021
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS021_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS022
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS022_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS023
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS023_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS024
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS024_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS025
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS025_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS026
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS026_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS027
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS027_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS028
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS028_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS029
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS029_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS030
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS030_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS031
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS031_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS032
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS032_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS033
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS033_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS034
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS034_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/h3get
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/h3get_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_freq
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_freq_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_blocked
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_blocked_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_connection_close
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_connection_close_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_crypto
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_crypto_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_handshake_done
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_handshake_done_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_max_streams
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_max_streams_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_message
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_message_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_mtu_discovery
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_mtu_discovery_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_connection_id
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_connection_id_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_token
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_token_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_padding
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_padding_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_challenge
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_challenge_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_response
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_response_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ping
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ping_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_rst_frame
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_rst_frame_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stop_sending
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stop_sending_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stream
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stream_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_streams_blocked
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_streams_blocked_junk
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_window_update
 create mode 100644 test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_window_update_junk

diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD
index cd8ffdd57553..57d96dfbec4b 100644
--- a/test/common/quic/BUILD
+++ b/test/common/quic/BUILD
@@ -1,8 +1,10 @@
 load(
     "//bazel:envoy_build_system.bzl",
+    "envoy_cc_fuzz_test",
     "envoy_cc_test",
     "envoy_cc_test_library",
     "envoy_package",
+    "envoy_proto_library",
     "envoy_select_enable_http_datagrams",
 )
 
@@ -379,3 +381,37 @@ envoy_cc_test(
         "@com_github_google_quiche//:quic_test_tools_test_utils_lib",
     ]),
 )
+
+envoy_proto_library(
+    name = "envoy_quic_h3_fuzz_proto",
+    srcs = ["envoy_quic_h3_fuzz.proto"],
+    deps = ["//test/fuzz:common_proto"],
+)
+
+envoy_cc_test_library(
+    name = "envoy_quic_h3_fuzz_helper_lib",
+    srcs = ["envoy_quic_h3_fuzz_helper.cc"],
+    hdrs = ["envoy_quic_h3_fuzz_helper.h"],
+    tags = ["nofips"],
+    deps = [
+        ":envoy_quic_h3_fuzz_proto_cc_proto",
+        "@com_github_google_quiche//:quic_test_tools_test_utils_lib",
+    ],
+)
+
+envoy_cc_fuzz_test(
+    name = "envoy_quic_h3_fuzz_test",
+    srcs = ["envoy_quic_h3_fuzz_test.cc"],
+    corpus = "envoy_quic_h3_fuzz_test_corpus",
+    tags = ["nofips"],
+    deps = [
+        ":envoy_quic_h3_fuzz_helper_lib",
+        ":test_proof_source_lib",
+        ":test_utils_lib",
+        "//source/common/quic:envoy_quic_alarm_factory_lib",
+        "//source/common/quic:envoy_quic_connection_helper_lib",
+        "//source/common/quic:envoy_quic_dispatcher_lib",
+        "//source/common/quic:server_codec_lib",
+        "//test/mocks/http:http_mocks",
+    ],
+)
diff --git a/test/common/quic/envoy_quic_h3_fuzz.proto b/test/common/quic/envoy_quic_h3_fuzz.proto
new file mode 100644
index 000000000000..8f768a867c13
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz.proto
@@ -0,0 +1,232 @@
+syntax = "proto3";
+
+package test.common.quic;
+import "test/fuzz/common.proto";
+
+message H3DataFrame {
+  bytes data = 1;
+}
+
+message H3HeaderFrame {
+  // Need to be converted to an encoded field section
+  test.fuzz.Headers headers = 1;
+}
+
+message H3CancelPushFrame {
+  uint64 push_id = 1;
+}
+
+message H3Setting {
+  // Uses only lower 2 bytes
+  uint64 identifier = 1;
+  uint64 value = 2;
+}
+
+message H3SettingsFrame {
+  repeated H3Setting settings = 1;
+}
+
+message H3PushPromiseFrame {
+  uint64 push_id = 1;
+  // Need to be converted to an encoded field section
+  test.fuzz.Headers headers = 2;
+}
+
+message H3GoAwayFrame {
+  uint64 push_id = 1;
+}
+
+message H3MaxPushIdFrame {
+  uint64 push_id = 1;
+}
+
+message H3Frame {
+  oneof frame {
+    H3DataFrame data = 1;
+    H3HeaderFrame headers = 2;
+    H3CancelPushFrame cancel_push = 3;
+    H3SettingsFrame settings = 4;
+    H3PushPromiseFrame push_promise = 5;
+    H3GoAwayFrame go_away = 6;
+    H3MaxPushIdFrame max_push_id = 7;
+  }
+}
+
+message QuicPaddingFrame {
+  int32 num_padding_bytes = 1;
+}
+
+message QuicStreamFrame {
+  bool unidirectional = 1;
+  uint32 type = 2;
+  uint32 id = 3;
+  bool fin = 4;
+  uint64 offset = 5;
+  oneof data {
+    H3Frame h3frame = 6;
+    bytes junk = 7;
+  }
+}
+
+message QuicHandshakeDoneFrame {
+  uint32 control_frame_id = 1;
+}
+
+message QuicCryptoFrame {
+  uint64 offset = 1;
+  bytes data = 2;
+}
+
+message QuicEcnCounters {
+  uint64 ect0 = 1;
+  uint64 ect1 = 2;
+  uint64 ce = 3;
+}
+
+message QuicAckFrame {
+  uint64 largest_acked = 1;
+  QuicEcnCounters ecn_counters = 4;
+}
+
+message QuicMtuDiscoveryFrame {
+}
+
+message QuicStopWaitingFrame {
+  uint64 least_unacked = 1;
+}
+
+message QuicPingFrame {
+  uint32 control_frame_id = 1;
+}
+
+message QuicRstStreamFrame {
+  uint32 control_frame_id = 1;
+  uint32 stream_id = 2;
+  uint32 error_code = 3;
+  uint64 bytes_written = 4;
+}
+
+message QuicConnectionCloseFrame {
+  // TODO transport_version = 1;
+  uint32 error_code = 2;
+  uint64 ietf_error = 3;
+  string error_phrase = 4;
+  uint64 transport_close_frame_type = 5;
+}
+
+message QuicGoAwayFrame {
+  uint32 control_frame_id = 1;
+  uint32 error_code = 2;
+  uint32 last_good_stream_id = 3;
+  string reason = 4;
+}
+
+message QuicWindowUpdateFrame {
+  uint32 control_frame_id = 1;
+  uint32 stream_id = 2;
+  uint64 max_data = 3;
+}
+
+message QuicBlockedFrame {
+  uint32 control_frame_id = 1;
+  uint32 stream_id = 2;
+  uint64 offset = 3;
+}
+
+message QuicNewConnectionIdFrame {
+  uint32 control_frame_id = 1;
+  bytes connection_id = 2;
+  uint64 sequence_number = 3;
+  bytes stateless_reset_token = 4;
+  uint64 retire_prior_to = 5;
+}
+
+message QuicRetireConnectionIdFrame {
+  uint32 control_frame_id = 1;
+  uint64 sequence_number = 2;
+}
+
+message QuicMaxStreamsFrame {
+  uint32 control_frame_id = 1;
+  uint32 stream_count = 2;
+  bool unidirectional = 3;
+}
+
+message QuicStreamsBlockedFrame {
+  uint32 control_frame_id = 1;
+  uint32 stream_count = 2;
+  bool unidirectional = 3;
+}
+
+message QuicPathResponseFrame {
+  uint32 control_frame_id = 1;
+  bytes data = 2;
+}
+
+message QuicPathChallengeFrame {
+  uint32 control_frame_id = 1;
+  bytes data = 2;
+}
+
+message QuicStopSendingFrame {
+  uint32 control_frame_id = 1;
+  uint32 stream_id = 2;
+  uint32 error_code = 3;
+}
+
+message QuicMessageFrame {
+  uint32 message_id = 1;
+  bytes data = 2;
+}
+
+message QuicNewTokenFrame {
+  uint32 control_frame_id = 1;
+  string token = 2;
+}
+
+message QuickAckFrequencyFrame {
+  uint32 control_frame_id = 1;
+  uint64 sequence_number = 2;
+  uint64 packet_tolerance = 3;
+  uint32 milliseconds = 4;
+}
+
+message QuicFrame {
+  oneof frame {
+    QuicPaddingFrame padding = 1;
+    QuicStreamFrame stream = 2;
+    QuicHandshakeDoneFrame handshake_done = 3;
+    QuicCryptoFrame crypto = 4;
+    QuicAckFrame ack = 5;
+    QuicMtuDiscoveryFrame mtu_discovery = 6;
+    QuicStopWaitingFrame stop_waiting = 7;
+    QuicPingFrame ping = 8;
+    QuicRstStreamFrame rst_stream = 9;
+    QuicConnectionCloseFrame connection_close = 10;
+    QuicGoAwayFrame go_away = 11;
+    QuicWindowUpdateFrame window_update = 12;
+    QuicBlockedFrame blocked = 13;
+    QuicNewConnectionIdFrame new_connection_id = 14;
+    QuicRetireConnectionIdFrame retire_connection_id = 15;
+    QuicMaxStreamsFrame max_streams = 16;
+    QuicStreamsBlockedFrame streams_blocked = 17;
+    QuicPathResponseFrame path_response = 18;
+    QuicPathChallengeFrame path_challenge = 19;
+    QuicStopSendingFrame stop_sending = 20;
+    QuicMessageFrame message_frame = 21;
+    QuicNewTokenFrame new_token = 22;
+    QuickAckFrequencyFrame ack_frequency = 23;
+  }
+}
+
+message QuicFrameOrJunk {
+  oneof frame {
+    QuicFrame qframe = 1;
+    bytes junk = 2;
+  }
+}
+
+message QuicH3FuzzCase {
+  // QUIC frames to be processed
+  repeated QuicFrameOrJunk frames = 1;
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_helper.cc b/test/common/quic/envoy_quic_h3_fuzz_helper.cc
new file mode 100644
index 000000000000..1994e83c7f40
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_helper.cc
@@ -0,0 +1,452 @@
+#include "test/common/quic/envoy_quic_h3_fuzz_helper.h"
+
+#include "quiche/common/quiche_data_writer.h"
+#include "quiche/quic/test_tools/quic_test_utils.h"
+
+namespace Envoy {
+namespace Quic {
+
+static uint64_t clampU64(uint64_t in) { return in & ((1ULL << 62) - 1); }
+
+using namespace test::common::quic;
+
+enum class Type : uint8_t {
+  Data = 0x00,
+  Headers = 0x01,
+  CancelPush = 0x03,
+  Settings = 0x04,
+  PushPromise = 0x05,
+  GoAway = 0x07,
+  MaxPushId = 0x0d,
+};
+
+class Delegate : public quic::QpackEncoder::DecoderStreamErrorDelegate {
+public:
+  void OnDecoderStreamError(quic::QuicErrorCode, absl::string_view) override{};
+};
+
+static std::string encodeHeaders(const spdy::Http2HeaderBlock& headers) {
+  static Delegate delegate;
+  quic::QpackEncoder encoder(&delegate);
+  return encoder.EncodeHeaderList(0, headers, nullptr);
+}
+
+static size_t buildRawFrame(quiche::QuicheDataWriter& dw, Type type, const std::string& payload) {
+  bool valid = true;
+  valid &= dw.WriteVarInt62(static_cast(type));
+  valid &= dw.WriteStringPieceVarInt62(payload);
+  return valid ? dw.length() : 0;
+}
+
+static size_t buildVarIntFrame(quiche::QuicheDataWriter& dw, Type type, uint64_t number) {
+  bool valid = true;
+  uint64_t s = quiche::QuicheDataWriter::GetVarInt62Len(clampU64(number));
+  valid &= dw.WriteVarInt62(static_cast(type));
+  valid &= dw.WriteVarInt62(s);
+  valid &= dw.WriteVarInt62(clampU64(number));
+  return valid ? dw.length() : 0;
+}
+
+static size_t buildSettingsFrame(quiche::QuicheDataWriter& dw,
+                                 std::vector>& settings) {
+  bool valid = true;
+  uint64_t slen = 0;
+  for (auto pair : settings) {
+    slen += quiche::QuicheDataWriter::GetVarInt62Len(clampU64(pair.first));
+    slen += quiche::QuicheDataWriter::GetVarInt62Len(clampU64(pair.second));
+  }
+  valid &= dw.WriteVarInt62(static_cast(Type::Settings));
+  valid &= dw.WriteVarInt62(slen);
+  for (auto pair : settings) {
+    valid &= dw.WriteVarInt62(clampU64(pair.first));
+    valid &= dw.WriteVarInt62(clampU64(pair.second));
+  }
+  return valid ? dw.length() : 0;
+}
+
+static size_t buildPushPromiseFrame(quiche::QuicheDataWriter& dw, uint64_t push_id,
+                                    const std::string& headers) {
+  bool valid = true;
+  uint64_t s = quiche::QuicheDataWriter::GetVarInt62Len(clampU64(push_id));
+  s += headers.size();
+
+  valid &= dw.WriteVarInt62(static_cast(Type::PushPromise));
+  valid &= dw.WriteVarInt62(s);
+  valid &= dw.WriteVarInt62(clampU64(push_id));
+  valid &= dw.WriteBytes(headers.data(), headers.size());
+  return valid ? dw.length() : 0;
+}
+
+std::string H3Serializer::serialize(bool unidirectional, uint32_t type, uint32_t id,
+                                    const H3Frame& h3frame) {
+  char buffer[kMaxPacketSize];
+  quiche::QuicheDataWriter dw(kMaxPacketSize, buffer);
+  if (unidirectional) {
+    if (open_unidirectional_streams_.find(id) == open_unidirectional_streams_.end()) {
+      dw.WriteVarInt62(static_cast(type));
+      open_unidirectional_streams_.insert(id);
+    }
+  }
+  switch (h3frame.frame_case()) {
+  case H3Frame::kData: {
+    const auto& f = h3frame.data();
+    size_t size = buildRawFrame(dw, Type::Data, f.data());
+    return {buffer, size};
+  }
+  case H3Frame::kHeaders: {
+    const auto& f = h3frame.headers();
+    spdy::Http2HeaderBlock headers;
+    for (const auto& hdr : f.headers().headers()) {
+      headers.AppendValueOrAddHeader(hdr.key(), hdr.value());
+    }
+    size_t size = buildRawFrame(dw, Type::Headers, encodeHeaders(headers));
+    return {buffer, size};
+  }
+  case H3Frame::kCancelPush: {
+    const auto& f = h3frame.cancel_push();
+    size_t size = buildVarIntFrame(dw, Type::CancelPush, f.push_id());
+    return {buffer, size};
+  }
+  case H3Frame::kSettings: {
+    const auto& f = h3frame.settings();
+    std::vector> values;
+    for (auto& setting : f.settings()) {
+      values.push_back(std::make_pair(setting.identifier(), setting.value()));
+    }
+    size_t size = buildSettingsFrame(dw, values);
+    return {buffer, size};
+  }
+  case H3Frame::kPushPromise: {
+    const auto& f = h3frame.push_promise();
+    uint64_t push_id = f.push_id();
+    spdy::Http2HeaderBlock headers;
+    for (auto& hdr : f.headers().headers()) {
+      headers.AppendValueOrAddHeader(hdr.key(), hdr.value());
+    }
+    size_t size = buildPushPromiseFrame(dw, push_id, encodeHeaders(headers));
+    return {buffer, size};
+  }
+  case H3Frame::kGoAway: {
+    const auto& f = h3frame.go_away();
+    size_t size = buildVarIntFrame(dw, Type::GoAway, f.push_id());
+    return {buffer, size};
+  }
+  case H3Frame::kMaxPushId: {
+    const auto& f = h3frame.max_push_id();
+    size_t size = buildVarIntFrame(dw, Type::MaxPushId, f.push_id());
+    return {buffer, size};
+  }
+  default:
+    break;
+  }
+  return "";
+}
+
+static quic::QuicConnectionId toConnectionId(const std::string& data) {
+  uint8_t size = std::min(static_cast(data.size()), quic::kQuicDefaultConnectionIdLength);
+  return {data.data(), size};
+}
+
+static quic::StatelessResetToken toStatelessResetToken(const std::string& data) {
+  quic::StatelessResetToken token = {0};
+  size_t to_copy = std::min(data.size(), token.size());
+  const char* start = data.data();
+  const char* end = start + to_copy;
+  std::copy(start, end, token.begin());
+  return token;
+}
+
+static quic::QuicPathFrameBuffer toPathFrameBuffer(const std::string& data) {
+  quic::QuicPathFrameBuffer buffer = {0};
+  size_t to_copy = std::min(data.size(), buffer.size());
+  const char* start = data.data();
+  const char* end = start + to_copy;
+  std::copy(start, end, buffer.begin());
+  return buffer;
+}
+
+static quic::QuicErrorCode toErrorCode(uint32_t) { return quic::QuicErrorCode::QUIC_NO_ERROR; }
+
+QuicPacketizer::QuicPacketizer(const quic::ParsedQuicVersion& quic_version,
+                               quic::QuicConnectionHelperInterface* connection_helper)
+    : quic_version_(quic_version), connection_helper_(connection_helper), packet_number_(0),
+      destination_connection_id_(quic::test::TestConnectionId()),
+      framer_({quic_version_}, connection_helper_->GetClock()->Now(), quic::Perspective::IS_CLIENT,
+              quic::kQuicDefaultConnectionIdLength),
+      h3serializer_(open_unidirectional_streams_) {
+  quic::Perspective p = quic::Perspective::IS_CLIENT;
+  framer_.SetEncrypter(quic::ENCRYPTION_INITIAL, std::make_unique(p));
+  framer_.SetEncrypter(quic::ENCRYPTION_HANDSHAKE, std::make_unique(p));
+  framer_.SetEncrypter(quic::ENCRYPTION_ZERO_RTT, std::make_unique(p));
+  framer_.SetEncrypter(quic::ENCRYPTION_FORWARD_SECURE, std::make_unique(p));
+}
+
+std::vector
+QuicPacketizer::serializePackets(const QuicH3FuzzCase& input) {
+  std::vector packets;
+  for (auto& quic_frame_or_junk : input.frames()) {
+    if (quic_frame_or_junk.has_qframe()) {
+      auto packet = serializePacket(quic_frame_or_junk.qframe());
+      if (packet) {
+        packets.push_back(std::move(packet));
+      }
+    } else if (quic_frame_or_junk.has_junk()) {
+      const std::string& junk = quic_frame_or_junk.junk();
+      auto packet = serializeJunkPacket(junk);
+      if (packet) {
+        packets.push_back(std::move(packet));
+      }
+    }
+  }
+  return packets;
+}
+
+void QuicPacketizer::reset() {
+  packet_number_ = quic::QuicPacketNumber(0);
+  open_unidirectional_streams_.clear();
+}
+
+QuicPacketizer::QuicPacketPtr QuicPacketizer::serializePacket(const QuicFrame& frame) {
+  switch (frame.frame_case()) {
+  case QuicFrame::kPadding: {
+    int padding = frame.padding().num_padding_bytes() & 0xff;
+    if (padding == 0) {
+      padding++;
+    }
+    auto quic_padding = quic::QuicPaddingFrame(padding);
+    return serialize(quic::QuicFrame(quic_padding));
+  }
+  case QuicFrame::kStream:
+    return serializeStreamFrame(frame.stream());
+  case QuicFrame::kHandshakeDone: {
+    const auto& f = frame.handshake_done();
+    auto handshake = quic::QuicHandshakeDoneFrame(f.control_frame_id());
+    return serialize(quic::QuicFrame(handshake));
+  }
+  case QuicFrame::kCrypto:
+    return serializeCryptoFrame(frame.crypto());
+  case QuicFrame::kAck:
+    return serializeAckFrame(frame.ack());
+  case QuicFrame::kMtuDiscovery: {
+    auto quic_mtu = quic::QuicMtuDiscoveryFrame();
+    return serialize(quic::QuicFrame(quic_mtu));
+  }
+  case QuicFrame::kStopWaiting:
+    // not possible in IETF mode
+    break;
+  case QuicFrame::kPing: {
+    const auto& f = frame.ping();
+    auto quic_ping = quic::QuicPingFrame(f.control_frame_id());
+    return serialize(quic::QuicFrame(quic_ping));
+  }
+  case QuicFrame::kRstStream: {
+    const auto& f = frame.rst_stream();
+    quic::QuicRstStreamErrorCode error_code =
+        static_cast(f.error_code());
+    auto reset_stream_frame = quic::QuicRstStreamFrame(f.control_frame_id(), f.stream_id(),
+                                                       error_code, clampU64(f.bytes_written()));
+    return serialize(quic::QuicFrame(&reset_stream_frame));
+  }
+  case QuicFrame::kConnectionClose: {
+    const auto& f = frame.connection_close();
+    quic::QuicErrorCode error_code = toErrorCode(f.error_code());
+    quic::QuicIetfTransportErrorCodes ietf_error =
+        static_cast(clampU64(f.ietf_error()));
+    auto connection_close_frame =
+        quic::QuicConnectionCloseFrame(quic_version_.transport_version, error_code, ietf_error,
+                                       f.error_phrase(), clampU64(f.transport_close_frame_type()));
+    return serialize(quic::QuicFrame(&connection_close_frame));
+  }
+  case QuicFrame::kGoAway: {
+    // not possible in IETF mode
+    return nullptr;
+  }
+  case QuicFrame::kWindowUpdate: {
+    const auto& f = frame.window_update();
+    auto quic_window =
+        quic::QuicWindowUpdateFrame(f.control_frame_id(), f.stream_id(), clampU64(f.max_data()));
+    return serialize(quic::QuicFrame(quic_window));
+  }
+  case QuicFrame::kBlocked: {
+    const auto& f = frame.blocked();
+    auto quic_blocked =
+        quic::QuicBlockedFrame(f.control_frame_id(), f.stream_id(), clampU64(f.offset()));
+    return serialize(quic::QuicFrame(quic_blocked));
+  }
+  case QuicFrame::kNewConnectionId:
+    return serializeNewConnectionIdFrame(frame.new_connection_id());
+  case QuicFrame::kRetireConnectionId: {
+    const auto& f = frame.retire_connection_id();
+    auto retire_connection_id_frame =
+        quic::QuicRetireConnectionIdFrame(f.control_frame_id(), clampU64(f.sequence_number()));
+    return serialize(quic::QuicFrame(&retire_connection_id_frame));
+  }
+  case QuicFrame::kMaxStreams: {
+    const auto& f = frame.max_streams();
+    auto quic_max_streams =
+        quic::QuicMaxStreamsFrame(f.control_frame_id(), f.stream_count(), f.unidirectional());
+    return serialize(quic::QuicFrame(quic_max_streams));
+  }
+  case QuicFrame::kStreamsBlocked: {
+    const auto& f = frame.streams_blocked();
+    auto quic_streams =
+        quic::QuicStreamsBlockedFrame(f.control_frame_id(), f.stream_count(), f.unidirectional());
+    return serialize(quic::QuicFrame(quic_streams));
+  }
+  case QuicFrame::kPathResponse: {
+    const auto& f = frame.path_response();
+    auto quic_path = quic::QuicPathResponseFrame(f.control_frame_id(), toPathFrameBuffer(f.data()));
+    return serialize(quic::QuicFrame(quic_path));
+  }
+  case QuicFrame::kPathChallenge: {
+    const auto& f = frame.path_challenge();
+    auto quic_path =
+        quic::QuicPathChallengeFrame(f.control_frame_id(), toPathFrameBuffer(f.data()));
+    return serialize(quic::QuicFrame(quic_path));
+  }
+  case QuicFrame::kStopSending: {
+    const auto& f = frame.stop_sending();
+    quic::QuicRstStreamErrorCode error_code =
+        static_cast(f.error_code());
+    auto quic_stop = quic::QuicStopSendingFrame(f.control_frame_id(), f.stream_id(), error_code);
+    return serialize(quic::QuicFrame(quic_stop));
+  }
+  case QuicFrame::kMessageFrame:
+    return serializeMessageFrame(frame.message_frame());
+  case QuicFrame::kNewToken:
+    return serializeNewTokenFrame(frame.new_token());
+  case QuicFrame::kAckFrequency: {
+    const auto& f = frame.ack_frequency();
+    auto delta = quic::QuicTime::Delta::FromMilliseconds(clampU64(f.milliseconds()));
+    auto ack_frequency = quic::QuicAckFrequencyFrame(
+        f.control_frame_id(), clampU64(f.sequence_number()), clampU64(f.packet_tolerance()), delta);
+    return serialize(quic::QuicFrame(&ack_frequency));
+  }
+  default:
+    break;
+  }
+  return nullptr;
+}
+
+QuicPacketizer::QuicPacketPtr QuicPacketizer::serializeJunkPacket(const std::string& data) {
+  char* buffer = new char[kMaxPacketSize];
+
+  quic::QuicPacketHeader header;
+  header.packet_number = packet_number_++;
+  header.destination_connection_id = destination_connection_id_;
+  header.source_connection_id = destination_connection_id_;
+
+  quic::QuicDataWriter writer(kMaxPacketSize, buffer);
+  quic::QuicFramer framer({quic_version_}, connection_helper_->GetClock()->Now(),
+                          quic::Perspective::IS_CLIENT, quic::kQuicDefaultConnectionIdLength);
+
+  auto encrypter = std::make_unique(quic::Perspective::IS_CLIENT);
+  framer.SetEncrypter(quic::ENCRYPTION_INITIAL, std::move(encrypter));
+
+  size_t length_field_offset = 0;
+  if (!framer.AppendIetfPacketHeader(header, &writer, &length_field_offset)) {
+    return nullptr;
+  }
+  size_t max_data_len = std::min(data.size(), writer.remaining());
+  writer.WriteBytes(data.data(), max_data_len);
+  framer.WriteIetfLongHeaderLength(header, &writer, length_field_offset, quic::ENCRYPTION_INITIAL);
+  return std::make_unique(buffer, writer.length(), true);
+}
+
+QuicPacketizer::QuicPacketPtr QuicPacketizer::serialize(quic::QuicFrame frame) {
+  char* buffer = new char[kMaxPacketSize];
+  quic::QuicFrames frames = {frame};
+  quic::QuicFramer framer({quic_version_}, connection_helper_->GetClock()->Now(),
+                          quic::Perspective::IS_CLIENT, quic::kQuicDefaultConnectionIdLength);
+
+  quic::QuicPacketHeader header;
+  header.packet_number = packet_number_++;
+  header.destination_connection_id = destination_connection_id_;
+  header.source_connection_id = destination_connection_id_;
+  auto encrypter = std::make_unique(quic::Perspective::IS_CLIENT);
+  framer.SetEncrypter(quic::ENCRYPTION_INITIAL, std::move(encrypter));
+  size_t size = framer.BuildDataPacket(header, frames, buffer, kMaxPacketSize,
+                                       quic::EncryptionLevel::ENCRYPTION_INITIAL);
+  return std::make_unique(buffer, size, true);
+}
+
+QuicPacketizer::QuicPacketPtr
+QuicPacketizer::serializeStreamFrame(const test::common::quic::QuicStreamFrame& frame) {
+  bool unidirectional = frame.unidirectional();
+  uint32_t type = frame.type();
+  uint32_t id = frame.id();
+  bool fin = frame.fin();
+  uint64_t offset = clampU64(frame.offset());
+  if (frame.has_h3frame()) {
+    const auto& f = frame.h3frame();
+    auto h3packet = h3serializer_.serialize(unidirectional, type, id, f);
+    if (!h3packet.empty()) {
+      return serialize(quic::QuicFrame(
+          quic::QuicStreamFrame(id, fin, offset, h3packet.data(), h3packet.size())));
+    }
+  } else if (frame.has_junk()) {
+    auto junk = frame.junk();
+    size_t len = std::min(junk.size(), 1024UL);
+    return serialize(quic::QuicFrame(quic::QuicStreamFrame(id, fin, offset, junk.data(), len)));
+  }
+  return nullptr;
+}
+
+QuicPacketizer::QuicPacketPtr
+QuicPacketizer::serializeNewTokenFrame(const test::common::quic::QuicNewTokenFrame& frame) {
+  char buffer[1024];
+  size_t len = std::min(frame.token().size(), sizeof(buffer));
+  memcpy(buffer, frame.token().data(), len);
+  absl::string_view token(buffer, len);
+  auto new_token = quic::QuicNewTokenFrame(frame.control_frame_id(), token);
+  return serialize(quic::QuicFrame(&new_token));
+}
+
+QuicPacketizer::QuicPacketPtr
+QuicPacketizer::serializeMessageFrame(const test::common::quic::QuicMessageFrame& frame) {
+  char buffer[1024];
+  auto message = frame.data();
+  size_t len = std::min(message.size(), sizeof(buffer));
+  memcpy(buffer, message.data(), len);
+  auto message_frame = quic::QuicMessageFrame(buffer, len);
+  return serialize(quic::QuicFrame(&message_frame));
+}
+
+QuicPacketizer::QuicPacketPtr
+QuicPacketizer::serializeCryptoFrame(const test::common::quic::QuicCryptoFrame& frame) {
+  char buffer[1024];
+  auto data = frame.data();
+  uint16_t len = std::min(data.size(), sizeof(buffer));
+  memcpy(buffer, data.data(), len);
+  auto crypto_frame = quic::QuicCryptoFrame(quic::EncryptionLevel::ENCRYPTION_INITIAL,
+                                            clampU64(frame.offset()), buffer, len);
+  return serialize(quic::QuicFrame(&crypto_frame));
+}
+
+QuicPacketizer::QuicPacketPtr
+QuicPacketizer::serializeAckFrame(const test::common::quic::QuicAckFrame& frame) {
+  auto largest_acked = quic::QuicPacketNumber(clampU64(frame.largest_acked()));
+  quic::QuicAckFrame ack_frame;
+  ack_frame.largest_acked = largest_acked;
+  ack_frame.packets.Add(largest_acked);
+  if (frame.has_ecn_counters()) {
+    const auto& c = frame.ecn_counters();
+    ack_frame.ecn_counters =
+        quic::QuicEcnCounts(clampU64(c.ect0()), clampU64(c.ect1()), clampU64(c.ce()));
+  }
+  return serialize(quic::QuicFrame(&ack_frame));
+}
+
+QuicPacketizer::QuicPacketPtr QuicPacketizer::serializeNewConnectionIdFrame(
+    const test::common::quic::QuicNewConnectionIdFrame& frame) {
+  quic::QuicNewConnectionIdFrame new_connection_id_frame;
+  new_connection_id_frame.control_frame_id = frame.control_frame_id();
+  new_connection_id_frame.connection_id = toConnectionId(frame.connection_id());
+  new_connection_id_frame.stateless_reset_token =
+      toStatelessResetToken(frame.stateless_reset_token());
+  new_connection_id_frame.sequence_number = clampU64(frame.sequence_number());
+  return serialize(quic::QuicFrame(&new_connection_id_frame));
+}
+
+} // namespace Quic
+} // namespace Envoy
diff --git a/test/common/quic/envoy_quic_h3_fuzz_helper.h b/test/common/quic/envoy_quic_h3_fuzz_helper.h
new file mode 100644
index 000000000000..d9e5a2999b9e
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_helper.h
@@ -0,0 +1,102 @@
+#include 
+
+#include "test/common/quic/envoy_quic_h3_fuzz.pb.h"
+
+#include "quiche/quic/core/crypto/null_decrypter.h"
+#include "quiche/quic/core/crypto/null_encrypter.h"
+#include "quiche/quic/core/quic_connection.h"
+#include "quiche/quic/core/quic_versions.h"
+
+namespace Envoy {
+namespace Quic {
+
+// This class serializes structured protobuf `HTTP/3` messages to bytes as they
+// would be sent over the wire.
+class H3Serializer {
+public:
+  static constexpr size_t kMaxPacketSize = 1024;
+  H3Serializer(std::set& streams) : open_unidirectional_streams_(streams){};
+  // This method serializes an `HTTP/3` frame given by `h3frame` to `std::string`.
+  // If `unidirectional` is true, `type` will give the type of unidirectional
+  // stream to be opened. `id` identifies an `HTTP/3` frame and is used to track
+  // the whether the stream is already in flight.
+  std::string serialize(bool unidirectional, uint32_t type, uint32_t id,
+                        const test::common::quic::H3Frame& h3frame);
+
+private:
+  std::set& open_unidirectional_streams_;
+};
+
+// This class serializes structured protobuf `QUIC + HTTP/3` messages to bytes as they
+// would be sent over the wire.
+class QuicPacketizer {
+public:
+  static constexpr size_t kMaxPacketSize = 1460;
+  using QuicPacketPtr = std::unique_ptr;
+  QuicPacketizer(const quic::ParsedQuicVersion& quic_version,
+                 quic::QuicConnectionHelperInterface* connection_helper);
+  std::vector serializePackets(const test::common::quic::QuicH3FuzzCase& input);
+  void reset();
+
+private:
+  QuicPacketPtr serializePacket(const test::common::quic::QuicFrame& frame);
+  QuicPacketPtr serializeJunkPacket(const std::string& data);
+  QuicPacketPtr serialize(quic::QuicFrame frame);
+  QuicPacketPtr serializeStreamFrame(const test::common::quic::QuicStreamFrame& frame);
+  QuicPacketPtr serializeNewTokenFrame(const test::common::quic::QuicNewTokenFrame& frame);
+  QuicPacketPtr serializeMessageFrame(const test::common::quic::QuicMessageFrame& frame);
+  QuicPacketPtr serializeCryptoFrame(const test::common::quic::QuicCryptoFrame& frame);
+  QuicPacketPtr serializeAckFrame(const test::common::quic::QuicAckFrame& frame);
+  QuicPacketPtr
+  serializeNewConnectionIdFrame(const test::common::quic::QuicNewConnectionIdFrame& frame);
+
+  quic::ParsedQuicVersion quic_version_;
+  quic::QuicConnectionHelperInterface* connection_helper_;
+  quic::QuicPacketNumber packet_number_;
+
+  quic::QuicConnectionId destination_connection_id_;
+  quic::QuicFramer framer_;
+
+  H3Serializer h3serializer_;
+
+  // Unidirectional streams are started by sending a variable-length integer
+  // indicating the stream type (RFC9114, Sec. 6.2). H3Serializer serializes the
+  // stream type into the stream payload upon seeing the stream for the first
+  // time. This set tracks opened unidirectional streams in this flight. So that
+  // multiple stream frames on the same stream will not cause the stream type to
+  // be serialized again.
+  std::set open_unidirectional_streams_;
+};
+
+// The following two classes handle the encryption and decryption in the fuzzer
+// and just pass the plain or cipher text as is. The classes override
+// the `EncryptPacket` method, because the `Null{En,De}crypter` calculates
+// a hash of the data for each packet, which is unnecessary in fuzzing.
+class FuzzEncrypter : public quic::NullEncrypter {
+public:
+  explicit FuzzEncrypter(quic::Perspective perspective) : NullEncrypter(perspective){};
+  bool EncryptPacket(uint64_t, absl::string_view, absl::string_view plaintext, char* output,
+                     size_t* output_length, size_t max_output_length) override {
+    ASSERT(plaintext.length() <= max_output_length);
+    memcpy(output, plaintext.data(), plaintext.length());
+    *output_length = plaintext.length();
+    return true;
+  };
+  size_t GetMaxPlaintextSize(size_t ciphertext_size) const override { return ciphertext_size; }
+  size_t GetCiphertextSize(size_t plaintext_size) const override { return plaintext_size; }
+};
+
+class FuzzDecrypter : public quic::NullDecrypter {
+public:
+  explicit FuzzDecrypter(quic::Perspective perspective) : NullDecrypter(perspective){};
+  bool DecryptPacket(uint64_t, absl::string_view, absl::string_view ciphertext, char* output,
+                     size_t* output_length, size_t max_output_length) override {
+    ASSERT(ciphertext.length() <= max_output_length);
+    memcpy(output, ciphertext.data(), ciphertext.length());
+    *output_length = ciphertext.length();
+    return true;
+  };
+};
+
+} // namespace Quic
+} // namespace Envoy
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test.cc b/test/common/quic/envoy_quic_h3_fuzz_test.cc
new file mode 100644
index 000000000000..977eece5bc52
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test.cc
@@ -0,0 +1,226 @@
+#include "source/common/quic/envoy_quic_alarm_factory.h"
+#include "source/common/quic/envoy_quic_connection_helper.h"
+#include "source/common/quic/envoy_quic_dispatcher.h"
+#include "source/common/quic/envoy_quic_server_connection.h"
+#include "source/common/quic/envoy_quic_server_session.h"
+
+#include "test/common/quic/envoy_quic_h3_fuzz.pb.h"
+#include "test/common/quic/envoy_quic_h3_fuzz_helper.h"
+#include "test/common/quic/test_proof_source.h"
+#include "test/common/quic/test_utils.h"
+#include "test/fuzz/fuzz_runner.h"
+#include "test/mocks/http/mocks.h"
+#include "test/mocks/network/mocks.h"
+
+#include "quiche/quic/core/deterministic_connection_id_generator.h"
+#include "quiche/quic/core/quic_crypto_server_stream.h"
+#include "quiche/quic/core/tls_server_handshaker.h"
+#include "quiche/quic/test_tools/quic_test_utils.h"
+
+namespace Envoy {
+namespace Quic {
+
+using namespace test::common::quic;
+
+// The following classes essentially mock the `QUIC` handshake
+class ProofSourceDetailsSetter {
+public:
+  virtual ~ProofSourceDetailsSetter() = default;
+  virtual void setProofSourceDetails(std::unique_ptr details) = 0;
+};
+
+class TestQuicCryptoServerStream : public quic::QuicCryptoServerStream,
+                                   public ProofSourceDetailsSetter {
+public:
+  ~TestQuicCryptoServerStream() override = default;
+  explicit TestQuicCryptoServerStream(const quic::QuicCryptoServerConfig* crypto_config,
+                                      quic::QuicCompressedCertsCache* compressed_certs_cache,
+                                      quic::QuicSession* session,
+                                      quic::QuicCryptoServerStreamBase::Helper* helper)
+      : quic::QuicCryptoServerStream(crypto_config, compressed_certs_cache, session, helper) {}
+  bool encryption_established() const override { return true; }
+  const EnvoyQuicProofSourceDetails* ProofSourceDetails() const override { return details_.get(); }
+  void setProofSourceDetails(std::unique_ptr details) override {
+    details_ = std::move(details);
+  }
+
+private:
+  std::unique_ptr details_;
+};
+
+class TestEnvoyQuicTlsServerHandshaker : public quic::TlsServerHandshaker,
+                                         public ProofSourceDetailsSetter {
+public:
+  ~TestEnvoyQuicTlsServerHandshaker() override = default;
+  TestEnvoyQuicTlsServerHandshaker(quic::QuicSession* session,
+                                   const quic::QuicCryptoServerConfig& crypto_config)
+      : quic::TlsServerHandshaker(session, &crypto_config),
+        params_(new quic::QuicCryptoNegotiatedParameters) {
+    params_->cipher_suite = 1;
+  }
+  bool encryption_established() const override { return true; }
+  const EnvoyQuicProofSourceDetails* ProofSourceDetails() const override { return details_.get(); }
+  void setProofSourceDetails(std::unique_ptr details) override {
+    details_ = std::move(details);
+  }
+  const quic::QuicCryptoNegotiatedParameters& crypto_negotiated_params() const override {
+    return *params_;
+  }
+
+private:
+  std::unique_ptr details_;
+  quiche::QuicheReferenceCountedPointer params_;
+};
+
+class EnvoyQuicTestCryptoServerStreamFactory : public EnvoyQuicCryptoServerStreamFactoryInterface {
+public:
+  ProtobufTypes::MessagePtr createEmptyConfigProto() override { return nullptr; }
+  std::string name() const override { return "quic.test_crypto_server_stream"; }
+
+  std::unique_ptr createEnvoyQuicCryptoServerStream(
+      const quic::QuicCryptoServerConfig* crypto_config,
+      quic::QuicCompressedCertsCache* compressed_certs_cache, quic::QuicSession* session,
+      quic::QuicCryptoServerStreamBase::Helper* helper,
+      OptRef /*transport_socket_factory*/,
+      Event::Dispatcher& /*dispatcher*/) override {
+    switch (session->connection()->version().handshake_protocol) {
+    case quic::PROTOCOL_QUIC_CRYPTO:
+      return std::make_unique(crypto_config, compressed_certs_cache,
+                                                          session, helper);
+    case quic::PROTOCOL_TLS1_3:
+      return std::make_unique(session, *crypto_config);
+    case quic::PROTOCOL_UNSUPPORTED:
+      ASSERT(false, "Unknown handshake protocol");
+    }
+    return nullptr;
+  }
+};
+
+QuicDispatcherStats generateStats(Stats::Scope& store) {
+  return {QUIC_DISPATCHER_STATS(POOL_COUNTER_PREFIX(store, "quic.dispatcher"))};
+}
+
+// On every fuzz case the harness creates a connection for a fixed `QUIC` version,
+// and establishes a `QUIC` session. The handshake/client hello are skipped.
+struct Harness {
+  Harness(quic::ParsedQuicVersion quic_version)
+      : quic_version_(quic_version), api_(Api::createApiForTest()),
+        dispatcher_(api_->allocateDispatcher("envoy_quic_h3_fuzzer_thread")),
+        version_manager_(quic::CurrentSupportedHttp3Versions()),
+        connection_helper_(std::unique_ptr(
+            new EnvoyQuicConnectionHelper(*dispatcher_.get()))),
+        alarm_factory_(std::unique_ptr(
+            new EnvoyQuicAlarmFactory(*dispatcher_.get(), *connection_helper_->GetClock()))),
+        packetizer_(quic_version, connection_helper_.get()),
+        peer_addr_(Network::Utility::getAddressWithPort(*Network::Utility::getIpv6LoopbackAddress(),
+                                                        12345)),
+        self_addr_(Network::Utility::getAddressWithPort(*Network::Utility::getIpv6LoopbackAddress(),
+                                                        54321)),
+        cli_addr_(peer_addr_->sockAddr(), peer_addr_->sockAddrLen()),
+        srv_addr_(self_addr_->sockAddr(), self_addr_->sockAddrLen()),
+        quic_stat_names_(mock_listener_config_.listenerScope().symbolTable()),
+        http3_stats_({ALL_HTTP3_CODEC_STATS(
+            POOL_COUNTER_PREFIX(mock_listener_config_.listenerScope(), "http3."),
+            POOL_GAUGE_PREFIX(mock_listener_config_.listenerScope(), "http3."))}),
+        crypto_config_(quic::QuicCryptoServerConfig::TESTING, quic::QuicRandom::GetInstance(),
+                       std::make_unique(), quic::KeyExchangeSource::Default()),
+        connection_stats_({QUIC_CONNECTION_STATS(
+            POOL_COUNTER_PREFIX(mock_listener_config_.listenerScope(), "quic.connection"))}) {
+    SetQuicFlag(quic_allow_chlo_buffering, false);
+    ON_CALL(writer_, WritePacket(_, _, _, _, _, _))
+        .WillByDefault(testing::Return(quic::WriteResult(quic::WRITE_STATUS_OK, 0)));
+    ON_CALL(http_connection_callbacks_, newStream(_, _))
+        .WillByDefault(Invoke([&](Http::ResponseEncoder&, bool) -> Http::RequestDecoder& {
+          return orphan_request_decoder_;
+        }));
+  }
+
+  void fuzz(const test::common::quic::QuicH3FuzzCase& input) {
+    auto connection_socket = Quic::createConnectionSocket(peer_addr_, self_addr_, nullptr);
+    auto connection = std::make_unique(
+        quic::test::TestConnectionId(), srv_addr_, cli_addr_, *connection_helper_, *alarm_factory_,
+        &writer_, false, quic::ParsedQuicVersionVector{quic_version_}, std::move(connection_socket),
+        generator_);
+
+    auto decrypter = std::make_unique(quic::Perspective::IS_SERVER);
+    auto encrypter = std::make_unique(quic::Perspective::IS_SERVER);
+    connection->InstallDecrypter(quic::EncryptionLevel::ENCRYPTION_FORWARD_SECURE,
+                                 std::move(decrypter));
+    connection->SetEncrypter(quic::EncryptionLevel::ENCRYPTION_FORWARD_SECURE,
+                             std::move(encrypter));
+    connection->SetDefaultEncryptionLevel(quic::EncryptionLevel::ENCRYPTION_FORWARD_SECURE);
+
+    auto stream_info = std::make_unique(
+        dispatcher_->timeSource(),
+        connection->connectionSocket()->connectionInfoProviderSharedPtr());
+    auto session = std::make_unique(
+        quic_config_, quic::ParsedQuicVersionVector{quic_version_}, std::move(connection), nullptr,
+        &crypto_stream_helper_, &crypto_config_, &compressed_certs_cache_, *dispatcher_.get(),
+        quic::kDefaultFlowControlSendWindow * 1.5, quic_stat_names_,
+        mock_listener_config_.listenerScope(), crypto_stream_factory_, std::move(stream_info),
+        connection_stats_);
+    session->Initialize();
+    session->setHeadersWithUnderscoreAction(envoy::config::core::v3::HttpProtocolOptions::ALLOW);
+    session->setHttp3Options(http3_options_);
+    session->setCodecStats(http3_stats_);
+    session->setHttpConnectionCallbacks(http_connection_callbacks_);
+    session->setMaxIncomingHeadersCount(100);
+    session->set_max_inbound_header_list_size(64 * 1024u);
+    setQuicConfigWithDefaultValues(session->config());
+    session->OnConfigNegotiated();
+
+    auto packets = packetizer_.serializePackets(input);
+    for (auto& p : packets) {
+      auto receipt_time = connection_helper_->GetClock()->Now();
+      // We have to make sure that the server only receives the correct
+      // connection ID in all packets.
+      quic::QuicReceivedPacket qrp(p->data(), p->length(), receipt_time, false);
+      session->ProcessUdpPacket(srv_addr_, cli_addr_, qrp);
+    }
+    session->connection()->CloseConnection(quic::QUIC_PEER_GOING_AWAY, "Fuzzer case done",
+                                           quic::ConnectionCloseBehavior::SILENT_CLOSE);
+    packetizer_.reset();
+  }
+
+  const quic::ParsedQuicVersion quic_version_;
+  Api::ApiPtr api_;
+  Event::DispatcherPtr dispatcher_;
+  quic::QuicVersionManager version_manager_;
+  std::unique_ptr connection_helper_;
+  std::unique_ptr alarm_factory_;
+  NiceMock writer_;
+  QuicPacketizer packetizer_;
+
+  Network::Address::InstanceConstSharedPtr peer_addr_;
+  Network::Address::InstanceConstSharedPtr self_addr_;
+  quic::QuicSocketAddress cli_addr_;
+  quic::QuicSocketAddress srv_addr_;
+
+  NiceMock mock_listener_config_;
+  QuicStatNames quic_stat_names_;
+  Http::Http3::CodecStats http3_stats_;
+
+  quic::QuicConfig quic_config_;
+  envoy::config::core::v3::Http3ProtocolOptions http3_options_{};
+  quic::DeterministicConnectionIdGenerator generator_{quic::kQuicDefaultConnectionIdLength};
+  quic::QuicCompressedCertsCache compressed_certs_cache_{100};
+  EnvoyQuicTestCryptoServerStreamFactory crypto_stream_factory_;
+  const quic::QuicCryptoServerConfig crypto_config_;
+  QuicConnectionStats connection_stats_;
+  NiceMock crypto_stream_helper_;
+  Http::MockServerConnectionCallbacks http_connection_callbacks_;
+  NiceMock orphan_request_decoder_;
+};
+
+std::unique_ptr harness;
+static void resetHarness() { harness = nullptr; };
+DEFINE_PROTO_FUZZER(const test::common::quic::QuicH3FuzzCase& input) {
+  if (harness == nullptr) {
+    harness = std::make_unique(quic::CurrentSupportedHttp3Versions()[0]);
+    atexit(resetHarness);
+  }
+  harness->fuzz(input);
+}
+
+} // namespace Quic
+} // namespace Envoy
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS000 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS000
new file mode 100644
index 000000000000..4fece40db250
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS000
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "get"
+            }
+            headers {
+              key: "connection"
+              value: "close"
+            }
+            headers {
+              key: "-retry-on"
+              value: "retriable-4xx"
+            }
+            headers {
+              key: "cache-control"
+              value: "no-cache"
+            }
+            headers {
+              key: "early-data"
+              value: "empty"
+            }
+            headers {
+              key: "http2-settings"
+              value: "empty"
+            }
+            headers {
+              key: "authorization"
+              value: "empty"
+            }
+            headers {
+              key: "x-squash-debug"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS000_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS000_junk
new file mode 100644
index 000000000000..eade4111a5c9
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS000_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\135\000\000\326\301\137\000\202\230\251\057\000\041\352\250\244\111\217\127\204\045\007\101\177\057\000\132\302\246\317\113\036\257\212\260\251\260\303\216\202\254\327\237\077\347\137\107\204\055\065\247\327\057\003\235\051\254\113\040\251\111\252\231\037\204\055\065\247\327\137\105\204\055\065\247\327\057\004\362\262\073\132\064\116\264\205\216\331\277\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\135\000\000\326\301\137\000\202\230\251\057\000\041\352\250\244\111\217\127\204\045\007\101\177\057\000\132\302\246\317\113\036\257\212\260\251\260\303\216\202\254\327\237\077\347\137\107\204\055\065\247\327\057\003\235\051\254\113\040\251\111\252\231\037\204\055\065\247\327\137\105\204\055\065\247\327\057\004\362\262\073\132\064\116\264\205\216\331\277\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS001 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS001
new file mode 100644
index 000000000000..da384bae1cff
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS001
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "head"
+            }
+            headers {
+              key: "proxy-authenticate"
+              value: "empty"
+            }
+            headers {
+              key: "grpc-timeout"
+              value: "empty"
+            }
+            headers {
+              key: "grpc-encoding"
+              value: "empty"
+            }
+            headers {
+              key: "-max-retries"
+              value: "empty"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "chunked"
+            }
+            headers {
+              key: "keep-alive"
+              value: "empty"
+            }
+            headers {
+              key: "proxy-authorization"
+              value: "empty"
+            }
+            headers {
+              key: "early-data"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS001_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS001_junk
new file mode 100644
index 000000000000..8b4bb0e7c34b
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS001_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\217\000\000\327\301\137\000\203\234\243\223\057\006\256\303\371\364\260\355\114\345\251\046\040\322\137\204\055\065\247\327\057\002\232\312\310\262\115\111\117\152\177\204\055\065\247\327\057\003\232\312\310\261\152\041\344\065\123\177\204\055\065\247\327\057\002\132\221\371\132\302\246\303\025\037\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\044\366\325\324\262\177\057\001\352\122\326\260\350\067\162\377\204\055\065\247\327\057\007\256\303\371\364\260\355\114\347\260\336\306\223\036\257\204\055\065\247\327\137\107\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\217\000\000\327\301\137\000\203\234\243\223\057\006\256\303\371\364\260\355\114\345\251\046\040\322\137\204\055\065\247\327\057\002\232\312\310\262\115\111\117\152\177\204\055\065\247\327\057\003\232\312\310\261\152\041\344\065\123\177\204\055\065\247\327\057\002\132\221\371\132\302\246\303\025\037\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\044\366\325\324\262\177\057\001\352\122\326\260\350\067\162\377\204\055\065\247\327\057\007\256\303\371\364\260\355\114\347\260\336\306\223\036\257\204\055\065\247\327\137\107\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS002 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS002
new file mode 100644
index 000000000000..d57448fbe3f7
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS002
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "post"
+            }
+            headers {
+              key: "content-range"
+              value: "empty"
+            }
+            headers {
+              key: "te"
+              value: "trailers"
+            }
+            headers {
+              key: "-retriable-header-names"
+              value: "empty"
+            }
+            headers {
+              key: "grpc-message-type"
+              value: "empty"
+            }
+            headers {
+              key: "access-control-allow-credentials"
+              value: "empty"
+            }
+            headers {
+              key: "-original-method"
+              value: "empty"
+            }
+            headers {
+              key: "-upstream-stream-duration-ms"
+              value: "empty"
+            }
+            headers {
+              key: "-decorator-operation"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS002_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS002_junk
new file mode 100644
index 000000000000..8f7843c8f7a8
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS002_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\230\000\000\326\301\137\000\203\254\350\117\057\002\041\352\111\152\112\326\016\251\213\204\055\065\247\327\042\164\145\206\115\203\065\005\261\037\057\011\132\302\246\303\016\072\012\264\345\034\205\261\152\207\111\121\204\055\065\247\327\057\005\232\312\310\265\045\102\007\061\126\117\252\313\204\055\065\247\327\137\072\204\055\065\247\327\057\005\130\366\032\143\124\072\026\244\251\234\362\177\204\055\065\247\327\057\015\132\332\320\233\012\072\126\102\154\050\351\132\113\154\032\114\172\226\245\037\204\055\065\247\327\057\007\132\102\220\366\015\047\261\143\326\133\006\223\036\257\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\230\000\000\326\301\137\000\203\254\350\117\057\002\041\352\111\152\112\326\016\251\213\204\055\065\247\327\042\164\145\206\115\203\065\005\261\037\057\011\132\302\246\303\016\072\012\264\345\034\205\261\152\207\111\121\204\055\065\247\327\057\005\232\312\310\265\045\102\007\061\126\117\252\313\204\055\065\247\327\137\072\204\055\065\247\327\057\005\130\366\032\143\124\072\026\244\251\234\362\177\204\055\065\247\327\057\015\132\332\320\233\012\072\126\102\154\050\351\132\113\154\032\114\172\226\245\037\204\055\065\247\327\057\007\132\102\220\366\015\047\261\143\326\133\006\223\036\257\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS003 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS003
new file mode 100644
index 000000000000..56a40e5a49d0
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS003
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "put"
+            }
+            headers {
+              key: "host"
+              value: "empty"
+            }
+            headers {
+              key: ":protocol"
+              value: "bytestream"
+            }
+            headers {
+              key: "x-forwarded-proto"
+              value: "empty"
+            }
+            headers {
+              key: "connection"
+              value: "http2-settings"
+            }
+            headers {
+              key: "content-type"
+              value: "application/grpc-web-text"
+            }
+            headers {
+              key: "-upstream-hostname"
+              value: "empty"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "identity"
+            }
+            headers {
+              key: "alt-svc"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS003_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS003_junk
new file mode 100644
index 000000000000..db6bde5e4b81
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS003_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\221\000\000\327\301\137\000\003\160\165\164\053\234\350\117\204\055\065\247\327\057\000\271\135\207\111\310\172\077\207\217\322\112\204\330\121\323\057\006\362\264\247\263\300\354\220\262\055\135\207\111\377\204\055\065\247\327\057\000\041\352\250\244\111\217\127\212\235\051\254\113\040\251\111\252\231\037\137\035\222\035\165\320\142\015\046\075\114\115\145\144\133\301\143\131\045\362\237\057\006\132\332\320\233\012\072\126\234\350\115\103\244\277\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\064\205\251\046\117\257\137\104\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\221\000\000\327\301\137\000\003\160\165\164\053\234\350\117\204\055\065\247\327\057\000\271\135\207\111\310\172\077\207\217\322\112\204\330\121\323\057\006\362\264\247\263\300\354\220\262\055\135\207\111\377\204\055\065\247\327\057\000\041\352\250\244\111\217\127\212\235\051\254\113\040\251\111\252\231\037\137\035\222\035\165\320\142\015\046\075\114\115\145\144\133\301\143\131\045\362\237\057\006\132\332\320\233\012\072\126\234\350\115\103\244\277\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\064\205\251\046\117\257\137\104\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS004 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS004
new file mode 100644
index 000000000000..081870144601
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS004
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "delete"
+            }
+            headers {
+              key: "-retry-on"
+              value: "retriable-status-codes"
+            }
+            headers {
+              key: "-downstream-service-cluster"
+              value: "empty"
+            }
+            headers {
+              key: "-upstream-alt-stat-name"
+              value: "empty"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "zstd"
+            }
+            headers {
+              key: "-retry-grpc-on"
+              value: "empty"
+            }
+            headers {
+              key: "-original-url"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS004_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS004_junk
new file mode 100644
index 000000000000..64375410a180
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS004_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\212\000\000\326\301\137\000\204\220\264\025\045\057\000\132\302\246\317\113\036\257\217\260\251\260\303\216\202\254\204\215\066\241\142\036\102\243\057\014\132\103\370\251\011\260\243\245\144\026\316\346\041\126\045\026\241\045\263\204\055\065\247\327\057\011\132\332\320\233\012\072\126\035\011\131\011\032\126\250\164\227\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\203\366\204\311\057\004\132\302\246\317\113\115\145\144\130\365\177\204\055\065\247\327\057\003\130\366\032\143\124\072\026\266\312\077\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\212\000\000\326\301\137\000\204\220\264\025\045\057\000\132\302\246\317\113\036\257\217\260\251\260\303\216\202\254\204\215\066\241\142\036\102\243\057\014\132\103\370\251\011\260\243\245\144\026\316\346\041\126\045\026\241\045\263\204\055\065\247\327\057\011\132\332\320\233\012\072\126\035\011\131\011\032\126\250\164\227\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\203\366\204\311\057\004\132\302\246\317\113\115\145\144\130\365\177\204\055\065\247\327\057\003\130\366\032\143\124\072\026\266\312\077\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS005 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS005
new file mode 100644
index 000000000000..6bc3474bac71
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS005
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "options"
+            }
+            headers {
+              key: "grpc-message"
+              value: "empty"
+            }
+            headers {
+              key: "x-forwarded-proto"
+              value: "empty"
+            }
+            headers {
+              key: "-is-timeout-retry"
+              value: "empty"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "br"
+            }
+            headers {
+              key: "user-agent"
+              value: "Envoy/HC"
+            }
+            headers {
+              key: "cache-control"
+              value: "no-transform"
+            }
+            headers {
+              key: "-upstream-host-address"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS005_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS005_junk
new file mode 100644
index 000000000000..bb98b3d17aa1
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS005_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\201\000\000\327\301\137\000\205\075\151\061\352\107\057\002\232\312\310\265\045\102\007\061\177\204\055\065\247\327\057\006\362\264\247\263\300\354\220\262\055\135\207\111\377\204\055\065\247\327\057\005\130\310\131\046\244\247\265\053\130\124\331\353\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\002\142\162\137\120\207\301\127\163\372\143\035\357\137\025\211\250\353\046\301\324\211\117\145\077\057\011\132\332\320\233\012\072\126\234\350\112\303\222\113\012\204\177\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\201\000\000\327\301\137\000\205\075\151\061\352\107\057\002\232\312\310\265\045\102\007\061\177\204\055\065\247\327\057\006\362\264\247\263\300\354\220\262\055\135\207\111\377\204\055\065\247\327\057\005\130\310\131\046\244\247\265\053\130\124\331\353\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\002\142\162\137\120\207\301\127\163\372\143\035\357\137\025\211\250\353\046\301\324\211\117\145\077\057\011\132\332\320\233\012\072\126\234\350\112\303\222\113\012\204\177\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS006 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS006
new file mode 100644
index 000000000000..c47f1ccf10d1
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS006
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "trace"
+            }
+            headers {
+              key: "-retry-on"
+              value: "connect-failure"
+            }
+            headers {
+              key: "-original-dst-host"
+              value: "empty"
+            }
+            headers {
+              key: "etag"
+              value: "empty"
+            }
+            headers {
+              key: "-immediate-health-check-fail"
+              value: "true"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "compress"
+            }
+            headers {
+              key: ":authority"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS006_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS006_junk
new file mode 100644
index 000000000000..9efef7ef062d
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS006_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\157\000\000\326\301\137\000\204\115\203\041\177\057\000\132\302\246\317\113\036\257\213\041\352\250\244\112\322\214\324\133\141\177\057\006\130\366\032\143\124\072\026\221\011\132\163\241\077\204\055\065\247\327\127\204\055\065\247\327\057\015\130\324\322\131\014\064\225\151\312\072\023\072\304\234\244\352\264\243\065\037\203\115\226\227\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\041\351\256\302\241\037\120\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\157\000\000\326\301\137\000\204\115\203\041\177\057\000\132\302\246\317\113\036\257\213\041\352\250\244\112\322\214\324\133\141\177\057\006\130\366\032\143\124\072\026\221\011\132\163\241\077\204\055\065\247\327\127\204\055\065\247\327\057\015\130\324\322\131\014\064\225\151\312\072\023\072\304\234\244\352\264\243\065\037\203\115\226\227\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\041\351\256\302\241\037\120\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS007 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS007
new file mode 100644
index 000000000000..0c5b5d4175bd
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS007
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "patch"
+            }
+            headers {
+              key: "-external-address"
+              value: "empty"
+            }
+            headers {
+              key: "x-client-trace-id"
+              value: "empty"
+            }
+            headers {
+              key: "date"
+              value: "empty"
+            }
+            headers {
+              key: "content-type"
+              value: "text/plain; charset=UTF-8"
+            }
+            headers {
+              key: "grpc-timeout"
+              value: "empty"
+            }
+            headers {
+              key: "-retry-on"
+              value: "gateway-error"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS007_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS007_junk
new file mode 100644
index 000000000000..b59cede096ee
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS007_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\161\000\000\327\301\137\000\204\254\151\044\377\057\005\130\276\122\133\052\035\013\016\111\054\052\021\204\055\065\247\327\057\005\362\261\050\061\152\112\311\260\144\052\306\223\204\055\065\247\327\126\204\055\065\247\327\137\035\223\111\174\245\212\350\031\252\373\120\223\216\304\025\060\160\337\205\147\277\057\002\232\312\310\262\115\111\117\152\177\204\055\065\247\327\057\000\132\302\246\317\113\036\257\212\230\151\057\201\372\130\266\130\173\077"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\161\000\000\327\301\137\000\204\254\151\044\377\057\005\130\276\122\133\052\035\013\016\111\054\052\021\204\055\065\247\327\057\005\362\261\050\061\152\112\311\260\144\052\306\223\204\055\065\247\327\126\204\055\065\247\327\137\035\223\111\174\245\212\350\031\252\373\120\223\216\304\025\060\160\337\205\147\277\057\002\232\312\310\262\115\111\117\152\177\204\055\065\247\327\057\000\132\302\246\317\113\036\257\212\230\151\057\201\372\130\266\130\173\077"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS008 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS008
new file mode 100644
index 000000000000..caa07dc49bf2
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS008
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "connect"
+            }
+            headers {
+              key: "range"
+              value: "empty"
+            }
+            headers {
+              key: "-internal"
+              value: "true"
+            }
+            headers {
+              key: "access-control-allow-headers"
+              value: "empty"
+            }
+            headers {
+              key: "-cluster"
+              value: "empty"
+            }
+            headers {
+              key: "connection"
+              value: "close"
+            }
+            headers {
+              key: "-downstream-service-node"
+              value: "empty"
+            }
+            headers {
+              key: "-upstream-rq-timeout-alt-response"
+              value: "empty"
+            }
+            headers {
+              key: "-expected-rq-timeout-ms"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS008_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS008_junk
new file mode 100644
index 000000000000..b123792e72cb
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS008_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\217\000\000\326\301\137\000\205\041\352\250\244\117\137\050\204\055\065\247\327\057\000\130\325\044\266\124\072\077\203\115\226\227\137\022\204\055\065\247\327\056\130\224\132\204\226\317\204\055\065\247\327\057\000\041\352\250\244\111\217\127\204\045\007\101\177\057\012\132\103\370\251\011\260\243\245\144\026\316\346\041\126\250\362\027\204\055\065\247\327\057\020\132\332\320\233\012\072\126\263\262\311\065\045\075\251\130\164\045\153\012\212\317\122\013\204\055\065\247\327\057\012\130\276\153\051\022\131\026\263\262\311\065\045\075\251\132\224\177\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\217\000\000\326\301\137\000\205\041\352\250\244\117\137\050\204\055\065\247\327\057\000\130\325\044\266\124\072\077\203\115\226\227\137\022\204\055\065\247\327\056\130\224\132\204\226\317\204\055\065\247\327\057\000\041\352\250\244\111\217\127\204\045\007\101\177\057\012\132\103\370\251\011\260\243\245\144\026\316\346\041\126\250\362\027\204\055\065\247\327\057\020\132\332\320\233\012\072\126\263\262\311\065\045\075\251\130\164\045\153\012\212\317\122\013\204\055\065\247\327\057\012\130\276\153\051\022\131\026\263\262\311\065\045\075\251\132\224\177\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS009 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS009
new file mode 100644
index 000000000000..1b22fee43732
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS009
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "get"
+            }
+            headers {
+              key: "grpc-status"
+              value: "empty"
+            }
+            headers {
+              key: "location"
+              value: "empty"
+            }
+            headers {
+              key: "x-request-id"
+              value: "empty"
+            }
+            headers {
+              key: "-cluster"
+              value: "empty"
+            }
+            headers {
+              key: "-expected-rq-timeout-ms"
+              value: "empty"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "gzip"
+            }
+            headers {
+              key: "if-modified-since"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS009_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS009_junk
new file mode 100644
index 000000000000..47aabc479e07
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS009_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\152\000\000\327\301\137\000\202\230\251\057\001\232\312\310\262\022\064\332\217\204\055\065\247\327\134\204\055\065\247\327\057\002\362\265\205\355\151\120\225\215\047\204\055\065\247\327\056\130\224\132\204\226\317\204\055\065\247\327\057\012\130\276\153\051\022\131\026\263\262\311\065\045\075\251\132\224\177\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\203\233\331\253\130\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\152\000\000\327\301\137\000\202\230\251\057\001\232\312\310\262\022\064\332\217\204\055\065\247\327\134\204\055\065\247\327\057\002\362\265\205\355\151\120\225\215\047\204\055\065\247\327\056\130\224\132\204\226\317\204\055\065\247\327\057\012\130\276\153\051\022\131\026\263\262\311\065\045\075\251\132\224\177\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\203\233\331\253\130\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS010 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS010
new file mode 100644
index 000000000000..9823cf9649e4
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS010
@@ -0,0 +1,63 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "head"
+            }
+            headers {
+              key: "-retry-on"
+              value: "gateway-error"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "br"
+            }
+            headers {
+              key: "last-modified"
+              value: "empty"
+            }
+            headers {
+              key: "expect"
+              value: "100-continue"
+            }
+            headers {
+              key: "grpc-message"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS010_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS010_junk
new file mode 100644
index 000000000000..90aedd7c13b0
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS010_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\124\000\000\326\301\137\000\203\234\243\223\057\000\132\302\246\317\113\036\257\212\230\151\057\201\372\130\266\130\173\077\057\005\115\203\251\022\226\305\213\121\017\041\252\233\002\142\162\132\204\055\065\247\327\055\057\232\312\104\377\210\010\000\261\017\122\115\125\245\057\002\232\312\310\265\045\102\007\061\177\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\124\000\000\326\301\137\000\203\234\243\223\057\000\132\302\246\317\113\036\257\212\230\151\057\201\372\130\266\130\173\077\057\005\115\203\251\022\226\305\213\121\017\041\252\233\002\142\162\132\204\055\065\247\327\055\057\232\312\104\377\210\010\000\261\017\122\115\125\245\057\002\232\312\310\265\045\102\007\061\177\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS011 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS011
new file mode 100644
index 000000000000..e7fedd95485a
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS011
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "post"
+            }
+            headers {
+              key: "-overloaded"
+              value: "true"
+            }
+            headers {
+              key: "-immediate-health-check-fail"
+              value: "true"
+            }
+            headers {
+              key: "accept-encoding"
+              value: "identity"
+            }
+            headers {
+              key: "www-authenticate"
+              value: "empty"
+            }
+            headers {
+              key: "if-range"
+              value: "empty"
+            }
+            headers {
+              key: "content-type"
+              value: "text/plain"
+            }
+            headers {
+              key: "x-forwarded-host"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS011_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS011_junk
new file mode 100644
index 000000000000..c9dbb29409bd
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS011_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\151\000\000\327\301\137\000\203\254\350\117\057\001\130\375\313\145\007\034\205\223\203\115\226\227\057\015\130\324\322\131\014\064\225\151\312\072\023\072\304\234\244\352\264\243\065\037\203\115\226\227\137\020\206\064\205\251\046\117\257\057\005\361\343\302\303\265\063\226\244\230\203\111\177\204\055\065\247\327\137\112\204\055\065\247\327\365\057\005\362\264\247\263\300\354\220\262\055\071\320\237\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\151\000\000\327\301\137\000\203\254\350\117\057\001\130\375\313\145\007\034\205\223\203\115\226\227\057\015\130\324\322\131\014\064\225\151\312\072\023\072\304\234\244\352\264\243\065\037\203\115\226\227\137\020\206\064\205\251\046\117\257\057\005\361\343\302\303\265\063\226\244\230\203\111\177\204\055\065\247\327\137\112\204\055\065\247\327\365\057\005\362\264\247\263\300\354\220\262\055\071\320\237\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS012 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS012
new file mode 100644
index 000000000000..0c61bb2bc930
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS012
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "put"
+            }
+            headers {
+              key: "content-type"
+              value: "application/grpc-web-text+proto"
+            }
+            headers {
+              key: "-upstream-rq-timeout-alt-response"
+              value: "empty"
+            }
+            headers {
+              key: "-retriable-header-names"
+              value: "empty"
+            }
+            headers {
+              key: "-hedge-on-per-try-timeout"
+              value: "empty"
+            }
+            headers {
+              key: "x-ot-span-context"
+              value: "empty"
+            }
+            headers {
+              key: "-degraded"
+              value: "empty"
+            }
+            headers {
+              key: ":authority"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS012_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS012_junk
new file mode 100644
index 000000000000..a8f64cc02548
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS012_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\231\000\000\326\301\137\000\003\160\165\164\137\035\227\035\165\320\142\015\046\075\114\115\145\144\133\301\143\131\045\362\237\367\135\207\111\377\057\020\132\332\320\233\012\072\126\263\262\311\065\045\075\251\130\164\045\153\012\212\317\122\013\204\055\065\247\327\057\011\132\302\246\303\016\072\012\264\345\034\205\261\152\207\111\121\204\055\065\247\327\057\013\132\162\311\061\126\075\113\126\133\026\115\236\226\111\251\051\355\117\204\055\065\247\327\057\005\362\261\322\262\053\035\113\020\365\044\276\123\204\055\065\247\327\057\000\132\102\315\140\344\054\237\204\055\065\247\327\120\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\231\000\000\326\301\137\000\003\160\165\164\137\035\227\035\165\320\142\015\046\075\114\115\145\144\133\301\143\131\045\362\237\367\135\207\111\377\057\020\132\332\320\233\012\072\126\263\262\311\065\045\075\251\130\164\045\153\012\212\317\122\013\204\055\065\247\327\057\011\132\302\246\303\016\072\012\264\345\034\205\261\152\207\111\121\204\055\065\247\327\057\013\132\162\311\061\126\075\113\126\133\026\115\236\226\111\251\051\355\117\204\055\065\247\327\057\005\362\261\322\262\053\035\113\020\365\044\276\123\204\055\065\247\327\057\000\132\102\315\140\344\054\237\204\055\065\247\327\120\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS013 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS013
new file mode 100644
index 000000000000..92ebcb203347
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS013
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "delete"
+            }
+            headers {
+              key: "content-encoding"
+              value: "zstd"
+            }
+            headers {
+              key: "proxy-connection"
+              value: "empty"
+            }
+            headers {
+              key: "x-forwarded-for"
+              value: "empty"
+            }
+            headers {
+              key: "-retry-on"
+              value: "connect-failure"
+            }
+            headers {
+              key: "content-type"
+              value: "text/event-stream"
+            }
+            headers {
+              key: "-upstream-service-time"
+              value: "empty"
+            }
+            headers {
+              key: "-downstream-service-cluster"
+              value: "empty"
+            }
+            headers {
+              key: "access-control-expose-headers"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS013_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS013_junk
new file mode 100644
index 000000000000..a939c28bce00
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS013_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\207\000\000\327\301\137\000\204\220\264\025\045\137\033\203\366\204\311\057\005\256\303\371\364\261\017\125\105\042\114\172\277\204\055\065\247\327\137\121\204\055\065\247\327\057\000\132\302\246\317\113\036\257\213\041\352\250\244\112\322\214\324\133\141\177\137\035\214\111\174\245\202\367\055\111\131\011\260\243\247\057\011\132\332\320\233\012\072\126\101\154\356\142\025\144\232\222\377\204\055\065\247\327\057\014\132\103\370\251\011\260\243\245\144\026\316\346\041\126\045\026\241\045\263\204\055\065\247\327\137\100\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\207\000\000\327\301\137\000\204\220\264\025\045\137\033\203\366\204\311\057\005\256\303\371\364\261\017\125\105\042\114\172\277\204\055\065\247\327\137\121\204\055\065\247\327\057\000\132\302\246\317\113\036\257\213\041\352\250\244\112\322\214\324\133\141\177\137\035\214\111\174\245\202\367\055\111\131\011\260\243\247\057\011\132\332\320\233\012\072\126\101\154\356\142\025\144\232\222\377\204\055\065\247\327\057\014\132\103\370\251\011\260\243\245\144\026\316\346\041\126\045\026\241\045\263\204\055\065\247\327\137\100\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS014 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS014
new file mode 100644
index 000000000000..985eba5cb0bf
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS014
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "options"
+            }
+            headers {
+              key: "upgrade"
+              value: "h2c"
+            }
+            headers {
+              key: "range"
+              value: "empty"
+            }
+            headers {
+              key: "x-forwarded-for"
+              value: "empty"
+            }
+            headers {
+              key: "x-content-type-options"
+              value: "empty"
+            }
+            headers {
+              key: "content-type"
+              value: "text/event-stream"
+            }
+            headers {
+              key: "server"
+              value: "empty"
+            }
+            headers {
+              key: "-retry-on"
+              value: "retriable-status-codes"
+            }
+            headers {
+              key: "te"
+              value: "trailers"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS014_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS014_junk
new file mode 100644
index 000000000000..a8a7545814f6
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS014_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\143\000\000\326\301\137\000\205\075\151\061\352\107\055\266\271\254\034\205\202\234\104\137\050\204\055\065\247\327\137\121\204\055\065\247\327\137\056\204\055\065\247\327\137\035\214\111\174\245\202\367\055\111\131\011\260\243\247\137\115\204\055\065\247\327\057\000\132\302\246\317\113\036\257\217\260\251\260\303\216\202\254\204\215\066\241\142\036\102\243\042\164\145\206\115\203\065\005\261\037"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\143\000\000\326\301\137\000\205\075\151\061\352\107\055\266\271\254\034\205\202\234\104\137\050\204\055\065\247\327\137\121\204\055\065\247\327\137\056\204\055\065\247\327\137\035\214\111\174\245\202\367\055\111\131\011\260\243\247\137\115\204\055\065\247\327\057\000\132\302\246\317\113\036\257\217\260\251\260\303\216\202\254\204\215\066\241\142\036\102\243\042\164\145\206\115\203\065\005\261\037"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS015 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS015
new file mode 100644
index 000000000000..47421ec9cabb
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS015
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "trace"
+            }
+            headers {
+              key: "-internal"
+              value: "true"
+            }
+            headers {
+              key: "-upstream-rq-timeout-ms"
+              value: "empty"
+            }
+            headers {
+              key: "-attempt-count"
+              value: "empty"
+            }
+            headers {
+              key: "-degraded"
+              value: "empty"
+            }
+            headers {
+              key: "set-cookie"
+              value: "empty"
+            }
+            headers {
+              key: "-upstream-alt-stat-name"
+              value: "empty"
+            }
+            headers {
+              key: "-retry-on"
+              value: "envoy-ratelimited"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS015_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS015_junk
new file mode 100644
index 000000000000..6b8dc9447d65
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS015_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\202\000\000\327\301\137\000\204\115\203\041\177\057\000\130\325\044\266\124\072\077\203\115\226\227\057\012\132\332\320\233\012\072\126\263\262\311\065\045\075\251\132\224\177\204\055\065\247\327\057\003\130\151\111\151\255\053\020\366\324\237\204\055\065\247\327\057\000\132\102\315\140\344\054\237\204\055\065\247\327\136\204\055\065\247\327\057\011\132\332\320\233\012\072\126\035\011\131\011\032\126\250\164\227\204\055\065\247\327\057\000\132\302\246\317\113\036\257\214\055\135\317\351\153\006\222\320\152\114\222\311"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\202\000\000\327\301\137\000\204\115\203\041\177\057\000\130\325\044\266\124\072\077\203\115\226\227\057\012\132\332\320\233\012\072\126\263\262\311\065\045\075\251\132\224\177\204\055\065\247\327\057\003\130\151\111\151\255\053\020\366\324\237\204\055\065\247\327\057\000\132\102\315\140\344\054\237\204\055\065\247\327\136\204\055\065\247\327\057\011\132\332\320\233\012\072\126\035\011\131\011\032\126\250\164\227\204\055\065\247\327\057\000\132\302\246\317\113\036\257\214\055\135\317\351\153\006\222\320\152\114\222\311"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS016 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS016
new file mode 100644
index 000000000000..bd09af7955b1
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS016
@@ -0,0 +1,63 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "patch"
+            }
+            headers {
+              key: "accept"
+              value: "empty"
+            }
+            headers {
+              key: "cookie"
+              value: "empty"
+            }
+            headers {
+              key: "content-type"
+              value: "text/html; charset=UTF-8"
+            }
+            headers {
+              key: "connection"
+              value: "keep-alive"
+            }
+            headers {
+              key: "access-control-request-headers"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS016_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS016_junk
new file mode 100644
index 000000000000..b2c531d3a25d
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS016_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\106\000\000\326\301\137\000\204\254\151\044\377\137\016\204\055\065\247\327\125\204\055\065\247\327\137\035\222\111\174\245\211\323\115\037\152\022\161\330\202\246\016\033\360\254\367\057\000\041\352\250\244\111\217\127\210\352\122\326\260\350\067\162\377\137\101\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\106\000\000\326\301\137\000\204\254\151\044\377\137\016\204\055\065\247\327\125\204\055\065\247\327\137\035\222\111\174\245\211\323\115\037\152\022\161\330\202\246\016\033\360\254\367\057\000\041\352\250\244\111\217\127\210\352\122\326\260\350\067\162\377\137\101\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS017 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS017
new file mode 100644
index 000000000000..2aad202302bd
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS017
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "connect"
+            }
+            headers {
+              key: "content-type"
+              value: "application/x-www-form-urlencoded"
+            }
+            headers {
+              key: "proxy-status"
+              value: "empty"
+            }
+            headers {
+              key: "-upstream-hostname"
+              value: "empty"
+            }
+            headers {
+              key: "-retry-on"
+              value: "refused-stream"
+            }
+            headers {
+              key: "keep-alive"
+              value: "empty"
+            }
+            headers {
+              key: "-ip-tags"
+              value: "empty"
+            }
+            headers {
+              key: "x-forwarded-host"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS017_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS017_junk
new file mode 100644
index 000000000000..1324b694d8b6
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS017_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\163\000\000\327\301\137\000\205\041\352\250\244\117\357\057\002\256\303\371\364\262\022\064\332\217\204\055\065\247\327\057\006\132\332\320\233\012\072\126\234\350\115\103\244\277\204\055\065\247\327\057\000\132\302\246\317\113\036\257\212\260\262\332\202\310\262\023\141\107\117\057\001\352\122\326\260\350\067\162\377\204\055\065\247\327\056\130\325\254\221\314\217\204\055\065\247\327\057\005\362\264\247\263\300\354\220\262\055\071\320\237\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\163\000\000\327\301\137\000\205\041\352\250\244\117\357\057\002\256\303\371\364\262\022\064\332\217\204\055\065\247\327\057\006\132\332\320\233\012\072\126\234\350\115\103\244\277\204\055\065\247\327\057\000\132\302\246\317\113\036\257\212\260\262\332\202\310\262\023\141\107\117\057\001\352\122\326\260\350\067\162\377\204\055\065\247\327\056\130\325\254\221\314\217\204\055\065\247\327\057\005\362\264\247\263\300\354\220\262\055\071\320\237\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS018 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS018
new file mode 100644
index 000000000000..debbafddb6f0
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS018
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "get"
+            }
+            headers {
+              key: "-is-timeout-retry"
+              value: "empty"
+            }
+            headers {
+              key: "content-type"
+              value: "text/plain; charset=UTF-8"
+            }
+            headers {
+              key: "x-forwarded-port"
+              value: "empty"
+            }
+            headers {
+              key: "-retry-on"
+              value: "retriable-4xx"
+            }
+            headers {
+              key: "upgrade"
+              value: "h2c"
+            }
+            headers {
+              key: "location"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS018_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS018_junk
new file mode 100644
index 000000000000..c33d18b6e5ff
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS018_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\150\000\000\326\301\137\000\202\230\251\057\005\130\310\131\046\244\247\265\053\130\124\331\353\204\055\065\247\327\137\035\223\111\174\245\212\350\031\252\373\120\223\216\304\025\060\160\337\205\147\277\057\005\362\264\247\263\300\354\220\262\055\131\354\117\204\055\065\247\327\057\000\132\302\246\317\113\036\257\212\260\251\260\303\216\202\254\327\237\077\055\266\271\254\034\205\202\234\104\134\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\150\000\000\326\301\137\000\202\230\251\057\005\130\310\131\046\244\247\265\053\130\124\331\353\204\055\065\247\327\137\035\223\111\174\245\212\350\031\252\373\120\223\216\304\025\060\160\337\205\147\277\057\005\362\264\247\263\300\354\220\262\055\131\354\117\204\055\065\247\327\057\000\132\302\246\317\113\036\257\212\260\251\260\303\216\202\254\327\237\077\055\266\271\254\034\205\202\234\104\134\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS019 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS019
new file mode 100644
index 000000000000..e96d89d34545
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS019
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "head"
+            }
+            headers {
+              key: "via"
+              value: "empty"
+            }
+            headers {
+              key: "-retriable-status-codes"
+              value: "empty"
+            }
+            headers {
+              key: "grpc-accept-encoding"
+              value: "identity"
+            }
+            headers {
+              key: "content-type"
+              value: "application/grpc-web-text"
+            }
+            headers {
+              key: "connection"
+              value: "upgrade"
+            }
+            headers {
+              key: "proxy-status"
+              value: "empty"
+            }
+            headers {
+              key: ":status"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS019_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS019_junk
new file mode 100644
index 000000000000..dc0f62bc0b91
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS019_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\174\000\000\327\301\137\000\203\234\243\223\043\166\151\141\204\055\065\247\327\057\011\132\302\246\303\016\072\012\262\022\064\332\205\210\171\012\217\204\055\065\247\327\057\007\232\312\310\260\310\102\326\225\213\121\017\041\252\233\206\064\205\251\046\117\257\137\035\222\035\165\320\142\015\046\075\114\115\145\144\133\301\143\131\045\362\237\057\000\041\352\250\244\111\217\127\205\266\271\254\034\205\057\002\256\303\371\364\262\022\064\332\217\204\055\065\247\327\137\011\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\174\000\000\327\301\137\000\203\234\243\223\043\166\151\141\204\055\065\247\327\057\011\132\302\246\303\016\072\012\262\022\064\332\205\210\171\012\217\204\055\065\247\327\057\007\232\312\310\260\310\102\326\225\213\121\017\041\252\233\206\064\205\251\046\117\257\137\035\222\035\165\320\142\015\046\075\114\115\145\144\133\301\143\131\045\362\237\057\000\041\352\250\244\111\217\127\205\266\271\254\034\205\057\002\256\303\371\364\262\022\064\332\217\204\055\065\247\327\137\011\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS020 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS020
new file mode 100644
index 000000000000..f9b706d1f1ff
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS020
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "post"
+            }
+            headers {
+              key: "-retry-on"
+              value: "retriable-headers"
+            }
+            headers {
+              key: "set-cookie"
+              value: "empty"
+            }
+            headers {
+              key: "content-type"
+              value: "application/grpc-web+proto"
+            }
+            headers {
+              key: "-external-address"
+              value: "empty"
+            }
+            headers {
+              key: "-original-method"
+              value: "empty"
+            }
+            headers {
+              key: "access-control-max-age"
+              value: "empty"
+            }
+            headers {
+              key: "-attempt-count"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS020_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS020_junk
new file mode 100644
index 000000000000..e745a7b45e49
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS020_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\211\000\000\326\301\137\000\203\254\350\117\057\000\132\302\246\317\113\036\257\214\260\251\260\303\216\202\255\071\107\041\154\107\136\204\055\065\247\327\137\035\223\035\165\320\142\015\046\075\114\115\145\144\133\301\143\377\165\330\164\237\057\005\130\276\122\133\052\035\013\016\111\054\052\021\204\055\065\247\327\057\005\130\366\032\143\124\072\026\244\251\234\362\177\204\055\065\247\327\057\010\031\010\124\041\142\036\244\330\172\026\244\176\126\034\305\204\055\065\247\327\057\003\130\151\111\151\255\053\020\366\324\237\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\211\000\000\326\301\137\000\203\254\350\117\057\000\132\302\246\317\113\036\257\214\260\251\260\303\216\202\255\071\107\041\154\107\136\204\055\065\247\327\137\035\223\035\165\320\142\015\046\075\114\115\145\144\133\301\143\377\165\330\164\237\057\005\130\276\122\133\052\035\013\016\111\054\052\021\204\055\065\247\327\057\005\130\366\032\143\124\072\026\244\251\234\362\177\204\055\065\247\327\057\010\031\010\124\041\142\036\244\330\172\026\244\176\126\034\305\204\055\065\247\327\057\003\130\151\111\151\255\053\020\366\324\237\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS021 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS021
new file mode 100644
index 000000000000..cfb6e5684b0b
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS021
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "put"
+            }
+            headers {
+              key: "content-type"
+              value: "application/grpc-web"
+            }
+            headers {
+              key: "x-squash-debug"
+              value: "empty"
+            }
+            headers {
+              key: "-decorator-operation"
+              value: "empty"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "chunked"
+            }
+            headers {
+              key: "-upstream-canary"
+              value: "empty"
+            }
+            headers {
+              key: "upgrade"
+              value: "websocket"
+            }
+            headers {
+              key: "grpc-status-details-bin"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS021_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS021_junk
new file mode 100644
index 000000000000..5b88de28cb1f
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS021_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\217\000\000\327\301\137\000\003\160\165\164\137\035\216\035\165\320\142\015\046\075\114\115\145\144\133\301\143\057\004\362\262\073\132\064\116\264\205\216\331\277\204\055\065\247\327\057\007\132\102\220\366\015\047\261\143\326\133\006\223\036\257\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\044\366\325\324\262\177\057\005\132\332\320\233\012\072\126\040\352\035\236\277\204\055\065\247\327\055\266\271\254\034\205\207\360\130\320\162\165\052\177\057\011\232\312\310\262\022\064\332\205\244\052\106\152\020\264\146\253\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\217\000\000\327\301\137\000\003\160\165\164\137\035\216\035\165\320\142\015\046\075\114\115\145\144\133\301\143\057\004\362\262\073\132\064\116\264\205\216\331\277\204\055\065\247\327\057\007\132\102\220\366\015\047\261\143\326\133\006\223\036\257\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\044\366\325\324\262\177\057\005\132\332\320\233\012\072\126\040\352\035\236\277\204\055\065\247\327\055\266\271\254\034\205\207\360\130\320\162\165\052\177\057\011\232\312\310\262\022\064\332\205\244\052\106\152\020\264\146\253\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS022 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS022
new file mode 100644
index 000000000000..fc49842e6965
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS022
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "delete"
+            }
+            headers {
+              key: "www-authenticate"
+              value: "empty"
+            }
+            headers {
+              key: "user-agent"
+              value: "Envoy/HC"
+            }
+            headers {
+              key: "server"
+              value: "empty"
+            }
+            headers {
+              key: "cookie"
+              value: "empty"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "gzip"
+            }
+            headers {
+              key: "-original-path"
+              value: "empty"
+            }
+            headers {
+              key: "pragma"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS022_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS022_junk
new file mode 100644
index 000000000000..46f91d01c629
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS022_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\143\000\000\326\301\137\000\204\220\264\025\045\057\005\361\343\302\303\265\063\226\244\230\203\111\177\204\055\065\247\327\137\120\207\301\127\163\372\143\035\357\137\115\204\055\065\247\327\125\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\203\233\331\253\057\003\130\366\032\143\124\072\026\254\151\237\204\055\065\247\327\055\256\301\315\110\377\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\143\000\000\326\301\137\000\204\220\264\025\045\057\005\361\343\302\303\265\063\226\244\230\203\111\177\204\055\065\247\327\137\120\207\301\127\163\372\143\035\357\137\115\204\055\065\247\327\125\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\203\233\331\253\057\003\130\366\032\143\124\072\026\254\151\237\204\055\065\247\327\055\256\301\315\110\377\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS023 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS023
new file mode 100644
index 000000000000..22635b30df58
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS023
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "options"
+            }
+            headers {
+              key: "-upstream-service-time"
+              value: "empty"
+            }
+            headers {
+              key: "authentication"
+              value: "empty"
+            }
+            headers {
+              key: "x-client-trace-id"
+              value: "empty"
+            }
+            headers {
+              key: "access-control-allow-methods"
+              value: "empty"
+            }
+            headers {
+              key: "cache-control"
+              value: "private"
+            }
+            headers {
+              key: "-original-url"
+              value: "empty"
+            }
+            headers {
+              key: "-not-forwarded"
+              value: "empty"
+            }
+            headers {
+              key: "grpc-status"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS023_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS023_junk
new file mode 100644
index 000000000000..df4b97d18921
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS023_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\207\000\000\327\301\137\000\205\075\151\061\352\107\057\011\132\332\320\233\012\072\126\101\154\356\142\025\144\232\222\377\204\055\065\247\327\057\003\035\251\234\265\044\304\032\114\172\277\204\055\065\247\327\057\005\362\261\050\061\152\112\311\260\144\052\306\223\204\055\065\247\327\137\075\204\055\065\247\327\137\025\205\256\303\167\032\113\057\003\130\366\032\143\124\072\026\266\312\077\204\055\065\247\327\057\003\132\243\245\151\117\147\201\331\041\144\204\055\065\247\327\057\001\232\312\310\262\022\064\332\217\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\207\000\000\327\301\137\000\205\075\151\061\352\107\057\011\132\332\320\233\012\072\126\101\154\356\142\025\144\232\222\377\204\055\065\247\327\057\003\035\251\234\265\044\304\032\114\172\277\204\055\065\247\327\057\005\362\261\050\061\152\112\311\260\144\052\306\223\204\055\065\247\327\137\075\204\055\065\247\327\137\025\205\256\303\167\032\113\057\003\130\366\032\143\124\072\026\266\312\077\204\055\065\247\327\057\003\132\243\245\151\117\147\201\331\041\144\204\055\065\247\327\057\001\232\312\310\262\022\064\332\217\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS024 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS024
new file mode 100644
index 000000000000..994989ced99a
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS024
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "trace"
+            }
+            headers {
+              key: "access-control-request-private-network"
+              value: "empty"
+            }
+            headers {
+              key: "cdn-loop"
+              value: "empty"
+            }
+            headers {
+              key: "proxy-authenticate"
+              value: "empty"
+            }
+            headers {
+              key: "-force-trace"
+              value: "empty"
+            }
+            headers {
+              key: "content-type"
+              value: "application/grpc-web+proto"
+            }
+            headers {
+              key: "-upstream-healthchecked-cluster"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS024_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS024_junk
new file mode 100644
index 000000000000..e766d838d737
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS024_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\220\000\000\326\301\137\000\204\115\203\041\177\057\024\031\010\124\041\142\036\244\330\172\026\260\275\255\052\022\265\166\033\270\322\125\252\052\174\036\316\277\204\055\065\247\327\056\044\225\055\101\317\137\204\055\065\247\327\057\006\256\303\371\364\260\355\114\345\251\046\040\322\137\204\055\065\247\327\057\002\132\123\330\102\254\233\006\102\377\204\055\065\247\327\137\035\223\035\165\320\142\015\046\075\114\115\145\144\133\301\143\377\165\330\164\237\057\017\132\332\320\233\012\072\126\234\243\241\063\222\162\223\251\144\130\224\132\204\226\317\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\220\000\000\326\301\137\000\204\115\203\041\177\057\024\031\010\124\041\142\036\244\330\172\026\260\275\255\052\022\265\166\033\270\322\125\252\052\174\036\316\277\204\055\065\247\327\056\044\225\055\101\317\137\204\055\065\247\327\057\006\256\303\371\364\260\355\114\345\251\046\040\322\137\204\055\065\247\327\057\002\132\123\330\102\254\233\006\102\377\204\055\065\247\327\137\035\223\035\165\320\142\015\046\075\114\115\145\144\133\301\143\377\165\330\164\237\057\017\132\332\320\233\012\072\126\234\243\241\063\222\162\223\251\144\130\224\132\204\226\317\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS025 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS025
new file mode 100644
index 000000000000..c80fa46a1152
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS025
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "patch"
+            }
+            headers {
+              key: "-ratelimited"
+              value: "true"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "compress"
+            }
+            headers {
+              key: "connection"
+              value: "keep-alive"
+            }
+            headers {
+              key: "x-forwarded-port"
+              value: "empty"
+            }
+            headers {
+              key: "-downstream-service-node"
+              value: "empty"
+            }
+            headers {
+              key: "vary"
+              value: "Accept-Encoding"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS025_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS025_junk
new file mode 100644
index 000000000000..0af8b5b86f11
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS025_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\172\000\000\327\301\137\000\204\254\151\044\377\057\002\132\301\244\264\032\223\044\262\177\203\115\226\227\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\041\351\256\302\241\037\057\000\041\352\250\244\111\217\127\210\352\122\326\260\350\067\162\377\057\005\362\264\247\263\300\354\220\262\055\131\354\117\204\055\065\247\327\057\012\132\103\370\251\011\260\243\245\144\026\316\346\041\126\250\362\027\204\055\065\247\327\137\054\213\204\204\055\151\133\005\104\074\206\252\157"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\172\000\000\327\301\137\000\204\254\151\044\377\057\002\132\301\244\264\032\223\044\262\177\203\115\226\227\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\041\351\256\302\241\037\057\000\041\352\250\244\111\217\127\210\352\122\326\260\350\067\162\377\057\005\362\264\247\263\300\354\220\262\055\131\354\117\204\055\065\247\327\057\012\132\103\370\251\011\260\243\245\144\026\316\346\041\126\250\362\027\204\055\065\247\327\137\054\213\204\204\055\151\133\005\104\074\206\252\157"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS026 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS026
new file mode 100644
index 000000000000..1387f05a4ade
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS026
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "connect"
+            }
+            headers {
+              key: "content-type"
+              value: "application/json"
+            }
+            headers {
+              key: "connection"
+              value: "upgrade"
+            }
+            headers {
+              key: "-upstream-canary"
+              value: "empty"
+            }
+            headers {
+              key: "if-match"
+              value: "empty"
+            }
+            headers {
+              key: "host"
+              value: "empty"
+            }
+            headers {
+              key: "proxy-authorization"
+              value: "empty"
+            }
+            headers {
+              key: "-retry-on"
+              value: "5xx"
+            }
+            headers {
+              key: "-upstream-healthchecked-cluster"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS026_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS026_junk
new file mode 100644
index 000000000000..aea6ec1a19ef
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS026_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\203\000\000\326\301\137\000\205\041\352\250\244\117\356\057\000\041\352\250\244\111\217\127\205\266\271\254\034\205\057\005\132\332\320\233\012\072\126\040\352\035\236\277\204\055\065\247\327\056\064\253\122\064\222\177\204\055\065\247\327\053\234\350\117\204\055\065\247\327\057\007\256\303\371\364\260\355\114\347\260\336\306\223\036\257\204\055\065\247\327\057\000\132\302\246\317\113\036\257\003\065\170\170\057\017\132\332\320\233\012\072\126\234\243\241\063\222\162\223\251\144\130\224\132\204\226\317\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\203\000\000\326\301\137\000\205\041\352\250\244\117\356\057\000\041\352\250\244\111\217\127\205\266\271\254\034\205\057\005\132\332\320\233\012\072\126\040\352\035\236\277\204\055\065\247\327\056\064\253\122\064\222\177\204\055\065\247\327\053\234\350\117\204\055\065\247\327\057\007\256\303\371\364\260\355\114\347\260\336\306\223\036\257\204\055\065\247\327\057\000\132\302\246\317\113\036\257\003\065\170\170\057\017\132\332\320\233\012\072\126\234\243\241\063\222\162\223\251\144\130\224\132\204\226\317\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS027 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS027
new file mode 100644
index 000000000000..8d39e7a92884
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS027
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "get"
+            }
+            headers {
+              key: "-hedge-on-per-try-timeout"
+              value: "empty"
+            }
+            headers {
+              key: "referer"
+              value: "empty"
+            }
+            headers {
+              key: "content-encoding"
+              value: "br"
+            }
+            headers {
+              key: "vary"
+              value: "*"
+            }
+            headers {
+              key: "date"
+              value: "empty"
+            }
+            headers {
+              key: "access-control-allow-private-network"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS027_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS027_junk
new file mode 100644
index 000000000000..09eda44b252e
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS027_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\124\000\000\327\301\137\000\202\230\251\057\013\132\162\311\061\126\075\113\126\133\026\115\236\226\111\251\051\355\117\204\055\065\247\327\135\204\055\065\247\327\352\137\054\001\052\126\204\055\065\247\327\057\023\031\010\124\041\142\036\244\330\172\026\035\024\037\302\325\330\156\343\111\126\250\251\360\173\072\377\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\124\000\000\327\301\137\000\202\230\251\057\013\132\162\311\061\126\075\113\126\133\026\115\236\226\111\251\051\355\117\204\055\065\247\327\135\204\055\065\247\327\352\137\054\001\052\126\204\055\065\247\327\057\023\031\010\124\041\142\036\244\330\172\026\035\024\037\302\325\330\156\343\111\126\250\251\360\173\072\377\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS028 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS028
new file mode 100644
index 000000000000..9acd250d4170
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS028
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "head"
+            }
+            headers {
+              key: "if-unmodified-since"
+              value: "empty"
+            }
+            headers {
+              key: "access-control-request-method"
+              value: "empty"
+            }
+            headers {
+              key: "content-type"
+              value: "application/grpc-web"
+            }
+            headers {
+              key: "cache-control"
+              value: "no-cache, max-age=0"
+            }
+            headers {
+              key: "-upstream-stream-duration-ms"
+              value: "empty"
+            }
+            headers {
+              key: "-ratelimited"
+              value: "true"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS028_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS028_junk
new file mode 100644
index 000000000000..584f090978ce
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS028_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\162\000\000\326\301\137\000\203\234\243\223\057\007\064\253\133\125\047\220\322\230\262\054\203\124\102\377\204\055\065\247\327\137\102\204\055\065\247\327\137\035\216\035\165\320\142\015\046\075\114\115\145\144\133\301\143\137\025\216\250\353\020\144\234\277\112\122\077\053\016\142\300\017\057\015\132\332\320\233\012\072\126\102\154\050\351\132\113\154\032\114\172\226\245\037\204\055\065\247\327\057\002\132\301\244\264\032\223\044\262\177\203\115\226\227"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\162\000\000\326\301\137\000\203\234\243\223\057\007\064\253\133\125\047\220\322\230\262\054\203\124\102\377\204\055\065\247\327\137\102\204\055\065\247\327\137\035\216\035\165\320\142\015\046\075\114\115\145\144\133\301\143\137\025\216\250\353\020\144\234\277\112\122\077\053\016\142\300\017\057\015\132\332\320\233\012\072\126\102\154\050\351\132\113\154\032\114\172\226\245\037\204\055\065\247\327\057\002\132\301\244\264\032\223\044\262\177\203\115\226\227"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS029 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS029
new file mode 100644
index 000000000000..cba46aacaac6
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS029
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "post"
+            }
+            headers {
+              key: "-force-trace"
+              value: "empty"
+            }
+            headers {
+              key: "access-control-allow-origin"
+              value: "empty"
+            }
+            headers {
+              key: "-retry-on"
+              value: "retriable-headers"
+            }
+            headers {
+              key: "x-forwarded-client-cert"
+              value: "empty"
+            }
+            headers {
+              key: "accept-encoding"
+              value: "*"
+            }
+            headers {
+              key: "-upstream-rq-per-try-timeout-ms"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS029_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS029_junk
new file mode 100644
index 000000000000..74505efb8506
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS029_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\160\000\000\327\301\137\000\203\254\350\117\057\002\132\123\330\102\254\233\006\102\377\204\055\065\247\327\137\024\204\055\065\247\327\057\000\132\302\246\317\113\036\257\214\260\251\260\303\216\202\255\071\107\041\154\107\057\012\362\264\247\263\300\354\220\262\054\112\014\132\222\261\013\142\177\204\055\065\247\327\137\020\001\052\057\017\132\332\320\233\012\072\126\263\262\325\226\305\223\147\245\222\152\112\173\122\265\050\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\160\000\000\327\301\137\000\203\254\350\117\057\002\132\123\330\102\254\233\006\102\377\204\055\065\247\327\137\024\204\055\065\247\327\057\000\132\302\246\317\113\036\257\214\260\251\260\303\216\202\255\071\107\041\154\107\057\012\362\264\247\263\300\354\220\262\054\112\014\132\222\261\013\142\177\204\055\065\247\327\137\020\001\052\057\017\132\332\320\233\012\072\126\263\262\325\226\305\223\147\245\222\152\112\173\122\265\050\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS030 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS030
new file mode 100644
index 000000000000..8760a3c4992d
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS030
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "put"
+            }
+            headers {
+              key: "proxy-connection"
+              value: "empty"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "identity"
+            }
+            headers {
+              key: "-retry-grpc-on"
+              value: "empty"
+            }
+            headers {
+              key: "-upstream-rq-timeout-ms"
+              value: "empty"
+            }
+            headers {
+              key: "upgrade"
+              value: "websocket"
+            }
+            headers {
+              key: "-original-dst-host"
+              value: "empty"
+            }
+            headers {
+              key: "-retriable-status-codes"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS030_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS030_junk
new file mode 100644
index 000000000000..b869e1e950d6
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS030_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\225\000\000\326\301\137\000\003\160\165\164\057\005\256\303\371\364\261\017\125\105\042\114\172\277\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\064\205\251\046\117\257\057\004\132\302\246\317\113\115\145\144\130\365\177\204\055\065\247\327\057\012\132\332\320\233\012\072\126\263\262\311\065\045\075\251\132\224\177\204\055\065\247\327\055\266\271\254\034\205\207\360\130\320\162\165\052\177\057\006\130\366\032\143\124\072\026\221\011\132\163\241\077\204\055\065\247\327\057\011\132\302\246\303\016\072\012\262\022\064\332\205\210\171\012\217\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\225\000\000\326\301\137\000\003\160\165\164\057\005\256\303\371\364\261\017\125\105\042\114\172\277\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\206\064\205\251\046\117\257\057\004\132\302\246\317\113\115\145\144\130\365\177\204\055\065\247\327\057\012\132\332\320\233\012\072\126\263\262\311\065\045\075\251\132\224\177\204\055\065\247\327\055\266\271\254\034\205\207\360\130\320\162\165\052\177\057\006\130\366\032\143\124\072\026\221\011\132\163\241\077\204\055\065\247\327\057\011\132\302\246\303\016\072\012\262\022\064\332\205\210\171\012\217\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS031 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS031
new file mode 100644
index 000000000000..c483a90328d4
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS031
@@ -0,0 +1,71 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "delete"
+            }
+            headers {
+              key: "content-type"
+              value: "text/html; charset=UTF-8"
+            }
+            headers {
+              key: "via"
+              value: "empty"
+            }
+            headers {
+              key: "expires"
+              value: "empty"
+            }
+            headers {
+              key: "content-range"
+              value: "empty"
+            }
+            headers {
+              key: "expect"
+              value: "100-continue"
+            }
+            headers {
+              key: "-overloaded"
+              value: "true"
+            }
+            headers {
+              key: "-original-path"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS031_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS031_junk
new file mode 100644
index 000000000000..cb22c7297195
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS031_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\162\000\000\327\301\137\000\204\220\264\025\045\137\035\222\111\174\245\211\323\115\037\152\022\161\330\202\246\016\033\360\254\367\043\166\151\141\204\055\065\247\327\055\057\232\315\141\121\204\055\065\247\327\057\002\041\352\111\152\112\326\016\251\213\204\055\065\247\327\055\057\232\312\104\377\210\010\000\261\017\122\115\125\245\057\001\130\375\313\145\007\034\205\223\203\115\226\227\057\003\130\366\032\143\124\072\026\254\151\237\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\162\000\000\327\301\137\000\204\220\264\025\045\137\035\222\111\174\245\211\323\115\037\152\022\161\330\202\246\016\033\360\254\367\043\166\151\141\204\055\065\247\327\055\057\232\315\141\121\204\055\065\247\327\057\002\041\352\111\152\112\326\016\251\213\204\055\065\247\327\055\057\232\312\104\377\210\010\000\261\017\122\115\125\245\057\001\130\375\313\145\007\034\205\223\203\115\226\227\057\003\130\366\032\143\124\072\026\254\151\237\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS032 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS032
new file mode 100644
index 000000000000..4baa0743ca40
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS032
@@ -0,0 +1,67 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "options"
+            }
+            headers {
+              key: "-retry-on"
+              value: "5xx"
+            }
+            headers {
+              key: "if-none-match"
+              value: "empty"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "deflate"
+            }
+            headers {
+              key: "age"
+              value: "empty"
+            }
+            headers {
+              key: "origin"
+              value: "empty"
+            }
+            headers {
+              key: "-upstream-rq-per-try-timeout-ms"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS032_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS032_junk
new file mode 100644
index 000000000000..9a9476d70860
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS032_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\135\000\000\326\301\137\000\205\075\151\061\352\107\057\000\132\302\246\317\113\036\257\003\065\170\170\131\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\205\220\262\320\064\227\122\204\055\065\247\327\137\113\204\055\065\247\327\057\017\132\332\320\233\012\072\126\263\262\325\226\305\223\147\245\222\152\112\173\122\265\050\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\135\000\000\326\301\137\000\205\075\151\061\352\107\057\000\132\302\246\317\113\036\257\003\065\170\170\131\204\055\065\247\327\057\005\115\203\251\022\226\305\213\121\017\041\252\233\205\220\262\320\064\227\122\204\055\065\247\327\137\113\204\055\065\247\327\057\017\132\332\320\233\012\072\126\263\262\325\226\305\223\147\245\222\152\112\173\122\265\050\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS033 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS033
new file mode 100644
index 000000000000..dba2c5ff4951
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS033
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "https"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "trace"
+            }
+            headers {
+              key: "x-request-id"
+              value: "empty"
+            }
+            headers {
+              key: ":protocol"
+              value: "bytestream"
+            }
+            headers {
+              key: "grpc-status-details-bin"
+              value: "empty"
+            }
+            headers {
+              key: "content-encoding"
+              value: "gzip"
+            }
+            headers {
+              key: "-not-forwarded"
+              value: "empty"
+            }
+            headers {
+              key: "-upstream-host-address"
+              value: "empty"
+            }
+            headers {
+              key: "-retry-on"
+              value: "reset"
+            }
+            headers {
+              key: "http2-settings"
+              value: "empty"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS033_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS033_junk
new file mode 100644
index 000000000000..d76eea1db558
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS033_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\213\000\000\327\301\137\000\204\115\203\041\177\057\002\362\265\205\355\151\120\225\215\047\204\055\065\247\327\057\000\271\135\207\111\310\172\077\207\217\322\112\204\330\121\323\057\011\232\312\310\262\022\064\332\205\244\052\106\152\020\264\146\253\204\055\065\247\327\353\057\003\132\243\245\151\117\147\201\331\041\144\204\055\065\247\327\057\011\132\332\320\233\012\072\126\234\350\112\303\222\113\012\204\177\204\055\065\247\327\057\000\132\302\246\317\113\036\257\204\260\250\052\177\057\003\235\051\254\113\040\251\111\252\231\037\204\055\065\247\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\213\000\000\327\301\137\000\204\115\203\041\177\057\002\362\265\205\355\151\120\225\215\047\204\055\065\247\327\057\000\271\135\207\111\310\172\077\207\217\322\112\204\330\121\323\057\011\232\312\310\262\022\064\332\205\244\052\106\152\020\264\146\253\204\055\065\247\327\353\057\003\132\243\245\151\117\147\201\331\041\144\204\055\065\247\327\057\011\132\332\320\233\012\072\126\234\350\112\303\222\113\012\204\177\204\055\065\247\327\057\000\132\302\246\317\113\036\257\204\260\250\052\177\057\003\235\051\254\113\040\251\111\252\231\037\204\055\065\247\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS034 b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS034
new file mode 100644
index 000000000000..21e2b95a9126
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS034
@@ -0,0 +1,75 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+        h3frame {
+          settings {}
+        }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      type: 0,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":scheme"
+              value: "http"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":method"
+              value: "patch"
+            }
+            headers {
+              key: "x-content-type-options"
+              value: "empty"
+            }
+            headers {
+              key: "-max-retries"
+              value: "empty"
+            }
+            headers {
+              key: "x-forwarded-client-cert"
+              value: "empty"
+            }
+            headers {
+              key: "content-type"
+              value: "application/x-www-form-urlencoded"
+            }
+            headers {
+              key: "transfer-encoding"
+              value: "deflate"
+            }
+            headers {
+              key: ":status"
+              value: "empty"
+            }
+            headers {
+              key: "-ip-tags"
+              value: "empty"
+            }
+            headers {
+              key: "connection"
+              value: "close"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS034_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS034_junk
new file mode 100644
index 000000000000..9ef23e51e6df
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/HDRS034_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\160\000\000\326\301\137\000\204\254\151\044\377\137\056\204\055\065\247\327\057\002\132\221\371\132\302\246\303\025\037\204\055\065\247\327\057\012\362\264\247\263\300\354\220\262\054\112\014\132\222\261\013\142\177\204\055\065\247\327\357\057\005\115\203\251\022\226\305\213\121\017\041\252\233\205\220\262\320\064\227\137\011\204\055\065\247\327\056\130\325\254\221\314\217\204\055\065\247\327\057\000\041\352\250\244\111\217\127\204\045\007\101\177"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\100\160\000\000\326\301\137\000\204\254\151\044\377\137\056\204\055\065\247\327\057\002\132\221\371\132\302\246\303\025\037\204\055\065\247\327\057\012\362\264\247\263\300\354\220\262\054\112\014\132\222\261\013\142\177\204\055\065\247\327\357\057\005\115\203\251\022\226\305\213\121\017\041\252\233\205\220\262\320\064\227\137\011\204\055\065\247\327\056\130\325\254\221\314\217\204\055\065\247\327\057\000\041\352\250\244\111\217\127\204\045\007\101\177"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/h3get b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/h3get
new file mode 100644
index 000000000000..f6606ca6dd60
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/h3get
@@ -0,0 +1,43 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 2,
+      fin: false,
+      offset: 0,
+      h3frame {
+        settings {
+        }
+      }
+    }
+  }
+}
+frames {
+  qframe {
+    stream {
+      unidirectional: false,
+      id: 4,
+      fin: false,
+      offset: 0,
+      h3frame {
+        headers {
+          headers {
+            headers {
+              key: ":method"
+              value: "get"
+            }
+            headers {
+              key: ":path"
+              value: "/"
+            }
+            headers {
+              key: ":scheme",
+              value: "https",
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/h3get_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/h3get_junk
new file mode 100644
index 000000000000..20fb144406c9
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/h3get_junk
@@ -0,0 +1,12 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\000\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\011\000\000\137\000\202\230\251\301\327"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\002\004\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\001\010\004\001\011\000\000\137\000\202\230\251\301\327"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack
new file mode 100644
index 000000000000..d6e11eedf16e
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack
@@ -0,0 +1,12 @@
+frames {
+  qframe {
+    ack {
+      largest_acked: 1000,
+      ecn_counters: {
+        ect0: 0,
+        ect1: 0,
+        ce: 0,
+      }
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_freq b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_freq
new file mode 100644
index 000000000000..2db5157c90ff
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_freq
@@ -0,0 +1,10 @@
+frames {
+  qframe {
+    ack_frequency {
+      control_frame_id: 0,
+      sequence_number: 1,
+      packet_tolerance: 1,
+      milliseconds: 100,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_freq_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_freq_junk
new file mode 100644
index 000000000000..5423dcce7e22
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_freq_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\100\257\001\001\200\001\206\240\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\100\257\001\001\200\001\206\240\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_junk
new file mode 100644
index 000000000000..2da59294d7c2
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ack_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\003\103\350\377\377\377\377\377\377\377\377\000\000\000\000\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\003\103\350\377\377\377\377\377\377\377\377\000\000\000\000\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_blocked b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_blocked
new file mode 100644
index 000000000000..fcba8bffe3ce
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_blocked
@@ -0,0 +1,9 @@
+frames {
+  qframe {
+    blocked {
+      control_frame_id: 0,
+      stream_id: 0,
+      offset: 100,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_blocked_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_blocked_junk
new file mode 100644
index 000000000000..cb915041c442
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_blocked_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\025\000\100\144"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\025\000\100\144"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_connection_close b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_connection_close
new file mode 100644
index 000000000000..2206c7c744e4
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_connection_close
@@ -0,0 +1,10 @@
+frames {
+  qframe {
+    connection_close {
+      error_code: 0,
+      ietf_error: 0,
+      error_phrase: "oops",
+      transport_close_frame_type: 0,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_connection_close_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_connection_close_junk
new file mode 100644
index 000000000000..ff08b7fb9999
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_connection_close_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\034\000\000\006\060\072\157\157\160\163"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\034\000\000\006\060\072\157\157\160\163"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_crypto b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_crypto
new file mode 100644
index 000000000000..bb256a876e47
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_crypto
@@ -0,0 +1,8 @@
+frames {
+  qframe {
+    crypto {
+      offset: 0,
+      data: "\000"
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_crypto_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_crypto_junk
new file mode 100644
index 000000000000..f738c1ff9a31
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_crypto_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\006\000\001\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\006\000\001\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_handshake_done b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_handshake_done
new file mode 100644
index 000000000000..fa6722c6c28e
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_handshake_done
@@ -0,0 +1,7 @@
+frames {
+  qframe {
+    handshake_done {
+      control_frame_id: 0,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_handshake_done_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_handshake_done_junk
new file mode 100644
index 000000000000..ce389fe570cb
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_handshake_done_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\036"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\036"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_max_streams b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_max_streams
new file mode 100644
index 000000000000..a80e269ffbc8
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_max_streams
@@ -0,0 +1,9 @@
+frames {
+  qframe {
+    max_streams {
+      control_frame_id: 0,
+      stream_count: 0,
+      unidirectional: true,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_max_streams_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_max_streams_junk
new file mode 100644
index 000000000000..762110383c01
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_max_streams_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\023\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\023\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_message b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_message
new file mode 100644
index 000000000000..1548007b256e
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_message
@@ -0,0 +1,8 @@
+frames {
+  qframe {
+    message_frame {
+      message_id: 0,
+      data: "\000",
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_message_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_message_junk
new file mode 100644
index 000000000000..34369c0c66bd
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_message_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\060"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\060"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_mtu_discovery b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_mtu_discovery
new file mode 100644
index 000000000000..4171425f5241
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_mtu_discovery
@@ -0,0 +1,5 @@
+frames {
+  qframe {
+    mtu_discovery {}
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_mtu_discovery_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_mtu_discovery_junk
new file mode 100644
index 000000000000..e8da00b3e73c
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_mtu_discovery_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\001"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\001"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_connection_id b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_connection_id
new file mode 100644
index 000000000000..523113f6a6de
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_connection_id
@@ -0,0 +1,11 @@
+frames {
+  qframe {
+    new_connection_id {
+      control_frame_id: 0,
+      connection_id: "\000\000\000\000",
+      sequence_number: 0,
+      stateless_reset_token: "\000\000\000\000",
+      retire_prior_to: 0,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_connection_id_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_connection_id_junk
new file mode 100644
index 000000000000..15243b273e8e
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_connection_id_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\030\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\030\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_token b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_token
new file mode 100644
index 000000000000..1ab8b8785304
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_token
@@ -0,0 +1,8 @@
+frames {
+  qframe {
+    new_token {
+      control_frame_id: 0,
+      token: "\000\000\000\000",
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_token_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_token_junk
new file mode 100644
index 000000000000..8efce0e31f30
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_new_token_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\007\004\000\000\000\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\007\004\000\000\000\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_padding b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_padding
new file mode 100644
index 000000000000..c55831e4a766
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_padding
@@ -0,0 +1,7 @@
+frames {
+  qframe {
+    padding {
+      num_padding_bytes: 127,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_padding_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_padding_junk
new file mode 100644
index 000000000000..c6b637e34a62
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_padding_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_challenge b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_challenge
new file mode 100644
index 000000000000..3e6be78e8f56
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_challenge
@@ -0,0 +1,8 @@
+frames {
+  qframe {
+    path_challenge {
+      control_frame_id: 0,
+      data: "\000",
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_challenge_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_challenge_junk
new file mode 100644
index 000000000000..df25f57d62a5
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_challenge_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\032\000\000\000\000\000\000\000\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\032\000\000\000\000\000\000\000\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_response b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_response
new file mode 100644
index 000000000000..16535b174232
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_response
@@ -0,0 +1,8 @@
+frames {
+  qframe {
+    path_response {
+      control_frame_id: 0,
+      data: "\000",
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_response_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_response_junk
new file mode 100644
index 000000000000..2aa6ee56b664
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_path_response_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\033\000\000\000\000\000\000\000\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\033\000\000\000\000\000\000\000\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ping b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ping
new file mode 100644
index 000000000000..538238368f0d
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ping
@@ -0,0 +1,7 @@
+frames {
+  qframe {
+    ping {
+      control_frame_id: 0,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ping_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ping_junk
new file mode 100644
index 000000000000..e8da00b3e73c
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_ping_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\001"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\001"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_rst_frame b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_rst_frame
new file mode 100644
index 000000000000..3d00a4f4a5f4
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_rst_frame
@@ -0,0 +1,10 @@
+frames {
+  qframe {
+    rst_stream {
+      control_frame_id: 0,
+      stream_id: 0,
+      error_code: 0,
+      bytes_written: 0,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_rst_frame_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_rst_frame_junk
new file mode 100644
index 000000000000..761b4513c8cc
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_rst_frame_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\004\000\101\000\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\004\000\101\000\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stop_sending b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stop_sending
new file mode 100644
index 000000000000..8f7dc25d952a
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stop_sending
@@ -0,0 +1,9 @@
+frames {
+  qframe {
+    stop_sending {
+      control_frame_id: 0,
+      stream_id: 0,
+      error_code: 0,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stop_sending_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stop_sending_junk
new file mode 100644
index 000000000000..df05dc0b3b85
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stop_sending_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\005\000\101\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\005\000\101\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stream b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stream
new file mode 100644
index 000000000000..d5b5f106a305
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stream
@@ -0,0 +1,12 @@
+frames {
+  qframe {
+    stream {
+      unidirectional: true,
+      type: 0,
+      id: 0,
+      fin: false,
+      offset: 0,
+      junk: "\000"
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stream_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stream_junk
new file mode 100644
index 000000000000..144dc8d1c6a3
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_stream_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\000\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\010\000\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_streams_blocked b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_streams_blocked
new file mode 100644
index 000000000000..d3dc513baef1
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_streams_blocked
@@ -0,0 +1,9 @@
+frames {
+  qframe {
+    streams_blocked {
+      control_frame_id: 0,
+      stream_count: 0,
+      unidirectional: true,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_streams_blocked_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_streams_blocked_junk
new file mode 100644
index 000000000000..4e998060b386
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_streams_blocked_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\027\000"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\027\000"
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_window_update b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_window_update
new file mode 100644
index 000000000000..3454b6a154b8
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_window_update
@@ -0,0 +1,9 @@
+frames {
+  qframe {
+    window_update {
+      control_frame_id: 0,
+      stream_id: 0,
+      max_data: 100,
+    }
+  }
+}
diff --git a/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_window_update_junk b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_window_update_junk
new file mode 100644
index 000000000000..990ad5d1d238
--- /dev/null
+++ b/test/common/quic/envoy_quic_h3_fuzz_test_corpus/quic_window_update_junk
@@ -0,0 +1,6 @@
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\021\000\100\144"
+}
+frames {
+  junk: "\103\000\000\000\000\000\000\000\052\000\000\000\000\021\000\100\144"
+}

From 6de02d23825523484941f1216c43c2b89f03752c Mon Sep 17 00:00:00 2001
From: Rama Chavali 
Date: Wed, 14 Jun 2023 21:28:16 +0530
Subject: [PATCH 534/740] config: add warming_state stat to clusters (#27910)

Signed-off-by: Rama Chavali 
---
 changelogs/current.yaml                            |  3 +++
 .../upstream/cluster_manager/cluster_stats.rst     |  3 +++
 envoy/upstream/upstream.h                          |  7 ++++---
 source/common/upstream/cluster_manager_impl.cc     | 10 +++++++++-
 source/common/upstream/upstream_impl.cc            |  1 +
 test/integration/ads_integration_test.cc           | 14 +++++++++++++-
 6 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 747ec9c010a9..a7b46730a34a 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -345,6 +345,9 @@ new_features:
 - area: zookeeper
   change: |
     Added the "addWatch" opcode support to the ZooKeeper proxy filter.
+- area: config
+  change: |
+    added a statistic :ref:`warming_state ` to indicate the current warming state of a cluster.
 
 deprecated:
 - area: access_log
diff --git a/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst b/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst
index 96e82847961c..3dd58c0d7a0c 100644
--- a/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst
+++ b/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst
@@ -28,6 +28,8 @@ upstreams and control plane xDS clusters.
   active_clusters, Gauge, Number of currently active (warmed) clusters
   warming_clusters, Gauge, Number of currently warming (not active) clusters
 
+.. _config_cluster_stats:
+
 Every cluster has a statistics tree rooted at *cluster..* with the following statistics:
 
 .. csv-table::
@@ -102,6 +104,7 @@ Every cluster has a statistics tree rooted at *cluster..* with the followi
   update_empty, Counter, Total cluster membership updates ending with empty cluster load assignment and continuing with previous config
   update_no_rebuild, Counter, Total successful cluster membership updates that didn't result in any cluster load balancing structure rebuilds
   version, Gauge, Hash of the contents from the last successful API fetch
+  warming_state, Gauge, Current cluster warming state
   max_host_weight, Gauge, Maximum weight of any host in the cluster
   bind_errors, Counter, Total errors binding the socket to the configured source address
   assignment_timeout_received, Counter, Total assignments received with endpoint lease information.
diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h
index 32bd2bddda55..21303beb2b29 100644
--- a/envoy/upstream/upstream.h
+++ b/envoy/upstream/upstream.h
@@ -224,7 +224,7 @@ class Host : virtual public HostDescription {
 
   /**
    * Set the timestamp of when the host has transitioned from unhealthy to healthy state via an
-   * active healchecking.
+   * active health checking.
    */
   virtual void setLastHcPassTime(MonotonicTime last_hc_pass_time) PURE;
 
@@ -596,7 +596,8 @@ class PrioritySet {
   COUNTER(update_failure)                                                                          \
   COUNTER(update_no_rebuild)                                                                       \
   COUNTER(update_success)                                                                          \
-  GAUGE(version, NeverImport)
+  GAUGE(version, NeverImport)                                                                      \
+  GAUGE(warming_state, NeverImport)
 
 /**
  * All cluster endpoints related stats.
@@ -610,7 +611,7 @@ class PrioritySet {
   GAUGE(membership_total, NeverImport)
 
 /**
- * All cluster loadbalancing related stats.
+ * All cluster load balancing related stats.
  */
 #define ALL_CLUSTER_LB_STATS(COUNTER, GAUGE, HISTOGRAM, TEXT_READOUT, STATNAME)                    \
   COUNTER(lb_healthy_panic)                                                                        \
diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc
index 3a892ed69db1..b4bd365e63e6 100644
--- a/source/common/upstream/cluster_manager_impl.cc
+++ b/source/common/upstream/cluster_manager_impl.cc
@@ -93,8 +93,13 @@ void ClusterManagerInitHelper::addCluster(ClusterManagerCluster& cm_cluster) {
   // server initialization.
   ASSERT(state_ != State::AllClustersInitialized);
 
-  const auto initialize_cb = [&cm_cluster, this] { onClusterInit(cm_cluster); };
+  const auto initialize_cb = [&cm_cluster, this] {
+    onClusterInit(cm_cluster);
+    cm_cluster.cluster().info()->configUpdateStats().warming_state_.set(0);
+  };
   Cluster& cluster = cm_cluster.cluster();
+
+  cluster.info()->configUpdateStats().warming_state_.set(1);
   if (cluster.initializePhase() == Cluster::InitializePhase::Primary) {
     // Remove the previous cluster before the cluster object is destroyed.
     primary_init_clusters_.insert_or_assign(cm_cluster.cluster().info()->name(), &cm_cluster);
@@ -749,6 +754,7 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::config::cluster::v3::Cl
   const auto previous_cluster =
       loadCluster(cluster, new_hash, version_info, true, warming_clusters_);
   auto& cluster_entry = warming_clusters_.at(cluster_name);
+  cluster_entry->cluster_->info()->configUpdateStats().warming_state_.set(1);
   if (!all_clusters_initialized) {
     ENVOY_LOG(debug, "add/update cluster {} during init", cluster_name);
     init_helper_.addCluster(*cluster_entry);
@@ -757,6 +763,8 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::config::cluster::v3::Cl
     cluster_entry->cluster_->initialize([this, cluster_name] {
       ENVOY_LOG(debug, "warming cluster {} complete", cluster_name);
       auto state_changed_cluster_entry = warming_clusters_.find(cluster_name);
+      state_changed_cluster_entry->second->cluster_->info()->configUpdateStats().warming_state_.set(
+          0);
       onClusterInit(*state_changed_cluster_entry->second);
     });
   }
diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc
index 718f2f68ec30..95af949905f5 100644
--- a/source/common/upstream/upstream_impl.cc
+++ b/source/common/upstream/upstream_impl.cc
@@ -1527,6 +1527,7 @@ void ClusterImplBase::onPreInitComplete() {
 }
 
 void ClusterImplBase::onInitDone() {
+  info()->configUpdateStats().warming_state_.set(0);
   if (health_checker_ && pending_initialize_health_checks_ == 0) {
     for (auto& host_set : prioritySet().hostSetsPerPriority()) {
       for (auto& host : host_set->hosts()) {
diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc
index 1c6c060800ca..104fd246939c 100644
--- a/test/integration/ads_integration_test.cc
+++ b/test/integration/ads_integration_test.cc
@@ -46,6 +46,7 @@ TEST_P(AdsIntegrationTest, BasicClusterInitialWarming) {
   sendDiscoveryResponse(
       cds_type_url, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1");
   test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1);
+  test_server_->waitForGaugeEq("cluster.cluster_0.warming_state", 1);
   EXPECT_TRUE(compareDiscoveryRequest(eds_type_url, "", {"cluster_0"}, {"cluster_0"}, {}));
   sendDiscoveryResponse(
       eds_type_url, {buildClusterLoadAssignment("cluster_0")},
@@ -53,6 +54,7 @@ TEST_P(AdsIntegrationTest, BasicClusterInitialWarming) {
 
   test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0);
   test_server_->waitForGaugeGe("cluster_manager.active_clusters", 2);
+  test_server_->waitForGaugeEq("cluster.cluster_0.warming_state", 0);
 }
 
 // Tests that the Envoy xDS client can handle updates to a subset of the subscribed resources from
@@ -71,6 +73,8 @@ TEST_P(AdsIntegrationTest, UpdateToSubsetOfResources) {
   sendDiscoveryResponse(cds_type_url, {cluster_0, cluster_1},
                                                              {cluster_0, cluster_1}, {}, "1");
   test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2);
+  test_server_->waitForGaugeEq("cluster.cluster_0.warming_state", 1);
+  test_server_->waitForGaugeEq("cluster.cluster_1.warming_state", 1);
   EXPECT_TRUE(compareDiscoveryRequest(eds_type_url, "", {cluster_0.name(), cluster_1.name()},
                                       {cluster_0.name(), cluster_1.name()}, {}));
   auto cla_0 = buildClusterLoadAssignment(cluster_0.name());
@@ -80,6 +84,8 @@ TEST_P(AdsIntegrationTest, UpdateToSubsetOfResources) {
 
   test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0);
   test_server_->waitForGaugeGe("cluster_manager.active_clusters", 4);
+  test_server_->waitForGaugeEq("cluster.cluster_0.warming_state", 0);
+  test_server_->waitForGaugeEq("cluster.cluster_0.warming_state", 0);
 
   // Send an update for one of the ClusterLoadAssignments only.
   cla_0.mutable_endpoints(0)->mutable_lb_endpoints(0)->mutable_load_balancing_weight()->set_value(
@@ -244,6 +250,8 @@ TEST_P(AdsIntegrationTest, ClusterSharingSecretWarming) {
   EXPECT_TRUE(compareDiscoveryRequest(sds_type_url, "", {"validation_context"},
                                       {"validation_context"}, {}));
   test_server_->waitForGaugeGe("cluster_manager.warming_clusters", 2);
+  test_server_->waitForGaugeEq("cluster.cluster_0.warming_state", 1);
+  test_server_->waitForGaugeEq("cluster.cluster_1.warming_state", 1);
 
   envoy::extensions::transport_sockets::tls::v3::Secret validation_context;
   TestUtility::loadFromYaml(fmt::format(R"EOF(
@@ -719,6 +727,7 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) {
       {buildCluster("warming_cluster_1")}, {"cluster_0"}, "2");
 
   test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1);
+  test_server_->waitForGaugeEq("cluster.warming_cluster_1.warming_state", 1);
 
   EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1",
                                       {"warming_cluster_1"}, {"warming_cluster_1"}, {"cluster_0"}));
@@ -729,8 +738,9 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) {
       {buildCluster("warming_cluster_1"), buildCluster("warming_cluster_2")},
       {buildCluster("warming_cluster_1"), buildCluster("warming_cluster_2")}, {}, "3");
   test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2);
-  // We would've got a Cluster discovery request with version 2 here, had the CDS not been paused.
+  test_server_->waitForGaugeEq("cluster.warming_cluster_2.warming_state", 1);
 
+  // We would've got a Cluster discovery request with version 2 here, had the CDS not been paused.
   EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1",
                                       {"warming_cluster_2", "warming_cluster_1"},
                                       {"warming_cluster_2"}, {}));
@@ -746,6 +756,8 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) {
 
   // Validate that clusters are warmed.
   test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0);
+  test_server_->waitForGaugeEq("cluster.warming_cluster_1.warming_state", 0);
+  test_server_->waitForGaugeEq("cluster.warming_cluster_2.warming_state", 0);
 
   // CDS is resumed and EDS response was acknowledged.
   // TODO (dmitri-d) remove the conditional when legacy mux implementations are removed.

From 9825a8453e1491827d7a2e67360cf2a2f630cff0 Mon Sep 17 00:00:00 2001
From: Ali Beyad 
Date: Wed, 14 Jun 2023 12:38:01 -0400
Subject: [PATCH 535/740] ci: Remove fetch-depth in Traffic Director workflow
 YAML (#27965)

Switching to use the default value of 1, as the workflow doesn't operate on pull requests.

Signed-off-by: Ali Beyad 
---
 .github/workflows/mobile-traffic_director.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/mobile-traffic_director.yml b/.github/workflows/mobile-traffic_director.yml
index 10e6c2b9ed90..85a9bdf0b892 100644
--- a/.github/workflows/mobile-traffic_director.yml
+++ b/.github/workflows/mobile-traffic_director.yml
@@ -30,10 +30,6 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 0
     - name: Add safe directory
       run: git config --global --add safe.directory /__w/envoy/envoy
     - name: 'Run GcpTrafficDirectorIntegrationTest'

From a0fed8f481914b099431cfece6a719b306b3bfca Mon Sep 17 00:00:00 2001
From: Kevin Baichoo 
Date: Wed, 14 Jun 2023 14:18:57 -0400
Subject: [PATCH 536/740] TLS Inspector: Enable tls_inspector buffer to
 dynamically size. (#27860)

* Enable tls_inspector buffer to dynamically size listener filter buffer.

Signed-off-by: Kevin Baichoo 
---
 .../tls_inspector/v3/tls_inspector.proto      |  9 ++
 changelogs/current.yaml                       |  7 ++
 .../listener/tls_inspector/tls_inspector.cc   | 17 +++-
 .../listener/tls_inspector/tls_inspector.h    |  8 +-
 .../tls_inspector_integration_test.cc         | 53 +++++++++-
 .../tls_inspector/tls_inspector_test.cc       | 97 +++++++++++++------
 6 files changed, 155 insertions(+), 36 deletions(-)

diff --git a/api/envoy/extensions/filters/listener/tls_inspector/v3/tls_inspector.proto b/api/envoy/extensions/filters/listener/tls_inspector/v3/tls_inspector.proto
index 8e5350cc7f12..db2d07c8da00 100644
--- a/api/envoy/extensions/filters/listener/tls_inspector/v3/tls_inspector.proto
+++ b/api/envoy/extensions/filters/listener/tls_inspector/v3/tls_inspector.proto
@@ -6,6 +6,7 @@ import "google/protobuf/wrappers.proto";
 
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
+import "validate/validate.proto";
 
 option java_package = "io.envoyproxy.envoy.extensions.filters.listener.tls_inspector.v3";
 option java_outer_classname = "TlsInspectorProto";
@@ -23,4 +24,12 @@ message TlsInspector {
 
   // Populate ``JA3`` fingerprint hash using data from the TLS Client Hello packet. Default is false.
   google.protobuf.BoolValue enable_ja3_fingerprinting = 1;
+
+  // The size in bytes of the initial buffer requested by the tls_inspector.
+  // If the filter needs to read additional bytes from the socket, the
+  // filter will double the buffer up to it's default maximum of 64KiB.
+  // If this size is not defined, defaults to maximum 64KiB that the
+  // tls inspector will consume.
+  google.protobuf.UInt32Value initial_read_buffer_size = 2
+      [(validate.rules).uint32 = {lt: 65537 gt: 255}];
 }
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index a7b46730a34a..157d83993d9a 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -251,6 +251,13 @@ new_features:
     the connection uses tls this records the tls client hello size. In cases
     where the connection doesn't use tls this records the amount of bytes the
     tls_inspector processed until it realized the connection was not using tls.
+- area: tls_inspector
+  change: |
+    added new configuration field :ref:`initial_read_buffer_size
+    `
+    to allow users to tune the buffer size requested by the filter. If
+    configured, and the filter needs additional bytes, the filter will double
+    the number of bytes requested up to the default 64KiB maximum.
 - area: access_log
   change: |
     added access log filter :ref:`log_type_filter `
diff --git a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc
index 31f849087866..e8ec24ec3fb2 100644
--- a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc
+++ b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc
@@ -54,8 +54,11 @@ Config::Config(
       ssl_ctx_(SSL_CTX_new(TLS_with_buffers_method())),
       enable_ja3_fingerprinting_(
           PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config, enable_ja3_fingerprinting, false)),
-      max_client_hello_size_(max_client_hello_size) {
-
+      max_client_hello_size_(max_client_hello_size),
+      initial_read_buffer_size_(
+          std::min(PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config, initial_read_buffer_size,
+                                                   max_client_hello_size),
+                   max_client_hello_size)) {
   if (max_client_hello_size_ > TLS_MAX_CLIENT_HELLO) {
     throw EnvoyException(fmt::format("max_client_hello_size of {} is greater than maximum of {}.",
                                      max_client_hello_size_, size_t(TLS_MAX_CLIENT_HELLO)));
@@ -92,7 +95,9 @@ Config::Config(
 
 bssl::UniquePtr Config::newSsl() { return bssl::UniquePtr{SSL_new(ssl_ctx_.get())}; }
 
-Filter::Filter(const ConfigSharedPtr& config) : config_(config), ssl_(config_->newSsl()) {
+Filter::Filter(const ConfigSharedPtr& config)
+    : config_(config), ssl_(config_->newSsl()),
+      requested_read_bytes_(config->initialReadBufferSize()) {
   SSL_set_app_data(ssl_.get(), this);
   SSL_set_accept_state(ssl_.get());
 }
@@ -183,12 +188,16 @@ ParseState Filter::parseClientHello(const void* data, size_t len,
   ParseState state = [this, ret]() {
     switch (SSL_get_error(ssl_.get(), ret)) {
     case SSL_ERROR_WANT_READ:
-      if (read_ == config_->maxClientHelloSize()) {
+      if (read_ == maxConfigReadBytes()) {
         // We've hit the specified size limit. This is an unreasonably large ClientHello;
         // indicate failure.
         config_->stats().client_hello_too_large_.inc();
         return ParseState::Error;
       }
+      if (read_ == requested_read_bytes_) {
+        // Double requested bytes up to the maximum configured.
+        requested_read_bytes_ = std::min(2 * requested_read_bytes_, maxConfigReadBytes());
+      }
       return ParseState::Continue;
     case SSL_ERROR_SSL:
       if (clienthello_success_) {
diff --git a/source/extensions/filters/listener/tls_inspector/tls_inspector.h b/source/extensions/filters/listener/tls_inspector/tls_inspector.h
index fd463f0234cb..cb02de823cb1 100644
--- a/source/extensions/filters/listener/tls_inspector/tls_inspector.h
+++ b/source/extensions/filters/listener/tls_inspector/tls_inspector.h
@@ -58,6 +58,7 @@ class Config {
   bssl::UniquePtr newSsl();
   bool enableJA3Fingerprinting() const { return enable_ja3_fingerprinting_; }
   uint32_t maxClientHelloSize() const { return max_client_hello_size_; }
+  uint32_t initialReadBufferSize() const { return initial_read_buffer_size_; }
 
   static constexpr size_t TLS_MAX_CLIENT_HELLO = 64 * 1024;
   static const unsigned TLS_MIN_SUPPORTED_VERSION;
@@ -68,6 +69,7 @@ class Config {
   bssl::UniquePtr ssl_ctx_;
   const bool enable_ja3_fingerprinting_;
   const uint32_t max_client_hello_size_;
+  const uint32_t initial_read_buffer_size_;
 };
 
 using ConfigSharedPtr = std::shared_ptr;
@@ -82,7 +84,7 @@ class Filter : public Network::ListenerFilter, Logger::LoggablemaxClientHelloSize(); }
+  size_t maxReadBytes() const override { return requested_read_bytes_; }
 
 private:
   ParseState parseClientHello(const void* data, size_t len, uint64_t bytes_already_processed);
@@ -90,6 +92,7 @@ class Filter : public Network::ListenerFilter, Logger::LoggablemaxClientHelloSize(); }
 
   ConfigSharedPtr config_;
   Network::ListenerFilterCallbacks* cb_{};
@@ -98,6 +101,9 @@ class Filter : public Network::ListenerFilter, Logger::Loggable listener_filter_disabled = absl::nullopt,
                                   bool enable_ja3_fingerprinting = false) {
-    config_helper_.renameListener("echo");
     std::string tls_inspector_config = ConfigHelper::tlsInspectorFilter(enable_ja3_fingerprinting);
     if (listener_filter_disabled.has_value()) {
       tls_inspector_config = appendMatcher(tls_inspector_config, listener_filter_disabled.value());
     }
+    initializeWithTlsInspector(ssl_client, log_format, tls_inspector_config);
+  }
+
+  void initializeWithTlsInspector(bool ssl_client, const std::string& log_format,
+                                  const std::string& tls_inspector_config) {
+    config_helper_.renameListener("echo");
     config_helper_.addListenerFilter(tls_inspector_config);
 
     config_helper_.addConfigModifier([ssl_client](
@@ -204,6 +209,52 @@ TEST_P(TlsInspectorIntegrationTest, JA3FingerprintIsSet) {
             115);
 }
 
+TEST_P(TlsInspectorIntegrationTest, RequestedBufferSizeCanGrow) {
+  const std::string small_initial_buffer_tls_inspector_config = R"EOF(
+    name: "envoy.filters.listener.tls_inspector"
+    typed_config:
+      "@type": type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
+      initial_read_buffer_size: 256
+  )EOF";
+  initializeWithTlsInspector(true, "%RESPONSE_CODE_DETAILS%",
+                             small_initial_buffer_tls_inspector_config);
+
+  Network::Address::InstanceConstSharedPtr address =
+      Ssl::getSslAddress(version_, lookupPort("echo"));
+
+  Ssl::ClientSslTransportOptions ssl_options;
+  ssl_options.setCipherSuites({"ECDHE-RSA-AES128-GCM-SHA256"});
+  ssl_options.setTlsVersion(envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLSv1_2);
+  const std::string really_long_sni(absl::StrCat(std::string(240, 'a'), ".foo.com"));
+  ssl_options.setSni(really_long_sni);
+  context_ = Ssl::createClientSslTransportSocketFactory(ssl_options, *context_manager_, *api_);
+  Network::TransportSocketPtr transport_socket = context_->createTransportSocket(
+      std::make_shared(
+          absl::string_view(""), std::vector(), std::vector{"envoyalpn"}),
+      nullptr);
+
+  client_ = dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(),
+                                                std::move(transport_socket), nullptr, nullptr);
+  client_->addConnectionCallbacks(connect_callbacks_);
+  client_->connect();
+
+  while (!connect_callbacks_.connected() && !connect_callbacks_.closed()) {
+    dispatcher_->run(Event::Dispatcher::RunType::NonBlock);
+  }
+
+  ASSERT(connect_callbacks_.connected());
+  client_->close(Network::ConnectionCloseType::NoFlush);
+
+  test_server_->waitUntilHistogramHasSamples("tls_inspector.bytes_processed");
+  auto bytes_processed_histogram = test_server_->histogram("tls_inspector.bytes_processed");
+  EXPECT_EQ(
+      TestUtility::readSampleCount(test_server_->server().dispatcher(), *bytes_processed_histogram),
+      1);
+  EXPECT_EQ(static_cast(TestUtility::readSampleSum(test_server_->server().dispatcher(),
+                                                        *bytes_processed_histogram)),
+            515);
+}
+
 INSTANTIATE_TEST_SUITE_P(IpVersions, TlsInspectorIntegrationTest,
                          testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
                          TestUtility::ipTestParamsToString);
diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc
index e9ae319ab3d3..0bd45b131c53 100644
--- a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc
+++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc
@@ -54,27 +54,39 @@ class TlsInspectorTest : public testing::TestWithParam(&file_event_callback_), ReturnNew>()));
     buffer_ = std::make_unique(
         *io_handle_, dispatcher_, [](bool) {}, [](Network::ListenerFilterBuffer&) {},
-        cfg_->maxClientHelloSize());
+        cfg_->initialReadBufferSize());
     filter_->onAccept(cb_);
   }
 
-  void mockSysCallForPeek(std::vector& client_hello) {
+  void mockSysCallForPeek(std::vector& client_hello, bool windows_recv = false) {
 #ifdef WIN32
-    EXPECT_CALL(os_sys_calls_, readv(_, _, _))
-        .WillOnce(Invoke(
-            [&client_hello](os_fd_t fd, const iovec* iov, int iovcnt) -> Api::SysCallSizeResult {
-              ASSERT(iov->iov_len >= client_hello.size());
-              memcpy(iov->iov_base, client_hello.data(), client_hello.size());
-              return Api::SysCallSizeResult{ssize_t(client_hello.size()), 0};
-            }))
-        .WillOnce(Return(Api::SysCallSizeResult{ssize_t(-1), SOCKET_ERROR_AGAIN}));
+    // In some cases the syscall used for windows is recv, not readv.
+    if (windows_recv) {
+      EXPECT_CALL(os_sys_calls_, recv(_, _, _, _))
+          .WillOnce(Invoke([=, &client_hello](os_fd_t, void* buffer, size_t length,
+                                              int) -> Api::SysCallSizeResult {
+            const size_t amount_to_copy = std::min(length, client_hello.size());
+            memcpy(buffer, client_hello.data(), amount_to_copy);
+            return Api::SysCallSizeResult{ssize_t(amount_to_copy), 0};
+          }));
+    } else {
+      EXPECT_CALL(os_sys_calls_, readv(_, _, _))
+          .WillOnce(Invoke(
+              [&client_hello](os_fd_t fd, const iovec* iov, int iovcnt) -> Api::SysCallSizeResult {
+                const size_t amount_to_copy = std::min(iov->iov_len, client_hello.size());
+                memcpy(iov->iov_base, client_hello.data(), amount_to_copy);
+                return Api::SysCallSizeResult{ssize_t(amount_to_copy), 0};
+              }))
+          .WillOnce(Return(Api::SysCallSizeResult{ssize_t(-1), SOCKET_ERROR_AGAIN}));
+    }
 #else
+    (void)windows_recv;
     EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK))
         .WillOnce(Invoke(
             [&client_hello](os_fd_t, void* buffer, size_t length, int) -> Api::SysCallSizeResult {
-              ASSERT(length >= client_hello.size());
-              memcpy(buffer, client_hello.data(), client_hello.size());
-              return Api::SysCallSizeResult{ssize_t(client_hello.size()), 0};
+              const size_t amount_to_copy = std::min(length, client_hello.size());
+              memcpy(buffer, client_hello.data(), amount_to_copy);
+              return Api::SysCallSizeResult{ssize_t(amount_to_copy), 0};
             }));
 #endif
   }
@@ -264,23 +276,7 @@ TEST_P(TlsInspectorTest, ClientHelloTooBig) {
       cfg_->maxClientHelloSize());
 
   filter_->onAccept(cb_);
-#ifdef WIN32
-  EXPECT_CALL(os_sys_calls_, recv(_, _, _, _))
-      .WillOnce(Invoke(
-          [=, &client_hello](os_fd_t, void* buffer, size_t length, int) -> Api::SysCallSizeResult {
-            ASSERT(length >= max_size);
-            memcpy(buffer, client_hello.data(), max_size);
-            return Api::SysCallSizeResult{ssize_t(max_size), 0};
-          }));
-#else
-  EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK))
-      .WillOnce(Invoke(
-          [=, &client_hello](os_fd_t, void* buffer, size_t length, int) -> Api::SysCallSizeResult {
-            ASSERT(length == max_size);
-            memcpy(buffer, client_hello.data(), length);
-            return Api::SysCallSizeResult{ssize_t(length), 0};
-          }));
-#endif
+  mockSysCallForPeek(client_hello, true);
   EXPECT_CALL(socket_, detectedTransportProtocol()).Times(::testing::AnyNumber());
   file_event_callback_(Event::FileReadyType::Read);
   auto state = filter_->onData(*buffer_);
@@ -442,6 +438,47 @@ TEST_P(TlsInspectorTest, EarlyTerminationShouldNotRecordBytesProcessed) {
   ASSERT_FALSE(store_.histogramRecordedValues("tls_inspector.bytes_processed"));
 }
 
+TEST_P(TlsInspectorTest, RequestedMaxReadSizeDoesNotGoBeyondMaxSize) {
+  envoy::extensions::filters::listener::tls_inspector::v3::TlsInspector proto_config;
+  const uint32_t initial_buffer_size = 15;
+  const size_t max_size = 50;
+  proto_config.mutable_initial_read_buffer_size()->set_value(initial_buffer_size);
+  cfg_ = std::make_shared(*store_.rootScope(), proto_config, max_size);
+  buffer_ = std::make_unique(
+      *io_handle_, dispatcher_, [](bool) {}, [](Network::ListenerFilterBuffer&) {},
+      cfg_->initialReadBufferSize());
+  std::vector client_hello = Tls::Test::generateClientHello(
+      std::get<0>(GetParam()), std::get<1>(GetParam()), "example.com", "\x02h2");
+
+  init();
+  EXPECT_CALL(socket_, setRequestedServerName(_)).Times(0);
+  EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0);
+  EXPECT_CALL(socket_, setDetectedTransportProtocol(_)).Times(0);
+
+  mockSysCallForPeek(client_hello, true);
+  file_event_callback_(Event::FileReadyType::Read);
+  auto state = filter_->onData(*buffer_);
+  EXPECT_EQ(Network::FilterStatus::StopIteration, state);
+  EXPECT_EQ(2 * initial_buffer_size, filter_->maxReadBytes());
+  buffer_->resetCapacity(2 * initial_buffer_size);
+
+  mockSysCallForPeek(client_hello, true);
+  file_event_callback_(Event::FileReadyType::Read);
+  state = filter_->onData(*buffer_);
+  EXPECT_EQ(Network::FilterStatus::StopIteration, state);
+  EXPECT_EQ(max_size, filter_->maxReadBytes());
+  buffer_->resetCapacity(max_size);
+
+  // The filter should not request a larger buffer as we've reached the max.
+  // It should close the connection.
+  mockSysCallForPeek(client_hello, true);
+  file_event_callback_(Event::FileReadyType::Read);
+  state = filter_->onData(*buffer_);
+  EXPECT_EQ(Network::FilterStatus::StopIteration, state);
+  EXPECT_EQ(max_size, filter_->maxReadBytes());
+  EXPECT_FALSE(io_handle_->isOpen());
+}
+
 } // namespace
 } // namespace TlsInspector
 } // namespace ListenerFilters

From b28128bfe96abaefcb5dfdef634d540adcafec73 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Wed, 14 Jun 2023 15:16:11 -0400
Subject: [PATCH 537/740] mobile: moving dump stats away from admin (#27944)

Risk Level: low
Testing: new integration test, existing stats tests
Docs Changes: n/a
Release Notes: n/a

co-authored-by: jmarantz (04aae9f)
Signed-off-by: Alyssa Wilk 
---
 mobile/library/cc/engine.cc                   | 12 ++++
 mobile/library/cc/engine.h                    |  1 +
 mobile/library/common/engine.cc               | 47 ++++++++++++++
 mobile/library/common/engine.h                |  6 ++
 mobile/library/common/main_interface.cc       | 63 ++++---------------
 .../integration/client_integration_test.cc    |  7 +++
 6 files changed, 85 insertions(+), 51 deletions(-)

diff --git a/mobile/library/cc/engine.cc b/mobile/library/cc/engine.cc
index b807119637e8..1b44741fccdd 100644
--- a/mobile/library/cc/engine.cc
+++ b/mobile/library/cc/engine.cc
@@ -1,5 +1,6 @@
 #include "engine.h"
 
+#include "library/common/data/utility.h"
 #include "library/common/main_interface.h"
 #include "library/common/types/c_types.h"
 
@@ -24,6 +25,17 @@ StreamClientSharedPtr Engine::streamClient() {
 
 PulseClientSharedPtr Engine::pulseClient() { return std::make_shared(); }
 
+std::string Engine::dumpStats() {
+  envoy_data data;
+  if (dump_stats(engine_, &data) == ENVOY_FAILURE) {
+    return "";
+  }
+  const std::string to_return = Data::Utility::copyToString(data);
+  release_envoy_data(data);
+
+  return to_return;
+}
+
 envoy_status_t Engine::terminate() {
   if (terminated_) {
     throw std::runtime_error("attempting to double terminate Engine");
diff --git a/mobile/library/cc/engine.h b/mobile/library/cc/engine.h
index 61261332e4e8..ea474033f37c 100644
--- a/mobile/library/cc/engine.h
+++ b/mobile/library/cc/engine.h
@@ -19,6 +19,7 @@ class Engine : public std::enable_shared_from_this {
 public:
   ~Engine();
 
+  std::string dumpStats();
   StreamClientSharedPtr streamClient();
   PulseClientSharedPtr pulseClient();
 
diff --git a/mobile/library/common/engine.cc b/mobile/library/common/engine.cc
index 125b443728fe..323caad30446 100644
--- a/mobile/library/common/engine.cc
+++ b/mobile/library/common/engine.cc
@@ -222,6 +222,53 @@ Network::ConnectivityManager& Engine::networkConnectivityManager() {
   return *connectivity_manager_;
 }
 
+void statsAsText(const std::map& all_stats,
+                 const std::vector& histograms,
+                 Buffer::Instance& response) {
+  for (const auto& stat : all_stats) {
+    response.addFragments({stat.first, ": ", absl::StrCat(stat.second), "\n"});
+  }
+  std::map all_histograms;
+  for (const Stats::ParentHistogramSharedPtr& histogram : histograms) {
+    if (histogram->used()) {
+      all_histograms.emplace(histogram->name(), histogram->quantileSummary());
+    }
+  }
+  for (const auto& histogram : all_histograms) {
+    response.addFragments({histogram.first, ": ", histogram.second, "\n"});
+  }
+}
+
+void handlerStats(Stats::Store& stats, Buffer::Instance& response) {
+  std::map all_stats;
+  for (const Stats::CounterSharedPtr& counter : stats.counters()) {
+    if (counter->used()) {
+      all_stats.emplace(counter->name(), counter->value());
+    }
+  }
+
+  for (const Stats::GaugeSharedPtr& gauge : stats.gauges()) {
+    if (gauge->used()) {
+      all_stats.emplace(gauge->name(), gauge->value());
+    }
+  }
+
+  std::vector histograms = stats.histograms();
+  stats.symbolTable().sortByStatNames(
+      histograms.begin(), histograms.end(),
+      [](const Stats::ParentHistogramSharedPtr& a) -> Stats::StatName { return a->statName(); });
+
+  statsAsText(all_stats, histograms, response);
+}
+
+Envoy::Buffer::OwnedImpl Engine::dumpStats() {
+  ASSERT(dispatcher_->isThreadSafe(), "flushStats must be called from the dispatcher's context");
+
+  Envoy::Buffer::OwnedImpl instance;
+  handlerStats(server_->stats(), instance);
+  return instance;
+}
+
 void Engine::flushStats() {
   ASSERT(dispatcher_->isThreadSafe(), "flushStats must be called from the dispatcher's context");
 
diff --git a/mobile/library/common/engine.h b/mobile/library/common/engine.h
index 5d0a3991f4ed..3f5298017baa 100644
--- a/mobile/library/common/engine.h
+++ b/mobile/library/common/engine.h
@@ -82,6 +82,12 @@ class Engine : public Logger::Loggable {
    */
   envoy_status_t makeAdminCall(absl::string_view path, absl::string_view method, envoy_data& out);
 
+  /**
+   * Dump Envoy stats into the returned buffer
+   * @returns a buffer with referenced stats dumped in Envoy's standard text format.
+   */
+  Buffer::OwnedImpl dumpStats();
+
   /**
    * Flush the stats sinks outside of a flushing interval.
    * Note: stat flushing is done asynchronously, this function will never block.
diff --git a/mobile/library/common/main_interface.cc b/mobile/library/common/main_interface.cc
index fbd0654da057..96c581ba6707 100644
--- a/mobile/library/common/main_interface.cc
+++ b/mobile/library/common/main_interface.cc
@@ -5,6 +5,7 @@
 
 #include "absl/synchronization/notification.h"
 #include "library/common/api/external.h"
+#include "library/common/data/utility.h"
 #include "library/common/engine.h"
 #include "library/common/engine_handle.h"
 #include "library/common/extensions/filters/http/platform_bridge/c_types.h"
@@ -92,58 +93,18 @@ envoy_status_t record_counter_inc(envoy_engine_t e, const char* elements, envoy_
       });
 }
 
-namespace {
-struct AdminCallContext {
-  envoy_status_t status_{};
-  envoy_data response_{};
-
-  absl::Mutex mutex_{};
-  absl::Notification data_received_{};
-};
-
-absl::optional blockingAdminCall(envoy_engine_t e, absl::string_view path,
-                                             absl::string_view method,
-                                             std::chrono::milliseconds timeout) {
-  // Use a shared ptr here so that we can safely exit this scope in case of a timeout,
-  // allowing the dispatched lambda to clean itself up when it's done.
-  auto context = std::make_shared();
-
-  auto status = Envoy::EngineHandle::runOnEngineDispatcher(
-      e, [context, path = std::string(path), method = std::string(method)](auto& engine) -> void {
-        absl::MutexLock lock(&context->mutex_);
-
-        context->status_ = engine.makeAdminCall(path, method, context->response_);
-        context->data_received_.Notify();
-      });
-
-  if (status == ENVOY_FAILURE) {
-    return {};
+envoy_status_t dump_stats(envoy_engine_t engine, envoy_data* out) {
+  absl::Notification stats_received;
+  if (Envoy::EngineHandle::runOnEngineDispatcher(
+          engine, ([out, &stats_received](auto& engine) -> void {
+            Envoy::Buffer::OwnedImpl dumped_stats = engine.dumpStats();
+            *out = Envoy::Data::Utility::toBridgeData(dumped_stats, 1024 * 1024 * 100);
+            stats_received.Notify();
+          })) == ENVOY_FAILURE) {
+    return ENVOY_FAILURE;
   }
-
-  if (context->data_received_.WaitForNotificationWithTimeout(absl::Milliseconds(timeout.count()))) {
-    absl::MutexLock lock(&context->mutex_);
-
-    if (context->status_ == ENVOY_FAILURE) {
-      return {};
-    }
-
-    return context->response_;
-  } else {
-    ENVOY_LOG_MISC(warn, "timed out waiting for admin response");
-  }
-
-  return {};
-}
-} // namespace
-
-envoy_status_t dump_stats(envoy_engine_t e, envoy_data* out) {
-  auto maybe_data = blockingAdminCall(e, "/stats?usedonly", "GET", std::chrono::milliseconds(100));
-  if (maybe_data) {
-    *out = *maybe_data;
-    return ENVOY_SUCCESS;
-  }
-
-  return ENVOY_FAILURE;
+  stats_received.WaitForNotification();
+  return ENVOY_SUCCESS;
 }
 
 void flush_stats(envoy_engine_t e) {
diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc
index 13b2e3997e89..0e0ff356772c 100644
--- a/mobile/test/common/integration/client_integration_test.cc
+++ b/mobile/test/common/integration/client_integration_test.cc
@@ -515,5 +515,12 @@ TEST_P(ClientIntegrationTest, TestAdmin) {
 }
 #endif
 
+TEST_P(ClientIntegrationTest, TestStats) {
+  initialize();
+
+  std::string stats = engine_->dumpStats();
+  EXPECT_TRUE((absl::StrContains(stats, "runtime.load_success: 1"))) << stats;
+}
+
 } // namespace
 } // namespace Envoy

From 1024260dcbd7adf30eaedaeff8c9525734a3bfca Mon Sep 17 00:00:00 2001
From: "Vikas Choudhary (vikasc)" 
Date: Thu, 15 Jun 2023 02:28:09 +0530
Subject: [PATCH 538/740] Oauth2: Fix ENVOY_BUG crash if request with cookies
 at callback endpoint (#27946)

Signed-off-by: Vikas Choudhary 
---
 .../extensions/filters/http/oauth2/filter.cc  |  1 +
 .../filters/http/oauth2/filter_test.cc        |  6 +++--
 .../http/oauth2/oauth_integration_test.cc     | 25 +++++++++++++++++++
 3 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/source/extensions/filters/http/oauth2/filter.cc b/source/extensions/filters/http/oauth2/filter.cc
index d3422448d46b..4fe2c5edce05 100644
--- a/source/extensions/filters/http/oauth2/filter.cc
+++ b/source/extensions/filters/http/oauth2/filter.cc
@@ -299,6 +299,7 @@ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& he
               {{Http::Headers::get().Status, std::to_string(enumToInt(Http::Code::Found))},
                {Http::Headers::get().Location, state}})};
       decoder_callbacks_->encodeHeaders(std::move(response_headers), true, REDIRECT_RACE);
+      return Http::FilterHeadersStatus::StopIteration;
     }
 
     // Continue on with the filter stack.
diff --git a/test/extensions/filters/http/oauth2/filter_test.cc b/test/extensions/filters/http/oauth2/filter_test.cc
index e0445410e3ec..5cae3970b5a6 100644
--- a/test/extensions/filters/http/oauth2/filter_test.cc
+++ b/test/extensions/filters/http/oauth2/filter_test.cc
@@ -948,7 +948,8 @@ TEST_F(OAuth2Test, OAuthTestUpdatePathAfterSuccessLegacyEncoding) {
 
   EXPECT_CALL(decoder_callbacks_,
               encodeHeaders_(HeaderMapEqualRef(&expected_response_headers), true));
-  EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false));
+  EXPECT_EQ(Http::FilterHeadersStatus::StopIteration,
+            filter_->decodeHeaders(request_headers, false));
 
   Http::TestRequestHeaderMapImpl final_request_headers{
       {Http::Headers::get().Host.get(), "traffic.example.com"},
@@ -1003,7 +1004,8 @@ TEST_F(OAuth2Test, OAuthTestUpdatePathAfterSuccess) {
 
   EXPECT_CALL(decoder_callbacks_,
               encodeHeaders_(HeaderMapEqualRef(&expected_response_headers), true));
-  EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false));
+  EXPECT_EQ(Http::FilterHeadersStatus::StopIteration,
+            filter_->decodeHeaders(request_headers, false));
 
   Http::TestRequestHeaderMapImpl final_request_headers{
       {Http::Headers::get().Host.get(), "traffic.example.com"},
diff --git a/test/extensions/filters/http/oauth2/oauth_integration_test.cc b/test/extensions/filters/http/oauth2/oauth_integration_test.cc
index 4dce37751660..ce4e68dfe4db 100644
--- a/test/extensions/filters/http/oauth2/oauth_integration_test.cc
+++ b/test/extensions/filters/http/oauth2/oauth_integration_test.cc
@@ -292,7 +292,32 @@ name: oauth
         validateHmac(response->headers(), headers.Host()->value().getStringView(), hmac_secret));
 
     EXPECT_EQ("302", response->headers().getStatusValue());
+    std::string hmac =
+        Http::Utility::parseSetCookieValue(response->headers(), default_cookie_names_.oauth_hmac_);
+    std::string oauth_expires = Http::Utility::parseSetCookieValue(
+        response->headers(), default_cookie_names_.oauth_expires_);
+
+    RELEASE_ASSERT(response->waitForEndStream(), "unexpected timeout");
+    codec_client_->close();
 
+    // Now try sending the cookies back
+    codec_client_ = makeHttpConnection(lookupPort("http"));
+    Http::TestRequestHeaderMapImpl headersWithCookie{
+        {":method", "GET"},
+        {":path", "/callback?code=foo&state=http%3A%2F%2Ftraffic.example.com%2Fnot%2F_oauth"},
+        {":scheme", "http"},
+        {"x-forwarded-proto", "http"},
+        {":authority", "authority"},
+        {"authority", "Bearer token"},
+        {"cookie", absl::StrCat(default_cookie_names_.oauth_hmac_, "=", hmac)},
+        {"cookie", absl::StrCat(default_cookie_names_.oauth_expires_, "=", oauth_expires)},
+    };
+    auto encoder_decoder2 = codec_client_->startRequest(headersWithCookie, true);
+    response = std::move(encoder_decoder2.second);
+    response->waitForHeaders();
+    EXPECT_EQ("302", response->headers().getStatusValue());
+    EXPECT_EQ("http://traffic.example.com/not/_oauth",
+              response->headers().Location()->value().getStringView());
     RELEASE_ASSERT(response->waitForEndStream(), "unexpected timeout");
     codec_client_->close();
   }

From 64cc9840f004b3d34eced6a58fa4ed83a4a88c8c Mon Sep 17 00:00:00 2001
From: Raven Black 
Date: Wed, 14 Jun 2023 17:16:31 -0700
Subject: [PATCH 539/740] Make ENVOY_MAKE_SOCKET_OPTION_NAME macro more
 explicit (#27966)

Make socket option macro more explicit

Signed-off-by: Raven Black 
---
 envoy/network/socket.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/envoy/network/socket.h b/envoy/network/socket.h
index 0f2cde06ff19..0f19735d0313 100644
--- a/envoy/network/socket.h
+++ b/envoy/network/socket.h
@@ -40,7 +40,7 @@ struct SocketOptionName {
 // ENVOY_MAKE_SOCKET_OPTION_NAME is a helper macro to generate a
 // SocketOptionName with a descriptive string name.
 #define ENVOY_MAKE_SOCKET_OPTION_NAME(level, option)                                               \
-  Network::SocketOptionName(level, option, #level "/" #option)
+  ::Envoy::Network::SocketOptionName(level, option, #level "/" #option)
 
 // Moved from source/common/network/socket_option_impl.h to avoid dependency loops.
 #ifdef IP_TRANSPARENT

From 91f738a300d6719dadb79731b0d9aa9874b45bc9 Mon Sep 17 00:00:00 2001
From: pianiststickman <34144687+pianiststickman@users.noreply.github.com>
Date: Wed, 14 Jun 2023 20:20:13 -0400
Subject: [PATCH 540/740] access logs: periodic bytes snapshot (#27947)

Compute delta of bytes during periodic snapshot.

Signed-off-by: Eugene Chan 
Co-authored-by: Paul Gallagher 
---
 changelogs/current.yaml                       |  5 ++
 envoy/stream_info/stream_info.h               | 57 ++++++++++++
 source/common/http/conn_manager_impl.cc       | 10 +++
 source/common/router/upstream_request.cc      | 12 +++
 source/common/stream_info/stream_info_impl.h  |  6 +-
 source/common/tcp_proxy/tcp_proxy.cc          |  5 ++
 test/common/http/conn_manager_impl_test.cc    | 37 +++++---
 .../common/router/router_upstream_log_test.cc | 90 +++++++++++++------
 test/common/tcp_proxy/tcp_proxy_test.cc       | 27 ++++++
 9 files changed, 206 insertions(+), 43 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 157d83993d9a..b050838d1f4d 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -355,6 +355,11 @@ new_features:
 - area: config
   change: |
     added a statistic :ref:`warming_state ` to indicate the current warming state of a cluster.
+- area: access_log
+  change: |
+    added bytes snapshotting for upstream and downstream logging that will be reset after every periodic log. Downstream
+    periodic loggers should read BytesMeter::bytesAtLastDownstreamPeriodicLog(), and upstream periodic loggers should read
+    BytesMeter::bytesAtLastUpstreamPeriodicLog().
 
 deprecated:
 - area: access_log
diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h
index d7176c3dcaec..198cd6255759 100644
--- a/envoy/stream_info/stream_info.h
+++ b/envoy/stream_info/stream_info.h
@@ -375,20 +375,77 @@ class DownstreamTiming {
 
 // Measure the number of bytes sent and received for a stream.
 struct BytesMeter {
+  BytesMeter() = default;
   uint64_t wireBytesSent() const { return wire_bytes_sent_; }
   uint64_t wireBytesReceived() const { return wire_bytes_received_; }
   uint64_t headerBytesSent() const { return header_bytes_sent_; }
   uint64_t headerBytesReceived() const { return header_bytes_received_; }
+
   void addHeaderBytesSent(uint64_t added_bytes) { header_bytes_sent_ += added_bytes; }
   void addHeaderBytesReceived(uint64_t added_bytes) { header_bytes_received_ += added_bytes; }
   void addWireBytesSent(uint64_t added_bytes) { wire_bytes_sent_ += added_bytes; }
   void addWireBytesReceived(uint64_t added_bytes) { wire_bytes_received_ += added_bytes; }
 
+  struct BytesSnapshot {
+    SystemTime snapshot_time;
+    uint64_t header_bytes_sent{};
+    uint64_t header_bytes_received{};
+    uint64_t wire_bytes_sent{};
+    uint64_t wire_bytes_received{};
+  };
+  void takeDownstreamPeriodicLoggingSnapshot(const SystemTime& snapshot_time) {
+    downstream_periodic_logging_bytes_snapshot_ = std::make_unique();
+
+    downstream_periodic_logging_bytes_snapshot_->snapshot_time = snapshot_time;
+    downstream_periodic_logging_bytes_snapshot_->header_bytes_sent = header_bytes_sent_;
+    downstream_periodic_logging_bytes_snapshot_->header_bytes_received = header_bytes_received_;
+    downstream_periodic_logging_bytes_snapshot_->wire_bytes_sent = wire_bytes_sent_;
+    downstream_periodic_logging_bytes_snapshot_->wire_bytes_received = wire_bytes_received_;
+  }
+  void takeUpstreamPeriodicLoggingSnapshot(const SystemTime& snapshot_time) {
+    upstream_periodic_logging_bytes_snapshot_ = std::make_unique();
+
+    upstream_periodic_logging_bytes_snapshot_->snapshot_time = snapshot_time;
+    upstream_periodic_logging_bytes_snapshot_->header_bytes_sent = header_bytes_sent_;
+    upstream_periodic_logging_bytes_snapshot_->header_bytes_received = header_bytes_received_;
+    upstream_periodic_logging_bytes_snapshot_->wire_bytes_sent = wire_bytes_sent_;
+    upstream_periodic_logging_bytes_snapshot_->wire_bytes_received = wire_bytes_received_;
+  }
+  const BytesSnapshot* bytesAtLastDownstreamPeriodicLog() const {
+    return downstream_periodic_logging_bytes_snapshot_.get();
+  }
+  const BytesSnapshot* bytesAtLastUpstreamPeriodicLog() const {
+    return upstream_periodic_logging_bytes_snapshot_.get();
+  }
+  // Adds the bytes from `existing` to `this`.
+  // Additionally, captures the snapshots on `existing` and adds them to `this`.
+  void captureExistingBytesMeter(BytesMeter& existing) {
+    // Add bytes accumulated on `this` to the pre-existing periodic bytes collectors.
+    if (existing.downstream_periodic_logging_bytes_snapshot_) {
+      downstream_periodic_logging_bytes_snapshot_ =
+          std::move(existing.downstream_periodic_logging_bytes_snapshot_);
+      existing.downstream_periodic_logging_bytes_snapshot_ = nullptr;
+    }
+    if (existing.upstream_periodic_logging_bytes_snapshot_) {
+      upstream_periodic_logging_bytes_snapshot_ =
+          std::move(existing.upstream_periodic_logging_bytes_snapshot_);
+      existing.upstream_periodic_logging_bytes_snapshot_ = nullptr;
+    }
+
+    // Accumulate existing bytes.
+    header_bytes_sent_ += existing.header_bytes_sent_;
+    header_bytes_received_ += existing.header_bytes_received_;
+    wire_bytes_sent_ += existing.wire_bytes_sent_;
+    wire_bytes_received_ += existing.wire_bytes_received_;
+  }
+
 private:
   uint64_t header_bytes_sent_{};
   uint64_t header_bytes_received_{};
   uint64_t wire_bytes_sent_{};
   uint64_t wire_bytes_received_{};
+  std::unique_ptr downstream_periodic_logging_bytes_snapshot_;
+  std::unique_ptr upstream_periodic_logging_bytes_snapshot_;
 };
 
 using BytesMeterSharedPtr = std::shared_ptr;
diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc
index 80995aa8fd1a..31fcaeec15cd 100644
--- a/source/common/http/conn_manager_impl.cc
+++ b/source/common/http/conn_manager_impl.cc
@@ -833,6 +833,16 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect
         filter_manager_.log(AccessLog::AccessLogType::DownstreamPeriodic);
         refreshAccessLogFlushTimer();
       }
+      const SystemTime now = connection_manager_.timeSource().systemTime();
+      // Downstream bytes meter is guaranteed to be non-null because ActiveStream and the timer
+      // event are created on the same thread that sets the meter in
+      // ConnectionManagerImpl::newStream.
+      filter_manager_.streamInfo().getDownstreamBytesMeter()->takeDownstreamPeriodicLoggingSnapshot(
+          now);
+      if (auto& upstream_bytes_meter = filter_manager_.streamInfo().getUpstreamBytesMeter();
+          upstream_bytes_meter != nullptr) {
+        upstream_bytes_meter->takeDownstreamPeriodicLoggingSnapshot(now);
+      }
     });
     refreshAccessLogFlushTimer();
   }
diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc
index 2d5f67f8a39d..b2bdd2564773 100644
--- a/source/common/router/upstream_request.cc
+++ b/source/common/router/upstream_request.cc
@@ -388,6 +388,18 @@ void UpstreamRequest::acceptHeadersFromRouter(bool end_stream) {
         upstreamLog(AccessLog::AccessLogType::UpstreamPeriodic);
         resetUpstreamLogFlushTimer();
       }
+      // Both downstream and upstream bytes meters may not be initialized when
+      // the timer goes off, e.g. if it takes longer than the interval for a
+      // connection to be initialized; check for nullptr.
+      auto& downstream_bytes_meter = stream_info_.getDownstreamBytesMeter();
+      auto& upstream_bytes_meter = stream_info_.getUpstreamBytesMeter();
+      const SystemTime now = parent_.callbacks()->dispatcher().timeSource().systemTime();
+      if (downstream_bytes_meter) {
+        downstream_bytes_meter->takeUpstreamPeriodicLoggingSnapshot(now);
+      }
+      if (upstream_bytes_meter) {
+        upstream_bytes_meter->takeUpstreamPeriodicLoggingSnapshot(now);
+      }
     });
 
     resetUpstreamLogFlushTimer();
diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h
index af11fcc817d3..767501aa2b34 100644
--- a/source/common/stream_info/stream_info_impl.h
+++ b/source/common/stream_info/stream_info_impl.h
@@ -327,11 +327,7 @@ struct StreamInfoImpl : public StreamInfo {
   }
 
   void setUpstreamBytesMeter(const BytesMeterSharedPtr& upstream_bytes_meter) override {
-    // Accumulate the byte measurement from previous upstream request during a retry.
-    upstream_bytes_meter->addWireBytesSent(upstream_bytes_meter_->wireBytesSent());
-    upstream_bytes_meter->addWireBytesReceived(upstream_bytes_meter_->wireBytesReceived());
-    upstream_bytes_meter->addHeaderBytesSent(upstream_bytes_meter_->headerBytesSent());
-    upstream_bytes_meter->addHeaderBytesReceived(upstream_bytes_meter_->headerBytesReceived());
+    upstream_bytes_meter->captureExistingBytesMeter(*upstream_bytes_meter_);
     upstream_bytes_meter_ = upstream_bytes_meter;
   }
 
diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc
index bf0e27ba30d0..fd5b12dbd144 100644
--- a/source/common/tcp_proxy/tcp_proxy.cc
+++ b/source/common/tcp_proxy/tcp_proxy.cc
@@ -850,6 +850,11 @@ void Filter::onAccessLogFlushInterval() {
     access_log->log(nullptr, nullptr, nullptr, getStreamInfo(),
                     AccessLog::AccessLogType::TcpPeriodic);
   }
+  const SystemTime now = read_callbacks_->connection().dispatcher().timeSource().systemTime();
+  getStreamInfo().getDownstreamBytesMeter()->takeDownstreamPeriodicLoggingSnapshot(now);
+  if (getStreamInfo().getUpstreamBytesMeter()) {
+    getStreamInfo().getUpstreamBytesMeter()->takeDownstreamPeriodicLoggingSnapshot(now);
+  }
   resetAccessLogFlushTimer();
 }
 
diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc
index befb0aa44d75..355dbd91cbc1 100644
--- a/test/common/http/conn_manager_impl_test.cc
+++ b/test/common/http/conn_manager_impl_test.cc
@@ -2554,6 +2554,7 @@ TEST_F(HttpConnectionManagerImplTest, TestPeriodicAccessLogging) {
       }));
   Event::MockTimer* periodic_log_timer;
   EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> Http::Status {
+    response_encoder_.stream_.bytes_meter_->addWireBytesReceived(4);
     periodic_log_timer = setUpTimer();
     EXPECT_CALL(*periodic_log_timer, enableTimer(*access_log_flush_interval_, _));
     decoder_ = &conn_manager_->newStream(response_encoder_);
@@ -2573,19 +2574,31 @@ TEST_F(HttpConnectionManagerImplTest, TestPeriodicAccessLogging) {
   conn_manager_->onData(fake_input, false);
 
   EXPECT_CALL(*handler, log(_, _, _, _, _))
-      .Times(2)
-      .WillRepeatedly(Invoke(
-          [&](const HeaderMap* request_headers, const HeaderMap* response_headers, const HeaderMap*,
-              const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType access_log_type) {
-            EXPECT_EQ(AccessLog::AccessLogType::DownstreamPeriodic, access_log_type);
-            EXPECT_EQ(&decoder_->streamInfo(), &stream_info);
-            EXPECT_THAT(request_headers, testing::NotNull());
-            EXPECT_THAT(response_headers, testing::IsNull());
-            EXPECT_EQ(stream_info.requestComplete(), absl::nullopt);
-          }));
+      .WillOnce(Invoke([&](const HeaderMap* request_headers, const HeaderMap* response_headers,
+                           const HeaderMap*, const StreamInfo::StreamInfo& stream_info,
+                           AccessLog::AccessLogType access_log_type) {
+        EXPECT_EQ(AccessLog::AccessLogType::DownstreamPeriodic, access_log_type);
+        EXPECT_EQ(&decoder_->streamInfo(), &stream_info);
+        EXPECT_THAT(request_headers, testing::NotNull());
+        EXPECT_THAT(response_headers, testing::IsNull());
+        EXPECT_EQ(stream_info.requestComplete(), absl::nullopt);
+        EXPECT_THAT(stream_info.getDownstreamBytesMeter()->bytesAtLastDownstreamPeriodicLog(),
+                    testing::IsNull());
+      }))
+      .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*,
+                          const StreamInfo::StreamInfo& stream_info,
+                          AccessLog::AccessLogType access_log_type) {
+        EXPECT_EQ(AccessLog::AccessLogType::DownstreamPeriodic, access_log_type);
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()
+                      ->bytesAtLastDownstreamPeriodicLog()
+                      ->wire_bytes_received,
+                  4);
+      }));
   // Pretend like some 30s has passed, and the log should be written.
   EXPECT_CALL(*periodic_log_timer, enableTimer(*access_log_flush_interval_, _)).Times(2);
   periodic_log_timer->invokeCallback();
+  // Add additional bytes.
+  response_encoder_.stream_.bytes_meter_->addWireBytesReceived(12);
   periodic_log_timer->invokeCallback();
   EXPECT_CALL(*handler, log(_, _, _, _, _))
       .WillOnce(Invoke([&](const HeaderMap* request_headers, const HeaderMap* response_headers,
@@ -2598,6 +2611,10 @@ TEST_F(HttpConnectionManagerImplTest, TestPeriodicAccessLogging) {
         EXPECT_THAT(stream_info.responseCodeDetails(),
                     testing::Optional(testing::StrEq("details")));
         EXPECT_THAT(stream_info.responseCode(), testing::Optional(200));
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()
+                      ->bytesAtLastDownstreamPeriodicLog()
+                      ->wire_bytes_received,
+                  4 + 12);
       }));
   filter->callbacks_->streamInfo().setResponseCodeDetails("");
   ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}};
diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc
index 62d9fd8dafc2..43e255f9861a 100644
--- a/test/common/router/router_upstream_log_test.cc
+++ b/test/common/router/router_upstream_log_test.cc
@@ -93,17 +93,17 @@ class RouterUpstreamLogTest : public testing::Test {
     ON_CALL(*cluster_info_, name()).WillByDefault(ReturnRef(cluster_name));
     ON_CALL(*cluster_info_, observabilityName()).WillByDefault(ReturnRef(cluster_name));
     ON_CALL(callbacks_.stream_info_, upstreamClusterInfo()).WillByDefault(Return(cluster_info_));
+    callbacks_.stream_info_.downstream_bytes_meter_ = std::make_shared();
     EXPECT_CALL(callbacks_.dispatcher_, deferredDelete_).Times(testing::AnyNumber());
 
-    if (upstream_log) {
-      auto upstream_log_options = router_proto.mutable_upstream_log_options();
-      upstream_log_options->set_flush_upstream_log_on_upstream_stream(
-          flush_upstream_log_on_upstream_stream);
-
-      if (enable_periodic_upstream_log) {
-        upstream_log_options->mutable_upstream_log_flush_interval()->set_seconds(1);
-      }
+    auto upstream_log_options = router_proto.mutable_upstream_log_options();
+    upstream_log_options->set_flush_upstream_log_on_upstream_stream(
+        flush_upstream_log_on_upstream_stream);
 
+    if (enable_periodic_upstream_log) {
+      upstream_log_options->mutable_upstream_log_flush_interval()->set_seconds(1);
+    }
+    if (upstream_log) {
       ON_CALL(*context_.access_log_manager_.file_, write(_))
           .WillByDefault(
               Invoke([&](absl::string_view data) { output_.push_back(std::string(data)); }));
@@ -116,6 +116,8 @@ class RouterUpstreamLogTest : public testing::Test {
     Stats::StatNameManagedStorage prefix("prefix", context_.scope().symbolTable());
     config_ = std::make_shared(prefix.statName(), context_,
                                              ShadowWriterPtr(new MockShadowWriter()), router_proto);
+    mock_upstream_log_ = std::make_shared>();
+    config_->upstream_logs_.push_back(mock_upstream_log_);
     router_ = std::make_shared(*config_, config_->default_stats_);
     router_->setDecoderFilterCallbacks(callbacks_);
     EXPECT_CALL(callbacks_.dispatcher_, pushTrackedObject(_)).Times(testing::AnyNumber());
@@ -279,6 +281,7 @@ class RouterUpstreamLogTest : public testing::Test {
   std::shared_ptr router_;
   std::shared_ptr> cluster_info_;
   NiceMock stream_info_;
+  std::shared_ptr> mock_upstream_log_;
 };
 
 TEST_F(RouterUpstreamLogTest, NoLogConfigured) {
@@ -450,20 +453,9 @@ name: accesslog
 }
 
 TEST_F(RouterUpstreamLogTest, PeriodicLog) {
-  const std::string yaml = R"EOF(
-name: accesslog
-typed_config:
-  "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
-  log_format:
-    text_format_source:
-      inline_string: "%ACCESS_LOG_TYPE%"
-  path: "/dev/null"
-  )EOF";
-
-  envoy::config::accesslog::v3::AccessLog upstream_log;
-  TestUtility::loadFromYaml(yaml, upstream_log);
-
-  init(absl::optional(upstream_log), false, true);
+  init(absl::nullopt,
+       /*flush_upstream_log_on_upstream_stream=*/false,
+       /*enable_periodic_upstream_log=*/true);
 
   NiceMock encoder;
   Http::ResponseDecoder* response_decoder = nullptr;
@@ -476,6 +468,7 @@ name: accesslog
             response_decoder = &decoder;
             EXPECT_CALL(encoder.stream_, connectionInfoProvider())
                 .WillRepeatedly(ReturnRef(connection_info1_));
+            encoder.stream_.bytes_meter_ = std::make_shared();
             callbacks.onPoolReady(encoder,
                                   context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_,
                                   stream_info_, Http::Protocol::Http10);
@@ -487,30 +480,71 @@ name: accesslog
 
   Http::TestRequestHeaderMapImpl headers;
   HttpTestUtility::addDefaultHeaders(headers);
+  callbacks_.stream_info_.downstream_bytes_meter_->addWireBytesReceived(10);
 
   EXPECT_CALL(*periodic_log_flush_, enableTimer(_, _));
   router_->decodeHeaders(headers, true);
 
   EXPECT_CALL(*router_->retry_state_, shouldRetryHeaders(_, _, _))
       .WillOnce(Return(RetryStatus::No));
+  encoder.stream_.bytes_meter_->addWireBytesReceived(9);
 
   EXPECT_CALL(*periodic_log_flush_, enableTimer(_, _));
+  EXPECT_CALL(*mock_upstream_log_, log(_, _, _, _, AccessLog::AccessLogType::UpstreamPeriodic))
+      .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*,
+                          const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) {
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 10);
+
+        EXPECT_THAT(stream_info.getDownstreamBytesMeter()->bytesAtLastUpstreamPeriodicLog(),
+                    testing::IsNull());
+        EXPECT_EQ(stream_info.getUpstreamBytesMeter()->wireBytesReceived(), 9);
+        EXPECT_THAT(stream_info.getUpstreamBytesMeter()->bytesAtLastUpstreamPeriodicLog(),
+                    testing::IsNull());
+      }));
   periodic_log_flush_->invokeCallback();
-  EXPECT_EQ(output_.size(), 1U);
-  EXPECT_EQ(output_.front(), AccessLogType_Name(AccessLog::AccessLogType::UpstreamPeriodic));
+
+  callbacks_.stream_info_.downstream_bytes_meter_->addWireBytesReceived(8);
+  encoder.stream_.bytes_meter_->addWireBytesReceived(7);
 
   EXPECT_CALL(*periodic_log_flush_, enableTimer(_, _));
+  EXPECT_CALL(*mock_upstream_log_, log(_, _, _, _, AccessLog::AccessLogType::UpstreamPeriodic))
+      .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*,
+                          const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) {
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 10 + 8);
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()
+                      ->bytesAtLastUpstreamPeriodicLog()
+                      ->wire_bytes_received,
+                  10);
+        EXPECT_EQ(stream_info.getUpstreamBytesMeter()->wireBytesReceived(), 9 + 7);
+        EXPECT_EQ(stream_info.getUpstreamBytesMeter()
+                      ->bytesAtLastUpstreamPeriodicLog()
+                      ->wire_bytes_received,
+                  9);
+      }));
   periodic_log_flush_->invokeCallback();
-  EXPECT_EQ(output_.size(), 2U);
-  EXPECT_EQ(output_.front(), AccessLogType_Name(AccessLog::AccessLogType::UpstreamPeriodic));
-  EXPECT_EQ(output_.back(), AccessLogType_Name(AccessLog::AccessLogType::UpstreamPeriodic));
 
   Http::ResponseHeaderMapPtr response_headers(new Http::TestResponseHeaderMapImpl());
   response_headers->setStatus(200);
 
+  callbacks_.stream_info_.downstream_bytes_meter_->addWireBytesReceived(6);
+  encoder.stream_.bytes_meter_->addWireBytesReceived(5);
+
   EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
               putHttpResponseCode(200));
-
+  EXPECT_CALL(*mock_upstream_log_, log(_, _, _, _, AccessLog::AccessLogType::UpstreamEnd))
+      .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*,
+                          const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) {
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 10 + 8 + 6);
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()
+                      ->bytesAtLastUpstreamPeriodicLog()
+                      ->wire_bytes_received,
+                  10 + 8);
+        EXPECT_EQ(stream_info.getUpstreamBytesMeter()->wireBytesReceived(), 9 + 7 + 5);
+        EXPECT_EQ(stream_info.getUpstreamBytesMeter()
+                      ->bytesAtLastUpstreamPeriodicLog()
+                      ->wire_bytes_received,
+                  9 + 7);
+      }));
   response_decoder->decodeHeaders(std::move(response_headers), false);
 
   EXPECT_CALL(*periodic_log_flush_, disableTimer());
diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc
index 735d9ba6ae41..7faa95fb5d2d 100644
--- a/test/common/tcp_proxy/tcp_proxy_test.cc
+++ b/test/common/tcp_proxy/tcp_proxy_test.cc
@@ -70,6 +70,9 @@ class TcpProxyTest : public TcpProxyTestBase {
     }
 
     configure(config);
+    mock_access_logger_ = std::make_shared>();
+    const_cast&>(config_->accessLogs())
+        .push_back(mock_access_logger_);
     upstream_local_address_ = Network::Utility::resolveUrl("tcp://2.2.2.2:50000");
     upstream_remote_address_ = Network::Utility::resolveUrl("tcp://127.0.0.1:80");
     for (uint32_t i = 0; i < connections; i++) {
@@ -160,6 +163,7 @@ class TcpProxyTest : public TcpProxyTestBase {
   // Saved api handle pointer. The pointer is assigned in setup() in most of the on demand cases.
   // In these cases, the mocked allocateOdCdsApi() takes the ownership.
   Upstream::MockOdCdsApiHandle* mock_odcds_api_handle_{};
+  std::shared_ptr> mock_access_logger_;
 };
 
 TEST_F(TcpProxyTest, ExplicitCluster) {
@@ -970,11 +974,34 @@ TEST_F(TcpProxyTest, IntermediateLogEntry) {
 
   // The timer will be enabled cyclically.
   EXPECT_CALL(*flush_timer, enableTimer(std::chrono::milliseconds(1000), _));
+  filter_callbacks_.connection_.stream_info_.downstream_bytes_meter_->addWireBytesReceived(10);
+  EXPECT_CALL(*mock_access_logger_, log(_, _, _, _, AccessLog::AccessLogType::TcpPeriodic))
+      .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*,
+                          const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) {
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 10);
+        EXPECT_THAT(stream_info.getDownstreamBytesMeter()->bytesAtLastDownstreamPeriodicLog(),
+                    testing::IsNull());
+      }));
   flush_timer->invokeCallback();
 
   // No valid duration until the connection is closed.
   EXPECT_EQ(access_log_data_.value(), AccessLogType_Name(AccessLog::AccessLogType::TcpPeriodic));
 
+  filter_callbacks_.connection_.stream_info_.downstream_bytes_meter_->addWireBytesReceived(9);
+  EXPECT_CALL(*mock_access_logger_, log(_, _, _, _, AccessLog::AccessLogType::TcpPeriodic))
+      .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*,
+                          const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) {
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 19);
+        EXPECT_EQ(stream_info.getDownstreamBytesMeter()
+                      ->bytesAtLastDownstreamPeriodicLog()
+                      ->wire_bytes_received,
+                  10);
+      }));
+  EXPECT_CALL(*flush_timer, enableTimer(std::chrono::milliseconds(1000), _));
+  flush_timer->invokeCallback();
+
+  EXPECT_CALL(*mock_access_logger_, log(_, _, _, _, AccessLog::AccessLogType::TcpConnectionEnd));
+
   filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
   filter_.reset();
 

From 6cebb9b2290cf7694270eb0f4cce0a098dd5a633 Mon Sep 17 00:00:00 2001
From: "Adi (Suissa) Peleg" 
Date: Wed, 14 Jun 2023 20:29:52 -0400
Subject: [PATCH 541/740] upstream: minor code refactor calling a function once
 (#27963)

Signed-off-by: Adi Suissa-Peleg 
---
 source/common/upstream/upstream_impl.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc
index 95af949905f5..8041988e0fbf 100644
--- a/source/common/upstream/upstream_impl.cc
+++ b/source/common/upstream/upstream_impl.cc
@@ -1471,10 +1471,10 @@ ClusterImplBase::partitionHostList(const HostVector& hosts) {
   auto excluded_list = std::make_shared();
 
   for (const auto& host : hosts) {
-    if (host->coarseHealth() == Host::Health::Healthy) {
+    const Host::Health health_status = host->coarseHealth();
+    if (health_status == Host::Health::Healthy) {
       healthy_list->get().emplace_back(host);
-    }
-    if (host->coarseHealth() == Host::Health::Degraded) {
+    } else if (health_status == Host::Health::Degraded) {
       degraded_list->get().emplace_back(host);
     }
     if (excludeBasedOnHealthFlag(*host)) {

From 779d7310edb9a27ea6519a226a808aa0391f3c97 Mon Sep 17 00:00:00 2001
From: ohadvano <49730675+ohadvano@users.noreply.github.com>
Date: Thu, 15 Jun 2023 04:10:17 +0300
Subject: [PATCH 542/740] debug log: add log to indicate which upstream
 connection is attached to downstream (#27856)

* Update conn_pool_base.h

Signed-off-by: ohadvano <49730675+ohadvano@users.noreply.github.com>

* Update conn_pool_base.cc

Signed-off-by: ohadvano <49730675+ohadvano@users.noreply.github.com>

* revert previous approach

Signed-off-by: ohadvano 

* add log in tcp_proxy

Signed-off-by: ohadvano 

* fix format

Signed-off-by: ohadvano 

* add condition

Signed-off-by: ohadvano 

* add log in upstream_request

Signed-off-by: ohadvano 

* fix format

Signed-off-by: ohadvano 

---------

Signed-off-by: ohadvano <49730675+ohadvano@users.noreply.github.com>
Signed-off-by: ohadvano 
---
 source/common/router/upstream_request.cc      |  6 ++++++
 source/common/tcp_proxy/upstream.cc           | 20 +++++++++++++++++--
 source/common/tcp_proxy/upstream.h            |  4 +++-
 .../upstreams/tcp/generic/config.cc           |  4 ++--
 4 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc
index b2bdd2564773..630465d000ac 100644
--- a/source/common/router/upstream_request.cc
+++ b/source/common/router/upstream_request.cc
@@ -663,6 +663,12 @@ void UpstreamRequest::onPoolReady(std::unique_ptr&& upstream,
     upstreamLog(AccessLog::AccessLogType::UpstreamPoolReady);
   }
 
+  if (address_provider.connectionID() && stream_info_.downstreamAddressProvider().connectionID()) {
+    ENVOY_LOG(debug, "Attached upstream connection [C{}] to downstream connection [C{}]",
+              address_provider.connectionID().value(),
+              stream_info_.downstreamAddressProvider().connectionID().value());
+  }
+
   for (auto* callback : upstream_callbacks_) {
     callback->onUpstreamConnectionEstablished();
   }
diff --git a/source/common/tcp_proxy/upstream.cc b/source/common/tcp_proxy/upstream.cc
index b186b136bfd4..ea9e2ba73eb1 100644
--- a/source/common/tcp_proxy/upstream.cc
+++ b/source/common/tcp_proxy/upstream.cc
@@ -166,8 +166,9 @@ void HttpUpstream::doneWriting() {
 
 TcpConnPool::TcpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
                          Upstream::LoadBalancerContext* context,
-                         Tcp::ConnectionPool::UpstreamCallbacks& upstream_callbacks)
-    : upstream_callbacks_(upstream_callbacks) {
+                         Tcp::ConnectionPool::UpstreamCallbacks& upstream_callbacks,
+                         StreamInfo::StreamInfo& downstream_info)
+    : upstream_callbacks_(upstream_callbacks), downstream_info_(downstream_info) {
   conn_pool_data_ = thread_local_cluster.tcpConnPool(Upstream::ResourcePriority::Default, context);
 }
 
@@ -199,6 +200,12 @@ void TcpConnPool::onPoolFailure(ConnectionPool::PoolFailureReason reason,
 
 void TcpConnPool::onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr&& conn_data,
                               Upstream::HostDescriptionConstSharedPtr host) {
+  if (downstream_info_.downstreamAddressProvider().connectionID()) {
+    ENVOY_LOG(debug, "Attached upstream connection [C{}] to downstream connection [C{}]",
+              conn_data->connection().id(),
+              downstream_info_.downstreamAddressProvider().connectionID().value());
+  }
+
   upstream_handle_ = nullptr;
   Tcp::ConnectionPool::ConnectionData* latched_data = conn_data.get();
   Network::Connection& connection = conn_data->connection();
@@ -261,6 +268,15 @@ void HttpConnPool::onPoolFailure(ConnectionPool::PoolFailureReason reason,
 void HttpConnPool::onPoolReady(Http::RequestEncoder& request_encoder,
                                Upstream::HostDescriptionConstSharedPtr host,
                                StreamInfo::StreamInfo& info, absl::optional) {
+  if (info.downstreamAddressProvider().connectionID() &&
+      downstream_info_.downstreamAddressProvider().connectionID()) {
+    // info.downstreamAddressProvider() is being called to get the upstream connection ID,
+    // because the StreamInfo object here is of the upstream connection.
+    ENVOY_LOG(debug, "Attached upstream connection [C{}] to downstream connection [C{}]",
+              info.downstreamAddressProvider().connectionID().value(),
+              downstream_info_.downstreamAddressProvider().connectionID().value());
+  }
+
   upstream_handle_ = nullptr;
   upstream_->setRequestEncoder(request_encoder,
                                host->transportSocketFactory().implementsSecureTransport());
diff --git a/source/common/tcp_proxy/upstream.h b/source/common/tcp_proxy/upstream.h
index 2cdf57cc83b0..feeea15a56c2 100644
--- a/source/common/tcp_proxy/upstream.h
+++ b/source/common/tcp_proxy/upstream.h
@@ -21,7 +21,8 @@ class TcpConnPool : public GenericConnPool, public Tcp::ConnectionPool::Callback
 public:
   TcpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
               Upstream::LoadBalancerContext* context,
-              Tcp::ConnectionPool::UpstreamCallbacks& upstream_callbacks);
+              Tcp::ConnectionPool::UpstreamCallbacks& upstream_callbacks,
+              StreamInfo::StreamInfo& downstream_info);
   ~TcpConnPool() override;
 
   bool valid() const { return conn_pool_data_.has_value(); }
@@ -41,6 +42,7 @@ class TcpConnPool : public GenericConnPool, public Tcp::ConnectionPool::Callback
   Tcp::ConnectionPool::Cancellable* upstream_handle_{};
   GenericConnectionPoolCallbacks* callbacks_{};
   Tcp::ConnectionPool::UpstreamCallbacks& upstream_callbacks_;
+  StreamInfo::StreamInfo& downstream_info_;
 };
 
 class HttpUpstream;
diff --git a/source/extensions/upstreams/tcp/generic/config.cc b/source/extensions/upstreams/tcp/generic/config.cc
index ed3770304c78..709323a2a04b 100644
--- a/source/extensions/upstreams/tcp/generic/config.cc
+++ b/source/extensions/upstreams/tcp/generic/config.cc
@@ -31,8 +31,8 @@ TcpProxy::GenericConnPoolPtr GenericConnPoolFactory::createGenericConnPool(
         thread_local_cluster, context, *config, upstream_callbacks, pool_type, downstream_info);
     return (ret->valid() ? std::move(ret) : nullptr);
   }
-  auto ret =
-      std::make_unique(thread_local_cluster, context, upstream_callbacks);
+  auto ret = std::make_unique(thread_local_cluster, context,
+                                                     upstream_callbacks, downstream_info);
   return (ret->valid() ? std::move(ret) : nullptr);
 }
 

From 43a247f6f2602d0809a5f03cb2326c2244ce05fd Mon Sep 17 00:00:00 2001
From: StarryNight 
Date: Thu, 15 Jun 2023 19:39:27 +0800
Subject: [PATCH 543/740] go extension: add lock in go side to avoid
 concurrency race (#27959)

* go extension: add lock in go side to avoid concurrency race

Signed-off-by: wangkai19 

* add some annotation

Signed-off-by: wangkai19 

* fix format

Signed-off-by: wangkai19 

* Kick CI

Signed-off-by: wangkai19 

* Kick CI

Signed-off-by: wangkai19 

---------

Signed-off-by: wangkai19 
---
 .../filters/http/source/go/pkg/http/capi.go   |  2 +-
 .../http/source/go/pkg/http/capi_impl.go      |  8 ++++--
 .../filters/http/source/go/pkg/http/filter.go | 15 +++++-----
 .../filters/http/source/go/pkg/http/type.go   | 28 +++++++++++++++++++
 4 files changed, 42 insertions(+), 11 deletions(-)

diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi.go b/contrib/golang/filters/http/source/go/pkg/http/capi.go
index 1e35e3c8f85e..40238541b8a5 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/capi.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/capi.go
@@ -43,7 +43,7 @@ type HttpCAPI interface {
 	HttpSetTrailer(r unsafe.Pointer, key *string, value *string, add bool)
 	HttpRemoveTrailer(r unsafe.Pointer, key *string)
 
-	HttpGetStringValue(r unsafe.Pointer, id int) (string, bool)
+	HttpGetStringValue(r *httpRequest, id int) (string, bool)
 	HttpGetIntegerValue(r unsafe.Pointer, id int) (uint64, bool)
 
 	// TODO: HttpGetDynamicMetadata(r unsafe.Pointer, filterName string) map[string]interface{}
diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
index 1cb8a4e65939..1ba63f71c238 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
@@ -216,11 +216,13 @@ func (c *httpCApiImpl) HttpRemoveTrailer(r unsafe.Pointer, key *string) {
 	handleCApiStatus(res)
 }
 
-func (c *httpCApiImpl) HttpGetStringValue(r unsafe.Pointer, id int) (string, bool) {
+func (c *httpCApiImpl) HttpGetStringValue(r *httpRequest, id int) (string, bool) {
 	var value string
-	// TODO: add a lock to protect filter->req_->strValue field in the Envoy side, from being writing concurrency,
+	// add a lock to protect filter->req_->strValue field in the Envoy side, from being writing concurrency,
 	// since there might be multiple concurrency goroutines invoking this API on the Go side.
-	res := C.envoyGoFilterHttpGetStringValue(r, C.int(id), unsafe.Pointer(&value))
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	res := C.envoyGoFilterHttpGetStringValue(unsafe.Pointer(r.req), C.int(id), unsafe.Pointer(&value))
 	if res == C.CAPIValueNotFound {
 		return "", false
 	}
diff --git a/contrib/golang/filters/http/source/go/pkg/http/filter.go b/contrib/golang/filters/http/source/go/pkg/http/filter.go
index 01b43e47b129..74def40b8996 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/filter.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/filter.go
@@ -63,6 +63,7 @@ type httpRequest struct {
 	pInfo          panicInfo
 	sema           sync.WaitGroup
 	waitingOnEnvoy int32
+	mutex          sync.Mutex
 }
 
 func (r *httpRequest) pluginName() string {
@@ -137,12 +138,12 @@ type streamInfo struct {
 }
 
 func (s *streamInfo) GetRouteName() string {
-	name, _ := cAPI.HttpGetStringValue(unsafe.Pointer(s.request.req), ValueRouteName)
+	name, _ := cAPI.HttpGetStringValue(s.request, ValueRouteName)
 	return name
 }
 
 func (s *streamInfo) FilterChainName() string {
-	name, _ := cAPI.HttpGetStringValue(unsafe.Pointer(s.request.req), ValueFilterChainName)
+	name, _ := cAPI.HttpGetStringValue(s.request, ValueFilterChainName)
 	return name
 }
 
@@ -164,7 +165,7 @@ func (s *streamInfo) ResponseCode() (uint32, bool) {
 }
 
 func (s *streamInfo) ResponseCodeDetails() (string, bool) {
-	return cAPI.HttpGetStringValue(unsafe.Pointer(s.request.req), ValueResponseCodeDetails)
+	return cAPI.HttpGetStringValue(s.request, ValueResponseCodeDetails)
 }
 
 func (s *streamInfo) AttemptCount() uint32 {
@@ -187,21 +188,21 @@ func (d *dynamicMetadata) Set(filterName string, key string, value interface{})
 }
 
 func (s *streamInfo) DownstreamLocalAddress() string {
-	address, _ := cAPI.HttpGetStringValue(unsafe.Pointer(s.request.req), ValueDownstreamLocalAddress)
+	address, _ := cAPI.HttpGetStringValue(s.request, ValueDownstreamLocalAddress)
 	return address
 }
 
 func (s *streamInfo) DownstreamRemoteAddress() string {
-	address, _ := cAPI.HttpGetStringValue(unsafe.Pointer(s.request.req), ValueDownstreamRemoteAddress)
+	address, _ := cAPI.HttpGetStringValue(s.request, ValueDownstreamRemoteAddress)
 	return address
 }
 
 func (s *streamInfo) UpstreamHostAddress() (string, bool) {
-	return cAPI.HttpGetStringValue(unsafe.Pointer(s.request.req), ValueUpstreamHostAddress)
+	return cAPI.HttpGetStringValue(s.request, ValueUpstreamHostAddress)
 }
 
 func (s *streamInfo) UpstreamClusterName() (string, bool) {
-	return cAPI.HttpGetStringValue(unsafe.Pointer(s.request.req), ValueUpstreamClusterName)
+	return cAPI.HttpGetStringValue(s.request, ValueUpstreamClusterName)
 }
 
 type filterState struct {
diff --git a/contrib/golang/filters/http/source/go/pkg/http/type.go b/contrib/golang/filters/http/source/go/pkg/http/type.go
index a9c35dd85ca1..2011190c7534 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/type.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/type.go
@@ -19,6 +19,7 @@ package http
 
 import (
 	"strconv"
+	"sync"
 	"unsafe"
 
 	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
@@ -38,6 +39,7 @@ type headerMapImpl struct {
 	headers     map[string][]string
 	headerNum   uint64
 	headerBytes uint64
+	mutex       sync.Mutex
 }
 
 // ByteSize return size of HeaderMap
@@ -62,6 +64,8 @@ func (h *requestOrResponseHeaderMapImpl) GetRaw(key string) string {
 }
 
 func (h *requestOrResponseHeaderMapImpl) Get(key string) (string, bool) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initHeaders()
 	value, ok := h.headers[key]
 	if !ok {
@@ -71,6 +75,8 @@ func (h *requestOrResponseHeaderMapImpl) Get(key string) (string, bool) {
 }
 
 func (h *requestOrResponseHeaderMapImpl) Values(key string) []string {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initHeaders()
 	value, ok := h.headers[key]
 	if !ok {
@@ -83,6 +89,8 @@ func (h *requestOrResponseHeaderMapImpl) Set(key, value string) {
 	// Get all header values first before setting a value, since the set operation may not take affects immediately
 	// when it's invoked in a Go thread, instead, it will post a callback to run in the envoy worker thread.
 	// Otherwise, we may get outdated values in a following Get call.
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initHeaders()
 	if h.headers != nil {
 		h.headers[key] = []string{value}
@@ -91,6 +99,8 @@ func (h *requestOrResponseHeaderMapImpl) Set(key, value string) {
 }
 
 func (h *requestOrResponseHeaderMapImpl) Add(key, value string) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initHeaders()
 	if h.headers != nil {
 		if hdrs, found := h.headers[key]; found {
@@ -106,12 +116,17 @@ func (h *requestOrResponseHeaderMapImpl) Del(key string) {
 	// Get all header values first before removing a key, since the del operation may not take affects immediately
 	// when it's invoked in a Go thread, instead, it will post a callback to run in the envoy worker thread.
 	// Otherwise, we may get outdated values in a following Get call.
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initHeaders()
 	delete(h.headers, key)
 	cAPI.HttpRemoveHeader(unsafe.Pointer(h.request.req), &key)
 }
 
 func (h *requestOrResponseHeaderMapImpl) Range(f func(key, value string) bool) {
+	// To avoid dead lock, methods with lock(Get, Values, Set, Add, Del) should not be used in func f.
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initHeaders()
 	for key, values := range h.headers {
 		for _, value := range values {
@@ -186,6 +201,8 @@ func (h *requestOrResponseTrailerMapImpl) GetRaw(key string) string {
 }
 
 func (h *requestOrResponseTrailerMapImpl) Get(key string) (string, bool) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initTrailers()
 	value, ok := h.headers[key]
 	if !ok {
@@ -195,6 +212,8 @@ func (h *requestOrResponseTrailerMapImpl) Get(key string) (string, bool) {
 }
 
 func (h *requestOrResponseTrailerMapImpl) Values(key string) []string {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initTrailers()
 	value, ok := h.headers[key]
 	if !ok {
@@ -207,6 +226,8 @@ func (h *requestOrResponseTrailerMapImpl) Set(key, value string) {
 	// Get all header values first before setting a value, since the set operation may not take affects immediately
 	// when it's invoked in a Go thread, instead, it will post a callback to run in the envoy worker thread.
 	// Otherwise, we may get outdated values in a following Get call.
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initTrailers()
 	if h.headers != nil {
 		h.headers[key] = []string{value}
@@ -216,6 +237,8 @@ func (h *requestOrResponseTrailerMapImpl) Set(key, value string) {
 }
 
 func (h *requestOrResponseTrailerMapImpl) Add(key, value string) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initTrailers()
 	if h.headers != nil {
 		if trailers, found := h.headers[key]; found {
@@ -228,12 +251,17 @@ func (h *requestOrResponseTrailerMapImpl) Add(key, value string) {
 }
 
 func (h *requestOrResponseTrailerMapImpl) Del(key string) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initTrailers()
 	delete(h.headers, key)
 	cAPI.HttpRemoveTrailer(unsafe.Pointer(h.request.req), &key)
 }
 
 func (h *requestOrResponseTrailerMapImpl) Range(f func(key, value string) bool) {
+	// To avoid dead lock, methods with lock(Get, Values, Set, Add, Del) should not be used in func f.
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 	h.initTrailers()
 	for key, values := range h.headers {
 		for _, value := range values {

From c6a9a24987ebaab94a529fbd1da1ab89ec480d81 Mon Sep 17 00:00:00 2001
From: Joseph Straceski 
Date: Thu, 15 Jun 2023 09:46:33 -0400
Subject: [PATCH 544/740] Adding per route matching to the extension with
 matcher. (#27088)

Adding per route match tree resolution to the extension with matcher.

Risk Level: Low
Testing: test/common/http/match_delegate/match_delegate_integration_test.cc, test/extensions/filters/http/composite/composite_filter_integration_test.cc, test/common/http/match_delegate/config_test.cc
Docs Changes: matching_api.rst
Release Notes: changelogs/current.yaml
Platform Specific Features: N/A

Signed-off-by: Joseph Straceski 
---
 .../matching/v3/extension_matcher.proto       |   6 +
 changelogs/current.yaml                       |  11 +
 source/common/http/match_delegate/BUILD       |   3 +
 source/common/http/match_delegate/config.cc   |  72 ++++++-
 source/common/http/match_delegate/config.h    |  46 +++-
 test/common/http/match_delegate/BUILD         |  17 ++
 .../common/http/match_delegate/config_test.cc |  81 +++++--
 .../match_delegate_integration_test.cc        | 104 +++++++++
 test/extensions/filters/http/composite/BUILD  |   2 +
 .../composite_filter_integration_test.cc      | 198 +++++++++++++++---
 .../filters/set_response_code_filter.cc       |  11 +-
 11 files changed, 485 insertions(+), 66 deletions(-)
 create mode 100644 test/common/http/match_delegate/match_delegate_integration_test.cc

diff --git a/api/envoy/extensions/common/matching/v3/extension_matcher.proto b/api/envoy/extensions/common/matching/v3/extension_matcher.proto
index 53dcf9d4b703..bef7c712eb2e 100644
--- a/api/envoy/extensions/common/matching/v3/extension_matcher.proto
+++ b/api/envoy/extensions/common/matching/v3/extension_matcher.proto
@@ -37,3 +37,9 @@ message ExtensionWithMatcher {
   config.core.v3.TypedExtensionConfig extension_config = 2
       [(validate.rules).message = {required: true}];
 }
+
+// Extra settings on a per virtualhost/route/weighted-cluster level.
+message ExtensionWithMatcherPerRoute {
+  // Matcher override.
+  xds.type.matcher.v3.Matcher xds_matcher = 1;
+}
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index b050838d1f4d..0fceea0650c6 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -287,6 +287,17 @@ new_features:
     added new field :ref:`connection_rate_limit
     `
     to limit reconnection rate to redis server to avoid reconnection storm.
+- area: match_delegate
+  change: |
+    added :ref:`per route configuration
+    ` to the
+    :ref:`ExtensionWithMatcher
+    ` filter.
+    Which allows the associated matcher to be defined on a per route basis.
+- area: match_delegate
+  change: |
+    If no matcher is set the :ref:`ExtensionWithMatcher
+    ` filter is now set to skip rather than erroring out.
 - area: access_log
   change: |
     added additional HCM access log option :ref:`flush_log_on_tunnel_successfully_established
diff --git a/source/common/http/match_delegate/BUILD b/source/common/http/match_delegate/BUILD
index 2e1fbaa93e01..f86049878266 100644
--- a/source/common/http/match_delegate/BUILD
+++ b/source/common/http/match_delegate/BUILD
@@ -19,11 +19,14 @@ envoy_cc_library(
     deps = [
         "//envoy/registry",
         "//envoy/server:filter_config_interface",
+        "//source/common/config:utility_lib",
+        "//source/common/http:utility_lib",
         "//source/common/http/matching:data_impl_lib",
         "//source/common/matcher:matcher_lib",
         "//source/extensions/filters/http/common:factory_base_lib",
         "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/common/matcher/action/v3:pkg_cc_proto",
+        "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto",
     ],
     alwayslink = LEGACY_ALWAYSLINK,
 )
diff --git a/source/common/http/match_delegate/config.cc b/source/common/http/match_delegate/config.cc
index 587c8de71104..9b2b77eaec5b 100644
--- a/source/common/http/match_delegate/config.cc
+++ b/source/common/http/match_delegate/config.cc
@@ -1,13 +1,18 @@
+#include "config.h"
 #include "source/common/http/match_delegate/config.h"
 
 #include 
 
 #include "envoy/http/filter.h"
 #include "envoy/registry/registry.h"
+#include "envoy/type/matcher/v3/http_inputs.pb.h"
+#include "envoy/type/matcher/v3/http_inputs.pb.validate.h"
 
 #include "source/common/config/utility.h"
+#include "source/common/http/utility.h"
 
 #include "absl/status/status.h"
+#include "xds/type/matcher/v3/http_inputs.pb.h"
 
 namespace Envoy {
 namespace Common {
@@ -96,9 +101,17 @@ struct DelegatingFactoryCallbacks : public Envoy::Http::FilterChainFactoryCallba
 
 void DelegatingStreamFilter::FilterMatchState::evaluateMatchTree(
     MatchDataUpdateFunc data_update_func) {
-  if (!has_match_tree_ || match_tree_evaluated_) {
+  if (match_tree_evaluated_) {
     return;
   }
+
+  // If no match tree is set, interpret as a skip.
+  if (!has_match_tree_) {
+    skip_filter_ = true;
+    match_tree_evaluated_ = true;
+    return;
+  }
+
   ASSERT(matching_data_ != nullptr);
   data_update_func(*matching_data_);
 
@@ -135,6 +148,18 @@ DelegatingStreamFilter::DelegatingStreamFilter(
 
 Envoy::Http::FilterHeadersStatus
 DelegatingStreamFilter::decodeHeaders(Envoy::Http::RequestHeaderMap& headers, bool end_stream) {
+  const auto* per_route_config =
+      Envoy::Http::Utility::resolveMostSpecificPerFilterConfig(
+          decoder_callbacks_);
+
+  if (per_route_config != nullptr) {
+    match_state_.setMatchTree(per_route_config->matchTree());
+  } else {
+    ENVOY_LOG(
+        trace,
+        "No per route config found, thus the matcher tree can not be built from per route config");
+  }
+
   match_state_.evaluateMatchTree([&headers](Envoy::Http::Matching::HttpMatchingDataImpl& data) {
     data.onRequestHeaders(headers);
   });
@@ -194,7 +219,6 @@ DelegatingStreamFilter::encode1xxHeaders(Envoy::Http::ResponseHeaderMap& headers
 
 Envoy::Http::FilterHeadersStatus
 DelegatingStreamFilter::encodeHeaders(Envoy::Http::ResponseHeaderMap& headers, bool end_stream) {
-
   match_state_.evaluateMatchTree([&headers](Envoy::Http::Matching::HttpMatchingDataImpl& data) {
     data.onResponseHeaders(headers);
   });
@@ -266,13 +290,12 @@ Envoy::Http::FilterFactoryCb MatchDelegateConfig::createFilterFactoryFromProtoTy
   Matcher::MatchTreeFactory
       matcher_factory(action_context, context.getServerFactoryContext(), validation_visitor);
-  Matcher::MatchTreeFactoryCb factory_cb;
+  absl::optional> factory_cb =
+      std::nullopt;
   if (proto_config.has_xds_matcher()) {
     factory_cb = matcher_factory.create(proto_config.xds_matcher());
   } else if (proto_config.has_matcher()) {
     factory_cb = matcher_factory.create(proto_config.matcher());
-  } else {
-    throw EnvoyException("one of `matcher` and `matcher_tree` must be set.");
   }
 
   if (!validation_visitor.errors().empty()) {
@@ -281,13 +304,50 @@ Envoy::Http::FilterFactoryCb MatchDelegateConfig::createFilterFactoryFromProtoTy
                                      validation_visitor.errors()[0]));
   }
 
-  Matcher::MatchTreeSharedPtr match_tree = factory_cb();
+  Matcher::MatchTreeSharedPtr match_tree = nullptr;
+
+  if (factory_cb.has_value()) {
+    match_tree = factory_cb.value()();
+  }
+
   return [filter_factory, match_tree](Envoy::Http::FilterChainFactoryCallbacks& callbacks) -> void {
     Factory::DelegatingFactoryCallbacks delegating_callbacks(callbacks, match_tree);
     return filter_factory(delegating_callbacks);
   };
 }
 
+Router::RouteSpecificFilterConfigConstSharedPtr
+MatchDelegateConfig::createRouteSpecificFilterConfigTyped(
+    const envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute& proto_config,
+    Server::Configuration::ServerFactoryContext& server_context,
+    ProtobufMessage::ValidationVisitor&) {
+  return std::make_shared(proto_config, server_context);
+}
+
+Matcher::MatchTreeSharedPtr
+FilterConfigPerRoute::createFilterMatchTree(
+    const envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute& proto_config,
+    Server::Configuration::ServerFactoryContext& server_context) {
+  auto requirements =
+      std::make_unique();
+  requirements->mutable_data_input_allow_list()->add_type_url(TypeUtil::descriptorFullNameToTypeUrl(
+      envoy::type::matcher::v3::HttpRequestHeaderMatchInput::descriptor()->full_name()));
+  requirements->mutable_data_input_allow_list()->add_type_url(TypeUtil::descriptorFullNameToTypeUrl(
+      xds::type::matcher::v3::HttpAttributesCelMatchInput::descriptor()->full_name()));
+  Envoy::Http::Matching::HttpFilterActionContext action_context{
+      fmt::format("http.{}.",
+                  server_context.scope().symbolTable().toString(server_context.scope().prefix())),
+      absl::nullopt, server_context};
+
+  Factory::MatchTreeValidationVisitor validation_visitor(*requirements);
+  Matcher::MatchTreeFactory
+      matcher_factory(action_context, server_context, validation_visitor);
+  Matcher::MatchTreeFactoryCb factory_cb =
+      matcher_factory.create(proto_config.xds_matcher());
+  return factory_cb();
+}
+
 /**
  * Static registration for the match delegate filter. @see RegisterFactory.
  * This match delegate filter is designed as delegate of all other HTTP filters. It will create
diff --git a/source/common/http/match_delegate/config.h b/source/common/http/match_delegate/config.h
index a5e7521d5aa4..3d15dec30146 100644
--- a/source/common/http/match_delegate/config.h
+++ b/source/common/http/match_delegate/config.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include 
+
 #include "envoy/extensions/common/matching/v3/extension_matcher.pb.validate.h"
 #include "envoy/extensions/filters/common/matcher/action/v3/skip_action.pb.h"
 #include "envoy/matcher/matcher.h"
@@ -17,7 +19,8 @@ namespace MatchDelegate {
 class SkipAction : public Matcher::ActionBase<
                        envoy::extensions::filters::common::matcher::action::v3::SkipFilter> {};
 
-class DelegatingStreamFilter : public Envoy::Http::StreamFilter {
+class DelegatingStreamFilter : public Logger::Loggable,
+                               public Envoy::Http::StreamFilter {
 public:
   using MatchDataUpdateFunc = std::function;
 
@@ -29,15 +32,23 @@ class DelegatingStreamFilter : public Envoy::Http::StreamFilter {
     void evaluateMatchTree(MatchDataUpdateFunc data_update_func);
     bool skipFilter() const { return skip_filter_; }
     void onStreamInfo(const StreamInfo::StreamInfo& stream_info) {
-      if (has_match_tree_ && matching_data_ == nullptr) {
+      if (matching_data_ == nullptr) {
         matching_data_ = std::make_shared(stream_info);
       }
     }
+
+    // The matcher from the per route config, if available, will override the matcher from the
+    // filter config.
+    void setMatchTree(Matcher::MatchTreeSharedPtr match_tree) {
+      match_tree_ = std::move(match_tree);
+      has_match_tree_ = match_tree_ != nullptr;
+    }
+
     void setBaseFilter(Envoy::Http::StreamFilterBase* base_filter) { base_filter_ = base_filter; }
 
   private:
     Matcher::MatchTreeSharedPtr match_tree_;
-    const bool has_match_tree_{};
+    bool has_match_tree_{};
     Envoy::Http::StreamFilterBase* base_filter_{};
 
     Envoy::Http::Matching::HttpMatchingDataImplSharedPtr matching_data_;
@@ -92,8 +103,10 @@ class DelegatingStreamFilter : public Envoy::Http::StreamFilter {
   Envoy::Http::StreamFilterBase* base_filter_{};
 };
 
-class MatchDelegateConfig : public Extensions::HttpFilters::Common::FactoryBase<
-                                envoy::extensions::common::matching::v3::ExtensionWithMatcher> {
+class MatchDelegateConfig
+    : public Extensions::HttpFilters::Common::FactoryBase<
+          envoy::extensions::common::matching::v3::ExtensionWithMatcher,
+          envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute> {
 public:
   // TODO(wbpcode): move this filter to 'source/extensions/filters/http'.
   MatchDelegateConfig() : FactoryBase("envoy.filters.http.match_delegate") {}
@@ -102,6 +115,29 @@ class MatchDelegateConfig : public Extensions::HttpFilters::Common::FactoryBase<
   Envoy::Http::FilterFactoryCb createFilterFactoryFromProtoTyped(
       const envoy::extensions::common::matching::v3::ExtensionWithMatcher& proto_config,
       const std::string&, Server::Configuration::FactoryContext& context) override;
+
+  Router::RouteSpecificFilterConfigConstSharedPtr createRouteSpecificFilterConfigTyped(
+      const envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute& proto_config,
+      Server::Configuration::ServerFactoryContext& context,
+      ProtobufMessage::ValidationVisitor& validation) override;
+};
+
+class FilterConfigPerRoute : public Router::RouteSpecificFilterConfig {
+public:
+  FilterConfigPerRoute(
+      const envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute& proto_config,
+      Server::Configuration::ServerFactoryContext& server_context)
+      : match_tree_(createFilterMatchTree(proto_config, server_context)) {}
+
+  const Matcher::MatchTreeSharedPtr& matchTree() const {
+    return match_tree_;
+  }
+
+private:
+  Matcher::MatchTreeSharedPtr createFilterMatchTree(
+      const envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute& proto_config,
+      Server::Configuration::ServerFactoryContext& server_context);
+  Matcher::MatchTreeSharedPtr match_tree_;
 };
 
 DECLARE_FACTORY(MatchDelegateConfig);
diff --git a/test/common/http/match_delegate/BUILD b/test/common/http/match_delegate/BUILD
index 254d0bf87350..7ca57f5c887c 100644
--- a/test/common/http/match_delegate/BUILD
+++ b/test/common/http/match_delegate/BUILD
@@ -18,3 +18,20 @@ envoy_cc_test(
         "//test/test_common:test_runtime_lib",
     ],
 )
+
+envoy_cc_test(
+    name = "match_delegate_integration_test",
+    srcs = ["match_delegate_integration_test.cc"],
+    deps = [
+        "//source/extensions/filters/http/common:factory_base_lib",
+        "//source/server/config_validation:server_lib",
+        "//test/integration:http_integration_lib",
+        "//test/integration/filters:set_response_code_filter_config_proto_cc_proto",
+        "//test/integration/filters:set_response_code_filter_lib",
+        "//test/mocks/server:options_mocks",
+        "//test/test_common:utility_lib",
+        "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
+        "@envoy_api//envoy/config/listener/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto",
+    ],
+)
diff --git a/test/common/http/match_delegate/config_test.cc b/test/common/http/match_delegate/config_test.cc
index 7cbcd104e3bb..662d60821602 100644
--- a/test/common/http/match_delegate/config_test.cc
+++ b/test/common/http/match_delegate/config_test.cc
@@ -100,6 +100,38 @@ TEST(MatchWrapper, WithMatcher) {
   cb(factory_callbacks);
 }
 
+TEST(MatchWrapper, PerRouteConfig) {
+  TestFactory test_factory;
+  Envoy::Registry::InjectFactory
+      inject_factory(test_factory);
+
+  const auto yaml = (R"EOF(
+xds_matcher:
+  matcher_tree:
+    input:
+      name: request-headers
+      typed_config:
+        "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput
+        header_name: default-matcher-header
+    exact_match_map:
+        map:
+            match:
+                action:
+                    name: skip
+                    typed_config:
+                        "@type": type.googleapis.com/envoy.extensions.filters.common.matcher.action.v3.SkipFilter
+)EOF");
+
+  MatchDelegateConfig factory;
+  NiceMock server_factory_context;
+  envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute config;
+  TestUtility::loadFromYamlAndValidate(yaml, config);
+  Router::RouteSpecificFilterConfigConstSharedPtr route_config =
+      factory.createRouteSpecificFilterConfig(config, server_factory_context,
+                                              ProtobufMessage::getNullValidationVisitor());
+  EXPECT_TRUE(route_config.get());
+}
+
 TEST(MatchWrapper, DEPRECATED_FEATURE_TEST(WithDeprecatedMatcher)) {
   TestFactory test_factory;
   Envoy::Registry::InjectFactory
@@ -186,27 +218,6 @@ TEST(MatchWrapper, QueryParamMatcherYaml) {
   EXPECT_TRUE(cb);
 }
 
-TEST(MatchWrapper, WithNoMatcher) {
-  TestFactory test_factory;
-  Envoy::Registry::InjectFactory
-      inject_factory(test_factory);
-
-  NiceMock factory_context;
-
-  const auto config =
-      TestUtility::parseYaml(R"EOF(
-extension_config:
-  name: test
-  typed_config:
-    "@type": type.googleapis.com/google.protobuf.StringValue
-)EOF");
-
-  MatchDelegateConfig match_delegate_config;
-  EXPECT_THROW_WITH_REGEX(
-      match_delegate_config.createFilterFactoryFromProto(config, "", factory_context),
-      EnvoyException, "one of `matcher` and `matcher_tree` must be set.");
-}
-
 TEST(MatchWrapper, WithMatcherInvalidDataInput) {
   TestFactory test_factory;
   Envoy::Registry::InjectFactory
@@ -330,6 +341,34 @@ createMatchTreeWithOnNoMatch(const std::string& name, const std::string& value)
   return tree;
 }
 
+// Test that the DelegatingStreamFilter skips when no matcher is defined.
+TEST(DelegatingFilterTest, WithNoMatcher) {
+  std::shared_ptr decoder_filter(
+      new Envoy::Http::MockStreamDecoderFilter());
+  NiceMock callbacks;
+
+  EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_));
+  EXPECT_CALL(*decoder_filter, decodeHeaders(_, _)).Times(0);
+  EXPECT_CALL(*decoder_filter, decodeData(_, _)).Times(0);
+  EXPECT_CALL(*decoder_filter, decodeMetadata(_)).Times(0);
+  EXPECT_CALL(*decoder_filter, decodeTrailers(_)).Times(0);
+  EXPECT_CALL(*decoder_filter, decodeComplete()).Times(0);
+
+  auto match_tree =
+      createMatchingTree(
+          "match_query", "match");
+  auto delegating_filter =
+      std::make_shared(nullptr, decoder_filter, nullptr);
+
+  Envoy::Http::RequestHeaderMapPtr request_headers{new Envoy::Http::TestRequestHeaderMapImpl{
+      {":authority", "host"}, {":path", "/"}, {":method", "GET"}, {"match-header", "not_match"}}};
+
+  delegating_filter->setDecoderFilterCallbacks(callbacks);
+  EXPECT_EQ(Envoy::Http::FilterHeadersStatus::Continue,
+            delegating_filter->decodeHeaders(*request_headers, false));
+  delegating_filter->decodeComplete();
+}
+
 TEST(DelegatingFilterTest, MatchTreeOnNoMatchSkipActionDecodingHeaders) {
   // The filter is added, but the `on_no_match` action will be invoked since it is not matched on
   // request header. The skip action is provided to `on_no_match` so we skip this filter.
diff --git a/test/common/http/match_delegate/match_delegate_integration_test.cc b/test/common/http/match_delegate/match_delegate_integration_test.cc
new file mode 100644
index 000000000000..dca1a61d2c7c
--- /dev/null
+++ b/test/common/http/match_delegate/match_delegate_integration_test.cc
@@ -0,0 +1,104 @@
+#include "envoy/config/bootstrap/v3/bootstrap.pb.h"
+#include "envoy/config/listener/v3/listener_components.pb.h"
+#include "envoy/extensions/common/matching/v3/extension_matcher.pb.validate.h"
+
+#include "source/common/http/match_delegate/config.h"
+
+#include "test/integration/filters/set_response_code_filter_config.pb.h"
+#include "test/integration/http_integration.h"
+#include "test/mocks/server/options.h"
+#include "test/test_common/utility.h"
+
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace Common {
+namespace Http {
+namespace MatchDelegate {
+namespace {
+
+using envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute;
+using Envoy::Protobuf::MapPair;
+using Envoy::ProtobufWkt::Any;
+
+class MatchDelegateInegrationTest : public testing::TestWithParam,
+                                    public HttpIntegrationTest {
+public:
+  MatchDelegateInegrationTest() : HttpIntegrationTest(Envoy::Http::CodecType::HTTP1, GetParam()) {}
+  void initialize() override {
+    config_helper_.prependFilter(default_config_);
+    HttpIntegrationTest::initialize();
+
+    codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http")));
+  }
+
+  const Envoy::Http::TestRequestHeaderMapImpl default_request_headers_ = {{":method", "GET"},
+                                                                          {":path", "/test"},
+                                                                          {":scheme", "http"},
+                                                                          {"match-header", "route"},
+                                                                          {":authority", "host"}};
+
+  const std::string default_config_ = R"EOF(
+      name: ext_proc
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher
+        extension_config:
+          name: set-response-code
+          typed_config:
+            "@type": type.googleapis.com/test.integration.filters.SetResponseCodeFilterConfig
+            code: 403
+    )EOF";
+  const std::string per_route_config_ = R"EOF(
+      xds_matcher:
+        matcher_tree:
+          input:
+            name: request-headers
+            typed_config:
+              "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput
+              header_name: match-header
+          exact_match_map:
+            map:
+              match:
+                action:
+                  name: skip
+                  typed_config:
+                    "@type": type.googleapis.com/envoy.extensions.filters.common.matcher.action.v3.SkipFilter
+    )EOF";
+};
+
+INSTANTIATE_TEST_SUITE_P(IpVersions, MatchDelegateInegrationTest,
+                         testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
+                         TestUtility::ipTestParamsToString);
+
+TEST_P(MatchDelegateInegrationTest, NoMatcherDefault) {
+  initialize();
+  Envoy::Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}};
+  auto response = sendRequestAndWaitForResponse(default_request_headers_, 0, response_headers, 0);
+  ASSERT_TRUE(response->waitForEndStream());
+  EXPECT_TRUE(response->complete());
+  EXPECT_THAT(response->headers(), Envoy::Http::HttpStatusIs("200"));
+}
+
+TEST_P(MatchDelegateInegrationTest, PerRouteConfig) {
+  config_helper_.addConfigModifier([this](ConfigHelper::HttpConnectionManager& cm) {
+    auto* vh = cm.mutable_route_config()->mutable_virtual_hosts()->Mutable(0);
+    auto* route = vh->mutable_routes()->Mutable(0);
+    route->mutable_route()->set_cluster("cluster_0");
+    route->mutable_match()->set_prefix("/test");
+    const auto matcher = TestUtility::parseYaml(per_route_config_);
+    Any cfg_any;
+    ASSERT_TRUE(cfg_any.PackFrom(matcher));
+    route->mutable_typed_per_filter_config()->insert(
+        MapPair("envoy.filters.http.match_delegate", cfg_any));
+  });
+  initialize();
+  auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_);
+  ASSERT_TRUE(response->waitForEndStream());
+  EXPECT_THAT(response->headers(), Envoy::Http::HttpStatusIs("403"));
+}
+
+} // namespace
+} // namespace MatchDelegate
+} // namespace Http
+} // namespace Common
+} // namespace Envoy
diff --git a/test/extensions/filters/http/composite/BUILD b/test/extensions/filters/http/composite/BUILD
index 4c60b323301d..f846deb7b778 100644
--- a/test/extensions/filters/http/composite/BUILD
+++ b/test/extensions/filters/http/composite/BUILD
@@ -42,11 +42,13 @@ envoy_extension_cc_test(
         "//test/integration:http_integration_lib",
         "//test/integration/filters:server_factory_context_filter_config_proto_cc_proto",
         "//test/integration/filters:server_factory_context_filter_lib",
+        "//test/integration/filters:set_response_code_filter_config_proto_cc_proto",
         "//test/integration/filters:set_response_code_filter_lib",
         "//test/proto:helloworld_proto_cc_proto",
         "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/http/composite/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto",
+        "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto",
     ],
 )
diff --git a/test/extensions/filters/http/composite/composite_filter_integration_test.cc b/test/extensions/filters/http/composite/composite_filter_integration_test.cc
index aaad8465ed98..fe75b1a45213 100644
--- a/test/extensions/filters/http/composite/composite_filter_integration_test.cc
+++ b/test/extensions/filters/http/composite/composite_filter_integration_test.cc
@@ -1,59 +1,129 @@
+#include 
+#include 
+
 #include "envoy/extensions/common/matching/v3/extension_matcher.pb.validate.h"
 #include "envoy/extensions/filters/http/composite/v3/composite.pb.h"
 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h"
+#include "envoy/matcher/matcher.h"
 #include "envoy/network/address.h"
+#include "envoy/type/matcher/v3/http_inputs.pb.h"
+#include "envoy/type/matcher/v3/http_inputs.pb.validate.h"
 
 #include "source/common/http/match_delegate/config.h"
 #include "source/common/http/matching/inputs.h"
 
 #include "test/common/grpc/grpc_client_integration.h"
 #include "test/common/http/common.h"
+#include "test/integration/filters/add_body_filter.pb.h"
 #include "test/integration/filters/server_factory_context_filter_config.pb.h"
+#include "test/integration/filters/set_response_code_filter_config.pb.h"
 #include "test/integration/http_integration.h"
 #include "test/mocks/http/mocks.h"
 #include "test/proto/helloworld.pb.h"
 #include "test/test_common/utility.h"
 
+#include "absl/strings/str_format.h"
 #include "gtest/gtest.h"
 
 namespace Envoy {
 
+namespace {
+
+using envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute;
+using envoy::extensions::filters::http::composite::v3::ExecuteFilterAction;
+using envoy::type::matcher::v3::HttpRequestHeaderMatchInput;
+using test::integration::filters::SetResponseCodeFilterConfig;
+using xds::type::matcher::v3::Matcher_OnMatch;
+
 class CompositeFilterIntegrationTest : public testing::TestWithParam,
                                        public HttpIntegrationTest {
 public:
   CompositeFilterIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {}
 
-  void initialize() override {
-    config_helper_.prependFilter(R"EOF(
-  name: composite
-  typed_config:
-    "@type": type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher
-    extension_config:
-      name: composite
+  ExtensionWithMatcherPerRoute createPerRouteConfig(
+      std::function base_action_function) {
+
+    ExtensionWithMatcherPerRoute per_route_config;
+    auto matcher_tree = per_route_config.mutable_xds_matcher()->mutable_matcher_tree();
+
+    auto matcher_input = matcher_tree->mutable_input();
+    matcher_input->set_name("request-headers");
+    HttpRequestHeaderMatchInput match_input;
+    match_input.set_header_name("match-header");
+    matcher_input->mutable_typed_config()->PackFrom(match_input);
+
+    auto map = matcher_tree->mutable_exact_match_map()->mutable_map();
+    Matcher_OnMatch match;
+    auto mutable_action = match.mutable_action();
+    mutable_action->set_name("composite-action");
+
+    ExecuteFilterAction filter_action;
+    base_action_function(filter_action.mutable_typed_config());
+    mutable_action->mutable_typed_config()->PackFrom(filter_action);
+
+    (*map)["match"] = match;
+    return per_route_config;
+  }
+
+  void addPerRouteResponseCodeFilter(const std::string& filter_name,
+                                     const std::string& route_prefix, const int& code,
+                                     bool response_prefix = false) {
+    SetResponseCodeFilterConfig set_response_code;
+    set_response_code.set_code(code);
+    if (response_prefix) {
+      set_response_code.set_prefix("skipLocalReplyAndContinue");
+    }
+    auto per_route_config = createPerRouteConfig([set_response_code](auto* cfg) {
+      cfg->set_name("set-response-code-filter");
+      cfg->mutable_typed_config()->PackFrom(set_response_code);
+    });
+    config_helper_.addConfigModifier(
+        [per_route_config, filter_name, route_prefix](ConfigHelper::HttpConnectionManager& cm) {
+          auto* vh = cm.mutable_route_config()->mutable_virtual_hosts(0);
+          auto* route = vh->mutable_routes()->Mutable(0);
+          route->mutable_match()->set_prefix(route_prefix);
+          route->mutable_route()->set_cluster("cluster_0");
+          (*route->mutable_typed_per_filter_config())[filter_name].PackFrom(per_route_config);
+        });
+  }
+
+  void prependCompositeFilter(const std::string& name = "composite") {
+    config_helper_.prependFilter(absl::StrFormat(R"EOF(
+      name: %s
       typed_config:
-        "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.Composite
-    xds_matcher:
-      matcher_tree:
-        input:
-          name: request-headers
+        "@type": type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher
+        extension_config:
+          name: composite
           typed_config:
-            "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput
-            header_name: match-header
-        exact_match_map:
-          map:
-            match:
-              action:
-                name: composite-action
-                typed_config:
-                  "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.ExecuteFilterAction
-                  typed_config:
-                    name: set-response-code
+            "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.Composite
+        xds_matcher:
+          matcher_tree:
+            input:
+              name: request-headers
+              typed_config:
+                "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput
+                header_name: match-header
+            exact_match_map:
+              map:
+                match:
+                  action:
+                    name: composite-action
                     typed_config:
-                      "@type": type.googleapis.com/test.integration.filters.SetResponseCodeFilterConfig
-                      code: 403
-    )EOF");
-    HttpIntegrationTest::initialize();
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.ExecuteFilterAction
+                      typed_config:
+                        name: set-response-code
+                        typed_config:
+                          "@type": type.googleapis.com/test.integration.filters.SetResponseCodeFilterConfig
+                          code: 403
+    )EOF",
+                                                 name));
   }
+
+  const Http::TestRequestHeaderMapImpl match_request_headers_ = {{":method", "GET"},
+                                                                 {":path", "/somepath"},
+                                                                 {":scheme", "http"},
+                                                                 {"match-header", "match"},
+                                                                 {":authority", "blah"}};
 };
 
 INSTANTIATE_TEST_SUITE_P(IpVersions, CompositeFilterIntegrationTest,
@@ -63,6 +133,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, CompositeFilterIntegrationTest,
 // Verifies that if we don't match the match action the request is proxied as normal, while if the
 // match action is hit we apply the specified filter to the stream.
 TEST_P(CompositeFilterIntegrationTest, TestBasic) {
+  prependCompositeFilter();
   initialize();
   codec_client_ = makeHttpConnection(lookupPort("http"));
 
@@ -76,17 +147,78 @@ TEST_P(CompositeFilterIntegrationTest, TestBasic) {
   }
 
   {
-    const Http::TestRequestHeaderMapImpl request_headers = {{":method", "GET"},
-                                                            {":path", "/somepath"},
-                                                            {":scheme", "http"},
-                                                            {"match-header", "match"},
-                                                            {":authority", "blah"}};
-    auto response = codec_client_->makeRequestWithBody(request_headers, 1024);
+    auto response = codec_client_->makeRequestWithBody(match_request_headers_, 1024);
     ASSERT_TRUE(response->waitForEndStream());
     EXPECT_THAT(response->headers(), Http::HttpStatusIs("403"));
   }
 }
 
+// Verifies function of the per-route config in the ExtensionWithMatcher class.
+TEST_P(CompositeFilterIntegrationTest, TestPerRoute) {
+  prependCompositeFilter();
+  addPerRouteResponseCodeFilter(/*filter_name=*/"composite", /*route_prefix=*/"/somepath",
+                                /*code=*/401);
+  initialize();
+  codec_client_ = makeHttpConnection(lookupPort("http"));
+
+  auto response = codec_client_->makeRequestWithBody(match_request_headers_, 1024);
+  ASSERT_TRUE(response->waitForEndStream());
+  EXPECT_THAT(response->headers(), Http::HttpStatusIs("401"));
+}
+
+// Test an empty match tree resolving with a per route config.
+TEST_P(CompositeFilterIntegrationTest, TestPerRouteEmptyMatcher) {
+  config_helper_.prependFilter(R"EOF(
+      name: composite
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher
+        extension_config:
+          name: composite
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.Composite
+    )EOF");
+  addPerRouteResponseCodeFilter(/*filter_name=*/"composite", /*route_prefix=*/"/somepath",
+                                /*code=*/402);
+  initialize();
+  codec_client_ = makeHttpConnection(lookupPort("http"));
+  auto response = codec_client_->makeRequestWithBody(match_request_headers_, 1024);
+  ASSERT_TRUE(response->waitForEndStream());
+  EXPECT_THAT(response->headers(), Http::HttpStatusIs("402"));
+}
+
+// Test that the specified filters apply per route configs to requests.
+TEST_P(CompositeFilterIntegrationTest, TestPerRouteEmptyMatcherMultipleFilters) {
+  config_helper_.prependFilter(R"EOF(
+      name: composite_2
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher
+        extension_config:
+          name: composite
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.Composite
+    )EOF");
+  config_helper_.prependFilter(R"EOF(
+      name: composite
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher
+        extension_config:
+          name: composite
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.http.composite.v3.Composite
+    )EOF");
+
+  addPerRouteResponseCodeFilter(/*filter_name=*/"composite", /*route_prefix=*/"/somepath",
+                                /*code=*/407, /*response_prefix=*/true);
+  addPerRouteResponseCodeFilter(/*filter_name=*/"composite_2", /*route_prefix=*/"/somepath",
+                                /*code=*/402);
+
+  initialize();
+  codec_client_ = makeHttpConnection(lookupPort("http"));
+  auto response = codec_client_->makeRequestWithBody(match_request_headers_, 1024);
+  ASSERT_TRUE(response->waitForEndStream());
+  EXPECT_THAT(response->headers(), Http::HttpStatusIs("402"));
+}
+} // namespace
 class CompositeFilterSeverContextIntegrationTest
     : public HttpIntegrationTest,
       public Grpc::GrpcClientIntegrationParamTestWithDeferredProcessing {
diff --git a/test/integration/filters/set_response_code_filter.cc b/test/integration/filters/set_response_code_filter.cc
index b18c4fe6a692..26a8b5bedaa3 100644
--- a/test/integration/filters/set_response_code_filter.cc
+++ b/test/integration/filters/set_response_code_filter.cc
@@ -17,7 +17,7 @@ namespace Envoy {
 class SetResponseCodeFilterConfig {
 public:
   SetResponseCodeFilterConfig(const std::string& prefix, uint32_t code, const std::string& body,
-                              Server::Configuration::FactoryContext& context)
+                              Server::Configuration::FactoryContextBase& context)
       : prefix_(prefix), code_(code), body_(body), tls_slot_(context.threadLocal()) {}
 
   const std::string prefix_;
@@ -59,6 +59,15 @@ class SetResponseCodeFilterFactory : public Extensions::HttpFilters::Common::Fac
       callbacks.addStreamFilter(std::make_shared(filter_config));
     };
   }
+  Http::FilterFactoryCb createFilterServerFactoryFromProtoTyped(
+      const test::integration::filters::SetResponseCodeFilterConfig& proto_config,
+      const std::string&, Server::Configuration::ServerFactoryContext& context) override {
+    auto filter_config = std::make_shared(
+        proto_config.prefix(), proto_config.code(), proto_config.body(), context);
+    return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void {
+      callbacks.addStreamFilter(std::make_shared(filter_config));
+    };
+  }
 };
 
 REGISTER_FACTORY(SetResponseCodeFilterFactory, Server::Configuration::NamedHttpFilterConfigFactory);

From d0d224939a162e8c31747f36f031921ff1bb031c Mon Sep 17 00:00:00 2001
From: "Vikas Choudhary (vikasc)" 
Date: Thu, 15 Jun 2023 21:10:32 +0530
Subject: [PATCH 545/740] Allow connect with 2xx status codes (#27974)

* Allow connect with 2xx status codes

Signed-off-by: Vikas Choudhary 
---
 changelogs/current.yaml                       |  4 +++
 source/common/router/upstream_codec_filter.cc |  5 ++-
 source/common/runtime/runtime_features.cc     |  1 +
 .../tcp_tunneling_integration_test.cc         | 33 +++++++++++++++++++
 4 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 0fceea0650c6..c8179abf17f9 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -94,6 +94,10 @@ minor_behavior_changes:
   change: |
     Added new type of gauge with type hidden. These stats are hidden from admin/stats-sinks but can shown with a
     query-parameter of ``/stats?hidden=include`` or ``/stats?hidden=showonly``.
+- area: upstream
+  change: |
+    Changed behavior of the unpausing connect with 2xx status codes. This change can be reverted temporarily by
+    setting the runtime guard ``envoy.reloadable_features.upstream_allow_connect_with_2xx`` to false.
 
 bug_fixes:
 # *Changes expected to improve the state of the world and are unlikely to have negative effects*
diff --git a/source/common/router/upstream_codec_filter.cc b/source/common/router/upstream_codec_filter.cc
index df00ef8813da..158d2b729713 100644
--- a/source/common/router/upstream_codec_filter.cc
+++ b/source/common/router/upstream_codec_filter.cc
@@ -146,7 +146,10 @@ void UpstreamCodecFilter::CodecBridge::decodeHeaders(Http::ResponseHeaderMapPtr&
       filter_.callbacks_->dispatcher().timeSource());
 
   if (filter_.callbacks_->upstreamCallbacks()->pausedForConnect() &&
-      Http::Utility::getResponseStatus(*headers) == 200) {
+      ((Http::Utility::getResponseStatus(*headers) == 200) ||
+       ((Runtime::runtimeFeatureEnabled(
+            "envoy.reloadable_features.upstream_allow_connect_with_2xx")) &&
+        (Http::CodeUtility::is2xx(Http::Utility::getResponseStatus(*headers)))))) {
     filter_.callbacks_->upstreamCallbacks()->setPausedForConnect(false);
     filter_.callbacks_->continueDecoding();
   }
diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc
index 750b702ec48e..f576b972e9bd 100644
--- a/source/common/runtime/runtime_features.cc
+++ b/source/common/runtime/runtime_features.cc
@@ -75,6 +75,7 @@ RUNTIME_GUARD(envoy_reloadable_features_thrift_connection_draining);
 RUNTIME_GUARD(envoy_reloadable_features_uhv_allow_malformed_url_encoding);
 RUNTIME_GUARD(envoy_reloadable_features_uhv_preserve_url_encoded_case);
 RUNTIME_GUARD(envoy_reloadable_features_uhv_translate_backslash_to_slash);
+RUNTIME_GUARD(envoy_reloadable_features_upstream_allow_connect_with_2xx);
 RUNTIME_GUARD(envoy_reloadable_features_upstream_wait_for_response_headers_before_disabling_read);
 RUNTIME_GUARD(envoy_reloadable_features_use_http3_header_normalisation);
 RUNTIME_GUARD(envoy_reloadable_features_validate_connect);
diff --git a/test/integration/tcp_tunneling_integration_test.cc b/test/integration/tcp_tunneling_integration_test.cc
index 2b2220997f72..d455fbd8ce4f 100644
--- a/test/integration/tcp_tunneling_integration_test.cc
+++ b/test/integration/tcp_tunneling_integration_test.cc
@@ -682,6 +682,39 @@ TEST_P(ProxyingConnectIntegrationTest, ProxyConnectWithIP) {
   cleanupUpstreamAndDownstream();
 }
 
+TEST_P(ProxyingConnectIntegrationTest, 2xxStatusCode) {
+  initialize();
+
+  // Send request headers.
+  codec_client_ = makeHttpConnection(lookupPort("http"));
+  connect_headers_.setHost("1.2.3.4:80");
+  auto encoder_decoder = codec_client_->startRequest(connect_headers_);
+  request_encoder_ = &encoder_decoder.first;
+  response_ = std::move(encoder_decoder.second);
+
+  // Wait for them to arrive upstream.
+  AssertionResult result =
+      fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_);
+  RELEASE_ASSERT(result, result.message());
+  result = fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_);
+  RELEASE_ASSERT(result, result.message());
+  ASSERT_TRUE(upstream_request_->waitForHeadersComplete());
+  EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().Method)[0]->value(), "CONNECT");
+
+  // Send valid response headers, in HTTP1 all status codes in the 2xx range
+  // are considered valid.
+  default_response_headers_.setStatus(enumToInt(Http::Code::Accepted));
+
+  // Send response headers
+  upstream_request_->encodeHeaders(default_response_headers_, false);
+
+  // Wait for them to arrive downstream.
+  response_->waitForHeaders();
+  EXPECT_EQ("202", response_->headers().getStatusValue());
+
+  cleanupUpstreamAndDownstream();
+}
+
 // Tunneling downstream TCP over an upstream HTTP CONNECT tunnel.
 class TcpTunnelingIntegrationTest : public HttpProtocolIntegrationTest {
 public:

From 677081a97c0694f4fb73262411af05b8a9e0d50f Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Thu, 15 Jun 2023 12:29:38 -0400
Subject: [PATCH 546/740] mobile: removing legacy admin APIs. (#27969)

Admin APIs had been made test only a while back. Now that we have stats support without admin, we can remove the admin APIs entirely.

Risk Level: low
Testing: n/a
Docs Changes: n/a
Release Notes: n/a (removing test code)

Signed-off-by: Alyssa Wilk 
---
 .github/workflows/mobile-android_build.yml    |  1 -
 .../workflows/mobile-compile_time_options.yml |  2 --
 .github/workflows/mobile-ios_tests.yml        |  2 --
 mobile/library/cc/engine_builder.cc           | 28 +++---------------
 mobile/library/cc/engine_builder.h            |  4 ---
 mobile/library/common/engine.cc               | 28 +-----------------
 mobile/library/common/engine.h                | 14 +--------
 mobile/library/common/engine_handle.cc        |  4 +--
 mobile/library/common/engine_handle.h         |  6 ++--
 mobile/library/common/jni/jni_interface.cc    | 25 +++++++---------
 mobile/library/common/main_interface.cc       |  5 ++--
 mobile/library/common/main_interface.h        |  4 +--
 .../engine/EnvoyConfiguration.java            | 24 +++++++--------
 .../envoymobile/engine/JniLibrary.java        |  4 +--
 .../impl/NativeCronvoyEngineBuilderImpl.java  | 25 ++++++++--------
 .../kotlin/io/envoyproxy/envoymobile/BUILD    |  5 +---
 .../envoyproxy/envoymobile/EngineBuilder.kt   |  2 --
 .../envoymobile/EngineBuilderAdminUtil.kt     | 21 --------------
 .../library/objective-c/EnvoyConfiguration.h  |  4 +--
 .../library/objective-c/EnvoyConfiguration.mm |  7 +----
 mobile/library/objective-c/EnvoyEngineImpl.mm |  2 +-
 mobile/library/swift/EngineBuilder.swift      | 21 --------------
 mobile/test/cc/unit/envoy_config_test.cc      | 16 ----------
 mobile/test/common/engine_test.cc             |  2 +-
 .../integration/client_integration_test.cc    |  7 -----
 mobile/test/common/main_interface_test.cc     | 22 +++++++-------
 .../engine/EnvoyConfigurationTest.kt          |  5 ----
 .../kotlin/apps/experimental/MainActivity.kt  |  2 --
 .../gcp_traffic_director_integration_test.cc  |  2 +-
 mobile/test/performance/test_binary_size.cc   |  2 +-
 mobile/test/swift/EngineBuilderTests.swift    | 29 -------------------
 .../apps/experimental/ViewController.swift    |  3 --
 32 files changed, 65 insertions(+), 263 deletions(-)
 delete mode 100644 mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilderAdminUtil.kt

diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml
index f791676c9084..81814c64ada9 100644
--- a/.github/workflows/mobile-android_build.yml
+++ b/.github/workflows/mobile-android_build.yml
@@ -214,7 +214,6 @@ jobs:
         cd mobile && ./bazelw build \
             $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \
             --fat_apk_cpu=x86_64 \
-            --define=admin_functionality=enabled \
             --define envoy_mobile_listener=enabled \
             //test/kotlin/apps/experimental:hello_envoy_kt
           adb install -r --no-incremental bazel-bin/test/kotlin/apps/experimental/hello_envoy_kt.apk
diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml
index 3c9693112247..196921a57e4c 100644
--- a/.github/workflows/mobile-compile_time_options.yml
+++ b/.github/workflows/mobile-compile_time_options.yml
@@ -57,7 +57,6 @@ jobs:
             --config=ios \
             $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \
             --define=signal_trace=disabled \
-            --define=admin_html=enabled \
             --define=envoy_mobile_request_compression=disabled \
             --define=envoy_mobile_stats_reporting=disabled \
             --define=envoy_mobile_swift_cxx_interop=disabled \
@@ -91,7 +90,6 @@ jobs:
             $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \
             --fat_apk_cpu=x86_64 \
             --define=signal_trace=disabled \
-            --define=admin_html=enabled \
             --define=envoy_mobile_request_compression=disabled \
             --define=envoy_enable_http_datagrams=disabled \
             --define=google_grpc=disabled \
diff --git a/.github/workflows/mobile-ios_tests.yml b/.github/workflows/mobile-ios_tests.yml
index e6f25e960d7e..adc3e19728ce 100644
--- a/.github/workflows/mobile-ios_tests.yml
+++ b/.github/workflows/mobile-ios_tests.yml
@@ -29,7 +29,6 @@ jobs:
     - name: 'Run swift library tests'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      # runs with admin enabled due to regression testing admin interface
       # runs with the listener enabled due to IdleTimeoutTest not setting up a test backend.
       run: |
         cd mobile && ./bazelw test \
@@ -39,7 +38,6 @@ jobs:
             --define envoy_mobile_listener=enabled \
             --build_tests_only \
             $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \
-            --define=admin_functionality=enabled \
             //test/swift/...
   objctests:
     if: ${{ needs.env.outputs.mobile_ios_tests == 'true' }}
diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc
index aed3ca25acf6..250217ebc01b 100644
--- a/mobile/library/cc/engine_builder.cc
+++ b/mobile/library/cc/engine_builder.cc
@@ -174,13 +174,6 @@ EngineBuilder& EngineBuilder::enableSocketTagging(bool socket_tagging_on) {
   return *this;
 }
 
-#ifdef ENVOY_ADMIN_FUNCTIONALITY
-EngineBuilder& EngineBuilder::enableAdminInterface(bool admin_interface_on) {
-  admin_interface_enabled_ = admin_interface_on;
-  return *this;
-}
-#endif
-
 #ifdef ENVOY_ENABLE_QUIC
 EngineBuilder& EngineBuilder::enableHttp3(bool http3_on) {
   enable_http3_ = http3_on;
@@ -851,23 +844,10 @@ std::unique_ptr EngineBuilder::generate
     list->add_patterns()->set_exact("cluster_manager.cluster_added");
   }
 
-  // Admin
-  if (admin_interface_enabled_) {
-#ifdef ENVOY_ADMIN_FUNCTIONALITY
-    auto* admin_address = bootstrap->mutable_admin()->mutable_address()->mutable_socket_address();
-    admin_address->set_address("::1");
-    admin_address->set_port_value(9901);
-#else
-    throw std::runtime_error("Admin functionality was not compiled in this build of Envoy Mobile");
-#endif
-  } else {
-    // Default Envoy mobile to the lightweight API listener. This is not
-    // supported if the admin interface is enabled.
-    envoy::config::listener::v3::ApiListenerManager api;
-    auto* listener_manager = bootstrap->mutable_listener_manager();
-    listener_manager->mutable_typed_config()->PackFrom(api);
-    listener_manager->set_name("envoy.listener_manager_impl.api");
-  }
+  envoy::config::listener::v3::ApiListenerManager api;
+  auto* listener_manager = bootstrap->mutable_listener_manager();
+  listener_manager->mutable_typed_config()->PackFrom(api);
+  listener_manager->set_name("envoy.listener_manager_impl.api");
 
   return bootstrap;
 }
diff --git a/mobile/library/cc/engine_builder.h b/mobile/library/cc/engine_builder.h
index 84b8ce489983..c2562a2ddfdd 100644
--- a/mobile/library/cc/engine_builder.h
+++ b/mobile/library/cc/engine_builder.h
@@ -92,9 +92,6 @@ class EngineBuilder {
   EngineBuilder& setForceAlwaysUsev6(bool value);
   EngineBuilder& addDnsPreresolveHostnames(const std::vector& hostnames);
   EngineBuilder& addNativeFilter(std::string name, std::string typed_config);
-#ifdef ENVOY_ADMIN_FUNCTIONALITY
-  EngineBuilder& enableAdminInterface(bool admin_interface_on);
-#endif
 
 #ifdef ENVOY_MOBILE_STATS_REPORTING
   EngineBuilder& addStatsSinks(std::vector stat_sinks);
@@ -162,7 +159,6 @@ class EngineBuilder {
 
   absl::flat_hash_map key_value_stores_{};
 
-  bool admin_interface_enabled_ = false;
   bool enable_interface_binding_ = false;
   bool enable_drain_post_dns_refresh_ = false;
   bool enforce_trust_chain_verification_ = true;
diff --git a/mobile/library/common/engine.cc b/mobile/library/common/engine.cc
index 323caad30446..f73cda268298 100644
--- a/mobile/library/common/engine.cc
+++ b/mobile/library/common/engine.cc
@@ -21,8 +21,7 @@ Engine::Engine(envoy_engine_callbacks callbacks, envoy_logger logger,
   Envoy::Api::External::registerApi(std::string(envoy_event_tracker_api_name), &event_tracker_);
 }
 
-envoy_status_t Engine::run(const std::string config, const std::string log_level,
-                           const std::string admin_address_path) {
+envoy_status_t Engine::run(const std::string config, const std::string log_level) {
   // Start the Envoy on the dedicated thread. Note: due to how the assignment operator works with
   // std::thread, main_thread_ is the same object after this call, but its state is replaced with
   // that of the temporary. The temporary object's state becomes the default state, which does
@@ -33,9 +32,6 @@ envoy_status_t Engine::run(const std::string config, const std::string log_level
     options->setLogLevel(options->parseAndValidateLogLevel(log_level.c_str()));
   }
   options->setConcurrency(1);
-  if (!admin_address_path.empty()) {
-    options->setAdminAddressPath(admin_address_path);
-  }
   return run(std::move(options));
 }
 
@@ -186,28 +182,6 @@ envoy_status_t Engine::recordCounterInc(const std::string& elements, envoy_stats
   return ENVOY_SUCCESS;
 }
 
-envoy_status_t Engine::makeAdminCall(absl::string_view path, absl::string_view method,
-                                     envoy_data& out) {
-  ENVOY_LOG(trace, "admin call {} {}", method, path);
-  if (!server_->admin()) {
-    ENVOY_LOG(warn, "admin support compiled out.");
-    return ENVOY_FAILURE;
-  }
-
-  ASSERT(dispatcher_->isThreadSafe(), "admin calls must be run from the dispatcher's context");
-  auto response_headers = Http::ResponseHeaderMapImpl::create();
-  std::string body;
-  const auto code = server_->admin()->request(path, method, *response_headers, body);
-  if (code != Http::Code::OK) {
-    ENVOY_LOG(warn, "admin call failed with status {} body {}", static_cast(code), body);
-    return ENVOY_FAILURE;
-  }
-
-  out = Data::Utility::copyToBridgeData(body);
-
-  return ENVOY_SUCCESS;
-}
-
 Event::ProvisionalDispatcher& Engine::dispatcher() { return *dispatcher_; }
 
 Http::Client& Engine::httpClient() {
diff --git a/mobile/library/common/engine.h b/mobile/library/common/engine.h
index 3f5298017baa..7a61b8366bf7 100644
--- a/mobile/library/common/engine.h
+++ b/mobile/library/common/engine.h
@@ -34,10 +34,8 @@ class Engine : public Logger::Loggable {
    * Run the engine with the provided configuration.
    * @param config, the Envoy bootstrap configuration to use.
    * @param log_level, the log level.
-   * @param admin_address_path to set --admin-address-path, or an empty string if not needed.
    */
-  envoy_status_t run(std::string config, std::string log_level,
-                     const std::string admin_address_path);
+  envoy_status_t run(std::string config, std::string log_level);
   envoy_status_t run(std::unique_ptr&& options);
 
   /**
@@ -72,16 +70,6 @@ class Engine : public Logger::Loggable {
   envoy_status_t recordCounterInc(const std::string& elements, envoy_stats_tags tags,
                                   uint64_t count);
 
-  /**
-   * Issue a call against the admin handler, populating the `out` parameter with the response if
-   * the call was successful.
-   * @param path the admin path to query.
-   * @param method the HTTP method to use (GET or POST).
-   * @param out the response body, populated if the call is successful.
-   * @returns ENVOY_SUCCESS if the call was successful and `out` was populated.
-   */
-  envoy_status_t makeAdminCall(absl::string_view path, absl::string_view method, envoy_data& out);
-
   /**
    * Dump Envoy stats into the returned buffer
    * @returns a buffer with referenced stats dumped in Envoy's standard text format.
diff --git a/mobile/library/common/engine_handle.cc b/mobile/library/common/engine_handle.cc
index 975386b7af23..51287f6daebe 100644
--- a/mobile/library/common/engine_handle.cc
+++ b/mobile/library/common/engine_handle.cc
@@ -17,9 +17,9 @@ envoy_engine_t EngineHandle::initEngine(envoy_engine_callbacks callbacks, envoy_
 }
 
 envoy_status_t EngineHandle::runEngine(envoy_engine_t handle, const char* config,
-                                       const char* log_level, const char* admin_address_path) {
+                                       const char* log_level) {
   if (auto engine = reinterpret_cast(handle)) {
-    engine->run(config, log_level, admin_address_path);
+    engine->run(config, log_level);
     return ENVOY_SUCCESS;
   }
   return ENVOY_FAILURE;
diff --git a/mobile/library/common/engine_handle.h b/mobile/library/common/engine_handle.h
index 1c97d8e5bfad..3b91029e0a3c 100644
--- a/mobile/library/common/engine_handle.h
+++ b/mobile/library/common/engine_handle.h
@@ -25,15 +25,13 @@ class EngineHandle {
 private:
   static envoy_engine_t initEngine(envoy_engine_callbacks callbacks, envoy_logger logger,
                                    envoy_event_tracker event_tracker);
-  static envoy_status_t runEngine(envoy_engine_t, const char* config, const char* log_level,
-                                  const char* admin_address_path);
+  static envoy_status_t runEngine(envoy_engine_t, const char* config, const char* log_level);
   static envoy_status_t terminateEngine(envoy_engine_t handle, bool release);
 
   // Allow a specific list of functions to access the internal setup/teardown functionality.
   friend envoy_engine_t(::init_engine)(envoy_engine_callbacks callbacks, envoy_logger logger,
                                        envoy_event_tracker event_tracker);
-  friend envoy_status_t(::run_engine)(envoy_engine_t, const char* config, const char* log_level,
-                                      const char* admin_address_path);
+  friend envoy_status_t(::run_engine)(envoy_engine_t, const char* config, const char* log_level);
   friend envoy_status_t(::terminate_engine)(envoy_engine_t engine, bool release);
 };
 
diff --git a/mobile/library/common/jni/jni_interface.cc b/mobile/library/common/jni/jni_interface.cc
index 327377d1de19..daeb92d36b9a 100644
--- a/mobile/library/common/jni/jni_interface.cc
+++ b/mobile/library/common/jni/jni_interface.cc
@@ -133,7 +133,7 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibra
 
   jint result;
   if (!bootstrap) {
-    result = run_engine(engine, string_config, string_log_level, "");
+    result = run_engine(engine, string_config, string_log_level);
   } else {
     auto options = std::make_unique();
     options->setConfigProto(std::move(bootstrap));
@@ -1195,11 +1195,10 @@ javaObjectArrayToStringPairVector(JNIEnv* env, jobjectArray entries) {
 }
 
 void configureBuilder(
-    JNIEnv* env, jstring grpc_stats_domain, jboolean admin_interface_enabled,
-    jlong connect_timeout_seconds, jlong dns_refresh_seconds,
-    jlong dns_failure_refresh_seconds_base, jlong dns_failure_refresh_seconds_max,
-    jlong dns_query_timeout_seconds, jlong dns_min_refresh_seconds,
-    jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache,
+    JNIEnv* env, jstring grpc_stats_domain, jlong connect_timeout_seconds,
+    jlong dns_refresh_seconds, jlong dns_failure_refresh_seconds_base,
+    jlong dns_failure_refresh_seconds_max, jlong dns_query_timeout_seconds,
+    jlong dns_min_refresh_seconds, jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache,
     jlong dns_cache_save_interval_seconds, jboolean enable_drain_post_dns_refresh,
     jboolean enable_http3, jboolean enable_gzip_decompression, jboolean enable_brotli_decompression,
     jboolean enable_socket_tagging, jboolean enable_interface_binding,
@@ -1231,9 +1230,6 @@ void configureBuilder(
 
   builder.setStreamIdleTimeoutSeconds((stream_idle_timeout_seconds));
   builder.setPerTryIdleTimeoutSeconds((per_try_idle_timeout_seconds));
-#ifdef ENVOY_ADMIN_FUNCTIONALITY
-  builder.enableAdminInterface(admin_interface_enabled == JNI_TRUE);
-#endif
   builder.enableGzipDecompression(enable_gzip_decompression == JNI_TRUE);
   builder.enableBrotliDecompression(enable_brotli_decompression == JNI_TRUE);
   builder.enableSocketTagging(enable_socket_tagging == JNI_TRUE);
@@ -1293,11 +1289,10 @@ void configureBuilder(
 }
 
 extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_createBootstrap(
-    JNIEnv* env, jclass, jstring grpc_stats_domain, jboolean admin_interface_enabled,
-    jlong connect_timeout_seconds, jlong dns_refresh_seconds,
-    jlong dns_failure_refresh_seconds_base, jlong dns_failure_refresh_seconds_max,
-    jlong dns_query_timeout_seconds, jlong dns_min_refresh_seconds,
-    jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache,
+    JNIEnv* env, jclass, jstring grpc_stats_domain, jlong connect_timeout_seconds,
+    jlong dns_refresh_seconds, jlong dns_failure_refresh_seconds_base,
+    jlong dns_failure_refresh_seconds_max, jlong dns_query_timeout_seconds,
+    jlong dns_min_refresh_seconds, jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache,
     jlong dns_cache_save_interval_seconds, jboolean enable_drain_post_dns_refresh,
     jboolean enable_http3, jboolean enable_gzip_decompression, jboolean enable_brotli_decompression,
     jboolean enable_socket_tagging, jboolean enable_interface_binding,
@@ -1314,7 +1309,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr
   Envoy::Platform::EngineBuilder builder;
 
   configureBuilder(
-      env, grpc_stats_domain, admin_interface_enabled, connect_timeout_seconds, dns_refresh_seconds,
+      env, grpc_stats_domain, connect_timeout_seconds, dns_refresh_seconds,
       dns_failure_refresh_seconds_base, dns_failure_refresh_seconds_max, dns_query_timeout_seconds,
       dns_min_refresh_seconds, dns_preresolve_hostnames, enable_dns_cache,
       dns_cache_save_interval_seconds, enable_drain_post_dns_refresh, enable_http3,
diff --git a/mobile/library/common/main_interface.cc b/mobile/library/common/main_interface.cc
index 96c581ba6707..8b280d272ac1 100644
--- a/mobile/library/common/main_interface.cc
+++ b/mobile/library/common/main_interface.cc
@@ -121,9 +121,8 @@ envoy_engine_t init_engine(envoy_engine_callbacks callbacks, envoy_logger logger
   return Envoy::EngineHandle::initEngine(callbacks, logger, event_tracker);
 }
 
-envoy_status_t run_engine(envoy_engine_t engine, const char* config, const char* log_level,
-                          const char* admin_path) {
-  return Envoy::EngineHandle::runEngine(engine, config, log_level, admin_path);
+envoy_status_t run_engine(envoy_engine_t engine, const char* config, const char* log_level) {
+  return Envoy::EngineHandle::runEngine(engine, config, log_level);
 }
 
 envoy_status_t terminate_engine(envoy_engine_t engine, bool release) {
diff --git a/mobile/library/common/main_interface.h b/mobile/library/common/main_interface.h
index 0704113ed2ce..cd4f763c6571 100644
--- a/mobile/library/common/main_interface.h
+++ b/mobile/library/common/main_interface.h
@@ -203,11 +203,9 @@ envoy_engine_t init_engine(envoy_engine_callbacks callbacks, envoy_logger logger
  * @param engine, handle to the engine to run.
  * @param config, the configuration blob to run envoy with.
  * @param log_level, the logging level to run envoy with.
- * @param admin_path, the file path to log the admin address to if desired.
  * @return envoy_status_t, the resulting status of the operation.
  */
-envoy_status_t run_engine(envoy_engine_t engine, const char* config, const char* log_level,
-                          const char* admin_path);
+envoy_status_t run_engine(envoy_engine_t engine, const char* config, const char* log_level);
 
 /**
  * Terminate an engine. Further interactions with a terminated engine, or streams created by a
diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java
index 036e5358a0cf..d98af79c1fd0 100644
--- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java
+++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java
@@ -28,7 +28,6 @@ public enum TrustChainVerification {
     ACCEPT_UNTRUSTED;
   }
 
-  public final Boolean adminInterfaceEnabled;
   public final String grpcStatsDomain;
   public final Integer connectTimeoutSeconds;
   public final Integer dnsRefreshSeconds;
@@ -81,8 +80,6 @@ public enum TrustChainVerification {
   /**
    * Create a new instance of the configuration.
    *
-   * @param adminInterfaceEnabled                         whether admin interface should be enabled
-   *     or not.
    * @param grpcStatsDomain                               the domain to flush stats to.
    * @param connectTimeoutSeconds                         timeout for new network connections to
    *     hosts in
@@ -152,11 +149,11 @@ public enum TrustChainVerification {
    *     could be empty.
    */
   public EnvoyConfiguration(
-      boolean adminInterfaceEnabled, String grpcStatsDomain, int connectTimeoutSeconds,
-      int dnsRefreshSeconds, int dnsFailureRefreshSecondsBase, int dnsFailureRefreshSecondsMax,
-      int dnsQueryTimeoutSeconds, int dnsMinRefreshSeconds, List dnsPreresolveHostnames,
-      boolean enableDNSCache, int dnsCacheSaveIntervalSeconds, boolean enableDrainPostDnsRefresh,
-      boolean enableHttp3, boolean enableGzipDecompression, boolean enableBrotliDecompression,
+      String grpcStatsDomain, int connectTimeoutSeconds, int dnsRefreshSeconds,
+      int dnsFailureRefreshSecondsBase, int dnsFailureRefreshSecondsMax, int dnsQueryTimeoutSeconds,
+      int dnsMinRefreshSeconds, List dnsPreresolveHostnames, boolean enableDNSCache,
+      int dnsCacheSaveIntervalSeconds, boolean enableDrainPostDnsRefresh, boolean enableHttp3,
+      boolean enableGzipDecompression, boolean enableBrotliDecompression,
       boolean enableSocketTagging, boolean enableInterfaceBinding,
       int h2ConnectionKeepaliveIdleIntervalMilliseconds, int h2ConnectionKeepaliveTimeoutSeconds,
       int maxConnectionsPerHost, int statsFlushSeconds, int streamIdleTimeoutSeconds,
@@ -172,7 +169,6 @@ public EnvoyConfiguration(
       String nodeRegion, String nodeZone, String nodeSubZone, String cdsResourcesLocator,
       Integer cdsTimeoutSeconds, boolean enableCds) {
     JniLibrary.load();
-    this.adminInterfaceEnabled = adminInterfaceEnabled;
     this.grpcStatsDomain = grpcStatsDomain;
     this.connectTimeoutSeconds = connectTimeoutSeconds;
     this.dnsRefreshSeconds = dnsRefreshSeconds;
@@ -249,11 +245,11 @@ public long createBootstrap() {
     byte[][] runtime_guards = JniBridgeUtility.mapToJniBytes(runtimeGuards);
 
     return JniLibrary.createBootstrap(
-        grpcStatsDomain, adminInterfaceEnabled, connectTimeoutSeconds, dnsRefreshSeconds,
-        dnsFailureRefreshSecondsBase, dnsFailureRefreshSecondsMax, dnsQueryTimeoutSeconds,
-        dnsMinRefreshSeconds, dns_preresolve, enableDNSCache, dnsCacheSaveIntervalSeconds,
-        enableDrainPostDnsRefresh, enableHttp3, enableGzipDecompression, enableBrotliDecompression,
-        enableSocketTagging, enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds,
+        grpcStatsDomain, connectTimeoutSeconds, dnsRefreshSeconds, dnsFailureRefreshSecondsBase,
+        dnsFailureRefreshSecondsMax, dnsQueryTimeoutSeconds, dnsMinRefreshSeconds, dns_preresolve,
+        enableDNSCache, dnsCacheSaveIntervalSeconds, enableDrainPostDnsRefresh, enableHttp3,
+        enableGzipDecompression, enableBrotliDecompression, enableSocketTagging,
+        enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds,
         h2ConnectionKeepaliveTimeoutSeconds, maxConnectionsPerHost, statsFlushSeconds,
         streamIdleTimeoutSeconds, perTryIdleTimeoutSeconds, appVersion, appId,
         enforceTrustChainVerification, filter_chain, stats_sinks,
diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java
index 84b54d3e5447..6fe419ff422c 100644
--- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java
+++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java
@@ -299,8 +299,8 @@ public static native Object callCertificateVerificationFromNative(byte[][] certC
    *
    */
   public static native long createBootstrap(
-      String grpcStatsDomain, boolean adminInterfaceEnabled, long connectTimeoutSeconds,
-      long dnsRefreshSeconds, long dnsFailureRefreshSecondsBase, long dnsFailureRefreshSecondsMax,
+      String grpcStatsDomain, long connectTimeoutSeconds, long dnsRefreshSeconds,
+      long dnsFailureRefreshSecondsBase, long dnsFailureRefreshSecondsMax,
       long dnsQueryTimeoutSeconds, long dnsMinRefreshSeconds, byte[][] dnsPreresolveHostnames,
       boolean enableDNSCache, long dnsCacheSaveIntervalSeconds, boolean enableDrainPostDnsRefresh,
       boolean enableHttp3, boolean enableGzipDecompression, boolean enableBrotliDecompression,
diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java
index 0f59c4ea7cf9..2091599c82b7 100644
--- a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java
+++ b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java
@@ -33,7 +33,6 @@ public class NativeCronvoyEngineBuilderImpl extends CronvoyEngineBuilderImpl {
   private final List nativeFilterChain = new ArrayList<>();
   private final EnvoyLogger mEnvoyLogger = null;
   private final EnvoyEventTracker mEnvoyEventTracker = null;
-  private boolean mAdminInterfaceEnabled = false;
   private String mGrpcStatsDomain = null;
   private int mConnectTimeoutSeconds = 30;
   private int mDnsRefreshSeconds = 60;
@@ -133,17 +132,17 @@ private EnvoyConfiguration createEnvoyConfiguration() {
     Map runtimeGuards = Collections.emptyMap();
 
     return new EnvoyConfiguration(
-        mAdminInterfaceEnabled, mGrpcStatsDomain, mConnectTimeoutSeconds, mDnsRefreshSeconds,
-        mDnsFailureRefreshSecondsBase, mDnsFailureRefreshSecondsMax, mDnsQueryTimeoutSeconds,
-        mDnsMinRefreshSeconds, mDnsPreresolveHostnames, mEnableDNSCache,
-        mDnsCacheSaveIntervalSeconds, mEnableDrainPostDnsRefresh, quicEnabled(),
-        mEnableGzipDecompression, brotliEnabled(), mEnableSocketTag, mEnableInterfaceBinding,
-        mH2ConnectionKeepaliveIdleIntervalMilliseconds, mH2ConnectionKeepaliveTimeoutSeconds,
-        mMaxConnectionsPerHost, mStatsFlushSeconds, mStreamIdleTimeoutSeconds,
-        mPerTryIdleTimeoutSeconds, mAppVersion, mAppId, mTrustChainVerification, nativeFilterChain,
-        platformFilterChain, stringAccessors, keyValueStores, statSinks, runtimeGuards,
-        mEnablePlatformCertificatesValidation, mRtdsLayerName, mRtdsTimeoutSeconds, mAdsAddress,
-        mAdsPort, mAdsToken, mAdsTokenLifetime, mAdsRootCerts, mNodeId, mNodeRegion, mNodeZone,
-        mNodeSubZone, mCdsResourcesLocator, mCdsTimeoutSeconds, mEnableCds);
+        mGrpcStatsDomain, mConnectTimeoutSeconds, mDnsRefreshSeconds, mDnsFailureRefreshSecondsBase,
+        mDnsFailureRefreshSecondsMax, mDnsQueryTimeoutSeconds, mDnsMinRefreshSeconds,
+        mDnsPreresolveHostnames, mEnableDNSCache, mDnsCacheSaveIntervalSeconds,
+        mEnableDrainPostDnsRefresh, quicEnabled(), mEnableGzipDecompression, brotliEnabled(),
+        mEnableSocketTag, mEnableInterfaceBinding, mH2ConnectionKeepaliveIdleIntervalMilliseconds,
+        mH2ConnectionKeepaliveTimeoutSeconds, mMaxConnectionsPerHost, mStatsFlushSeconds,
+        mStreamIdleTimeoutSeconds, mPerTryIdleTimeoutSeconds, mAppVersion, mAppId,
+        mTrustChainVerification, nativeFilterChain, platformFilterChain, stringAccessors,
+        keyValueStores, statSinks, runtimeGuards, mEnablePlatformCertificatesValidation,
+        mRtdsLayerName, mRtdsTimeoutSeconds, mAdsAddress, mAdsPort, mAdsToken, mAdsTokenLifetime,
+        mAdsRootCerts, mNodeId, mNodeRegion, mNodeZone, mNodeSubZone, mCdsResourcesLocator,
+        mCdsTimeoutSeconds, mEnableCds);
   }
 }
diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/BUILD b/mobile/library/kotlin/io/envoyproxy/envoymobile/BUILD
index 764ac6eac45c..a4ea5c339633 100644
--- a/mobile/library/kotlin/io/envoyproxy/envoymobile/BUILD
+++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/BUILD
@@ -1,4 +1,4 @@
-load("@envoy//bazel:envoy_build_system.bzl", "envoy_select_admin_functionality", "envoy_select_enable_http3", "envoy_select_envoy_mobile_request_compression")
+load("@envoy//bazel:envoy_build_system.bzl", "envoy_select_enable_http3", "envoy_select_envoy_mobile_request_compression")
 load("@envoy_mobile//bazel:android_artifacts.bzl", "android_artifacts")
 load("@envoy_mobile//bazel:kotlin_lib.bzl", "envoy_mobile_kt_library")
 load("@io_bazel_rules_kotlin//kotlin:android.bzl", "kt_android_library")
@@ -84,9 +84,6 @@ envoy_mobile_kt_library(
     ) + envoy_select_enable_http3(
         ["EngineBuilderHTTP3Util.kt"],
         "@envoy",
-    ) + envoy_select_admin_functionality(
-        ["EngineBuilderAdminUtil.kt"],
-        "@envoy",
     ),
     visibility = ["//visibility:public"],
     deps = [
diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt
index 2445ec84d16d..de7a7d930e8e 100644
--- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt
+++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt
@@ -42,7 +42,6 @@ open class EngineBuilder(
     EnvoyEngineImpl(onEngineRunning, logger, eventTracker)
   }
   private var logLevel = LogLevel.INFO
-  internal var adminInterfaceEnabled = false
   private var grpcStatsDomain: String? = null
   private var connectTimeoutSeconds = 30
   private var dnsRefreshSeconds = 60
@@ -637,7 +636,6 @@ open class EngineBuilder(
   @Suppress("LongMethod")
   fun build(): Engine {
     val engineConfiguration = EnvoyConfiguration(
-      adminInterfaceEnabled,
       grpcStatsDomain,
       connectTimeoutSeconds,
       dnsRefreshSeconds,
diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilderAdminUtil.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilderAdminUtil.kt
deleted file mode 100644
index bc1b3722ab6e..000000000000
--- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilderAdminUtil.kt
+++ /dev/null
@@ -1,21 +0,0 @@
-package io.envoyproxy.envoymobile
-
-/**
- * Utility to enable admin interface.
- */
-object EngineBuilderAdminUtil {
-  /**
-   * Enable admin interface on 127.0.0.1:9901 address. Admin interface is intended to be
-   * used for development/debugging purposes only. Enabling it in production may open
-   * your app to security vulnerabilities.
-   *
-   * Note this will not work with the default production build, as it builds with admin
-   * functionality disabled via --define=admin_functionality=disabled
-   *
-   * @return this builder.
-   */
-  fun EngineBuilder.enableAdminInterface(): EngineBuilder {
-    this.adminInterfaceEnabled = true
-    return this
-  }
-}
diff --git a/mobile/library/objective-c/EnvoyConfiguration.h b/mobile/library/objective-c/EnvoyConfiguration.h
index 7c2afa305c42..3716815181d8 100644
--- a/mobile/library/objective-c/EnvoyConfiguration.h
+++ b/mobile/library/objective-c/EnvoyConfiguration.h
@@ -11,7 +11,6 @@ NS_ASSUME_NONNULL_BEGIN
 /// Typed configuration that may be used for starting Envoy.
 @interface EnvoyConfiguration : NSObject
 
-@property (nonatomic, assign) BOOL adminInterfaceEnabled;
 @property (nonatomic, strong, nullable) NSString *grpcStatsDomain;
 @property (nonatomic, assign) UInt32 connectTimeoutSeconds;
 @property (nonatomic, assign) UInt32 dnsFailureRefreshSecondsBase;
@@ -63,8 +62,7 @@ NS_ASSUME_NONNULL_BEGIN
 /**
  Create a new instance of the configuration.
  */
-- (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled
-                                  grpcStatsDomain:(nullable NSString *)grpcStatsDomain
+- (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain
                             connectTimeoutSeconds:(UInt32)connectTimeoutSeconds
                                 dnsRefreshSeconds:(UInt32)dnsRefreshSeconds
                      dnsFailureRefreshSecondsBase:(UInt32)dnsFailureRefreshSecondsBase
diff --git a/mobile/library/objective-c/EnvoyConfiguration.mm b/mobile/library/objective-c/EnvoyConfiguration.mm
index b75d01bb85a2..25b20b0734a4 100644
--- a/mobile/library/objective-c/EnvoyConfiguration.mm
+++ b/mobile/library/objective-c/EnvoyConfiguration.mm
@@ -67,8 +67,7 @@ @implementation EMODirectResponse
 
 @implementation EnvoyConfiguration
 
-- (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled
-                                  grpcStatsDomain:(nullable NSString *)grpcStatsDomain
+- (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain
                             connectTimeoutSeconds:(UInt32)connectTimeoutSeconds
                                 dnsRefreshSeconds:(UInt32)dnsRefreshSeconds
                      dnsFailureRefreshSecondsBase:(UInt32)dnsFailureRefreshSecondsBase
@@ -129,7 +128,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled
     return nil;
   }
 
-  self.adminInterfaceEnabled = adminInterfaceEnabled;
   self.grpcStatsDomain = grpcStatsDomain;
   self.connectTimeoutSeconds = connectTimeoutSeconds;
   self.dnsRefreshSeconds = dnsRefreshSeconds;
@@ -273,9 +271,6 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled
     builder.addCdsLayer([self.cdsResourcesLocator toCXXString], self.cdsTimeoutSeconds);
   }
 #endif
-#ifdef ENVOY_ADMIN_FUNCTIONALITY
-  builder.enableAdminInterface(self.adminInterfaceEnabled);
-#endif
 
   return builder;
 }
diff --git a/mobile/library/objective-c/EnvoyEngineImpl.mm b/mobile/library/objective-c/EnvoyEngineImpl.mm
index 5bea922541be..3cb922742fed 100644
--- a/mobile/library/objective-c/EnvoyEngineImpl.mm
+++ b/mobile/library/objective-c/EnvoyEngineImpl.mm
@@ -546,7 +546,7 @@ - (int)runWithYAML:(NSString *)yaml
   [self startObservingLifecycleNotifications];
 
   @try {
-    return (int)run_engine(_engineHandle, yaml.UTF8String, logLevel.UTF8String, "");
+    return (int)run_engine(_engineHandle, yaml.UTF8String, logLevel.UTF8String);
   } @catch (NSException *exception) {
     [self logException:exception];
     return kEnvoyFailure;
diff --git a/mobile/library/swift/EngineBuilder.swift b/mobile/library/swift/EngineBuilder.swift
index 806548792066..f555caff27d6 100644
--- a/mobile/library/swift/EngineBuilder.swift
+++ b/mobile/library/swift/EngineBuilder.swift
@@ -19,7 +19,6 @@ open class EngineBuilder: NSObject {
     case custom(String)
   }
 
-  private var adminInterfaceEnabled = false
   private var grpcStatsDomain: String?
   private var connectTimeoutSeconds: UInt32 = 30
   private var dnsFailureRefreshSecondsBase: UInt32 = 2
@@ -620,22 +619,6 @@ open class EngineBuilder: NSObject {
   }
 #endif
 
-#if ENVOY_ADMIN_FUNCTIONALITY
-  /// Enable admin interface on 127.0.0.1:9901 address. Admin interface is intended to be
-  /// used for development/debugging purposes only. Enabling it in production may open
-  /// your app to security vulnerabilities.
-  ///
-  /// Note this will not work with the default production build, as it builds with admin
-  /// functionality disabled via --define=admin_functionality=disabled
-  ///
-  /// - returns: This builder.
-  @discardableResult
-  public func enableAdminInterface() -> Self {
-    self.adminInterfaceEnabled = true
-    return self
-  }
-#endif
-
 #if canImport(EnvoyCxxSwiftInterop)
   /// Use Swift's experimental C++ interop support to generate the bootstrap object
   /// instead of going through the Objective-C layer.
@@ -694,7 +677,6 @@ open class EngineBuilder: NSObject {
 
   func makeConfig() -> EnvoyConfiguration {
     EnvoyConfiguration(
-      adminInterfaceEnabled: self.adminInterfaceEnabled,
       grpcStatsDomain: self.grpcStatsDomain,
       connectTimeoutSeconds: self.connectTimeoutSeconds,
       dnsRefreshSeconds: self.dnsRefreshSeconds,
@@ -762,9 +744,6 @@ private extension EngineBuilder {
   func generateBootstrap() -> Bootstrap {
     var cxxBuilder = Envoy.Platform.EngineBuilder()
     cxxBuilder.addLogLevel(self.logLevel.toCXX())
-#if ENVOY_ADMIN_FUNCTIONALITY
-    cxxBuilder.enableAdminInterface(self.adminInterfaceEnabled)
-#endif
     if let grpcStatsDomain = self.grpcStatsDomain {
       cxxBuilder.addGrpcStatsDomain(grpcStatsDomain.toCXX())
     }
diff --git a/mobile/test/cc/unit/envoy_config_test.cc b/mobile/test/cc/unit/envoy_config_test.cc
index 200814196abb..bb42d1d69c53 100644
--- a/mobile/test/cc/unit/envoy_config_test.cc
+++ b/mobile/test/cc/unit/envoy_config_test.cc
@@ -45,9 +45,6 @@ TEST(TestConfig, ConfigIsApplied) {
       .setRuntimeGuard("test_feature_false", true)
       .enableDnsCache(true, /* save_interval_seconds */ 101)
       .addDnsPreresolveHostnames({"lyft.com", "google.com"})
-#ifdef ENVOY_ADMIN_FUNCTIONALITY
-      .enableAdminInterface(true)
-#endif
       .setForceAlwaysUsev6(true)
 #ifdef ENVOY_GOOGLE_GRPC
       .setNodeId("my_test_node")
@@ -177,19 +174,6 @@ TEST(TestConfig, PerTryIdleTimeout) {
   EXPECT_THAT(bootstrap->ShortDebugString(), HasSubstr("per_try_idle_timeout { seconds: 42 }"));
 }
 
-#ifdef ENVOY_ADMIN_FUNCTIONALITY
-TEST(TestConfig, EnableAdminInterface) {
-  EngineBuilder engine_builder;
-
-  std::unique_ptr bootstrap = engine_builder.generateBootstrap();
-  EXPECT_FALSE(bootstrap->has_admin());
-
-  engine_builder.enableAdminInterface(true);
-  bootstrap = engine_builder.generateBootstrap();
-  EXPECT_TRUE(bootstrap->has_admin());
-}
-#endif
-
 TEST(TestConfig, EnableInterfaceBinding) {
   EngineBuilder engine_builder;
 
diff --git a/mobile/test/common/engine_test.cc b/mobile/test/common/engine_test.cc
index b08f569078ed..1b0715bb2ac1 100644
--- a/mobile/test/common/engine_test.cc
+++ b/mobile/test/common/engine_test.cc
@@ -18,7 +18,7 @@ struct TestEngineHandle {
     Platform::EngineBuilder builder;
     auto bootstrap = builder.generateBootstrap();
     std::string yaml = Envoy::MessageUtil::getYamlStringFromMessage(*bootstrap);
-    run_engine(handle_, yaml.c_str(), level.c_str(), "");
+    run_engine(handle_, yaml.c_str(), level.c_str());
   }
 
   envoy_status_t terminate() { return terminate_engine(handle_, /* release */ false); }
diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc
index 0e0ff356772c..59fcf1137566 100644
--- a/mobile/test/common/integration/client_integration_test.cc
+++ b/mobile/test/common/integration/client_integration_test.cc
@@ -508,13 +508,6 @@ TEST_P(ClientIntegrationTest, TestRuntimeSet) {
   EXPECT_FALSE(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.test_feature_true"));
 }
 
-#ifdef ENVOY_ADMIN_FUNCTIONALITY
-TEST_P(ClientIntegrationTest, TestAdmin) {
-  builder_.enableAdminInterface(true);
-  initialize();
-}
-#endif
-
 TEST_P(ClientIntegrationTest, TestStats) {
   initialize();
 
diff --git a/mobile/test/common/main_interface_test.cc b/mobile/test/common/main_interface_test.cc
index f790147da438..a7a600b0e91c 100644
--- a/mobile/test/common/main_interface_test.cc
+++ b/mobile/test/common/main_interface_test.cc
@@ -160,7 +160,7 @@ TEST_F(MainInterfaceTest, BasicStream) {
                                     } /*on_exit*/,
                                     &engine_cbs_context /*context*/};
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, {});
-  run_engine(engine_handle, BUFFERED_TEST_CONFIG.c_str(), level.c_str(), "");
+  run_engine(engine_handle, BUFFERED_TEST_CONFIG.c_str(), level.c_str());
 
   ASSERT_TRUE(
       engine_cbs_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(10)));
@@ -226,7 +226,7 @@ TEST_F(MainInterfaceTest, SendMetadata) {
   // There is nothing functional about the config used to run the engine, as the created stream is
   // only used for send_metadata.
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, {});
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
 
   ASSERT_TRUE(
       engine_cbs_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(10)));
@@ -266,7 +266,7 @@ TEST_F(MainInterfaceTest, ResetStream) {
   // There is nothing functional about the config used to run the engine, as the created stream is
   // immediately reset.
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, {});
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
 
   ASSERT_TRUE(
       engine_cbs_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(10)));
@@ -337,7 +337,7 @@ TEST_F(MainInterfaceTest, RegisterPlatformApi) {
 
   // Using the minimal envoy mobile config that allows for running the engine.
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, {});
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
 
   ASSERT_TRUE(
       engine_cbs_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(10)));
@@ -368,7 +368,7 @@ TEST(EngineTest, RecordCounter) {
                                     &test_context /*context*/};
   EXPECT_EQ(ENVOY_FAILURE, record_counter_inc(0, "counter", envoy_stats_notags, 1));
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, {});
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
   ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(3)));
   EXPECT_EQ(ENVOY_SUCCESS, record_counter_inc(engine_handle, "counter", envoy_stats_notags, 1));
 
@@ -406,7 +406,7 @@ TEST(EngineTest, Logger) {
                       &test_context};
 
   envoy_engine_t engine_handle = init_engine(engine_cbs, logger, {});
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
   ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(3)));
 
   ASSERT_TRUE(test_context.on_log.WaitForNotificationWithTimeout(absl::Seconds(3)));
@@ -431,7 +431,7 @@ TEST(EngineTest, EventTrackerRegistersDefaultAPI) {
                                     &test_context /*context*/};
 
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, {});
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
 
   // A default event tracker is registered in external API registry.
   const auto registered_event_tracker =
@@ -475,7 +475,7 @@ TEST(EngineTest, EventTrackerRegistersAPI) {
                                     &test_context /*context*/};
 
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, event_tracker);
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
 
   ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(3)));
   const auto registered_event_tracker =
@@ -518,7 +518,7 @@ TEST(EngineTest, EventTrackerRegistersAssertionFailureRecordAction) {
       &test_context /*context*/};
 
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, event_tracker);
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
 
   ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(3)));
   // Simulate a failed assertion by invoking a debug assertion failure
@@ -558,7 +558,7 @@ TEST(EngineTest, EventTrackerRegistersEnvoyBugRecordAction) {
                                     &test_context /*context*/};
 
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, event_tracker);
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
 
   ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(3)));
   // Simulate an envoy bug by invoking an Envoy bug failure
@@ -585,7 +585,7 @@ TEST_F(MainInterfaceTest, ResetConnectivityState) {
                                     } /*on_exit*/,
                                     &test_context /*context*/};
   envoy_engine_t engine_handle = init_engine(engine_cbs, {}, {});
-  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str(), "");
+  run_engine(engine_handle, MINIMAL_TEST_CONFIG.c_str(), LEVEL_DEBUG.c_str());
   ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(3)));
 
   ASSERT_EQ(ENVOY_SUCCESS, reset_connectivity_state(engine_handle));
diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt
index 80969da60de4..93d71c2006e3 100644
--- a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt
+++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt
@@ -68,7 +68,6 @@ class TestEnvoyHTTPFilterFactory(name : String) : EnvoyHTTPFilterFactory {
 class EnvoyConfigurationTest {
 
   fun buildTestEnvoyConfiguration(
-    adminInterfaceEnabled: Boolean = false,
     grpcStatsDomain: String = "stats.example.com",
     connectTimeoutSeconds: Int = 123,
     dnsRefreshSeconds: Int = 234,
@@ -116,7 +115,6 @@ class EnvoyConfigurationTest {
 
   ): EnvoyConfiguration {
     return EnvoyConfiguration(
-      adminInterfaceEnabled,
       grpcStatsDomain,
       connectTimeoutSeconds,
       dnsRefreshSeconds,
@@ -174,8 +172,6 @@ class EnvoyConfigurationTest {
     val resolvedTemplate = TestJni.createYaml(envoyConfiguration)
     assertThat(resolvedTemplate).contains("connect_timeout: 123s")
 
-    assertThat(resolvedTemplate).doesNotContain("admin: *admin_interface")
-
     // DNS
     assertThat(resolvedTemplate).contains("dns_refresh_rate: 234s")
     assertThat(resolvedTemplate).contains("base_interval: 345s")
@@ -243,7 +239,6 @@ class EnvoyConfigurationTest {
   fun `configuration resolves with alternate values`() {
     JniLibrary.loadTestLibrary()
     val envoyConfiguration = buildTestEnvoyConfiguration(
-      adminInterfaceEnabled = false,
       grpcStatsDomain = "",
       enableDrainPostDnsRefresh = true,
       enableDNSCache = true,
diff --git a/mobile/test/kotlin/apps/experimental/MainActivity.kt b/mobile/test/kotlin/apps/experimental/MainActivity.kt
index 731cb51b00fc..6a76e94ae47a 100644
--- a/mobile/test/kotlin/apps/experimental/MainActivity.kt
+++ b/mobile/test/kotlin/apps/experimental/MainActivity.kt
@@ -12,7 +12,6 @@ import androidx.recyclerview.widget.RecyclerView
 import io.envoyproxy.envoymobile.android.SharedPreferencesStore
 import io.envoyproxy.envoymobile.AndroidEngineBuilder
 import io.envoyproxy.envoymobile.CompressionAlgorithm
-import io.envoyproxy.envoymobile.EngineBuilderAdminUtil.enableAdminInterface
 import io.envoyproxy.envoymobile.Element
 import io.envoyproxy.envoymobile.Engine
 import io.envoyproxy.envoymobile.LogLevel
@@ -57,7 +56,6 @@ class MainActivity : Activity() {
       .addPlatformFilter(::DemoFilter)
       .addPlatformFilter(::BufferDemoFilter)
       .addPlatformFilter(::AsyncDemoFilter)
-      .enableAdminInterface()
       .enableDNSCache(true)
       // required by DNS cache
       .addKeyValueStore("reserved.platform_store", SharedPreferencesStore(preferences))
diff --git a/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc b/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc
index 0c21ae35e020..3afeb9fb8f3b 100644
--- a/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc
+++ b/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc
@@ -147,7 +147,7 @@ TEST_P(GcpTrafficDirectorIntegrationTest, AdsDynamicClusters) {
   //      5. base_clear
   ASSERT_TRUE(waitForGaugeGe("cluster_manager.active_clusters", 5));
 
-  // TODO(abeyad): Once we have a Envoy Mobile stats/admin API, we can use it to check the
+  // TODO(abeyad): Once we have a Envoy Mobile stats API, we can use it to check the
   // actual cluster names.
 }
 
diff --git a/mobile/test/performance/test_binary_size.cc b/mobile/test/performance/test_binary_size.cc
index 436b05d5e556..07165c65a8b1 100644
--- a/mobile/test/performance/test_binary_size.cc
+++ b/mobile/test/performance/test_binary_size.cc
@@ -5,4 +5,4 @@
 // This binary is used to perform stripped down binary size investigations of the Envoy codebase.
 // Please refer to the development docs for more information:
 // https://envoymobile.io/docs/envoy-mobile/latest/development/performance/binary_size.html
-int main() { return run_engine(0, nullptr, nullptr, nullptr); }
+int main() { return run_engine(0, nullptr, nullptr); }
diff --git a/mobile/test/swift/EngineBuilderTests.swift b/mobile/test/swift/EngineBuilderTests.swift
index bfd83b08438b..98795b7b3f05 100644
--- a/mobile/test/swift/EngineBuilderTests.swift
+++ b/mobile/test/swift/EngineBuilderTests.swift
@@ -67,35 +67,6 @@ final class EngineBuilderTests: XCTestCase {
     self.waitForExpectations(timeout: 0.01)
   }
 
-  func testAdminInterfaceIsDisabledByDefault() {
-    let expectation = self.expectation(description: "Run called with disabled admin interface")
-    MockEnvoyEngine.onRunWithConfig = { config, _ in
-      XCTAssertFalse(config.adminInterfaceEnabled)
-      expectation.fulfill()
-    }
-
-    _ = EngineBuilder()
-      .addEngineType(MockEnvoyEngine.self)
-      .build()
-    self.waitForExpectations(timeout: 0.01)
-  }
-
-#if ENVOY_ADMIN_FUNCTIONALITY
-  func testEnablingAdminInterfaceAddsToConfigurationWhenRunningEnvoy() {
-    let expectation = self.expectation(description: "Run called with enabled admin interface")
-    MockEnvoyEngine.onRunWithConfig = { config, _ in
-      XCTAssertTrue(config.adminInterfaceEnabled)
-      expectation.fulfill()
-    }
-
-    _ = EngineBuilder()
-      .addEngineType(MockEnvoyEngine.self)
-      .enableAdminInterface()
-      .build()
-    self.waitForExpectations(timeout: 0.01)
-  }
-#endif
-
   func testEnablingInterfaceBindingAddsToConfigurationWhenRunningEnvoy() {
     let expectation = self.expectation(description: "Run called with enabled interface binding")
     MockEnvoyEngine.onRunWithConfig = { config, _ in
diff --git a/mobile/test/swift/apps/experimental/ViewController.swift b/mobile/test/swift/apps/experimental/ViewController.swift
index 0636e680808a..36e6c6b9ec94 100644
--- a/mobile/test/swift/apps/experimental/ViewController.swift
+++ b/mobile/test/swift/apps/experimental/ViewController.swift
@@ -22,9 +22,6 @@ final class ViewController: UITableViewController {
       .addPlatformFilter(DemoFilter.init)
       .addPlatformFilter(BufferDemoFilter.init)
       .addPlatformFilter(AsyncDemoFilter.init)
-#if ENVOY_ADMIN_FUNCTIONALITY
-      .enableAdminInterface()
-#endif
       .enableDNSCache(true)
       // required by DNS cache
       .addKeyValueStore(name: "reserved.platform_store", keyValueStore: UserDefaults.standard)

From fe290c2c7601b4951bff2f3695ede775bc620830 Mon Sep 17 00:00:00 2001
From: Pradeep Rao <84025829+pradeepcrao@users.noreply.github.com>
Date: Thu, 15 Jun 2023 12:29:58 -0400
Subject: [PATCH 547/740] Temporary fix for ext_proc coverage issue. (#27992)

Signed-off-by: pcrao 
---
 test/per_file_coverage.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh
index 0fb812f5e751..a54367841c8f 100755
--- a/test/per_file_coverage.sh
+++ b/test/per_file_coverage.sh
@@ -35,6 +35,7 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/filters/common/fault:94.5"
 "source/extensions/filters/common/rbac:90.5"
 "source/extensions/filters/http/cache:93.4"
+"source/extensions/filters/http/ext_proc:96.0" # TODO(tyxia) Improve coverage for ext_proc/config.cc
 "source/extensions/filters/http/grpc_json_transcoder:95.6"
 "source/extensions/filters/http/ip_tagging:88.0"
 "source/extensions/filters/http/kill_request:91.7" # Death tests don't report LCOV

From 35514452d10ac56f5afca0c177149f0cdd030aa7 Mon Sep 17 00:00:00 2001
From: ohadvano <49730675+ohadvano@users.noreply.github.com>
Date: Thu, 15 Jun 2023 20:26:16 +0300
Subject: [PATCH 548/740] logging: extract ENVOY_TAGGED_LOG tags to JSON
 properties (#27960)

Signed-off-by: ohadvano 
---
 source/common/common/logger.cc    | 104 +++++++++++++++++++++++++++---
 source/common/common/logger.h     |  42 ++++++++++++
 test/common/common/logger_test.cc | 104 ++++++++++++++++++++++++++++++
 3 files changed, 241 insertions(+), 9 deletions(-)

diff --git a/source/common/common/logger.cc b/source/common/common/logger.cc
index 74cee4d084e4..5a1723ed5e3e 100644
--- a/source/common/common/logger.cc
+++ b/source/common/common/logger.cc
@@ -253,6 +253,8 @@ void Registry::setLogFormat(const std::string& log_format) {
   for (Logger& logger : allLoggers()) {
     Utility::setLogFormatForLogger(logger.getLogger(), log_format);
   }
+
+  json_log_format_set_ = false;
 }
 
 absl::Status Registry::setJsonLogFormat(const Protobuf::Message& log_format_struct) {
@@ -276,10 +278,29 @@ absl::Status Registry::setJsonLogFormat(const Protobuf::Message& log_format_stru
     return absl::InvalidArgumentError("Usage of %_ is unavailable for JSON log formats");
   }
 
+  // Since the format is now a JSON struct, it is guaranteed that it ends with '}'.
+  // Here we replace '}' with '%*}' to enable the option to add more JSON properties,
+  // in case ENVOY_TAGGED_LOG is used. If there are no tags, '%*' will be replaced by
+  // an empty string, falling back to the original log format. If there are log tags,
+  // '%*' will be replaced by the serialized tags as JSON properties.
+  format_as_json.replace(format_as_json.rfind('}'), 1, "%*}");
+
+  // To avoid performance impact for '%j' flag in case JSON logs are not enabled,
+  // all occurrences of '%j' will be replaced with '%+' which removes the tags from
+  // the message string before writing it to the output as escaped JSON string.
+  size_t flag_index = format_as_json.find("%j");
+  while (flag_index != std::string::npos) {
+    format_as_json.replace(flag_index, 2, "%+");
+    flag_index = format_as_json.find("%j");
+  }
+
   setLogFormat(format_as_json);
+  json_log_format_set_ = true;
   return absl::OkStatus();
 }
 
+bool Registry::json_log_format_set_ = false;
+
 Logger* Registry::logger(const std::string& log_name) {
   Logger* logger_to_return = nullptr;
   for (Logger& logger : loggers()) {
@@ -304,6 +325,17 @@ void setLogFormatForLogger(spdlog::logger& logger, const std::string& log_format
       ->add_flag(
           CustomFlagFormatter::EscapeMessageJsonString::Placeholder)
       .set_pattern(log_format);
+
+  formatter
+      ->add_flag(
+          CustomFlagFormatter::ExtractedTags::Placeholder)
+      .set_pattern(log_format);
+
+  formatter
+      ->add_flag(
+          CustomFlagFormatter::ExtractedMessage::Placeholder)
+      .set_pattern(log_format);
+
   logger.set_formatter(std::move(formatter));
 }
 
@@ -313,18 +345,51 @@ std::string serializeLogTags(const std::map& tags) {
   }
 
   std::stringstream tags_stream;
-  tags_stream << "[Tags: ";
-  for (const auto& tag : tags) {
-    tags_stream << "\"" << tag.first << "\":\"" << tag.second << "\",";
+  tags_stream << TagsPrefix;
+  if (Registry::jsonLogFormatSet()) {
+    for (const auto& tag : tags) {
+      tags_stream << "\"";
+
+      auto required_space = JsonEscaper::extraSpace(tag.first);
+      if (required_space == 0) {
+        tags_stream << tag.first;
+      } else {
+        tags_stream << JsonEscaper::escapeString(tag.first, required_space);
+      }
+      tags_stream << "\":\"";
+
+      required_space = JsonEscaper::extraSpace(tag.second);
+      if (required_space == 0) {
+        tags_stream << tag.second;
+      } else {
+        tags_stream << JsonEscaper::escapeString(tag.second, required_space);
+      }
+      tags_stream << "\",";
+    }
+  } else {
+    for (const auto& tag : tags) {
+      tags_stream << "\"" << tag.first << "\":\"" << tag.second << "\",";
+    }
   }
 
   std::string serialized = tags_stream.str();
   serialized.pop_back();
-  serialized += "] ";
+  serialized += TagsSuffix;
 
   return serialized;
 }
 
+void escapeMessageJsonString(absl::string_view payload, spdlog::memory_buf_t& dest) {
+  const uint64_t required_space = JsonEscaper::extraSpace(payload);
+  if (required_space == 0) {
+    dest.append(payload.data(), payload.data() + payload.size());
+    return;
+  }
+
+  const std::string escaped = JsonEscaper::escapeString(payload, required_space);
+  dest.append(escaped.data(), escaped.data() + escaped.size());
+}
+
 } // namespace Utility
 
 namespace CustomFlagFormatter {
@@ -340,13 +405,34 @@ void EscapeMessageJsonString::format(const spdlog::details::log_msg& msg, const
                                      spdlog::memory_buf_t& dest) {
 
   absl::string_view payload = absl::string_view(msg.payload.data(), msg.payload.size());
-  const uint64_t required_space = JsonEscaper::extraSpace(payload);
-  if (required_space == 0) {
-    dest.append(payload.data(), payload.data() + payload.size());
+  Envoy::Logger::Utility::escapeMessageJsonString(payload, dest);
+}
+
+void ExtractedTags::format(const spdlog::details::log_msg& msg, const std::tm&,
+                           spdlog::memory_buf_t& dest) {
+  absl::string_view payload = absl::string_view(msg.payload.data(), msg.payload.size());
+  if (payload.rfind(Utility::TagsPrefix, 0) == std::string::npos) {
     return;
   }
-  const std::string escaped = JsonEscaper::escapeString(payload, required_space);
-  dest.append(escaped.data(), escaped.data() + escaped.size());
+
+  auto tags_end_pos = payload.find(Utility::TagsSuffixForSearch) + 1;
+  dest.append(&JsonPropertyDeimilter, &JsonPropertyDeimilter + 1);
+  dest.append(payload.data() + Utility::TagsPrefix.size(), payload.data() + tags_end_pos);
+}
+
+void ExtractedMessage::format(const spdlog::details::log_msg& msg, const std::tm&,
+                              spdlog::memory_buf_t& dest) {
+  absl::string_view payload = absl::string_view(msg.payload.data(), msg.payload.size());
+  if (payload.rfind(Utility::TagsPrefix, 0) == std::string::npos) {
+    Envoy::Logger::Utility::escapeMessageJsonString(payload, dest);
+    return;
+  }
+
+  auto tags_end_pos =
+      payload.find(Utility::TagsSuffixForSearch) + Utility::TagsSuffixForSearch.size();
+  auto original_message =
+      absl::string_view(payload.data() + tags_end_pos, payload.size() - tags_end_pos);
+  Envoy::Logger::Utility::escapeMessageJsonString(original_message, dest);
 }
 
 } // namespace CustomFlagFormatter
diff --git a/source/common/common/logger.h b/source/common/common/logger.h
index 720e9b1bb68d..45ded0852277 100644
--- a/source/common/common/logger.h
+++ b/source/common/common/logger.h
@@ -359,6 +359,11 @@ class Registry {
    */
   static absl::Status setJsonLogFormat(const Protobuf::Message& log_format_struct);
 
+  /**
+   * @return true if JSON log format was set using setJsonLogFormat.
+   */
+  static bool jsonLogFormatSet() { return json_log_format_set_; }
+
   /**
    * @return std::vector& the installed loggers.
    */
@@ -376,6 +381,8 @@ class Registry {
    * @return std::vector& return the installed loggers.
    */
   static std::vector& allLoggers();
+
+  static bool json_log_format_set_;
 };
 
 /**
@@ -396,6 +403,10 @@ template  class Loggable {
 
 namespace Utility {
 
+constexpr static absl::string_view TagsPrefix = "[Tags: ";
+constexpr static absl::string_view TagsSuffix = "] ";
+constexpr static absl::string_view TagsSuffixForSearch = "\"] ";
+
 /**
  * Sets the log format for a specific logger.
  */
@@ -403,9 +414,15 @@ void setLogFormatForLogger(spdlog::logger& logger, const std::string& log_format
 
 /**
  * Serializes custom log tags to a string that will be prepended to the log message.
+ * In case JSON logging is enabled, the keys and values will be serialized with JSON escaping.
  */
 std::string serializeLogTags(const std::map& tags);
 
+/**
+ * Escapes the payload to a JSON string and writes the output to the destination buffer.
+ */
+void escapeMessageJsonString(absl::string_view payload, spdlog::memory_buf_t& dest);
+
 } // namespace Utility
 
 // Contains custom flags to introduce user defined flags in log pattern. Reference:
@@ -450,6 +467,31 @@ class EscapeMessageJsonString : public spdlog::custom_flag_formatter {
   constexpr static char Placeholder = 'j';
 };
 
+class ExtractedTags : public spdlog::custom_flag_formatter {
+public:
+  void format(const spdlog::details::log_msg& msg, const std::tm& tm,
+              spdlog::memory_buf_t& dest) override;
+
+  std::unique_ptr clone() const override {
+    return spdlog::details::make_unique();
+  }
+
+  constexpr static char Placeholder = '*';
+  constexpr static char JsonPropertyDeimilter = ',';
+};
+
+class ExtractedMessage : public spdlog::custom_flag_formatter {
+public:
+  void format(const spdlog::details::log_msg& msg, const std::tm& tm,
+              spdlog::memory_buf_t& dest) override;
+
+  std::unique_ptr clone() const override {
+    return spdlog::details::make_unique();
+  }
+
+  constexpr static char Placeholder = '+';
+};
+
 } // namespace CustomFlagFormatter
 } // namespace Logger
 
diff --git a/test/common/common/logger_test.cc b/test/common/common/logger_test.cc
index d9e039a4d517..86a4e9803176 100644
--- a/test/common/common/logger_test.cc
+++ b/test/common/common/logger_test.cc
@@ -131,6 +131,14 @@ class LoggerCustomFlagsTest : public testing::TestWithParam {
         ->add_flag(
             CustomFlagFormatter::EscapeMessageJsonString::Placeholder)
         .set_pattern(pattern);
+    formatter
+        ->add_flag(
+            CustomFlagFormatter::ExtractedTags::Placeholder)
+        .set_pattern(pattern);
+    formatter
+        ->add_flag(
+            CustomFlagFormatter::ExtractedMessage::Placeholder)
+        .set_pattern(pattern);
     logger_->set_formatter(std::move(formatter));
     logger_->set_level(spdlog::level::info);
 
@@ -177,6 +185,25 @@ TEST_P(LoggerCustomFlagsTest, LogMessageAsJsonStringEscaped) {
       "StreamAggregatedResources gRPC config stream closed: 14, connection error: desc = "
       "\\\"transport: Error while dialing dial tcp [::1]:15012: connect: connection refused\\\"");
 }
+
+TEST_P(LoggerCustomFlagsTest, LogMessageWithTagsAndExtractMessage) {
+  expectLogMessage("%+", "message", "message");
+  expectLogMessage("%+", " message", " message");
+  expectLogMessage("%+", "[Tags: \"key\":\"val\"] message1", "message1");
+  expectLogMessage("%+", "[Tags: \"key1\":\"val1\",\"key2\":\"val2\"] message2", "message2");
+  expectLogMessage("%+", "[Tags: \"key\":\"val\"] mes] sge3", "mes] sge3");
+  expectLogMessage("%+", "[Tags: \"key\":\"val\"] mes\"] sge4", "mes\\\"] sge4");
+}
+
+TEST_P(LoggerCustomFlagsTest, LogMessageWithTagsAndExtractTags) {
+  expectLogMessage("%*", "message1", "");
+  expectLogMessage("%*", "[Tags: \"key\":\"val\"] message1", ",\"key\":\"val\"");
+  expectLogMessage("%*", "[Tags: \"key1\":\"val1\",\"key2\":\"val2\"] message2",
+                   ",\"key1\":\"val1\",\"key2\":\"val2\"");
+  expectLogMessage("%*", "[Tags: \"key\":\"val\"] mes] sge3", ",\"key\":\"val\"");
+  expectLogMessage("%*", "[Tags: \"key\":\"val\"] mes\"] sge4", ",\"key\":\"val\"");
+}
+
 class NamedLogTest : public Loggable, public testing::Test {};
 
 TEST_F(NamedLogTest, NamedLogsAreSentToSink) {
@@ -268,6 +295,7 @@ TEST(LoggerTest, TestJsonFormatError) {
   // This scenario shouldn't happen in production, the test is added mainly for coverage.
   auto status = Envoy::Logger::Registry::setJsonLogFormat(log_struct);
   EXPECT_FALSE(status.ok());
+  EXPECT_FALSE(Envoy::Logger::Registry::jsonLogFormatSet());
   EXPECT_EQ("INVALID_ARGUMENT: Provided struct cannot be serialized as JSON string",
             status.ToString());
 }
@@ -282,6 +310,7 @@ TEST(LoggerTest, TestJsonFormatNonEscapedThrows) {
 
     auto status = Envoy::Logger::Registry::setJsonLogFormat(log_struct);
     EXPECT_FALSE(status.ok());
+    EXPECT_FALSE(Envoy::Logger::Registry::jsonLogFormatSet());
     EXPECT_EQ("INVALID_ARGUMENT: Usage of %v is unavailable for JSON log formats",
               status.ToString());
   }
@@ -293,6 +322,7 @@ TEST(LoggerTest, TestJsonFormatNonEscapedThrows) {
 
     auto status = Envoy::Logger::Registry::setJsonLogFormat(log_struct);
     EXPECT_FALSE(status.ok());
+    EXPECT_FALSE(Envoy::Logger::Registry::jsonLogFormatSet());
     EXPECT_EQ("INVALID_ARGUMENT: Usage of %_ is unavailable for JSON log formats",
               status.ToString());
   }
@@ -302,6 +332,7 @@ TEST(LoggerTest, TestJsonFormatEmptyStruct) {
   ProtobufWkt::Struct log_struct;
   Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
   EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok());
+  EXPECT_TRUE(Envoy::Logger::Registry::jsonLogFormatSet());
 
   MockLogSink sink(Envoy::Logger::Registry::getSink());
   EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto& log) {
@@ -319,6 +350,7 @@ TEST(LoggerTest, TestJsonFormatNullAndFixedField) {
   (*log_struct.mutable_fields())["NullField"].set_null_value(ProtobufWkt::NULL_VALUE);
   Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
   EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok());
+  EXPECT_TRUE(Envoy::Logger::Registry::jsonLogFormatSet());
 
   MockLogSink sink(Envoy::Logger::Registry::getSink());
   EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto&) {
@@ -337,6 +369,7 @@ TEST(LoggerTest, TestJsonFormat) {
   (*log_struct.mutable_fields())["Message"].set_string_value("%j");
   Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
   EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok());
+  EXPECT_TRUE(Envoy::Logger::Registry::jsonLogFormatSet());
 
   MockLogSink sink(Envoy::Logger::Registry::getSink());
   EXPECT_CALL(sink, log(_, _))
@@ -371,6 +404,7 @@ TEST(LoggerTest, TestJsonFormatWithNestedJsonMessage) {
   (*log_struct.mutable_fields())["FixedValue"].set_string_value("Fixed");
   Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
   EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok());
+  EXPECT_TRUE(Envoy::Logger::Registry::jsonLogFormatSet());
 
   MockLogSink sink(Envoy::Logger::Registry::getSink());
   EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto& log) {
@@ -401,6 +435,15 @@ TEST(LoggerUtilityTest, TestSerializeLogTags) {
   EXPECT_EQ("[Tags: \"key1\":\"value1\",\"key2\":\"value2\",\"key3\":\"value3\"] ",
             Envoy::Logger::Utility::serializeLogTags(
                 {{"key1", "value1"}, {"key2", "value2"}, {"key3", "value3"}}));
+
+  // Entries that require JSON escaping
+  EXPECT_EQ("[Tags: "
+            "\"ke\\\"y2\":\"value2\",\"ke\\\"y4\":\"va\\\"lue4\",\"key1\":\"value1\",\"key3\":"
+            "\"va\\\"lue3\"] ",
+            Envoy::Logger::Utility::serializeLogTags({{"key1", "value1"},
+                                                      {"ke\"y2", "value2"},
+                                                      {"key3", "va\"lue3"},
+                                                      {"ke\"y4", "va\"lue4"}}));
 }
 
 class ClassForTaggedLog : public Envoy::Logger::Loggable {
@@ -412,8 +455,13 @@ class ClassForTaggedLog : public Envoy::Logger::Loggable tags_{{"key", "val"}};
+  std::map tags_with_escaping_{{"key", "val"}, {"ke\"y", "v\"al"}};
 };
 
 TEST(TaggedLogTest, TestTaggedLog) {
@@ -425,11 +473,67 @@ TEST(TaggedLogTest, TestTaggedLog) {
       }))
       .WillOnce(Invoke([](auto msg, auto&) {
         EXPECT_THAT(msg, HasSubstr("[Tags: \"key_inline\":\"val\"] fake message val"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_THAT(msg,
+                    HasSubstr("[Tags: \"ke\"y\":\"v\"al\",\"key\":\"val\"] fake me\"ssage val"));
       }));
 
   ClassForTaggedLog object;
   object.logMessageWithPreCreatedTags();
   object.logMessageWithInlineTags();
+  object.logMessageWithEscaping();
+}
+
+TEST(TaggedLogTest, TestTaggedLogWithJsonFormat) {
+  ProtobufWkt::Struct log_struct;
+  (*log_struct.mutable_fields())["Level"].set_string_value("%l");
+  (*log_struct.mutable_fields())["Message"].set_string_value("%j");
+  Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
+  EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok());
+  EXPECT_TRUE(Envoy::Logger::Registry::jsonLogFormatSet());
+
+  MockLogSink sink(Envoy::Logger::Registry::getSink());
+  EXPECT_CALL(sink, log(_, _))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
+        EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
+        EXPECT_THAT(msg, HasSubstr("\"Message\":\"fake message val\""));
+        EXPECT_THAT(msg, HasSubstr("\"key\":\"val\""));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
+        EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
+        EXPECT_THAT(msg, HasSubstr("\"Message\":\"fake me\\\"ssage val\""));
+        EXPECT_THAT(msg, HasSubstr("\"ke\\\"y\":\"v\\\"al\""));
+        EXPECT_THAT(msg, HasSubstr("\"key\":\"val\""));
+      }));
+
+  ClassForTaggedLog object;
+  object.logMessageWithPreCreatedTags();
+  object.logMessageWithEscaping();
+}
+
+TEST(TaggedLogTest, TestTaggedLogWithJsonFormatMultipleJFlags) {
+  ProtobufWkt::Struct log_struct;
+  (*log_struct.mutable_fields())["Level"].set_string_value("%l");
+  (*log_struct.mutable_fields())["Message1"].set_string_value("%j");
+  (*log_struct.mutable_fields())["Message2"].set_string_value("%j");
+  Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
+  EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok());
+  EXPECT_TRUE(Envoy::Logger::Registry::jsonLogFormatSet());
+
+  MockLogSink sink(Envoy::Logger::Registry::getSink());
+  EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto&) {
+    EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
+    EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
+    EXPECT_THAT(msg, HasSubstr("\"Message1\":\"fake message val\""));
+    EXPECT_THAT(msg, HasSubstr("\"Message2\":\"fake message val\""));
+    EXPECT_THAT(msg, HasSubstr("\"key\":\"val\""));
+  }));
+
+  ClassForTaggedLog object;
+  object.logMessageWithPreCreatedTags();
 }
 
 } // namespace

From 3470853c91a153b28c2f2274ef8797e05f631d76 Mon Sep 17 00:00:00 2001
From: botengyao 
Date: Thu, 15 Jun 2023 14:40:14 -0400
Subject: [PATCH 549/740] hc: add pragma once (#27971)

add pragma once to header

Signed-off-by: Boteng Yao 
---
 envoy/upstream/health_check_event_sink.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/envoy/upstream/health_check_event_sink.h b/envoy/upstream/health_check_event_sink.h
index aed7792d057d..9f82edda1458 100644
--- a/envoy/upstream/health_check_event_sink.h
+++ b/envoy/upstream/health_check_event_sink.h
@@ -1,3 +1,5 @@
+#pragma once
+
 #include "envoy/config/typed_config.h"
 #include "envoy/server/health_checker_config.h"
 

From ad5f4c6cbfcc7e09fcdd09289e13ac72fcffbce4 Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Thu, 15 Jun 2023 15:11:57 -0400
Subject: [PATCH 550/740] health checker: removing exceptions (#27859)

Risk Level: medium
Testing: new tests
Docs Changes: n/a
Release Notes: n/a
Part of envoyproxy/envoy-mobile#176

Signed-off-by: Alyssa Wilk 
---
 envoy/common/exception.h                      | 15 ++++++
 .../common/upstream/cluster_factory_impl.cc   |  6 ++-
 .../upstream/health_checker_event_logger.h    |  1 -
 source/common/upstream/health_checker_impl.cc | 12 ++---
 source/common/upstream/health_checker_impl.h  |  8 +--
 .../upstream/health_discovery_service.cc      | 51 +++++++++++++------
 .../upstream/health_discovery_service.h       | 17 ++++---
 .../http/health_checker_impl.cc               |  4 +-
 .../http/health_checker_impl.h                |  2 +-
 .../tcp/health_checker_impl.cc                | 11 ++--
 .../health_checkers/tcp/health_checker_impl.h |  2 +-
 test/common/upstream/hds_test.cc              | 45 +++++++++++++---
 .../upstream/health_checker_impl_test.cc      | 21 ++++----
 .../health_checkers/redis/config_test.cc      |  1 +
 .../health_checkers/thrift/config_test.cc     |  1 +
 tools/code_format/config.yaml                 |  1 -
 16 files changed, 140 insertions(+), 58 deletions(-)

diff --git a/envoy/common/exception.h b/envoy/common/exception.h
index d5b9255ead1e..823fd7021ebd 100644
--- a/envoy/common/exception.h
+++ b/envoy/common/exception.h
@@ -11,4 +11,19 @@ class EnvoyException : public std::runtime_error {
 public:
   EnvoyException(const std::string& message) : std::runtime_error(message) {}
 };
+
+// Simple macro to handle bridging functions which return absl::StatusOr, and
+// functions which throw errors.
+//
+// The completely unnecessary throw action argument is just so 'throw' appears
+// at the call site, so format checks about use of exceptions are triggered.
+#define THROW_IF_STATUS_NOT_OK(variable, throw_action)                                             \
+  if (!variable.status().ok()) {                                                                   \
+    throw_action EnvoyException(std::string(variable.status().message()));                         \
+  }
+
+#define RETURN_IF_STATUS_NOT_OK(variable)                                                          \
+  if (!variable.status().ok()) {                                                                   \
+    return variable.status();                                                                      \
+  }
 } // namespace Envoy
diff --git a/source/common/upstream/cluster_factory_impl.cc b/source/common/upstream/cluster_factory_impl.cc
index ea782481749b..ea8cdca4da25 100644
--- a/source/common/upstream/cluster_factory_impl.cc
+++ b/source/common/upstream/cluster_factory_impl.cc
@@ -105,10 +105,12 @@ ClusterFactoryImplBase::create(const envoy::config::cluster::v3::Cluster& cluste
     if (cluster.health_checks().size() != 1) {
       return absl::InvalidArgumentError("Multiple health checks not supported");
     } else {
-      new_cluster_pair.first->setHealthChecker(HealthCheckerFactory::create(
+      auto checker_or_error = HealthCheckerFactory::create(
           cluster.health_checks()[0], *new_cluster_pair.first, server_context.runtime(),
           server_context.mainThreadDispatcher(), server_context.accessLogManager(),
-          context.messageValidationVisitor(), server_context.api()));
+          context.messageValidationVisitor(), server_context.api());
+      RETURN_IF_STATUS_NOT_OK(checker_or_error);
+      new_cluster_pair.first->setHealthChecker(checker_or_error.value());
     }
   }
 
diff --git a/source/common/upstream/health_checker_event_logger.h b/source/common/upstream/health_checker_event_logger.h
index fe67cd9a8f87..5b4d2b526b9f 100644
--- a/source/common/upstream/health_checker_event_logger.h
+++ b/source/common/upstream/health_checker_event_logger.h
@@ -36,7 +36,6 @@ class HealthCheckEventLoggerImpl : public HealthCheckEventLogger {
       file_ = log_manager.createAccessLog(Filesystem::FilePathAndType{
           Filesystem::DestinationType::File, health_check_config.event_log_path()});
     }
-
     for (const auto& config : health_check_config.event_logger()) {
       auto& factory = Config::Utility::getAndCheckFactory(config);
       event_sinks_.push_back(factory.createHealthCheckEventSink(config.typed_config(), context));
diff --git a/source/common/upstream/health_checker_impl.cc b/source/common/upstream/health_checker_impl.cc
index 3ccae085a020..d9f14b733f78 100644
--- a/source/common/upstream/health_checker_impl.cc
+++ b/source/common/upstream/health_checker_impl.cc
@@ -47,7 +47,7 @@ const std::string& HealthCheckerFactory::getHostname(const HostSharedPtr& host,
   return cluster->name();
 }
 
-HealthCheckerSharedPtr HealthCheckerFactory::create(
+absl::StatusOr HealthCheckerFactory::create(
     const envoy::config::core::v3::HealthCheck& health_check_config, Upstream::Cluster& cluster,
     Runtime::Loader& runtime, Event::Dispatcher& dispatcher,
     AccessLog::AccessLogManager& log_manager,
@@ -56,7 +56,7 @@ HealthCheckerSharedPtr HealthCheckerFactory::create(
 
   switch (health_check_config.health_checker_case()) {
   case envoy::config::core::v3::HealthCheck::HealthCheckerCase::HEALTH_CHECKER_NOT_SET:
-    throw EnvoyException("invalid cluster config");
+    return absl::InvalidArgumentError("invalid cluster config");
   case envoy::config::core::v3::HealthCheck::HealthCheckerCase::kHttpHealthCheck:
     factory = &Config::Utility::getAndCheckFactoryByName<
         Server::Configuration::CustomHealthCheckerFactory>("envoy.health_checkers.http");
@@ -67,8 +67,8 @@ HealthCheckerSharedPtr HealthCheckerFactory::create(
     break;
   case envoy::config::core::v3::HealthCheck::HealthCheckerCase::kGrpcHealthCheck:
     if (!(cluster.info()->features() & Upstream::ClusterInfo::Features::HTTP2)) {
-      throw EnvoyException(fmt::format("{} cluster must support HTTP/2 for gRPC healthchecking",
-                                       cluster.info()->name()));
+      return absl::InvalidArgumentError(fmt::format(
+          "{} cluster must support HTTP/2 for gRPC healthchecking", cluster.info()->name()));
     }
     factory = &Config::Utility::getAndCheckFactoryByName<
         Server::Configuration::CustomHealthCheckerFactory>("envoy.health_checkers.grpc");
@@ -94,7 +94,7 @@ HealthCheckerSharedPtr HealthCheckerFactory::create(
   return factory->createCustomHealthChecker(health_check_config, *context);
 }
 
-PayloadMatcher::MatchSegments PayloadMatcher::loadProtoBytes(
+absl::StatusOr PayloadMatcher::loadProtoBytes(
     const Protobuf::RepeatedPtrField& byte_array) {
   MatchSegments result;
 
@@ -103,7 +103,7 @@ PayloadMatcher::MatchSegments PayloadMatcher::loadProtoBytes(
     if (entry.has_text()) {
       decoded = Hex::decode(entry.text());
       if (decoded.empty()) {
-        throw EnvoyException(fmt::format("invalid hex string '{}'", entry.text()));
+        return absl::InvalidArgumentError(fmt::format("invalid hex string '{}'", entry.text()));
       }
     } else {
       decoded.assign(entry.binary().begin(), entry.binary().end());
diff --git a/source/common/upstream/health_checker_impl.h b/source/common/upstream/health_checker_impl.h
index e6233a9b5773..a5ae845b1b52 100644
--- a/source/common/upstream/health_checker_impl.h
+++ b/source/common/upstream/health_checker_impl.h
@@ -92,7 +92,7 @@ class HealthCheckerFactory : public Logger::Loggable
                                         const std::string& config_hostname,
                                         const ClusterInfoConstSharedPtr& cluster);
   /**
-   * Create a health checker.
+   * Create a health checker or return an error.
    * @param health_check_config supplies the health check proto.
    * @param cluster supplies the owning cluster.
    * @param runtime supplies the runtime loader.
@@ -100,9 +100,9 @@ class HealthCheckerFactory : public Logger::Loggable
    * @param log_manager supplies the log_manager.
    * @param validation_visitor message validation visitor instance.
    * @param api reference to the Api object
-   * @return a health checker.
+   * @return a health checker or creation error.
    */
-  static HealthCheckerSharedPtr
+  static absl::StatusOr
   create(const envoy::config::core::v3::HealthCheck& health_check_config,
          Upstream::Cluster& cluster, Runtime::Loader& runtime, Event::Dispatcher& dispatcher,
          AccessLog::AccessLogManager& log_manager,
@@ -159,7 +159,7 @@ class PayloadMatcher {
 public:
   using MatchSegments = std::list>;
 
-  static MatchSegments loadProtoBytes(
+  static absl::StatusOr loadProtoBytes(
       const Protobuf::RepeatedPtrField& byte_array);
   static bool match(const MatchSegments& expected, const Buffer::Instance& buffer);
 };
diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc
index 08815cb4acdd..3a3db1cdb6eb 100644
--- a/source/common/upstream/health_discovery_service.cc
+++ b/source/common/upstream/health_discovery_service.cc
@@ -195,10 +195,11 @@ envoy::config::cluster::v3::Cluster HdsDelegate::createClusterConfig(
   return cluster_config;
 }
 
-void HdsDelegate::updateHdsCluster(HdsClusterPtr cluster,
-                                   const envoy::config::cluster::v3::Cluster& cluster_config,
-                                   const envoy::config::core::v3::BindConfig& bind_config) {
-  cluster->update(cluster_config, bind_config, info_factory_, tls_);
+absl::Status
+HdsDelegate::updateHdsCluster(HdsClusterPtr cluster,
+                              const envoy::config::cluster::v3::Cluster& cluster_config,
+                              const envoy::config::core::v3::BindConfig& bind_config) {
+  return cluster->update(cluster_config, bind_config, info_factory_, tls_);
 }
 
 HdsClusterPtr
@@ -216,7 +217,7 @@ HdsDelegate::createHdsCluster(const envoy::config::cluster::v3::Cluster& cluster
   return new_cluster;
 }
 
-void HdsDelegate::processMessage(
+absl::Status HdsDelegate::processMessage(
     std::unique_ptr&& message) {
   ENVOY_LOG(debug, "New health check response message {} ", message->DebugString());
   ASSERT(message);
@@ -239,7 +240,11 @@ void HdsDelegate::processMessage(
       if (cluster_map_pair != hds_clusters_name_map_.end()) {
         // We have a previous cluster with this name, update.
         cluster_ptr = cluster_map_pair->second;
-        updateHdsCluster(cluster_ptr, cluster_config, cluster_health_check.upstream_bind_config());
+        absl::Status status = updateHdsCluster(cluster_ptr, cluster_config,
+                                               cluster_health_check.upstream_bind_config());
+        if (!status.ok()) {
+          return status;
+        }
       } else {
         // There is no cluster with this name previously or its an empty string, so just create a
         // new cluster.
@@ -269,6 +274,7 @@ void HdsDelegate::processMessage(
   hds_clusters_ = std::move(hds_clusters);
 
   // TODO: add stats reporting for number of clusters added, removed, and reused.
+  return absl::OkStatus();
 }
 
 void HdsDelegate::onReceiveMessage(
@@ -301,8 +307,14 @@ void HdsDelegate::onReceiveMessage(
   // Set response
   auto server_response_ms = PROTOBUF_GET_MS_OR_DEFAULT(*message, interval, 1000);
 
-  // Process the HealthCheckSpecifier message.
-  processMessage(std::move(message));
+  /// Process the HealthCheckSpecifier message.
+  absl::Status status = processMessage(std::move(message));
+  if (!status.ok()) {
+    stats_.errors_.inc();
+    ENVOY_LOG(warn, "Unable to validate health check specifier: {}", status.message());
+    // Do not continue processing message
+    return;
+  }
 
   stats_.updates_.inc();
 
@@ -377,9 +389,9 @@ HdsCluster::HdsCluster(Server::Configuration::ServerFactoryContext& server_conte
       std::make_shared(std::move(hosts_by_locality), false);
 }
 
-void HdsCluster::update(envoy::config::cluster::v3::Cluster cluster,
-                        const envoy::config::core::v3::BindConfig& bind_config,
-                        ClusterInfoFactory& info_factory, ThreadLocal::SlotAllocator& tls) {
+absl::Status HdsCluster::update(envoy::config::cluster::v3::Cluster cluster,
+                                const envoy::config::core::v3::BindConfig& bind_config,
+                                ClusterInfoFactory& info_factory, ThreadLocal::SlotAllocator& tls) {
 
   // check to see if the config changed. If it did, update.
   const uint64_t config_hash = MessageUtil::hash(cluster);
@@ -402,11 +414,15 @@ void HdsCluster::update(envoy::config::cluster::v3::Cluster cluster,
     updateHosts(cluster_.load_assignment().endpoints(), update_cluster_info);
 
     // Check to see if any of the health checkers have changed.
-    updateHealthchecks(cluster_.health_checks());
+    absl::Status status = updateHealthchecks(cluster_.health_checks());
+    if (!status.ok()) {
+      return status;
+    }
   }
+  return absl::OkStatus();
 }
 
-void HdsCluster::updateHealthchecks(
+absl::Status HdsCluster::updateHealthchecks(
     const Protobuf::RepeatedPtrField& health_checks) {
   std::vector health_checkers;
   HealthCheckerMap health_checkers_map;
@@ -420,10 +436,12 @@ void HdsCluster::updateHealthchecks(
       health_checkers.push_back(health_checker->second);
     } else {
       // If it does not, create a new one.
-      auto new_health_checker = Upstream::HealthCheckerFactory::create(
+      auto checker_or_error = Upstream::HealthCheckerFactory::create(
           health_check, *this, server_context_.runtime(), server_context_.mainThreadDispatcher(),
           server_context_.accessLogManager(), server_context_.messageValidationVisitor(),
           server_context_.api());
+      RETURN_IF_STATUS_NOT_OK(checker_or_error);
+      auto new_health_checker = checker_or_error.value();
       health_checkers_map.insert({health_check, new_health_checker});
       health_checkers.push_back(new_health_checker);
 
@@ -437,6 +455,7 @@ void HdsCluster::updateHealthchecks(
   health_checkers_map_ = std::move(health_checkers_map);
 
   // TODO: add stats reporting for number of health checkers added, removed, and reused.
+  return absl::OkStatus();
 }
 
 void HdsCluster::updateHosts(
@@ -532,11 +551,13 @@ ProdClusterInfoFactory::createClusterInfo(const CreateClusterInfoParams& params)
 
 void HdsCluster::initHealthchecks() {
   for (auto& health_check : cluster_.health_checks()) {
-    auto health_checker = Upstream::HealthCheckerFactory::create(
+    auto health_checker_or_error = Upstream::HealthCheckerFactory::create(
         health_check, *this, server_context_.runtime(), server_context_.mainThreadDispatcher(),
         server_context_.accessLogManager(), server_context_.messageValidationVisitor(),
         server_context_.api());
+    THROW_IF_STATUS_NOT_OK(health_checker_or_error, throw);
 
+    auto health_checker = health_checker_or_error.value();
     health_checkers_.push_back(health_checker);
     health_checkers_map_.insert({health_check, health_checker});
     health_checker->start();
diff --git a/source/common/upstream/health_discovery_service.h b/source/common/upstream/health_discovery_service.h
index 11c7d0a8aa27..7880649b0319 100644
--- a/source/common/upstream/health_discovery_service.h
+++ b/source/common/upstream/health_discovery_service.h
@@ -64,9 +64,9 @@ class HdsCluster : public Cluster, Logger::Loggable {
   const Outlier::Detector* outlierDetector() const override { return outlier_detector_.get(); }
   void initialize(std::function callback) override;
   // Compare changes in the cluster proto, and update parts of the cluster as needed.
-  void update(envoy::config::cluster::v3::Cluster cluster,
-              const envoy::config::core::v3::BindConfig& bind_config,
-              ClusterInfoFactory& info_factory, ThreadLocal::SlotAllocator& tls);
+  absl::Status update(envoy::config::cluster::v3::Cluster cluster,
+                      const envoy::config::core::v3::BindConfig& bind_config,
+                      ClusterInfoFactory& info_factory, ThreadLocal::SlotAllocator& tls);
   // Creates healthcheckers and adds them to the list, then does initial start.
   void initHealthchecks();
 
@@ -98,7 +98,7 @@ class HdsCluster : public Cluster, Logger::Loggable {
   HealthCheckerMap health_checkers_map_;
   TimeSource& time_source_;
 
-  void updateHealthchecks(
+  absl::Status updateHealthchecks(
       const Protobuf::RepeatedPtrField& health_checks);
   void
   updateHosts(const Protobuf::RepeatedPtrField&
@@ -157,12 +157,13 @@ class HdsDelegate : Grpc::AsyncStreamCallbacks&& message);
+  absl::Status
+  processMessage(std::unique_ptr&& message);
   envoy::config::cluster::v3::Cluster
   createClusterConfig(const envoy::service::health::v3::ClusterHealthCheck& cluster_health_check);
-  void updateHdsCluster(HdsClusterPtr cluster,
-                        const envoy::config::cluster::v3::Cluster& cluster_health_check,
-                        const envoy::config::core::v3::BindConfig& bind_config);
+  absl::Status updateHdsCluster(HdsClusterPtr cluster,
+                                const envoy::config::cluster::v3::Cluster& cluster_health_check,
+                                const envoy::config::core::v3::BindConfig& bind_config);
   HdsClusterPtr createHdsCluster(const envoy::config::cluster::v3::Cluster& cluster_health_check,
                                  const envoy::config::core::v3::BindConfig& bind_config);
   HdsDelegateStats stats_;
diff --git a/source/extensions/health_checkers/http/health_checker_impl.cc b/source/extensions/health_checkers/http/health_checker_impl.cc
index c19da7eee4e4..49b07984222e 100644
--- a/source/extensions/health_checkers/http/health_checker_impl.cc
+++ b/source/extensions/health_checkers/http/health_checker_impl.cc
@@ -63,7 +63,6 @@ HttpHealthCheckerImpl::HttpHealthCheckerImpl(const Cluster& cluster,
                                              HealthCheckEventLoggerPtr&& event_logger)
     : HealthCheckerImplBase(cluster, config, dispatcher, runtime, random, std::move(event_logger)),
       path_(config.http_health_check().path()), host_value_(config.http_health_check().host()),
-      receive_bytes_(PayloadMatcher::loadProtoBytes(config.http_health_check().receive())),
       method_(getMethod(config.http_health_check().method())),
       response_buffer_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(
           config.http_health_check(), response_buffer_size, kDefaultMaxBytesInBuffer)),
@@ -75,6 +74,9 @@ HttpHealthCheckerImpl::HttpHealthCheckerImpl(const Cluster& cluster,
                            static_cast(Http::Code::OK)),
       codec_client_type_(codecClientType(config.http_health_check().codec_client_type())),
       random_generator_(random) {
+  auto bytes_or_error = PayloadMatcher::loadProtoBytes(config.http_health_check().receive());
+  THROW_IF_STATUS_NOT_OK(bytes_or_error, throw);
+  receive_bytes_ = bytes_or_error.value();
   if (config.http_health_check().has_service_name_matcher()) {
     service_name_matcher_.emplace(config.http_health_check().service_name_matcher());
   }
diff --git a/source/extensions/health_checkers/http/health_checker_impl.h b/source/extensions/health_checkers/http/health_checker_impl.h
index bb3caca64477..6230166aee8e 100644
--- a/source/extensions/health_checkers/http/health_checker_impl.h
+++ b/source/extensions/health_checkers/http/health_checker_impl.h
@@ -164,7 +164,7 @@ class HttpHealthCheckerImpl : public HealthCheckerImplBase {
 
   const std::string path_;
   const std::string host_value_;
-  const PayloadMatcher::MatchSegments receive_bytes_;
+  PayloadMatcher::MatchSegments receive_bytes_;
   const envoy::config::core::v3::RequestMethod method_;
   uint64_t response_buffer_size_;
   absl::optional>
diff --git a/source/extensions/health_checkers/tcp/health_checker_impl.cc b/source/extensions/health_checkers/tcp/health_checker_impl.cc
index aefab127a1d2..0c1b540157f8 100644
--- a/source/extensions/health_checkers/tcp/health_checker_impl.cc
+++ b/source/extensions/health_checkers/tcp/health_checker_impl.cc
@@ -53,9 +53,14 @@ TcpHealthCheckerImpl::TcpHealthCheckerImpl(const Cluster& cluster,
         if (!config.tcp_health_check().send().text().empty()) {
           send_repeated.Add()->CopyFrom(config.tcp_health_check().send());
         }
-        return PayloadMatcher::loadProtoBytes(send_repeated);
-      }()),
-      receive_bytes_(PayloadMatcher::loadProtoBytes(config.tcp_health_check().receive())) {}
+        auto bytes_or_error = PayloadMatcher::loadProtoBytes(send_repeated);
+        THROW_IF_STATUS_NOT_OK(bytes_or_error, throw);
+        return bytes_or_error.value();
+      }()) {
+  auto bytes_or_error = PayloadMatcher::loadProtoBytes(config.tcp_health_check().receive());
+  THROW_IF_STATUS_NOT_OK(bytes_or_error, throw);
+  receive_bytes_ = bytes_or_error.value();
+}
 
 TcpHealthCheckerImpl::TcpActiveHealthCheckSession::~TcpActiveHealthCheckSession() {
   ASSERT(client_ == nullptr);
diff --git a/source/extensions/health_checkers/tcp/health_checker_impl.h b/source/extensions/health_checkers/tcp/health_checker_impl.h
index f8c525bc8235..4ac85c934270 100644
--- a/source/extensions/health_checkers/tcp/health_checker_impl.h
+++ b/source/extensions/health_checkers/tcp/health_checker_impl.h
@@ -101,7 +101,7 @@ class TcpHealthCheckerImpl : public HealthCheckerImplBase {
   }
 
   const PayloadMatcher::MatchSegments send_bytes_;
-  const PayloadMatcher::MatchSegments receive_bytes_;
+  PayloadMatcher::MatchSegments receive_bytes_;
 };
 
 } // namespace Upstream
diff --git a/test/common/upstream/hds_test.cc b/test/common/upstream/hds_test.cc
index e3ee0677e7c1..dd51ad0a6152 100644
--- a/test/common/upstream/hds_test.cc
+++ b/test/common/upstream/hds_test.cc
@@ -51,7 +51,7 @@ class HdsDelegateFriend {
   void processPrivateMessage(
       HdsDelegate& hd,
       std::unique_ptr&& message) {
-    hd.processMessage(std::move(message));
+    ASSERT_TRUE(hd.processMessage(std::move(message)).ok());
   };
   HdsDelegateStats getStats(HdsDelegate& hd) { return hd.stats_; };
 };
@@ -118,7 +118,7 @@ class HdsTest : public testing::Test {
 
   // Creates a HealthCheckSpecifier message that contains one endpoint and one
   // healthcheck
-  envoy::service::health::v3::HealthCheckSpecifier* createSimpleMessage() {
+  envoy::service::health::v3::HealthCheckSpecifier* createSimpleMessage(bool http = true) {
     envoy::service::health::v3::HealthCheckSpecifier* msg =
         new envoy::service::health::v3::HealthCheckSpecifier;
     msg->mutable_interval()->set_seconds(1);
@@ -130,10 +130,11 @@ class HdsTest : public testing::Test {
     health_check->mutable_health_checks(0)->mutable_unhealthy_threshold()->set_value(2);
     health_check->mutable_health_checks(0)->mutable_healthy_threshold()->set_value(2);
     health_check->mutable_health_checks(0)->mutable_grpc_health_check();
-    health_check->mutable_health_checks(0)->mutable_http_health_check()->set_codec_client_type(
-        envoy::type::v3::HTTP1);
-    health_check->mutable_health_checks(0)->mutable_http_health_check()->set_path("/healthcheck");
-
+    if (http) {
+      health_check->mutable_health_checks(0)->mutable_http_health_check()->set_codec_client_type(
+          envoy::type::v3::HTTP1);
+      health_check->mutable_health_checks(0)->mutable_http_health_check()->set_path("/healthcheck");
+    }
     auto* locality_endpoints = health_check->add_locality_endpoints();
     // add locality information to this endpoint set of one endpoint.
     auto* locality = locality_endpoints->mutable_locality();
@@ -465,6 +466,38 @@ TEST_F(HdsTest, TestProcessMessageMissingFieldsWithFallback) {
   EXPECT_EQ(hds_delegate_friend_.getStats(*hds_delegate_).requests_.value(), 2);
 }
 
+// Test if processMessage exits gracefully if the update fails
+TEST_F(HdsTest, TestProcessMessageInvalidFieldsWithFallback) {
+  EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_));
+  EXPECT_CALL(async_stream_, sendMessageRaw_(_, _));
+  createHdsDelegate();
+
+  // Create Message
+  message.reset(createSimpleMessage());
+
+  Network::MockClientConnection* connection = new NiceMock();
+  EXPECT_CALL(server_context_.dispatcher_, createClientConnection_(_, _, _, _))
+      .WillRepeatedly(Return(connection));
+  EXPECT_CALL(*server_response_timer_, enableTimer(_, _));
+  EXPECT_CALL(test_factory_, createClusterInfo(_)).WillOnce(Return(cluster_info_));
+  EXPECT_CALL(*connection, setBufferLimits(_));
+  EXPECT_CALL(server_context_.dispatcher_, deferredDelete_(_));
+  // Process message
+  hds_delegate_->onReceiveMessage(std::move(message));
+  connection->raiseEvent(Network::ConnectionEvent::Connected);
+
+  // Create a invalid message: grpc health checks require an H2 cluster
+  message.reset(createSimpleMessage(false));
+
+  // Pass invalid message through. Should increment stat_ errors upon
+  // getting a bad message.
+  hds_delegate_->onReceiveMessage(std::move(message));
+
+  // Check Correctness by verifying one request and one error has been generated in stat_
+  EXPECT_EQ(hds_delegate_friend_.getStats(*hds_delegate_).errors_.value(), 1);
+  EXPECT_EQ(hds_delegate_friend_.getStats(*hds_delegate_).requests_.value(), 2);
+}
+
 // Test if sendResponse() retains the structure of all endpoints ingested in the specifier
 // from onReceiveMessage(). This verifies that all endpoints are grouped by the correct
 // cluster and the correct locality.
diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc
index 8c44cd9b1f31..24d1bd48e124 100644
--- a/test/common/upstream/health_checker_impl_test.cc
+++ b/test/common/upstream/health_checker_impl_test.cc
@@ -81,10 +81,11 @@ TEST(HealthCheckerFactoryTest, GrpcHealthCheckHTTP2NotConfiguredException) {
   NiceMock validation_visitor;
   Api::MockApi api;
 
-  EXPECT_THROW_WITH_MESSAGE(
-      HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, runtime, dispatcher,
-                                   log_manager, validation_visitor, api),
-      EnvoyException, "fake_cluster cluster must support HTTP/2 for gRPC healthchecking");
+  EXPECT_EQ(HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, runtime,
+                                         dispatcher, log_manager, validation_visitor, api)
+                .status()
+                .message(),
+            "fake_cluster cluster must support HTTP/2 for gRPC healthchecking");
 }
 
 TEST(HealthCheckerFactoryTest, CreateGrpc) {
@@ -103,6 +104,7 @@ TEST(HealthCheckerFactoryTest, CreateGrpc) {
             dynamic_cast(
                 HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, runtime,
                                              dispatcher, log_manager, validation_visitor, api)
+                    .value()
                     .get()));
 }
 
@@ -4063,7 +4065,8 @@ TEST(PayloadMatcher, loadJsonBytes) {
     repeated_payload.Add()->set_text("39000000");
     repeated_payload.Add()->set_text("EEEEEEEE");
 
-    PayloadMatcher::MatchSegments segments = PayloadMatcher::loadProtoBytes(repeated_payload);
+    PayloadMatcher::MatchSegments segments =
+        PayloadMatcher::loadProtoBytes(repeated_payload).value();
     EXPECT_EQ(2U, segments.size());
   }
 
@@ -4071,14 +4074,14 @@ TEST(PayloadMatcher, loadJsonBytes) {
     Protobuf::RepeatedPtrField repeated_payload;
     repeated_payload.Add()->set_text("4");
 
-    EXPECT_THROW(PayloadMatcher::loadProtoBytes(repeated_payload), EnvoyException);
+    EXPECT_FALSE(PayloadMatcher::loadProtoBytes(repeated_payload).status().ok());
   }
 
   {
     Protobuf::RepeatedPtrField repeated_payload;
     repeated_payload.Add()->set_text("gg");
 
-    EXPECT_THROW(PayloadMatcher::loadProtoBytes(repeated_payload), EnvoyException);
+    EXPECT_FALSE(PayloadMatcher::loadProtoBytes(repeated_payload).status().ok());
   }
 }
 
@@ -4091,7 +4094,7 @@ TEST(PayloadMatcher, matchHexText) {
   repeated_payload.Add()->set_text("01");
   repeated_payload.Add()->set_text("02");
 
-  PayloadMatcher::MatchSegments segments = PayloadMatcher::loadProtoBytes(repeated_payload);
+  PayloadMatcher::MatchSegments segments = PayloadMatcher::loadProtoBytes(repeated_payload).value();
 
   Buffer::OwnedImpl buffer;
   EXPECT_FALSE(PayloadMatcher::match(segments, buffer));
@@ -4118,7 +4121,7 @@ TEST(PayloadMatcher, matchBinary) {
   Protobuf::RepeatedPtrField repeated_payload;
   repeated_payload.Add()->set_binary(std::string({0x01, 0x02}));
 
-  PayloadMatcher::MatchSegments segments = PayloadMatcher::loadProtoBytes(repeated_payload);
+  PayloadMatcher::MatchSegments segments = PayloadMatcher::loadProtoBytes(repeated_payload).value();
 
   Buffer::OwnedImpl buffer;
   EXPECT_FALSE(PayloadMatcher::match(segments, buffer));
diff --git a/test/extensions/health_checkers/redis/config_test.cc b/test/extensions/health_checkers/redis/config_test.cc
index 644bf0538e26..a183dbde4582 100644
--- a/test/extensions/health_checkers/redis/config_test.cc
+++ b/test/extensions/health_checkers/redis/config_test.cc
@@ -122,6 +122,7 @@ TEST(HealthCheckerFactoryTest, CreateRedisViaUpstreamHealthCheckerFactory) {
                 Upstream::HealthCheckerFactory::create(
                     Upstream::parseHealthCheckFromV3Yaml(yaml), cluster, runtime, dispatcher,
                     log_manager, ProtobufMessage::getStrictValidationVisitor(), api)
+                    .value()
                     .get()));
 }
 } // namespace
diff --git a/test/extensions/health_checkers/thrift/config_test.cc b/test/extensions/health_checkers/thrift/config_test.cc
index 601ec2709e80..924610631f92 100644
--- a/test/extensions/health_checkers/thrift/config_test.cc
+++ b/test/extensions/health_checkers/thrift/config_test.cc
@@ -196,6 +196,7 @@ TEST(HealthCheckerFactoryTest, CreateThriftViaUpstreamHealthCheckerFactory) {
                 Upstream::HealthCheckerFactory::create(
                     Upstream::parseHealthCheckFromV3Yaml(yaml), cluster, runtime, dispatcher,
                     log_manager, ProtobufMessage::getStrictValidationVisitor(), api)
+                    .value()
                     .get()));
 }
 
diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml
index 5129d452f7d8..4e9adbb9aef1 100644
--- a/tools/code_format/config.yaml
+++ b/tools/code_format/config.yaml
@@ -97,7 +97,6 @@ paths:
     - source/common/upstream/cds_api_helper.cc
     - source/common/upstream/thread_aware_lb_impl.cc
     - source/common/upstream/load_balancer_impl.cc
-    - source/common/upstream/health_checker_impl.cc
     - source/common/upstream/cluster_manager_impl.cc
     - source/common/upstream/od_cds_api_impl.cc
     - source/common/upstream/cds_api_impl.cc

From f00166c4cc95b5d3aea14af40a7afe0e849cc78d Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Thu, 15 Jun 2023 15:45:25 -0400
Subject: [PATCH 551/740] =?UTF-8?q?tools:=20disallowing=20raw=20try=20for?=
 =?UTF-8?q?=20extensions=20in=20favor=20of=20TRY=5FASSERT=5FMAIN=E2=80=A6?=
 =?UTF-8?q?=20(#27987)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

also fixing a bug in multi-catch in the main build (not yet used) and replacing two catch calls with CATCH macros while I'm in here

Risk Level: low
Testing: n/a
Docs Changes: n/a
Release Notes: n/a
Part of envoyproxy/envoy-mobile#176 #27412
Signed-off-by: Alyssa Wilk 
---
 source/common/common/thread.h                 |  2 +-
 .../common/aws/credentials_provider_impl.cc   | 10 ++++----
 source/extensions/common/tap/admin.cc         |  5 ++--
 .../http/aws_lambda/aws_lambda_filter.cc      |  5 ++--
 .../aws_request_signing_filter.cc             | 10 ++++----
 .../filters/http/composite/action.h           |  7 +++---
 .../extensions/filters/http/lua/lua_filter.cc | 20 +++++++---------
 .../filters/http/oauth2/oauth_client.cc       |  5 ++--
 .../network/common/redis/client_impl.cc       |  5 ++--
 .../network/dubbo_proxy/active_message.cc     |  8 ++++---
 .../network/dubbo_proxy/conn_manager.cc       | 10 ++++----
 .../filters/network/mongo_proxy/proxy.cc      |  5 ++--
 .../filters/network/mongo_proxy/utility.cc    |  6 ++---
 .../network/redis_proxy/conn_pool_impl.cc     | 10 ++++----
 .../network/redis_proxy/proxy_filter.cc       |  5 ++--
 .../network/thrift_proxy/conn_manager.cc      | 23 ++++++++++---------
 .../thrift_proxy/router/shadow_writer_impl.h  |  8 +++----
 .../network/zookeeper_proxy/decoder.cc        | 10 ++++----
 .../filters/udp/dns_filter/dns_filter.cc      |  8 ++++---
 .../grpc_credentials/aws_iam/config.cc        |  5 ++--
 .../skywalking/skywalking_tracer_impl.cc      |  5 ++--
 source/extensions/tracers/xray/config.cc      |  5 ++--
 .../tracers/xray/localized_sampling.cc        |  5 ++--
 .../tracers/zipkin/zipkin_tracer_impl.cc      |  5 ++--
 source/server/admin/html/active_stats.js      |  4 ++--
 source/server/server.cc                       |  8 +++----
 tools/code_format/check_format.py             |  6 ++---
 tools/code_format/config.yaml                 |  2 --
 28 files changed, 102 insertions(+), 105 deletions(-)

diff --git a/source/common/common/thread.h b/source/common/common/thread.h
index a1d651efe679..f1d2128a2204 100644
--- a/source/common/common/thread.h
+++ b/source/common/common/thread.h
@@ -255,7 +255,7 @@ class MainThread {
 #endif
 
 #ifdef ENVOY_DISABLE_EXCEPTIONS
-#define MULTI_CATCH(ExceptionType, Handler)
+#define MULTI_CATCH(ExceptionType, Handler, Handler2)
 #else
 #define MULTI_CATCH(Exception, Handler, Handler2)                                                  \
   catch (Exception) {                                                                              \
diff --git a/source/extensions/common/aws/credentials_provider_impl.cc b/source/extensions/common/aws/credentials_provider_impl.cc
index ee83614e0885..88709fdbeb6a 100644
--- a/source/extensions/common/aws/credentials_provider_impl.cc
+++ b/source/extensions/common/aws/credentials_provider_impl.cc
@@ -246,9 +246,8 @@ void InstanceProfileCredentialsProvider::extractCredentials(
     return;
   }
   Json::ObjectSharedPtr document_json;
-  try {
-    document_json = Json::Factory::loadFromString(credential_document_value);
-  } catch (EnvoyException& e) {
+  TRY_NEEDS_AUDIT { document_json = Json::Factory::loadFromString(credential_document_value); }
+  END_TRY catch (EnvoyException& e) {
     ENVOY_LOG(error, "Could not parse AWS credentials document: {}", e.what());
     return;
   }
@@ -298,9 +297,8 @@ void TaskRoleCredentialsProvider::extractCredentials(const std::string& credenti
     return;
   }
   Json::ObjectSharedPtr document_json;
-  try {
-    document_json = Json::Factory::loadFromString(credential_document_value);
-  } catch (EnvoyException& e) {
+  TRY_NEEDS_AUDIT { document_json = Json::Factory::loadFromString(credential_document_value); }
+  END_TRY catch (EnvoyException& e) {
     ENVOY_LOG(error, "Could not parse AWS credentials document from the task role: {}", e.what());
     return;
   }
diff --git a/source/extensions/common/tap/admin.cc b/source/extensions/common/tap/admin.cc
index 122277eaac0e..e97a3dcb741c 100644
--- a/source/extensions/common/tap/admin.cc
+++ b/source/extensions/common/tap/admin.cc
@@ -55,12 +55,11 @@ Http::Code AdminHandler::handler(Http::HeaderMap&, Buffer::Instance& response,
   }
 
   envoy::admin::v3::TapRequest tap_request;
-  try {
+  TRY_NEEDS_AUDIT {
     MessageUtil::loadFromYamlAndValidate(admin_stream.getRequestBody()->toString(), tap_request,
                                          ProtobufMessage::getStrictValidationVisitor());
-  } catch (EnvoyException& e) {
-    return badRequest(response, e.what());
   }
+  END_TRY catch (EnvoyException& e) { return badRequest(response, e.what()); }
 
   ENVOY_LOG(debug, "tap admin request for config_id={}", tap_request.config_id());
   if (config_id_map_.count(tap_request.config_id()) == 0) {
diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc
index ba88de25429c..f47f5b8915d2 100644
--- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc
+++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc
@@ -312,10 +312,11 @@ void Filter::dejsonizeResponse(Http::ResponseHeaderMap& headers, const Buffer::I
                                Buffer::Instance& body) {
   using source::extensions::filters::http::aws_lambda::Response;
   Response json_resp;
-  try {
+  TRY_NEEDS_AUDIT {
     MessageUtil::loadFromJson(json_buf.toString(), json_resp,
                               ProtobufMessage::getNullValidationVisitor());
-  } catch (EnvoyException& ex) {
+  }
+  END_TRY catch (EnvoyException& ex) {
     // We would only get here if all of the following are true:
     // 1- Passthrough is set to false
     // 2- Lambda returned a 200 OK
diff --git a/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.cc b/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.cc
index beefccd8c800..f9705dca40bf 100644
--- a/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.cc
+++ b/source/extensions/filters/http/aws_request_signing/aws_request_signing_filter.cc
@@ -46,7 +46,7 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers,
     return Http::FilterHeadersStatus::StopIteration;
   }
 
-  try {
+  TRY_NEEDS_AUDIT {
     ENVOY_LOG(debug, "aws request signing from decodeHeaders use_unsigned_payload: {}",
               use_unsigned_payload);
     if (use_unsigned_payload) {
@@ -55,7 +55,8 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers,
       config.signer().signEmptyPayload(headers);
     }
     config.stats().signing_added_.inc();
-  } catch (const EnvoyException& e) {
+  }
+  END_TRY catch (const EnvoyException& e) {
     // TODO: sign should not throw to avoid exceptions in the request path
     ENVOY_LOG(debug, "signing failed: {}", e.what());
     config.stats().signing_failed_.inc();
@@ -82,13 +83,14 @@ Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_strea
   auto& hashing_util = Envoy::Common::Crypto::UtilitySingleton::get();
   const std::string hash = Hex::encode(hashing_util.getSha256Digest(decoding_buffer));
 
-  try {
+  TRY_NEEDS_AUDIT {
     ENVOY_LOG(debug, "aws request signing from decodeData");
     ASSERT(request_headers_ != nullptr);
     config.signer().sign(*request_headers_, hash);
     config.stats().signing_added_.inc();
     config.stats().payload_signing_added_.inc();
-  } catch (const EnvoyException& e) {
+  }
+  END_TRY catch (const EnvoyException& e) {
     // TODO: sign should not throw to avoid exceptions in the request path
     ENVOY_LOG(debug, "signing failed: {}", e.what());
     config.stats().signing_failed_.inc();
diff --git a/source/extensions/filters/http/composite/action.h b/source/extensions/filters/http/composite/action.h
index 86b72de5ce7c..62f28135ae7c 100644
--- a/source/extensions/filters/http/composite/action.h
+++ b/source/extensions/filters/http/composite/action.h
@@ -45,18 +45,19 @@ class ExecuteFilterActionFactory
 
     // TODO(tyxia) Update the logic later to create the filter from the `factoryContext` first if it
     // is present.
-    try {
+    TRY_NEEDS_AUDIT {
       if (context.server_factory_context_.has_value()) {
         callback = factory.createFilterServerFactoryFromProto(
             *message, context.stat_prefix_, context.server_factory_context_.value());
       }
-    } catch (EnvoyException& e) {
+    }
+    END_TRY CATCH(EnvoyException & e, {
       // First, we try to create the delegated filter creation callback from server factory context.
       // If it failed (i.e., the corresponding filter doesn't support this method), we log this
       // message and fallback to creating the filter from factory context.
       ENVOY_LOG(trace,
                 absl::StrCat(e.what(), ", fallback to creating the filter from factory context."));
-    }
+    });
 
     if (callback == nullptr) {
       RELEASE_ASSERT(context.factory_context_.has_value(),
diff --git a/source/extensions/filters/http/lua/lua_filter.cc b/source/extensions/filters/http/lua/lua_filter.cc
index 83526dbde05b..0f4b53f22934 100644
--- a/source/extensions/filters/http/lua/lua_filter.cc
+++ b/source/extensions/filters/http/lua/lua_filter.cc
@@ -456,12 +456,11 @@ void StreamHandleWrapper::onSuccess(const Http::AsyncClient::Request&,
     state_ = State::Running;
     markLive();
 
-    try {
+    TRY_NEEDS_AUDIT {
       resumeCoroutine(2, yield_callback_);
       markDead();
-    } catch (const Filters::Common::Lua::LuaException& e) {
-      filter_.scriptError(e);
     }
+    END_TRY catch (const Filters::Common::Lua::LuaException& e) { filter_.scriptError(e); }
 
     if (state_ == State::Running) {
       headers_continued_ = true;
@@ -864,12 +863,11 @@ Filter::doHeaders(StreamHandleRef& handle, Filters::Common::Lua::CoroutinePtr& c
                true);
 
   Http::FilterHeadersStatus status = Http::FilterHeadersStatus::Continue;
-  try {
+  TRY_NEEDS_AUDIT {
     status = handle.get()->start(function_ref);
     handle.markDead();
-  } catch (const Filters::Common::Lua::LuaException& e) {
-    scriptError(e);
   }
+  END_TRY catch (const Filters::Common::Lua::LuaException& e) { scriptError(e); }
 
   return status;
 }
@@ -878,13 +876,12 @@ Http::FilterDataStatus Filter::doData(StreamHandleRef& handle, Buffer::Instance&
                                       bool end_stream) {
   Http::FilterDataStatus status = Http::FilterDataStatus::Continue;
   if (handle.get() != nullptr) {
-    try {
+    TRY_NEEDS_AUDIT {
       handle.markLive();
       status = handle.get()->onData(data, end_stream);
       handle.markDead();
-    } catch (const Filters::Common::Lua::LuaException& e) {
-      scriptError(e);
     }
+    END_TRY catch (const Filters::Common::Lua::LuaException& e) { scriptError(e); }
   }
 
   return status;
@@ -893,13 +890,12 @@ Http::FilterDataStatus Filter::doData(StreamHandleRef& handle, Buffer::Instance&
 Http::FilterTrailersStatus Filter::doTrailers(StreamHandleRef& handle, Http::HeaderMap& trailers) {
   Http::FilterTrailersStatus status = Http::FilterTrailersStatus::Continue;
   if (handle.get() != nullptr) {
-    try {
+    TRY_NEEDS_AUDIT {
       handle.markLive();
       status = handle.get()->onTrailers(trailers);
       handle.markDead();
-    } catch (const Filters::Common::Lua::LuaException& e) {
-      scriptError(e);
     }
+    END_TRY catch (const Filters::Common::Lua::LuaException& e) { scriptError(e); }
   }
 
   return status;
diff --git a/source/extensions/filters/http/oauth2/oauth_client.cc b/source/extensions/filters/http/oauth2/oauth_client.cc
index d74588cc4480..21fd343beb09 100644
--- a/source/extensions/filters/http/oauth2/oauth_client.cc
+++ b/source/extensions/filters/http/oauth2/oauth_client.cc
@@ -93,9 +93,10 @@ void OAuth2ClientImpl::onSuccess(const Http::AsyncClient::Request&,
   const std::string response_body = message->bodyAsString();
 
   envoy::extensions::http_filters::oauth2::OAuthResponse response;
-  try {
+  TRY_NEEDS_AUDIT {
     MessageUtil::loadFromJson(response_body, response, ProtobufMessage::getNullValidationVisitor());
-  } catch (EnvoyException& e) {
+  }
+  END_TRY catch (EnvoyException& e) {
     ENVOY_LOG(debug, "Error parsing response body, received exception: {}", e.what());
     ENVOY_LOG(debug, "Response body: {}", response_body);
     parent_->sendUnauthorizedResponse();
diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc
index 53b578d305e5..121afcff70a0 100644
--- a/source/extensions/filters/network/common/redis/client_impl.cc
+++ b/source/extensions/filters/network/common/redis/client_impl.cc
@@ -161,9 +161,8 @@ void ClientImpl::onConnectOrOpTimeout() {
 }
 
 void ClientImpl::onData(Buffer::Instance& data) {
-  try {
-    decoder_->decode(data);
-  } catch (ProtocolError&) {
+  TRY_NEEDS_AUDIT { decoder_->decode(data); }
+  END_TRY catch (ProtocolError&) {
     putOutlierEvent(Upstream::Outlier::Result::ExtOriginRequestFailed);
     host_->cluster().trafficStats()->upstream_cx_protocol_error_.inc();
     host_->stats().rq_error_.inc();
diff --git a/source/extensions/filters/network/dubbo_proxy/active_message.cc b/source/extensions/filters/network/dubbo_proxy/active_message.cc
index f95351aab143..9d019f747f19 100644
--- a/source/extensions/filters/network/dubbo_proxy/active_message.cc
+++ b/source/extensions/filters/network/dubbo_proxy/active_message.cc
@@ -375,7 +375,7 @@ void ActiveMessage::startUpstreamResponse() {
 DubboFilters::UpstreamResponseStatus ActiveMessage::upstreamData(Buffer::Instance& buffer) {
   ASSERT(response_decoder_ != nullptr);
 
-  try {
+  TRY_NEEDS_AUDIT {
     auto status = response_decoder_->onData(buffer);
     if (status == DubboFilters::UpstreamResponseStatus::Complete) {
       if (requestId() != response_decoder_->requestId()) {
@@ -390,12 +390,14 @@ DubboFilters::UpstreamResponseStatus ActiveMessage::upstreamData(Buffer::Instanc
     }
 
     return status;
-  } catch (const DownstreamConnectionCloseException& ex) {
+  }
+  END_TRY catch (const DownstreamConnectionCloseException& ex) {
     ENVOY_CONN_LOG(error, "dubbo response: exception ({})", parent_.connection(), ex.what());
     onReset();
     parent_.stats().response_error_caused_connection_close_.inc();
     return DubboFilters::UpstreamResponseStatus::Reset;
-  } catch (const EnvoyException& ex) {
+  }
+  catch (const EnvoyException& ex) {
     ENVOY_CONN_LOG(error, "dubbo response: exception ({})", parent_.connection(), ex.what());
     parent_.stats().response_decoding_error_.inc();
 
diff --git a/source/extensions/filters/network/dubbo_proxy/conn_manager.cc b/source/extensions/filters/network/dubbo_proxy/conn_manager.cc
index 692f2d183286..f431fef69d23 100644
--- a/source/extensions/filters/network/dubbo_proxy/conn_manager.cc
+++ b/source/extensions/filters/network/dubbo_proxy/conn_manager.cc
@@ -97,13 +97,14 @@ void ConnectionManager::dispatch() {
     return;
   }
 
-  try {
+  TRY_NEEDS_AUDIT {
     bool underflow = false;
     while (!underflow) {
       decoder_->onData(request_buffer_, underflow);
     }
     return;
-  } catch (const EnvoyException& ex) {
+  }
+  END_TRY catch (const EnvoyException& ex) {
     ENVOY_CONN_LOG(error, "dubbo error: {}", read_callbacks_->connection(), ex.what());
     read_callbacks_->connection().close(Network::ConnectionCloseType::NoFlush);
     stats_.request_decoding_error_.inc();
@@ -121,11 +122,12 @@ void ConnectionManager::sendLocalReply(MessageMetadata& metadata,
   DubboFilters::DirectResponse::ResponseType result =
       DubboFilters::DirectResponse::ResponseType::ErrorReply;
 
-  try {
+  TRY_NEEDS_AUDIT {
     Buffer::OwnedImpl buffer;
     result = response.encode(metadata, *protocol_, buffer);
     read_callbacks_->connection().write(buffer, end_stream);
-  } catch (const EnvoyException& ex) {
+  }
+  END_TRY catch (const EnvoyException& ex) {
     ENVOY_CONN_LOG(error, "dubbo error: {}", read_callbacks_->connection(), ex.what());
   }
 
diff --git a/source/extensions/filters/network/mongo_proxy/proxy.cc b/source/extensions/filters/network/mongo_proxy/proxy.cc
index 86db1078f659..b046721afed5 100644
--- a/source/extensions/filters/network/mongo_proxy/proxy.cc
+++ b/source/extensions/filters/network/mongo_proxy/proxy.cc
@@ -335,9 +335,8 @@ void ProxyFilter::doDecode(Buffer::Instance& buffer) {
     decoder_ = createDecoder(*this);
   }
 
-  try {
-    decoder_->onData(buffer);
-  } catch (EnvoyException& e) {
+  TRY_NEEDS_AUDIT { decoder_->onData(buffer); }
+  END_TRY catch (EnvoyException& e) {
     ENVOY_LOG(info, "mongo decoding error: {}", e.what());
     stats_.decoding_error_.inc();
     sniffing_ = false;
diff --git a/source/extensions/filters/network/mongo_proxy/utility.cc b/source/extensions/filters/network/mongo_proxy/utility.cc
index 421eceaa9b64..ecdff4da9074 100644
--- a/source/extensions/filters/network/mongo_proxy/utility.cc
+++ b/source/extensions/filters/network/mongo_proxy/utility.cc
@@ -4,6 +4,7 @@
 
 #include "envoy/common/exception.h"
 
+#include "source/common/common/thread.h"
 #include "source/common/json/json_loader.h"
 
 namespace Envoy {
@@ -101,12 +102,11 @@ std::string QueryMessageInfo::parseCallingFunction(const QueryMessage& query) {
 }
 
 std::string QueryMessageInfo::parseCallingFunctionJson(const std::string& json_string) {
-  try {
+  TRY_NEEDS_AUDIT {
     Json::ObjectSharedPtr json = Json::Factory::loadFromString(json_string);
     return json->getString("callingFunction");
-  } catch (Json::Exception&) {
-    return "";
   }
+  END_TRY catch (Json::Exception&) { return ""; }
 }
 
 QueryMessageInfo::QueryType QueryMessageInfo::parseType(const QueryMessage& query) {
diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc
index 0330bfee1077..6e315a42ffa6 100644
--- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc
+++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc
@@ -348,11 +348,10 @@ Common::Redis::Client::PoolRequest* InstanceImpl::ThreadLocalPool::makeRequestTo
     if (!absl::SimpleAtoi(ip_port, &ip_port_number) || (ip_port_number > 65535)) {
       return nullptr;
     }
-    try {
+    TRY_NEEDS_AUDIT {
       address_ptr = std::make_shared(ip_address, ip_port_number);
-    } catch (const EnvoyException&) {
-      return nullptr;
     }
+    END_TRY catch (const EnvoyException&) { return nullptr; }
     host_address_map_key = address_ptr->asString();
   }
 
@@ -371,11 +370,10 @@ Common::Redis::Client::PoolRequest* InstanceImpl::ThreadLocalPool::makeRequestTo
       if (!absl::SimpleAtoi(ip_port, &ip_port_number) || (ip_port_number > 65535)) {
         return nullptr;
       }
-      try {
+      TRY_NEEDS_AUDIT {
         address_ptr = std::make_shared(ip_address, ip_port_number);
-      } catch (const EnvoyException&) {
-        return nullptr;
       }
+      END_TRY catch (const EnvoyException&) { return nullptr; }
     }
     Upstream::HostSharedPtr new_host{new Upstream::HostImpl(
         cluster_->info(), "", address_ptr, nullptr, 1, envoy::config::core::v3::Locality(),
diff --git a/source/extensions/filters/network/redis_proxy/proxy_filter.cc b/source/extensions/filters/network/redis_proxy/proxy_filter.cc
index cd66f63fa78c..0cdc572bfdf2 100644
--- a/source/extensions/filters/network/redis_proxy/proxy_filter.cc
+++ b/source/extensions/filters/network/redis_proxy/proxy_filter.cc
@@ -210,10 +210,11 @@ void ProxyFilter::onResponse(PendingRequest& request, Common::Redis::RespValuePt
 }
 
 Network::FilterStatus ProxyFilter::onData(Buffer::Instance& data, bool) {
-  try {
+  TRY_NEEDS_AUDIT {
     decoder_->decode(data);
     return Network::FilterStatus::Continue;
-  } catch (Common::Redis::ProtocolError&) {
+  }
+  END_TRY catch (Common::Redis::ProtocolError&) {
     config_->stats_.downstream_cx_protocol_error_.inc();
     Common::Redis::RespValue error;
     error.type(Common::Redis::RespType::Error);
diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.cc b/source/extensions/filters/network/thrift_proxy/conn_manager.cc
index c71d621bd512..77bc7f02da24 100644
--- a/source/extensions/filters/network/thrift_proxy/conn_manager.cc
+++ b/source/extensions/filters/network/thrift_proxy/conn_manager.cc
@@ -71,7 +71,7 @@ void ConnectionManager::dispatch() {
     return;
   }
 
-  try {
+  TRY_NEEDS_AUDIT {
     bool underflow = false;
     while (!underflow) {
       FilterStatus status = decoder_->onData(request_buffer_, underflow);
@@ -82,7 +82,8 @@ void ConnectionManager::dispatch() {
     }
 
     return;
-  } catch (const AppException& ex) {
+  }
+  END_TRY catch (const AppException& ex) {
     ENVOY_LOG(debug, "thrift application exception: {}", ex.what());
     if (rpcs_.empty()) {
       MessageMetadata metadata;
@@ -90,7 +91,8 @@ void ConnectionManager::dispatch() {
     } else {
       sendLocalReply(*(*rpcs_.begin())->metadata_, ex, true);
     }
-  } catch (const EnvoyException& ex) {
+  }
+  catch (const EnvoyException& ex) {
     ENVOY_CONN_LOG(debug, "thrift error: {}", read_callbacks_->connection(), ex.what());
 
     if (rpcs_.empty()) {
@@ -868,11 +870,8 @@ FilterStatus ConnectionManager::ActiveRpc::messageBegin(MessageMetadataSharedPtr
 
   FilterStatus result = FilterStatus::StopIteration;
   absl::optional error;
-  try {
-    result = applyDecoderFilters(DecoderEvent::MessageBegin, metadata);
-  } catch (const std::bad_function_call& e) {
-    error = std::string(e.what());
-  }
+  TRY_NEEDS_AUDIT { result = applyDecoderFilters(DecoderEvent::MessageBegin, metadata); }
+  END_TRY catch (const std::bad_function_call& e) { error = std::string(e.what()); }
 
   const auto& route_ptr = route();
   const std::string& cluster_name = route_ptr ? route_ptr->routeEntry()->clusterName() : "-";
@@ -1077,20 +1076,22 @@ void ConnectionManager::ActiveRpc::startUpstreamResponse(Transport& transport, P
 ThriftFilters::ResponseStatus ConnectionManager::ActiveRpc::upstreamData(Buffer::Instance& buffer) {
   ASSERT(response_decoder_ != nullptr);
 
-  try {
+  TRY_NEEDS_AUDIT {
     if (response_decoder_->onData(buffer)) {
       // Completed upstream response.
       parent_.doDeferredRpcDestroy(*this);
       return ThriftFilters::ResponseStatus::Complete;
     }
     return ThriftFilters::ResponseStatus::MoreData;
-  } catch (const AppException& ex) {
+  }
+  END_TRY catch (const AppException& ex) {
     ENVOY_LOG(error, "thrift response application error: {}", ex.what());
     parent_.stats_.response_decoding_error_.inc();
 
     sendLocalReply(ex, true);
     return ThriftFilters::ResponseStatus::Reset;
-  } catch (const EnvoyException& ex) {
+  }
+  catch (const EnvoyException& ex) {
     ENVOY_CONN_LOG(error, "thrift response error: {}", parent_.read_callbacks_->connection(),
                    ex.what());
     parent_.stats_.response_decoding_error_.inc();
diff --git a/source/extensions/filters/network/thrift_proxy/router/shadow_writer_impl.h b/source/extensions/filters/network/thrift_proxy/router/shadow_writer_impl.h
index bc02d856cab7..47076b3d72c1 100644
--- a/source/extensions/filters/network/thrift_proxy/router/shadow_writer_impl.h
+++ b/source/extensions/filters/network/thrift_proxy/router/shadow_writer_impl.h
@@ -31,11 +31,9 @@ struct NullResponseDecoder : public DecoderCallbacks, public ProtocolConverter {
     upstream_buffer_.move(data);
 
     bool underflow = false;
-    try {
-      underflow = onData();
-    } catch (const AppException&) {
-      return ThriftFilters::ResponseStatus::Reset;
-    } catch (const EnvoyException&) {
+    TRY_NEEDS_AUDIT { underflow = onData(); }
+    END_TRY catch (const AppException&) { return ThriftFilters::ResponseStatus::Reset; }
+    catch (const EnvoyException&) {
       return ThriftFilters::ResponseStatus::Reset;
     }
 
diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.cc b/source/extensions/filters/network/zookeeper_proxy/decoder.cc
index 190d7ae518fc..f30830bb1d9c 100644
--- a/source/extensions/filters/network/zookeeper_proxy/decoder.cc
+++ b/source/extensions/filters/network/zookeeper_proxy/decoder.cc
@@ -551,7 +551,7 @@ void DecoderImpl::decodeAndBufferHelper(Buffer::Instance& data, DecodeType dtype
   bool has_full_packets = false;
 
   while (offset < data_len) {
-    try {
+    TRY_NEEDS_AUDIT {
       // Peek packet length.
       len = helper_.peekInt32(data, offset);
       ensureMinLength(len, dtype == DecodeType::READ ? XID_LENGTH + INT_LENGTH
@@ -561,7 +561,8 @@ void DecoderImpl::decodeAndBufferHelper(Buffer::Instance& data, DecodeType dtype
       if (offset <= data_len) {
         has_full_packets = true;
       }
-    } catch (const EnvoyException& e) {
+    }
+    END_TRY catch (const EnvoyException& e) {
       ENVOY_LOG(debug, "zookeeper_proxy: decoding exception {}", e.what());
       callbacks_.onDecodeError();
       return;
@@ -599,7 +600,7 @@ void DecoderImpl::decodeAndBufferHelper(Buffer::Instance& data, DecodeType dtype
 void DecoderImpl::decode(Buffer::Instance& data, DecodeType dtype, uint64_t full_packets_len) {
   uint64_t offset = 0;
 
-  try {
+  TRY_NEEDS_AUDIT {
     while (offset < full_packets_len) {
       // Reset the helper's cursor, to ensure the current message stays within the
       // allowed max length, even when it's different than the declared length
@@ -622,7 +623,8 @@ void DecoderImpl::decode(Buffer::Instance& data, DecodeType dtype, uint64_t full
         break;
       }
     }
-  } catch (const EnvoyException& e) {
+  }
+  END_TRY catch (const EnvoyException& e) {
     ENVOY_LOG(debug, "zookeeper_proxy: decoding exception {}", e.what());
     callbacks_.onDecodeError();
   }
diff --git a/source/extensions/filters/udp/dns_filter/dns_filter.cc b/source/extensions/filters/udp/dns_filter/dns_filter.cc
index aec85805c8e6..c56a16770ac2 100644
--- a/source/extensions/filters/udp/dns_filter/dns_filter.cc
+++ b/source/extensions/filters/udp/dns_filter/dns_filter.cc
@@ -197,16 +197,18 @@ bool DnsFilterEnvoyConfig::loadServerConfig(
 
   const auto& datasource = config.external_dns_table();
   bool data_source_loaded = false;
-  try {
+  TRY_NEEDS_AUDIT {
     // Data structure is deduced from the file extension. If the data is not read an exception
     // is thrown. If no table can be read, the filter will refer all queries to an external
     // DNS server, if configured, otherwise all queries will be responded to with Name Error.
     MessageUtil::loadFromFile(datasource.filename(), table,
                               ProtobufMessage::getNullValidationVisitor(), api_);
     data_source_loaded = true;
-  } catch (const ProtobufMessage::UnknownProtoFieldException& e) {
+  }
+  END_TRY catch (const ProtobufMessage::UnknownProtoFieldException& e) {
     ENVOY_LOG(warn, "Invalid field in DNS Filter datasource configuration: {}", e.what());
-  } catch (const EnvoyException& e) {
+  }
+  catch (const EnvoyException& e) {
     ENVOY_LOG(warn, "Filesystem DNS Filter config update failure: {}", e.what());
   }
   return data_source_loaded;
diff --git a/source/extensions/grpc_credentials/aws_iam/config.cc b/source/extensions/grpc_credentials/aws_iam/config.cc
index 0dbba166d326..42f7f28a9c41 100644
--- a/source/extensions/grpc_credentials/aws_iam/config.cc
+++ b/source/extensions/grpc_credentials/aws_iam/config.cc
@@ -100,9 +100,8 @@ AwsIamHeaderAuthenticator::GetMetadata(grpc::string_ref service_url, grpc::strin
   auto message = buildMessageToSign(absl::string_view(service_url.data(), service_url.length()),
                                     absl::string_view(method_name.data(), method_name.length()));
 
-  try {
-    signer_->sign(message, false);
-  } catch (const EnvoyException& e) {
+  TRY_NEEDS_AUDIT { signer_->sign(message, false); }
+  END_TRY catch (const EnvoyException& e) {
     return grpc::Status(grpc::StatusCode::INTERNAL, e.what());
   }
 
diff --git a/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc b/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc
index f727f3ead197..21671c8cfe26 100644
--- a/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc
+++ b/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc
@@ -67,11 +67,12 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config&, Tracing::TraceContext
     // throw exception that not be wrapped by TracerException. See
     // https://github.com/SkyAPM/cpp2sky/issues/117. So, we need to catch all exceptions here to
     // avoid Envoy crash in the runtime.
-    try {
+    TRY_NEEDS_AUDIT {
       SpanContextPtr span_context =
           createSpanContext(toStdStringView(header_value_string)); // NOLINT(std::string_view)
       tracing_context = tracing_context_factory_->create(span_context);
-    } catch (std::exception& e) {
+    }
+    END_TRY catch (std::exception& e) {
       ENVOY_LOG(
           warn,
           "New SkyWalking Span/Segment with previous span context cannot be created for error: {}",
diff --git a/source/extensions/tracers/xray/config.cc b/source/extensions/tracers/xray/config.cc
index f297c5c7d841..9421eb0678e0 100644
--- a/source/extensions/tracers/xray/config.cc
+++ b/source/extensions/tracers/xray/config.cc
@@ -22,10 +22,11 @@ Tracing::DriverSharedPtr
 XRayTracerFactory::createTracerDriverTyped(const envoy::config::trace::v3::XRayConfig& proto_config,
                                            Server::Configuration::TracerFactoryContext& context) {
   std::string sampling_rules_json;
-  try {
+  TRY_NEEDS_AUDIT {
     sampling_rules_json = Config::DataSource::read(proto_config.sampling_rule_manifest(), true,
                                                    context.serverFactoryContext().api());
-  } catch (EnvoyException& e) {
+  }
+  END_TRY catch (EnvoyException& e) {
     ENVOY_LOG(error, "Failed to read sampling rules manifest because of {}.", e.what());
   }
 
diff --git a/source/extensions/tracers/xray/localized_sampling.cc b/source/extensions/tracers/xray/localized_sampling.cc
index 3c0248733103..551ebfe66b9d 100644
--- a/source/extensions/tracers/xray/localized_sampling.cc
+++ b/source/extensions/tracers/xray/localized_sampling.cc
@@ -94,9 +94,8 @@ LocalizedSamplingManifest::LocalizedSamplingManifest(const std::string& rule_jso
   }
 
   ProtobufWkt::Struct document;
-  try {
-    MessageUtil::loadFromJson(rule_json, document);
-  } catch (EnvoyException& e) {
+  TRY_NEEDS_AUDIT { MessageUtil::loadFromJson(rule_json, document); }
+  END_TRY catch (EnvoyException& e) {
     fail("invalid JSON format");
     return;
   }
diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc
index 0a76f617f400..1eaf50a36de8 100644
--- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc
+++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc
@@ -118,7 +118,7 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config,
   SpanPtr new_zipkin_span;
   SpanContextExtractor extractor(trace_context);
   bool sampled{extractor.extractSampled(tracing_decision)};
-  try {
+  TRY_NEEDS_AUDIT {
     auto ret_span_context = extractor.extractSpanContext(sampled);
     if (!ret_span_context.second) {
       // Create a root Zipkin span. No context was found in the headers.
@@ -129,9 +129,8 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config,
       new_zipkin_span = tracer.startSpan(config, std::string(trace_context.host()),
                                          stream_info.startTime(), ret_span_context.first);
     }
-  } catch (const ExtractorException& e) {
-    return std::make_unique();
   }
+  END_TRY catch (const ExtractorException& e) { return std::make_unique(); }
 
   // Return the active Zipkin span.
   return std::make_unique(*new_zipkin_span, tracer);
diff --git a/source/server/admin/html/active_stats.js b/source/server/admin/html/active_stats.js
index ca6bf6723981..0e05bebfef1e 100644
--- a/source/server/admin/html/active_stats.js
+++ b/source/server/admin/html/active_stats.js
@@ -97,11 +97,11 @@ async function loadStats() {
   const url = prefix + '/stats?format=json&usedonly&histogram_buckets=detailed&' +
         params.map(makeQueryParam).join('&');
 
-  try {
+  TRY_NEEDS_AUDIT {
     const response = await fetch(url);
     const data = await response.json();
     renderStats(data);
-  } catch (e) {
+  } END_TRY catch (e) {
     statusDiv.textContent = 'Error fetching ' + url + ': ' + e;
   }
 
diff --git a/source/server/server.cc b/source/server/server.cc
index 91c72e529197..b8c2ca1ade67 100644
--- a/source/server/server.cc
+++ b/source/server/server.cc
@@ -780,10 +780,10 @@ void InstanceImpl::onRuntimeReady() {
   // Initializing can throw exceptions, so catch these.
   TRY_ASSERT_MAIN_THREAD { clusterManager().initializeSecondaryClusters(bootstrap_); }
   END_TRY
-  catch (const EnvoyException& e) {
+  CATCH(const EnvoyException& e, {
     ENVOY_LOG(warn, "Skipping initialization of secondary cluster: {}", e.what());
     shutdown();
-  }
+  });
 
   if (bootstrap_.has_hds_config()) {
     const auto& hds_config = bootstrap_.hds_config();
@@ -799,10 +799,10 @@ void InstanceImpl::onRuntimeReady() {
           stats_store_, *ssl_context_manager_, info_factory_);
     }
     END_TRY
-    catch (const EnvoyException& e) {
+    CATCH(const EnvoyException& e, {
       ENVOY_LOG(warn, "Skipping initialization of HDS cluster: {}", e.what());
       shutdown();
-    }
+    });
   }
 
   // If there is no global limit to the number of active connections, warn on startup.
diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py
index 18297edba85e..9931ed4eed8b 100755
--- a/tools/code_format/check_format.py
+++ b/tools/code_format/check_format.py
@@ -322,9 +322,7 @@ def allow_listed_for_unpack_to(self, file_path):
         ]
 
     def allow_listed_for_raw_try(self, file_path):
-        # TODO(chaoqin-li1123): Exclude some important extensions from ALLOWLIST.
-        return file_path in self.config.paths["raw_try"]["include"] or file_path.startswith(
-            "./source/extensions")
+        return file_path in self.config.paths["raw_try"]["include"]
 
     def deny_listed_for_exceptions(self, file_path):
         # Returns true when it is a non test header file or the file_path is in DENYLIST or
@@ -624,7 +622,7 @@ def check_source_line(self, line, file_path, report_error):
         if " try {" in line and file_path.startswith(
                 "./source") and not self.allow_listed_for_raw_try(file_path):
             report_error(
-                "Don't use raw try, use TRY_ASSERT_MAIN_THREAD if on the main thread otherwise don't use exceptions."
+                "Don't use raw try, use TRY_ASSERT_MAIN_THREAD and CATCH if on the main thread otherwise don't use exceptions."
             )
         if "__attribute__((packed))" in line and file_path != "./envoy/common/platform.h":
             # __attribute__((packed)) is not supported by MSVC, we have a PACKED_STRUCT macro that
diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml
index 4e9adbb9aef1..525423746a50 100644
--- a/tools/code_format/config.yaml
+++ b/tools/code_format/config.yaml
@@ -246,9 +246,7 @@ paths:
   # Files that are allowed to use try without main thread assertion.
   raw_try:
     include:
-    - source/common/common/regex.cc
     - source/common/common/thread.h
-    - source/common/network/utility.cc
 
   # Files matching these exact names can reference real-world time. These include the class
   # definitions for real-world time, the construction of them in main(), and perf annotation.

From 70e8eedf63f021fb1acbeeb2f47ed03d6b26fa90 Mon Sep 17 00:00:00 2001
From: Tianyu <72890320+tyxia@users.noreply.github.com>
Date: Thu, 15 Jun 2023 19:51:05 -0400
Subject: [PATCH 552/740] ext_proc: add unit test covergae (#27995)

Signed-off-by: tyxia 
---
 .../filters/http/ext_proc/config_test.cc      | 33 +++++++++++++++++++
 test/per_file_coverage.sh                     |  1 -
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/test/extensions/filters/http/ext_proc/config_test.cc b/test/extensions/filters/http/ext_proc/config_test.cc
index 525c278f3b23..fd1f9eb2a628 100644
--- a/test/extensions/filters/http/ext_proc/config_test.cc
+++ b/test/extensions/filters/http/ext_proc/config_test.cc
@@ -46,6 +46,39 @@ TEST(HttpExtProcConfigTest, CorrectConfig) {
   cb(filter_callback);
 }
 
+TEST(HttpExtProcConfigTest, CorrectConfigServerContext) {
+  std::string yaml = R"EOF(
+  grpc_service:
+    google_grpc:
+      target_uri: ext_proc_server
+      stat_prefix: google
+  failure_mode_allow: true
+  request_attributes: 'Foo, Bar, Baz'
+  response_attributes: More
+  processing_mode:
+    request_header_mode: send
+    response_header_mode: skip
+    request_body_mode: streamed
+    response_body_mode: buffered
+    request_trailer_mode: skip
+    response_trailer_mode: send
+  filter_metadata:
+    hello: "world"
+  )EOF";
+
+  ExternalProcessingFilterConfig factory;
+  ProtobufTypes::MessagePtr proto_config = factory.createEmptyConfigProto();
+  TestUtility::loadFromYaml(yaml, *proto_config);
+
+  testing::NiceMock context;
+  EXPECT_CALL(context, messageValidationVisitor());
+  Http::FilterFactoryCb cb =
+      factory.createFilterServerFactoryFromProto(*proto_config, "stats", context);
+  Http::MockFilterChainFactoryCallbacks filter_callback;
+  EXPECT_CALL(filter_callback, addStreamFilter(_));
+  cb(filter_callback);
+}
+
 } // namespace
 } // namespace ExternalProcessing
 } // namespace HttpFilters
diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh
index a54367841c8f..0fb812f5e751 100755
--- a/test/per_file_coverage.sh
+++ b/test/per_file_coverage.sh
@@ -35,7 +35,6 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/filters/common/fault:94.5"
 "source/extensions/filters/common/rbac:90.5"
 "source/extensions/filters/http/cache:93.4"
-"source/extensions/filters/http/ext_proc:96.0" # TODO(tyxia) Improve coverage for ext_proc/config.cc
 "source/extensions/filters/http/grpc_json_transcoder:95.6"
 "source/extensions/filters/http/ip_tagging:88.0"
 "source/extensions/filters/http/kill_request:91.7" # Death tests don't report LCOV

From 8d0275fef1ef8f94e1d6d59e69bd887213bf699c Mon Sep 17 00:00:00 2001
From: alyssawilk 
Date: Thu, 15 Jun 2023 19:57:34 -0400
Subject: [PATCH 553/740] request uid: removing exceptions (#27997)

This will cause implementers of the request ID extension to have to return absl::StatusOK on creation. They do not need to remove exceptions from their implementation.

Risk Level: low
Testing: n/a
Docs Changes: n/a
Release Notes: n/a
part of envoyproxy/envoy-mobile#176

Signed-off-by: Alyssa Wilk 
---
 source/common/http/request_id_extension_impl.cc              | 4 ++--
 source/common/http/request_id_extension_impl.h               | 5 +++--
 .../filters/network/http_connection_manager/config.cc        | 4 +++-
 tools/code_format/config.yaml                                | 1 -
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/source/common/http/request_id_extension_impl.cc b/source/common/http/request_id_extension_impl.cc
index bbe173d4d6b6..ed90f218c089 100644
--- a/source/common/http/request_id_extension_impl.cc
+++ b/source/common/http/request_id_extension_impl.cc
@@ -5,7 +5,7 @@
 namespace Envoy {
 namespace Http {
 
-RequestIDExtensionSharedPtr RequestIDExtensionFactory::fromProto(
+absl::StatusOr RequestIDExtensionFactory::fromProto(
     const envoy::extensions::filters::network::http_connection_manager::v3::RequestIDExtension&
         config,
     Server::Configuration::CommonFactoryContext& context) {
@@ -14,7 +14,7 @@ RequestIDExtensionSharedPtr RequestIDExtensionFactory::fromProto(
       Registry::FactoryRegistry::getFactoryByType(
           type);
   if (factory == nullptr) {
-    throw EnvoyException(
+    return absl::InvalidArgumentError(
         fmt::format("Didn't find a registered implementation for type: '{}'", type));
   }
 
diff --git a/source/common/http/request_id_extension_impl.h b/source/common/http/request_id_extension_impl.h
index 1da07a88ed9f..01cf8984ca00 100644
--- a/source/common/http/request_id_extension_impl.h
+++ b/source/common/http/request_id_extension_impl.h
@@ -12,9 +12,10 @@ namespace Http {
 class RequestIDExtensionFactory {
 public:
   /**
-   * Read a RequestIDExtension definition from proto and create it.
+   * Read a RequestIDExtension definition from proto and create it or return an
+   * error status.
    */
-  static RequestIDExtensionSharedPtr fromProto(
+  static absl::StatusOr fromProto(
       const envoy::extensions::filters::network::http_connection_manager::v3::RequestIDExtension&
           config,
       Server::Configuration::CommonFactoryContext& context);
diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc
index 46477c6d60d7..cb871957e80a 100644
--- a/source/extensions/filters/network/http_connection_manager/config.cc
+++ b/source/extensions/filters/network/http_connection_manager/config.cc
@@ -413,7 +413,9 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig(
     final_rid_config.mutable_typed_config()->PackFrom(
         envoy::extensions::request_id::uuid::v3::UuidRequestIdConfig());
   }
-  request_id_extension_ = Http::RequestIDExtensionFactory::fromProto(final_rid_config, context_);
+  auto extension_or_error = Http::RequestIDExtensionFactory::fromProto(final_rid_config, context_);
+  THROW_IF_STATUS_NOT_OK(extension_or_error, throw);
+  request_id_extension_ = extension_or_error.value();
 
   // Check if IP detection extensions were configured, otherwise fall back to XFF.
   auto ip_detection_extensions = config.original_ip_detection_extensions();
diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml
index 525423746a50..7a3747fc1fcb 100644
--- a/tools/code_format/config.yaml
+++ b/tools/code_format/config.yaml
@@ -122,7 +122,6 @@ paths:
     - source/common/stats/tag_producer_impl.cc
     - source/common/http/http2/codec_impl.cc
     - source/common/http/filter_chain_helper.cc
-    - source/common/http/request_id_extension_impl.cc
     - source/common/http/utility.cc
     - source/common/http/hash_policy.cc
     - source/common/http/utility.h

From 8b9136cb57646b911899c3fb96a6fed744bf1a4f Mon Sep 17 00:00:00 2001
From: Tony Han 
Date: Fri, 16 Jun 2023 09:03:32 +0800
Subject: [PATCH 554/740] lds: pause SRDS when LDS is updated (#27252)

Now RDS is paused when LDS is updated, so also pause SRDS.

Risk Level: Low
Testing: unit tests
Docs Changes: n/a
Release Notes: added
Platform Specific Features: n/a
Signed-off-by: Bing Han 
---
 changelogs/current.yaml                       |   3 +
 .../listener_manager/lds_api.cc               |   2 +
 test/integration/ads_integration_test.cc      | 105 ++++++++++++++++++
 3 files changed, 110 insertions(+)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index c8179abf17f9..f8ce0af6e040 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -375,6 +375,9 @@ new_features:
     added bytes snapshotting for upstream and downstream logging that will be reset after every periodic log. Downstream
     periodic loggers should read BytesMeter::bytesAtLastDownstreamPeriodicLog(), and upstream periodic loggers should read
     BytesMeter::bytesAtLastUpstreamPeriodicLog().
+- area: lds
+  change: |
+    pause SRDS when LDS is updated.
 
 deprecated:
 - area: access_log
diff --git a/source/extensions/listener_managers/listener_manager/lds_api.cc b/source/extensions/listener_managers/listener_manager/lds_api.cc
index 8a1235317a6e..321ec3d3cc3d 100644
--- a/source/extensions/listener_managers/listener_manager/lds_api.cc
+++ b/source/extensions/listener_managers/listener_manager/lds_api.cc
@@ -4,6 +4,7 @@
 #include "envoy/config/core/v3/config_source.pb.h"
 #include "envoy/config/listener/v3/listener.pb.h"
 #include "envoy/config/route/v3/route.pb.h"
+#include "envoy/config/route/v3/scoped_route.pb.h"
 #include "envoy/service/discovery/v3/discovery.pb.h"
 #include "envoy/stats/scope.h"
 
@@ -46,6 +47,7 @@ void LdsApiImpl::onConfigUpdate(const std::vector& a
   if (cm_.adsMux()) {
     const std::vector paused_xds_types{
         Config::getTypeUrl(),
+        Config::getTypeUrl(),
         Config::getTypeUrl()};
     maybe_resume_rds_sds = cm_.adsMux()->pause(paused_xds_types);
   }
diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc
index 104fd246939c..f9d4e54f2466 100644
--- a/test/integration/ads_integration_test.cc
+++ b/test/integration/ads_integration_test.cc
@@ -2379,4 +2379,109 @@ TEST_P(XdsTpAdsIntegrationTest, EdsAlternatingLedsUsage) {
   makeSingleRequest();
 }
 
+// Make sure two listeners send only a single SRDS request.
+TEST_P(AdsIntegrationTest, SrdsPausedDuringLds) {
+  initialize();
+  const auto lds_type_url = Config::TypeUrl::get().Listener;
+  const auto cds_type_url = Config::TypeUrl::get().Cluster;
+  const auto eds_type_url = Config::TypeUrl::get().ClusterLoadAssignment;
+  const auto srds_type_url = Config::TypeUrl::get().ScopedRouteConfiguration;
+  const auto rds_type_url = Config::TypeUrl::get().RouteConfiguration;
+
+  EXPECT_TRUE(compareDiscoveryRequest(cds_type_url, "", {}, {}, {}, true));
+  sendDiscoveryResponse(Config::TypeUrl::get().Cluster,
+                                                             {buildCluster("cluster_0")},
+                                                             {buildCluster("cluster_0")}, {}, "1");
+  EXPECT_TRUE(compareDiscoveryRequest(eds_type_url, "", {"cluster_0"}, {"cluster_0"}, {}));
+  sendDiscoveryResponse(
+      eds_type_url, {buildClusterLoadAssignment("cluster_0")},
+      {buildClusterLoadAssignment("cluster_0")}, {}, "1");
+
+  EXPECT_TRUE(compareDiscoveryRequest(cds_type_url, "1", {}, {}, {}));
+
+  std::vector listeners;
+  const std::string hcm = R"EOF(
+        filters:
+        - name: http
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+            codec_type: HTTP2
+            stat_prefix: ads_test
+            scoped_routes:
+              name: scoped_routes
+              scope_key_builder:
+                fragments:
+                - header_value_extractor:
+                    name: X-Route-Selector
+                    element_separator: ","
+                    element:
+                      separator: =
+                      key: vip
+              rds_config_source:
+                resource_api_version: V3
+                ads: {}
+              scoped_rds:
+                scoped_rds_config_source:
+                  resource_api_version: V3
+                  ads: {}
+            http_filters:
+            - name: envoy.filters.http.router
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+    )EOF";
+  for (int i = 0; i < 2; ++i) {
+    auto listener = ConfigHelper::buildBaseListener(
+        "listener", Network::Test::getLoopbackAddressString(ipVersion()), hcm);
+    listener.set_name(absl::StrCat("listener_", i));
+    listeners.push_back(std::move(listener));
+  }
+
+  EXPECT_TRUE(compareDiscoveryRequest(lds_type_url, "", {}, {}, {}));
+  sendDiscoveryResponse(lds_type_url, listeners, listeners,
+                                                               {}, "1");
+
+  test_server_->waitForCounterEq("listener_manager.listener_added", 2);
+
+  EXPECT_TRUE(compareDiscoveryRequest(eds_type_url, "1", {}, {}, {}));
+
+  // Expect a single request for SRDS resources.
+  EXPECT_TRUE(compareDiscoveryRequest(srds_type_url, "", {}, {}, {}));
+
+  EXPECT_TRUE(compareDiscoveryRequest(lds_type_url, "1", {}, {}, {}));
+
+  std::vector scoped_route_configs;
+  for (int i = 0; i < 2; ++i) {
+    envoy::config::route::v3::ScopedRouteConfiguration scoped_route;
+    TestUtility::loadFromYaml(fmt::format(
+                                  R"EOF(
+        name: scoped_route_{}
+        route_configuration_name: route_{}
+        key:
+          fragments:
+          - string_key: ip_{}
+      )EOF",
+                                  i, i, i),
+                              scoped_route);
+    scoped_route_configs.push_back(scoped_route);
+  }
+
+  sendDiscoveryResponse(
+      srds_type_url, scoped_route_configs, scoped_route_configs, {}, "1");
+
+  EXPECT_TRUE(compareDiscoveryRequest(rds_type_url, "", {"route_0", "route_1"},
+                                      {"route_0", "route_1"}, {}));
+
+  EXPECT_TRUE(compareDiscoveryRequest(srds_type_url, "1", {}, {}, {}));
+
+  sendDiscoveryResponse(
+      rds_type_url,
+      {buildRouteConfig("route_0", "cluster_0"), buildRouteConfig("route_1", "cluster_0")},
+      {buildRouteConfig("route_0", "cluster_0"), buildRouteConfig("route_1", "cluster_0")}, {},
+      "1");
+
+  EXPECT_TRUE(compareDiscoveryRequest(rds_type_url, "1", {"route_0", "route_1"}, {}, {}));
+
+  test_server_->waitForCounterEq("listener_manager.listener_create_success", 2);
+}
+
 } // namespace Envoy

From ca78605647433ace54d6a9c723c9e14c1718ddf5 Mon Sep 17 00:00:00 2001
From: doujiang24 
Date: Fri, 16 Jun 2023 09:38:05 +0800
Subject: [PATCH 555/740] golang filter: remove config on the Go side while
 deconstructing filter config. (#27892)

* golang filter: remove config on the Go side while deconstructing filter config.

Signed-off-by: doujiang24 

* fix comment: update merged_config_id_ where parent_id changed.

Signed-off-by: doujiang24 

---------

Signed-off-by: doujiang24 
---
 contrib/golang/common/dso/dso.cc              |  8 ++++
 contrib/golang/common/dso/dso.h               |  3 ++
 contrib/golang/common/dso/test/dso_test.cc    | 22 ++++++++++-
 contrib/golang/common/dso/test/mocks.h        |  1 +
 .../common/dso/test/test_data/simple.go       | 15 ++++++-
 .../filters/http/source/golang_filter.cc      | 39 ++++++++++++++++---
 .../filters/http/source/golang_filter.h       | 12 +++---
 7 files changed, 87 insertions(+), 13 deletions(-)

diff --git a/contrib/golang/common/dso/dso.cc b/contrib/golang/common/dso/dso.cc
index 71fb1c89e29c..18b8018af197 100644
--- a/contrib/golang/common/dso/dso.cc
+++ b/contrib/golang/common/dso/dso.cc
@@ -50,6 +50,9 @@ HttpFilterDsoImpl::HttpFilterDsoImpl(const std::string dso_name) : HttpFilterDso
   loaded_ &= dlsymInternal(
       envoy_go_filter_merge_http_plugin_config_, handler_, dso_name,
       "envoyGoFilterMergeHttpPluginConfig");
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_destroy_http_plugin_config_, handler_, dso_name,
+      "envoyGoFilterDestroyHttpPluginConfig");
   loaded_ &= dlsymInternal(
       envoy_go_filter_on_http_header_, handler_, dso_name, "envoyGoFilterOnHttpHeader");
   loaded_ &= dlsymInternal(
@@ -72,6 +75,11 @@ GoUint64 HttpFilterDsoImpl::envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUi
   return envoy_go_filter_merge_http_plugin_config_(p0, p1, p2, p3);
 }
 
+void HttpFilterDsoImpl::envoyGoFilterDestroyHttpPluginConfig(GoUint64 p0) {
+  ASSERT(envoy_go_filter_destroy_http_plugin_config_ != nullptr);
+  return envoy_go_filter_destroy_http_plugin_config_(p0);
+}
+
 GoUint64 HttpFilterDsoImpl::envoyGoFilterOnHttpHeader(httpRequest* p0, GoUint64 p1, GoUint64 p2,
                                                       GoUint64 p3) {
   ASSERT(envoy_go_filter_on_http_header_ != nullptr);
diff --git a/contrib/golang/common/dso/dso.h b/contrib/golang/common/dso/dso.h
index 9e687efe8dfe..540d4b97aa3b 100644
--- a/contrib/golang/common/dso/dso.h
+++ b/contrib/golang/common/dso/dso.h
@@ -34,6 +34,7 @@ class HttpFilterDso : public Dso {
                                                     GoUint64 p3) PURE;
   virtual GoUint64 envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2,
                                                       GoUint64 p3) PURE;
+  virtual void envoyGoFilterDestroyHttpPluginConfig(GoUint64 p0) PURE;
   virtual GoUint64 envoyGoFilterOnHttpHeader(httpRequest* p0, GoUint64 p1, GoUint64 p2,
                                              GoUint64 p3) PURE;
   virtual GoUint64 envoyGoFilterOnHttpData(httpRequest* p0, GoUint64 p1, GoUint64 p2,
@@ -51,6 +52,7 @@ class HttpFilterDsoImpl : public HttpFilterDso {
                                             GoUint64 p3) override;
   GoUint64 envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2,
                                               GoUint64 p3) override;
+  void envoyGoFilterDestroyHttpPluginConfig(GoUint64 p0) override;
   GoUint64 envoyGoFilterOnHttpHeader(httpRequest* p0, GoUint64 p1, GoUint64 p2,
                                      GoUint64 p3) override;
   GoUint64 envoyGoFilterOnHttpData(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) override;
@@ -62,6 +64,7 @@ class HttpFilterDsoImpl : public HttpFilterDso {
                                                       GoUint64 p3) = {nullptr};
   GoUint64 (*envoy_go_filter_merge_http_plugin_config_)(GoUint64 p0, GoUint64 p1, GoUint64 p2,
                                                         GoUint64 p3) = {nullptr};
+  void (*envoy_go_filter_destroy_http_plugin_config_)(GoUint64 p0) = {nullptr};
   GoUint64 (*envoy_go_filter_on_http_header_)(httpRequest* p0, GoUint64 p1, GoUint64 p2,
                                               GoUint64 p3) = {nullptr};
   GoUint64 (*envoy_go_filter_on_http_data_)(httpRequest* p0, GoUint64 p1, GoUint64 p2,
diff --git a/contrib/golang/common/dso/test/dso_test.cc b/contrib/golang/common/dso/test/dso_test.cc
index a3d0fd807497..95912d855b60 100644
--- a/contrib/golang/common/dso/test/dso_test.cc
+++ b/contrib/golang/common/dso/test/dso_test.cc
@@ -21,7 +21,7 @@ std::string genSoPath(std::string name) {
 TEST(DsoInstanceTest, SimpleAPI) {
   auto path = genSoPath("simple.so");
   HttpFilterDsoPtr dso(new HttpFilterDsoImpl(path));
-  EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 0), 100);
+  EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 100), 100);
 }
 
 TEST(DsoManagerTest, Pub) {
@@ -40,7 +40,7 @@ TEST(DsoManagerTest, Pub) {
   // get after load http filter dso
   dso = DsoManager::getDsoByPluginName(plugin_name);
   EXPECT_NE(dso, nullptr);
-  EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 0), 100);
+  EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 200), 200);
 
   // second time load http filter dso
   dso = DsoManager::load(id, path, plugin_name);
@@ -60,6 +60,24 @@ TEST(DsoInstanceTest, BadSo) {
   EXPECT_EQ(dso->loaded(), false);
 }
 
+// remove plugin config
+TEST(DsoInstanceTest, RemovePluginConfig) {
+  auto path = genSoPath("simple.so");
+  HttpFilterDsoPtr dso(new HttpFilterDsoImpl(path));
+  EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 300), 300);
+  // new again, return 0, since it's already existing
+  EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 300), 0);
+
+  // remove it
+  dso->envoyGoFilterDestroyHttpPluginConfig(300);
+  // new again, after removed.
+  EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 300), 300);
+
+  // remove twice should be ok
+  dso->envoyGoFilterDestroyHttpPluginConfig(300);
+  dso->envoyGoFilterDestroyHttpPluginConfig(300);
+}
+
 } // namespace
 } // namespace Dso
 } // namespace Envoy
diff --git a/contrib/golang/common/dso/test/mocks.h b/contrib/golang/common/dso/test/mocks.h
index 21287d6e9c82..5cc346ea7385 100644
--- a/contrib/golang/common/dso/test/mocks.h
+++ b/contrib/golang/common/dso/test/mocks.h
@@ -15,6 +15,7 @@ class MockHttpFilterDsoImpl : public HttpFilterDso {
               (GoUint64 p0, GoUint64 p1, GoUint64 p2, GoUint64 p3));
   MOCK_METHOD(GoUint64, envoyGoFilterMergeHttpPluginConfig,
               (GoUint64 p0, GoUint64 p1, GoUint64 p2, GoUint64 p3));
+  MOCK_METHOD(void, envoyGoFilterDestroyHttpPluginConfig, (GoUint64 p0));
   MOCK_METHOD(GoUint64, envoyGoFilterOnHttpHeader,
               (httpRequest * p0, GoUint64 p1, GoUint64 p2, GoUint64 p3));
   MOCK_METHOD(GoUint64, envoyGoFilterOnHttpData,
diff --git a/contrib/golang/common/dso/test/test_data/simple.go b/contrib/golang/common/dso/test/test_data/simple.go
index 86c6ac0c1be8..adcff7b0a9bc 100644
--- a/contrib/golang/common/dso/test/test_data/simple.go
+++ b/contrib/golang/common/dso/test/test_data/simple.go
@@ -7,13 +7,26 @@ typedef struct {
 */
 import "C"
 
+import (
+	"sync"
+)
+
+var configCache = &sync.Map{}
+
 //export envoyGoFilterNewHttpPluginConfig
 func envoyGoFilterNewHttpPluginConfig(namePtr, nameLen, configPtr, configLen uint64) uint64 {
-	return 100
+	// already existing return 0, just for testing the destroy api.
+	if _, ok := configCache.Load(configLen); ok {
+		return 0
+	}
+	// mark this configLen already existing
+	configCache.Store(configLen, configLen)
+	return configLen
 }
 
 //export envoyGoFilterDestroyHttpPluginConfig
 func envoyGoFilterDestroyHttpPluginConfig(id uint64) {
+	configCache.Delete(id)
 }
 
 //export envoyGoFilterMergeHttpPluginConfig
diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc
index 4b9518a6cfa2..a3fda0e9d4fe 100644
--- a/contrib/golang/filters/http/source/golang_filter.cc
+++ b/contrib/golang/filters/http/source/golang_filter.cc
@@ -1198,6 +1198,12 @@ FilterConfig::FilterConfig(
   ENVOY_LOG(debug, "golang filter new plugin config, id: {}", config_id_);
 };
 
+FilterConfig::~FilterConfig() {
+  if (config_id_ > 0) {
+    dso_lib_->envoyGoFilterDestroyHttpPluginConfig(config_id_);
+  }
+}
+
 uint64_t FilterConfig::getConfigId() { return config_id_; }
 
 FilterConfigPerRoute::FilterConfigPerRoute(
@@ -1251,10 +1257,17 @@ RoutePluginConfig::RoutePluginConfig(
             config_id_);
 };
 
-uint64_t RoutePluginConfig::getConfigId() {
+RoutePluginConfig::~RoutePluginConfig() {
+  absl::WriterMutexLock lock(&mutex_);
   if (config_id_ > 0) {
-    return config_id_;
+    dso_lib_->envoyGoFilterDestroyHttpPluginConfig(config_id_);
   }
+  if (merged_config_id_ > 0) {
+    dso_lib_->envoyGoFilterDestroyHttpPluginConfig(merged_config_id_);
+  }
+}
+
+uint64_t RoutePluginConfig::getConfigId() {
   if (dso_lib_ == nullptr) {
     dso_lib_ = Dso::DsoManager::getDsoByPluginName(plugin_name_);
     ASSERT(dso_lib_ != nullptr, "load at the request time, so it should not be null");
@@ -1270,12 +1283,26 @@ uint64_t RoutePluginConfig::getConfigId() {
 };
 
 uint64_t RoutePluginConfig::getMergedConfigId(uint64_t parent_id) {
+  {
+    // this is the fast path for most cases.
+    absl::ReaderMutexLock lock(&mutex_);
+    if (merged_config_id_ > 0 && cached_parent_id_ == parent_id) {
+      return merged_config_id_;
+    }
+  }
+  absl::WriterMutexLock lock(&mutex_);
   if (merged_config_id_ > 0) {
-    return merged_config_id_;
+    if (cached_parent_id_ == parent_id) {
+      return merged_config_id_;
+    }
+    // upper level config changed, merged_config_id_ is outdated.
+    dso_lib_->envoyGoFilterDestroyHttpPluginConfig(merged_config_id_);
   }
 
-  config_id_ = getConfigId();
-  RELEASE_ASSERT(config_id_, "TODO: terminate request or passthrough");
+  if (config_id_ == 0) {
+    config_id_ = getConfigId();
+    RELEASE_ASSERT(config_id_, "TODO: terminate request or passthrough");
+  }
 
   auto name_ptr = reinterpret_cast(plugin_name_.data());
   merged_config_id_ = dso_lib_->envoyGoFilterMergeHttpPluginConfig(name_ptr, plugin_name_.length(),
@@ -1283,6 +1310,8 @@ uint64_t RoutePluginConfig::getMergedConfigId(uint64_t parent_id) {
   ASSERT(merged_config_id_, "config id is always grows");
   ENVOY_LOG(debug, "golang filter merge '{}' plugin config, from {} + {} to {}", plugin_name_,
             parent_id, config_id_, merged_config_id_);
+
+  cached_parent_id_ = parent_id;
   return merged_config_id_;
 };
 
diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h
index 762c7f5043f7..f959a60f08c5 100644
--- a/contrib/golang/filters/http/source/golang_filter.h
+++ b/contrib/golang/filters/http/source/golang_filter.h
@@ -30,8 +30,7 @@ class FilterConfig : Logger::Loggable {
   FilterConfig(const envoy::extensions::filters::http::golang::v3alpha::Config& proto_config,
                Dso::HttpFilterDsoPtr dso_lib, const std::string& stats_prefix,
                Server::Configuration::FactoryContext& context);
-  // TODO: delete config in Go
-  virtual ~FilterConfig() = default;
+  ~FilterConfig();
 
   const std::string& soId() const { return so_id_; }
   const std::string& soPath() const { return so_path_; }
@@ -57,8 +56,7 @@ class RoutePluginConfig : Logger::Loggable {
 public:
   RoutePluginConfig(const std::string plugin_name,
                     const envoy::extensions::filters::http::golang::v3alpha::RouterPlugin& config);
-  // TODO: delete plugin config in Go
-  ~RoutePluginConfig() = default;
+  ~RoutePluginConfig();
   uint64_t getConfigId();
   uint64_t getMergedConfigId(uint64_t parent_id);
 
@@ -68,7 +66,11 @@ class RoutePluginConfig : Logger::Loggable {
 
   Dso::HttpFilterDsoPtr dso_lib_;
   uint64_t config_id_{0};
-  uint64_t merged_config_id_{0};
+  // since these two fields are updated in worker threads, we need to protect them with a mutex.
+  uint64_t merged_config_id_ ABSL_GUARDED_BY(mutex_){0};
+  uint64_t cached_parent_id_ ABSL_GUARDED_BY(mutex_){0};
+
+  absl::Mutex mutex_;
 };
 
 using RoutePluginConfigPtr = std::shared_ptr;

From 2ef8e732057d4f1e2eaa40b149dc5b763253226d Mon Sep 17 00:00:00 2001
From: StarryNight 
Date: Fri, 16 Jun 2023 17:25:54 +0800
Subject: [PATCH 556/740] add StarryVae to go extension code owner (#28005)

Signed-off-by: wangkai19 
---
 CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index 3a2176501fb6..93f7cf37e1a1 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -347,7 +347,7 @@ extensions/filters/http/oauth2 @derekargueta @mattklein123
 /contrib/client_ssl_auth/ @UNOWNED @UNOWNED
 /contrib/common/sqlutils/ @cpakulski @cpakulski
 /contrib/dynamo/ @UNOWNED @UNOWNED
-/contrib/golang/ @doujiang24 @wangfakang
+/contrib/golang/ @doujiang24 @wangfakang @StarryVae
 /contrib/squash/ @yuval-k @alyssawilk
 /contrib/kafka/ @mattklein123 @adamkotwasinski
 /contrib/rocketmq_proxy/ @aaron-ai @lizhanhui @lizan

From abf3ffdb86f55ff0e52416066ab88450bbd4a2d5 Mon Sep 17 00:00:00 2001
From: yanavlasov 
Date: Fri, 16 Jun 2023 08:14:03 -0400
Subject: [PATCH 557/740] Fix tsan error in the
 ext_proc_grpc_fuzz_test_persistent test (#28000)

Signed-off-by: Yan Avlasov 
---
 test/extensions/filters/http/ext_proc/test_processor.cc | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/test/extensions/filters/http/ext_proc/test_processor.cc b/test/extensions/filters/http/ext_proc/test_processor.cc
index 26094afbd131..b64f1de0bedd 100644
--- a/test/extensions/filters/http/ext_proc/test_processor.cc
+++ b/test/extensions/filters/http/ext_proc/test_processor.cc
@@ -20,12 +20,6 @@ grpc::Status ProcessorWrapper::Process(
     (*context_callback_)(ctx);
   }
   callback_(stream);
-  if (testing::Test::HasFatalFailure()) {
-    // This is not strictly necessary, but it may help in troubleshooting to
-    // ensure that we return a bad gRPC status if an "ASSERT" failed in the
-    // processor.
-    return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Fatal test error");
-  }
   return grpc::Status::OK;
 }
 

From ce4d425e62cd86554c2adcbff9210dbd40c4cc3f Mon Sep 17 00:00:00 2001
From: Kuo-Chung Hsu 
Date: Fri, 16 Jun 2023 05:31:12 -0700
Subject: [PATCH 558/740] thrift_proxy: remove a misleading macro call (#28004)

Signed-off-by: kuochunghsu 
---
 source/extensions/filters/network/thrift_proxy/router/config.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/source/extensions/filters/network/thrift_proxy/router/config.cc b/source/extensions/filters/network/thrift_proxy/router/config.cc
index 251fda72788a..1a5fdfd779e9 100644
--- a/source/extensions/filters/network/thrift_proxy/router/config.cc
+++ b/source/extensions/filters/network/thrift_proxy/router/config.cc
@@ -16,8 +16,6 @@ namespace Router {
 ThriftFilters::FilterFactoryCb RouterFilterConfig::createFilterFactoryFromProtoTyped(
     const envoy::extensions::filters::network::thrift_proxy::router::v3::Router& proto_config,
     const std::string& stat_prefix, Server::Configuration::FactoryContext& context) {
-  UNREFERENCED_PARAMETER(proto_config);
-
   auto stats =
       std::make_shared(stat_prefix, context.scope(), context.localInfo());
   auto shadow_writer = std::make_shared(

From ef9387f7336d136c5d1525f9c75176a4ae87cb75 Mon Sep 17 00:00:00 2001
From: Kevin Baichoo 
Date: Fri, 16 Jun 2023 14:55:48 -0400
Subject: [PATCH 559/740] Listener: Bound the number of connections that can be
 accepted per socket event  (#27773)

* Bound the number of connections that can be accepted per socket event on
listeners.

Signed-off-by: Kevin Baichoo 
---
 api/envoy/config/listener/v3/listener.proto   | 13 +++-
 changelogs/current.yaml                       |  8 ++
 envoy/event/dispatcher.h                      |  8 +-
 envoy/network/listener.h                      |  8 ++
 source/common/event/dispatcher_impl.cc        | 15 ++--
 source/common/event/dispatcher_impl.h         |  2 +-
 source/common/network/tcp_listener_impl.cc    | 19 +++--
 source/common/network/tcp_listener_impl.h     |  4 +-
 .../listener_manager/active_tcp_listener.cc   |  4 +-
 .../listener_manager/listener_impl.cc         |  6 ++
 .../listener_manager/listener_impl.h          |  4 +
 source/server/admin/admin.h                   |  3 +
 source/server/config_validation/dispatcher.cc |  6 --
 source/server/config_validation/dispatcher.h  |  5 +-
 test/common/http/codec_client_test.cc         |  5 +-
 test/common/network/connection_impl_test.cc   | 11 ++-
 test/common/network/listener_impl_test.cc     | 73 +++++++++++++++++--
 .../active_internal_listener_test.cc          |  3 +
 .../proxy_protocol_regression_test.cc         |  3 +
 .../common/fuzz/listener_filter_fuzzer.h      |  3 +
 .../proxy_protocol/proxy_protocol_test.cc     |  6 ++
 .../dns_resolver/cares/dns_impl_test.cc       |  3 +-
 ...ected_resource_monitor_integration_test.cc | 46 ++++++++++++
 .../transport_sockets/tls/ssl_socket_test.cc  | 47 ++++++++----
 test/integration/fake_upstream.h              |  3 +
 test/mocks/event/mocks.h                      |  7 +-
 test/mocks/event/wrapped_dispatcher.h         |  5 +-
 test/mocks/network/mocks.cc                   |  4 +
 test/mocks/network/mocks.h                    |  1 +
 test/server/config_validation/server_test.cc  |  4 +-
 test/server/connection_handler_test.cc        | 26 ++++---
 31 files changed, 276 insertions(+), 79 deletions(-)

diff --git a/api/envoy/config/listener/v3/listener.proto b/api/envoy/config/listener/v3/listener.proto
index 2016d2bb8ec5..5099dfa3e589 100644
--- a/api/envoy/config/listener/v3/listener.proto
+++ b/api/envoy/config/listener/v3/listener.proto
@@ -53,7 +53,7 @@ message ListenerCollection {
   repeated xds.core.v3.CollectionEntry entries = 1;
 }
 
-// [#next-free-field: 34]
+// [#next-free-field: 35]
 message Listener {
   option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Listener";
 
@@ -339,6 +339,17 @@ message Listener {
   // provided net.core.somaxconn will be used on Linux and 128 otherwise.
   google.protobuf.UInt32Value tcp_backlog_size = 24;
 
+  // The maximum number of connections to accept from the kernel per socket
+  // event. Envoy may decide to close these connections after accepting them
+  // from the kernel e.g. due to load shedding, or other policies.
+  // If there are more than max_connections_to_accept_per_socket_event
+  // connections pending accept, connections over this threshold will be
+  // accepted in later event loop iterations.
+  // If no value is provided Envoy will accept all connections pending accept
+  // from the kernel.
+  google.protobuf.UInt32Value max_connections_to_accept_per_socket_event = 34
+      [(validate.rules).uint32 = {gt: 0}];
+
   // Whether the listener should bind to the port. A listener that doesn't
   // bind can only receive connections redirected from other listeners that set
   // :ref:`use_original_dst `
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index f8ce0af6e040..c2ce3ad46ef8 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -218,6 +218,14 @@ new_features:
     added Runtime feature ``envoy.reloadable_features.max_request_headers_size_kb`` to override the default value of
     :ref:`max request headers size
     `.
+- area: listeners
+  change: |
+    added :ref:`max_connections_to_accept_per_socket_event
+    `
+    that sets the maximum number of new connections to be accepted per socket
+    event on a listener. If there are more connections to be accepted beyond
+    the maximum, the remaining connections would be processed in later
+    dispatcher loop iterations.
 - area: load shed point
   change: |
     added load shed point ``envoy.load_shed_points.http_connection_manager_decode_headers`` that rejects new http streams
diff --git a/envoy/event/dispatcher.h b/envoy/event/dispatcher.h
index 73c4cb80bd4f..7d568d279354 100644
--- a/envoy/event/dispatcher.h
+++ b/envoy/event/dispatcher.h
@@ -232,15 +232,13 @@ class Dispatcher : public DispatcherBase, public ScopeTracker {
    * @param socket supplies the socket to listen on.
    * @param cb supplies the callbacks to invoke for listener events.
    * @param runtime supplies the runtime for this server.
-   * @param bind_to_port controls whether the listener binds to a transport port or not.
-   * @param ignore_global_conn_limit controls whether the listener is limited by the global
-   * connection limit.
+   * @param listener_config configuration for the TCP listener to be created.
    * @return Network::ListenerPtr a new listener that is owned by the caller.
    */
   virtual Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket,
                                               Network::TcpListenerCallbacks& cb,
-                                              Runtime::Loader& runtime, bool bind_to_port,
-                                              bool ignore_global_conn_limit) PURE;
+                                              Runtime::Loader& runtime,
+                                              const Network::ListenerConfig& listener_config) PURE;
 
   /**
    * Creates a logical udp listener on a specific port.
diff --git a/envoy/network/listener.h b/envoy/network/listener.h
index 833ce6985c6f..5802ca480ae4 100644
--- a/envoy/network/listener.h
+++ b/envoy/network/listener.h
@@ -25,6 +25,9 @@
 namespace Envoy {
 namespace Network {
 
+// Set this to the maximum value which effectively accepts all connections.
+constexpr uint32_t DefaultMaxConnectionsToAcceptPerSocketEvent = UINT32_MAX;
+
 class ActiveUdpListenerFactory;
 class UdpListenerWorkerRouter;
 
@@ -240,6 +243,11 @@ class ListenerConfig {
    */
   virtual uint32_t tcpBacklogSize() const PURE;
 
+  /**
+   * @return the maximum number of connections that will be accepted for a given socket event.
+   */
+  virtual uint32_t maxConnectionsToAcceptPerSocketEvent() const PURE;
+
   /**
    * @return init manager of the listener.
    */
diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc
index 3219606e1cbe..1723e02f41a6 100644
--- a/source/common/event/dispatcher_impl.cc
+++ b/source/common/event/dispatcher_impl.cc
@@ -191,14 +191,15 @@ Filesystem::WatcherPtr DispatcherImpl::createFilesystemWatcher() {
   return Filesystem::WatcherPtr{new Filesystem::WatcherImpl(*this, file_system_)};
 }
 
-Network::ListenerPtr DispatcherImpl::createListener(Network::SocketSharedPtr&& socket,
-                                                    Network::TcpListenerCallbacks& cb,
-                                                    Runtime::Loader& runtime, bool bind_to_port,
-                                                    bool ignore_global_conn_limit) {
+Network::ListenerPtr
+DispatcherImpl::createListener(Network::SocketSharedPtr&& socket, Network::TcpListenerCallbacks& cb,
+                               Runtime::Loader& runtime,
+                               const Network::ListenerConfig& listener_config) {
   ASSERT(isThreadSafe());
-  return std::make_unique(*this, random_generator_, runtime,
-                                                    std::move(socket), cb, bind_to_port,
-                                                    ignore_global_conn_limit);
+  return std::make_unique(
+      *this, random_generator_, runtime, std::move(socket), cb, listener_config.bindToPort(),
+      listener_config.ignoreGlobalConnLimit(),
+      listener_config.maxConnectionsToAcceptPerSocketEvent());
 }
 
 Network::UdpListenerPtr
diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h
index fc3e9b2ca6c9..c051c4c088f7 100644
--- a/source/common/event/dispatcher_impl.h
+++ b/source/common/event/dispatcher_impl.h
@@ -77,7 +77,7 @@ class DispatcherImpl : Logger::Loggable,
   Filesystem::WatcherPtr createFilesystemWatcher() override;
   Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket,
                                       Network::TcpListenerCallbacks& cb, Runtime::Loader& runtime,
-                                      bool bind_to_port, bool ignore_global_conn_limit) override;
+                                      const Network::ListenerConfig& listener_config) override;
   Network::UdpListenerPtr
   createUdpListener(Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb,
                     const envoy::config::core::v3::UdpSocketConfig& config) override;
diff --git a/source/common/network/tcp_listener_impl.cc b/source/common/network/tcp_listener_impl.cc
index 395ac19caadb..c9cbe1efa833 100644
--- a/source/common/network/tcp_listener_impl.cc
+++ b/source/common/network/tcp_listener_impl.cc
@@ -40,8 +40,9 @@ void TcpListenerImpl::onSocketEvent(short flags) {
   ASSERT(bind_to_port_);
   ASSERT(flags & (Event::FileReadyType::Read));
 
-  // TODO(fcoras): Add limit on number of accepted calls per wakeup
-  while (1) {
+  uint32_t connections_accepted_from_kernel_count = 0;
+  for (; connections_accepted_from_kernel_count < max_connections_to_accept_per_socket_event_;
+       ++connections_accepted_from_kernel_count) {
     if (!socket_->ioHandle().isOpen()) {
       PANIC(fmt::format("listener accept failure: {}", errorDetails(errno)));
     }
@@ -90,18 +91,24 @@ void TcpListenerImpl::onSocketEvent(short flags) {
     cb_.onAccept(
         std::make_unique(std::move(io_handle), local_address, remote_address));
   }
+
+  ENVOY_LOG_MISC(trace, "TcpListener accepted {} new connections.",
+                 connections_accepted_from_kernel_count);
 }
 
 TcpListenerImpl::TcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random,
                                  Runtime::Loader& runtime, SocketSharedPtr socket,
                                  TcpListenerCallbacks& cb, bool bind_to_port,
-                                 bool ignore_global_conn_limit)
+                                 bool ignore_global_conn_limit,
+                                 uint32_t max_connections_to_accept_per_socket_event)
     : BaseListenerImpl(dispatcher, std::move(socket)), cb_(cb), random_(random), runtime_(runtime),
       bind_to_port_(bind_to_port), reject_fraction_(0.0),
-      ignore_global_conn_limit_(ignore_global_conn_limit) {
+      ignore_global_conn_limit_(ignore_global_conn_limit),
+      max_connections_to_accept_per_socket_event_(max_connections_to_accept_per_socket_event) {
   if (bind_to_port) {
-    // Although onSocketEvent drains to completion, use level triggered mode to avoid potential
-    // loss of the trigger due to transient accept errors.
+    // Use level triggered mode to avoid potential loss of the trigger due to
+    // transient accept errors or early termination due to accepting
+    // max_connections_to_accept_per_socket_event connections.
     socket_->ioHandle().initializeFileEvent(
         dispatcher, [this](uint32_t events) -> void { onSocketEvent(events); },
         Event::FileTriggerType::Level, Event::FileReadyType::Read);
diff --git a/source/common/network/tcp_listener_impl.h b/source/common/network/tcp_listener_impl.h
index ba404c57ba59..5cd08691665d 100644
--- a/source/common/network/tcp_listener_impl.h
+++ b/source/common/network/tcp_listener_impl.h
@@ -18,7 +18,8 @@ class TcpListenerImpl : public BaseListenerImpl {
 public:
   TcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random,
                   Runtime::Loader& runtime, SocketSharedPtr socket, TcpListenerCallbacks& cb,
-                  bool bind_to_port, bool ignore_global_conn_limit);
+                  bool bind_to_port, bool ignore_global_conn_limit,
+                  uint32_t max_connections_to_accept_per_socket_event);
   ~TcpListenerImpl() override {
     if (bind_to_port_) {
       socket_->ioHandle().resetFileEvents();
@@ -46,6 +47,7 @@ class TcpListenerImpl : public BaseListenerImpl {
   bool bind_to_port_;
   UnitFloat reject_fraction_;
   const bool ignore_global_conn_limit_;
+  const uint32_t max_connections_to_accept_per_socket_event_;
   Server::LoadShedPoint* listener_accept_{nullptr};
 };
 
diff --git a/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc b/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc
index 7dbad4b4a883..879ef13fabc9 100644
--- a/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc
+++ b/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc
@@ -19,9 +19,7 @@ ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent,
                                      Network::ConnectionBalancer& connection_balancer)
     : OwnedActiveStreamListenerBase(
           parent, parent.dispatcher(),
-          parent.dispatcher().createListener(std::move(socket), *this, runtime, config.bindToPort(),
-                                             config.ignoreGlobalConnLimit()),
-          config),
+          parent.dispatcher().createListener(std::move(socket), *this, runtime, config), config),
       tcp_conn_handler_(parent), connection_balancer_(connection_balancer),
       listen_address_(listen_address) {
   connection_balancer_.registerHandler(*this);
diff --git a/source/extensions/listener_managers/listener_manager/listener_impl.cc b/source/extensions/listener_managers/listener_manager/listener_impl.cc
index 47909c0de470..54a41c347129 100644
--- a/source/extensions/listener_managers/listener_manager/listener_impl.cc
+++ b/source/extensions/listener_managers/listener_manager/listener_impl.cc
@@ -320,6 +320,9 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config,
       added_via_api_(added_via_api), workers_started_(workers_started), hash_(hash),
       tcp_backlog_size_(
           PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, tcp_backlog_size, ENVOY_TCP_BACKLOG_SIZE)),
+      max_connections_to_accept_per_socket_event_(
+          PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_connections_to_accept_per_socket_event,
+                                          Network::DefaultMaxConnectionsToAcceptPerSocketEvent)),
       validation_visitor_(
           added_via_api_ ? parent_.server_.messageValidationContext().dynamicValidationVisitor()
                          : parent_.server_.messageValidationContext().staticValidationVisitor()),
@@ -446,6 +449,9 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin,
       workers_started_(workers_started), hash_(hash),
       tcp_backlog_size_(
           PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, tcp_backlog_size, ENVOY_TCP_BACKLOG_SIZE)),
+      max_connections_to_accept_per_socket_event_(
+          PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_connections_to_accept_per_socket_event,
+                                          Network::DefaultMaxConnectionsToAcceptPerSocketEvent)),
       validation_visitor_(
           added_via_api_ ? parent_.server_.messageValidationContext().dynamicValidationVisitor()
                          : parent_.server_.messageValidationContext().staticValidationVisitor()),
diff --git a/source/extensions/listener_managers/listener_manager/listener_impl.h b/source/extensions/listener_managers/listener_manager/listener_impl.h
index 4dfbd43b5a1d..7a2a0c57bcb0 100644
--- a/source/extensions/listener_managers/listener_manager/listener_impl.h
+++ b/source/extensions/listener_managers/listener_manager/listener_impl.h
@@ -377,6 +377,9 @@ class ListenerImpl final : public Network::ListenerConfig,
     return access_logs_;
   }
   uint32_t tcpBacklogSize() const override { return tcp_backlog_size_; }
+  uint32_t maxConnectionsToAcceptPerSocketEvent() const override {
+    return max_connections_to_accept_per_socket_event_;
+  }
   Init::Manager& initManager() override;
   bool ignoreGlobalConnLimit() const override { return ignore_global_conn_limit_; }
   envoy::config::core::v3::TrafficDirection direction() const override {
@@ -485,6 +488,7 @@ class ListenerImpl final : public Network::ListenerConfig,
   const bool workers_started_;
   const uint64_t hash_;
   const uint32_t tcp_backlog_size_;
+  const uint32_t max_connections_to_accept_per_socket_event_;
   ProtobufMessage::ValidationVisitor& validation_visitor_;
   const bool ignore_global_conn_limit_;
 
diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h
index fc5f3cf43ac3..da1a8729dcfe 100644
--- a/source/server/admin/admin.h
+++ b/source/server/admin/admin.h
@@ -428,6 +428,9 @@ class AdminImpl : public Admin,
       return empty_access_logs_;
     }
     uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; }
+    uint32_t maxConnectionsToAcceptPerSocketEvent() const override {
+      return Network::DefaultMaxConnectionsToAcceptPerSocketEvent;
+    }
     Init::Manager& initManager() override { return *init_manager_; }
     bool ignoreGlobalConnLimit() const override { return ignore_global_conn_limit_; }
 
diff --git a/source/server/config_validation/dispatcher.cc b/source/server/config_validation/dispatcher.cc
index 9e88c3975476..86fc214bba91 100644
--- a/source/server/config_validation/dispatcher.cc
+++ b/source/server/config_validation/dispatcher.cc
@@ -17,11 +17,5 @@ Network::ClientConnectionPtr ValidationDispatcher::createClientConnection(
                                                              transport_options);
 }
 
-Network::ListenerPtr ValidationDispatcher::createListener(Network::SocketSharedPtr&&,
-                                                          Network::TcpListenerCallbacks&,
-                                                          Runtime::Loader&, bool, bool) {
-  return nullptr;
-}
-
 } // namespace Event
 } // namespace Envoy
diff --git a/source/server/config_validation/dispatcher.h b/source/server/config_validation/dispatcher.h
index a4aa824a00c1..743ef2b34445 100644
--- a/source/server/config_validation/dispatcher.h
+++ b/source/server/config_validation/dispatcher.h
@@ -24,8 +24,9 @@ class ValidationDispatcher : public DispatcherImpl {
       Network::TransportSocketPtr&&, const Network::ConnectionSocket::OptionsSharedPtr& options,
       const Network::TransportSocketOptionsConstSharedPtr& transport_options) override;
   Network::ListenerPtr createListener(Network::SocketSharedPtr&&, Network::TcpListenerCallbacks&,
-                                      Runtime::Loader& runtime, bool bind_to_port,
-                                      bool ignore_global_conn_limit) override;
+                                      Runtime::Loader&, const Network::ListenerConfig&) override {
+    return nullptr;
+  }
 };
 
 } // namespace Event
diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc
index 2936e784cb36..b8031d7bc3bb 100644
--- a/test/common/http/codec_client_test.cc
+++ b/test/common/http/codec_client_test.cc
@@ -495,8 +495,9 @@ class CodecNetworkTest : public Event::TestUsingSimulatedTime,
     Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection(
         socket->connectionInfoProvider().localAddress(), source_address_,
         Network::Test::createRawBufferSocket(), nullptr, nullptr);
-    upstream_listener_ =
-        dispatcher_->createListener(std::move(socket), listener_callbacks_, runtime_, true, false);
+    NiceMock listener_config;
+    upstream_listener_ = dispatcher_->createListener(std::move(socket), listener_callbacks_,
+                                                     runtime_, listener_config);
     client_connection_ = client_connection.get();
     client_connection_->addConnectionCallbacks(client_callbacks_);
 
diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc
index ba0bae34a3e0..4208bb5abf8d 100644
--- a/test/common/network/connection_impl_test.cc
+++ b/test/common/network/connection_impl_test.cc
@@ -158,7 +158,9 @@ class ConnectionImplTestBase {
       dispatcher_ = api_->allocateDispatcher("test_thread");
     }
     socket_ = std::make_shared(address);
-    listener_ = dispatcher_->createListener(socket_, listener_callbacks_, runtime_, true, false);
+    NiceMock listener_config;
+    listener_ =
+        dispatcher_->createListener(socket_, listener_callbacks_, runtime_, listener_config);
     client_connection_ = std::make_unique(
         *dispatcher_, socket_->connectionInfoProvider().localAddress(), source_address_,
         createTransportSocket(), socket_options_, transport_socket_options_);
@@ -1426,7 +1428,8 @@ TEST_P(ConnectionImplTest, BindFailureTest) {
   dispatcher_ = api_->allocateDispatcher("test_thread");
   socket_ = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(GetParam()));
-  listener_ = dispatcher_->createListener(socket_, listener_callbacks_, runtime_, true, false);
+  NiceMock listener_config;
+  listener_ = dispatcher_->createListener(socket_, listener_callbacks_, runtime_, listener_config);
 
   client_connection_ = dispatcher_->createClientConnection(
       socket_->connectionInfoProvider().localAddress(), source_address_,
@@ -2913,7 +2916,9 @@ class ReadBufferLimitTest : public ConnectionImplTest {
     dispatcher_ = api_->allocateDispatcher("test_thread");
     socket_ = std::make_shared(
         Network::Test::getCanonicalLoopbackAddress(GetParam()));
-    listener_ = dispatcher_->createListener(socket_, listener_callbacks_, runtime_, true, false);
+    NiceMock listener_config;
+    listener_ =
+        dispatcher_->createListener(socket_, listener_callbacks_, runtime_, listener_config);
 
     client_connection_ = dispatcher_->createClientConnection(
         socket_->connectionInfoProvider().localAddress(),
diff --git a/test/common/network/listener_impl_test.cc b/test/common/network/listener_impl_test.cc
index 82230025b1e9..11522a18a265 100644
--- a/test/common/network/listener_impl_test.cc
+++ b/test/common/network/listener_impl_test.cc
@@ -38,8 +38,9 @@ static void errorCallbackTest(Address::IpVersion version) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version));
   Network::MockTcpListenerCallbacks listener_callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher->createListener(socket, listener_callbacks, runtime, true, false);
+      dispatcher->createListener(socket, listener_callbacks, runtime, listener_config);
 
   Network::ClientConnectionPtr client_connection = dispatcher->createClientConnection(
       socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(),
@@ -72,8 +73,16 @@ class TestTcpListenerImpl : public TcpListenerImpl {
   TestTcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random_generator,
                       Runtime::Loader& runtime, SocketSharedPtr socket, TcpListenerCallbacks& cb,
                       bool bind_to_port, bool ignore_global_conn_limit)
+      : TestTcpListenerImpl(dispatcher, random_generator, runtime, std::move(socket), cb,
+                            bind_to_port, ignore_global_conn_limit,
+                            Network::DefaultMaxConnectionsToAcceptPerSocketEvent) {}
+
+  TestTcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random_generator,
+                      Runtime::Loader& runtime, SocketSharedPtr socket, TcpListenerCallbacks& cb,
+                      bool bind_to_port, bool ignore_global_conn_limit,
+                      uint32_t max_connections_to_accept_per_socket_event)
       : TcpListenerImpl(dispatcher, random_generator, runtime, std::move(socket), cb, bind_to_port,
-                        ignore_global_conn_limit) {}
+                        ignore_global_conn_limit, max_connections_to_accept_per_socket_event) {}
 
   MOCK_METHOD(Address::InstanceConstSharedPtr, getLocalAddress, (os_fd_t fd));
 };
@@ -129,8 +138,9 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitEnforcement) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks listener_callbacks;
-  Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, listener_callbacks, scoped_runtime.loader(), true, false);
+  NiceMock listener_config;
+  Network::ListenerPtr listener = dispatcher_->createListener(
+      socket, listener_callbacks, scoped_runtime.loader(), listener_config);
 
   std::vector client_connections;
   std::vector server_connections;
@@ -194,8 +204,10 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitListenerOptOut) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks listener_callbacks;
-  Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, listener_callbacks, scoped_runtime.loader(), true, true);
+  NiceMock listener_config;
+  EXPECT_CALL(listener_config, ignoreGlobalConnLimit()).WillOnce(Return(true));
+  Network::ListenerPtr listener = dispatcher_->createListener(
+      socket, listener_callbacks, scoped_runtime.loader(), listener_config);
 
   std::vector client_connections;
   std::vector server_connections;
@@ -595,6 +607,55 @@ TEST_P(TcpListenerImplTest, EachQueuedConnectionShouldQueryTheLoadShedPoint) {
   dispatcher_->run(Event::Dispatcher::RunType::NonBlock);
 }
 
+TEST_P(TcpListenerImplTest, ShouldOnlyAcceptTheMaxNumberOfConnectionsConfiguredPerSocketEvent) {
+  auto socket = std::make_shared(
+      Network::Test::getCanonicalLoopbackAddress(version_));
+  MockTcpListenerCallbacks listener_callbacks;
+  Random::MockRandomGenerator random_generator;
+  NiceMock runtime;
+  const uint32_t max_connections_to_accept_per_socket_event = 1;
+  TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket,
+                               listener_callbacks, true, false,
+                               max_connections_to_accept_per_socket_event);
+
+  // Create two client connections, they should get accepted.
+  MockConnectionCallbacks connection_callbacks1;
+  ClientConnectionPtr client_connection1 = dispatcher_->createClientConnection(
+      socket->connectionInfoProvider().localAddress(), Address::InstanceConstSharedPtr(),
+      Network::Test::createRawBufferSocket(), nullptr, nullptr);
+  client_connection1->addConnectionCallbacks(connection_callbacks1);
+  client_connection1->connect();
+
+  MockConnectionCallbacks connection_callbacks2;
+  ClientConnectionPtr client_connection2 = dispatcher_->createClientConnection(
+      socket->connectionInfoProvider().localAddress(), Address::InstanceConstSharedPtr(),
+      Network::Test::createRawBufferSocket(), nullptr, nullptr);
+  client_connection2->addConnectionCallbacks(connection_callbacks2);
+  client_connection2->connect();
+
+  EXPECT_CALL(connection_callbacks1, onEvent(ConnectionEvent::Connected));
+  EXPECT_CALL(connection_callbacks2, onEvent(ConnectionEvent::Connected));
+  // Save the sever sockets so the connections do not close after being
+  // accepted.
+  std::vector server_sockets;
+  EXPECT_CALL(listener_callbacks, onAccept_(_))
+      .WillRepeatedly(
+          Invoke([this, &server_sockets](Network::ConnectionSocketPtr& server_socket) -> void {
+            server_sockets.push_back(std::move(server_socket));
+            if (server_sockets.size() == 2) {
+              dispatcher_->exit();
+            }
+          }));
+  //  Check the logs that they are accepted at different socket events
+  EXPECT_LOG_CONTAINS_N_TIMES("trace", "accepted 1 new connections", 2,
+                              { dispatcher_->run(Event::Dispatcher::RunType::Block); });
+
+  EXPECT_CALL(connection_callbacks1, onEvent(ConnectionEvent::LocalClose));
+  client_connection1->close(ConnectionCloseType::NoFlush);
+  EXPECT_CALL(connection_callbacks2, onEvent(ConnectionEvent::LocalClose));
+  client_connection2->close(ConnectionCloseType::NoFlush);
+}
+
 } // namespace
 } // namespace Network
 } // namespace Envoy
diff --git a/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc b/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc
index a63d2d5e1cb9..21dc77c17843 100644
--- a/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc
+++ b/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc
@@ -335,6 +335,9 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable {
     server_ = std::make_unique(*dispatcher_);
     socket_ = std::make_shared(
         Network::Test::getCanonicalLoopbackAddress(GetParam()));
-    listener_ = dispatcher_->createListener(socket_, *server_, runtime_, true, false);
+    NiceMock listener_config;
+    listener_ = dispatcher_->createListener(socket_, *server_, runtime_, listener_config);
     updateDnsResolverOptions();
 
     // Create a resolver options on stack here to emulate what actually happens in envoy bootstrap.
diff --git a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
index d0766536b1bb..95909558ea2d 100644
--- a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
+++ b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
@@ -178,4 +178,50 @@ TEST_P(OverloadIntegrationTest, NoNewStreamsWhenOverloaded) {
   codec_client_->close();
 }
 
+class ListenerMaxConnectionPerSocketEventTest : public OverloadIntegrationTest {};
+
+INSTANTIATE_TEST_SUITE_P(IpVersions, ListenerMaxConnectionPerSocketEventTest,
+                         testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
+                         TestUtility::ipTestParamsToString);
+
+TEST_P(ListenerMaxConnectionPerSocketEventTest, AcceptsConnectionsUpToTheMaximumPerSocketEvent) {
+  auto set_max_connections_per_socket_event_to_two =
+      [](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+        for (auto& listener_config : *bootstrap.mutable_static_resources()->mutable_listeners()) {
+          listener_config.mutable_max_connections_to_accept_per_socket_event()->set_value(2);
+        }
+      };
+  config_helper_.addConfigModifier(set_max_connections_per_socket_event_to_two);
+
+  initialize();
+  // Put envoy in overloaded state and check that it doesn't accept the new client connection.
+  updateResource(file_updater_1_, 0.95);
+  test_server_->waitForGaugeEq("overload.envoy.overload_actions.stop_accepting_connections.active",
+                               1);
+
+  // The TCP stack will accept the connections, but the Envoy listener will not
+  // not yet acknowledge the connection.
+  std::vector client_codecs;
+  for (int i = 0; i < 10; ++i) {
+    client_codecs.push_back(makeHttpConnection(makeClientConnection((lookupPort("http")))));
+    ASSERT_TRUE(client_codecs[i]->connected());
+  }
+
+  const std::string downstream_cx_active = (version_ == Network::Address::IpVersion::v4)
+                                               ? "listener.127.0.0.1_0.downstream_cx_active"
+                                               : "listener.[__1]_0.downstream_cx_active";
+  test_server_->waitForGaugeEq(downstream_cx_active, 0);
+
+  EXPECT_LOG_CONTAINS_N_TIMES("trace", "accepted 2 new connections", 5, {
+    // Reduce load a little to allow the connection to be accepted.
+    updateResource(file_updater_1_, 0.8);
+
+    // As we are using level trigger for listeners, all new connections get recognized.
+    test_server_->waitForGaugeEq(downstream_cx_active, 10);
+  });
+
+  std::for_each(client_codecs.begin(), client_codecs.end(),
+                [](IntegrationCodecClientPtr& client_codec) { client_codec->close(); });
+}
+
 } // namespace Envoy
diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc
index e036a7b6c1d4..ffda4d0ab62f 100644
--- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc
+++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc
@@ -359,8 +359,9 @@ void testUtil(const TestUtilOptions& options) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(options.version()));
   Network::MockTcpListenerCallbacks callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher->createListener(socket, callbacks, runtime, true, false);
+      dispatcher->createListener(socket, callbacks, runtime, listener_config);
 
   envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context;
   TestUtility::loadFromYaml(TestEnvironment::substitute(options.clientCtxYaml()),
@@ -701,8 +702,9 @@ void testUtilV2(const TestUtilOptionsV2& options) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(options.version()));
   NiceMock callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher->createListener(socket, callbacks, runtime, true, false);
+      dispatcher->createListener(socket, callbacks, runtime, listener_config);
 
   Stats::TestUtil::TestStore client_stats_store;
   Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system);
@@ -2978,8 +2980,9 @@ TEST_P(SslSocketTest, FlushCloseDuringHandshake) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, callbacks, runtime_, true, false);
+      dispatcher_->createListener(socket, callbacks, runtime_, listener_config);
 
   Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection(
       socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(),
@@ -3034,8 +3037,9 @@ TEST_P(SslSocketTest, HalfClose) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks listener_callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, listener_callbacks, runtime_, true, false);
+      dispatcher_->createListener(socket, listener_callbacks, runtime_, listener_config);
   std::shared_ptr server_read_filter(new Network::MockReadFilter());
   std::shared_ptr client_read_filter(new Network::MockReadFilter());
 
@@ -3116,8 +3120,9 @@ TEST_P(SslSocketTest, ShutdownWithCloseNotify) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks listener_callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, listener_callbacks, runtime_, true, false);
+      dispatcher_->createListener(socket, listener_callbacks, runtime_, listener_config);
   std::shared_ptr server_read_filter(new Network::MockReadFilter());
   std::shared_ptr client_read_filter(new Network::MockReadFilter());
 
@@ -3204,8 +3209,9 @@ TEST_P(SslSocketTest, ShutdownWithoutCloseNotify) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks listener_callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, listener_callbacks, runtime_, true, false);
+      dispatcher_->createListener(socket, listener_callbacks, runtime_, listener_config);
   std::shared_ptr server_read_filter(new Network::MockReadFilter());
   std::shared_ptr client_read_filter(new Network::MockReadFilter());
 
@@ -3308,8 +3314,9 @@ TEST_P(SslSocketTest, ClientAuthMultipleCAs) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, callbacks, runtime_, true, false);
+      dispatcher_->createListener(socket, callbacks, runtime_, listener_config);
 
   const std::string client_ctx_yaml = R"EOF(
   common_tls_context:
@@ -3408,10 +3415,11 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1,
       Network::Test::getCanonicalLoopbackAddress(ip_version));
   NiceMock callbacks;
   Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread"));
+  NiceMock listener_config;
   Network::ListenerPtr listener1 =
-      dispatcher->createListener(socket1, callbacks, runtime, true, false);
+      dispatcher->createListener(socket1, callbacks, runtime, listener_config);
   Network::ListenerPtr listener2 =
-      dispatcher->createListener(socket2, callbacks, runtime, true, false);
+      dispatcher->createListener(socket2, callbacks, runtime, listener_config);
 
   envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context;
   TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml), client_tls_context);
@@ -3550,9 +3558,10 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml
   auto tcp_socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(ip_version));
   NiceMock callbacks;
+  NiceMock listener_config;
   Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread"));
   Network::ListenerPtr listener =
-      dispatcher->createListener(tcp_socket, callbacks, runtime, true, false);
+      dispatcher->createListener(tcp_socket, callbacks, runtime, listener_config);
 
   envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context;
   TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml), client_tls_context);
@@ -4153,10 +4162,11 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) {
   auto socket2 = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, callbacks, runtime_, true, false);
+      dispatcher_->createListener(socket, callbacks, runtime_, listener_config);
   Network::ListenerPtr listener2 =
-      dispatcher_->createListener(socket2, callbacks, runtime_, true, false);
+      dispatcher_->createListener(socket2, callbacks, runtime_, listener_config);
   const std::string client_ctx_yaml = R"EOF(
   common_tls_context:
     tls_certificates:
@@ -4272,10 +4282,11 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version));
   NiceMock callbacks;
+  NiceMock listener_config;
   Api::ApiPtr api = Api::createApiForTest(server_stats_store, time_system_);
   Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread"));
   Network::ListenerPtr listener =
-      dispatcher->createListener(socket, callbacks, runtime_, true, false);
+      dispatcher->createListener(socket, callbacks, runtime_, listener_config);
 
   Network::ConnectionPtr server_connection;
   Network::MockConnectionCallbacks server_connection_callbacks;
@@ -4534,8 +4545,9 @@ TEST_P(SslSocketTest, SslError) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, callbacks, runtime_, true, false);
+      dispatcher_->createListener(socket, callbacks, runtime_, listener_config);
 
   Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection(
       socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(),
@@ -5063,8 +5075,9 @@ TEST_P(SslSocketTest, SetSignatureAlgorithms) {
   auto socket = std::make_shared(
       Network::Test::getCanonicalLoopbackAddress(version_));
   Network::MockTcpListenerCallbacks callbacks;
+  NiceMock listener_config;
   Network::ListenerPtr listener =
-      dispatcher_->createListener(socket, callbacks, runtime_, true, false);
+      dispatcher_->createListener(socket, callbacks, runtime_, listener_config);
 
   const std::string client_ctx_yaml = R"EOF(
   common_tls_context:
@@ -5764,7 +5777,9 @@ class SslReadBufferLimitTest : public SslSocketTest {
 
     socket_ = std::make_shared(
         Network::Test::getCanonicalLoopbackAddress(version_));
-    listener_ = dispatcher_->createListener(socket_, listener_callbacks_, runtime_, true, false);
+    NiceMock listener_config;
+    listener_ =
+        dispatcher_->createListener(socket_, listener_callbacks_, runtime_, listener_config);
 
     TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml_), upstream_tls_context_);
     auto client_cfg =
diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h
index c0b99df16891..6991b87509f5 100644
--- a/test/integration/fake_upstream.h
+++ b/test/integration/fake_upstream.h
@@ -879,6 +879,9 @@ class FakeUpstream : Logger::Loggable,
     }
     ResourceLimit& openConnections() override { return connection_resource_; }
     uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; }
+    uint32_t maxConnectionsToAcceptPerSocketEvent() const override {
+      return Network::DefaultMaxConnectionsToAcceptPerSocketEvent;
+    }
     Init::Manager& initManager() override { return *init_manager_; }
     bool ignoreGlobalConnLimit() const override { return false; }
 
diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h
index 3967d14e3979..4ca4a2276a43 100644
--- a/test/mocks/event/mocks.h
+++ b/test/mocks/event/mocks.h
@@ -72,9 +72,8 @@ class MockDispatcher : public Dispatcher {
 
   Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket,
                                       Network::TcpListenerCallbacks& cb, Runtime::Loader& runtime,
-                                      bool bind_to_port, bool ignore_global_conn_limit) override {
-    return Network::ListenerPtr{
-        createListener_(std::move(socket), cb, runtime, bind_to_port, ignore_global_conn_limit)};
+                                      const Network::ListenerConfig& listener_config) override {
+    return Network::ListenerPtr{createListener_(std::move(socket), cb, runtime, listener_config)};
   }
 
   Network::UdpListenerPtr
@@ -143,7 +142,7 @@ class MockDispatcher : public Dispatcher {
   MOCK_METHOD(Filesystem::Watcher*, createFilesystemWatcher_, ());
   MOCK_METHOD(Network::Listener*, createListener_,
               (Network::SocketSharedPtr && socket, Network::TcpListenerCallbacks& cb,
-               Runtime::Loader& runtime, bool bind_to_port, bool ignore_global_conn_limit));
+               Runtime::Loader& runtime, const Network::ListenerConfig& listener_config));
   MOCK_METHOD(Network::UdpListener*, createUdpListener_,
               (Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb,
                const envoy::config::core::v3::UdpSocketConfig& config));
diff --git a/test/mocks/event/wrapped_dispatcher.h b/test/mocks/event/wrapped_dispatcher.h
index b0c5e4c57efa..90d9bbc12341 100644
--- a/test/mocks/event/wrapped_dispatcher.h
+++ b/test/mocks/event/wrapped_dispatcher.h
@@ -62,9 +62,8 @@ class WrappedDispatcher : public Dispatcher {
 
   Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket,
                                       Network::TcpListenerCallbacks& cb, Runtime::Loader& runtime,
-                                      bool bind_to_port, bool ignore_global_conn_limit) override {
-    return impl_.createListener(std::move(socket), cb, runtime, bind_to_port,
-                                ignore_global_conn_limit);
+                                      const Network::ListenerConfig& listener_config) override {
+    return impl_.createListener(std::move(socket), cb, runtime, listener_config);
   }
 
   Network::UdpListenerPtr
diff --git a/test/mocks/network/mocks.cc b/test/mocks/network/mocks.cc
index 25dc8d522698..ac6f489df395 100644
--- a/test/mocks/network/mocks.cc
+++ b/test/mocks/network/mocks.cc
@@ -43,6 +43,10 @@ MockListenerConfig::MockListenerConfig()
       .WillByDefault(Return(socket_));
   ON_CALL(*this, listenerScope()).WillByDefault(ReturnRef(*store_.rootScope()));
   ON_CALL(*this, name()).WillByDefault(ReturnRef(name_));
+  ON_CALL(*this, maxConnectionsToAcceptPerSocketEvent())
+      .WillByDefault(Return(Network::DefaultMaxConnectionsToAcceptPerSocketEvent));
+  ON_CALL(*this, ignoreGlobalConnLimit()).WillByDefault(Return(false));
+  ON_CALL(*this, bindToPort()).WillByDefault(Return(true));
 }
 MockListenerConfig::~MockListenerConfig() = default;
 
diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h
index 70e69638f139..4f680680b0ce 100644
--- a/test/mocks/network/mocks.h
+++ b/test/mocks/network/mocks.h
@@ -445,6 +445,7 @@ class MockListenerConfig : public ListenerConfig {
   MOCK_METHOD(ConnectionBalancer&, connectionBalancer, (const Network::Address::Instance&));
   MOCK_METHOD(ResourceLimit&, openConnections, ());
   MOCK_METHOD(uint32_t, tcpBacklogSize, (), (const));
+  MOCK_METHOD(uint32_t, maxConnectionsToAcceptPerSocketEvent, (), (const));
   MOCK_METHOD(Init::Manager&, initManager, ());
   MOCK_METHOD(bool, ignoreGlobalConnLimit, (), (const));
 
diff --git a/test/server/config_validation/server_test.cc b/test/server/config_validation/server_test.cc
index bc883c65ba6f..18d82bb87d32 100644
--- a/test/server/config_validation/server_test.cc
+++ b/test/server/config_validation/server_test.cc
@@ -201,7 +201,9 @@ TEST_P(ValidationServerTest, DummyMethodsTest) {
   server.admin()->startHttpListener({}, "", nullptr, nullptr, nullptr);
 
   Network::MockTcpListenerCallbacks listener_callbacks;
-  server.dispatcher().createListener(nullptr, listener_callbacks, server.runtime(), false, false);
+  Network::MockListenerConfig listener_config;
+  server.dispatcher().createListener(nullptr, listener_callbacks, server.runtime(),
+                                     listener_config);
 
   server.dnsResolver()->resolve("", Network::DnsLookupFamily::All, nullptr);
 }
diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc
index 376ed11245fa..4eb3f9329a9c 100644
--- a/test/server/connection_handler_test.cc
+++ b/test/server/connection_handler_test.cc
@@ -156,6 +156,9 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::LoggablesocketFactory(), getListenSocket(_))
         .WillOnce(Return(listeners_.back()->sockets_[0]));
     if (socket_type == Network::Socket::Type::Stream) {
-      EXPECT_CALL(dispatcher_, createListener_(_, _, _, _, _))
-          .WillOnce(Invoke([listener, listener_callbacks](
-                               Network::SocketSharedPtr&&, Network::TcpListenerCallbacks& cb,
-                               Runtime::Loader&, bool, bool) -> Network::Listener* {
-            if (listener_callbacks != nullptr) {
-              *listener_callbacks = &cb;
-            }
-            return listener;
-          }));
+      EXPECT_CALL(dispatcher_, createListener_(_, _, _, _))
+          .WillOnce(Invoke(
+              [listener, listener_callbacks](Network::SocketSharedPtr&&,
+                                             Network::TcpListenerCallbacks& cb, Runtime::Loader&,
+                                             const Network::ListenerConfig&) -> Network::Listener* {
+                if (listener_callbacks != nullptr) {
+                  *listener_callbacks = &cb;
+                }
+                return listener;
+              }));
     } else {
       EXPECT_CALL(dispatcher_, createUdpListener_(_, _, _))
           .WillOnce(
@@ -384,11 +388,11 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggablesockets_[i]->connection_info_provider_->setLocalAddress(addresses[i]);
 
       if (socket_type == Network::Socket::Type::Stream) {
-        EXPECT_CALL(dispatcher_, createListener_(_, _, _, _, _))
+        EXPECT_CALL(dispatcher_, createListener_(_, _, _, _))
             .WillOnce(
                 Invoke([i, &mock_listeners, &listener_callbacks_map](
                            Network::SocketSharedPtr&& socket, Network::TcpListenerCallbacks& cb,
-                           Runtime::Loader&, bool, bool) -> Network::Listener* {
+                           Runtime::Loader&, const Network::ListenerConfig&) -> Network::Listener* {
                   auto listener_callbacks_iter = listener_callbacks_map.find(
                       socket->connectionInfoProvider().localAddress()->asString());
                   EXPECT_NE(listener_callbacks_iter, listener_callbacks_map.end());

From d839ec6777b6bb91234a918cdd579069004564a3 Mon Sep 17 00:00:00 2001
From: code 
Date: Sat, 17 Jun 2023 03:27:00 +0800
Subject: [PATCH 560/740] minor fix: update the new interface name (#27986)

* minor fix: update the new interface name

Signed-off-by: wbpcode 
---
 envoy/server/filter_config.h                              | 4 ++--
 source/extensions/filters/http/common/factory_base.h      | 8 ++++----
 source/extensions/filters/http/composite/action.h         | 2 +-
 source/extensions/filters/http/ext_proc/config.cc         | 3 ++-
 source/extensions/filters/http/ext_proc/config.h          | 2 +-
 test/extensions/filters/http/ext_proc/config_test.cc      | 2 +-
 test/integration/filters/server_factory_context_filter.cc | 2 +-
 test/integration/filters/set_response_code_filter.cc      | 2 +-
 8 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/envoy/server/filter_config.h b/envoy/server/filter_config.h
index ace1d61cd619..e05d5e6c5384 100644
--- a/envoy/server/filter_config.h
+++ b/envoy/server/filter_config.h
@@ -262,8 +262,8 @@ class NamedHttpFilterConfigFactory : public virtual HttpFilterConfigFactoryBase
    * @return Http::FilterFactoryCb the factory creation function.
    */
   virtual Http::FilterFactoryCb
-  createFilterServerFactoryFromProto(const Protobuf::Message&, const std::string&,
-                                     Server::Configuration::ServerFactoryContext&) {
+  createFilterFactoryFromProtoWithServerContext(const Protobuf::Message&, const std::string&,
+                                                Server::Configuration::ServerFactoryContext&) {
     ExceptionUtil::throwEnvoyException(
         "Creating filter factory from server factory context is not supported");
     return nullptr;
diff --git a/source/extensions/filters/http/common/factory_base.h b/source/extensions/filters/http/common/factory_base.h
index 4e13c4b8bf1e..c36b710ef80c 100644
--- a/source/extensions/filters/http/common/factory_base.h
+++ b/source/extensions/filters/http/common/factory_base.h
@@ -77,18 +77,18 @@ class FactoryBase : public CommonFactoryBase,
                                     const std::string& stats_prefix,
                                     Server::Configuration::FactoryContext& context) PURE;
 
-  Envoy::Http::FilterFactoryCb createFilterServerFactoryFromProto(
+  Envoy::Http::FilterFactoryCb createFilterFactoryFromProtoWithServerContext(
       const Protobuf::Message& proto_config, const std::string& stats_prefix,
       Server::Configuration::ServerFactoryContext& server_context) override {
-    return createFilterServerFactoryFromProtoTyped(
+    return createFilterFactoryFromProtoWithServerContextTyped(
         MessageUtil::downcastAndValidate(
             proto_config, server_context.messageValidationVisitor()),
         stats_prefix, server_context);
   }
 
   virtual Envoy::Http::FilterFactoryCb
-  createFilterServerFactoryFromProtoTyped(const ConfigProto&, const std::string&,
-                                          Server::Configuration::ServerFactoryContext&) {
+  createFilterFactoryFromProtoWithServerContextTyped(const ConfigProto&, const std::string&,
+                                                     Server::Configuration::ServerFactoryContext&) {
     ExceptionUtil::throwEnvoyException(
         "Creating filter factory from server factory context is not supported");
     return nullptr;
diff --git a/source/extensions/filters/http/composite/action.h b/source/extensions/filters/http/composite/action.h
index 62f28135ae7c..9457614a9b84 100644
--- a/source/extensions/filters/http/composite/action.h
+++ b/source/extensions/filters/http/composite/action.h
@@ -47,7 +47,7 @@ class ExecuteFilterActionFactory
     // is present.
     TRY_NEEDS_AUDIT {
       if (context.server_factory_context_.has_value()) {
-        callback = factory.createFilterServerFactoryFromProto(
+        callback = factory.createFilterFactoryFromProtoWithServerContext(
             *message, context.stat_prefix_, context.server_factory_context_.value());
       }
     }
diff --git a/source/extensions/filters/http/ext_proc/config.cc b/source/extensions/filters/http/ext_proc/config.cc
index 5ccf5a51f1be..7a0240e84e03 100644
--- a/source/extensions/filters/http/ext_proc/config.cc
+++ b/source/extensions/filters/http/ext_proc/config.cc
@@ -36,7 +36,8 @@ ExternalProcessingFilterConfig::createRouteSpecificFilterConfigTyped(
   return std::make_shared(proto_config);
 }
 
-Http::FilterFactoryCb ExternalProcessingFilterConfig::createFilterServerFactoryFromProtoTyped(
+Http::FilterFactoryCb
+ExternalProcessingFilterConfig::createFilterFactoryFromProtoWithServerContextTyped(
     const envoy::extensions::filters::http::ext_proc::v3::ExternalProcessor& proto_config,
     const std::string& stats_prefix, Server::Configuration::ServerFactoryContext& server_context) {
   const uint32_t message_timeout_ms =
diff --git a/source/extensions/filters/http/ext_proc/config.h b/source/extensions/filters/http/ext_proc/config.h
index 12a03d6710db..a2912466eb6b 100644
--- a/source/extensions/filters/http/ext_proc/config.h
+++ b/source/extensions/filters/http/ext_proc/config.h
@@ -32,7 +32,7 @@ class ExternalProcessingFilterConfig
       Server::Configuration::ServerFactoryContext& context,
       ProtobufMessage::ValidationVisitor& validator) override;
 
-  Http::FilterFactoryCb createFilterServerFactoryFromProtoTyped(
+  Http::FilterFactoryCb createFilterFactoryFromProtoWithServerContextTyped(
       const envoy::extensions::filters::http::ext_proc::v3::ExternalProcessor& proto_config,
       const std::string& stats_prefix,
       Server::Configuration::ServerFactoryContext& server_context) override;
diff --git a/test/extensions/filters/http/ext_proc/config_test.cc b/test/extensions/filters/http/ext_proc/config_test.cc
index fd1f9eb2a628..914e454ebd5b 100644
--- a/test/extensions/filters/http/ext_proc/config_test.cc
+++ b/test/extensions/filters/http/ext_proc/config_test.cc
@@ -73,7 +73,7 @@ TEST(HttpExtProcConfigTest, CorrectConfigServerContext) {
   testing::NiceMock context;
   EXPECT_CALL(context, messageValidationVisitor());
   Http::FilterFactoryCb cb =
-      factory.createFilterServerFactoryFromProto(*proto_config, "stats", context);
+      factory.createFilterFactoryFromProtoWithServerContext(*proto_config, "stats", context);
   Http::MockFilterChainFactoryCallbacks filter_callback;
   EXPECT_CALL(filter_callback, addStreamFilter(_));
   cb(filter_callback);
diff --git a/test/integration/filters/server_factory_context_filter.cc b/test/integration/filters/server_factory_context_filter.cc
index abc532d900fa..cea3f7ee5457 100644
--- a/test/integration/filters/server_factory_context_filter.cc
+++ b/test/integration/filters/server_factory_context_filter.cc
@@ -133,7 +133,7 @@ class ServerFactoryContextFilterFactory
     return nullptr;
   }
 
-  Http::FilterFactoryCb createFilterServerFactoryFromProtoTyped(
+  Http::FilterFactoryCb createFilterFactoryFromProtoWithServerContextTyped(
       const test::integration::filters::ServerFactoryContextFilterConfig& proto_config,
       const std::string&, Server::Configuration::ServerFactoryContext& server_context) override {
     FilterConfigSharedPtr filter_config =
diff --git a/test/integration/filters/set_response_code_filter.cc b/test/integration/filters/set_response_code_filter.cc
index 26a8b5bedaa3..f2cc3dc8e633 100644
--- a/test/integration/filters/set_response_code_filter.cc
+++ b/test/integration/filters/set_response_code_filter.cc
@@ -59,7 +59,7 @@ class SetResponseCodeFilterFactory : public Extensions::HttpFilters::Common::Fac
       callbacks.addStreamFilter(std::make_shared(filter_config));
     };
   }
-  Http::FilterFactoryCb createFilterServerFactoryFromProtoTyped(
+  Http::FilterFactoryCb createFilterFactoryFromProtoWithServerContextTyped(
       const test::integration::filters::SetResponseCodeFilterConfig& proto_config,
       const std::string&, Server::Configuration::ServerFactoryContext& context) override {
     auto filter_config = std::make_shared(

From 61958d40059a3d901b6f0e453d769f58a3dd2e6a Mon Sep 17 00:00:00 2001
From: botengyao 
Date: Fri, 16 Jun 2023 15:44:20 -0400
Subject: [PATCH 561/740] hc: expose server context to hc context (#27948)

* expose server context

Signed-off-by: Boteng Yao 
---
 envoy/server/BUILD                            |  1 +
 envoy/server/health_checker_config.h          |  6 +++
 .../common/upstream/cluster_factory_impl.cc   |  6 +--
 .../upstream/health_checker_event_logger.h    | 10 ++--
 source/common/upstream/health_checker_impl.cc | 15 +++---
 source/common/upstream/health_checker_impl.h  | 30 +++++------
 .../upstream/health_discovery_service.cc      | 12 ++---
 test/common/upstream/BUILD                    |  2 +-
 .../upstream/health_checker_impl_test.cc      | 54 +++++++------------
 test/extensions/health_checkers/redis/BUILD   |  5 +-
 .../health_checkers/redis/config_test.cc      | 25 +++------
 test/extensions/health_checkers/thrift/BUILD  |  5 +-
 .../health_checkers/thrift/config_test.cc     | 25 +++------
 test/mocks/server/BUILD                       |  1 +
 .../server/health_checker_factory_context.cc  |  1 +
 .../server/health_checker_factory_context.h   |  3 ++
 16 files changed, 81 insertions(+), 120 deletions(-)

diff --git a/envoy/server/BUILD b/envoy/server/BUILD
index ea069d7bf252..7649e4a728fb 100644
--- a/envoy/server/BUILD
+++ b/envoy/server/BUILD
@@ -96,6 +96,7 @@ envoy_cc_library(
     name = "health_checker_config_interface",
     hdrs = ["health_checker_config.h"],
     deps = [
+        ":factory_context_interface",
         "//envoy/common:random_generator_interface",
         "//envoy/config:typed_config_interface",
         "//envoy/upstream:health_checker_interface",
diff --git a/envoy/server/health_checker_config.h b/envoy/server/health_checker_config.h
index 9ee5477420fa..e329735d0c64 100644
--- a/envoy/server/health_checker_config.h
+++ b/envoy/server/health_checker_config.h
@@ -4,6 +4,7 @@
 #include "envoy/config/core/v3/health_check.pb.h"
 #include "envoy/config/typed_config.h"
 #include "envoy/runtime/runtime.h"
+#include "envoy/server/factory_context.h"
 #include "envoy/upstream/health_checker.h"
 
 namespace Envoy {
@@ -52,6 +53,11 @@ class HealthCheckerFactoryContext {
    */
   virtual AccessLog::AccessLogManager& accessLogManager() PURE;
 
+  /*
+   * @return Server context.
+   */
+  virtual Server::Configuration::ServerFactoryContext& serverFactoryContext() PURE;
+
   /**
    * Set the event logger to the context, nullptr is accepted since
    * the default in the context is nullptr.
diff --git a/source/common/upstream/cluster_factory_impl.cc b/source/common/upstream/cluster_factory_impl.cc
index ea8cdca4da25..50b720a7bdd5 100644
--- a/source/common/upstream/cluster_factory_impl.cc
+++ b/source/common/upstream/cluster_factory_impl.cc
@@ -105,10 +105,8 @@ ClusterFactoryImplBase::create(const envoy::config::cluster::v3::Cluster& cluste
     if (cluster.health_checks().size() != 1) {
       return absl::InvalidArgumentError("Multiple health checks not supported");
     } else {
-      auto checker_or_error = HealthCheckerFactory::create(
-          cluster.health_checks()[0], *new_cluster_pair.first, server_context.runtime(),
-          server_context.mainThreadDispatcher(), server_context.accessLogManager(),
-          context.messageValidationVisitor(), server_context.api());
+      auto checker_or_error = HealthCheckerFactory::create(cluster.health_checks()[0],
+                                                           *new_cluster_pair.first, server_context);
       RETURN_IF_STATUS_NOT_OK(checker_or_error);
       new_cluster_pair.first->setHealthChecker(checker_or_error.value());
     }
diff --git a/source/common/upstream/health_checker_event_logger.h b/source/common/upstream/health_checker_event_logger.h
index 5b4d2b526b9f..a05d3a5f0196 100644
--- a/source/common/upstream/health_checker_event_logger.h
+++ b/source/common/upstream/health_checker_event_logger.h
@@ -25,16 +25,16 @@ namespace Upstream {
 
 class HealthCheckEventLoggerImpl : public HealthCheckEventLogger {
 public:
-  HealthCheckEventLoggerImpl(AccessLog::AccessLogManager& log_manager, TimeSource& time_source,
-                             const envoy::config::core::v3::HealthCheck& health_check_config,
+  HealthCheckEventLoggerImpl(const envoy::config::core::v3::HealthCheck& health_check_config,
                              Server::Configuration::HealthCheckerFactoryContext& context)
-      : time_source_(time_source) {
+      : time_source_(context.serverFactoryContext().mainThreadDispatcher().timeSource()) {
 
     // TODO(botengyao): Remove the file_ creation here into the file based health check
     // event sink. In this way you can remove the file_ based code from the createHealthCheckEvent
     if (!health_check_config.event_log_path().empty() /* deprecated */) {
-      file_ = log_manager.createAccessLog(Filesystem::FilePathAndType{
-          Filesystem::DestinationType::File, health_check_config.event_log_path()});
+      file_ = context.serverFactoryContext().accessLogManager().createAccessLog(
+          Filesystem::FilePathAndType{Filesystem::DestinationType::File,
+                                      health_check_config.event_log_path()});
     }
     for (const auto& config : health_check_config.event_logger()) {
       auto& factory = Config::Utility::getAndCheckFactory(config);
diff --git a/source/common/upstream/health_checker_impl.cc b/source/common/upstream/health_checker_impl.cc
index d9f14b733f78..b318107f0cf0 100644
--- a/source/common/upstream/health_checker_impl.cc
+++ b/source/common/upstream/health_checker_impl.cc
@@ -47,11 +47,10 @@ const std::string& HealthCheckerFactory::getHostname(const HostSharedPtr& host,
   return cluster->name();
 }
 
-absl::StatusOr HealthCheckerFactory::create(
-    const envoy::config::core::v3::HealthCheck& health_check_config, Upstream::Cluster& cluster,
-    Runtime::Loader& runtime, Event::Dispatcher& dispatcher,
-    AccessLog::AccessLogManager& log_manager,
-    ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api) {
+absl::StatusOr
+HealthCheckerFactory::create(const envoy::config::core::v3::HealthCheck& health_check_config,
+                             Upstream::Cluster& cluster,
+                             Server::Configuration::ServerFactoryContext& server_context) {
   Server::Configuration::CustomHealthCheckerFactory* factory = nullptr;
 
   switch (health_check_config.health_checker_case()) {
@@ -81,14 +80,12 @@ absl::StatusOr HealthCheckerFactory::create(
   }
 
   std::unique_ptr context(
-      new HealthCheckerFactoryContextImpl(cluster, runtime, dispatcher, validation_visitor, api,
-                                          log_manager));
+      new HealthCheckerFactoryContextImpl(cluster, server_context));
 
   if (!health_check_config.event_log_path().empty() /* deprecated */ ||
       !health_check_config.event_logger().empty()) {
     HealthCheckEventLoggerPtr event_logger;
-    event_logger = std::make_unique(
-        log_manager, dispatcher.timeSource(), health_check_config, *context);
+    event_logger = std::make_unique(health_check_config, *context);
     context->setEventLogger(std::move(event_logger));
   }
   return factory->createCustomHealthChecker(health_check_config, *context);
diff --git a/source/common/upstream/health_checker_impl.h b/source/common/upstream/health_checker_impl.h
index a5ae845b1b52..040173b0d95a 100644
--- a/source/common/upstream/health_checker_impl.h
+++ b/source/common/upstream/health_checker_impl.h
@@ -52,12 +52,13 @@ struct HealthCheckerEqualTo {
  */
 class HealthCheckerFactoryContextImpl : public Server::Configuration::HealthCheckerFactoryContext {
 public:
-  HealthCheckerFactoryContextImpl(Upstream::Cluster& cluster, Envoy::Runtime::Loader& runtime,
-                                  Event::Dispatcher& dispatcher,
-                                  ProtobufMessage::ValidationVisitor& validation_visitor,
-                                  Api::Api& api, AccessLog::AccessLogManager& log_manager)
-      : cluster_(cluster), runtime_(runtime), dispatcher_(dispatcher),
-        validation_visitor_(validation_visitor), log_manager_(log_manager), api_(api) {}
+  HealthCheckerFactoryContextImpl(Upstream::Cluster& cluster,
+                                  Server::Configuration::ServerFactoryContext& server_context)
+      : cluster_(cluster), runtime_(server_context.runtime()),
+        dispatcher_(server_context.mainThreadDispatcher()),
+        validation_visitor_(server_context.messageValidationVisitor()),
+        log_manager_(server_context.accessLogManager()), api_(server_context.api()),
+        server_context_(server_context) {}
   Upstream::Cluster& cluster() override { return cluster_; }
   Envoy::Runtime::Loader& runtime() override { return runtime_; }
   Event::Dispatcher& mainThreadDispatcher() override { return dispatcher_; }
@@ -72,6 +73,10 @@ class HealthCheckerFactoryContextImpl : public Server::Configuration::HealthChec
     event_logger_ = std::move(event_logger);
   }
 
+  Server::Configuration::ServerFactoryContext& serverFactoryContext() override {
+    return server_context_;
+  };
+
 private:
   Upstream::Cluster& cluster_;
   Envoy::Runtime::Loader& runtime_;
@@ -80,6 +85,7 @@ class HealthCheckerFactoryContextImpl : public Server::Configuration::HealthChec
   AccessLog::AccessLogManager& log_manager_;
   Api::Api& api_;
   HealthCheckEventLoggerPtr event_logger_;
+  Server::Configuration::ServerFactoryContext& server_context_;
 };
 
 /**
@@ -95,18 +101,12 @@ class HealthCheckerFactory : public Logger::Loggable
    * Create a health checker or return an error.
    * @param health_check_config supplies the health check proto.
    * @param cluster supplies the owning cluster.
-   * @param runtime supplies the runtime loader.
-   * @param dispatcher supplies the dispatcher.
-   * @param log_manager supplies the log_manager.
-   * @param validation_visitor message validation visitor instance.
-   * @param api reference to the Api object
-   * @return a health checker or creation error.
+   * @param server_context reference to the Server context object
+   * @return a health checker.
    */
   static absl::StatusOr
   create(const envoy::config::core::v3::HealthCheck& health_check_config,
-         Upstream::Cluster& cluster, Runtime::Loader& runtime, Event::Dispatcher& dispatcher,
-         AccessLog::AccessLogManager& log_manager,
-         ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api);
+         Upstream::Cluster& cluster, Server::Configuration::ServerFactoryContext& server_context);
 };
 
 /**
diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc
index 3a3db1cdb6eb..bb6d798ca682 100644
--- a/source/common/upstream/health_discovery_service.cc
+++ b/source/common/upstream/health_discovery_service.cc
@@ -436,10 +436,8 @@ absl::Status HdsCluster::updateHealthchecks(
       health_checkers.push_back(health_checker->second);
     } else {
       // If it does not, create a new one.
-      auto checker_or_error = Upstream::HealthCheckerFactory::create(
-          health_check, *this, server_context_.runtime(), server_context_.mainThreadDispatcher(),
-          server_context_.accessLogManager(), server_context_.messageValidationVisitor(),
-          server_context_.api());
+      auto checker_or_error =
+          Upstream::HealthCheckerFactory::create(health_check, *this, server_context_);
       RETURN_IF_STATUS_NOT_OK(checker_or_error);
       auto new_health_checker = checker_or_error.value();
       health_checkers_map.insert({health_check, new_health_checker});
@@ -551,10 +549,8 @@ ProdClusterInfoFactory::createClusterInfo(const CreateClusterInfoParams& params)
 
 void HdsCluster::initHealthchecks() {
   for (auto& health_check : cluster_.health_checks()) {
-    auto health_checker_or_error = Upstream::HealthCheckerFactory::create(
-        health_check, *this, server_context_.runtime(), server_context_.mainThreadDispatcher(),
-        server_context_.accessLogManager(), server_context_.messageValidationVisitor(),
-        server_context_.api());
+    auto health_checker_or_error =
+        Upstream::HealthCheckerFactory::create(health_check, *this, server_context_);
     THROW_IF_STATUS_NOT_OK(health_checker_or_error, throw);
 
     auto health_checker = health_checker_or_error.value();
diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD
index 341abe8ab9f0..94251a06e891 100644
--- a/test/common/upstream/BUILD
+++ b/test/common/upstream/BUILD
@@ -189,11 +189,11 @@ envoy_cc_test(
         "//test/common/http:common_lib",
         "//test/mocks:common_lib",
         "//test/mocks/access_log:access_log_mocks",
-        "//test/mocks/api:api_mocks",
         "//test/mocks/http:http_mocks",
         "//test/mocks/network:network_mocks",
         "//test/mocks/protobuf:protobuf_mocks",
         "//test/mocks/runtime:runtime_mocks",
+        "//test/mocks/server:factory_context_mocks",
         "//test/mocks/upstream:cluster_info_mocks",
         "//test/mocks/upstream:cluster_priority_set_mocks",
         "//test/mocks/upstream:health_check_event_logger_mocks",
diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc
index 24d1bd48e124..b27eccbea538 100644
--- a/test/common/upstream/health_checker_impl_test.cc
+++ b/test/common/upstream/health_checker_impl_test.cc
@@ -28,12 +28,12 @@
 #include "test/common/http/common.h"
 #include "test/common/upstream/utility.h"
 #include "test/mocks/access_log/mocks.h"
-#include "test/mocks/api/mocks.h"
 #include "test/mocks/common.h"
 #include "test/mocks/http/mocks.h"
 #include "test/mocks/network/mocks.h"
 #include "test/mocks/protobuf/mocks.h"
 #include "test/mocks/runtime/mocks.h"
+#include "test/mocks/server/factory_context.h"
 #include "test/mocks/upstream/cluster_info.h"
 #include "test/mocks/upstream/cluster_priority_set.h"
 #include "test/mocks/upstream/health_check_event_logger.h"
@@ -76,13 +76,9 @@ TEST(HealthCheckerFactoryTest, GrpcHealthCheckHTTP2NotConfiguredException) {
   EXPECT_CALL(*cluster.info_, features()).WillRepeatedly(Return(0));
 
   Runtime::MockLoader runtime;
-  Event::MockDispatcher dispatcher;
-  AccessLog::MockAccessLogManager log_manager;
-  NiceMock validation_visitor;
-  Api::MockApi api;
+  NiceMock server_context;
 
-  EXPECT_EQ(HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, runtime,
-                                         dispatcher, log_manager, validation_visitor, api)
+  EXPECT_EQ(HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, server_context)
                 .status()
                 .message(),
             "fake_cluster cluster must support HTTP/2 for gRPC healthchecking");
@@ -94,16 +90,11 @@ TEST(HealthCheckerFactoryTest, CreateGrpc) {
   EXPECT_CALL(*cluster.info_, features())
       .WillRepeatedly(Return(Upstream::ClusterInfo::Features::HTTP2));
 
-  Runtime::MockLoader runtime;
-  Event::MockDispatcher dispatcher;
-  AccessLog::MockAccessLogManager log_manager;
-  NiceMock validation_visitor;
-  NiceMock api;
+  NiceMock server_context;
 
   EXPECT_NE(nullptr,
             dynamic_cast(
-                HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, runtime,
-                                             dispatcher, log_manager, validation_visitor, api)
+                HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, server_context)
                     .value()
                     .get()));
 }
@@ -6390,20 +6381,16 @@ TEST(Printer, HealthTransitionPrinter) {
 }
 
 TEST(HealthCheckEventLoggerImplTest, All) {
-  AccessLog::MockAccessLogManager log_manager;
-  std::shared_ptr file(new AccessLog::MockAccessLogFile());
-  EXPECT_CALL(log_manager, createAccessLog(Filesystem::FilePathAndType{
-                               Filesystem::DestinationType::File, "foo"}))
-      .WillOnce(Return(file));
-
   envoy::config::core::v3::HealthCheck health_check_config;
   health_check_config.set_event_log_path("foo");
 
   NiceMock cluster;
-  Runtime::MockLoader runtime;
-  Event::MockDispatcher dispatcher;
-  NiceMock validation_visitor;
-  NiceMock api;
+  NiceMock server_context;
+
+  std::shared_ptr file(new AccessLog::MockAccessLogFile());
+  EXPECT_CALL(server_context.access_log_manager_, createAccessLog(Filesystem::FilePathAndType{
+                                                      Filesystem::DestinationType::File, "foo"}))
+      .WillOnce(Return(file));
 
   std::shared_ptr host(new NiceMock());
   NiceMock cluster_info;
@@ -6411,14 +6398,13 @@ TEST(HealthCheckEventLoggerImplTest, All) {
   ON_CALL(*host, cluster()).WillByDefault(ReturnRef(cluster_info));
   ON_CALL(*host, metadata()).WillByDefault(Return(metadata));
 
-  HealthCheckerFactoryContextImpl context(cluster, runtime, dispatcher, validation_visitor, api,
-                                          log_manager);
+  HealthCheckerFactoryContextImpl context(cluster, server_context);
 
   Event::SimulatedTimeSystem time_system;
   // This is rendered as "2009-02-13T23:31:31.234Z".a
   time_system.setSystemTime(std::chrono::milliseconds(1234567891234));
 
-  HealthCheckEventLoggerImpl event_logger(log_manager, time_system, health_check_config, context);
+  HealthCheckEventLoggerImpl event_logger(health_check_config, context);
 
   EXPECT_CALL(*file, write(absl::string_view{
                          "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{"
@@ -6480,15 +6466,12 @@ TEST(HealthCheckEventLoggerImplTest, OneEventLogger) {
   config.set_event_log_path("foo");
   event_log->mutable_typed_config()->PackFrom(config);
 
-  NiceMock log_manager;
+  NiceMock server_context;
   StringViewSaver file_log_data;
-  ON_CALL(*log_manager.file_, write(_)).WillByDefault(SaveArg<0>(&file_log_data));
+  ON_CALL(*server_context.access_log_manager_.file_, write(_))
+      .WillByDefault(SaveArg<0>(&file_log_data));
 
   NiceMock cluster;
-  Runtime::MockLoader runtime;
-  Event::MockDispatcher dispatcher;
-  NiceMock validation_visitor;
-  NiceMock api;
 
   std::shared_ptr host(new NiceMock());
   NiceMock cluster_info;
@@ -6496,14 +6479,13 @@ TEST(HealthCheckEventLoggerImplTest, OneEventLogger) {
   ON_CALL(*host, cluster()).WillByDefault(ReturnRef(cluster_info));
   ON_CALL(*host, metadata()).WillByDefault(Return(metadata));
 
-  HealthCheckerFactoryContextImpl context(cluster, runtime, dispatcher, validation_visitor, api,
-                                          log_manager);
+  HealthCheckerFactoryContextImpl context(cluster, server_context);
 
   Event::SimulatedTimeSystem time_system;
   // This is rendered as "2009-02-13T23:31:31.234Z".a
   time_system.setSystemTime(std::chrono::milliseconds(1234567891234));
 
-  HealthCheckEventLoggerImpl event_logger(log_manager, time_system, health_check_config, context);
+  HealthCheckEventLoggerImpl event_logger(health_check_config, context);
 
   event_logger.logEjectUnhealthy(envoy::data::core::v3::HTTP, host, envoy::data::core::v3::ACTIVE);
   EXPECT_EQ(file_log_data, "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{"
diff --git a/test/extensions/health_checkers/redis/BUILD b/test/extensions/health_checkers/redis/BUILD
index b9430c4e2fe1..cc2d951494e6 100644
--- a/test/extensions/health_checkers/redis/BUILD
+++ b/test/extensions/health_checkers/redis/BUILD
@@ -44,13 +44,10 @@ envoy_extension_cc_test(
         "//source/common/upstream:health_checker_lib",
         "//source/extensions/health_checkers/redis:config",
         "//test/common/upstream:utility_lib",
-        "//test/mocks/access_log:access_log_mocks",
-        "//test/mocks/network:network_mocks",
-        "//test/mocks/runtime:runtime_mocks",
+        "//test/mocks/server:factory_context_mocks",
         "//test/mocks/server:health_checker_factory_context_mocks",
         "//test/mocks/upstream:health_checker_mocks",
         "//test/mocks/upstream:priority_set_mocks",
-        "//test/test_common:test_runtime_lib",
         "@envoy_api//envoy/extensions/filters/network/redis_proxy/v3:pkg_cc_proto",
     ],
 )
diff --git a/test/extensions/health_checkers/redis/config_test.cc b/test/extensions/health_checkers/redis/config_test.cc
index a183dbde4582..f5b7610e5018 100644
--- a/test/extensions/health_checkers/redis/config_test.cc
+++ b/test/extensions/health_checkers/redis/config_test.cc
@@ -5,13 +5,10 @@
 #include "source/extensions/health_checkers/redis/config.h"
 
 #include "test/common/upstream/utility.h"
-#include "test/mocks/access_log/mocks.h"
-#include "test/mocks/network/mocks.h"
-#include "test/mocks/runtime/mocks.h"
+#include "test/mocks/server/factory_context.h"
 #include "test/mocks/server/health_checker_factory_context.h"
 #include "test/mocks/upstream/health_checker.h"
 #include "test/mocks/upstream/priority_set.h"
-#include "test/test_common/test_runtime.h"
 
 namespace Envoy {
 namespace Extensions {
@@ -111,19 +108,13 @@ TEST(HealthCheckerFactoryTest, CreateRedisViaUpstreamHealthCheckerFactory) {
     )EOF";
 
   NiceMock cluster;
-  Runtime::MockLoader runtime;
-  Random::MockRandomGenerator random;
-  Event::MockDispatcher dispatcher;
-  AccessLog::MockAccessLogManager log_manager;
-  NiceMock api;
-
-  EXPECT_NE(nullptr,
-            dynamic_cast(
-                Upstream::HealthCheckerFactory::create(
-                    Upstream::parseHealthCheckFromV3Yaml(yaml), cluster, runtime, dispatcher,
-                    log_manager, ProtobufMessage::getStrictValidationVisitor(), api)
-                    .value()
-                    .get()));
+  NiceMock server_context;
+
+  EXPECT_NE(nullptr, dynamic_cast(
+                         Upstream::HealthCheckerFactory::create(
+                             Upstream::parseHealthCheckFromV3Yaml(yaml), cluster, server_context)
+                             .value()
+                             .get()));
 }
 } // namespace
 } // namespace RedisHealthChecker
diff --git a/test/extensions/health_checkers/thrift/BUILD b/test/extensions/health_checkers/thrift/BUILD
index 6917babae997..05deaadae37c 100644
--- a/test/extensions/health_checkers/thrift/BUILD
+++ b/test/extensions/health_checkers/thrift/BUILD
@@ -73,12 +73,9 @@ envoy_extension_cc_test(
         "//source/common/upstream:health_checker_lib",
         "//source/extensions/health_checkers/thrift:config",
         "//test/common/upstream:utility_lib",
-        "//test/mocks/access_log:access_log_mocks",
-        "//test/mocks/network:network_mocks",
-        "//test/mocks/runtime:runtime_mocks",
+        "//test/mocks/server:factory_context_mocks",
         "//test/mocks/server:health_checker_factory_context_mocks",
         "//test/mocks/upstream:health_checker_mocks",
         "//test/mocks/upstream:priority_set_mocks",
-        "//test/test_common:test_runtime_lib",
     ],
 )
diff --git a/test/extensions/health_checkers/thrift/config_test.cc b/test/extensions/health_checkers/thrift/config_test.cc
index 924610631f92..280ab53339ab 100644
--- a/test/extensions/health_checkers/thrift/config_test.cc
+++ b/test/extensions/health_checkers/thrift/config_test.cc
@@ -4,13 +4,10 @@
 #include "source/extensions/health_checkers/thrift/config.h"
 
 #include "test/common/upstream/utility.h"
-#include "test/mocks/access_log/mocks.h"
-#include "test/mocks/network/mocks.h"
-#include "test/mocks/runtime/mocks.h"
+#include "test/mocks/server/factory_context.h"
 #include "test/mocks/server/health_checker_factory_context.h"
 #include "test/mocks/upstream/health_checker.h"
 #include "test/mocks/upstream/priority_set.h"
-#include "test/test_common/test_runtime.h"
 
 namespace Envoy {
 namespace Extensions {
@@ -185,19 +182,13 @@ TEST(HealthCheckerFactoryTest, CreateThriftViaUpstreamHealthCheckerFactory) {
     )EOF";
 
   NiceMock cluster;
-  Runtime::MockLoader runtime;
-  Random::MockRandomGenerator random;
-  Event::MockDispatcher dispatcher;
-  AccessLog::MockAccessLogManager log_manager;
-  NiceMock api;
-
-  EXPECT_NE(nullptr,
-            dynamic_cast(
-                Upstream::HealthCheckerFactory::create(
-                    Upstream::parseHealthCheckFromV3Yaml(yaml), cluster, runtime, dispatcher,
-                    log_manager, ProtobufMessage::getStrictValidationVisitor(), api)
-                    .value()
-                    .get()));
+  NiceMock server_context;
+
+  EXPECT_NE(nullptr, dynamic_cast(
+                         Upstream::HealthCheckerFactory::create(
+                             Upstream::parseHealthCheckFromV3Yaml(yaml), cluster, server_context)
+                             .value()
+                             .get()));
 }
 
 } // namespace
diff --git a/test/mocks/server/BUILD b/test/mocks/server/BUILD
index 8ebe31e1ec73..f0a0deb36052 100644
--- a/test/mocks/server/BUILD
+++ b/test/mocks/server/BUILD
@@ -281,6 +281,7 @@ envoy_cc_mock(
         "//test/mocks/protobuf:protobuf_mocks",
         "//test/mocks/router:router_mocks",
         "//test/mocks/runtime:runtime_mocks",
+        "//test/mocks/server:factory_context_mocks",
         "//test/mocks/upstream:cluster_priority_set_mocks",
         "//test/mocks/upstream:health_check_event_logger_mocks",
         "//test/mocks/upstream:health_checker_mocks",
diff --git a/test/mocks/server/health_checker_factory_context.cc b/test/mocks/server/health_checker_factory_context.cc
index f8a3ed362352..4826e4405e68 100644
--- a/test/mocks/server/health_checker_factory_context.cc
+++ b/test/mocks/server/health_checker_factory_context.cc
@@ -20,6 +20,7 @@ MockHealthCheckerFactoryContext::MockHealthCheckerFactoryContext() {
       .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor()));
   ON_CALL(*this, api()).WillByDefault(ReturnRef(api_));
   ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_));
+  ON_CALL(*this, serverFactoryContext()).WillByDefault(ReturnRef(server_context_));
 }
 
 MockHealthCheckerFactoryContext::~MockHealthCheckerFactoryContext() = default;
diff --git a/test/mocks/server/health_checker_factory_context.h b/test/mocks/server/health_checker_factory_context.h
index 85d8af23b9bd..5fad6da1259b 100644
--- a/test/mocks/server/health_checker_factory_context.h
+++ b/test/mocks/server/health_checker_factory_context.h
@@ -9,6 +9,7 @@
 #include "test/mocks/protobuf/mocks.h"
 #include "test/mocks/router/mocks.h"
 #include "test/mocks/runtime/mocks.h"
+#include "test/mocks/server/factory_context.h"
 #include "test/mocks/upstream/cluster_priority_set.h"
 #include "test/mocks/upstream/health_check_event_logger.h"
 #include "test/mocks/upstream/health_checker.h"
@@ -32,6 +33,7 @@ class MockHealthCheckerFactoryContext : public virtual HealthCheckerFactoryConte
   MOCK_METHOD(Api::Api&, api, ());
   MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, ());
   MOCK_METHOD(void, setEventLogger, (Upstream::HealthCheckEventLoggerPtr));
+  MOCK_METHOD(Server::Configuration::ServerFactoryContext&, serverFactoryContext, ());
 
   Upstream::HealthCheckEventLoggerPtr eventLogger() override {
     if (!event_logger_) {
@@ -47,6 +49,7 @@ class MockHealthCheckerFactoryContext : public virtual HealthCheckerFactoryConte
   testing::NiceMock api_{};
   testing::NiceMock access_log_manager_;
   std::unique_ptr> event_logger_;
+  testing::NiceMock server_context_;
 };
 
 } // namespace Configuration

From 46f5af2948eab4280814a24330bff495017e3d0c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:29:15 +0100
Subject: [PATCH 562/740] build(deps): bump yapf from 0.33.0 to 0.40.0 in
 /tools/base (#27952)

Bumps [yapf](https://github.com/google/yapf) from 0.33.0 to 0.40.0.
- [Changelog](https://github.com/google/yapf/blob/main/CHANGELOG)
- [Commits](https://github.com/google/yapf/commits)

---
updated-dependencies:
- dependency-name: yapf
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tools/base/requirements.txt | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt
index 689b131a2d73..b7b24775a20c 100644
--- a/tools/base/requirements.txt
+++ b/tools/base/requirements.txt
@@ -774,6 +774,10 @@ imagesize==1.4.1 \
     --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \
     --hash=sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a
     # via sphinx
+importlib-metadata==6.6.0 \
+    --hash=sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed \
+    --hash=sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705
+    # via yapf
 jinja2==3.1.2 \
     --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \
     --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61
@@ -998,6 +1002,10 @@ pep8-naming==0.13.3 \
     --hash=sha256:1705f046dfcd851378aac3be1cd1551c7c1e5ff363bacad707d43007877fa971 \
     --hash=sha256:1a86b8c71a03337c97181917e2b472f0f5e4ccb06844a0d6f0a33522549e7a80
     # via -r requirements.in
+platformdirs==3.5.3 \
+    --hash=sha256:0ade98a4895e87dc51d47151f7d2ec290365a585151d97b4d8d6312ed6132fed \
+    --hash=sha256:e48fabd87db8f3a7df7150a4a5ea22c546ee8bc39bc2473244730d4b56d2cc4e
+    # via yapf
 ply==3.11 \
     --hash=sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3 \
     --hash=sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce
@@ -1395,9 +1403,9 @@ yamllint==1.32.0 \
     --hash=sha256:d01dde008c65de5b235188ab3110bebc59d18e5c65fc8a58267cd211cd9df34a \
     --hash=sha256:d97a66e48da820829d96077d76b8dfbe6c6140f106e558dae87e81ac4e6b30b7
     # via envoy-code-check
-yapf==0.33.0 \
-    --hash=sha256:4c2b59bd5ffe46f3a7da48df87596877189148226ce267c16e8b44240e51578d \
-    --hash=sha256:da62bdfea3df3673553351e6246abed26d9fe6780e548a5af9e70f6d2b4f5b9a
+yapf==0.40.0 \
+    --hash=sha256:63386557876c2686cb29d8c422b2d1e484f29f7a278821f91515b5c5adf569a7 \
+    --hash=sha256:7eeb8c404e386f16e24cbd785103dbc573f51cbb68e65a35f4392e0233f3d7bc
     # via
     #   -r requirements.in
     #   envoy-code-check
@@ -1479,6 +1487,10 @@ yarl==1.9.2 \
     # via
     #   -r requirements.in
     #   aiohttp
+zipp==3.15.0 \
+    --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \
+    --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556
+    # via importlib-metadata
 zstandard==0.21.0 \
     --hash=sha256:0aad6090ac164a9d237d096c8af241b8dcd015524ac6dbec1330092dba151657 \
     --hash=sha256:0bdbe350691dec3078b187b8304e6a9c4d9db3eb2d50ab5b1d748533e746d099 \

From 6036072d7ca298c7f816f71193337d4f743ac9ef Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:30:29 +0100
Subject: [PATCH 563/740] build(deps): bump protobuf from 4.23.2 to 4.23.3 in
 /examples/grpc-bridge/client (#27975)

build(deps): bump protobuf in /examples/grpc-bridge/client

Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 4.23.2 to 4.23.3.
- [Release notes](https://github.com/protocolbuffers/protobuf/releases)
- [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py)
- [Commits](https://github.com/protocolbuffers/protobuf/compare/v4.23.2...v4.23.3)

---
updated-dependencies:
- dependency-name: protobuf
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/grpc-bridge/client/requirements.txt | 28 ++++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt
index 4044bb58fc62..ec9135051317 100644
--- a/examples/grpc-bridge/client/requirements.txt
+++ b/examples/grpc-bridge/client/requirements.txt
@@ -112,20 +112,20 @@ idna==3.2 \
     --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \
     --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3
     # via requests
-protobuf==4.23.2 \
-    --hash=sha256:09310bce43353b46d73ba7e3bca78273b9bc50349509b9698e64d288c6372c2a \
-    --hash=sha256:20874e7ca4436f683b64ebdbee2129a5a2c301579a67d1a7dda2cdf62fb7f5f7 \
-    --hash=sha256:25e3370eda26469b58b602e29dff069cfaae8eaa0ef4550039cc5ef8dc004511 \
-    --hash=sha256:281342ea5eb631c86697e1e048cb7e73b8a4e85f3299a128c116f05f5c668f8f \
-    --hash=sha256:384dd44cb4c43f2ccddd3645389a23ae61aeb8cfa15ca3a0f60e7c3ea09b28b3 \
-    --hash=sha256:54a533b971288af3b9926e53850c7eb186886c0c84e61daa8444385a4720297f \
-    --hash=sha256:6c081863c379bb1741be8f8193e893511312b1d7329b4a75445d1ea9955be69e \
-    --hash=sha256:86df87016d290143c7ce3be3ad52d055714ebaebb57cc659c387e76cfacd81aa \
-    --hash=sha256:8da6070310d634c99c0db7df48f10da495cc283fd9e9234877f0cd182d43ab7f \
-    --hash=sha256:b2cfab63a230b39ae603834718db74ac11e52bccaaf19bf20f5cce1a84cf76df \
-    --hash=sha256:c52cfcbfba8eb791255edd675c1fe6056f723bf832fa67f0442218f8817c076e \
-    --hash=sha256:ce744938406de1e64b91410f473736e815f28c3b71201302612a68bf01517fea \
-    --hash=sha256:efabbbbac1ab519a514579ba9ec52f006c28ae19d97915951f69fa70da2c9e91
+protobuf==4.23.3 \
+    --hash=sha256:0149053336a466e3e0b040e54d0b615fc71de86da66791c592cc3c8d18150bf8 \
+    --hash=sha256:08fe19d267608d438aa37019236db02b306e33f6b9902c3163838b8e75970223 \
+    --hash=sha256:29660574cd769f2324a57fb78127cda59327eb6664381ecfe1c69731b83e8288 \
+    --hash=sha256:2991f5e7690dab569f8f81702e6700e7364cc3b5e572725098215d3da5ccc6ac \
+    --hash=sha256:3b01a5274ac920feb75d0b372d901524f7e3ad39c63b1a2d55043f3887afe0c1 \
+    --hash=sha256:3bcbeb2bf4bb61fe960dd6e005801a23a43578200ea8ceb726d1f6bd0e562ba1 \
+    --hash=sha256:447b9786ac8e50ae72cae7a2eec5c5df6a9dbf9aa6f908f1b8bda6032644ea62 \
+    --hash=sha256:514b6bbd54a41ca50c86dd5ad6488afe9505901b3557c5e0f7823a0cf67106fb \
+    --hash=sha256:5cb9e41188737f321f4fce9a4337bf40a5414b8d03227e1d9fbc59bc3a216e35 \
+    --hash=sha256:7a92beb30600332a52cdadbedb40d33fd7c8a0d7f549c440347bc606fb3fe34b \
+    --hash=sha256:84ea0bd90c2fdd70ddd9f3d3fc0197cc24ecec1345856c2b5ba70e4d99815359 \
+    --hash=sha256:aca6e86a08c5c5962f55eac9b5bd6fce6ed98645d77e8bfc2b952ecd4a8e4f6a \
+    --hash=sha256:cc14358a8742c4e06b1bfe4be1afbdf5c9f6bd094dff3e14edb78a1513893ff5
     # via
     #   -r requirements.in
     #   grpcio-tools

From 184f8dad8207ffad40a0e12308d822bbc6aa8cda Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:31:47 +0100
Subject: [PATCH 564/740] build(deps): bump golang from `419bc89` to `8e927ec`
 in /examples/shared/golang (#27956)

build(deps): bump golang in /examples/shared/golang

Bumps golang from `419bc89` to `8e927ec`.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/golang/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile
index 3e3ac40744eb..aba3dda1e87b 100644
--- a/examples/shared/golang/Dockerfile
+++ b/examples/shared/golang/Dockerfile
@@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache
 
 
-FROM golang:1.20.5-bullseye@sha256:419bc8954c0e08c539830c8669ccd116a063303481c748fabd09d8fd6d4e2c5f as golang-base
+FROM golang:1.20.5-bullseye@sha256:8e927ecc8ba530bde779a125b95d2d7f0c9c7ef63df3d492d2e99dd599f69ab8 as golang-base
 
 
 FROM golang-base as golang-control-plane-builder

From 755bc031790c15c1c29661278c45a891675e07dd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:32:05 +0100
Subject: [PATCH 565/740] build(deps): bump python from `1966141` to `03cec81`
 in /examples/shared/python (#27931)

build(deps): bump python in /examples/shared/python

Bumps python from `1966141` to `03cec81`.

---
updated-dependencies:
- dependency-name: python
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/python/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/python/Dockerfile b/examples/shared/python/Dockerfile
index e152f2165b9d..cd1cce759148 100644
--- a/examples/shared/python/Dockerfile
+++ b/examples/shared/python/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11.4-slim-bullseye@sha256:1966141ab594e175852a033da2a38f0cb042b5b92896c22073f8477f96f43b06 as python-base
+FROM python:3.11.4-slim-bullseye@sha256:03cec818d5c99fd643011552ad955952fae5c9bf17cbd9fca5980f0b59662a64 as python-base
 RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache
 ARG PYTHON_REQUIREMENTS_FILE=aiohttp/requirements.txt

From 0307b1010250560436d8a1d6b11ef8ce5760a8c0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:33:03 +0100
Subject: [PATCH 566/740] build(deps): bump debian from `7606bef` to `924df86`
 in /examples/shared/golang (#27934)

build(deps): bump debian in /examples/shared/golang

Bumps debian from `7606bef` to `924df86`.

---
updated-dependencies:
- dependency-name: debian
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/golang/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile
index aba3dda1e87b..aeeefbcdf60c 100644
--- a/examples/shared/golang/Dockerfile
+++ b/examples/shared/golang/Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:bullseye-slim@sha256:7606bef5684b393434f06a50a3d1a09808fee5a0240d37da5d181b1b121e7637 as os-base
+FROM debian:bullseye-slim@sha256:924df86f8aad741a0134b2de7d8e70c5c6863f839caadef62609c1be1340daf5 as os-base
 RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache
 

From 28f0ff0af6867da44d449d6797a7aa0a577b2e9d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:33:43 +0100
Subject: [PATCH 567/740] build(deps): bump node from `f52e0eb` to `873d0db` in
 /examples/shared/node (#27933)

build(deps): bump node in /examples/shared/node

Bumps node from `f52e0eb` to `873d0db`.

---
updated-dependencies:
- dependency-name: node
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/node/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile
index f44be4a0ec27..2f5b1867b05e 100644
--- a/examples/shared/node/Dockerfile
+++ b/examples/shared/node/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:20.3-bullseye-slim@sha256:f52e0eb0f31863051b56d76d191b283c2b49ac084762eddfeb1afb54791b250b as node-base
+FROM node:20.3-bullseye-slim@sha256:873d0db3312a942fd77d99117d2dbfc7e38c8cf51ab3a2157aa98ec5e9197ad8 as node-base
 
 
 FROM node-base as node-http-auth

From 5b3a6fb7926703c96a1e09f4bf62bb5ea100dc78 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:34:28 +0100
Subject: [PATCH 568/740] build(deps): bump postgres from `31c9342` to
 `2642aec` in /examples/shared/postgres (#27929)

build(deps): bump postgres in /examples/shared/postgres

Bumps postgres from `31c9342` to `2642aec`.

---
updated-dependencies:
- dependency-name: postgres
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/postgres/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile
index 523758fea8e5..b4f150cc54f5 100644
--- a/examples/shared/postgres/Dockerfile
+++ b/examples/shared/postgres/Dockerfile
@@ -1,3 +1,3 @@
-FROM postgres:latest@sha256:31c9342603866f29206a06b77c8fed48b3c3f70d710a4be4e8216b134f92d0df
+FROM postgres:latest@sha256:2642aecef7a61b5eafd45138c5b380845db1be3ab9dfaaa5133660341bc203e8
 COPY docker-healthcheck.sh /usr/local/bin/
 HEALTHCHECK CMD ["docker-healthcheck.sh"]

From f963f898f0de8766db553b1e76a7dbecc20684a3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:35:15 +0100
Subject: [PATCH 569/740] build(deps): bump jaegertracing/all-in-one from
 `4f85b75` to `fc197c4` in /examples/shared/jaeger (#27979)

build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger

Bumps jaegertracing/all-in-one from `4f85b75` to `fc197c4`.

---
updated-dependencies:
- dependency-name: jaegertracing/all-in-one
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/jaeger/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile
index 8fdd5df9af5c..e2eeeaf1f0cd 100644
--- a/examples/shared/jaeger/Dockerfile
+++ b/examples/shared/jaeger/Dockerfile
@@ -1,4 +1,4 @@
-FROM jaegertracing/all-in-one@sha256:4f85b75673c0863ef869f3e54d043f842935351ef0856d5ddad1709da4f17354
+FROM jaegertracing/all-in-one@sha256:fc197c49334b8f82cb4b4aba0557d3b0a4364f78867f3abaa19a1e16b4af6019
 HEALTHCHECK \
     --interval=1s \
     --timeout=1s \

From b8230f1a8a25949b522d3c4f88554f1fe9f6248c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:36:13 +0100
Subject: [PATCH 570/740] build(deps): bump mysql from `4bae986` to `15f0692`
 in /examples/mysql (#28006)

Bumps mysql from `4bae986` to `15f0692`.

---
updated-dependencies:
- dependency-name: mysql
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/mysql/Dockerfile-mysql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/mysql/Dockerfile-mysql b/examples/mysql/Dockerfile-mysql
index d1038cf5ca42..3d6b40847586 100644
--- a/examples/mysql/Dockerfile-mysql
+++ b/examples/mysql/Dockerfile-mysql
@@ -1 +1 @@
-FROM mysql:8.0.33@sha256:4bae98614cd6ad1aecbdd32ff1b37b93fb0ee22b069469e7bc9679bacef1abd2
+FROM mysql:8.0.33@sha256:15f069202c46cf861ce429423ae3f8dfa6423306fbf399eaef36094ce30dd75c

From 6e19b400851841c0045a8d0fcbf8df2a88bd29ca Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:36:33 +0100
Subject: [PATCH 571/740] build(deps): bump redis from `f972469` to `6ccde0c`
 in /examples/redis (#28007)

Bumps redis from `f972469` to `6ccde0c`.

---
updated-dependencies:
- dependency-name: redis
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/redis/Dockerfile-redis | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/redis/Dockerfile-redis b/examples/redis/Dockerfile-redis
index aa02190f6263..5651cd91b725 100644
--- a/examples/redis/Dockerfile-redis
+++ b/examples/redis/Dockerfile-redis
@@ -1 +1 @@
-FROM redis@sha256:f9724694a0b97288d2255ff2b69642dfba7f34c8e41aaf0a59d33d10d8a42687
+FROM redis@sha256:6ccde0cb87c24c09b1970802b10e702ab4491ac932b79f69682609787379fa42

From c89cc2ab48769efd726a0db8c8d75d516554a529 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:37:26 +0100
Subject: [PATCH 572/740] build(deps): bump nginx from `af296b1` to `593dac2`
 in /examples/local_ratelimit (#27978)

build(deps): bump nginx in /examples/local_ratelimit

Bumps nginx from `af296b1` to `593dac2`.

---
updated-dependencies:
- dependency-name: nginx
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/local_ratelimit/Dockerfile-nginx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/local_ratelimit/Dockerfile-nginx b/examples/local_ratelimit/Dockerfile-nginx
index d4d5b1681b33..7eaceae2ac3a 100644
--- a/examples/local_ratelimit/Dockerfile-nginx
+++ b/examples/local_ratelimit/Dockerfile-nginx
@@ -1 +1 @@
-FROM nginx@sha256:af296b188c7b7df99ba960ca614439c99cb7cf252ed7bbc23e90cfda59092305
+FROM nginx@sha256:593dac25b7733ffb7afe1a72649a43e574778bf025ad60514ef40f6b5d606247

From 0ee9bf24520413271b8998e88988da47a3800aba Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:38:12 +0100
Subject: [PATCH 573/740] build(deps): bump alpine from `02bb6f4` to `82d1e9d`
 in /examples/opentelemetry (#27980)

build(deps): bump alpine in /examples/opentelemetry

Bumps alpine from `02bb6f4` to `82d1e9d`.

---
updated-dependencies:
- dependency-name: alpine
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/opentelemetry/Dockerfile-opentelemetry | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry
index 38a00b8e466d..50132701f29f 100644
--- a/examples/opentelemetry/Dockerfile-opentelemetry
+++ b/examples/opentelemetry/Dockerfile-opentelemetry
@@ -1,4 +1,4 @@
-FROM alpine:3.18@sha256:02bb6f428431fbc2809c5d1b41eab5a68350194fb508869a33cb1af4444c9b11 as otelc_curl
+FROM alpine:3.18@sha256:82d1e9d7ed48a7523bdebc18cf6290bdb97b82302a8a9c27d4fe885949ea94d1 as otelc_curl
 RUN apk --update add curl
 
 FROM otel/opentelemetry-collector:latest@sha256:324e2c7bdd1ecd58c1a1347737174371b4917d71a05cc4403ce3fb83bcddf836

From 908af8a94fac533e693c396bad6bbbb00cc10b6f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 17 Jun 2023 08:38:50 +0100
Subject: [PATCH 574/740] build(deps): bump google.golang.org/grpc from 1.55.0
 to 1.56.0 in /examples/ext_authz/auth/grpc-service (#28008)

build(deps): bump google.golang.org/grpc

Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.55.0 to 1.56.0.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.55.0...v1.56.0)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/ext_authz/auth/grpc-service/go.mod |  2 +-
 examples/ext_authz/auth/grpc-service/go.sum | 12 ++++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/examples/ext_authz/auth/grpc-service/go.mod b/examples/ext_authz/auth/grpc-service/go.mod
index 8bc653ebc6e5..fbe4b7c6cf53 100644
--- a/examples/ext_authz/auth/grpc-service/go.mod
+++ b/examples/ext_authz/auth/grpc-service/go.mod
@@ -6,5 +6,5 @@ require (
 	github.com/envoyproxy/go-control-plane v0.11.1
 	github.com/golang/protobuf v1.5.3
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e
-	google.golang.org/grpc v1.55.0
+	google.golang.org/grpc v1.56.0
 )
diff --git a/examples/ext_authz/auth/grpc-service/go.sum b/examples/ext_authz/auth/grpc-service/go.sum
index 3eb080a7e70a..9049678cee91 100644
--- a/examples/ext_authz/auth/grpc-service/go.sum
+++ b/examples/ext_authz/auth/grpc-service/go.sum
@@ -171,6 +171,7 @@ cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvj
 cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
 cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs=
 cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU=
+cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE=
 cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
@@ -632,8 +633,9 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74 h1:zlUubfBUxApscKFsF4VSvvfhsBNTBu0eF/ddvpo96yk=
 github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -650,12 +652,14 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34=
 github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
+github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q=
 github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM=
 github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo=
 github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w=
 github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
+github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
 github.com/envoyproxy/protoc-gen-validate v1.0.1 h1:kt9FtLiooDc0vbwTLhdg3dyNX1K9Qwa1EK9LcD4jVUQ=
 github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
@@ -1015,6 +1019,7 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
 golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
+golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1415,6 +1420,7 @@ google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVix
 google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
 google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
 google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
+google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
 google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY=
 google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw=
 google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk=
@@ -1463,10 +1469,12 @@ google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD
 google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
 google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
 google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
+google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
 google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
 google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
-google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
+google.golang.org/grpc v1.56.0 h1:+y7Bs8rtMd07LeXmL3NxcTLn7mUkbKZqEpPhMNkwJEE=
+google.golang.org/grpc v1.56.0/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

From 08042d42a8c263a10575d2384b66d749c89dba26 Mon Sep 17 00:00:00 2001
From: Zhewei Hu 
Date: Sat, 17 Jun 2023 14:51:53 -0700
Subject: [PATCH 575/740] [ZK filter] Fix 'xid not found' decoding error due to
 control requests xid incorrect handling (#27912)

Signed-off-by: Zhewei Hu 
---
 .../network/zookeeper_proxy/decoder.cc        | 75 ++++++++++++++-----
 .../filters/network/zookeeper_proxy/decoder.h | 10 ++-
 .../network/zookeeper_proxy/filter_test.cc    | 56 ++++++++++++++
 3 files changed, 123 insertions(+), 18 deletions(-)

diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.cc b/source/extensions/filters/network/zookeeper_proxy/decoder.cc
index f30830bb1d9c..b54a81724d41 100644
--- a/source/extensions/filters/network/zookeeper_proxy/decoder.cc
+++ b/source/extensions/filters/network/zookeeper_proxy/decoder.cc
@@ -69,21 +69,21 @@ void DecoderImpl::decodeOnData(Buffer::Instance& data, uint64_t& offset) {
   switch (static_cast(xid)) {
   case XidCodes::ConnectXid:
     parseConnect(data, offset, len);
-    requests_by_xid_[xid] = {OpCodes::Connect, std::move(start_time)};
+    control_requests_by_xid_[xid].push({OpCodes::Connect, std::move(start_time)});
     return;
   case XidCodes::PingXid:
     offset += OPCODE_LENGTH;
     callbacks_.onPing();
-    requests_by_xid_[xid] = {OpCodes::Ping, std::move(start_time)};
+    control_requests_by_xid_[xid].push({OpCodes::Ping, std::move(start_time)});
     return;
   case XidCodes::AuthXid:
     parseAuthRequest(data, offset, len);
-    requests_by_xid_[xid] = {OpCodes::SetAuth, std::move(start_time)};
+    control_requests_by_xid_[xid].push({OpCodes::SetAuth, std::move(start_time)});
     return;
   case XidCodes::SetWatchesXid:
     offset += OPCODE_LENGTH;
     parseSetWatchesRequest(data, offset, len);
-    requests_by_xid_[xid] = {OpCodes::SetWatches, std::move(start_time)};
+    control_requests_by_xid_[xid].push({OpCodes::SetWatches, std::move(start_time)});
     return;
   default:
     // WATCH_XID is generated by the server, so that and everything
@@ -191,19 +191,21 @@ void DecoderImpl::decodeOnWrite(Buffer::Instance& data, uint64_t& offset) {
   std::chrono::milliseconds latency;
   OpCodes opcode;
 
-  if (xid_code != XidCodes::WatchXid) {
-    // Find the corresponding request for this XID.
-    const auto it = requests_by_xid_.find(xid);
-
-    // If this fails, it's either a server-side bug or a malformed packet.
-    if (it == requests_by_xid_.end()) {
-      throw EnvoyException("xid not found");
-    }
-
-    latency = std::chrono::duration_cast(time_source_.monotonicTime() -
-                                                                    it->second.start_time);
-    opcode = it->second.opcode;
-    requests_by_xid_.erase(it);
+  switch (xid_code) {
+  case XidCodes::ConnectXid:
+    ABSL_FALLTHROUGH_INTENDED;
+  case XidCodes::PingXid:
+    ABSL_FALLTHROUGH_INTENDED;
+  case XidCodes::AuthXid:
+    ABSL_FALLTHROUGH_INTENDED;
+  case XidCodes::SetWatchesXid:
+    latency = fetchControlRequestData(xid, opcode);
+    break;
+  case XidCodes::WatchXid:
+    // WATCH_XID is generated by the server, no need to fetch opcode and latency here.
+    break;
+  default:
+    latency = fetchDataRequestData(xid, opcode);
   }
 
   // Connect responses are special, they have no full reply header
@@ -663,6 +665,45 @@ bool DecoderImpl::maybeReadBool(Buffer::Instance& data, uint64_t& offset) {
   return false;
 }
 
+std::chrono::milliseconds DecoderImpl::fetchControlRequestData(const int32_t xid, OpCodes& opcode) {
+  // Find the corresponding request queue for this XID.
+  const auto it = control_requests_by_xid_.find(xid);
+
+  // If this fails, it's either a server-side bug or a malformed packet.
+  if (it == control_requests_by_xid_.end()) {
+    throw EnvoyException(fmt::format("control request xid {} not found", xid));
+  }
+
+  std::queue& rq_queue = it->second;
+  if (rq_queue.empty()) {
+    throw EnvoyException(fmt::format("control request queue for {} is empty", xid));
+  }
+
+  std::chrono::milliseconds latency = std::chrono::duration_cast(
+      time_source_.monotonicTime() - rq_queue.front().start_time);
+  opcode = rq_queue.front().opcode;
+  rq_queue.pop();
+
+  return latency;
+}
+
+std::chrono::milliseconds DecoderImpl::fetchDataRequestData(const int32_t xid, OpCodes& opcode) {
+  // Find the corresponding request for this XID.
+  const auto it = requests_by_xid_.find(xid);
+
+  // If this fails, it's either a server-side bug or a malformed packet.
+  if (it == requests_by_xid_.end()) {
+    throw EnvoyException(fmt::format("xid {} not found", xid));
+  }
+
+  std::chrono::milliseconds latency = std::chrono::duration_cast(
+      time_source_.monotonicTime() - it->second.start_time);
+  opcode = it->second.opcode;
+  requests_by_xid_.erase(it);
+
+  return latency;
+}
+
 } // namespace ZooKeeperProxy
 } // namespace NetworkFilters
 } // namespace Extensions
diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.h b/source/extensions/filters/network/zookeeper_proxy/decoder.h
index 36278932524c..c401a77cd616 100644
--- a/source/extensions/filters/network/zookeeper_proxy/decoder.h
+++ b/source/extensions/filters/network/zookeeper_proxy/decoder.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include 
+#include 
 #include 
 
 #include "envoy/common/platform.h"
@@ -183,12 +184,19 @@ class DecoderImpl : public Decoder, Logger::Loggable {
   void parseWatchEvent(Buffer::Instance& data, uint64_t& offset, uint32_t len, int64_t zxid,
                        int32_t error);
   bool maybeReadBool(Buffer::Instance& data, uint64_t& offset);
+  std::chrono::milliseconds fetchControlRequestData(const int32_t xid, OpCodes& opcode);
+  std::chrono::milliseconds fetchDataRequestData(const int32_t xid, OpCodes& opcode);
 
   DecoderCallbacks& callbacks_;
   const uint32_t max_packet_bytes_;
   BufferHelper helper_;
   TimeSource& time_source_;
-  absl::node_hash_map requests_by_xid_;
+  absl::flat_hash_map requests_by_xid_;
+  // Different from transaction ids of data requests, the transaction ids (XidCodes) of same kind of
+  // control requests are always the same. Therefore, we use a queue for each kind of control
+  // request, so we can differentiate different control requests with the same xid and calculate
+  // their response latency.
+  absl::flat_hash_map> control_requests_by_xid_;
   Buffer::OwnedImpl zk_filter_read_buffer_;
   Buffer::OwnedImpl zk_filter_write_buffer_;
 };
diff --git a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc
index b619800fc55b..c442584266b1 100644
--- a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc
+++ b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc
@@ -1381,6 +1381,62 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithOneOnDataCall) {
                config_->stats().create_resp_slow_, 1001, 2001, 2);
 }
 
+// |REQ1|REQ2|
+// (onData1  )
+TEST_F(ZooKeeperFilterTest, MultipleControlRequestsWithOneOnDataCall) {
+  initialize();
+
+  // Request (onData1).
+  Buffer::OwnedImpl data = encodeAuth("digest");
+  data.add(encodeAuth("digest"));
+
+  EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false));
+  EXPECT_EQ(2UL, store_.counter("test.zookeeper.auth.digest_rq").value());
+  EXPECT_EQ(72UL, config_->stats().request_bytes_.value());
+  EXPECT_EQ(0UL, config_->stats().decoder_error_.value());
+
+  // Responses.
+  testResponse({{{"opname", "auth_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}},
+               config_->stats().auth_resp_, config_->stats().auth_resp_fast_,
+               config_->stats().auth_resp_slow_, enumToSignedInt(XidCodes::AuthXid), 2000);
+  testResponse({{{"opname", "auth_response"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}},
+               config_->stats().auth_resp_, config_->stats().auth_resp_fast_,
+               config_->stats().auth_resp_slow_, enumToSignedInt(XidCodes::AuthXid), 2001, 2);
+}
+
+// |REQ1|REQ2|
+// (onData1  )
+TEST_F(ZooKeeperFilterTest, MixedControlAndDataRequestsWithOneOnDataCall) {
+  initialize();
+
+  // Request (onData1).
+  Buffer::OwnedImpl rq_data = encodeAuth("digest");
+  rq_data.add(encodeCreateRequest("/foo", "bar", CreateFlags::Persistent, false, 1000,
+                                  enumToSignedInt(OpCodes::Create)));
+
+  EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(rq_data, false));
+  EXPECT_EQ(1UL, store_.counter("test.zookeeper.auth.digest_rq").value());
+  EXPECT_EQ(1UL, config_->stats().create_rq_.value());
+  EXPECT_EQ(71UL, config_->stats().request_bytes_.value());
+  EXPECT_EQ(0UL, config_->stats().decoder_error_.value());
+
+  // Responses.
+  testResponse({{{"opname", "auth_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}},
+               config_->stats().auth_resp_, config_->stats().auth_resp_fast_,
+               config_->stats().auth_resp_slow_, enumToSignedInt(XidCodes::AuthXid), 2000);
+
+  Buffer::OwnedImpl resp_data = encodeResponseHeader(1000, 2001, 0);
+  expectSetDynamicMetadata(
+      {{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}});
+  EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(resp_data, false));
+  EXPECT_EQ(1UL, config_->stats().create_resp_.value());
+  EXPECT_EQ(1UL, config_->stats().create_resp_fast_.value());
+  EXPECT_EQ(0UL, config_->stats().create_resp_slow_.value());
+  EXPECT_EQ(40UL, config_->stats().response_bytes_.value());
+  EXPECT_EQ(0UL, config_->stats().decoder_error_.value());
+  EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.create_resp_latency"));
+}
+
 // |REQ1 -------|REQ2 ---------|
 // (onData1)(onData2  )(onData3)
 TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls) {

From 5253a42be7b396cc100a65d026a79eb8497b9925 Mon Sep 17 00:00:00 2001
From: deveshkandpal1224 <100540598+deveshkandpal1224@users.noreply.github.com>
Date: Sat, 17 Jun 2023 21:48:51 -0400
Subject: [PATCH 576/740] redis_proxy: add formatter support for
 catch_all_route (#27902)

Signed-off-by: deveshkandpal1224 
---
 changelogs/current.yaml                       |  5 ++
 .../network/redis_proxy/router_impl.cc        | 19 +++----
 .../filters/network/redis_proxy/router_impl.h |  2 +-
 .../network/redis_proxy/router_impl_test.cc   | 53 ++++++++++++++-----
 4 files changed, 55 insertions(+), 24 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index c2ce3ad46ef8..5ba5d1445041 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -160,6 +160,11 @@ bug_fixes:
     - `CVE-2023-31147 `_.
     - `CVE-2023-31124 `_.
     - `CVE-2023-32067 `_.
+- area: redis_proxy
+  change: |
+    Fixes a bug where route properties such as key_formatter,
+    prefix and remove_prefix do not take effect when configured for :ref:`catch_all_route
+    `.
 
 removed_config_or_runtime:
 # *Normally occurs at the end of the* :ref:`deprecation period `
diff --git a/source/extensions/filters/network/redis_proxy/router_impl.cc b/source/extensions/filters/network/redis_proxy/router_impl.cc
index 87f4770f8927..4ea7b9e37ee6 100644
--- a/source/extensions/filters/network/redis_proxy/router_impl.cc
+++ b/source/extensions/filters/network/redis_proxy/router_impl.cc
@@ -87,17 +87,18 @@ RouteSharedPtr PrefixRoutes::upstreamPool(std::string& key) {
     value = prefix_lookup_table_.findLongestPrefix(key.c_str());
   }
 
-  if (value != nullptr) {
-    if (value->removePrefix()) {
-      key.erase(0, value->prefix().length());
-    }
-    if (!value->keyFormatter().empty()) {
-      formatKey(key, value->keyFormatter());
-    }
-    return value;
+  if (value == nullptr) {
+    // prefix route not found, default to catch all route.
+    value = catch_all_route_;
   }
 
-  return catch_all_route_;
+  if (value->removePrefix()) {
+    key.erase(0, value->prefix().length());
+  }
+  if (!value->keyFormatter().empty()) {
+    formatKey(key, value->keyFormatter());
+  }
+  return value;
 }
 
 void PrefixRoutes::formatKey(std::string& key, std::string redis_key_formatter) {
diff --git a/source/extensions/filters/network/redis_proxy/router_impl.h b/source/extensions/filters/network/redis_proxy/router_impl.h
index 643644bfe25a..45f9463412a6 100644
--- a/source/extensions/filters/network/redis_proxy/router_impl.h
+++ b/source/extensions/filters/network/redis_proxy/router_impl.h
@@ -87,7 +87,7 @@ class PrefixRoutes : public Router {
   TrieLookupTable prefix_lookup_table_;
   const bool case_insensitive_;
   Upstreams upstreams_;
-  RouteSharedPtr catch_all_route_;
+  PrefixSharedPtr catch_all_route_;
   Network::ReadFilterCallbacks* callbacks_{};
   const std::string redis_key_formatter_command_ = "%KEY%";
 };
diff --git a/test/extensions/filters/network/redis_proxy/router_impl_test.cc b/test/extensions/filters/network/redis_proxy/router_impl_test.cc
index 1ea598c3a48f..961ebc6bf9d5 100644
--- a/test/extensions/filters/network/redis_proxy/router_impl_test.cc
+++ b/test/extensions/filters/network/redis_proxy/router_impl_test.cc
@@ -49,19 +49,6 @@ createPrefixRoutes() {
   return prefix_routes;
 }
 
-TEST(PrefixRoutesTest, MissingCatchAll) {
-  Upstreams upstreams;
-  upstreams.emplace("fake_clusterA", std::make_shared());
-  upstreams.emplace("fake_clusterB", std::make_shared());
-
-  Runtime::MockLoader runtime_;
-
-  PrefixRoutes router(createPrefixRoutes(), std::move(upstreams), runtime_);
-
-  std::string key("c:bar");
-  EXPECT_EQ(nullptr, router.upstreamPool(key));
-}
-
 TEST(PrefixRoutesTest, RoutedToCatchAll) {
   auto upstream_c = std::make_shared();
 
@@ -96,7 +83,45 @@ TEST(PrefixRoutesTest, RoutedToLongestPrefix) {
   EXPECT_EQ(upstream_a, router.upstreamPool(key)->upstream());
 }
 
-TEST(PrefixRoutesTest, TestKeyPrefixFormatter) {
+TEST(PrefixRoutesTest, TestFormatterWithCatchAllRoute) {
+  auto upstream_catch_all = std::make_shared();
+  NiceMock filter_callbacks;
+  NiceMock connection;
+  NiceMock stream_info;
+  TestEnvironment::setEnvVar("ENVOY_TEST_ENV", "catchAllEnv", 1);
+  Envoy::Cleanup cleanup([]() { TestEnvironment::unsetEnvVar("ENVOY_TEST_ENV"); });
+  const std::string format =
+      "{%KEY%}-%ENVIRONMENT(ENVOY_TEST_ENV)%-%FILTER_STATE(redisKey)%-{%KEY%}";
+
+  stream_info.filterState()->setData(
+      "redisKey", std::make_unique("subjectCN"),
+      StreamInfo::FilterState::StateType::ReadOnly);
+
+  ON_CALL(filter_callbacks, connection()).WillByDefault(ReturnRef(connection));
+  ON_CALL(connection, streamInfo()).WillByDefault(ReturnRef(stream_info));
+
+  Upstreams upstreams;
+  upstreams.emplace("fake_clusterA", std::make_shared());
+  upstreams.emplace("fake_clusterB", std::make_shared());
+  upstreams.emplace("fake_catchAllCluster", upstream_catch_all);
+
+  Runtime::MockLoader runtime_;
+  auto prefix_routes = createPrefixRoutes();
+  {
+    auto* route = prefix_routes.mutable_catch_all_route();
+    route->set_cluster("fake_catchAllCluster");
+    route->set_remove_prefix(true);
+    route->set_prefix("removeMe_");
+    route->set_key_formatter(format);
+  }
+  PrefixRoutes router(prefix_routes, std::move(upstreams), runtime_);
+  router.setReadFilterCallback(&filter_callbacks);
+  std::string key("removeMe_catchAllKey");
+  EXPECT_EQ(upstream_catch_all, router.upstreamPool(key)->upstream());
+  EXPECT_EQ("{catchAllKey}-catchAllEnv-subjectCN-{catchAllKey}", key);
+}
+
+TEST(PrefixRoutesTest, TestFormatterWithPrefixRoute) {
   auto upstream_c = std::make_shared();
   NiceMock filter_callbacks;
   NiceMock connection;

From 652d23db03fcf90ecd36a24293fc02dc3a2a07bd Mon Sep 17 00:00:00 2001
From: Pradeep Rao <84025829+pradeepcrao@users.noreply.github.com>
Date: Sat, 17 Jun 2023 21:49:14 -0400
Subject: [PATCH 577/740] Happy Eyeballs for EDS clusters (#27881)

Signed-off-by: pcrao 
---
 .../endpoint/v3/endpoint_components.proto     | 12 +++++
 changelogs/current.yaml                       |  4 ++
 .../upstream/connection_pooling.rst           | 13 +++--
 source/common/upstream/upstream_impl.cc       |  4 ++
 source/common/upstream/upstream_impl.h        |  3 ++
 source/extensions/clusters/eds/eds.cc         | 10 +++-
 .../clusters/static/static_cluster.cc         |  2 +-
 test/common/upstream/upstream_impl_test.cc    |  4 +-
 test/extensions/clusters/eds/BUILD            |  1 +
 test/extensions/clusters/eds/eds_test.cc      | 53 ++++++++++++++++++-
 10 files changed, 98 insertions(+), 8 deletions(-)

diff --git a/api/envoy/config/endpoint/v3/endpoint_components.proto b/api/envoy/config/endpoint/v3/endpoint_components.proto
index 247f2a501935..62598f5d0f31 100644
--- a/api/envoy/config/endpoint/v3/endpoint_components.proto
+++ b/api/envoy/config/endpoint/v3/endpoint_components.proto
@@ -57,6 +57,11 @@ message Endpoint {
     bool disable_active_health_check = 4;
   }
 
+  message AdditionalAddress {
+    // Additional address that is associated with the endpoint.
+    core.v3.Address address = 1;
+  }
+
   // The upstream host address.
   //
   // .. attention::
@@ -82,6 +87,13 @@ message Endpoint {
   // that require a hostname, like
   // :ref:`auto_host_rewrite `.
   string hostname = 3;
+
+  // An ordered list of addresses that together with `address` comprise the
+  // list of addresses for an endpoint. The address given in the `address` is
+  // prepended to this list. It is assumed that the list must already be
+  // sorted by preference order of the addresses. This will only be supported
+  // for STATIC and EDS clusters.
+  repeated AdditionalAddress additional_addresses = 4;
 }
 
 // An Endpoint that Envoy can route traffic to.
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 5ba5d1445041..9db57a81bcfa 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -94,6 +94,10 @@ minor_behavior_changes:
   change: |
     Added new type of gauge with type hidden. These stats are hidden from admin/stats-sinks but can shown with a
     query-parameter of ``/stats?hidden=include`` or ``/stats?hidden=showonly``.
+- area: eds
+  change: |
+    Added the ability to specify mulitple addresses for a host in an EDS cluster. Connections to the host with more than one
+    address will be established using the Happy Eyeballs algorithm.
 - area: upstream
   change: |
     Changed behavior of the unpausing connect with 2xx status codes. This change can be reverted temporarily by
diff --git a/docs/root/intro/arch_overview/upstream/connection_pooling.rst b/docs/root/intro/arch_overview/upstream/connection_pooling.rst
index ad2a816aac53..f66061663656 100644
--- a/docs/root/intro/arch_overview/upstream/connection_pooling.rst
+++ b/docs/root/intro/arch_overview/upstream/connection_pooling.rst
@@ -86,13 +86,20 @@ production burn time, but is considered ready for use.
 Happy Eyeballs Support
 ----------------------
 
-Envoy supports Happy Eyeballs, `RFC6555 `_,
-for upstream TCP connections.  For clusters which use
+Envoy supports Happy Eyeballs, `RFC8305 `_,
+for upstream TCP connections. For clusters which use
 :ref:`LOGICAL_DNS`,
 this behavior is configured by setting the DNS IP address resolution policy in
 :ref:`config.cluster.v3.Cluster.DnsLookupFamily `
 to the :ref:`ALL ` option to return
-both IPv4 and IPv6 addresses. The returned addresses will be sorted according the the Happy Eyeballs
+both IPv4 and IPv6 addresses. For clusters which use
+:ref:`EDS`, this behavior is configured
+by specifying additional IP addresses for a host using the
+:ref:`additional_addresses ` field.
+The addresses specified in this field will be appended in a list to the one specified in
+:ref:`address `
+
+The list of all addresses will be sorted according the the Happy Eyeballs
 specification and a connection will be attempted to the first in the list. If this connection succeeds,
 it will be used. If it fails, an attempt will be made to the next on the list. If after 300ms the connection
 is still connecting, then a backup connection attempt will be made to the next address on the list.
diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc
index 8041988e0fbf..232e32a807b0 100644
--- a/source/common/upstream/upstream_impl.cc
+++ b/source/common/upstream/upstream_impl.cc
@@ -1854,6 +1854,7 @@ void PriorityStateManager::initializePriorityFor(
 
 void PriorityStateManager::registerHostForPriority(
     const std::string& hostname, Network::Address::InstanceConstSharedPtr address,
+    const std::vector& address_list,
     const envoy::config::endpoint::v3::LocalityLbEndpoints& locality_lb_endpoint,
     const envoy::config::endpoint::v3::LbEndpoint& lb_endpoint, TimeSource& time_source) {
   auto metadata = lb_endpoint.has_metadata()
@@ -1863,6 +1864,9 @@ void PriorityStateManager::registerHostForPriority(
       parent_.info(), hostname, address, metadata, lb_endpoint.load_balancing_weight().value(),
       locality_lb_endpoint.locality(), lb_endpoint.endpoint().health_check_config(),
       locality_lb_endpoint.priority(), lb_endpoint.health_status(), time_source);
+  if (!address_list.empty()) {
+    host->setAddressList(address_list);
+  }
   registerHostForPriority(host, locality_lb_endpoint);
 }
 
diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h
index ee759e33f9cb..276225475197 100644
--- a/source/common/upstream/upstream_impl.h
+++ b/source/common/upstream/upstream_impl.h
@@ -285,6 +285,7 @@ class HostDescriptionImpl : virtual public HostDescription,
 
   void setAddressList(const std::vector& address_list) {
     address_list_ = address_list;
+    ASSERT(address_list_.empty() || *address_list_.front() == *address_);
   }
 
 protected:
@@ -311,6 +312,7 @@ class HostDescriptionImpl : virtual public HostDescription,
   const std::string hostname_;
   const std::string health_checks_hostname_;
   Network::Address::InstanceConstSharedPtr address_;
+  // The first entry in the address_list_ should match the value in address_.
   std::vector address_list_;
   Network::Address::InstanceConstSharedPtr health_check_address_;
   std::atomic canary_;
@@ -1235,6 +1237,7 @@ class PriorityStateManager : protected Logger::Loggable {
   // lb_endpoint health status is unhealthy, draining or timeout.
   void registerHostForPriority(
       const std::string& hostname, Network::Address::InstanceConstSharedPtr address,
+      const std::vector& address_list,
       const envoy::config::endpoint::v3::LocalityLbEndpoints& locality_lb_endpoint,
       const envoy::config::endpoint::v3::LbEndpoint& lb_endpoint, TimeSource& time_source);
 
diff --git a/source/extensions/clusters/eds/eds.cc b/source/extensions/clusters/eds/eds.cc
index d8e480a97106..c9ba84898922 100644
--- a/source/extensions/clusters/eds/eds.cc
+++ b/source/extensions/clusters/eds/eds.cc
@@ -125,6 +125,14 @@ void EdsClusterImpl::BatchUpdateHelper::updateLocalityEndpoints(
     const envoy::config::endpoint::v3::LocalityLbEndpoints& locality_lb_endpoint,
     PriorityStateManager& priority_state_manager, absl::flat_hash_set& all_new_hosts) {
   const auto address = parent_.resolveProtoAddress(lb_endpoint.endpoint().address());
+  std::vector address_list;
+  if (!lb_endpoint.endpoint().additional_addresses().empty()) {
+    address_list.push_back(address);
+    for (const auto& additional_address : lb_endpoint.endpoint().additional_addresses()) {
+      address_list.emplace_back(parent_.resolveProtoAddress(additional_address.address()));
+    }
+  }
+
   // When the configuration contains duplicate hosts, only the first one will be retained.
   const auto address_as_string = address->asString();
   if (all_new_hosts.count(address_as_string) > 0) {
@@ -132,7 +140,7 @@ void EdsClusterImpl::BatchUpdateHelper::updateLocalityEndpoints(
   }
 
   priority_state_manager.registerHostForPriority(lb_endpoint.endpoint().hostname(), address,
-                                                 locality_lb_endpoint, lb_endpoint,
+                                                 address_list, locality_lb_endpoint, lb_endpoint,
                                                  parent_.time_source_);
   all_new_hosts.emplace(address_as_string);
 }
diff --git a/source/extensions/clusters/static/static_cluster.cc b/source/extensions/clusters/static/static_cluster.cc
index 963755a5aacd..025f761c51ce 100644
--- a/source/extensions/clusters/static/static_cluster.cc
+++ b/source/extensions/clusters/static/static_cluster.cc
@@ -31,7 +31,7 @@ StaticClusterImpl::StaticClusterImpl(const envoy::config::cluster::v3::Cluster&
     for (const auto& lb_endpoint : locality_lb_endpoint.lb_endpoints()) {
       priority_state_manager_->registerHostForPriority(
           lb_endpoint.endpoint().hostname(), resolveProtoAddress(lb_endpoint.endpoint().address()),
-          locality_lb_endpoint, lb_endpoint, dispatcher.timeSource());
+          {}, locality_lb_endpoint, lb_endpoint, dispatcher.timeSource());
     }
   }
 }
diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc
index 76c6320909d6..1b63edb01691 100644
--- a/test/common/upstream/upstream_impl_test.cc
+++ b/test/common/upstream/upstream_impl_test.cc
@@ -1422,8 +1422,8 @@ TEST_F(HostImplTest, CreateConnectionHappyEyeballs) {
   Network::MockTransportSocketFactory socket_factory;
 
   std::vector address_list = {
-      Network::Utility::resolveUrl("tcp://10.0.0.1:1235"),
       address,
+      Network::Utility::resolveUrl("tcp://10.0.0.1:1235"),
   };
   host->setAddressList(address_list);
   auto connection = new testing::StrictMock();
@@ -1472,8 +1472,8 @@ TEST_F(HostImplTest, ProxyOverridesHappyEyeballs) {
   Network::MockTransportSocketFactory socket_factory;
 
   std::vector address_list = {
-      Network::Utility::resolveUrl("tcp://10.0.0.1:1235"),
       address,
+      Network::Utility::resolveUrl("tcp://10.0.0.1:1235"),
   };
   host->setAddressList(address_list);
   auto connection = new testing::StrictMock();
diff --git a/test/extensions/clusters/eds/BUILD b/test/extensions/clusters/eds/BUILD
index b38d03fe9077..823ae6b55d79 100644
--- a/test/extensions/clusters/eds/BUILD
+++ b/test/extensions/clusters/eds/BUILD
@@ -30,6 +30,7 @@ envoy_cc_test(
         "//test/mocks/ssl:ssl_mocks",
         "//test/mocks/upstream:cluster_manager_mocks",
         "//test/mocks/upstream:health_checker_mocks",
+        "//test/test_common:simulated_time_system_lib",
         "//test/test_common:utility_lib",
         "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
diff --git a/test/extensions/clusters/eds/eds_test.cc b/test/extensions/clusters/eds/eds_test.cc
index 3457970f8ae9..df77453512a5 100644
--- a/test/extensions/clusters/eds/eds_test.cc
+++ b/test/extensions/clusters/eds/eds_test.cc
@@ -22,6 +22,7 @@
 #include "test/mocks/ssl/mocks.h"
 #include "test/mocks/upstream/cluster_manager.h"
 #include "test/mocks/upstream/health_checker.h"
+#include "test/test_common/simulated_time_system.h"
 #include "test/test_common/utility.h"
 
 #include "gmock/gmock.h"
@@ -33,7 +34,7 @@ namespace Envoy {
 namespace Upstream {
 namespace {
 
-class EdsTest : public testing::Test {
+class EdsTest : public testing::Test, public Event::TestUsingSimulatedTime {
 public:
   EdsTest() { resetCluster(); }
 
@@ -133,6 +134,7 @@ class EdsTest : public testing::Test {
   }
 
   void initialize() {
+    EXPECT_CALL(server_context_, timeSource()).WillRepeatedly(testing::ReturnRef(simTime()));
     EXPECT_CALL(*server_context_.cluster_manager_.subscription_factory_.subscription_, start(_));
     cluster_->initialize([this] { initialized_ = true; });
   }
@@ -379,6 +381,55 @@ TEST_F(EdsTest, EndpointWeightChangeCausesRebuild) {
   EXPECT_EQ(new_hosts[0]->weight(), 31);
 }
 
+// Verify that host weight changes cause a full rebuild.
+TEST_F(EdsTest, DualStackEndpoint) {
+  envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment;
+  cluster_load_assignment.set_cluster_name("fare");
+
+  // Add dual stack endpoint
+  auto* endpoints = cluster_load_assignment.add_endpoints();
+  auto* endpoint = endpoints->add_lb_endpoints();
+  endpoint->mutable_endpoint()->mutable_address()->mutable_socket_address()->set_address("::1");
+  endpoint->mutable_endpoint()->mutable_address()->mutable_socket_address()->set_port_value(80);
+  auto* socket_address = endpoint->mutable_endpoint()
+                             ->mutable_additional_addresses()
+                             ->Add()
+                             ->mutable_address()
+                             ->mutable_socket_address();
+  socket_address->set_address("1.2.3.5");
+  socket_address->set_port_value(80);
+
+  endpoint->mutable_load_balancing_weight()->set_value(30);
+
+  initialize();
+  doOnConfigUpdateVerifyNoThrow(cluster_load_assignment);
+  EXPECT_TRUE(initialized_);
+  EXPECT_EQ(0UL,
+            stats_.findCounterByString("cluster.name.update_no_rebuild").value().get().value());
+  EXPECT_EQ(30UL, stats_.findGaugeByString("cluster.name.max_host_weight").value().get().value());
+  auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+  EXPECT_EQ(hosts.size(), 1);
+  EXPECT_EQ(hosts[0]->weight(), 30);
+
+  testing::StrictMock dispatcher;
+  Network::TransportSocketOptionsConstSharedPtr transport_socket_options;
+  Network::ConnectionSocket::OptionsSharedPtr options;
+
+  auto connection = new testing::StrictMock();
+  EXPECT_CALL(*connection, setBufferLimits(1048576));
+  EXPECT_CALL(*connection, addConnectionCallbacks(_));
+  EXPECT_CALL(*connection, connectionInfoSetter());
+  // The underlying connection should be created with the first address in the list.
+  EXPECT_CALL(dispatcher, createClientConnection_(hosts[0]->address(), _, _, _))
+      .WillOnce(Return(connection));
+  EXPECT_CALL(dispatcher, createTimer_(_));
+
+  Envoy::Upstream::Host::CreateConnectionData connection_data =
+      hosts[0]->createConnection(dispatcher, options, transport_socket_options);
+  // The created connection will be wrapped in a HappyEyeballsConnectionImpl.
+  EXPECT_NE(connection, connection_data.connection_.get());
+}
+
 // Validate that onConfigUpdate() updates the endpoint metadata.
 TEST_F(EdsTest, EndpointMetadata) {
   envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment;

From 37f764e056505632ae12a22d215fa46305be4053 Mon Sep 17 00:00:00 2001
From: Ali Beyad 
Date: Sun, 18 Jun 2023 00:08:57 -0400
Subject: [PATCH 578/740] mobile: Optimize the Swift-C++ interop
 generateBootstrap bridge (#28023)

This change passes a reference, instead of a copy, to the
generateBootstrapPtr bridge function. Copying the EngineBuilder can be
quite expensive, since it can hold lots of large strings (certs, filter
configs, etc).

The Swift bridge's generateBootstrap method needs to be declared
`mutating` because the EngineBuilder reference is not constant (const&).
The method parameter cannot be const& because of an interop oddity
described in
https://github.com/apple/swift/blob/main/docs/CppInteroperability/InteropOddities.md.

Signed-off-by: Ali Beyad 
---
 .../swift/EnvoyCxxSwiftInterop/cxx_swift_interop.h   |  2 +-
 mobile/library/swift/cxx/Bootstrap.swift             | 12 ++++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.h b/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.h
index d1506c3914ca..d6e9e0d4f293 100644
--- a/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.h
+++ b/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.h
@@ -37,7 +37,7 @@ inline void raw_header_map_set(Platform::RawHeaderMap& map, std::string key,
 
 // Smart pointers aren't currently supported by Swift / C++ interop, so we "erase"
 // it into a `BootstrapPtr` / `intptr_t`, which we can import from Swift.
-inline BootstrapPtr generateBootstrapPtr(Platform::EngineBuilder builder) {
+inline BootstrapPtr generateBootstrapPtr(Platform::EngineBuilder& builder) {
   return reinterpret_cast(builder.generateBootstrap().release());
 }
 
diff --git a/mobile/library/swift/cxx/Bootstrap.swift b/mobile/library/swift/cxx/Bootstrap.swift
index 9877c3778bb9..1fe00d2c8824 100644
--- a/mobile/library/swift/cxx/Bootstrap.swift
+++ b/mobile/library/swift/cxx/Bootstrap.swift
@@ -20,8 +20,16 @@ final class Bootstrap {
 }
 
 extension Envoy.Platform.EngineBuilder {
+  /// Overrides the generateBootstrap() virtual function in the C++ EngineBuilder.
+  ///
+  /// NOTE: This function is `mutating` because we pass in the EngineBuilder by reference to
+  /// CxxSwift.generateBootstrapPtr(). The CxxSwift.generateBootstrapPtr() function cannot take
+  /// a const reference (const&) because passing by const reference is bridged as a value type
+  /// (see https://github.com/apple/swift/blob/main/docs/CppInteroperability/InteropOddities.md)
+  /// and EngineBuilder can't be a value type because it has virtual functions.
+  ///
   /// - returns: A generated bootstrap object.
-  func generateBootstrap() -> Bootstrap {
-    Bootstrap(pointer: Envoy.CxxSwift.generateBootstrapPtr(self))
+  mutating func generateBootstrap() -> Bootstrap {
+    Bootstrap(pointer: Envoy.CxxSwift.generateBootstrapPtr(&self))
   }
 }

From a2a153d65743100367871685de7b88e4afb88256 Mon Sep 17 00:00:00 2001
From: Alex Ding <39857214+nmdzl@users.noreply.github.com>
Date: Tue, 20 Jun 2023 00:08:40 -0400
Subject: [PATCH 579/740] ext_proc: Move setUpstreamHost from openStream to
 closeStream (#27855)

* setUpstreamHost on stream close

Signed-off-by: Alex Ding 

* fix

Signed-off-by: Alex Ding 

* Move setUpstreamInfo to openStream

Signed-off-by: Alex Ding <39857214+nmdzl@users.noreply.github.com>

* Add missing member variable

Signed-off-by: Alex Ding <39857214+nmdzl@users.noreply.github.com>

* Remove a redundant check

Signed-off-by: Alex Ding <39857214+nmdzl@users.noreply.github.com>

* move to onReceiveMessage

Signed-off-by: Alex Ding 

* also add to onGrpcError and onGrpcClose

Signed-off-by: Alex Ding 

* Run clang-format

Signed-off-by: Alex Ding 

* move to closeStream only

Signed-off-by: Alex Ding 

* Remove redundant check.

Signed-off-by: Alex Ding <39857214+nmdzl@users.noreply.github.com>

---------

Signed-off-by: Alex Ding 
Signed-off-by: Alex Ding <39857214+nmdzl@users.noreply.github.com>
---
 source/extensions/filters/http/ext_proc/ext_proc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc
index 0ef4f84c8274..3704a204ec80 100644
--- a/source/extensions/filters/http/ext_proc/ext_proc.cc
+++ b/source/extensions/filters/http/ext_proc/ext_proc.cc
@@ -137,7 +137,6 @@ Filter::StreamOpenState Filter::openStream() {
     // have a proper implementation of streamInfo.
     if (grpc_service_.has_envoy_grpc()) {
       logging_info_->setClusterInfo(stream_->streamInfo().upstreamClusterInfo());
-      logging_info_->setUpstreamHost(stream_->streamInfo().upstreamInfo()->upstreamHost());
     }
   }
   return StreamOpenState::Ok;
@@ -148,6 +147,7 @@ void Filter::closeStream() {
     if (grpc_service_.has_envoy_grpc()) {
       logging_info_->setBytesSent(stream_->streamInfo().bytesSent());
       logging_info_->setBytesReceived(stream_->streamInfo().bytesReceived());
+      logging_info_->setUpstreamHost(stream_->streamInfo().upstreamInfo()->upstreamHost());
     }
     ENVOY_LOG(debug, "Calling close on stream");
     if (stream_->close()) {

From 72e964aaadddb1607fbfdfb207a1c2d7a808e43c Mon Sep 17 00:00:00 2001
From: Tianyu <72890320+tyxia@users.noreply.github.com>
Date: Tue, 20 Jun 2023 09:57:05 -0400
Subject: [PATCH 580/740] RLQS: Improve the test coverage (#28022)

* remove untested code for now

Signed-off-by: tyxia 
---
 .../filters/http/rate_limit_quota/filter.cc          | 12 +++++-------
 .../filters/http/rate_limit_quota/filter.h           |  1 -
 .../filters/http/rate_limit_quota/matcher.cc         |  3 ---
 .../filters/http/rate_limit_quota/matcher.h          |  1 -
 .../filters/http/rate_limit_quota/filter_test.cc     | 12 ++++++++++++
 test/per_file_coverage.sh                            |  1 -
 6 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/source/extensions/filters/http/rate_limit_quota/filter.cc b/source/extensions/filters/http/rate_limit_quota/filter.cc
index 79e4792adaa9..8c65c580b4c7 100644
--- a/source/extensions/filters/http/rate_limit_quota/filter.cc
+++ b/source/extensions/filters/http/rate_limit_quota/filter.cc
@@ -63,10 +63,10 @@ RateLimitQuotaFilter::requestMatching(const Http::RequestHeaderMap& headers) {
   if (matcher_ == nullptr) {
     return absl::InternalError("Matcher tree has not been initialized yet.");
   } else {
-    data_ptr_->onRequestHeaders(headers);
-    // TODO(tyxia) This function should trigger the CEL expression matching. Here, we need to
-    // implement the custom_matcher and factory, also statically register it so that CEL matching
-    // will be triggered with its own match() method.
+    // TODO(tyxia) Update the logic with the CEL matcher and other attributes besides header.
+    if (!headers.empty()) {
+      data_ptr_->onRequestHeaders(headers);
+    }
     auto match_result = Matcher::evaluateMatch(*matcher_, *data_ptr_);
 
     if (match_result.match_state_ == Matcher::MatchState::MatchComplete) {
@@ -78,13 +78,11 @@ RateLimitQuotaFilter::requestMatching(const Http::RequestHeaderMap& headers) {
       }
     } else {
       // The returned state from `evaluateMatch` function is `MatchState::UnableToMatch` here.
-      return absl::InternalError("Unable to match the request.");
+      return absl::InternalError("Unable to match due to the required data not being available.");
     }
   }
 }
 
-void RateLimitQuotaFilter::onComplete(const RateLimitQuotaBucketSettings&, RateLimitStatus) {}
-
 } // namespace RateLimitQuota
 } // namespace HttpFilters
 } // namespace Extensions
diff --git a/source/extensions/filters/http/rate_limit_quota/filter.h b/source/extensions/filters/http/rate_limit_quota/filter.h
index e71b300f4e57..362dd71d5609 100644
--- a/source/extensions/filters/http/rate_limit_quota/filter.h
+++ b/source/extensions/filters/http/rate_limit_quota/filter.h
@@ -72,7 +72,6 @@ class RateLimitQuotaFilter : public Http::PassThroughFilter,
     return *data_ptr_;
   }
 
-  void onComplete(const RateLimitQuotaBucketSettings&, RateLimitStatus);
   ~RateLimitQuotaFilter() override = default;
 
 private:
diff --git a/source/extensions/filters/http/rate_limit_quota/matcher.cc b/source/extensions/filters/http/rate_limit_quota/matcher.cc
index bed4d857284e..8cffa982a694 100644
--- a/source/extensions/filters/http/rate_limit_quota/matcher.cc
+++ b/source/extensions/filters/http/rate_limit_quota/matcher.cc
@@ -46,9 +46,6 @@ RateLimitOnMatchAction::generateBucketId(const Http::Matching::HttpMatchingDataI
       if (!str.empty()) {
         // Build the bucket id from the matched result.
         bucket_id.mutable_bucket()->insert({bucket_id_key, str});
-      } else {
-        // TODO(tyxia) Nothing hits this line at this moment.
-        return absl::InternalError("Empty resulting data from custom value config.");
       }
       break;
     }
diff --git a/source/extensions/filters/http/rate_limit_quota/matcher.h b/source/extensions/filters/http/rate_limit_quota/matcher.h
index af59371844c1..254d631cca1f 100644
--- a/source/extensions/filters/http/rate_limit_quota/matcher.h
+++ b/source/extensions/filters/http/rate_limit_quota/matcher.h
@@ -40,7 +40,6 @@ class RateLimitOnMatchAction : public Matcher::ActionBase,
   absl::StatusOr generateBucketId(const Http::Matching::HttpMatchingDataImpl& data,
                                             Server::Configuration::FactoryContext& factory_context,
                                             RateLimitQuotaValidationVisitor& visitor) const;
-  RateLimitQuotaBucketSettings bucketSettings() const { return setting_; }
 
 private:
   RateLimitQuotaBucketSettings setting_;
diff --git a/test/extensions/filters/http/rate_limit_quota/filter_test.cc b/test/extensions/filters/http/rate_limit_quota/filter_test.cc
index f2be91f45667..e6888999226b 100644
--- a/test/extensions/filters/http/rate_limit_quota/filter_test.cc
+++ b/test/extensions/filters/http/rate_limit_quota/filter_test.cc
@@ -405,6 +405,18 @@ TEST_F(FilterTest, RequestMatchingFailed) {
   EXPECT_EQ(match.status().message(), "Matching completed but no match result was found.");
 }
 
+TEST_F(FilterTest, RequestMatchingFailedWithEmptyHeader) {
+  addMatcherConfig(MatcherConfigType::Valid);
+  createFilter();
+  Http::TestRequestHeaderMapImpl empty_header = {};
+  // Perform request matching.
+  auto match = filter_->requestMatching(empty_header);
+  // Not_OK status is expected to be returned because the matching failed due to empty headers.
+  EXPECT_FALSE(match.ok());
+  EXPECT_EQ(match.status().message(),
+            "Unable to match due to the required data not being available.");
+}
+
 TEST_F(FilterTest, RequestMatchingFailedWithNoCallback) {
   addMatcherConfig(MatcherConfigType::Valid);
   createFilter(/*set_callback*/ false);
diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh
index 0fb812f5e751..a9c1113fcc2f 100755
--- a/test/per_file_coverage.sh
+++ b/test/per_file_coverage.sh
@@ -39,7 +39,6 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/filters/http/ip_tagging:88.0"
 "source/extensions/filters/http/kill_request:91.7" # Death tests don't report LCOV
 "source/extensions/filters/http/wasm:1.9"
-"source/extensions/filters/http/rate_limit_quota:94.8" # TODO(tyxia) WIP. Improve coverage as development continues
 "source/extensions/filters/listener/original_dst:82.9"
 "source/extensions/filters/listener/original_src:92.1"
 "source/extensions/filters/network/common:96.2"

From 3436b8e40616d329b47ef89bd7cfa45c425adac9 Mon Sep 17 00:00:00 2001
From: yanavlasov 
Date: Tue, 20 Jun 2023 09:58:35 -0400
Subject: [PATCH 581/740] jwt_authn: Add upper bound on JWT cache_duration
 (#28017)

* Add upper bound on JWT cache_duration

Signed-off-by: Yan Avlasov 
---
 .../filters/http/jwt_authn/v3/config.proto    |  5 ++-
 test/extensions/filters/http/jwt_authn/BUILD  |  3 ++
 .../jwt_authn_corpus/out_of_range_duration    | 37 +++++++++++++++++++
 3 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100644 test/extensions/filters/http/jwt_authn/jwt_authn_corpus/out_of_range_duration

diff --git a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto
index 11e6af453af0..bf88896e7030 100644
--- a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto
+++ b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto
@@ -330,7 +330,10 @@ message RemoteJwks {
 
   // Duration after which the cached JWKS should be expired. If not specified, default cache
   // duration is 10 minutes.
-  google.protobuf.Duration cache_duration = 2;
+  google.protobuf.Duration cache_duration = 2 [(validate.rules).duration = {
+    lt {seconds: 9000000000}
+    gte {nanos: 1000000}
+  }];
 
   // Fetch Jwks asynchronously in the main thread before the listener is activated.
   // Fetched Jwks can be used by all worker threads.
diff --git a/test/extensions/filters/http/jwt_authn/BUILD b/test/extensions/filters/http/jwt_authn/BUILD
index 49d91ab3ca62..b69ee00fd156 100644
--- a/test/extensions/filters/http/jwt_authn/BUILD
+++ b/test/extensions/filters/http/jwt_authn/BUILD
@@ -147,6 +147,9 @@ envoy_extension_cc_test(
     srcs = ["filter_integration_test.cc"],
     extension_names = ["envoy.filters.http.jwt_authn"],
     shard_count = 6,
+    tags = [
+        "cpu:3",
+    ],
     deps = [
         "//source/common/router:string_accessor_lib",
         "//source/extensions/filters/http/jwt_authn:config",
diff --git a/test/extensions/filters/http/jwt_authn/jwt_authn_corpus/out_of_range_duration b/test/extensions/filters/http/jwt_authn/jwt_authn_corpus/out_of_range_duration
new file mode 100644
index 000000000000..0c0c344f0894
--- /dev/null
+++ b/test/extensions/filters/http/jwt_authn/jwt_authn_corpus/out_of_range_duration
@@ -0,0 +1,37 @@
+config {
+  providers {
+    key: "\005"
+    value {
+      remote_jwks {
+        http_uri {
+          uri: "https:/pubkey_server/pubkey_phttps:/pubkey_server/pubkey_pathath"
+          cluster: "2"
+          timeout {
+            seconds: 68719476736
+          }
+        }
+        cache_duration {
+        }
+      }
+    }
+  }
+  rules {
+    match {
+      prefix: ""
+    }
+    requires {
+      allow_missing {
+      }
+    }
+  }
+}
+request_data {
+  headers {
+    headers {
+      key: "authorization"
+      value: "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwczovL2V4cGxlLmNvbSIsImF1ZCI6ImV4YW1wbGVfc2VydmljZSIsImV4cCI6MjAwMTAwMTAwMX0.n45uWZfIBZwCIPiL0K8Ca3tmm-ZlsDrC79_vXCspPwk5oxdSn983tuC9GfVWKXWUMHe11DsB02b19Ow-fmoEzooTFn65Ml7G34nW07amyM6lETiMhNzyiunctplOr6xKKJHmzTUhfTirvDeG-q9n24-8lH7GP8GgHvDlgSM9OY7TGp81bRcnZBmxim_UzHoYO3_c8OP4ZX3xG5PfihVk5G0g6wcHrO70w0_64JgkKRCrLHMJSrhIgp9NHel_CNOnL0AjQKe9IGblJrMuouqYYS0zEWwmOVUWUSxQkoLpldQUVefcfjQeGjz8IlvktRa77FYexfP590ACPyXrivtsxg"
+    }
+  }
+}
+remote_jwks: "{\"keys\":[{\"kty\":\"RSA\",\"alg\":\"RS256\",\"use\":\"sig\",\"kid\":\"62a93512c9ee4c7f8067b5a216dade2763d32a47\",\"n\":\"0YWnm_eplO9BFtUUUUUUU\0055H8mYDW11HolzZmTQpRoLV8ZoHbHEaTfqX_aYahIw\",\"e\":\"AQAB\"},{\"kty\":\"RSA\",\"alg\":\"RS256\",\"use\":\"sig\",\"kid\":\"b3319a147514df7ee5e4bcdee51350cc890cc89e\",\"n\":\"qDi7Tx4DhNDDdk0iUtJSPZliUHJBI_pj8M-2Mn_oA8jBuI8YKwBqYkZCN2I95Q\",\"e\":\"AQAB\"}]}"
+num_calls: 4

From 56f5071551ed05765c26eca661afca9fe117c542 Mon Sep 17 00:00:00 2001
From: yanjunxiang-google
 <78807980+yanjunxiang-google@users.noreply.github.com>
Date: Tue, 20 Jun 2023 10:05:51 -0400
Subject: [PATCH 582/740] Applying same ext_proc filter checker rule for
 immediate response. (#27925)

* Applying same ext_proc filter checker rule for immediate response.

Signed-off-by: Yanjun Xiang 
---
 changelogs/current.yaml                       |  4 ++
 source/common/runtime/runtime_features.cc     |  1 +
 .../filters/http/ext_proc/ext_proc.cc         | 14 ++++--
 test/extensions/filters/http/ext_proc/BUILD   |  1 +
 .../ext_proc/ext_proc_integration_test.cc     | 47 +++++++++++++++++++
 5 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 9db57a81bcfa..47f4e5dbfb2d 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -15,6 +15,10 @@ behavior_changes:
     When ``append_x_forwarded_host`` is enabled for a given route action it is now only appended iff it is different from the last
     value in the list. This resolves issues where a retry caused the same value to be appended multiple times. This
     behavioral change can be temporarily reverted by setting runtime guard ``envoy_reloadable_features_append_xfh_idempotent`` to false.
+- area: ext_proc
+  change: |
+    Apply header mutation rules from the ext_proc config to the ImmediateResponse. This behavior change can be temporarily
+    reverted by setting the runtime guard ``envoy_reloadable_features_immediate_response_use_filter_mutation_rule`` to false.
 
 minor_behavior_changes:
 # *Changes that may cause incompatibilities for some users, but should not for most*
diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc
index f576b972e9bd..05dca2e7c39c 100644
--- a/source/common/runtime/runtime_features.cc
+++ b/source/common/runtime/runtime_features.cc
@@ -51,6 +51,7 @@ RUNTIME_GUARD(envoy_reloadable_features_http_ext_auth_failure_mode_allow_header_
 RUNTIME_GUARD(envoy_reloadable_features_http_filter_avoid_reentrant_local_reply);
 RUNTIME_GUARD(envoy_reloadable_features_http_reject_path_with_fragment);
 RUNTIME_GUARD(envoy_reloadable_features_ignore_optional_option_from_hcm_for_route_config);
+RUNTIME_GUARD(envoy_reloadable_features_immediate_response_use_filter_mutation_rule);
 RUNTIME_GUARD(envoy_reloadable_features_initialize_upstream_filters);
 RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name);
 RUNTIME_GUARD(envoy_reloadable_features_no_full_scan_certs_on_sni_mismatch);
diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc
index 3704a204ec80..318ed7a21c40 100644
--- a/source/extensions/filters/http/ext_proc/ext_proc.cc
+++ b/source/extensions/filters/http/ext_proc/ext_proc.cc
@@ -781,9 +781,17 @@ void Filter::sendImmediateResponse(const ImmediateResponse& response) {
           : absl::nullopt;
   const auto mutate_headers = [this, &response](Http::ResponseHeaderMap& headers) {
     if (response.has_headers()) {
-      const auto mut_status = MutationUtils::applyHeaderMutations(
-          response.headers(), headers, false, immediateResponseChecker().checker(),
-          stats_.rejected_header_mutations_);
+      absl::Status mut_status;
+      if (Runtime::runtimeFeatureEnabled(
+              "envoy.reloadable_features.immediate_response_use_filter_mutation_rule")) {
+        mut_status = MutationUtils::applyHeaderMutations(response.headers(), headers, false,
+                                                         config().mutationChecker(),
+                                                         stats_.rejected_header_mutations_);
+      } else {
+        mut_status = MutationUtils::applyHeaderMutations(response.headers(), headers, false,
+                                                         immediateResponseChecker().checker(),
+                                                         stats_.rejected_header_mutations_);
+      }
       if (!mut_status.ok()) {
         ENVOY_LOG_EVERY_POW_2(error, "Immediate response mutations failed with {}",
                               mut_status.message());
diff --git a/test/extensions/filters/http/ext_proc/BUILD b/test/extensions/filters/http/ext_proc/BUILD
index 8a77756a3aab..58c52ad997bc 100644
--- a/test/extensions/filters/http/ext_proc/BUILD
+++ b/test/extensions/filters/http/ext_proc/BUILD
@@ -126,6 +126,7 @@ envoy_extension_cc_test(
         "//source/extensions/filters/http/ext_proc:config",
         "//test/common/http:common_lib",
         "//test/integration:http_integration_lib",
+        "//test/test_common:test_runtime_lib",
         "//test/test_common:utility_lib",
         "@envoy_api//envoy/extensions/filters/http/ext_proc/v3:pkg_cc_proto",
         "@envoy_api//envoy/service/ext_proc/v3:pkg_cc_proto",
diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc
index 3fef3b403fa1..726c69c14a48 100644
--- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc
+++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc
@@ -9,6 +9,7 @@
 #include "test/common/http/common.h"
 #include "test/extensions/filters/http/ext_proc/utils.h"
 #include "test/integration/http_integration.h"
+#include "test/test_common/test_runtime.h"
 #include "test/test_common/utility.h"
 
 #include "absl/strings/str_cat.h"
@@ -67,6 +68,8 @@ class ExtProcIntegrationTest : public HttpIntegrationTest,
   }
 
   void initializeConfig() {
+    scoped_runtime_.mergeValues(
+        {{"envoy.reloadable_features.send_header_value_in_bytes", filter_mutation_rule_}});
     config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
       // Ensure "HTTP2 with no prior knowledge." Necessary for gRPC and for headers
       ConfigHelper::setHttp2(
@@ -381,6 +384,8 @@ class ExtProcIntegrationTest : public HttpIntegrationTest,
   std::vector grpc_upstreams_;
   FakeHttpConnectionPtr processor_connection_;
   FakeStreamPtr processor_stream_;
+  TestScopedRuntime scoped_runtime_;
+  std::string filter_mutation_rule_{"false"};
 };
 
 INSTANTIATE_TEST_SUITE_P(
@@ -1255,6 +1260,48 @@ TEST_P(ExtProcIntegrationTest, GetAndRespondImmediatelyWithBadStatus) {
   EXPECT_EQ("{\"reason\": \"Because\"}", response->body());
 }
 
+// Test the filter using an ext_proc server that responds to the request_header message
+// by sending back an immediate_response message with system header mutation.
+TEST_P(ExtProcIntegrationTest, GetAndRespondImmediatelyWithSystemHeaderMutation) {
+  filter_mutation_rule_ = "true";
+  proto_config_.mutable_mutation_rules()->mutable_disallow_is_error()->set_value(true);
+  // Disallow system header in the mutation rule config.
+  proto_config_.mutable_mutation_rules()->mutable_disallow_system()->set_value(true);
+  initializeConfig();
+  HttpIntegrationTest::initialize();
+  auto response = sendDownstreamRequest(absl::nullopt);
+  processAndRespondImmediately(*grpc_upstreams_[0], true, [](ImmediateResponse& immediate) {
+    immediate.mutable_status()->set_code(envoy::type::v3::StatusCode::Unauthorized);
+    auto* hdr = immediate.mutable_headers()->add_set_headers();
+    // Adding system header in the ext_proc response.
+    hdr->mutable_header()->set_key(":foo");
+    hdr->mutable_header()->set_value("bar");
+  });
+  verifyDownstreamResponse(*response, 401);
+  // The added system header is not sent to the client.
+  EXPECT_THAT(response->headers(), HasNoHeader(":foo"));
+}
+
+// Test the filter using an ext_proc server that responds to the request_header message
+// by sending back an immediate_response message with x-envoy header mutation.
+TEST_P(ExtProcIntegrationTest, GetAndRespondImmediatelyWithEnvoyHeaderMutation) {
+  filter_mutation_rule_ = "true";
+  proto_config_.mutable_mutation_rules()->mutable_disallow_is_error()->set_value(true);
+  proto_config_.mutable_mutation_rules()->mutable_allow_envoy()->set_value(false);
+  initializeConfig();
+  HttpIntegrationTest::initialize();
+  auto response = sendDownstreamRequest(absl::nullopt);
+  processAndRespondImmediately(*grpc_upstreams_[0], true, [](ImmediateResponse& immediate) {
+    immediate.mutable_status()->set_code(envoy::type::v3::StatusCode::Unauthorized);
+    auto* hdr = immediate.mutable_headers()->add_set_headers();
+    // Adding x-envoy header is not allowed.
+    hdr->mutable_header()->set_key("x-envoy-foo");
+    hdr->mutable_header()->set_value("bar");
+  });
+  verifyDownstreamResponse(*response, 401);
+  EXPECT_THAT(response->headers(), HasNoHeader("x-envoy-foo"));
+}
+
 // Test the filter with request body buffering enabled using
 // an ext_proc server that responds to the request_body message
 // by modifying a header that should cause an error.

From 4222f18fe3eb34a9530d3b1d916bcb8f18dc627d Mon Sep 17 00:00:00 2001
From: Ali Beyad 
Date: Tue, 20 Jun 2023 10:33:58 -0400
Subject: [PATCH 583/740] mobile: Improve the xDS APIs on EngineBuilder
 (#28001)

This change does a few things:
1. Makes the setNodeXXX APIs available even when GOOGLE_GRPC isn't enabled in the build. The node metadata APIs are useful in contexts other than for xDS.
2. Removes the setAggregatedDiscoveryServce, addCdsLayer, and addRtdsLayer methods in favor of a single setXds method, which takes in a newly introduced XdsBuilder to build up the xDS config.
3. A side benefit is the xDS server address/port are passed into the XdsBuilder's constructor, so we don't need to check in all of the xDS calls that "setAggregatedDiscoveryService" was called or not.
4. Fixes up some variable naming.

Signed-off-by: Ali Beyad 
---
 mobile/docs/root/api/starting_envoy.rst       |  73 ++---
 mobile/library/cc/engine_builder.cc           | 179 ++++++-----
 mobile/library/cc/engine_builder.h            | 117 ++++++--
 mobile/library/common/jni/jni_interface.cc    |  83 +++---
 .../engine/EnvoyConfiguration.java            |  60 ++--
 .../envoymobile/engine/JniLibrary.java        |   9 +-
 .../impl/NativeCronvoyEngineBuilderImpl.java  |  17 +-
 .../envoyproxy/envoymobile/EngineBuilder.kt   | 216 ++++++++------
 .../library/objective-c/EnvoyConfiguration.h  |  42 +--
 .../library/objective-c/EnvoyConfiguration.mm |  72 +++--
 mobile/library/swift/EngineBuilder.swift      | 277 ++++++++++++------
 mobile/test/cc/unit/envoy_config_test.cc      |  69 +++--
 .../integration/cds_integration_test.cc       |   7 +-
 .../integration/rtds_integration_test.cc      |  18 +-
 .../engine/EnvoyConfigurationTest.kt          |  36 +--
 .../envoymobile/EngineBuilderTest.kt          |  18 +-
 .../gcp_traffic_director_integration_test.cc  |  10 +-
 mobile/test/swift/EngineBuilderTests.swift    |  23 +-
 18 files changed, 782 insertions(+), 544 deletions(-)

diff --git a/mobile/docs/root/api/starting_envoy.rst b/mobile/docs/root/api/starting_envoy.rst
index 67b68a5a101a..2bff205e92fb 100644
--- a/mobile/docs/root/api/starting_envoy.rst
+++ b/mobile/docs/root/api/starting_envoy.rst
@@ -495,67 +495,44 @@ Note that Envoy will fail to start up in debug mode if an unknown guard is speci
   // Swift
   builder.setRuntimeGuard("feature", true)
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-``addRtdsLayer``
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Adds an RTDS layer to the bootstrap configuration.
-Requires that ADS be configured via `setAggregatedDiscoveryService()`.
-See the following link for details:
-https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/runtime/v3/rtds.proto
-
-**Example**::
-
-  // Kotlin
-  builder.addRtdsLayer(layerName = "rtds_layer_name", timeoutSeconds = 10)
-
-  // Swift
-  builder.addRTDSLayer(layerName: "rtds_layer_name", timeoutSeconds: 10)
-
-  // C++
-  builder.addRtdsLayer("rtds_layer_name", 10)
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-``addCdsLayer``
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Adds a CDS layer to the bootstrap configuration.
-Requires that ADS be configured via `setAggregatedDiscoveryService()`.
-See the following link for details:
-https://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cds
-
-**Example**::
-
-  // Kotlin
-  builder.addCdsLayer(resourcesLocator = "xdstp://td-location.com/cluster_location", timeoutSeconds = 10)
-
-  // Swift
-  builder.addCDSLayer(resourcesLocator: "xdstp://td-location.com/cluster_location", timeoutSeconds: 10)
-
-  // C++
-  builder.addCdsLayer("xdstp://td-location.com/cluster_location", 10)
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``setAggregatedDiscoveryService``
+``setXds``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Adds ADS to bootstrap configuration, for instance to be used with RTDS and CDS layers.
-Optional params allow configuring a JWT token and SSL. See the following link for details:
-https://www.envoyproxy.io/docs/envoy/latest/configuration/overview/xds_api#config-overview-ads
+Sets the Bootstrap configuration up with `xDS `_
+to fetch dynamic configuration from an xDS management server. The xDS management server must
+support the ADS protocol. At this moment, only the State-of-the-World (SotW) xDS protocol is
+supported, not the Delta protocol. The Envoy Mobile client will communicate with the configured
+xDS management server over gRPC.
+
+Use the XdsBuilder class to configure the xDS for the Envoy Mobile engine.  For example, the
+`addRuntimeDiscoveryService()` method can be used to configure the
+`Runtime Discovery Service (RTDS) `_
+and the `addClusterDiscoveryService()` method to configure the
+`Cluster Discovery Service (CDS) `_.
 
 Parameters:
-address, port, (optional) jwt_token, (optional) jwt_token_lifetime_seconds, (optional) ssl_root_certs
+xds_builder
 
 **Example**::
 
   // Kotlin
-  builder.setAggregatedDiscoveryService(address = "192.168.1.1", port = 0)
+  val xdsBuilder = new XdsBuilder(address = "my_xds_server.com", port = 443)
+                       .addRuntimeDiscoveryService("my_rtds_resource")
+                       .addClusterDiscoveryService()
+  builder.setXds(xdsBuilder)
 
   // Swift
-  builder.setAggregatedDiscoveryService(address: "192.168.1.1", port: 0)
+  var xdsBuilder = XdsBuilder(address: "my_xds_server.com", port: 443)
+                       .addRuntimeDiscoveryService("my_rtds_resource")
+                       .addClusterDiscoveryService()
+  builder.setXds(xdsBuilder)
 
   // C++
-  builder.setAggregatedDiscoveryService("192.168.1.1", 0)
+  XdsBuilder xds_builder(/*address=*/"my_xds_server.com", /*port=*/443);
+  xds_builder.addRuntimeDiscoveryService("my_rtds_resource")
+      .addClusterDiscoveryService();
+  builder.setXds(std::move(xds_builder));
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 ``setNodeId``
diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc
index 250217ebc01b..82ea4c6c9abf 100644
--- a/mobile/library/cc/engine_builder.cc
+++ b/mobile/library/cc/engine_builder.cc
@@ -41,6 +41,97 @@
 namespace Envoy {
 namespace Platform {
 
+#ifdef ENVOY_GOOGLE_GRPC
+XdsBuilder::XdsBuilder(std::string xds_server_address, const int xds_server_port)
+    : xds_server_address_(std::move(xds_server_address)), xds_server_port_(xds_server_port) {}
+
+XdsBuilder& XdsBuilder::setJwtAuthenticationToken(std::string token,
+                                                  const int token_lifetime_in_seconds) {
+  jwt_token_ = std::move(token);
+  jwt_token_lifetime_in_seconds_ =
+      token_lifetime_in_seconds > 0 ? token_lifetime_in_seconds : DefaultJwtTokenLifetimeSeconds;
+  return *this;
+}
+
+XdsBuilder& XdsBuilder::setSslRootCerts(std::string root_certs) {
+  ssl_root_certs_ = root_certs;
+  return *this;
+}
+
+XdsBuilder& XdsBuilder::addRuntimeDiscoveryService(std::string resource_name,
+                                                   const int timeout_in_seconds) {
+  rtds_resource_name_ = std::move(resource_name);
+  rtds_timeout_in_seconds_ = timeout_in_seconds > 0 ? timeout_in_seconds : DefaultXdsTimeout;
+  return *this;
+}
+
+XdsBuilder& XdsBuilder::addClusterDiscoveryService(std::string cds_resources_locator,
+                                                   const int timeout_in_seconds) {
+  enable_cds_ = true;
+  cds_resources_locator_ = std::move(cds_resources_locator);
+  cds_timeout_in_seconds_ = timeout_in_seconds > 0 ? timeout_in_seconds : DefaultXdsTimeout;
+  return *this;
+}
+
+void XdsBuilder::build(envoy::config::bootstrap::v3::Bootstrap* bootstrap) const {
+  auto* ads_config = bootstrap->mutable_dynamic_resources()->mutable_ads_config();
+  ads_config->set_transport_api_version(envoy::config::core::v3::ApiVersion::V3);
+  ads_config->set_set_node_on_first_message_only(true);
+  ads_config->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC);
+  auto& grpc_service = *ads_config->add_grpc_services();
+  grpc_service.mutable_google_grpc()->set_target_uri(
+      fmt::format(R"({}:{})", xds_server_address_, xds_server_port_));
+  grpc_service.mutable_google_grpc()->set_stat_prefix("ads");
+  if (!ssl_root_certs_.empty()) {
+    grpc_service.mutable_google_grpc()
+        ->mutable_channel_credentials()
+        ->mutable_ssl_credentials()
+        ->mutable_root_certs()
+        ->set_inline_string(ssl_root_certs_);
+  }
+  if (!jwt_token_.empty()) {
+    auto& jwt = *grpc_service.mutable_google_grpc()
+                     ->add_call_credentials()
+                     ->mutable_service_account_jwt_access();
+    jwt.set_json_key(jwt_token_);
+    jwt.set_token_lifetime_seconds(jwt_token_lifetime_in_seconds_);
+  }
+
+  if (!rtds_resource_name_.empty()) {
+    auto* layered_runtime = bootstrap->mutable_layered_runtime();
+    auto* layer = layered_runtime->add_layers();
+    layer->set_name("rtds_layer");
+    auto* rtds_layer = layer->mutable_rtds_layer();
+    rtds_layer->set_name(rtds_resource_name_);
+    auto* rtds_config = rtds_layer->mutable_rtds_config();
+    rtds_config->mutable_ads();
+    rtds_config->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3);
+    rtds_config->mutable_initial_fetch_timeout()->set_seconds(rtds_timeout_in_seconds_);
+  }
+
+  if (enable_cds_) {
+    auto* cds_config = bootstrap->mutable_dynamic_resources()->mutable_cds_config();
+    if (cds_resources_locator_.empty()) {
+      cds_config->mutable_ads();
+    } else {
+      bootstrap->mutable_dynamic_resources()->set_cds_resources_locator(cds_resources_locator_);
+      cds_config->mutable_api_config_source()->set_api_type(
+          envoy::config::core::v3::ApiConfigSource::AGGREGATED_GRPC);
+      cds_config->mutable_api_config_source()->set_transport_api_version(
+          envoy::config::core::v3::ApiVersion::V3);
+    }
+    cds_config->mutable_initial_fetch_timeout()->set_seconds(cds_timeout_in_seconds_);
+    cds_config->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3);
+    bootstrap->add_node_context_params("cluster");
+    // Stat prefixes that we use in tests.
+    auto* list =
+        bootstrap->mutable_stats_config()->mutable_stats_matcher()->mutable_inclusion_list();
+    list->add_patterns()->set_exact("cluster_manager.active_clusters");
+    list->add_patterns()->set_exact("cluster_manager.cluster_added");
+  }
+}
+#endif
+
 EngineBuilder::EngineBuilder() : callbacks_(std::make_shared()) {
 #ifndef ENVOY_ENABLE_QUIC
   enable_http3_ = false;
@@ -200,7 +291,7 @@ EngineBuilder& EngineBuilder::enforceTrustChainVerification(bool trust_chain_ver
   enforce_trust_chain_verification_ = trust_chain_verification_on;
   return *this;
 }
-#ifdef ENVOY_GOOGLE_GRPC
+
 EngineBuilder& EngineBuilder::setNodeId(std::string node_id) {
   node_id_ = std::move(node_id);
   return *this;
@@ -212,30 +303,9 @@ EngineBuilder& EngineBuilder::setNodeLocality(std::string region, std::string zo
   return *this;
 }
 
-EngineBuilder& EngineBuilder::setAggregatedDiscoveryService(std::string address, const int port,
-                                                            std::string jwt_token,
-                                                            const int jwt_token_lifetime_seconds,
-                                                            std::string ssl_root_certs) {
-  ads_address_ = address;
-  ads_port_ = port;
-  ads_jwt_token_ = std::move(jwt_token);
-  ads_jwt_token_lifetime_seconds_ =
-      jwt_token_lifetime_seconds == 0 ? DefaultJwtTokenLifetimeSeconds : jwt_token_lifetime_seconds;
-  ads_ssl_root_certs_ = std::move(ssl_root_certs);
-  return *this;
-}
-
-EngineBuilder& EngineBuilder::addRtdsLayer(std::string layer_name, const int timeout_seconds) {
-  rtds_layer_name_ = std::move(layer_name);
-  rtds_timeout_seconds_ = timeout_seconds == 0 ? DefaultXdsTimeout : timeout_seconds;
-  return *this;
-}
-
-EngineBuilder& EngineBuilder::addCdsLayer(std::string cds_resources_locator,
-                                          const int timeout_seconds) {
-  enable_cds_ = true;
-  cds_resources_locator_ = std::move(cds_resources_locator);
-  cds_timeout_seconds_ = timeout_seconds == 0 ? DefaultXdsTimeout : timeout_seconds;
+#ifdef ENVOY_GOOGLE_GRPC
+EngineBuilder& EngineBuilder::setXds(XdsBuilder xds_builder) {
+  xds_builder_ = std::move(xds_builder);
   return *this;
 }
 #endif
@@ -787,62 +857,11 @@ std::unique_ptr EngineBuilder::generate
   }
 
   bootstrap->mutable_dynamic_resources();
-  if ((!rtds_layer_name_.empty() || enable_cds_) && ads_address_.empty()) {
-    throw std::runtime_error("ADS must be configured when using xDS");
-  }
-  if (!rtds_layer_name_.empty()) {
-    auto* layered_runtime = bootstrap->mutable_layered_runtime();
-    auto* layer = layered_runtime->add_layers();
-    layer->set_name(rtds_layer_name_);
-    auto* rtds_layer = layer->mutable_rtds_layer();
-    rtds_layer->set_name(rtds_layer_name_);
-    auto* rtds_config = rtds_layer->mutable_rtds_config();
-    rtds_config->mutable_ads();
-    rtds_config->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3);
-    rtds_config->mutable_initial_fetch_timeout()->set_seconds(rtds_timeout_seconds_);
-  }
-  if (!ads_address_.empty()) {
-    std::string target_uri = fmt::format(R"({}:{})", ads_address_, ads_port_);
-    auto* ads_config = bootstrap->mutable_dynamic_resources()->mutable_ads_config();
-    ads_config->set_transport_api_version(envoy::config::core::v3::ApiVersion::V3);
-    ads_config->set_set_node_on_first_message_only(true);
-    ads_config->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC);
-    auto& grpc_service = *ads_config->add_grpc_services();
-    grpc_service.mutable_google_grpc()->set_target_uri(target_uri);
-    grpc_service.mutable_google_grpc()->set_stat_prefix("ads");
-    if (!ads_ssl_root_certs_.empty()) {
-      grpc_service.mutable_google_grpc()
-          ->mutable_channel_credentials()
-          ->mutable_ssl_credentials()
-          ->mutable_root_certs()
-          ->set_inline_string(ads_ssl_root_certs_);
-    }
-    if (!ads_jwt_token_.empty()) {
-      auto& jwt = *grpc_service.mutable_google_grpc()
-                       ->add_call_credentials()
-                       ->mutable_service_account_jwt_access();
-      jwt.set_json_key(ads_jwt_token_);
-      jwt.set_token_lifetime_seconds(ads_jwt_token_lifetime_seconds_);
-    }
-  }
-  if (enable_cds_) {
-    auto* cds_config = bootstrap->mutable_dynamic_resources()->mutable_cds_config();
-    if (cds_resources_locator_.empty()) {
-      cds_config->mutable_ads();
-    } else {
-      bootstrap->mutable_dynamic_resources()->set_cds_resources_locator(cds_resources_locator_);
-      cds_config->mutable_api_config_source()->set_api_type(
-          envoy::config::core::v3::ApiConfigSource::AGGREGATED_GRPC);
-      cds_config->mutable_api_config_source()->set_transport_api_version(
-          envoy::config::core::v3::ApiVersion::V3);
-    }
-    cds_config->mutable_initial_fetch_timeout()->set_seconds(cds_timeout_seconds_);
-    cds_config->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3);
-    bootstrap->add_node_context_params("cluster");
-    // add a stat prefix we use in test
-    list->add_patterns()->set_exact("cluster_manager.active_clusters");
-    list->add_patterns()->set_exact("cluster_manager.cluster_added");
+#ifdef ENVOY_GOOGLE_GRPC
+  if (xds_builder_) {
+    xds_builder_->build(bootstrap.get());
   }
+#endif
 
   envoy::config::listener::v3::ApiListenerManager api;
   auto* listener_manager = bootstrap->mutable_listener_manager();
diff --git a/mobile/library/cc/engine_builder.h b/mobile/library/cc/engine_builder.h
index c2562a2ddfdd..3c6a2640b5c2 100644
--- a/mobile/library/cc/engine_builder.h
+++ b/mobile/library/cc/engine_builder.h
@@ -20,9 +20,12 @@
 namespace Envoy {
 namespace Platform {
 
-constexpr int DefaultJwtTokenLifetimeSeconds = 31536000; // 1 year
+constexpr int DefaultJwtTokenLifetimeSeconds = 60 * 60 * 24 * 90; // 90 days
 constexpr int DefaultXdsTimeout = 5;
 
+// Forward declaration so it can be referenced by XdsBuilder.
+class EngineBuilder;
+
 // Represents the locality information in the Bootstrap's node, as defined in:
 // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/base.proto#envoy-v3-api-msg-config-core-v3-locality
 struct NodeLocality {
@@ -31,6 +34,85 @@ struct NodeLocality {
   std::string sub_zone;
 };
 
+#ifdef ENVOY_GOOGLE_GRPC
+// A class for building the xDS configuration for the Envoy Mobile engine.
+// xDS is a protocol for dynamic configuration of Envoy instances, more information can be found in:
+// https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol.
+//
+// This class is typically used as input to the EngineBuilder's setXds() method.
+class XdsBuilder final {
+public:
+  // `xds_server_address`: the host name or IP address of the xDS management server. The xDS server
+  //                       must support the ADS protocol
+  //                       (https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration#aggregated-xds-ads).
+  // `xds_server_port`: the port on which the xDS management server listens for ADS discovery
+  //                    requests.
+  XdsBuilder(std::string xds_server_address, const int xds_server_port);
+
+  // Sets JWT as the authentication method to the xDS management server, using the given token.
+  //
+  // `token`: the JWT token used to authenticate the client to the xDS management server.
+  // `token_lifetime_in_seconds`:  the lifetime of the JWT token, in seconds. If none
+  //                              (or 0) is specified, then DefaultJwtTokenLifetimeSeconds is used.
+  XdsBuilder&
+  setJwtAuthenticationToken(std::string token,
+                            int token_lifetime_in_seconds = DefaultJwtTokenLifetimeSeconds);
+
+  // Sets the PEM-encoded server root certificates used to negotiate the TLS handshake for the gRPC
+  // connection. If no root certs are specified, the operating system defaults are used.
+  XdsBuilder& setSslRootCerts(std::string root_certs);
+
+  // Adds Runtime Discovery Service (RTDS) to the Runtime layers of the Bootstrap configuration,
+  // to retrieve dynamic runtime configuration via the xDS management server.
+  //
+  // `resource_name`: The runtime config resource to subscribe to.
+  // `timeout_in_seconds`:  specifies the `initial_fetch_timeout` field on the
+  //    api.v3.core.ConfigSource. Unlike the ConfigSource default of 15s, we set a default fetch
+  //    timeout value of 5s, to prevent mobile app initialization from stalling. The default
+  //    parameter value may change through the course of experimentation and no assumptions should
+  //    be made of its exact value.
+  XdsBuilder& addRuntimeDiscoveryService(std::string resource_name,
+                                         int timeout_in_seconds = DefaultXdsTimeout);
+
+  // Adds the Cluster Discovery Service (CDS) configuration for retrieving dynamic cluster resources
+  // via the xDS management server.
+  //
+  // `cds_resources_locator`:  the xdstp:// URI for subscribing to the cluster resources.
+  //    If not using xdstp, then `cds_resources_locator` should be set to the empty string.
+  // `timeout_in_seconds`:  specifies the `initial_fetch_timeout` field on the
+  //    api.v3.core.ConfigSource. Unlike the ConfigSource default of 15s, we set a default fetch
+  //    timeout value of 5s, to prevent mobile app initialization from stalling. The default
+  //    parameter value may change through the course of experimentation and no assumptions should
+  //    be made of its exact value.
+  XdsBuilder& addClusterDiscoveryService(std::string cds_resources_locator = "",
+                                         int timeout_in_seconds = DefaultXdsTimeout);
+
+protected:
+  // Sets the xDS configuration specified on this XdsBuilder instance on the Bootstrap proto
+  // provided as an input parameter.
+  //
+  // This method takes in a modifiable Bootstrap proto pointer because returning a new Bootstrap
+  // proto would rely on proto's MergeFrom behavior, which can lead to unexpected results in the
+  // Bootstrap config.
+  void build(envoy::config::bootstrap::v3::Bootstrap* bootstrap) const;
+
+private:
+  // Required so that EngineBuilder can call the XdsBuilder's protected build() method.
+  friend class EngineBuilder;
+
+  std::string xds_server_address_;
+  int xds_server_port_;
+  std::string jwt_token_;
+  int jwt_token_lifetime_in_seconds_ = DefaultJwtTokenLifetimeSeconds;
+  std::string ssl_root_certs_;
+  std::string rtds_resource_name_;
+  int rtds_timeout_in_seconds_ = DefaultXdsTimeout;
+  bool enable_cds_ = false;
+  std::string cds_resources_locator_;
+  int cds_timeout_in_seconds_ = DefaultXdsTimeout;
+};
+#endif
+
 // The C++ Engine builder creates a structured bootstrap proto and modifies it through parameters
 // set through the EngineBuilder API calls to produce the Bootstrap config that the Engine is
 // created from.
@@ -69,24 +151,15 @@ class EngineBuilder {
   EngineBuilder& enableDrainPostDnsRefresh(bool drain_post_dns_refresh_on);
   EngineBuilder& enforceTrustChainVerification(bool trust_chain_verification_on);
   EngineBuilder& enablePlatformCertificatesValidation(bool platform_certificates_validation_on);
-#ifdef ENVOY_GOOGLE_GRPC
   // Sets the node.id field in the Bootstrap configuration.
   EngineBuilder& setNodeId(std::string node_id);
   // Sets the node.locality field in the Bootstrap configuration.
   EngineBuilder& setNodeLocality(std::string region, std::string zone, std::string sub_zone);
-  // Adds an ADS layer. Note that only the state-of-the-world gRPC protocol is supported, not Delta
-  // gRPC.
-  EngineBuilder& setAggregatedDiscoveryService(std::string address, const int port,
-                                               std::string jwt_token = "",
-                                               int jwt_token_lifetime_seconds = 0,
-                                               std::string ssl_root_certs = "");
-  // Adds an RTDS layer to default config. Requires that ADS be configured.
-  EngineBuilder& addRtdsLayer(std::string layer_name, const int timeout_seconds = 0);
-  // Adds a CDS layer to default config. Requires that ADS be configured via
-  // setAggregatedDiscoveryService(). If `cds_resources_locator` is non-empty, the xdstp namespace
-  // is used for identifying resources. If not using xdstp, then set `cds_resources_locator` to the
-  // empty string.
-  EngineBuilder& addCdsLayer(std::string cds_resources_locator = "", const int timeout_seconds = 0);
+#ifdef ENVOY_GOOGLE_GRPC
+  // Sets the xDS configuration for the Envoy Mobile engine.
+  //
+  // `xds_builder`: the XdsBuilder instance used to specify the xDS configuration options.
+  EngineBuilder& setXds(XdsBuilder xds_builder);
 #endif
   EngineBuilder& enableDnsCache(bool dns_cache_on, int save_interval_seconds = 1);
   EngineBuilder& setForceAlwaysUsev6(bool value);
@@ -142,18 +215,8 @@ class EngineBuilder {
   bool brotli_decompression_filter_ = false;
   bool socket_tagging_filter_ = false;
   bool platform_certificates_validation_on_ = false;
-  std::string rtds_layer_name_ = "";
-  int rtds_timeout_seconds_;
   std::string node_id_;
   absl::optional node_locality_ = absl::nullopt;
-  std::string ads_address_ = ""; // make absl:optional?
-  int ads_port_;
-  std::string ads_jwt_token_;
-  int ads_jwt_token_lifetime_seconds_;
-  std::string ads_ssl_root_certs_;
-  std::string cds_resources_locator_;
-  int cds_timeout_seconds_;
-  bool enable_cds_ = false;
   bool dns_cache_on_ = false;
   int dns_cache_save_interval_seconds_ = 1;
 
@@ -173,6 +236,10 @@ class EngineBuilder {
 
   std::vector> runtime_guards_;
   absl::flat_hash_map string_accessors_;
+
+#ifdef ENVOY_GOOGLE_GRPC
+  absl::optional xds_builder_ = absl::nullopt;
+#endif
 };
 
 using EngineBuilderSharedPtr = std::shared_ptr;
diff --git a/mobile/library/common/jni/jni_interface.cc b/mobile/library/common/jni/jni_interface.cc
index daeb92d36b9a..6371db7e7862 100644
--- a/mobile/library/common/jni/jni_interface.cc
+++ b/mobile/library/common/jni/jni_interface.cc
@@ -1194,24 +1194,23 @@ javaObjectArrayToStringPairVector(JNIEnv* env, jobjectArray entries) {
   return ret;
 }
 
-void configureBuilder(
-    JNIEnv* env, jstring grpc_stats_domain, jlong connect_timeout_seconds,
-    jlong dns_refresh_seconds, jlong dns_failure_refresh_seconds_base,
-    jlong dns_failure_refresh_seconds_max, jlong dns_query_timeout_seconds,
-    jlong dns_min_refresh_seconds, jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache,
-    jlong dns_cache_save_interval_seconds, jboolean enable_drain_post_dns_refresh,
-    jboolean enable_http3, jboolean enable_gzip_decompression, jboolean enable_brotli_decompression,
-    jboolean enable_socket_tagging, jboolean enable_interface_binding,
-    jlong h2_connection_keepalive_idle_interval_milliseconds,
-    jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host,
-    jlong stats_flush_seconds, jlong stream_idle_timeout_seconds,
-    jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id,
-    jboolean trust_chain_verification, jobjectArray filter_chain, jobjectArray stat_sinks,
-    jboolean enable_platform_certificates_validation, jobjectArray runtime_guards,
-    jstring rtds_layer_name, jlong rtds_timeout_seconds, jstring ads_address, jlong ads_port,
-    jstring ads_token, jlong ads_token_lifetime, jstring ads_root_certs, jstring node_id,
-    jstring node_region, jstring node_zone, jstring node_sub_zone, jstring cds_resources_locator,
-    jlong cds_timeout_seconds, jboolean enable_cds, Envoy::Platform::EngineBuilder& builder) {
+void configureBuilder(JNIEnv* env, jstring grpc_stats_domain, jlong connect_timeout_seconds,
+                      jlong dns_refresh_seconds, jlong dns_failure_refresh_seconds_base,
+                      jlong dns_failure_refresh_seconds_max, jlong dns_query_timeout_seconds,
+                      jlong dns_min_refresh_seconds, jobjectArray dns_preresolve_hostnames,
+                      jboolean enable_dns_cache, jlong dns_cache_save_interval_seconds,
+                      jboolean enable_drain_post_dns_refresh, jboolean enable_http3,
+                      jboolean enable_gzip_decompression, jboolean enable_brotli_decompression,
+                      jboolean enable_socket_tagging, jboolean enable_interface_binding,
+                      jlong h2_connection_keepalive_idle_interval_milliseconds,
+                      jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host,
+                      jlong stats_flush_seconds, jlong stream_idle_timeout_seconds,
+                      jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id,
+                      jboolean trust_chain_verification, jobjectArray filter_chain,
+                      jobjectArray stat_sinks, jboolean enable_platform_certificates_validation,
+                      jobjectArray runtime_guards, jstring node_id, jstring node_region,
+                      jstring node_zone, jstring node_sub_zone,
+                      Envoy::Platform::EngineBuilder& builder) {
   builder.addConnectTimeoutSeconds((connect_timeout_seconds));
   builder.addDnsRefreshSeconds((dns_refresh_seconds));
   builder.addDnsFailureRefreshSeconds((dns_failure_refresh_seconds_base),
@@ -1262,11 +1261,6 @@ void configureBuilder(
 
   std::vector hostnames = javaObjectArrayToStringVector(env, dns_preresolve_hostnames);
   builder.addDnsPreresolveHostnames(hostnames);
-#ifdef ENVOY_GOOGLE_GRPC
-  std::string native_rtds_layer_name = getCppString(env, rtds_layer_name);
-  if (!native_rtds_layer_name.empty()) {
-    builder.addRtdsLayer(native_rtds_layer_name, rtds_timeout_seconds);
-  }
   std::string native_node_id = getCppString(env, node_id);
   if (!native_node_id.empty()) {
     builder.setNodeId(native_node_id);
@@ -1276,16 +1270,6 @@ void configureBuilder(
     builder.setNodeLocality(native_node_region, getCppString(env, node_zone),
                             getCppString(env, node_sub_zone));
   }
-  std::string native_ads_address = getCppString(env, ads_address);
-  if (!native_ads_address.empty()) {
-    builder.setAggregatedDiscoveryService(native_ads_address, ads_port,
-                                          getCppString(env, ads_token), ads_token_lifetime,
-                                          getCppString(env, ads_root_certs));
-  }
-  if (enable_cds == JNI_TRUE) {
-    builder.addCdsLayer(getCppString(env, cds_resources_locator), cds_timeout_seconds);
-  }
-#endif
 }
 
 extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_createBootstrap(
@@ -1302,8 +1286,8 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr
     jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id,
     jboolean trust_chain_verification, jobjectArray filter_chain, jobjectArray stat_sinks,
     jboolean enable_platform_certificates_validation, jobjectArray runtime_guards,
-    jstring rtds_layer_name, jlong rtds_timeout_seconds, jstring ads_address, jlong ads_port,
-    jstring ads_token, jlong ads_token_lifetime, jstring ads_root_certs, jstring node_id,
+    jstring rtds_resource_name, jlong rtds_timeout_seconds, jstring xds_address, jlong xds_port,
+    jstring xds_jwt_token, jlong xds_jwt_token_lifetime, jstring xds_root_certs, jstring node_id,
     jstring node_region, jstring node_zone, jstring node_sub_zone, jstring cds_resources_locator,
     jlong cds_timeout_seconds, jboolean enable_cds) {
   Envoy::Platform::EngineBuilder builder;
@@ -1318,9 +1302,32 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr
       h2_connection_keepalive_timeout_seconds, max_connections_per_host, stats_flush_seconds,
       stream_idle_timeout_seconds, per_try_idle_timeout_seconds, app_version, app_id,
       trust_chain_verification, filter_chain, stat_sinks, enable_platform_certificates_validation,
-      runtime_guards, rtds_layer_name, rtds_timeout_seconds, ads_address, ads_port, ads_token,
-      ads_token_lifetime, ads_root_certs, node_id, node_region, node_zone, node_sub_zone,
-      cds_resources_locator, cds_timeout_seconds, enable_cds, builder);
+      runtime_guards, node_id, node_region, node_zone, node_sub_zone, builder);
+
+#ifdef ENVOY_GOOGLE_GRPC
+  std::string native_xds_address = getCppString(env, xds_address);
+  if (!native_xds_address.empty()) {
+    Envoy::Platform::XdsBuilder xds_builder(std::move(native_xds_address), xds_port);
+    std::string native_jwt_token = getCppString(env, xds_jwt_token);
+    if (!native_jwt_token.empty()) {
+      xds_builder.setJwtAuthenticationToken(std::move(native_jwt_token), xds_jwt_token_lifetime);
+    }
+    std::string native_root_certs = getCppString(env, xds_root_certs);
+    if (!native_root_certs.empty()) {
+      xds_builder.setSslRootCerts(std::move(native_root_certs));
+    }
+    std::string native_rtds_resource_name = getCppString(env, rtds_resource_name);
+    if (!native_rtds_resource_name.empty()) {
+      xds_builder.addRuntimeDiscoveryService(std::move(native_rtds_resource_name),
+                                             rtds_timeout_seconds);
+    }
+    if (enable_cds == JNI_TRUE) {
+      xds_builder.addClusterDiscoveryService(getCppString(env, cds_resources_locator),
+                                             cds_timeout_seconds);
+    }
+    builder.setXds(std::move(xds_builder));
+  }
+#endif
 
   return reinterpret_cast(builder.generateBootstrap().release());
 }
diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java
index d98af79c1fd0..9832b60d4e25 100644
--- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java
+++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java
@@ -60,13 +60,13 @@ public enum TrustChainVerification {
   public final List statSinks;
   public final Map runtimeGuards;
   public final Boolean enablePlatformCertificatesValidation;
-  public final String rtdsLayerName;
+  public final String rtdsResourceName;
   public final Integer rtdsTimeoutSeconds;
-  public final String adsAddress;
-  public final Integer adsPort;
-  public final String adsToken;
-  public final Integer adsTokenLifetime;
-  public final String adsRootCerts;
+  public final String xdsAddress;
+  public final Integer xdsPort;
+  public final String xdsJwtToken;
+  public final Integer xdsJwtTokenLifetime;
+  public final String xdsRootCerts;
   public final String nodeId;
   public final String nodeRegion;
   public final String nodeZone;
@@ -129,20 +129,20 @@ public enum TrustChainVerification {
    * @param stringAccessors                               platform string accessors to register.
    * @param keyValueStores                                platform key-value store implementations.
    * @param enablePlatformCertificatesValidation          whether to use the platform verifier.
-   * @param rtdsLayerName                                 the RTDS layer name for this client.
+   * @param rtdsResourceName                                 the RTDS layer name for this client.
    * @param rtdsTimeoutSeconds                            the timeout for RTDS fetches.
-   * @param adsAddress                                    the address for the ADS server.
-   * @param adsPort                                       the port for the ADS server.
-   * @param adsToken                                      the token to use for authenticating with
-   *                                                      the ADS server.
-   * @param adsTokenLifetime                              the lifetime of the ADS token.
-   * @param adsRootCerts                                  the root certificates to use for
-   *     validating the ADS server.
-   * @param nodeId                                        the node ID to use for the ADS server.
-   * @param nodeRegion                                    the node region to use for the ADS server.
-   * @param nodeZone                                      the node zone to use for the ADS server.
-   * @param nodeSubZone                                   the node sub-zone to use for the ADS
-   *     server.
+   * @param xdsAddress                                    the address for the xDS management server.
+   * @param xdsPort                                       the port for the xDS server.
+   * @param xdsJwtToken                                   the JWT token to use for authenticating
+   *                                                      with the xDS server.
+   * @param xdsTokenLifetime                              the lifetime of the JWT token.
+   * @param xdsRootCerts                                  the root certificates to use for the TLS
+   *                                                      handshake during connection establishment
+   *                                                      with the xDS management server.
+   * @param nodeId                                        the node ID in the Node metadata.
+   * @param nodeRegion                                    the node region in the Node metadata.
+   * @param nodeZone                                      the node zone in the Node metadata.
+   * @param nodeSubZone                                   the node sub-zone in the Node metadata.
    * @param cdsResourcesLocator                           the resources locator for CDS.
    * @param cdsTimeoutSeconds                             the timeout for CDS fetches.
    * @param enableCds                                     enables CDS, used because all CDS params
@@ -164,8 +164,8 @@ public EnvoyConfiguration(
       Map stringAccessors,
       Map keyValueStores, List statSinks,
       Map runtimeGuards, boolean enablePlatformCertificatesValidation,
-      String rtdsLayerName, Integer rtdsTimeoutSeconds, String adsAddress, Integer adsPort,
-      String adsToken, Integer adsTokenLifetime, String adsRootCerts, String nodeId,
+      String rtdsResourceName, Integer rtdsTimeoutSeconds, String xdsAddress, Integer xdsPort,
+      String xdsJwtToken, Integer xdsJwtTokenLifetime, String xdsRootCerts, String nodeId,
       String nodeRegion, String nodeZone, String nodeSubZone, String cdsResourcesLocator,
       Integer cdsTimeoutSeconds, boolean enableCds) {
     JniLibrary.load();
@@ -217,13 +217,13 @@ public EnvoyConfiguration(
       this.runtimeGuards.put(guardAndValue.getKey(), String.valueOf(guardAndValue.getValue()));
     }
     this.enablePlatformCertificatesValidation = enablePlatformCertificatesValidation;
-    this.rtdsLayerName = rtdsLayerName;
+    this.rtdsResourceName = rtdsResourceName;
     this.rtdsTimeoutSeconds = rtdsTimeoutSeconds;
-    this.adsAddress = adsAddress;
-    this.adsPort = adsPort;
-    this.adsToken = adsToken;
-    this.adsTokenLifetime = adsTokenLifetime;
-    this.adsRootCerts = adsRootCerts;
+    this.xdsAddress = xdsAddress;
+    this.xdsPort = xdsPort;
+    this.xdsJwtToken = xdsJwtToken;
+    this.xdsJwtTokenLifetime = xdsJwtTokenLifetime;
+    this.xdsRootCerts = xdsRootCerts;
     this.nodeId = nodeId;
     this.nodeRegion = nodeRegion;
     this.nodeZone = nodeZone;
@@ -253,9 +253,9 @@ public long createBootstrap() {
         h2ConnectionKeepaliveTimeoutSeconds, maxConnectionsPerHost, statsFlushSeconds,
         streamIdleTimeoutSeconds, perTryIdleTimeoutSeconds, appVersion, appId,
         enforceTrustChainVerification, filter_chain, stats_sinks,
-        enablePlatformCertificatesValidation, runtime_guards, rtdsLayerName, rtdsTimeoutSeconds,
-        adsAddress, adsPort, adsToken, adsTokenLifetime, adsRootCerts, nodeId, nodeRegion, nodeZone,
-        nodeSubZone, cdsResourcesLocator, cdsTimeoutSeconds, enableCds);
+        enablePlatformCertificatesValidation, runtime_guards, rtdsResourceName, rtdsTimeoutSeconds,
+        xdsAddress, xdsPort, xdsJwtToken, xdsJwtTokenLifetime, xdsRootCerts, nodeId, nodeRegion,
+        nodeZone, nodeSubZone, cdsResourcesLocator, cdsTimeoutSeconds, enableCds);
   }
 
   static class ConfigurationException extends RuntimeException {
diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java
index 6fe419ff422c..98182676420c 100644
--- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java
+++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java
@@ -309,8 +309,9 @@ public static native long createBootstrap(
       long maxConnectionsPerHost, long statsFlushSeconds, long streamIdleTimeoutSeconds,
       long perTryIdleTimeoutSeconds, String appVersion, String appId,
       boolean trustChainVerification, byte[][] filterChain, byte[][] statSinks,
-      boolean enablePlatformCertificatesValidation, byte[][] runtimeGuards, String rtdsLayerName,
-      long rtdsTimeoutSeconds, String adsAddress, long adsPort, String adsToken,
-      long adsTokenLifetime, String adsRootCerts, String nodeId, String nodeRegion, String nodeZone,
-      String nodeSubZone, String cdsResourcesLocator, long cdsTimeoutSeconds, boolean enableCds);
+      boolean enablePlatformCertificatesValidation, byte[][] runtimeGuards, String rtdsResourceName,
+      long rtdsTimeoutSeconds, String xdsAddress, long xdsPort, String xdsJwtToken,
+      long xdsJwtTokenLifetime, String xdsRootCerts, String nodeId, String nodeRegion,
+      String nodeZone, String nodeSubZone, String cdsResourcesLocator, long cdsTimeoutSeconds,
+      boolean enableCds);
 }
diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java
index 2091599c82b7..ef40522f6a20 100644
--- a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java
+++ b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java
@@ -61,20 +61,10 @@ public class NativeCronvoyEngineBuilderImpl extends CronvoyEngineBuilderImpl {
   private String mAppId = "unspecified";
   private TrustChainVerification mTrustChainVerification = VERIFY_TRUST_CHAIN;
   private boolean mEnablePlatformCertificatesValidation = true;
-  private String mRtdsLayerName = "";
-  private int mRtdsTimeoutSeconds = 0;
-  private String mAdsAddress = "";
-  private int mAdsPort = 0;
-  private String mAdsToken = "";
-  private int mAdsTokenLifetime = 0;
-  private String mAdsRootCerts = "";
   private String mNodeId = "";
   private String mNodeRegion = "";
   private String mNodeZone = "";
   private String mNodeSubZone = "";
-  private String mCdsResourcesLocator = "";
-  private int mCdsTimeoutSeconds = 0;
-  private boolean mEnableCds = false;
 
   /**
    * Builder for Native Cronet Engine. Default config enables SPDY, disables QUIC and HTTP cache.
@@ -141,8 +131,9 @@ mEnableDrainPostDnsRefresh, quicEnabled(), mEnableGzipDecompression, brotliEnabl
         mStreamIdleTimeoutSeconds, mPerTryIdleTimeoutSeconds, mAppVersion, mAppId,
         mTrustChainVerification, nativeFilterChain, platformFilterChain, stringAccessors,
         keyValueStores, statSinks, runtimeGuards, mEnablePlatformCertificatesValidation,
-        mRtdsLayerName, mRtdsTimeoutSeconds, mAdsAddress, mAdsPort, mAdsToken, mAdsTokenLifetime,
-        mAdsRootCerts, mNodeId, mNodeRegion, mNodeZone, mNodeSubZone, mCdsResourcesLocator,
-        mCdsTimeoutSeconds, mEnableCds);
+        /*rtdsResourceName=*/"", /*rtdsTimeoutSeconds=*/0, /*xdsAddress=*/"",
+        /*xdsPort=*/0, /*xdsJwtToken=*/"", /*xdsJwtTokenLifetime=*/0, /*xdsSslRootCerts=*/"",
+        mNodeId, mNodeRegion, mNodeZone, mNodeSubZone, /*cdsResourcesLocator=*/"",
+        /*cdsTimeoutSeconds=*/0, /*enableCds=*/false);
   }
 }
diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt
index de7a7d930e8e..d6ddb8bd6bca 100644
--- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt
+++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt
@@ -27,6 +27,117 @@ class Standard : BaseConfiguration()
  */
 class Custom(val yaml: String) : BaseConfiguration()
 
+/**
+  * Builder for generating the xDS configuration for the Envoy Mobile engine.
+  * xDS is a protocol for dynamic configuration of Envoy instances, more information can be found in
+  * https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol.
+  *
+  * This class is typically used as input to the EngineBuilder's setXds() method.
+ */
+open class XdsBuilder (
+  internal val xdsServerAddress: String,
+  internal val xdsServerPort: Int
+) {
+  companion object {
+    private const val DEFAULT_JWT_TOKEN_LIFETIME_IN_SECONDS: Int = 60 * 60 * 24 * 90 // 90 days
+    private const val DEFAULT_XDS_TIMEOUT_IN_SECONDS: Int = 5
+  }
+
+  internal var jwtToken: String? = null
+  internal var jwtTokenLifetimeInSeconds: Int = DEFAULT_JWT_TOKEN_LIFETIME_IN_SECONDS
+  internal var sslRootCerts: String? = null
+  internal var rtdsResourceName: String? = null
+  internal var rtdsTimeoutInSeconds: Int = DEFAULT_XDS_TIMEOUT_IN_SECONDS
+  internal var enableCds: Boolean = false
+  internal var cdsResourcesLocator: String? = null
+  internal var cdsTimeoutInSeconds: Int = DEFAULT_XDS_TIMEOUT_IN_SECONDS
+
+  /**
+   * Sets JWT as the authentication method to the xDS management server, using the given token.
+   *
+   * @param token The JWT token used to authenticate the client to the xDS management server.
+   * @param tokenLifetimeInSeconds  The lifetime of the JWT token, in seconds. If none
+   *                               (or 0) is specified, then defaultJwtTokenLifetimeSeconds is
+   *                               used.
+   *
+   * @return this builder.
+   */
+  fun setJwtAuthenticationToken(
+    token: String,
+    tokenLifetimeInSeconds: Int = DEFAULT_JWT_TOKEN_LIFETIME_IN_SECONDS
+  ): XdsBuilder {
+    this.jwtToken = token
+    this.jwtTokenLifetimeInSeconds = if (tokenLifetimeInSeconds > 0)
+        tokenLifetimeInSeconds else
+        DEFAULT_JWT_TOKEN_LIFETIME_IN_SECONDS
+    return this
+  }
+
+  /**
+   * Sets the PEM-encoded server root certificates used to negotiate the TLS handshake for the gRPC
+   * connection. If no root certs are specified, the operating system defaults are used.
+   *
+   * @param rootCerts The PEM-encoded server root certificates.
+   *
+   * @return this builder.
+   */
+  fun setSslRootCerts(rootCerts: String): XdsBuilder {
+    this.sslRootCerts = rootCerts
+    return this
+  }
+
+  /**
+   * Adds Runtime Discovery Service (RTDS) to the Runtime layers of the Bootstrap configuration,
+   * to retrieve dynamic runtime configuration via the xDS management server.
+   *
+   * @param resourceName The runtime config resource to subscribe to.
+   * @param timeoutInSeconds  specifies the `initial_fetch_timeout` field on the
+   *     api.v3.core.ConfigSource. Unlike the ConfigSource default of 15s, we set a default fetch
+   *     timeout value of 5s, to prevent mobile app initialization from stalling. The default
+   *     parameter value may change through the course of experimentation and no assumptions should
+   *     be made of its exact value.
+   *
+   * @return this builder.
+   */
+  fun addRuntimeDiscoveryService(
+    resourceName: String,
+    timeoutInSeconds: Int = DEFAULT_XDS_TIMEOUT_IN_SECONDS
+  ): XdsBuilder {
+    this.rtdsResourceName = resourceName
+    this.rtdsTimeoutInSeconds = timeoutOrXdsDefault(timeoutInSeconds)
+    return this
+  }
+
+  /**
+   * Adds the Cluster Discovery Service (CDS) configuration for retrieving dynamic cluster
+   * resources via the xDS management server.
+   *
+   * @param cdsResourcesLocator  the xdstp:// URI for subscribing to the cluster
+   *     resources. If not using xdstp, then `cds_resources_locator` should be set to the empty
+   *     string.
+   * @param timeoutInSeconds  specifies the `initial_fetch_timeout` field on the
+   *     api.v3.core.ConfigSource. Unlike the ConfigSource default of 15s, we set a default fetch
+   *     timeout value of 5s, to prevent mobile app initialization from stalling. The default
+   *     parameter value may change through the course of experimentation and no assumptions should
+   *     be made of its exact value.
+   *
+   * @return this builder.
+   */
+  public fun addClusterDiscoveryService(
+    cdsResourcesLocator: String? = null,
+    timeoutInSeconds: Int = DEFAULT_XDS_TIMEOUT_IN_SECONDS
+  ): XdsBuilder {
+    this.enableCds = true
+    this.cdsResourcesLocator = cdsResourcesLocator
+    this.cdsTimeoutInSeconds = timeoutOrXdsDefault(timeoutInSeconds)
+    return this
+  }
+
+  private fun timeoutOrXdsDefault(timeout: Int): Int {
+    return if (timeout > 0) timeout else DEFAULT_XDS_TIMEOUT_IN_SECONDS
+  }
+}
+
 /**
  * Builder used for creating and running a new `Engine` instance.
  */
@@ -73,20 +184,11 @@ open class EngineBuilder(
   private var keyValueStores = mutableMapOf()
   private var statsSinks = listOf()
   private var enablePlatformCertificatesValidation = false
-  private var rtdsLayerName: String = ""
-  private var rtdsTimeoutSeconds: Int = 0
-  private var adsAddress: String = ""
-  private var adsPort: Int = 0
-  private var adsJwtToken: String = ""
-  private var adsJwtTokenLifetimeSeconds: Int = 0
-  private var adsSslRootCerts: String = ""
   private var nodeId: String = ""
   private var nodeRegion: String = ""
   private var nodeZone: String = ""
   private var nodeSubZone: String = ""
-  private var cdsResourcesLocator: String = ""
-  private var cdsTimeoutSeconds: Int = 0
-  private var enableCds: Boolean = false
+  private var xdsBuilder: XdsBuilder? = null
 
   /**
    * Add a log level to use with Envoy.
@@ -546,72 +648,14 @@ open class EngineBuilder(
   }
 
   /**
-  * Adds an ADS layer.
-  * Note that only the state-of-the-world gRPC protocol is supported, not Delta gRPC.
-  *
-  * @param address the network address of the server.
-  *
-  * @param port the port of the server.
-  *
-  * @param jwtToken the JWT token.
-  *
-  * @param jwtTokenLifetimeSeconds the lifetime of the JWT token. If zero,
-  *                                a default value is set in engine_builder.h.
-  *
-  * @param sslRootCerts the SSL root certificates.
-  *
-  * @return this builder.
-  */
-  fun setAggregatedDiscoveryService(
-    address: String,
-    port: Int,
-    jwtToken: String = "",
-    jwtTokenLifetimeSeconds: Int = 0,
-    sslRootCerts: String = ""
-  ): EngineBuilder {
-    this.adsAddress = address
-    this.adsPort = port
-    this.adsJwtToken = jwtToken
-    this.adsJwtTokenLifetimeSeconds = jwtTokenLifetimeSeconds
-    this.adsSslRootCerts = sslRootCerts
-    return this
-  }
-
-  /**
-  * Adds a CDS layer.
-  *
-  * @param resourcesLocator The xdstp resource URI for fetching clusters.
-  *                         If empty, xdstp is not used and a wildcard is inferred.
-  *
-  * @param timeoutSeconds The timeout in seconds. If zero, a default value is
-  *                       set in engine_builder.h.
-  *
-  * @return this builder.
-  */
-  fun addCdsLayer(
-    resourcesLocator: String = "",
-    timeoutSeconds: Int = 0,
-  ): EngineBuilder {
-    this.cdsResourcesLocator = resourcesLocator
-    this.cdsTimeoutSeconds = timeoutSeconds
-    this.enableCds = true
-    return this
-  }
-
-
-  /**
-  * Adds an RTDS layer to default config. Requires that ADS be configured.
-  *
-  * @param layerName the layer name.
-  *
-  * @param timeoutSeconds The timeout in seconds. If zero, a default value is
-  *                       set in engine_builder.h.
-  *
-  * @return this builder.
-  */
-  fun addRtdsLayer(layerName: String, timeoutSeconds: Int = 0): EngineBuilder {
-    this.rtdsLayerName = layerName
-    this.rtdsTimeoutSeconds = timeoutSeconds
+   * Sets the xDS configuration for the Envoy Mobile engine.
+   *
+   * @param xdsBuilder The XdsBuilder instance from which to construct the xDS configuration.
+   *
+   * @return this builder.
+   */
+  fun setXds(xdsBuilder: XdsBuilder): EngineBuilder {
+    this.xdsBuilder = xdsBuilder
     return this
   }
 
@@ -668,20 +712,20 @@ open class EngineBuilder(
       statsSinks,
       runtimeGuards,
       enablePlatformCertificatesValidation,
-      rtdsLayerName,
-      rtdsTimeoutSeconds,
-      adsAddress,
-      adsPort,
-      adsJwtToken,
-      adsJwtTokenLifetimeSeconds,
-      adsSslRootCerts,
+      xdsBuilder?.rtdsResourceName,
+      xdsBuilder?.rtdsTimeoutInSeconds ?: 0,
+      xdsBuilder?.xdsServerAddress,
+      xdsBuilder?.xdsServerPort ?: 0,
+      xdsBuilder?.jwtToken,
+      xdsBuilder?.jwtTokenLifetimeInSeconds ?: 0,
+      xdsBuilder?.sslRootCerts,
       nodeId,
       nodeRegion,
       nodeZone,
       nodeSubZone,
-      cdsResourcesLocator,
-      cdsTimeoutSeconds,
-      enableCds,
+      xdsBuilder?.cdsResourcesLocator,
+      xdsBuilder?.cdsTimeoutInSeconds ?: 0,
+      xdsBuilder?.enableCds ?: false,
     )
 
 
diff --git a/mobile/library/objective-c/EnvoyConfiguration.h b/mobile/library/objective-c/EnvoyConfiguration.h
index 3716815181d8..a1df1950bbc3 100644
--- a/mobile/library/objective-c/EnvoyConfiguration.h
+++ b/mobile/library/objective-c/EnvoyConfiguration.h
@@ -35,28 +35,28 @@ NS_ASSUME_NONNULL_BEGIN
 @property (nonatomic, assign) UInt32 statsFlushSeconds;
 @property (nonatomic, assign) UInt32 streamIdleTimeoutSeconds;
 @property (nonatomic, assign) UInt32 perTryIdleTimeoutSeconds;
-@property (nonatomic, strong) NSString *appVersion;
-@property (nonatomic, strong) NSString *appId;
+@property (nonatomic, strong, nullable) NSString *appVersion;
+@property (nonatomic, strong, nullable) NSString *appId;
 @property (nonatomic, strong) NSDictionary *runtimeGuards;
 @property (nonatomic, strong) NSArray *nativeFilterChain;
 @property (nonatomic, strong) NSArray *httpPlatformFilterFactories;
 @property (nonatomic, strong) NSDictionary *stringAccessors;
 @property (nonatomic, strong) NSDictionary> *keyValueStores;
 @property (nonatomic, strong) NSArray *statsSinks;
-@property (nonatomic, strong, nullable) NSString *rtdsLayerName;
-@property (nonatomic, assign) UInt32 rtdsTimeoutSeconds;
-@property (nonatomic, strong, nullable) NSString *adsAddress;
-@property (nonatomic, assign) UInt32 adsPort;
-@property (nonatomic, strong, nullable) NSString *adsJwtToken;
-@property (nonatomic, assign) UInt32 adsJwtTokenLifetimeSeconds;
-@property (nonatomic, strong, nullable) NSString *adsSslRootCerts;
 @property (nonatomic, strong, nullable) NSString *nodeId;
 @property (nonatomic, strong, nullable) NSString *nodeRegion;
 @property (nonatomic, strong, nullable) NSString *nodeZone;
 @property (nonatomic, strong, nullable) NSString *nodeSubZone;
-@property (nonatomic, strong) NSString *cdsResourcesLocator;
-@property (nonatomic, assign) UInt32 cdsTimeoutSeconds;
+@property (nonatomic, strong, nullable) NSString *xdsServerAddress;
+@property (nonatomic, assign) UInt32 xdsServerPort;
+@property (nonatomic, strong, nullable) NSString *xdsJwtToken;
+@property (nonatomic, assign) UInt32 xdsJwtTokenLifetimeSeconds;
+@property (nonatomic, strong, nullable) NSString *xdsSslRootCerts;
+@property (nonatomic, strong, nullable) NSString *rtdsResourceName;
+@property (nonatomic, assign) UInt32 rtdsTimeoutSeconds;
 @property (nonatomic, assign) BOOL enableCds;
+@property (nonatomic, strong, nullable) NSString *cdsResourcesLocator;
+@property (nonatomic, assign) UInt32 cdsTimeoutSeconds;
 @property (nonatomic, assign) intptr_t bootstrapPointer;
 
 /**
@@ -102,20 +102,20 @@ NS_ASSUME_NONNULL_BEGIN
                                        (NSDictionary> *)
                                            keyValueStores
                                        statsSinks:(NSArray *)statsSinks
-                                    rtdsLayerName:(nullable NSString *)rtdsLayerName
-                               rtdsTimeoutSeconds:(UInt32)rtdsTimeoutSeconds
-                                       adsAddress:(nullable NSString *)adsAddress
-                                          adsPort:(UInt32)adsPort
-                                      adsJwtToken:(nullable NSString *)adsJwtToken
-                       adsJwtTokenLifetimeSeconds:(UInt32)adsJwtTokenLifetimeSeconds
-                                  adsSslRootCerts:(nullable NSString *)adsSslRootCerts
                                            nodeId:(nullable NSString *)nodeId
                                        nodeRegion:(nullable NSString *)nodeRegion
                                          nodeZone:(nullable NSString *)nodeZone
                                       nodeSubZone:(nullable NSString *)nodeSubZone
-                              cdsResourcesLocator:(NSString *)cdsResourcesLocator
-                                cdsTimeoutSeconds:(UInt32)cdsTimeoutSeconds
-                                        enableCds:(BOOL)enableCds;
+                                 xdsServerAddress:(nullable NSString *)xdsServerAddress
+                                    xdsServerPort:(UInt32)xdsServerPort
+                                      xdsJwtToken:(nullable NSString *)xdsJwtToken
+                       xdsJwtTokenLifetimeSeconds:(UInt32)xdsJwtTokenLifetimeSeconds
+                                  xdsSslRootCerts:(nullable NSString *)xdsSslRootCerts
+                                 rtdsResourceName:(nullable NSString *)rtdsResourceName
+                               rtdsTimeoutSeconds:(UInt32)rtdsTimeoutSeconds
+                                        enableCds:(BOOL)enableCds
+                              cdsResourcesLocator:(nullable NSString *)cdsResourcesLocator
+                                cdsTimeoutSeconds:(UInt32)cdsTimeoutSeconds;
 
 /**
  Generate a string description of the C++ Envoy bootstrap from this configuration.
diff --git a/mobile/library/objective-c/EnvoyConfiguration.mm b/mobile/library/objective-c/EnvoyConfiguration.mm
index 25b20b0734a4..b7f62bbc2c2c 100644
--- a/mobile/library/objective-c/EnvoyConfiguration.mm
+++ b/mobile/library/objective-c/EnvoyConfiguration.mm
@@ -107,22 +107,20 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain
                                        (NSDictionary> *)
                                            keyValueStores
                                        statsSinks:(NSArray *)statsSinks
-                                    rtdsLayerName:(NSString *)rtdsLayerName
+                                           nodeId:(nullable NSString *)nodeId
+                                       nodeRegion:(nullable NSString *)nodeRegion
+                                         nodeZone:(nullable NSString *)nodeZone
+                                      nodeSubZone:(nullable NSString *)nodeSubZone
+                                 xdsServerAddress:(nullable NSString *)xdsServerAddress
+                                    xdsServerPort:(UInt32)xdsServerPort
+                                      xdsJwtToken:(nullable NSString *)xdsJwtToken
+                       xdsJwtTokenLifetimeSeconds:(UInt32)xdsJwtTokenLifetimeSeconds
+                                  xdsSslRootCerts:(nullable NSString *)xdsSslRootCerts
+                                 rtdsResourceName:(nullable NSString *)rtdsResourceName
                                rtdsTimeoutSeconds:(UInt32)rtdsTimeoutSeconds
-                                       adsAddress:(NSString *)adsAddress
-                                          adsPort:(UInt32)adsPort
-                                      adsJwtToken:(NSString *)adsJwtToken
-                       adsJwtTokenLifetimeSeconds:(UInt32)adsJwtTokenLifetimeSeconds
-                                  adsSslRootCerts:(NSString *)adsSslRootCerts
-                                           nodeId:(NSString *)nodeId
-                                       nodeRegion:(NSString *)nodeRegion
-                                         nodeZone:(NSString *)nodeZone
-                                      nodeSubZone:(NSString *)nodeSubZone
-                              cdsResourcesLocator:(NSString *)cdsResourcesLocator
-                                cdsTimeoutSeconds:(UInt32)cdsTimeoutSeconds
                                         enableCds:(BOOL)enableCds
-
-{
+                              cdsResourcesLocator:(nullable NSString *)cdsResourcesLocator
+                                cdsTimeoutSeconds:(UInt32)cdsTimeoutSeconds {
   self = [super init];
   if (!self) {
     return nil;
@@ -161,17 +159,17 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain
   self.stringAccessors = stringAccessors;
   self.keyValueStores = keyValueStores;
   self.statsSinks = statsSinks;
-  self.rtdsLayerName = rtdsLayerName;
-  self.rtdsTimeoutSeconds = rtdsTimeoutSeconds;
-  self.adsAddress = adsAddress;
-  self.adsPort = adsPort;
-  self.adsJwtToken = adsJwtToken;
-  self.adsJwtTokenLifetimeSeconds = adsJwtTokenLifetimeSeconds;
-  self.adsSslRootCerts = adsSslRootCerts;
   self.nodeId = nodeId;
   self.nodeRegion = nodeRegion;
   self.nodeZone = nodeZone;
   self.nodeSubZone = nodeSubZone;
+  self.xdsServerAddress = xdsServerAddress;
+  self.xdsServerPort = xdsServerPort;
+  self.xdsJwtToken = xdsJwtToken;
+  self.xdsJwtTokenLifetimeSeconds = xdsJwtTokenLifetimeSeconds;
+  self.xdsSslRootCerts = xdsSslRootCerts;
+  self.rtdsResourceName = rtdsResourceName;
+  self.rtdsTimeoutSeconds = rtdsTimeoutSeconds;
   self.cdsResourcesLocator = cdsResourcesLocator;
   self.cdsTimeoutSeconds = cdsTimeoutSeconds;
   self.enableCds = enableCds;
@@ -251,7 +249,6 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain
   builder.addStatsFlushSeconds(self.statsFlushSeconds);
 #endif
 
-#ifdef ENVOY_GOOGLE_GRPC
   if (self.nodeRegion != nil) {
     builder.setNodeLocality([self.nodeRegion toCXXString], [self.nodeZone toCXXString],
                             [self.nodeSubZone toCXXString]);
@@ -259,16 +256,27 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain
   if (self.nodeId != nil) {
     builder.setNodeId([self.nodeId toCXXString]);
   }
-  if (self.rtdsLayerName != nil) {
-    builder.addRtdsLayer([self.rtdsLayerName toCXXString], self.rtdsTimeoutSeconds);
-  }
-  if (self.adsAddress != nil) {
-    builder.setAggregatedDiscoveryService(
-        [self.adsAddress toCXXString], self.adsPort, [self.adsJwtToken toCXXString],
-        self.adsJwtTokenLifetimeSeconds, [self.adsSslRootCerts toCXXString]);
-  }
-  if (self.enableCds) {
-    builder.addCdsLayer([self.cdsResourcesLocator toCXXString], self.cdsTimeoutSeconds);
+
+#ifdef ENVOY_GOOGLE_GRPC
+  if (self.xdsServerAddress != nil) {
+    Envoy::Platform::XdsBuilder xdsBuilder([self.xdsServerAddress toCXXString], self.xdsServerPort);
+    if (self.xdsJwtToken != nil) {
+      xdsBuilder.setJwtAuthenticationToken([self.xdsJwtToken toCXXString],
+                                           self.xdsJwtTokenLifetimeSeconds);
+    }
+    if (self.xdsSslRootCerts != nil) {
+      xdsBuilder.setSslRootCerts([self.xdsSslRootCerts toCXXString]);
+    }
+    if (self.rtdsResourceName != nil) {
+      xdsBuilder.addRuntimeDiscoveryService([self.rtdsResourceName toCXXString],
+                                            self.rtdsTimeoutSeconds);
+    }
+    if (self.enableCds) {
+      xdsBuilder.addClusterDiscoveryService(
+          self.cdsResourcesLocator != nil ? [self.cdsResourcesLocator toCXXString] : "",
+          self.cdsTimeoutSeconds);
+    }
+    builder.setXds(xdsBuilder);
   }
 #endif
 
diff --git a/mobile/library/swift/EngineBuilder.swift b/mobile/library/swift/EngineBuilder.swift
index f555caff27d6..1ebb7a473d76 100644
--- a/mobile/library/swift/EngineBuilder.swift
+++ b/mobile/library/swift/EngineBuilder.swift
@@ -6,6 +6,120 @@ import Foundation
 
 // swiftlint:disable file_length
 
+#if ENVOY_GOOGLE_GRPC
+/// Builder for generating the xDS configuration for the Envoy Mobile engine.
+/// xDS is a protocol for dynamic configuration of Envoy instances, more information can be found in
+/// https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol.
+///
+/// This class is typically used as input to the EngineBuilder's setXds() method.
+@objcMembers
+open class XdsBuilder: NSObject {
+  public static let defaultJwtTokenLifetimeInSeconds: UInt32 = 60 * 60 * 24 * 90 // 90 days
+  public static let defaultXdsTimeoutInSeconds: UInt32 = 5
+
+  let xdsServerAddress: String
+  let xdsServerPort: UInt32
+  var jwtToken: String?
+  var jwtTokenLifetimeInSeconds: UInt32 = XdsBuilder.defaultJwtTokenLifetimeInSeconds
+  var sslRootCerts: String?
+  var rtdsResourceName: String?
+  var rtdsTimeoutInSeconds: UInt32 = 0
+  var enableCds: Bool = false
+  var cdsResourcesLocator: String?
+  var cdsTimeoutInSeconds: UInt32 = 0
+
+  /// Initialize a new builder for xDS configuration.
+  ///
+  /// - parameter xdsServerAddress: The host name or IP address of the xDS management server.
+  /// - parameter xdsServerPort:    The port on which the server listens for client connections.
+  public init(xdsServerAddress: String, xdsServerPort: UInt32) {
+    self.xdsServerAddress = xdsServerAddress
+    self.xdsServerPort = xdsServerPort
+  }
+
+  /// Sets JWT as the authentication method to the xDS management server, using the given token.
+  ///
+  /// - parameter token:                  The JWT token used to authenticate the client to the xDS
+  ///                                     management server.
+  /// - parameter tokenLifetimeInSeconds:  the lifetime of the JWT token, in seconds. If
+  ///                                     none (or 0) is specified, then
+  ///                                     defaultJwtTokenLifetimeSeconds is used.
+  ///
+  /// - returns: This builder.
+  @discardableResult
+  public func setJwtAuthenticationToken(
+    token: String,
+    tokenLifetimeInSeconds: UInt32 = XdsBuilder.defaultJwtTokenLifetimeInSeconds) -> Self {
+    self.jwtToken = token
+    self.jwtTokenLifetimeInSeconds = (tokenLifetimeInSeconds > 0) ?
+        tokenLifetimeInSeconds : XdsBuilder.defaultJwtTokenLifetimeInSeconds
+    return self
+  }
+
+  /// Sets the PEM-encoded server root certificates used to negotiate the TLS handshake for the gRPC
+  /// connection. If no root certs are specified, the operating system defaults are used.
+  ///
+  /// - parameter rootCerts: The PEM-encoded server root certificates.
+  ///
+  /// - returns: This builder.
+  @discardableResult
+  public func setSslRootCerts(rootCerts: String) -> Self {
+    self.sslRootCerts = rootCerts
+    return self
+  }
+
+  /// Adds Runtime Discovery Service (RTDS) to the Runtime layers of the Bootstrap configuration,
+  /// to retrieve dynamic runtime configuration via the xDS management server.
+  ///
+  /// - parameter resourceName:     The runtime config resource to subscribe to.
+  /// - parameter timeoutInSeconds:  specifies the `initial_fetch_timeout` field on the
+  ///                               api.v3.core.ConfigSource. Unlike the ConfigSource default of
+  ///                               15s, we set a default fetch timeout value of 5s, to prevent
+  ///                               mobile app initialization from stalling. The default parameter
+  ///                               value may change through the course of experimentation and no
+  ///                               assumptions should be made of its exact value.
+  ///
+  /// - returns: This builder.
+  @discardableResult
+  public func addRuntimeDiscoveryService(
+    resourceName: String,
+    timeoutInSeconds: UInt32 = XdsBuilder.defaultXdsTimeoutInSeconds) -> Self {
+    self.rtdsResourceName = resourceName
+    self.rtdsTimeoutInSeconds = timeoutOrXdsDefault(timeoutInSeconds)
+    return self
+  }
+
+  /// Adds the Cluster Discovery Service (CDS) configuration for retrieving dynamic cluster
+  /// resources via the xDS management server.
+  ///
+  /// - parameter cdsResourcesLocator:  the xdstp:// URI for subscribing to the cluster
+  ///                                  resources. If not using xdstp, then `cds_resources_locator`
+  ///                                  should be set to the empty string.
+  /// - parameter timeoutInSeconds:     specifies the `initial_fetch_timeout` field on the
+  ///                                  api.v3.core.ConfigSource. Unlike the ConfigSource default of
+  ///                                  15s, we set a default fetch timeout value of 5s, to prevent
+  ///                                  mobile app initialization from stalling. The default
+  ///                                  parameter value may change through the course of
+  ///                                  experimentation and no assumptions should be made of its
+  ///                                  exact value.
+  ///
+  /// - returns: This builder.
+  @discardableResult
+  public func addClusterDiscoveryService(
+    cdsResourcesLocator: String? = nil,
+    timeoutInSeconds: UInt32 = XdsBuilder.defaultXdsTimeoutInSeconds) -> Self {
+    self.enableCds = true
+    self.cdsResourcesLocator = cdsResourcesLocator
+    self.cdsTimeoutInSeconds = timeoutOrXdsDefault(timeoutInSeconds)
+    return self
+  }
+
+  private func timeoutOrXdsDefault(_ timeout: UInt32) -> UInt32 {
+    return timeout > 0 ? timeout : XdsBuilder.defaultXdsTimeoutInSeconds
+  }
+}
+#endif
+
 /// Builder used for creating and running a new Engine instance.
 @objcMembers
 open class EngineBuilder: NSObject {
@@ -59,20 +173,13 @@ open class EngineBuilder: NSObject {
   private var keyValueStores: [String: EnvoyKeyValueStore] = [:]
   private var runtimeGuards: [String: Bool] = [:]
   private var statsSinks: [String] = []
-  private var rtdsLayerName: String?
-  private var rtdsTimeoutSeconds: UInt32 = 0
-  private var adsAddress: String?
-  private var adsPort: UInt32 = 0
-  private var adsJwtToken: String?
-  private var adsJwtTokenLifetimeSeconds: UInt32 = 0
-  private var adsSslRootCerts: String?
   private var nodeID: String?
   private var nodeRegion: String?
   private var nodeZone: String?
   private var nodeSubZone: String?
-  private var cdsResourcesLocator: String = ""
-  private var cdsTimeoutSeconds: UInt32 = 0
-  private var enableCds: Bool = false
+#if ENVOY_GOOGLE_GRPC
+  private var xdsBuilder: XdsBuilder?
+#endif
   private var enableSwiftBootstrap = false
 
   // MARK: - Public
@@ -530,8 +637,6 @@ open class EngineBuilder: NSObject {
     return self
   }
 
-#if ENVOY_GOOGLE_GRPC
-
   /// Sets the node.id field in the Bootstrap configuration.
   ///
   /// - parameter nodeID: The node ID.
@@ -562,59 +667,16 @@ open class EngineBuilder: NSObject {
     return self
   }
 
-  /// Adds an aggregated discovery service layer to the configuration.
-  ///
-  /// - parameter address:                 The network address of the server.
-  /// - parameter port:                    The port of the server.
-  /// - parameter jwtToken:                The JWT token.
-  /// - parameter jwtTokenLifetimeSeconds: The JWT token lifetime in seconds. If zero, a
-  ///                                      default value is set in engine_builder.h.
-  /// - parameter sslRootCerts:            The SSL root certificates.
-  ///
-  /// - returns: This builder.
-  @discardableResult
-  public func setAggregatedDiscoveryService(
-    address: String,
-    port: UInt32,
-    jwtToken: String = "",
-    jwtTokenLifetimeSeconds: UInt32 = 0,
-    sslRootCerts: String = ""
-  ) -> Self {
-    self.adsAddress = address
-    self.adsPort = port
-    self.adsJwtToken = jwtToken
-    self.adsJwtTokenLifetimeSeconds = jwtTokenLifetimeSeconds
-    self.adsSslRootCerts = sslRootCerts
-    return self
-  }
-
-  /// Adds an RTDS layer to the configuration.
-  ///
-  /// - parameter layerName:      The layer name.
-  /// - parameter timeoutSeconds: The timeout in seconds. If zero, a default value is set in
-  ///                             engine_builder.h.
-  ///
-  /// - returns: This builder.
-  @discardableResult
-  public func addRTDSLayer(name layerName: String, timeoutSeconds: UInt32 = 0) -> Self {
-    self.rtdsLayerName = layerName
-    self.rtdsTimeoutSeconds = timeoutSeconds
-    return self
-  }
-
-  /// Adds a CDS layer to the configuration.
+#if ENVOY_GOOGLE_GRPC
+  /// Sets the xDS configuration for the Envoy Mobile engine.
   ///
-  /// - parameter resourcesLocator: The xdstp resource URI for fetching clusters.
-  ///                               If empty, xdstp is not used and a wildcard is inferred.
-  /// - parameter timeoutSeconds:   The timeout in seconds. If zero, a default value is set in
-  ///                               engine_builder.h.
+  /// - parameter xdsBuilder: The XdsBuilder instance which specifies the xDS config options.
+  ///                         The EngineBuilder takes ownership over the xds_builder.
   ///
   /// - returns: This builder.
   @discardableResult
-  public func addCDSLayer(resourcesLocator: String = "", timeoutSeconds: UInt32 = 0) -> Self {
-    self.cdsResourcesLocator = resourcesLocator
-    self.cdsTimeoutSeconds = timeoutSeconds
-    self.enableCds = true
+  public func setXds(_ xdsBuilder: XdsBuilder) -> Self {
+    self.xdsBuilder = xdsBuilder
     return self
   }
 #endif
@@ -676,7 +738,31 @@ open class EngineBuilder: NSObject {
   }
 
   func makeConfig() -> EnvoyConfiguration {
-    EnvoyConfiguration(
+    var xdsServerAddress: String?
+    var xdsServerPort: UInt32 = 0
+    var xdsJwtToken: String?
+    var xdsJwtTokenLifetimeSeconds: UInt32 = 0
+    var xdsSslRootCerts: String?
+    var rtdsResourceName: String?
+    var rtdsTimeoutSeconds: UInt32 = 0
+    var enableCds: Bool = false
+    var cdsResourcesLocator: String?
+    var cdsTimeoutSeconds: UInt32 = 0
+
+#if ENVOY_GOOGLE_GRPC
+    xdsServerAddress = self.xdsBuilder?.xdsServerAddress
+    xdsServerPort = self.xdsBuilder?.xdsServerPort ?? 0
+    xdsJwtToken = self.xdsBuilder?.jwtToken
+    xdsJwtTokenLifetimeSeconds = self.xdsBuilder?.jwtTokenLifetimeInSeconds ?? 0
+    xdsSslRootCerts = self.xdsBuilder?.sslRootCerts
+    rtdsResourceName = self.xdsBuilder?.rtdsResourceName
+    rtdsTimeoutSeconds = self.xdsBuilder?.rtdsTimeoutInSeconds ?? 0
+    enableCds = self.xdsBuilder?.enableCds ?? false
+    cdsResourcesLocator = self.xdsBuilder?.cdsResourcesLocator
+    cdsTimeoutSeconds = self.xdsBuilder?.cdsTimeoutInSeconds ?? 0
+#endif
+
+    return EnvoyConfiguration(
       grpcStatsDomain: self.grpcStatsDomain,
       connectTimeoutSeconds: self.connectTimeoutSeconds,
       dnsRefreshSeconds: self.dnsRefreshSeconds,
@@ -710,20 +796,20 @@ open class EngineBuilder: NSObject {
       stringAccessors: self.stringAccessors,
       keyValueStores: self.keyValueStores,
       statsSinks: self.statsSinks,
-      rtdsLayerName: self.rtdsLayerName,
-      rtdsTimeoutSeconds: self.rtdsTimeoutSeconds,
-      adsAddress: self.adsAddress,
-      adsPort: self.adsPort,
-      adsJwtToken: self.adsJwtToken,
-      adsJwtTokenLifetimeSeconds: self.adsJwtTokenLifetimeSeconds,
-      adsSslRootCerts: self.adsSslRootCerts,
       nodeId: self.nodeID,
       nodeRegion: self.nodeRegion,
       nodeZone: self.nodeZone,
       nodeSubZone: self.nodeSubZone,
-      cdsResourcesLocator: self.cdsResourcesLocator,
-      cdsTimeoutSeconds: self.cdsTimeoutSeconds,
-      enableCds: self.enableCds
+      xdsServerAddress: xdsServerAddress,
+      xdsServerPort: xdsServerPort,
+      xdsJwtToken: xdsJwtToken,
+      xdsJwtTokenLifetimeSeconds: xdsJwtTokenLifetimeSeconds,
+      xdsSslRootCerts: xdsSslRootCerts,
+      rtdsResourceName: rtdsResourceName,
+      rtdsTimeoutSeconds: rtdsTimeoutSeconds,
+      enableCds: enableCds,
+      cdsResourcesLocator: cdsResourcesLocator,
+      cdsTimeoutSeconds: cdsTimeoutSeconds
     )
   }
 
@@ -794,7 +880,6 @@ private extension EngineBuilder {
 
     cxxBuilder.addStatsSinks(self.statsSinks.toCXX())
 
-#if ENVOY_GOOGLE_GRPC
     if
       let nodeRegion = self.nodeRegion,
       let nodeZone = self.nodeZone,
@@ -807,29 +892,35 @@ private extension EngineBuilder {
       cxxBuilder.setNodeId(nodeID.toCXX())
     }
 
-    if let rtdsLayerName = self.rtdsLayerName {
-      cxxBuilder.addRtdsLayer(rtdsLayerName.toCXX(), Int32(self.rtdsTimeoutSeconds))
-    }
+    generateXds(&cxxBuilder)
 
-    if
-      let adsAddress = self.adsAddress,
-      let adsJwtToken = self.adsJwtToken,
-      let adsSslRootCerts = self.adsSslRootCerts
-    {
-      cxxBuilder.setAggregatedDiscoveryService(
-        adsAddress.toCXX(),
-        Int32(self.adsPort),
-        adsJwtToken.toCXX(),
-        Int32(self.adsJwtTokenLifetimeSeconds),
-        adsSslRootCerts.toCXX()
-      )
-    }
-    if self.enableCds {
-      cxxBuilder.addCdsLayer(self.cdsResourcesLocator.toCXX(), Int32(self.cdsTimeoutSeconds))
+    return cxxBuilder.generateBootstrap()
+  }
+
+  private func generateXds(_ cxxBuilder: inout Envoy.Platform.EngineBuilder) {
+#if ENVOY_GOOGLE_GRPC
+    if let xdsBuilder = self.xdsBuilder {
+      var cxxXdsBuilder = Envoy.Platform.XdsBuilder(xdsBuilder.xdsServerAddress.toCXX(),
+                                                    Int32(xdsBuilder.xdsServerPort))
+      if let xdsJwtToken = xdsBuilder.jwtToken {
+        cxxXdsBuilder.setJwtAuthenticationToken(xdsJwtToken.toCXX(),
+                                                Int32(xdsBuilder.jwtTokenLifetimeInSeconds))
+      }
+      if let xdsSslRootCerts = xdsBuilder.sslRootCerts {
+        cxxXdsBuilder.setSslRootCerts(xdsSslRootCerts.toCXX())
+      }
+      if let rtdsResourceName = xdsBuilder.rtdsResourceName {
+        cxxXdsBuilder.addRuntimeDiscoveryService(rtdsResourceName.toCXX(),
+                                                 Int32(xdsBuilder.rtdsTimeoutInSeconds))
+      }
+      if xdsBuilder.enableCds {
+        cxxXdsBuilder.addClusterDiscoveryService(
+          xdsBuilder.cdsResourcesLocator?.toCXX() ?? "".toCXX(),
+          Int32(xdsBuilder.cdsTimeoutInSeconds))
+      }
+      cxxBuilder.setXds(cxxXdsBuilder)
     }
 #endif
-    return cxxBuilder.generateBootstrap()
   }
-  // swiftlint:enable cyclomatic_complexity
 }
 #endif
diff --git a/mobile/test/cc/unit/envoy_config_test.cc b/mobile/test/cc/unit/envoy_config_test.cc
index bb42d1d69c53..6e215c7e0de5 100644
--- a/mobile/test/cc/unit/envoy_config_test.cc
+++ b/mobile/test/cc/unit/envoy_config_test.cc
@@ -46,9 +46,6 @@ TEST(TestConfig, ConfigIsApplied) {
       .enableDnsCache(true, /* save_interval_seconds */ 101)
       .addDnsPreresolveHostnames({"lyft.com", "google.com"})
       .setForceAlwaysUsev6(true)
-#ifdef ENVOY_GOOGLE_GRPC
-      .setNodeId("my_test_node")
-#endif
       .setDeviceOs("probably-ubuntu-on-CI");
 
   std::unique_ptr bootstrap = engine_builder.generateBootstrap();
@@ -91,6 +88,32 @@ TEST(TestConfig, MultiFlag) {
   EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_true\" value { bool_value: false }"));
 }
 
+TEST(TestConfig, CopyConstructor) {
+  EngineBuilder engine_builder;
+  engine_builder.setRuntimeGuard("test_feature_false", true).enableGzipDecompression(false);
+
+  std::unique_ptr bootstrap = engine_builder.generateBootstrap();
+  std::string bootstrap_str = bootstrap->ShortDebugString();
+  EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_false\" value { bool_value: true }"));
+  EXPECT_THAT(bootstrap_str, Not(HasSubstr("envoy.filters.http.decompressor")));
+
+  EngineBuilder engine_builder_copy(engine_builder);
+  engine_builder_copy.enableGzipDecompression(true);
+  XdsBuilder xdsBuilder("FAKE_XDS_SERVER", 0);
+  xdsBuilder.addClusterDiscoveryService();
+  engine_builder_copy.setXds(xdsBuilder);
+  bootstrap_str = engine_builder_copy.generateBootstrap()->ShortDebugString();
+  EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_false\" value { bool_value: true }"));
+  EXPECT_THAT(bootstrap_str, HasSubstr("envoy.filters.http.decompressor"));
+  EXPECT_THAT(bootstrap_str, HasSubstr("FAKE_XDS_SERVER"));
+
+  EngineBuilder engine_builder_copy2(engine_builder_copy);
+  bootstrap_str = engine_builder_copy2.generateBootstrap()->ShortDebugString();
+  EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_false\" value { bool_value: true }"));
+  EXPECT_THAT(bootstrap_str, HasSubstr("envoy.filters.http.decompressor"));
+  EXPECT_THAT(bootstrap_str, HasSubstr("FAKE_XDS_SERVER"));
+}
+
 TEST(TestConfig, ConfigIsValid) {
   EngineBuilder engine_builder;
   std::unique_ptr bootstrap = engine_builder.generateBootstrap();
@@ -263,22 +286,13 @@ TEST(TestConfig, DisableHttp3) {
       Not(HasSubstr("envoy.extensions.filters.http.alternate_protocols_cache.v3.FilterConfig")));
 #endif
 }
-#ifdef ENVOY_GOOGLE_GRPC
-TEST(TestConfig, RtdsWithoutAds) {
-  EngineBuilder engine_builder;
-  engine_builder.addRtdsLayer("some rtds layer");
-  try {
-    engine_builder.generateBootstrap();
-    FAIL() << "Expected std::runtime_error";
-  } catch (std::runtime_error& err) {
-    EXPECT_EQ(err.what(), std::string("ADS must be configured when using xDS"));
-  }
-}
 
-TEST(TestConfig, AdsConfig) {
+#ifdef ENVOY_GOOGLE_GRPC
+TEST(TestConfig, XdsConfig) {
   EngineBuilder engine_builder;
-  engine_builder.setAggregatedDiscoveryService(/*target_uri=*/"fake-td.googleapis.com",
-                                               /*port=*/12345);
+  XdsBuilder xds_builder(/*xds_server_address=*/"fake-td.googleapis.com",
+                         /*xds_server_port=*/12345);
+  engine_builder.setXds(std::move(xds_builder));
   std::unique_ptr bootstrap = engine_builder.generateBootstrap();
   auto& ads_config = bootstrap->dynamic_resources().ads_config();
   EXPECT_EQ(ads_config.api_type(), envoy::config::core::v3::ApiConfigSource::GRPC);
@@ -294,10 +308,12 @@ TEST(TestConfig, AdsConfig) {
   EXPECT_THAT(ads_config.grpc_services(0).google_grpc().call_credentials(), SizeIs(0));
 
   // With security credentials.
-  engine_builder.setAggregatedDiscoveryService(/*target_uri=*/"fake-td.googleapis.com",
-                                               /*port=*/12345, /*jwt_token=*/"my_jwt_token",
-                                               /*jwt_token_lifetime_seconds=*/500,
-                                               /*ssl_root_certs=*/"my_root_cert");
+  xds_builder =
+      XdsBuilder(/*xds_server_address=*/"fake-td.googleapis.com", /*xds_server_port=*/12345);
+  xds_builder.setJwtAuthenticationToken(/*token=*/"my_jwt_token",
+                                        /*token_lifetime_in_seconds=*/500);
+  xds_builder.setSslRootCerts(/*root_certs=*/"my_root_cert");
+  engine_builder.setXds(std::move(xds_builder));
   bootstrap = engine_builder.generateBootstrap();
   auto& ads_config_with_tokens = bootstrap->dynamic_resources().ads_config();
   EXPECT_EQ(ads_config_with_tokens.api_type(), envoy::config::core::v3::ApiConfigSource::GRPC);
@@ -415,7 +431,6 @@ TEST(TestConfig, DISABLED_StringAccessors) {
   release_envoy_data(data);
 }
 
-#ifdef ENVOY_GOOGLE_GRPC
 TEST(TestConfig, SetNodeId) {
   EngineBuilder engine_builder;
   const std::string default_node_id = "envoy-mobile";
@@ -438,20 +453,24 @@ TEST(TestConfig, SetNodeLocality) {
   EXPECT_EQ(bootstrap->node().locality().sub_zone(), sub_zone);
 }
 
+#ifdef ENVOY_GOOGLE_GRPC
 TEST(TestConfig, AddCdsLayer) {
+  XdsBuilder xds_builder(/*xds_server_address=*/"fake-xds-server", /*xds_server_port=*/12345);
+  xds_builder.addClusterDiscoveryService();
   EngineBuilder engine_builder;
-  engine_builder.setAggregatedDiscoveryService(/*address=*/"fake-xds-server", /*port=*/12345);
+  engine_builder.setXds(std::move(xds_builder));
 
-  engine_builder.addCdsLayer();
   std::unique_ptr bootstrap = engine_builder.generateBootstrap();
   EXPECT_EQ(bootstrap->dynamic_resources().cds_resources_locator(), "");
   EXPECT_EQ(bootstrap->dynamic_resources().cds_config().initial_fetch_timeout().seconds(),
             /*default_timeout=*/5);
 
+  xds_builder = XdsBuilder(/*xds_server_address=*/"fake-xds-server", /*xds_server_port=*/12345);
   const std::string cds_resources_locator =
       "xdstp://traffic-director-global.xds.googleapis.com/envoy.config.cluster.v3.Cluster";
   const int timeout_seconds = 300;
-  engine_builder.addCdsLayer(cds_resources_locator, timeout_seconds);
+  xds_builder.addClusterDiscoveryService(cds_resources_locator, timeout_seconds);
+  engine_builder.setXds(std::move(xds_builder));
   bootstrap = engine_builder.generateBootstrap();
   EXPECT_EQ(bootstrap->dynamic_resources().cds_resources_locator(), cds_resources_locator);
   EXPECT_EQ(bootstrap->dynamic_resources().cds_config().initial_fetch_timeout().seconds(),
diff --git a/mobile/test/common/integration/cds_integration_test.cc b/mobile/test/common/integration/cds_integration_test.cc
index 0720e4a9ce9c..639c108f6283 100644
--- a/mobile/test/common/integration/cds_integration_test.cc
+++ b/mobile/test/common/integration/cds_integration_test.cc
@@ -19,15 +19,14 @@ class CdsIntegrationTest : public XdsIntegrationTest {
   void createEnvoy() override {
     sotw_or_delta_ = sotwOrDelta();
     const std::string target_uri = Network::Test::getLoopbackAddressUrlString(ipVersion());
-    builder_.setAggregatedDiscoveryService(target_uri,
-                                           fake_upstreams_[1]->localAddress()->ip()->port());
-
+    Platform::XdsBuilder xds_builder(target_uri, fake_upstreams_[1]->localAddress()->ip()->port());
     std::string cds_resources_locator;
     if (use_xdstp_) {
       cds_namespace_ = "xdstp://" + target_uri + "/envoy.config.cluster.v3.Cluster";
       cds_resources_locator = cds_namespace_ + "/*";
     }
-    builder_.addCdsLayer(cds_resources_locator, /*timeout_seconds=*/1);
+    xds_builder.addClusterDiscoveryService(cds_resources_locator, /*timeout_in_seconds=*/1);
+    builder_.setXds(std::move(xds_builder));
 
     XdsIntegrationTest::createEnvoy();
   }
diff --git a/mobile/test/common/integration/rtds_integration_test.cc b/mobile/test/common/integration/rtds_integration_test.cc
index 658c2c07dedd..19b356cba7a6 100644
--- a/mobile/test/common/integration/rtds_integration_test.cc
+++ b/mobile/test/common/integration/rtds_integration_test.cc
@@ -18,10 +18,12 @@ class RtdsIntegrationTest : public XdsIntegrationTest {
     initializeXdsStream();
   }
   void createEnvoy() override {
-    builder_.setAggregatedDiscoveryService(Network::Test::getLoopbackAddressUrlString(ipVersion()),
-                                           fake_upstreams_[1]->localAddress()->ip()->port());
+    Platform::XdsBuilder xds_builder(
+        /*xds_server_address=*/Network::Test::getLoopbackAddressUrlString(ipVersion()),
+        /*xds_server_port=*/fake_upstreams_[1]->localAddress()->ip()->port());
     // Add the layered runtime config, which includes the RTDS layer.
-    builder_.addRtdsLayer("some_rtds_layer", 1);
+    xds_builder.addRuntimeDiscoveryService("some_rtds_resource", /*timeout_in_seconds=*/1);
+    builder_.setXds(std::move(xds_builder));
     XdsIntegrationTest::createEnvoy();
   }
 
@@ -57,15 +59,15 @@ TEST_P(RtdsIntegrationTest, RtdsReload) {
   const std::string load_success_counter = "runtime.load_success";
   uint64_t load_success_value = getCounterValue(load_success_counter);
   // Send a RTDS request and get back the RTDS response.
-  EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Runtime, "", {"some_rtds_layer"},
-                                      {"some_rtds_layer"}, {}, true));
-  auto some_rtds_layer = TestUtility::parseYaml(R"EOF(
-    name: some_rtds_layer
+  EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Runtime, "", {"some_rtds_resource"},
+                                      {"some_rtds_resource"}, {}, true));
+  auto some_rtds_resource = TestUtility::parseYaml(R"EOF(
+    name: some_rtds_resource
     layer:
       envoy.reloadable_features.test_feature_false: True
   )EOF");
   sendDiscoveryResponse(
-      Config::TypeUrl::get().Runtime, {some_rtds_layer}, {some_rtds_layer}, {}, "1");
+      Config::TypeUrl::get().Runtime, {some_rtds_resource}, {some_rtds_resource}, {}, "1");
   // Wait until the RTDS updates from the DiscoveryResponse have been applied.
   ASSERT_TRUE(waitForCounterGe(load_success_counter, load_success_value + 1));
 
diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt
index 93d71c2006e3..25d8dfd670c4 100644
--- a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt
+++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt
@@ -98,13 +98,13 @@ class EnvoyConfigurationTest {
     runtimeGuards: Map = emptyMap(),
     statSinks: MutableList = mutableListOf(),
     enablePlatformCertificatesValidation: Boolean = false,
-    rtdsLayerName: String = "",
+    rtdsResourceName: String = "",
     rtdsTimeoutSeconds: Int = 0,
-    adsAddress: String = "",
-    adsPort: Int = 0,
-    adsJwtToken: String = "",
-    adsJwtTokenLifetimeSeconds: Int = 0,
-    adsSslRootCerts: String = "",
+    xdsAddress: String = "",
+    xdsPort: Int = 0,
+    xdsJwtToken: String = "",
+    xdsJwtTokenLifetimeSeconds: Int = 0,
+    xdsSslRootCerts: String = "",
     nodeId: String = "",
     nodeRegion: String = "",
     nodeZone: String = "",
@@ -147,13 +147,13 @@ class EnvoyConfigurationTest {
       statSinks,
       runtimeGuards,
       enablePlatformCertificatesValidation,
-      rtdsLayerName,
+      rtdsResourceName,
       rtdsTimeoutSeconds,
-      adsAddress,
-      adsPort,
-      adsJwtToken,
-      adsJwtTokenLifetimeSeconds,
-      adsSslRootCerts,
+      xdsAddress,
+      xdsPort,
+      xdsJwtToken,
+      xdsJwtTokenLifetimeSeconds,
+      xdsSslRootCerts,
       nodeId,
       nodeRegion,
       nodeZone,
@@ -311,10 +311,10 @@ class EnvoyConfigurationTest {
   }
 
   @Test
-  fun `test adding RTDS and ADS`() {
+  fun `test adding RTDS`() {
     JniLibrary.loadTestLibrary()
     val envoyConfiguration = buildTestEnvoyConfiguration(
-      rtdsLayerName = "fake_rtds_layer", rtdsTimeoutSeconds = 5432, adsAddress = "FAKE_ADDRESS", adsPort = 0
+      rtdsResourceName = "fake_rtds_layer", rtdsTimeoutSeconds = 5432, xdsAddress = "FAKE_ADDRESS", xdsPort = 0
     )
 
     val resolvedTemplate = TestJni.createYaml(envoyConfiguration)
@@ -328,7 +328,7 @@ class EnvoyConfigurationTest {
   fun `test adding RTDS and CDS`() {
     JniLibrary.loadTestLibrary()
     val envoyConfiguration = buildTestEnvoyConfiguration(
-      cdsResourcesLocator = "FAKE_CDS_LOCATOR", cdsTimeoutSeconds = 356, adsAddress = "FAKE_ADDRESS", adsPort = 0, enableCds = true
+      cdsResourcesLocator = "FAKE_CDS_LOCATOR", cdsTimeoutSeconds = 356, xdsAddress = "FAKE_ADDRESS", xdsPort = 0, enableCds = true
     )
 
     val resolvedTemplate = TestJni.createYaml(envoyConfiguration)
@@ -342,7 +342,7 @@ class EnvoyConfigurationTest {
   fun `test not using enableCds`() {
     JniLibrary.loadTestLibrary()
     val envoyConfiguration = buildTestEnvoyConfiguration(
-      cdsResourcesLocator = "FAKE_CDS_LOCATOR", cdsTimeoutSeconds = 356, adsAddress = "FAKE_ADDRESS", adsPort = 0
+      cdsResourcesLocator = "FAKE_CDS_LOCATOR", cdsTimeoutSeconds = 356, xdsAddress = "FAKE_ADDRESS", xdsPort = 0
     )
 
     val resolvedTemplate = TestJni.createYaml(envoyConfiguration)
@@ -355,7 +355,7 @@ class EnvoyConfigurationTest {
   fun `test enableCds with default string`() {
     JniLibrary.loadTestLibrary()
     val envoyConfiguration = buildTestEnvoyConfiguration(
-      enableCds = true, adsAddress = "FAKE_ADDRESS", adsPort = 0
+      enableCds = true, xdsAddress = "FAKE_ADDRESS", xdsPort = 0
     )
 
     val resolvedTemplate = TestJni.createYaml(envoyConfiguration)
@@ -368,7 +368,7 @@ class EnvoyConfigurationTest {
   fun `test RTDS default timeout`() {
     JniLibrary.loadTestLibrary()
     val envoyConfiguration = buildTestEnvoyConfiguration(
-      rtdsLayerName = "fake_rtds_layer", adsAddress = "FAKE_ADDRESS", adsPort = 0
+      rtdsResourceName = "fake_rtds_layer", xdsAddress = "FAKE_ADDRESS", xdsPort = 0
     )
 
     val resolvedTemplate = TestJni.createYaml(envoyConfiguration)
diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt
index 8d3d1af832b6..f727c23dc2ba 100644
--- a/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt
+++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt
@@ -205,14 +205,22 @@ class EngineBuilderTest {
   }
 
   @Test
-  fun `specifying RTDS and ADS works`() {
+  fun `specifying xDS works`() {
+    var xdsBuilder = XdsBuilder("fake_test_address", 0)
+    xdsBuilder.setJwtAuthenticationToken("my_jwt_token")
+    xdsBuilder.setSslRootCerts("my_root_certs")
+    xdsBuilder.addRuntimeDiscoveryService("some_rtds_resource")
+    xdsBuilder.addClusterDiscoveryService("xdstp://fake_test_address/envoy.config.cluster.v3.Cluster/xyz")
     engineBuilder = EngineBuilder(Standard())
     engineBuilder.addEngineType { envoyEngine }
-    engineBuilder.addRtdsLayer("rtds_layer_name")
-    engineBuilder.setAggregatedDiscoveryService("fake_test_address", 0)
+    engineBuilder.setXds(xdsBuilder)
+
     val engine = engineBuilder.build() as EngineImpl
-    assertThat(engine.envoyConfiguration.rtdsLayerName).isEqualTo("rtds_layer_name")
-    assertThat(engine.envoyConfiguration.adsAddress).isEqualTo("fake_test_address")
+    assertThat(engine.envoyConfiguration.xdsAddress).isEqualTo("fake_test_address")
+    assertThat(engine.envoyConfiguration.xdsJwtToken).isEqualTo("my_jwt_token")
+    assertThat(engine.envoyConfiguration.xdsRootCerts).isEqualTo("my_root_certs")
+    assertThat(engine.envoyConfiguration.rtdsResourceName).isEqualTo("some_rtds_resource")
+    assertThat(engine.envoyConfiguration.cdsResourcesLocator).isEqualTo("xdstp://fake_test_address/envoy.config.cluster.v3.Cluster/xyz")
   }
 
   @Test
diff --git a/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc b/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc
index 3afeb9fb8f3b..182320d68a71 100644
--- a/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc
+++ b/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc
@@ -98,12 +98,14 @@ class GcpTrafficDirectorIntegrationTest
         TestEnvironment::runfilesPath("test/config/integration/certs/google_root_certs.pem")));
 
     // TODO(abeyad): switch to using API key authentication instead of a JWT token.
+    Platform::XdsBuilder xds_builder(/*xds_server_address=*/std::string(TD_API_ENDPOINT),
+                                     /*xds_server_port=*/443);
+    xds_builder.setJwtAuthenticationToken(jwtToken(), Platform::DefaultJwtTokenLifetimeSeconds);
+    xds_builder.setSslRootCerts(std::move(root_certs));
+    xds_builder.addClusterDiscoveryService();
     builder_.addLogLevel(Platform::LogLevel::trace)
         .setNodeId(absl::Substitute("projects/$0/networks/default/nodes/111222333444", PROJECT_ID))
-        .addCdsLayer()
-        .setAggregatedDiscoveryService(std::string(TD_API_ENDPOINT), /*port=*/443, jwtToken(),
-                                       Platform::DefaultJwtTokenLifetimeSeconds,
-                                       std::move(root_certs));
+        .setXds(std::move(xds_builder));
 
     // Other test knobs.
     skip_tag_extraction_rule_check_ = true;
diff --git a/mobile/test/swift/EngineBuilderTests.swift b/mobile/test/swift/EngineBuilderTests.swift
index 98795b7b3f05..5e0ff03e0d69 100644
--- a/mobile/test/swift/EngineBuilderTests.swift
+++ b/mobile/test/swift/EngineBuilderTests.swift
@@ -357,22 +357,24 @@ final class EngineBuilderTests: XCTestCase {
   }
 
 #if ENVOY_GOOGLE_GRPC
-  func testAddingRtdsAndAdsConfigurationWhenRunningEnvoy() {
+  func testAddingRtdsConfigurationWhenRunningEnvoy() {
+    let xdsBuilder = XdsBuilder(xdsServerAddress: "FAKE_SWIFT_ADDRESS", xdsServerPort: 0)
+      .addRuntimeDiscoveryService(resourceName: "some_rtds_resource", timeoutInSeconds: 14325)
     let bootstrapDebugDescription = EngineBuilder()
       .addEngineType(MockEnvoyEngine.self)
-      .addRTDSLayer(name: "rtds_layer_name", timeoutSeconds: 14325)
-      .setAggregatedDiscoveryService(address: "FAKE_SWIFT_ADDRESS", port: 0)
+      .setXds(xdsBuilder)
       .bootstrapDebugDescription()
-    XCTAssertTrue(bootstrapDebugDescription.contains("rtds_layer_name"))
+    XCTAssertTrue(bootstrapDebugDescription.contains("some_rtds_resource"))
     XCTAssertTrue(bootstrapDebugDescription.contains("initial_fetch_timeout { seconds: 14325 }"))
     XCTAssertTrue(bootstrapDebugDescription.contains("FAKE_SWIFT_ADDRESS"))
   }
 
-  func testAddingCdsAndAdsConfigurationWhenRunningEnvoy() {
+  func testAddingCdsConfigurationWhenRunningEnvoy() {
+    let xdsBuilder = XdsBuilder(xdsServerAddress: "FAKE_SWIFT_ADDRESS", xdsServerPort: 0)
+      .addClusterDiscoveryService(cdsResourcesLocator: "FAKE_CDS_LOCATOR", timeoutInSeconds: 2543)
     let bootstrapDebugDescription = EngineBuilder()
       .addEngineType(MockEnvoyEngine.self)
-      .addCDSLayer(resourcesLocator: "FAKE_CDS_LOCATOR", timeoutSeconds: 2543)
-      .setAggregatedDiscoveryService(address: "FAKE_SWIFT_ADDRESS", port: 0)
+      .setXds(xdsBuilder)
       .bootstrapDebugDescription()
     XCTAssertTrue(bootstrapDebugDescription.contains("FAKE_CDS_LOCATOR"))
     XCTAssertTrue(bootstrapDebugDescription.contains("initial_fetch_timeout { seconds: 2543 }"))
@@ -380,14 +382,16 @@ final class EngineBuilderTests: XCTestCase {
   }
 
   func testAddingDefaultCdsConfigurationWhenRunningEnvoy() {
+    let xdsBuilder = XdsBuilder(xdsServerAddress: "FAKE_SWIFT_ADDRESS", xdsServerPort: 0)
+      .addClusterDiscoveryService()
     let bootstrapDebugDescription = EngineBuilder()
       .addEngineType(MockEnvoyEngine.self)
-      .addCDSLayer()
-      .setAggregatedDiscoveryService(address: "FAKE_SWIFT_ADDRESS", port: 0)
+      .setXds(xdsBuilder)
       .bootstrapDebugDescription()
     XCTAssertTrue(bootstrapDebugDescription.contains("cds_config {"))
     XCTAssertTrue(bootstrapDebugDescription.contains("initial_fetch_timeout { seconds: 5 }"))
   }
+#endif
 
   func testXDSDefaultValues() {
     // rtds, ads, node_id, node_locality
@@ -417,7 +421,6 @@ final class EngineBuilderTests: XCTestCase {
     XCTAssertTrue(bootstrapDebugDescription.contains(#"zone: "SWIFT_ZONE""#))
     XCTAssertTrue(bootstrapDebugDescription.contains(#"sub_zone: "SWIFT_SUB""#))
   }
-#endif
 
   func testAddingKeyValueStoreToConfigurationWhenRunningEnvoy() {
     let expectation = self.expectation(description: "Run called with expected data")

From 1ea97aec7e15015b2b892202a159260cb41dee60 Mon Sep 17 00:00:00 2001
From: Kuat 
Date: Tue, 20 Jun 2023 09:24:48 -0700
Subject: [PATCH 584/740] cel: use singleton builder (#27861)

Commit Message: Consolidate the CEL expression builder construction under a shared singleton to amortize the cost and make the lifecycle explicit (expressions must not outlive the builder).
Additional Description: This required the following changes to the factory interface to get access to the singleton manager:

```C++
// before in substitution formatter
CommandParserFactory::createCommandParserFromProto(const Protobuf::Message&);
// after
CommandParserFactory::createCommandParserFromProto(const Protobuf::Message&,
                  Server::Configuration::CommonFactoryContext&);

// before in access log filter extension
FilterFactory::fromProto(const envoy::config::accesslog::v3::AccessLogFilter&,
       Runtime::Loader& , Random::RandomGenerator&,
       ProtobufMessage::ValidationVisitor&);
// after
FilterFactory::fromProto(const envoy::config::accesslog::v3::AccessLogFilter&,
       Server::Configuration::CommonFactoryContext&);
```

There are three remaining instances of CEL builder in rate limit descriptor, Wasm, and RBAC, which will need a follow-up.

Risk Level: low, extensions
Testing: refactor
Docs Changes: none
Release Notes: none
---
 envoy/formatter/BUILD                         |  1 +
 envoy/formatter/substitution_formatter.h      |  6 +++-
 source/common/access_log/access_log_impl.cc   | 33 +++++++++----------
 source/common/access_log/access_log_impl.h    | 20 +++++------
 .../formatter/substitution_format_string.cc   |  2 +-
 source/common/local_reply/local_reply.cc      |  4 +--
 .../access_loggers/filters/cel/cel.cc         |  6 ++--
 .../access_loggers/filters/cel/cel.h          |  3 +-
 .../access_loggers/filters/cel/config.cc      | 22 ++++---------
 .../access_loggers/filters/cel/config.h       |  8 ++---
 source/extensions/filters/common/expr/BUILD   |  1 +
 .../filters/common/expr/evaluator.cc          | 10 ++++++
 .../filters/common/expr/evaluator.h           | 15 +++++++++
 source/extensions/formatter/cel/cel.cc        |  8 ++---
 source/extensions/formatter/cel/cel.h         |  9 ++---
 source/extensions/formatter/cel/config.cc     |  7 ++--
 source/extensions/formatter/cel/config.h      |  3 +-
 .../extensions/formatter/metadata/config.cc   |  4 +--
 source/extensions/formatter/metadata/config.h |  3 +-
 .../formatter/req_without_query/config.cc     |  3 +-
 .../formatter/req_without_query/config.h      |  3 +-
 .../input_matchers/cel_matcher/config.h       | 15 +++------
 .../input_matchers/cel_matcher/matcher.cc     | 13 ++++----
 .../input_matchers/cel_matcher/matcher.h      |  6 ++--
 .../common/access_log/access_log_impl_test.cc |  8 ++---
 test/common/formatter/command_extension.cc    | 10 +++---
 test/common/formatter/command_extension.h     | 12 +++++--
 test/extensions/formatter/cel/cel_test.cc     |  6 ++--
 28 files changed, 129 insertions(+), 112 deletions(-)

diff --git a/envoy/formatter/BUILD b/envoy/formatter/BUILD
index 8c691b773bad..bf7979b7057f 100644
--- a/envoy/formatter/BUILD
+++ b/envoy/formatter/BUILD
@@ -15,6 +15,7 @@ envoy_cc_library(
         "//envoy/access_log:access_log_interface",
         "//envoy/config:typed_config_interface",
         "//envoy/http:header_map_interface",
+        "//envoy/server:factory_context_interface",
         "//envoy/stream_info:stream_info_interface",
     ],
 )
diff --git a/envoy/formatter/substitution_formatter.h b/envoy/formatter/substitution_formatter.h
index 6c4c3547cfba..719b1dabd0d2 100644
--- a/envoy/formatter/substitution_formatter.h
+++ b/envoy/formatter/substitution_formatter.h
@@ -7,6 +7,7 @@
 #include "envoy/common/pure.h"
 #include "envoy/config/typed_config.h"
 #include "envoy/http/header_map.h"
+#include "envoy/server/factory_context.h"
 #include "envoy/stream_info/stream_info.h"
 
 namespace Envoy {
@@ -122,10 +123,13 @@ class CommandParserFactory : public Config::TypedFactory {
    * Creates a particular CommandParser implementation.
    *
    * @param config supplies the configuration for the command parser.
+   * @param context supplies the factory context.
    * @return CommandParserPtr the CommandParser which will be used in
    * SubstitutionFormatParser::parse() when evaluating an access log format string.
    */
-  virtual CommandParserPtr createCommandParserFromProto(const Protobuf::Message& config) PURE;
+  virtual CommandParserPtr
+  createCommandParserFromProto(const Protobuf::Message& config,
+                               Server::Configuration::CommonFactoryContext& context) PURE;
 
   std::string category() const override { return "envoy.formatter"; }
 };
diff --git a/source/common/access_log/access_log_impl.cc b/source/common/access_log/access_log_impl.cc
index 91bafcdb5ce2..7df906595462 100644
--- a/source/common/access_log/access_log_impl.cc
+++ b/source/common/access_log/access_log_impl.cc
@@ -53,8 +53,10 @@ bool ComparisonFilter::compareAgainstValue(uint64_t lhs) const {
 }
 
 FilterPtr FilterFactory::fromProto(const envoy::config::accesslog::v3::AccessLogFilter& config,
-                                   Runtime::Loader& runtime, Random::RandomGenerator& random,
-                                   ProtobufMessage::ValidationVisitor& validation_visitor) {
+                                   Server::Configuration::CommonFactoryContext& context) {
+  Runtime::Loader& runtime = context.runtime();
+  Random::RandomGenerator& random = context.api().randomGenerator();
+  ProtobufMessage::ValidationVisitor& validation_visitor = context.messageValidationVisitor();
   switch (config.filter_specifier_case()) {
   case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kStatusCodeFilter:
     return FilterPtr{new StatusCodeFilter(config.status_code_filter(), runtime)};
@@ -67,9 +69,9 @@ FilterPtr FilterFactory::fromProto(const envoy::config::accesslog::v3::AccessLog
   case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kRuntimeFilter:
     return FilterPtr{new RuntimeFilter(config.runtime_filter(), runtime, random)};
   case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kAndFilter:
-    return FilterPtr{new AndFilter(config.and_filter(), runtime, random, validation_visitor)};
+    return FilterPtr{new AndFilter(config.and_filter(), context)};
   case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kOrFilter:
-    return FilterPtr{new OrFilter(config.or_filter(), runtime, random, validation_visitor)};
+    return FilterPtr{new OrFilter(config.or_filter(), context)};
   case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kHeaderFilter:
     return FilterPtr{new HeaderFilter(config.header_filter())};
   case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kResponseFlagFilter:
@@ -87,7 +89,7 @@ FilterPtr FilterFactory::fromProto(const envoy::config::accesslog::v3::AccessLog
     {
       auto& factory =
           Config::Utility::getAndCheckFactory(config.extension_filter());
-      return factory.createFilter(config.extension_filter(), runtime, random);
+      return factory.createFilter(config.extension_filter(), context);
     }
   case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::FILTER_SPECIFIER_NOT_SET:
     PANIC_DUE_TO_PROTO_UNSET;
@@ -158,25 +160,22 @@ bool RuntimeFilter::evaluate(const StreamInfo::StreamInfo& stream_info,
 
 OperatorFilter::OperatorFilter(
     const Protobuf::RepeatedPtrField& configs,
-    Runtime::Loader& runtime, Random::RandomGenerator& random,
-    ProtobufMessage::ValidationVisitor& validation_visitor) {
+    Server::Configuration::CommonFactoryContext& context) {
   for (const auto& config : configs) {
-    auto filter = FilterFactory::fromProto(config, runtime, random, validation_visitor);
+    auto filter = FilterFactory::fromProto(config, context);
     if (filter != nullptr) {
       filters_.emplace_back(std::move(filter));
     }
   }
 }
 
-OrFilter::OrFilter(const envoy::config::accesslog::v3::OrFilter& config, Runtime::Loader& runtime,
-                   Random::RandomGenerator& random,
-                   ProtobufMessage::ValidationVisitor& validation_visitor)
-    : OperatorFilter(config.filters(), runtime, random, validation_visitor) {}
+OrFilter::OrFilter(const envoy::config::accesslog::v3::OrFilter& config,
+                   Server::Configuration::CommonFactoryContext& context)
+    : OperatorFilter(config.filters(), context) {}
 
 AndFilter::AndFilter(const envoy::config::accesslog::v3::AndFilter& config,
-                     Runtime::Loader& runtime, Random::RandomGenerator& random,
-                     ProtobufMessage::ValidationVisitor& validation_visitor)
-    : OperatorFilter(config.filters(), runtime, random, validation_visitor) {}
+                     Server::Configuration::CommonFactoryContext& context)
+    : OperatorFilter(config.filters(), context) {}
 
 bool OrFilter::evaluate(const StreamInfo::StreamInfo& info,
                         const Http::RequestHeaderMap& request_headers,
@@ -339,9 +338,7 @@ InstanceSharedPtr makeAccessLogInstance(const envoy::config::accesslog::v3::Acce
                                         FactoryContext& context) {
   FilterPtr filter;
   if (config.has_filter()) {
-    filter = FilterFactory::fromProto(config.filter(), context.runtime(),
-                                      context.api().randomGenerator(),
-                                      context.messageValidationVisitor());
+    filter = FilterFactory::fromProto(config.filter(), context);
   }
 
   auto& factory =
diff --git a/source/common/access_log/access_log_impl.h b/source/common/access_log/access_log_impl.h
index 2305181720ef..0a0fe9a05775 100644
--- a/source/common/access_log/access_log_impl.h
+++ b/source/common/access_log/access_log_impl.h
@@ -32,8 +32,7 @@ class FilterFactory {
    * Read a filter definition from proto and instantiate a concrete filter class.
    */
   static FilterPtr fromProto(const envoy::config::accesslog::v3::AccessLogFilter& config,
-                             Runtime::Loader& runtime, Random::RandomGenerator& random,
-                             ProtobufMessage::ValidationVisitor& validation_visitor);
+                             Server::Configuration::CommonFactoryContext& context);
 };
 
 /**
@@ -87,8 +86,7 @@ class OperatorFilter : public Filter {
 public:
   OperatorFilter(
       const Protobuf::RepeatedPtrField& configs,
-      Runtime::Loader& runtime, Random::RandomGenerator& random,
-      ProtobufMessage::ValidationVisitor& validation_visitor);
+      Server::Configuration::CommonFactoryContext& context);
 
 protected:
   std::vector filters_;
@@ -99,9 +97,8 @@ class OperatorFilter : public Filter {
  */
 class AndFilter : public OperatorFilter {
 public:
-  AndFilter(const envoy::config::accesslog::v3::AndFilter& config, Runtime::Loader& runtime,
-            Random::RandomGenerator& random,
-            ProtobufMessage::ValidationVisitor& validation_visitor);
+  AndFilter(const envoy::config::accesslog::v3::AndFilter& config,
+            Server::Configuration::CommonFactoryContext& context);
 
   // AccessLog::Filter
   bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers,
@@ -114,8 +111,8 @@ class AndFilter : public OperatorFilter {
  */
 class OrFilter : public OperatorFilter {
 public:
-  OrFilter(const envoy::config::accesslog::v3::OrFilter& config, Runtime::Loader& runtime,
-           Random::RandomGenerator& random, ProtobufMessage::ValidationVisitor& validation_visitor);
+  OrFilter(const envoy::config::accesslog::v3::OrFilter& config,
+           Server::Configuration::CommonFactoryContext& context);
 
   // AccessLog::Filter
   bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers,
@@ -280,12 +277,11 @@ class ExtensionFilterFactory : public Config::TypedFactory {
    * implementation is unable to produce a filter with the provided parameters, it should throw an
    * EnvoyException. The returned pointer should never be nullptr.
    * @param config supplies the custom configuration for this filter type.
-   * @param runtime supplies the runtime loader.
-   * @param random supplies the random generator.
+   * @param context supplies the server factory context.
    * @return an instance of extension filter implementation from a config proto.
    */
   virtual FilterPtr createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config,
-                                 Runtime::Loader& runtime, Random::RandomGenerator& random) PURE;
+                                 Server::Configuration::CommonFactoryContext& context) PURE;
 
   std::string category() const override { return "envoy.access_loggers.extension_filters"; }
 };
diff --git a/source/common/formatter/substitution_format_string.cc b/source/common/formatter/substitution_format_string.cc
index 9eede2c4ddee..ab838fc8904c 100644
--- a/source/common/formatter/substitution_format_string.cc
+++ b/source/common/formatter/substitution_format_string.cc
@@ -30,7 +30,7 @@ FormatterPtr SubstitutionFormatStringUtils::fromProtoConfig(
     }
     auto typed_config = Envoy::Config::Utility::translateAnyToFactoryConfig(
         formatter.typed_config(), context.messageValidationVisitor(), *factory);
-    auto parser = factory->createCommandParserFromProto(*typed_config);
+    auto parser = factory->createCommandParserFromProto(*typed_config, context);
     if (!parser) {
       throw EnvoyException(absl::StrCat("Failed to create command parser: ", formatter.name()));
     }
diff --git a/source/common/local_reply/local_reply.cc b/source/common/local_reply/local_reply.cc
index edc51b177be3..f3c744ccf01e 100644
--- a/source/common/local_reply/local_reply.cc
+++ b/source/common/local_reply/local_reply.cc
@@ -56,9 +56,7 @@ class ResponseMapper {
       const envoy::extensions::filters::network::http_connection_manager::v3::ResponseMapper&
           config,
       Server::Configuration::FactoryContext& context)
-      : filter_(AccessLog::FilterFactory::fromProto(config.filter(), context.runtime(),
-                                                    context.api().randomGenerator(),
-                                                    context.messageValidationVisitor())) {
+      : filter_(AccessLog::FilterFactory::fromProto(config.filter(), context)) {
     if (config.has_status_code()) {
       status_code_ = static_cast(config.status_code().value());
     }
diff --git a/source/extensions/access_loggers/filters/cel/cel.cc b/source/extensions/access_loggers/filters/cel/cel.cc
index f0c694396363..25684a54afdd 100644
--- a/source/extensions/access_loggers/filters/cel/cel.cc
+++ b/source/extensions/access_loggers/filters/cel/cel.cc
@@ -9,9 +9,9 @@ namespace CEL {
 namespace Expr = Envoy::Extensions::Filters::Common::Expr;
 
 CELAccessLogExtensionFilter::CELAccessLogExtensionFilter(
-    Expr::Builder& builder, const google::api::expr::v1alpha1::Expr& input_expr)
-    : parsed_expr_(input_expr) {
-  compiled_expr_ = Expr::createExpression(builder, parsed_expr_);
+    Expr::BuilderInstanceSharedPtr builder, const google::api::expr::v1alpha1::Expr& input_expr)
+    : builder_(builder), parsed_expr_(input_expr) {
+  compiled_expr_ = Expr::createExpression(builder_->builder(), parsed_expr_);
 }
 
 bool CELAccessLogExtensionFilter::evaluate(const StreamInfo::StreamInfo& stream_info,
diff --git a/source/extensions/access_loggers/filters/cel/cel.h b/source/extensions/access_loggers/filters/cel/cel.h
index dc851eadb748..e8138df97281 100644
--- a/source/extensions/access_loggers/filters/cel/cel.h
+++ b/source/extensions/access_loggers/filters/cel/cel.h
@@ -17,7 +17,7 @@ namespace CEL {
 
 class CELAccessLogExtensionFilter : public AccessLog::Filter {
 public:
-  CELAccessLogExtensionFilter(Extensions::Filters::Common::Expr::Builder&,
+  CELAccessLogExtensionFilter(Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr,
                               const google::api::expr::v1alpha1::Expr&);
 
   bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers,
@@ -26,6 +26,7 @@ class CELAccessLogExtensionFilter : public AccessLog::Filter {
                 AccessLog::AccessLogType access_log_type) const override;
 
 private:
+  Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr builder_;
   const google::api::expr::v1alpha1::Expr parsed_expr_;
   Extensions::Filters::Common::Expr::ExpressionPtr compiled_expr_;
 };
diff --git a/source/extensions/access_loggers/filters/cel/config.cc b/source/extensions/access_loggers/filters/cel/config.cc
index 0df41fc19c52..2431a2e9dbf2 100644
--- a/source/extensions/access_loggers/filters/cel/config.cc
+++ b/source/extensions/access_loggers/filters/cel/config.cc
@@ -15,13 +15,11 @@ namespace Filters {
 namespace CEL {
 
 Envoy::AccessLog::FilterPtr CELAccessLogExtensionFilterFactory::createFilter(
-    const envoy::config::accesslog::v3::ExtensionFilter& config, Runtime::Loader&,
-    Random::RandomGenerator&) {
+    const envoy::config::accesslog::v3::ExtensionFilter& config,
+    Server::Configuration::CommonFactoryContext& context) {
 
-  // TODO(douglas-reid): use factory_context validation. likely needs update to
-  // createFilter signature to pass in validation visitor.
-  auto factory_config = Config::Utility::translateToFactoryConfig(
-      config, Envoy::ProtobufMessage::getNullValidationVisitor(), *this);
+  auto factory_config =
+      Config::Utility::translateToFactoryConfig(config, context.messageValidationVisitor(), *this);
 
 #if defined(USE_CEL_PARSER)
   envoy::extensions::access_loggers::filters::cel::v3::ExpressionFilter cel_config =
@@ -34,8 +32,8 @@ Envoy::AccessLog::FilterPtr CELAccessLogExtensionFilterFactory::createFilter(
                          parse_status.status().ToString());
   }
 
-  return std::make_unique(getOrCreateBuilder(),
-                                                       parse_status.value().expr());
+  return std::make_unique(
+      Extensions::Filters::Common::Expr::getBuilder(context), parse_status.value().expr());
 #else
   throw EnvoyException("CEL is not available for use in this environment.");
 #endif
@@ -45,14 +43,6 @@ ProtobufTypes::MessagePtr CELAccessLogExtensionFilterFactory::createEmptyConfigP
   return std::make_unique();
 }
 
-Extensions::Filters::Common::Expr::Builder&
-CELAccessLogExtensionFilterFactory::getOrCreateBuilder() {
-  if (expr_builder_ == nullptr) {
-    expr_builder_ = Extensions::Filters::Common::Expr::createBuilder(nullptr);
-  }
-  return *expr_builder_;
-}
-
 /**
  * Static registration for the CELAccessLogExtensionFilter. @see RegisterFactory.
  */
diff --git a/source/extensions/access_loggers/filters/cel/config.h b/source/extensions/access_loggers/filters/cel/config.h
index 0266af6fbb0a..383a02950d04 100644
--- a/source/extensions/access_loggers/filters/cel/config.h
+++ b/source/extensions/access_loggers/filters/cel/config.h
@@ -19,14 +19,10 @@ namespace CEL {
 class CELAccessLogExtensionFilterFactory : public Envoy::AccessLog::ExtensionFilterFactory {
 public:
   Envoy::AccessLog::FilterPtr
-  createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config, Runtime::Loader&,
-               Random::RandomGenerator&) override;
+  createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config,
+               Server::Configuration::CommonFactoryContext& context) override;
   ProtobufTypes::MessagePtr createEmptyConfigProto() override;
   std::string name() const override { return "envoy.access_loggers.extension_filters.cel"; }
-
-private:
-  Extensions::Filters::Common::Expr::Builder& getOrCreateBuilder();
-  Extensions::Filters::Common::Expr::BuilderPtr expr_builder_;
 };
 
 } // namespace CEL
diff --git a/source/extensions/filters/common/expr/BUILD b/source/extensions/filters/common/expr/BUILD
index 717c298ad31d..ebeb45ce0adb 100644
--- a/source/extensions/filters/common/expr/BUILD
+++ b/source/extensions/filters/common/expr/BUILD
@@ -14,6 +14,7 @@ envoy_cc_library(
     hdrs = ["evaluator.h"],
     deps = [
         ":context_lib",
+        "//envoy/singleton:manager_interface",
         "//source/common/http:utility_lib",
         "//source/common/protobuf",
         "@com_google_cel_cpp//eval/public:activation",
diff --git a/source/extensions/filters/common/expr/evaluator.cc b/source/extensions/filters/common/expr/evaluator.cc
index 1722892e5102..f7f965e9b3b9 100644
--- a/source/extensions/filters/common/expr/evaluator.cc
+++ b/source/extensions/filters/common/expr/evaluator.cc
@@ -1,6 +1,7 @@
 #include "source/extensions/filters/common/expr/evaluator.h"
 
 #include "envoy/common/exception.h"
+#include "envoy/singleton/manager.h"
 
 #include "eval/public/builtin_func_registrar.h"
 #include "eval/public/cel_expr_builder_factory.h"
@@ -84,6 +85,7 @@ ActivationPtr createActivation(const StreamInfo::StreamInfo& info,
 }
 
 BuilderPtr createBuilder(Protobuf::Arena* arena) {
+  ASSERT_IS_MAIN_OR_TEST_THREAD();
   google::api::expr::runtime::InterpreterOptions options;
 
   // Security-oriented defaults
@@ -110,6 +112,14 @@ BuilderPtr createBuilder(Protobuf::Arena* arena) {
   return builder;
 }
 
+SINGLETON_MANAGER_REGISTRATION(expression_builder);
+
+BuilderInstanceSharedPtr getBuilder(Server::Configuration::CommonFactoryContext& context) {
+  return context.singletonManager().getTyped(
+      SINGLETON_MANAGER_REGISTERED_NAME(expression_builder),
+      [] { return std::make_shared(createBuilder(nullptr)); });
+}
+
 ExpressionPtr createExpression(Builder& builder, const google::api::expr::v1alpha1::Expr& expr) {
   google::api::expr::v1alpha1::SourceInfo source_info;
   auto cel_expression_status = builder.CreateExpression(&expr, &source_info);
diff --git a/source/extensions/filters/common/expr/evaluator.h b/source/extensions/filters/common/expr/evaluator.h
index 63ed1e1266b8..656a9c7e676f 100644
--- a/source/extensions/filters/common/expr/evaluator.h
+++ b/source/extensions/filters/common/expr/evaluator.h
@@ -57,11 +57,26 @@ ActivationPtr createActivation(const StreamInfo::StreamInfo& info,
                                const ::Envoy::Http::ResponseHeaderMap* response_headers,
                                const ::Envoy::Http::ResponseTrailerMap* response_trailers);
 
+// Shared expression builder instance.
+class BuilderInstance : public Singleton::Instance {
+public:
+  explicit BuilderInstance(BuilderPtr builder) : builder_(std::move(builder)) {}
+  Builder& builder() { return *builder_; }
+
+private:
+  BuilderPtr builder_;
+};
+
+using BuilderInstanceSharedPtr = std::shared_ptr;
+
 // Creates an expression builder. The optional arena is used to enable constant folding
 // for intermediate evaluation results.
 // Throws an exception if fails to construct an expression builder.
 BuilderPtr createBuilder(Protobuf::Arena* arena);
 
+// Gets the singleton expression builder. Must be called on the main thread.
+BuilderInstanceSharedPtr getBuilder(Server::Configuration::CommonFactoryContext& context);
+
 // Creates an interpretable expression from a protobuf representation.
 // Throws an exception if fails to construct a runtime expression.
 ExpressionPtr createExpression(Builder& builder, const google::api::expr::v1alpha1::Expr& expr);
diff --git a/source/extensions/formatter/cel/cel.cc b/source/extensions/formatter/cel/cel.cc
index b0802e81139c..0802bd0661f2 100644
--- a/source/extensions/formatter/cel/cel.cc
+++ b/source/extensions/formatter/cel/cel.cc
@@ -15,11 +15,11 @@ namespace Formatter {
 
 namespace Expr = Filters::Common::Expr;
 
-CELFormatter::CELFormatter(Expr::Builder& builder,
+CELFormatter::CELFormatter(Expr::BuilderInstanceSharedPtr expr_builder,
                            const google::api::expr::v1alpha1::Expr& input_expr,
                            absl::optional& max_length)
-    : parsed_expr_(input_expr), max_length_(max_length) {
-  compiled_expr_ = Expr::createExpression(builder, parsed_expr_);
+    : expr_builder_(expr_builder), parsed_expr_(input_expr), max_length_(max_length) {
+  compiled_expr_ = Expr::createExpression(expr_builder_->builder(), parsed_expr_);
 }
 
 absl::optional CELFormatter::format(const Http::RequestHeaderMap& request_headers,
@@ -66,7 +66,7 @@ CELFormatterCommandParser::parse(const std::string& command, const std::string&
                            parse_status.status().ToString());
     }
 
-    return std::make_unique(*expr_builder_, parse_status.value().expr(), max_length);
+    return std::make_unique(expr_builder_, parse_status.value().expr(), max_length);
   }
 
   return nullptr;
diff --git a/source/extensions/formatter/cel/cel.h b/source/extensions/formatter/cel/cel.h
index d54e881d6c26..91e8a3ffb73c 100644
--- a/source/extensions/formatter/cel/cel.h
+++ b/source/extensions/formatter/cel/cel.h
@@ -14,7 +14,7 @@ namespace Formatter {
 
 class CELFormatter : public ::Envoy::Formatter::FormatterProvider {
 public:
-  CELFormatter(Extensions::Filters::Common::Expr::Builder&,
+  CELFormatter(Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr,
                const google::api::expr::v1alpha1::Expr&, absl::optional&);
 
   absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&,
@@ -25,6 +25,7 @@ class CELFormatter : public ::Envoy::Formatter::FormatterProvider {
                                  absl::string_view, AccessLog::AccessLogType) const override;
 
 private:
+  Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr expr_builder_;
   const google::api::expr::v1alpha1::Expr parsed_expr_;
   const absl::optional max_length_;
   Extensions::Filters::Common::Expr::ExpressionPtr compiled_expr_;
@@ -32,14 +33,14 @@ class CELFormatter : public ::Envoy::Formatter::FormatterProvider {
 
 class CELFormatterCommandParser : public ::Envoy::Formatter::CommandParser {
 public:
-  CELFormatterCommandParser()
-      : expr_builder_(Extensions::Filters::Common::Expr::createBuilder(nullptr)){};
+  CELFormatterCommandParser(Server::Configuration::CommonFactoryContext& context)
+      : expr_builder_(Extensions::Filters::Common::Expr::getBuilder(context)){};
   ::Envoy::Formatter::FormatterProviderPtr parse(const std::string& command,
                                                  const std::string& subcommand,
                                                  absl::optional& max_length) const override;
 
 private:
-  Extensions::Filters::Common::Expr::BuilderPtr expr_builder_;
+  Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr expr_builder_;
 };
 
 } // namespace Formatter
diff --git a/source/extensions/formatter/cel/config.cc b/source/extensions/formatter/cel/config.cc
index 79534aed08d8..2b2933c00f1f 100644
--- a/source/extensions/formatter/cel/config.cc
+++ b/source/extensions/formatter/cel/config.cc
@@ -8,11 +8,12 @@ namespace Envoy {
 namespace Extensions {
 namespace Formatter {
 
-::Envoy::Formatter::CommandParserPtr
-CELFormatterFactory::createCommandParserFromProto(const Protobuf::Message&) {
+::Envoy::Formatter::CommandParserPtr CELFormatterFactory::createCommandParserFromProto(
+    const Protobuf::Message&, Server::Configuration::CommonFactoryContext& context) {
 #if defined(USE_CEL_PARSER)
-  return std::make_unique();
+  return std::make_unique(context);
 #else
+  UNREFERENCED_PARAMETER(context);
   throw EnvoyException("CEL is not available for use in this environment.");
 #endif
 }
diff --git a/source/extensions/formatter/cel/config.h b/source/extensions/formatter/cel/config.h
index 4016ade6b0c6..ab2750dded5a 100644
--- a/source/extensions/formatter/cel/config.h
+++ b/source/extensions/formatter/cel/config.h
@@ -10,7 +10,8 @@ namespace Formatter {
 class CELFormatterFactory : public ::Envoy::Formatter::CommandParserFactory {
 public:
   ::Envoy::Formatter::CommandParserPtr
-  createCommandParserFromProto(const Protobuf::Message&) override;
+  createCommandParserFromProto(const Protobuf::Message&,
+                               Server::Configuration::CommonFactoryContext&) override;
   ProtobufTypes::MessagePtr createEmptyConfigProto() override;
   std::string name() const override;
 };
diff --git a/source/extensions/formatter/metadata/config.cc b/source/extensions/formatter/metadata/config.cc
index 7aa27b882910..1ec2cbc65cbe 100644
--- a/source/extensions/formatter/metadata/config.cc
+++ b/source/extensions/formatter/metadata/config.cc
@@ -8,8 +8,8 @@ namespace Envoy {
 namespace Extensions {
 namespace Formatter {
 
-::Envoy::Formatter::CommandParserPtr
-MetadataFormatterFactory::createCommandParserFromProto(const Protobuf::Message&) {
+::Envoy::Formatter::CommandParserPtr MetadataFormatterFactory::createCommandParserFromProto(
+    const Protobuf::Message&, Server::Configuration::CommonFactoryContext&) {
   return std::make_unique();
 }
 
diff --git a/source/extensions/formatter/metadata/config.h b/source/extensions/formatter/metadata/config.h
index ec080f8d806d..c615b5c10bd6 100644
--- a/source/extensions/formatter/metadata/config.h
+++ b/source/extensions/formatter/metadata/config.h
@@ -9,7 +9,8 @@ namespace Formatter {
 class MetadataFormatterFactory : public ::Envoy::Formatter::CommandParserFactory {
 public:
   ::Envoy::Formatter::CommandParserPtr
-  createCommandParserFromProto(const Protobuf::Message&) override;
+  createCommandParserFromProto(const Protobuf::Message&,
+                               Server::Configuration::CommonFactoryContext&) override;
   ProtobufTypes::MessagePtr createEmptyConfigProto() override;
   std::string name() const override;
 };
diff --git a/source/extensions/formatter/req_without_query/config.cc b/source/extensions/formatter/req_without_query/config.cc
index 76035de46951..3caf609aaabf 100644
--- a/source/extensions/formatter/req_without_query/config.cc
+++ b/source/extensions/formatter/req_without_query/config.cc
@@ -9,7 +9,8 @@ namespace Extensions {
 namespace Formatter {
 
 ::Envoy::Formatter::CommandParserPtr
-ReqWithoutQueryFactory::createCommandParserFromProto(const Protobuf::Message&) {
+ReqWithoutQueryFactory::createCommandParserFromProto(const Protobuf::Message&,
+                                                     Server::Configuration::CommonFactoryContext&) {
   return std::make_unique();
 }
 
diff --git a/source/extensions/formatter/req_without_query/config.h b/source/extensions/formatter/req_without_query/config.h
index 71b33f3904b0..a648197f392f 100644
--- a/source/extensions/formatter/req_without_query/config.h
+++ b/source/extensions/formatter/req_without_query/config.h
@@ -9,7 +9,8 @@ namespace Formatter {
 class ReqWithoutQueryFactory : public ::Envoy::Formatter::CommandParserFactory {
 public:
   ::Envoy::Formatter::CommandParserPtr
-  createCommandParserFromProto(const Protobuf::Message&) override;
+  createCommandParserFromProto(const Protobuf::Message&,
+                               Server::Configuration::CommonFactoryContext&) override;
   ProtobufTypes::MessagePtr createEmptyConfigProto() override;
   std::string name() const override;
 };
diff --git a/source/extensions/matching/input_matchers/cel_matcher/config.h b/source/extensions/matching/input_matchers/cel_matcher/config.h
index d894ebe7cb8a..608db0ff63f1 100644
--- a/source/extensions/matching/input_matchers/cel_matcher/config.h
+++ b/source/extensions/matching/input_matchers/cel_matcher/config.h
@@ -20,18 +20,15 @@ class CelInputMatcherFactory : public ::Envoy::Matcher::InputMatcherFactory {
 public:
   InputMatcherFactoryCb
   createInputMatcherFactoryCb(const Protobuf::Message& config,
-                              Server::Configuration::ServerFactoryContext&) override {
-    if (expr_builder_ == nullptr) {
-      expr_builder_ = Extensions::Filters::Common::Expr::createBuilder(nullptr);
-    }
-
+                              Server::Configuration::ServerFactoryContext& context) override {
     const auto& cel_matcher_config =
         dynamic_cast(config);
     CelMatcherSharedPtr cel_matcher =
         std::make_shared<::xds::type::matcher::v3::CelMatcher>(cel_matcher_config);
 
-    return [cel_matcher = std::move(cel_matcher), this] {
-      return std::make_unique(cel_matcher, *expr_builder_);
+    return [cel_matcher = std::move(cel_matcher), &context] {
+      return std::make_unique(cel_matcher,
+                                               Filters::Common::Expr::getBuilder(context));
     };
   }
 
@@ -40,10 +37,6 @@ class CelInputMatcherFactory : public ::Envoy::Matcher::InputMatcherFactory {
   }
 
   std::string name() const override { return "envoy.matching.matchers.cel_matcher"; }
-
-private:
-  // Expression builder must outlive the compiled expression.
-  BuilderPtr expr_builder_;
 };
 
 } // namespace CelMatcher
diff --git a/source/extensions/matching/input_matchers/cel_matcher/matcher.cc b/source/extensions/matching/input_matchers/cel_matcher/matcher.cc
index d6d61bc81142..86d5b6fd56fa 100644
--- a/source/extensions/matching/input_matchers/cel_matcher/matcher.cc
+++ b/source/extensions/matching/input_matchers/cel_matcher/matcher.cc
@@ -9,17 +9,18 @@ namespace CelMatcher {
 using ::Envoy::Extensions::Matching::Http::CelInput::CelMatchData;
 using ::xds::type::v3::CelExpression;
 
-CelInputMatcher::CelInputMatcher(CelMatcherSharedPtr cel_matcher, Builder& builder)
-    : cel_matcher_(std::move(cel_matcher)) {
+CelInputMatcher::CelInputMatcher(CelMatcherSharedPtr cel_matcher,
+                                 Filters::Common::Expr::BuilderInstanceSharedPtr builder)
+    : builder_(builder), cel_matcher_(std::move(cel_matcher)) {
   const CelExpression& input_expr = cel_matcher_->expr_match();
   switch (input_expr.expr_specifier_case()) {
   case CelExpression::ExprSpecifierCase::kParsedExpr:
-    compiled_expr_ =
-        Filters::Common::Expr::createExpression(builder, input_expr.parsed_expr().expr());
+    compiled_expr_ = Filters::Common::Expr::createExpression(builder_->builder(),
+                                                             input_expr.parsed_expr().expr());
     return;
   case CelExpression::ExprSpecifierCase::kCheckedExpr:
-    compiled_expr_ =
-        Filters::Common::Expr::createExpression(builder, input_expr.checked_expr().expr());
+    compiled_expr_ = Filters::Common::Expr::createExpression(builder_->builder(),
+                                                             input_expr.checked_expr().expr());
     return;
   case CelExpression::ExprSpecifierCase::EXPR_SPECIFIER_NOT_SET:
     PANIC_DUE_TO_PROTO_UNSET;
diff --git a/source/extensions/matching/input_matchers/cel_matcher/matcher.h b/source/extensions/matching/input_matchers/cel_matcher/matcher.h
index 5ac9e366c675..57e15a02c6c7 100644
--- a/source/extensions/matching/input_matchers/cel_matcher/matcher.h
+++ b/source/extensions/matching/input_matchers/cel_matcher/matcher.h
@@ -26,13 +26,12 @@ using ::Envoy::Matcher::MatchingDataType;
 using CelMatcher = ::xds::type::matcher::v3::CelMatcher;
 using CompiledExpressionPtr = std::unique_ptr;
 using BaseActivationPtr = std::unique_ptr;
-using Builder = google::api::expr::runtime::CelExpressionBuilder;
-using BuilderPtr = std::unique_ptr;
 using CelMatcherSharedPtr = std::shared_ptr<::xds::type::matcher::v3::CelMatcher>;
 
 class CelInputMatcher : public InputMatcher, public Logger::Loggable {
 public:
-  CelInputMatcher(CelMatcherSharedPtr cel_matcher, Builder& builder);
+  CelInputMatcher(CelMatcherSharedPtr cel_matcher,
+                  Filters::Common::Expr::BuilderInstanceSharedPtr builder);
 
   bool match(const MatchingDataType& input) override;
 
@@ -42,6 +41,7 @@ class CelInputMatcher : public InputMatcher, public Logger::Loggable(
             *factory_config);
@@ -1740,9 +1740,9 @@ class SampleExtensionFilterFactory : public ExtensionFilterFactory {
   ~SampleExtensionFilterFactory() override = default;
 
   FilterPtr createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config,
-                         Runtime::Loader&, Random::RandomGenerator&) override {
+                         Server::Configuration::CommonFactoryContext& context) override {
     auto factory_config = Config::Utility::translateToFactoryConfig(
-        config, Envoy::ProtobufMessage::getNullValidationVisitor(), *this);
+        config, context.messageValidationVisitor(), *this);
 
     ProtobufWkt::Struct struct_config =
         *dynamic_cast(factory_config.get());
diff --git a/test/common/formatter/command_extension.cc b/test/common/formatter/command_extension.cc
index 99dd2f7f6cfa..33695e85c639 100644
--- a/test/common/formatter/command_extension.cc
+++ b/test/common/formatter/command_extension.cc
@@ -31,7 +31,8 @@ FormatterProviderPtr TestCommandParser::parse(const std::string& command, const
 }
 
 CommandParserPtr
-TestCommandFactory::createCommandParserFromProto(const Protobuf::Message& message) {
+TestCommandFactory::createCommandParserFromProto(const Protobuf::Message& message,
+                                                 Server::Configuration::CommonFactoryContext&) {
   // Cast the config message to the actual type to test that it was constructed properly.
   [[maybe_unused]] const auto config = dynamic_cast(message);
   return std::make_unique();
@@ -68,8 +69,8 @@ FormatterProviderPtr AdditionalCommandParser::parse(const std::string& command,
   return nullptr;
 }
 
-CommandParserPtr
-AdditionalCommandFactory::createCommandParserFromProto(const Protobuf::Message& message) {
+CommandParserPtr AdditionalCommandFactory::createCommandParserFromProto(
+    const Protobuf::Message& message, Server::Configuration::CommonFactoryContext&) {
   // Cast the config message to the actual type to test that it was constructed properly.
   [[maybe_unused]] const auto config = dynamic_cast(message);
   return std::make_unique();
@@ -86,7 +87,8 @@ ProtobufTypes::MessagePtr AdditionalCommandFactory::createEmptyConfigProto() {
 std::string AdditionalCommandFactory::name() const { return "envoy.formatter.AdditionalFormatter"; }
 
 CommandParserPtr
-FailCommandFactory::createCommandParserFromProto(const Protobuf::Message& message) {
+FailCommandFactory::createCommandParserFromProto(const Protobuf::Message& message,
+                                                 Server::Configuration::CommonFactoryContext&) {
   // Cast the config message to the actual type to test that it was constructed properly.
   [[maybe_unused]] const auto config = dynamic_cast(message);
   return nullptr;
diff --git a/test/common/formatter/command_extension.h b/test/common/formatter/command_extension.h
index ad67936078f9..b6d07e4bcfd0 100644
--- a/test/common/formatter/command_extension.h
+++ b/test/common/formatter/command_extension.h
@@ -29,7 +29,9 @@ class TestCommandParser : public CommandParser {
 
 class TestCommandFactory : public CommandParserFactory {
 public:
-  CommandParserPtr createCommandParserFromProto(const Protobuf::Message&) override;
+  CommandParserPtr
+  createCommandParserFromProto(const Protobuf::Message&,
+                               Server::Configuration::CommonFactoryContext&) override;
   std::set configTypes() override;
   ProtobufTypes::MessagePtr createEmptyConfigProto() override;
   std::string name() const override;
@@ -54,7 +56,9 @@ class AdditionalCommandParser : public CommandParser {
 
 class AdditionalCommandFactory : public CommandParserFactory {
 public:
-  CommandParserPtr createCommandParserFromProto(const Protobuf::Message&) override;
+  CommandParserPtr
+  createCommandParserFromProto(const Protobuf::Message&,
+                               Server::Configuration::CommonFactoryContext&) override;
   std::set configTypes() override;
   ProtobufTypes::MessagePtr createEmptyConfigProto() override;
   std::string name() const override;
@@ -62,7 +66,9 @@ class AdditionalCommandFactory : public CommandParserFactory {
 
 class FailCommandFactory : public CommandParserFactory {
 public:
-  CommandParserPtr createCommandParserFromProto(const Protobuf::Message&) override;
+  CommandParserPtr
+  createCommandParserFromProto(const Protobuf::Message&,
+                               Server::Configuration::CommonFactoryContext&) override;
   std::set configTypes() override;
   ProtobufTypes::MessagePtr createEmptyConfigProto() override;
   std::string name() const override;
diff --git a/test/extensions/formatter/cel/cel_test.cc b/test/extensions/formatter/cel/cel_test.cc
index e97a30fe1b3a..e573f21e33b7 100644
--- a/test/extensions/formatter/cel/cel_test.cc
+++ b/test/extensions/formatter/cel/cel_test.cc
@@ -32,7 +32,7 @@ class CELFormatterTest : public ::testing::Test {
 
 #ifdef USE_CEL_PARSER
 TEST_F(CELFormatterTest, TestFormatValue) {
-  auto cel_parser = std::make_unique();
+  auto cel_parser = std::make_unique(context_);
   absl::optional max_length = absl::nullopt;
   auto formatter = cel_parser->parse("CEL", "request.headers[':method']", max_length);
   EXPECT_THAT(formatter->formatValue(request_headers_, response_headers_, response_trailers_,
@@ -41,14 +41,14 @@ TEST_F(CELFormatterTest, TestFormatValue) {
 }
 
 TEST_F(CELFormatterTest, TestParseFail) {
-  auto cel_parser = std::make_unique();
+  auto cel_parser = std::make_unique(context_);
   absl::optional max_length = absl::nullopt;
   EXPECT_EQ(nullptr,
             cel_parser->parse("INVALID_CMD", "requests.headers['missing_headers']", max_length));
 }
 
 TEST_F(CELFormatterTest, TestNullFormatValue) {
-  auto cel_parser = std::make_unique();
+  auto cel_parser = std::make_unique(context_);
   absl::optional max_length = absl::nullopt;
   auto formatter = cel_parser->parse("CEL", "requests.headers['missing_headers']", max_length);
   EXPECT_THAT(formatter->formatValue(request_headers_, response_headers_, response_trailers_,

From 41f8652f6197de2dd2ad990761f2741d7622d77e Mon Sep 17 00:00:00 2001
From: Robert Femmer <114982872+robertfemmer@users.noreply.github.com>
Date: Tue, 20 Jun 2023 20:51:58 +0200
Subject: [PATCH 585/740] fuzz: Add fuzzer targeting HCM redirect (#27453)

Signed-off-by: Robert Femmer 
---
 test/common/http/BUILD                        |  39 ++
 test/common/http/hcm_router_fuzz.proto        |  79 +++
 .../direct_response_flow                      |  28 +
 .../internal_redirect_flow                    |  49 ++
 test/common/http/hcm_router_fuzz_test.cc      | 613 ++++++++++++++++++
 5 files changed, 808 insertions(+)
 create mode 100644 test/common/http/hcm_router_fuzz.proto
 create mode 100644 test/common/http/hcm_router_fuzz_corpus/direct_response_flow
 create mode 100644 test/common/http/hcm_router_fuzz_corpus/internal_redirect_flow
 create mode 100644 test/common/http/hcm_router_fuzz_test.cc

diff --git a/test/common/http/BUILD b/test/common/http/BUILD
index 43f38fb16128..f4f7aaeb001a 100644
--- a/test/common/http/BUILD
+++ b/test/common/http/BUILD
@@ -594,3 +594,42 @@ envoy_cc_test(
         "//test/test_common:utility_lib",
     ],
 )
+
+envoy_proto_library(
+    name = "hcm_router_fuzz_proto",
+    srcs = ["hcm_router_fuzz.proto"],
+    deps = [
+        "//test/fuzz:common_proto",
+        "@envoy_api//envoy/extensions/filters/http/router/v3:pkg",
+    ],
+)
+
+envoy_cc_fuzz_test(
+    name = "hcm_router_fuzz_test",
+    srcs = ["hcm_router_fuzz_test.cc"],
+    corpus = "hcm_router_fuzz_corpus",
+    deps = [
+        ":conn_manager_impl_test_base_lib",
+        ":hcm_router_fuzz_proto_cc_proto",
+        "//source/common/http:header_map_lib",
+        "//source/common/http:headers_lib",
+        "//source/common/router:config_lib",
+        "//source/extensions/upstreams/http/generic:config",
+        "//test/fuzz:utility_lib",
+        "//test/mocks/access_log:access_log_mocks",
+        "//test/mocks/http:http_mocks",
+        "//test/mocks/local_info:local_info_mocks",
+        "//test/mocks/network:network_mocks",
+        "//test/mocks/router:router_mocks",
+        "//test/mocks/runtime:runtime_mocks",
+        "//test/mocks/server:factory_context_mocks",
+        "//test/mocks/server:server_mocks",
+        "//test/mocks/ssl:ssl_mocks",
+        "//test/mocks/tracing:tracing_mocks",
+        "//test/mocks/upstream:cluster_manager_mocks",
+        "//test/test_common:registry_lib",
+        "//test/test_common:simulated_time_system_lib",
+        "//test/test_common:test_runtime_lib",
+        "@envoy_api//envoy/extensions/upstreams/http/generic/v3:pkg_cc_proto",
+    ],
+)
diff --git a/test/common/http/hcm_router_fuzz.proto b/test/common/http/hcm_router_fuzz.proto
new file mode 100644
index 000000000000..b9fdd4e6342b
--- /dev/null
+++ b/test/common/http/hcm_router_fuzz.proto
@@ -0,0 +1,79 @@
+syntax = "proto3";
+
+package test.common.http;
+
+import "test/fuzz/common.proto";
+
+enum Clusters {
+  DEFAULT0 = 0;
+  DEFAULT1 = 1;
+  DEFAULT2 = 2;
+}
+
+message AdvanceTimeAction {
+  uint32 milliseconds = 1;
+}
+
+message ReconfigureClusterAction {
+  Clusters cluster = 1;
+  bool maintenance_mode = 2;
+  bool enable_internal_redirect = 3;
+  bool allow_cross_scheme_redirect = 4;
+  bool allow_early_data = 5;
+}
+
+message RequestHeaderAction {
+  Clusters cluster = 1;
+  test.fuzz.Headers headers = 2;
+  bool end_stream = 3;
+}
+
+message RequestDataAction {
+  Clusters cluster = 1;
+  uint32 stream = 2;
+  bytes data = 3;
+  bool end_stream = 4;
+}
+
+message RequestTrailerAction {
+  Clusters cluster = 1;
+  uint32 stream = 2;
+  test.fuzz.Headers trailers = 3;
+}
+
+message RespondHeaderAction {
+  Clusters cluster = 1;
+  uint32 stream = 2;
+  test.fuzz.Headers headers = 3;
+  bool end_stream = 4;
+}
+
+message RespondDataAction {
+  Clusters cluster = 1;
+  uint32 stream = 2;
+  bytes data = 3;
+  bool end_stream = 4;
+}
+
+message RespondTrailerAction {
+  Clusters cluster = 1;
+  uint32 stream = 2;
+  test.fuzz.Headers trailers = 3;
+}
+
+message FuzzAction {
+  oneof action {
+    AdvanceTimeAction advance_time = 1;
+    ReconfigureClusterAction reconfigure_cluster = 2;
+    RequestHeaderAction request_header = 3;
+    RequestDataAction request_data = 4;
+    RequestTrailerAction request_trailer = 5;
+    RespondHeaderAction respond_header = 6;
+    RespondDataAction respond_data = 7;
+    RespondTrailerAction respond_trailer = 8;
+  }
+}
+
+message RedirectFuzzCase {
+  repeated FuzzAction actions = 1;
+}
diff --git a/test/common/http/hcm_router_fuzz_corpus/direct_response_flow b/test/common/http/hcm_router_fuzz_corpus/direct_response_flow
new file mode 100644
index 000000000000..3577516db93b
--- /dev/null
+++ b/test/common/http/hcm_router_fuzz_corpus/direct_response_flow
@@ -0,0 +1,28 @@
+actions {
+  request_header {
+    cluster: 2
+    headers {
+      headers {
+        key: ":method"
+        value: "get"
+      }
+      headers {
+        key: ":path"
+        value: "/default2"
+      }
+      headers {
+        key: ":scheme"
+        value: "https"
+      }
+    }
+    end_stream: false
+  }
+}
+actions {
+  request_data {
+    cluster: 0
+    stream: 0
+    data: "foo"
+    end_stream: true
+  }
+}
diff --git a/test/common/http/hcm_router_fuzz_corpus/internal_redirect_flow b/test/common/http/hcm_router_fuzz_corpus/internal_redirect_flow
new file mode 100644
index 000000000000..46e782ce5d4a
--- /dev/null
+++ b/test/common/http/hcm_router_fuzz_corpus/internal_redirect_flow
@@ -0,0 +1,49 @@
+actions {
+  request_header {
+    headers {
+      headers {
+        key: ":method"
+        value: "get"
+      }
+      headers {
+        key: ":path"
+        value: "/default0"
+      }
+      headers {
+        key: ":scheme"
+        value: "https"
+      }
+    }
+    end_stream: true
+  }
+}
+actions {
+  respond_header {
+    cluster: 0
+    stream: 0
+    headers {
+      headers {
+        key: ":status"
+        value: "301"
+      }
+      headers {
+        key: "location"
+        value: "http://localhost/default1"
+      }
+    }
+    end_stream: true
+  }
+}
+actions {
+  respond_header {
+    cluster: 1
+    stream: 0
+    headers {
+      headers {
+        key: ":status"
+        value: "200"
+      }
+    }
+    end_stream: true
+  }
+}
diff --git a/test/common/http/hcm_router_fuzz_test.cc b/test/common/http/hcm_router_fuzz_test.cc
new file mode 100644
index 000000000000..b21618137d15
--- /dev/null
+++ b/test/common/http/hcm_router_fuzz_test.cc
@@ -0,0 +1,613 @@
+#include "envoy/extensions/upstreams/http/generic/v3/generic_connection_pool.pb.h"
+#include "envoy/http/header_map.h"
+#include "envoy/network/filter.h"
+#include "envoy/router/router.h"
+
+#include "source/common/http/conn_manager_impl.h"
+#include "source/common/http/date_provider_impl.h"
+#include "source/common/router/router.h"
+#include "source/common/router/upstream_codec_filter.h"
+#include "source/common/router/upstream_request.h"
+#include "source/extensions/upstreams/http/http/upstream_request.h"
+#include "source/extensions/upstreams/http/tcp/upstream_request.h"
+
+#include "test/common/http/conn_manager_impl_test_base.h"
+#include "test/common/http/hcm_router_fuzz.pb.h"
+#include "test/common/stats/stat_test_utility.h"
+#include "test/fuzz/fuzz_runner.h"
+#include "test/fuzz/utility.h"
+#include "test/mocks/network/mocks.h"
+#include "test/mocks/router/mocks.h"
+#include "test/mocks/server/factory_context.h"
+#include "test/test_common/registry.h"
+#include "test/test_common/test_runtime.h"
+
+using testing::InvokeWithoutArgs;
+using testing::Return;
+using testing::ReturnRef;
+
+namespace Envoy {
+namespace Http {
+
+class FuzzConfig;
+
+using FuzzCase = test::common::http::RedirectFuzzCase;
+using FuzzAction = test::common::http::FuzzAction;
+using ActionCase = test::common::http::FuzzAction::ActionCase;
+
+// An instance of this class will be installed in the filter chain
+// for downstream connections by the `HTTP` connection manager.
+class RouterFuzzFilter : public Router::Filter {
+public:
+  using Router::Filter::Filter;
+  static StreamDecoderFilterSharedPtr create(Router::FilterConfig& config) {
+    auto fuzz_filter = new RouterFuzzFilter(config, config.default_stats_);
+    return StreamDecoderFilterSharedPtr{fuzz_filter};
+  }
+  // Filter
+  Router::RetryStatePtr createRetryState(const Router::RetryPolicy&, RequestHeaderMap&,
+                                         const Upstream::ClusterInfo&,
+                                         const Router::VirtualCluster*,
+                                         Router::RouteStatsContextOptRef, Runtime::Loader&,
+                                         Random::RandomGenerator&, Event::Dispatcher&, TimeSource&,
+                                         Upstream::ResourcePriority) override {
+    EXPECT_EQ(nullptr, retry_state_);
+    retry_state_ = new NiceMock();
+
+    if (reject_all_hosts_) {
+      // Set up RetryState to always reject the host
+      ON_CALL(*retry_state_, shouldSelectAnotherHost(_)).WillByDefault(Return(true));
+    }
+    if (retry_425_response_) {
+      ON_CALL(*retry_state_, wouldRetryFromRetriableStatusCode(Code::TooEarly))
+          .WillByDefault(Return(true));
+    }
+    return Router::RetryStatePtr{retry_state_};
+  }
+
+  const Network::Connection* downstreamConnection() const override {
+    return &downstream_connection_;
+  }
+
+  NiceMock downstream_connection_;
+  Router::MockRetryState* retry_state_{};
+  bool reject_all_hosts_ = false;
+  bool retry_425_response_ = false;
+};
+
+// We track stream state here to prevent illegal operations, e.g. applying an
+// encodeData() to the codec after encodeTrailers(). This is necessary to
+// maintain the preconditions for operations on the codec at the API level. Of
+// course, it's the codecs must be robust to wire-level violations. We
+// explore these violations via MutateAction and SwapAction at the connection
+// buffer level.
+enum class StreamState { PendingHeaders, PendingDataOrTrailers, Closed };
+
+// Convert from test `proto` Headers to a variant of `TestHeaderMapImpl`.
+// In contrast to `Fuzz::fromHeaders`, invalid characters are replaced, so
+// that the fuzzer can proceed regardless.
+template 
+inline T fromHeaders(
+    const test::fuzz::Headers& headers,
+    const absl::node_hash_set& ignore_headers = absl::node_hash_set(),
+    absl::node_hash_set include_headers = absl::node_hash_set()) {
+  T header_map;
+  for (const auto& header : headers.headers()) {
+    if (ignore_headers.find(absl::AsciiStrToLower(header.key())) == ignore_headers.end()) {
+      header_map.addCopy(Fuzz::replaceInvalidCharacters(header.key()),
+                         Fuzz::replaceInvalidCharacters(header.value()));
+    }
+    include_headers.erase(absl::AsciiStrToLower(header.key()));
+  }
+  // Add dummy headers for non-present headers that must be included.
+  for (const auto& header : include_headers) {
+    header_map.addCopy(header, "dummy");
+  }
+  return header_map;
+}
+
+// This class mocks the downstream requests and serves as initial entry point
+// for the fuzzer
+class FuzzDownstream {
+public:
+  FuzzDownstream(ConnectionManagerImpl& conn_manager) : conn_manager_(conn_manager) {
+    // If sendLocalReply is called:
+    ON_CALL(encoder_, encodeHeaders(_, true))
+        .WillByDefault(Invoke([this](const ResponseHeaderMap&, bool end_stream) -> void {
+          request_state_ = end_stream ? StreamState::Closed : StreamState::PendingDataOrTrailers;
+        }));
+  }
+
+  void sendHeaders(const test::fuzz::Headers& request_headers, bool end_stream,
+                   absl::string_view path) {
+    if (request_state_ == StreamState::PendingHeaders) {
+      request_state_ = end_stream ? StreamState::Closed : StreamState::PendingDataOrTrailers;
+      auto headers = std::make_unique(
+          fromHeaders(request_headers, {}, {"host"}));
+      if (headers->Method() == nullptr) {
+        headers->setReferenceKey(Headers::get().Method, "GET");
+      }
+      headers->setReferenceKey(Headers::get().Path, path);
+      decoder_ = &conn_manager_.newStream(encoder_);
+      decoder_->decodeHeaders(std::move(headers), end_stream);
+    }
+  }
+
+  void sendData(const std::string& request_data, bool end_stream) {
+    if (request_state_ == StreamState::PendingDataOrTrailers) {
+      request_state_ = end_stream ? StreamState::Closed : StreamState::PendingDataOrTrailers;
+      Buffer::OwnedImpl data(request_data);
+      decoder_->decodeData(data, end_stream);
+    }
+  }
+
+  void sendTrailers(const test::fuzz::Headers& request_trailers) {
+    if (request_state_ == StreamState::PendingDataOrTrailers) {
+      auto trailers = std::make_unique(
+          fromHeaders(request_trailers));
+      decoder_->decodeTrailers(std::move(trailers));
+      request_state_ = StreamState::Closed;
+    }
+  }
+
+  ConnectionManagerImpl& conn_manager_;
+  RequestDecoder* decoder_{};
+  NiceMock encoder_;
+  StreamState request_state_{StreamState::PendingHeaders};
+};
+using FuzzDownstreamPtr = std::unique_ptr;
+
+// This class mocks the upstream and serves as entry point for responses.
+class FuzzUpstream {
+public:
+  FuzzUpstream(Http::ResponseDecoder& decoder) : decoder_(decoder) {
+    ON_CALL(mock_request_encoder_, encodeHeaders(_, _))
+        .WillByDefault(Invoke([](const Http::RequestHeaderMap&, bool) { return okStatus(); }));
+    ON_CALL(mock_request_encoder_.stream_, resetStream(_))
+        .WillByDefault(
+            Invoke([this](StreamResetReason) { response_state_ = StreamState::Closed; }));
+  }
+
+  void sendHeaders(Http::ResponseHeaderMapPtr&& headers, bool end_stream) {
+    if (response_state_ == StreamState::PendingHeaders) {
+      response_state_ = end_stream ? StreamState::Closed : StreamState::PendingDataOrTrailers;
+      decoder_.decodeHeaders(std::move(headers), end_stream);
+    }
+  }
+
+  void sendData(Buffer::Instance& data, bool end_stream) {
+    if (response_state_ == StreamState::PendingDataOrTrailers) {
+      response_state_ = end_stream ? StreamState::Closed : StreamState::PendingDataOrTrailers;
+      decoder_.decodeData(data, end_stream);
+    }
+  }
+
+  void sendTrailers(Http::ResponseTrailerMapPtr&& trailers) {
+    if (response_state_ == StreamState::PendingDataOrTrailers) {
+      response_state_ = StreamState::Closed;
+      decoder_.decodeTrailers(std::move(trailers));
+    }
+  }
+
+  Http::ResponseDecoder& decoder_;
+  StreamState response_state_{StreamState::PendingHeaders};
+  NiceMock mock_request_encoder_;
+};
+
+// This class mocks an upstream cluster. It holds the set of
+// `FuzzUpstream` instances, which result from `FuzzDownstream`
+// requests
+class FuzzCluster {
+public:
+  FuzzCluster(FuzzConfig& cfg, absl::string_view path) : cfg_(cfg), name_(path), path_(path) {
+
+    // `mock_route_` is still necessary to keep around when a direct response
+    // entry is added to this cluster
+    mock_route_ = new Router::MockRoute();
+    ON_CALL(mock_route_->route_entry_, clusterName()).WillByDefault(ReturnRef(name_));
+    ON_CALL(mock_route_->route_entry_.internal_redirect_policy_, enabled())
+        .WillByDefault(Return(internal_redirect_policy_enabled_));
+    ON_CALL(mock_route_->route_entry_.internal_redirect_policy_, shouldRedirectForResponseCode(_))
+        .WillByDefault(
+            Invoke([](const Http::Code& code) { return code == Http::Code::MovedPermanently; }));
+    ON_CALL(mock_route_->route_entry_.internal_redirect_policy_, maxInternalRedirects())
+        .WillByDefault(Return(10));
+    ON_CALL(mock_route_->route_entry_.internal_redirect_policy_, isCrossSchemeRedirectAllowed())
+        .WillByDefault(Return(cross_scheme_redirect_allowed_));
+    ON_CALL(mock_route_->route_entry_.early_data_policy_, allowsEarlyDataForRequest(_))
+        .WillByDefault(Return(allows_early_data_for_request_));
+    route_ = Router::RouteConstSharedPtr(mock_route_);
+
+    ON_CALL(*tlc_.cluster_.info_, maintenanceMode()).WillByDefault(Return(maintenance_));
+  }
+
+  void newUpstream(Router::GenericConnectionPoolCallbacks* request,
+                   absl::optional protocol) {
+    auto upstream = std::make_unique(request->upstreamToDownstream());
+    auto stream = std::make_unique(
+        request->upstreamToDownstream(), &upstream->mock_request_encoder_);
+    Upstream::HostDescriptionConstSharedPtr host =
+        std::make_shared();
+    request->onPoolReady(std::move(stream), host, mock_stream_info_.downstreamAddressProvider(),
+                         mock_stream_info_, protocol);
+    upstreams_.push_back(std::move(upstream));
+  }
+
+  void sendHeaders(uint32_t stream, const test::fuzz::Headers& response_headers,
+                   bool end_stream) const {
+    FuzzUpstream* s = select(stream);
+    if (s) {
+      auto headers = std::make_unique(
+          fromHeaders(response_headers, {}, {"status"}));
+      uint64_t rc;
+      if (!absl::SimpleAtoi(headers->getStatusValue(), &rc)) {
+        headers->setStatus(302);
+      }
+      s->sendHeaders(std::move(headers), end_stream);
+    }
+  }
+
+  void sendData(uint32_t stream, const std::string& data, bool end_stream) const {
+    FuzzUpstream* s = select(stream);
+    if (s) {
+      Buffer::OwnedImpl buf(data);
+      s->sendData(buf, end_stream);
+    }
+  }
+
+  void sendTrailers(uint32_t stream, const test::fuzz::Headers& response_trailers) const {
+    FuzzUpstream* s = select(stream);
+    if (s) {
+      auto trailers = std::make_unique(
+          Fuzz::fromHeaders(response_trailers));
+      s->sendTrailers(std::move(trailers));
+    }
+  }
+
+  FuzzUpstream* select(uint32_t stream) const {
+    if (upstreams_.empty()) {
+      return nullptr;
+    }
+
+    size_t idx = stream % upstreams_.size();
+    return upstreams_[idx].get();
+  }
+
+  void addDirectResponse(Http::Code code, const std::string& body, const std::string& new_uri,
+                         const std::string& route_name) {
+    direct_response_entry_ = std::make_unique();
+    route_name_ = route_name;
+    direct_response_body_ = body;
+    ON_CALL(*direct_response_entry_, responseCode()).WillByDefault(Return(code));
+    ON_CALL(*direct_response_entry_, responseBody())
+        .WillByDefault(ReturnRef(direct_response_body_));
+    ON_CALL(*direct_response_entry_, newUri(_)).WillByDefault(Return(new_uri));
+    ON_CALL(*direct_response_entry_, routeName()).WillByDefault(ReturnRef(route_name_));
+    ON_CALL(*mock_route_, directResponseEntry())
+        .WillByDefault(Return(direct_response_entry_.get()));
+  }
+
+  const absl::string_view getPath() const { return absl::string_view(path_); }
+
+  void reset() {
+    internal_redirect_policy_enabled_ = true;
+    cross_scheme_redirect_allowed_ = true;
+    allows_early_data_for_request_ = true;
+    maintenance_ = false;
+    upstreams_.clear();
+  }
+
+  void reconfigure(bool internal_redirect, bool allow_cross_scheme, bool allow_early_data,
+                   bool maintenance) {
+    if (upstreams_.empty()) {
+      internal_redirect_policy_enabled_ = internal_redirect;
+      cross_scheme_redirect_allowed_ = allow_cross_scheme;
+      allows_early_data_for_request_ = allow_early_data;
+    }
+    maintenance_ = maintenance;
+  }
+
+  FuzzConfig& cfg_;
+  std::string name_;
+  absl::string_view path_;
+  Upstream::MockThreadLocalCluster tlc_;
+  bool internal_redirect_policy_enabled_{true};
+  bool cross_scheme_redirect_allowed_{true};
+  bool allows_early_data_for_request_{true};
+
+  Router::MockRoute* mock_route_;
+  Router::RouteConstSharedPtr route_;
+
+  bool maintenance_{false};
+  StreamInfo::MockStreamInfo mock_stream_info_;
+
+  std::vector> upstreams_;
+  std::unique_ptr direct_response_entry_{};
+
+  std::string route_name_{};
+  std::string direct_response_body_{};
+};
+
+// This class holds the upstream `FuzzCluster` instances. This has nothing
+// to do with the cluster manager in envoy.
+class FuzzClusterManager {
+public:
+  using FuzzClusterPtr = std::unique_ptr;
+
+  void createDefaultClusters(FuzzConfig& cfg) {
+    // Create a set of clusters which allows to model most possible scenarios.
+    // Adding a cluster here needs to be reflected in the `hcm_router_fuzz_test.proto`.
+    FuzzClusterPtr default0 = std::make_unique(cfg, "/default0");
+    clusters_.push_back(std::move(default0));
+
+    FuzzClusterPtr default1 = std::make_unique(cfg, "/default1");
+    clusters_.push_back(std::move(default1));
+
+    FuzzClusterPtr default2 = std::make_unique(cfg, "/default2");
+    default2->addDirectResponse(Code::Found, "", "/default0", "/default0");
+    clusters_.push_back(std::move(default2));
+  }
+
+  FuzzCluster& selectOneCluster(uint32_t selection) {
+    return *clusters_[selection % clusters_.size()];
+  }
+
+  FuzzCluster* selectClusterByName(absl::string_view name) {
+    for (auto& cluster : clusters_) {
+      if (cluster->path_ == name) {
+        return cluster.get();
+      }
+    }
+    return nullptr;
+  }
+
+  void reset() {
+    for (auto& cluster : clusters_) {
+      cluster->reset();
+    }
+  }
+
+  Router::RouteConstSharedPtr route(const Http::RequestHeaderMap& request_map) {
+    absl::string_view path = request_map.Path()->value().getStringView();
+    FuzzCluster* cluster = selectClusterByName(path);
+    if (!cluster) {
+      return nullptr;
+    }
+    return cluster->route_;
+  }
+
+  Upstream::ThreadLocalCluster* getThreadLocalCluster(absl::string_view name) {
+    FuzzCluster* cluster = selectClusterByName(name);
+    if (!cluster) {
+      return nullptr;
+    }
+    return &cluster->tlc_;
+  }
+
+private:
+  std::vector> clusters_;
+};
+
+// Register a custom ConnPoolFactory, which will mock the upstream
+// connections to the mock cluster management.
+class FuzzGenericConnPoolFactory : public Router::GenericConnPoolFactory {
+public:
+  FuzzGenericConnPoolFactory(FuzzClusterManager& cluster_manager)
+      : cluster_manager_(cluster_manager) {}
+  std::string name() const override { return "envoy.filters.connection_pools.http.generic"; }
+  std::string category() const override { return "envoy.upstreams"; }
+  Router::GenericConnPoolPtr createGenericConnPool(Upstream::ThreadLocalCluster&, bool is_connect,
+                                                   const Router::RouteEntry& route_entry,
+                                                   absl::optional protocol,
+                                                   Upstream::LoadBalancerContext*) const override {
+    if (is_connect) {
+      return nullptr;
+    }
+    FuzzCluster* cluster = cluster_manager_.selectClusterByName(route_entry.clusterName());
+    if (cluster == nullptr) {
+      return nullptr;
+    }
+    auto conn_pool = std::make_unique();
+    ON_CALL(*conn_pool, newStream(_))
+        .WillByDefault(Invoke([cluster, protocol](Router::GenericConnectionPoolCallbacks* request) {
+          cluster->newUpstream(request, protocol);
+        }));
+    return conn_pool;
+  }
+
+  ProtobufTypes::MessagePtr createEmptyConfigProto() override {
+    return std::make_unique<
+        envoy::extensions::upstreams::http::generic::v3::GenericConnectionPoolProto>();
+  }
+
+private:
+  FuzzClusterManager& cluster_manager_;
+};
+
+class FuzzConfig : public HttpConnectionManagerImplMixin {
+public:
+  FuzzConfig(Protobuf::RepeatedPtrField strict_headers_to_check)
+      : pool_(fake_stats_.symbolTable()), fuzz_conn_pool_factory_(cluster_manager_),
+        reg_(fuzz_conn_pool_factory_), router_context_(fake_stats_.symbolTable()),
+        shadow_writer_(new NiceMock()),
+        filter_config_(pool_.add("fuzz_filter"), local_info_, *fake_stats_.rootScope(), cm_,
+                       runtime_, random_, Router::ShadowWriterPtr{shadow_writer_},
+                       true /*emit_dynamic_stats*/, false /*start_child_span*/,
+                       true /*suppress_envoy_headers*/, false /*respect_expected_rq_timeout*/,
+                       true /*suppress_grpc_request_failure_code_stats*/,
+                       false /*flush_upstream_log_on_upstream_stream*/,
+                       std::move(strict_headers_to_check), time_system_.timeSystem(), http_context_,
+                       router_context_) {
+    cluster_manager_.createDefaultClusters(*this);
+    // Install the `RouterFuzzFilter` here
+    ON_CALL(filter_factory_, createFilterChain(_))
+        .WillByDefault(Invoke([this](FilterChainManager& manager) -> bool {
+          FilterFactoryCb decoder_filter_factory = [this](FilterChainFactoryCallbacks& callbacks) {
+            callbacks.addStreamDecoderFilter(RouterFuzzFilter::create(filter_config_));
+          };
+          manager.applyFilterFactoryCb({}, decoder_filter_factory);
+          return true;
+        }));
+    ON_CALL(*route_config_provider_.route_config_, route(_, _, _, _))
+        .WillByDefault(
+            Invoke([this](const Router::RouteCallback&, const Http::RequestHeaderMap& request_map,
+                          const Envoy::StreamInfo::StreamInfo&,
+                          uint64_t) { return cluster_manager_.route(request_map); }));
+    ON_CALL(cm_, getThreadLocalCluster(_)).WillByDefault(Invoke([this](absl::string_view cluster) {
+      return cluster_manager_.getThreadLocalCluster(cluster);
+    }));
+  }
+
+  ServerConnectionPtr createCodec(Network::Connection&, const Buffer::Instance&,
+                                  ServerConnectionCallbacks&, Server::OverloadManager&) override {
+    if (codec_ == nullptr) {
+      codec_ = new NiceMock();
+    }
+    ON_CALL(*codec_, dispatch(_)).WillByDefault(Return(Http::okStatus()));
+    auto codec = ServerConnectionPtr{codec_};
+    codec_ = nullptr;
+    return codec;
+  }
+
+  FuzzClusterManager& getFuzzClusterManager() { return cluster_manager_; }
+
+  NiceMock cm_;
+  Event::SimulatedTimeSystem time_system_;
+
+private:
+  Stats::StatNamePool pool_;
+  FuzzClusterManager cluster_manager_;
+  FuzzGenericConnPoolFactory fuzz_conn_pool_factory_;
+  Registry::InjectFactory reg_;
+  Router::ContextImpl router_context_;
+  NiceMock local_info_;
+  NiceMock runtime_;
+  Router::MockShadowWriter* shadow_writer_;
+  Router::FilterConfig filter_config_;
+};
+
+class Harness {
+public:
+  Harness() : hcm_config_(Protobuf::RepeatedPtrField{}) {
+    ON_CALL(filter_callbacks_.connection_, close(_, _)).WillByDefault(InvokeWithoutArgs([this]() {
+      closed_ = true;
+    }));
+  }
+
+  void fuzz(const FuzzCase& input) {
+    hcm_ = std::make_unique(
+        hcm_config_, drain_close_, random_, hcm_config_.http_context_, runtime_, local_info_,
+        hcm_config_.cm_, overload_manager_, hcm_config_.time_system_);
+    hcm_->initializeReadFilterCallbacks(filter_callbacks_);
+    Buffer::OwnedImpl data;
+    hcm_->onData(data, false);
+    FuzzClusterManager& cluster_manager = hcm_config_.getFuzzClusterManager();
+
+    for (const auto& action : input.actions()) {
+      if (closed_) {
+        break;
+      }
+      switch (action.action_case()) {
+      case ActionCase::kAdvanceTime: {
+        const auto& a = action.advance_time();
+        hcm_config_.time_system_.timeSystem().advanceTimeWait(
+            std::chrono::milliseconds(a.milliseconds()));
+        break;
+      }
+      case ActionCase::kReconfigureCluster: {
+        const auto& a = action.reconfigure_cluster();
+        FuzzCluster& cluster = cluster_manager.selectOneCluster(a.cluster());
+        cluster.reconfigure(a.enable_internal_redirect(), a.allow_cross_scheme_redirect(),
+                            a.allow_early_data(), a.maintenance_mode());
+
+        break;
+      }
+      case ActionCase::kRequestHeader: {
+        auto& a = action.request_header();
+        FuzzCluster& cluster = cluster_manager.selectOneCluster(a.cluster());
+        FuzzDownstreamPtr stream = std::make_unique(*hcm_);
+        streams_.push_back(std::move(stream));
+        streams_.back()->sendHeaders(a.headers(), a.end_stream(), cluster.getPath());
+        break;
+      }
+      case ActionCase::kRequestData:
+        if (!streams_.empty()) {
+          const auto& a = action.request_data();
+          FuzzDownstream& s = *streams_[a.stream() % streams_.size()];
+          s.sendData(a.data(), a.end_stream());
+        }
+        break;
+      case ActionCase::kRequestTrailer:
+        if (!streams_.empty()) {
+          auto& a = action.request_trailer();
+          FuzzDownstream& s = *streams_[a.stream() % streams_.size()];
+          s.sendTrailers(a.trailers());
+        }
+        break;
+      case ActionCase::kRespondHeader: {
+        const auto& a = action.respond_header();
+        FuzzCluster& cluster = cluster_manager.selectOneCluster(a.cluster());
+        cluster.sendHeaders(a.stream(), a.headers(), a.end_stream());
+        break;
+      }
+      case ActionCase::kRespondData: {
+        const auto& a = action.respond_data();
+        FuzzCluster& cluster = cluster_manager.selectOneCluster(a.cluster());
+        cluster.sendData(a.stream(), a.data(), a.end_stream());
+        break;
+      }
+      case ActionCase::kRespondTrailer: {
+        const auto& a = action.respond_trailer();
+        FuzzCluster& cluster = cluster_manager.selectOneCluster(a.cluster());
+        cluster.sendTrailers(a.stream(), a.trailers());
+        break;
+      }
+      default:
+        break;
+      }
+    }
+    hcm_->onEvent(Network::ConnectionEvent::RemoteClose);
+    cluster_manager.reset();
+    streams_.clear();
+  }
+
+private:
+  FuzzConfig hcm_config_;
+  NiceMock drain_close_;
+  NiceMock random_;
+  NiceMock runtime_;
+  NiceMock local_info_;
+  NiceMock overload_manager_;
+  NiceMock filter_callbacks_;
+  std::unique_ptr hcm_;
+
+  bool closed_{false};
+  std::vector streams_;
+};
+
+// Macro to disable use of allocations that persist across fuzz runs.
+// Enabling it leads to significant slowdown, however, disabling it
+// may incur false positives.
+#ifdef _DISABLE_STATIC_HARNESS
+
+DEFINE_PROTO_FUZZER(FuzzCase& input) {
+  auto harness = std::make_unique();
+  harness->fuzz(input);
+}
+
+#else
+
+static std::unique_ptr harness = nullptr;
+static void cleanup() { harness = nullptr; }
+DEFINE_PROTO_FUZZER(FuzzCase& input) {
+  if (harness == nullptr) {
+    harness = std::make_unique();
+    atexit(cleanup);
+  }
+  harness->fuzz(input);
+}
+
+#endif
+
+} // namespace Http
+} // namespace Envoy

From 7801df6af9000ae31bdd89b080e0d797501cbd18 Mon Sep 17 00:00:00 2001
From: Xin 
Date: Tue, 20 Jun 2023 16:11:08 -0400
Subject: [PATCH 586/740] add deferred_creation util into stats (#27899)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Commit Message:
With lots of clusters and route-tables in a cloud proxy, we are seeing tons of RAM been spent on stats while most of the stats are never inc-ed due to traffic pattern(or long tail). We are thinking that we can lazy init cluster stats() so that the RAM is only allocated when it's required.

To achieve that we need to have finer grained stats group, e.g. configUpdateStats() are frequently updated by config management server, while upstream_xxx are only required when there is traffic for the cluster, for this sub-group we can save RAM by lazy init it.

Introduce a new stats utility in this PR such that the nested StatsStruct is only instantiated when any of "->" or "*xx." operator is used.

Cribbed from PR #23921
Please see that PR for how it is used.

Additional Description:
Risk Level: LOW，utility lib not used yet.
Testing: unit test and speed test.
Docs Changes:
Release Notes:
Platform Specific Features:

Signed-off-by: Xin Zhuang 
---
 api/envoy/config/bootstrap/v3/bootstrap.proto |  18 +-
 changelogs/current.yaml                       |   5 +
 envoy/server/configuration.h                  |   5 +
 envoy/stats/stats.h                           |  35 +++
 envoy/stats/stats_macros.h                    |   6 +-
 source/common/stats/BUILD                     |  12 +
 source/common/stats/deferred_creation.h       | 119 +++++++++
 source/docs/stats.md                          |   8 +
 source/server/configuration_impl.cc           |  15 +-
 source/server/configuration_impl.h            |   4 +
 test/common/stats/BUILD                       |  36 +++
 .../deferred_creation_stats_speed_test.cc     | 240 ++++++++++++++++++
 .../stats/deferred_creation_stats_test.cc     | 200 +++++++++++++++
 test/mocks/server/server_factory_context.h    |   1 +
 tools/spelling/spelling_dictionary.txt        |   1 +
 15 files changed, 696 insertions(+), 9 deletions(-)
 create mode 100644 source/common/stats/deferred_creation.h
 create mode 100644 test/common/stats/deferred_creation_stats_speed_test.cc
 create mode 100644 test/common/stats/deferred_creation_stats_test.cc

diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto
index 43e3a33a3f15..30d05dac50c8 100644
--- a/api/envoy/config/bootstrap/v3/bootstrap.proto
+++ b/api/envoy/config/bootstrap/v3/bootstrap.proto
@@ -41,7 +41,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // ` for more detail.
 
 // Bootstrap :ref:`configuration overview `.
-// [#next-free-field: 39]
+// [#next-free-field: 40]
 message Bootstrap {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.bootstrap.v2.Bootstrap";
@@ -124,6 +124,18 @@ message Bootstrap {
     LogFormat log_format = 1;
   }
 
+  message DeferredStatOptions {
+    // When the flag is enabled, Envoy will lazily initialize a subset of the stats (see below).
+    // This will save memory and CPU cycles when creating the objects that own these stats, if those
+    // stats are never referenced throughout the lifetime of the process. However, it will incur additional
+    // memory overhead for these objects, and a small increase of CPU usage when a at least one of the stats
+    // is updated for the first time.
+    // Groups of stats that will be lazily initialized:
+    // - Cluster traffic stats: a subgroup of the :ref:`cluster statistics `
+    // that are used when requests are routed to the cluster.
+    bool enable_deferred_creation_stats = 1;
+  }
+
   reserved 10, 11;
 
   reserved "runtime";
@@ -186,6 +198,10 @@ message Bootstrap {
   // Optional set of stats sinks.
   repeated metrics.v3.StatsSink stats_sinks = 6;
 
+  // Options to control behaviors of deferred creation compatible stats.
+  // [#not-implemented-hide:]
+  DeferredStatOptions deferred_stat_options = 39;
+
   // Configuration for internal processing of stats.
   metrics.v3.StatsConfig stats_config = 13;
 
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 47f4e5dbfb2d..b8b1cb8b9eae 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -264,6 +264,11 @@ new_features:
     added new configuration field :ref:`key_formatter
     ` to format redis key.
     The field supports using %KEY% as a formatter command for substituting the redis key as part of the substitution formatter expression.
+- area: stats
+  change: |
+    added config :ref:`enable_deferred_creation_stats
+    `.
+    When set to true, enables deferred instantiation on supported stats structures.
 - area: ratelimit
   change: |
     added new configuration field :ref:`domain
diff --git a/envoy/server/configuration.h b/envoy/server/configuration.h
index c716ca7393a0..945a175f6650 100644
--- a/envoy/server/configuration.h
+++ b/envoy/server/configuration.h
@@ -84,6 +84,11 @@ class StatsConfig {
    * @return bool indicator to flush stats on-demand via the admin interface instead of on a timer.
    */
   virtual bool flushOnAdmin() const PURE;
+
+  /**
+   * @return true if deferred creation of stats is enabled.
+   */
+  virtual bool enableDeferredCreationStats() const PURE;
 };
 
 /**
diff --git a/envoy/stats/stats.h b/envoy/stats/stats.h
index b2768144c60f..040dc392d1fb 100644
--- a/envoy/stats/stats.h
+++ b/envoy/stats/stats.h
@@ -217,5 +217,40 @@ using SizeFn = std::function;
  */
 template  using StatFn = std::function;
 
+/**
+ * Interface for stats lazy initialization.
+ * To save memory and CPU consumption on blocks of stats that are never referenced throughout the
+ * process lifetime, they can be encapsulated in a DeferredCreationCompatibleInterface. Then the
+Envoy
+ * bootstrap configuration can be set to defer the instantiation of those block. Note that when the
+ * blocks of stats are created, they carry an extra 60~100 byte overhead (depending on worker thread
+ * count) due to internal bookkeeping data structures. The overhead when deferred stats are disabled
+ * is just 8 bytes.
+* See more context: https://github.com/envoyproxy/envoy/issues/23575
+ */
+template  class DeferredCreationCompatibleInterface {
+public:
+  // Helper function to get-or-create and return the StatsStructType object.
+  virtual StatsStructType& getOrCreate() PURE;
+
+  virtual ~DeferredCreationCompatibleInterface() = default;
+};
+
+// A helper class for a lazy compatible stats struct type.
+template  class DeferredCreationCompatibleStats {
+public:
+  explicit DeferredCreationCompatibleStats(
+      std::unique_ptr> d)
+      : data_(std::move(d)) {}
+  // Allows move construct and assign.
+  DeferredCreationCompatibleStats& operator=(DeferredCreationCompatibleStats&&) noexcept = default;
+  DeferredCreationCompatibleStats(DeferredCreationCompatibleStats&&) noexcept = default;
+
+  inline StatsStructType* operator->() { return &data_->getOrCreate(); };
+  inline StatsStructType& operator*() { return data_->getOrCreate(); };
+
+private:
+  std::unique_ptr> data_;
+};
 } // namespace Stats
 } // namespace Envoy
diff --git a/envoy/stats/stats_macros.h b/envoy/stats/stats_macros.h
index bec6b6bf722d..47caffb36d95 100644
--- a/envoy/stats/stats_macros.h
+++ b/envoy/stats/stats_macros.h
@@ -158,6 +158,9 @@ static inline std::string statPrefixJoin(absl::string_view prefix, absl::string_
  */
 #define MAKE_STATS_STRUCT(StatsStruct, StatNamesStruct, ALL_STATS)                                 \
   struct StatsStruct {                                                                             \
+    /* Also referenced in Stats::createDeferredCompatibleStats. */                                 \
+    using StatNameType = StatNamesStruct;                                                          \
+    static const absl::string_view typeName() { return #StatsStruct; }                             \
     StatsStruct(const StatNamesStruct& stat_names, Envoy::Stats::Scope& scope,                     \
                 Envoy::Stats::StatName prefix = Envoy::Stats::StatName())                          \
         : stat_names_(stat_names)                                                                  \
@@ -165,9 +168,8 @@ static inline std::string statPrefixJoin(absl::string_view prefix, absl::string_
                         MAKE_STATS_STRUCT_HISTOGRAM_HELPER_,                                       \
                         MAKE_STATS_STRUCT_TEXT_READOUT_HELPER_,                                    \
                         MAKE_STATS_STRUCT_STATNAME_HELPER_) {}                                     \
-    const StatNamesStruct& stat_names_;                                                            \
+    const StatNameType& stat_names_;                                                               \
     ALL_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, GENERATE_HISTOGRAM_STRUCT,           \
               GENERATE_TEXT_READOUT_STRUCT, GENERATE_STATNAME_STRUCT)                              \
   }
-
 } // namespace Envoy
diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD
index d49ad82d7fc1..b795283f7718 100644
--- a/source/common/stats/BUILD
+++ b/source/common/stats/BUILD
@@ -36,6 +36,18 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "deferred_creation",
+    hdrs = ["deferred_creation.h"],
+    deps = [
+        "//envoy/stats:stats_interface",
+        "//source/common/common:cleanup_lib",
+        "//source/common/common:thread_lib",
+        "//source/common/stats:symbol_table_lib",
+        "//source/common/stats:utility_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "histogram_lib",
     srcs = ["histogram_impl.cc"],
diff --git a/source/common/stats/deferred_creation.h b/source/common/stats/deferred_creation.h
new file mode 100644
index 000000000000..4e2732728355
--- /dev/null
+++ b/source/common/stats/deferred_creation.h
@@ -0,0 +1,119 @@
+#pragma once
+
+#include "envoy/common/pure.h"
+#include "envoy/stats/scope.h"
+#include "envoy/stats/stats.h"
+
+#include "source/common/common/cleanup.h"
+#include "source/common/common/thread.h"
+#include "source/common/stats/symbol_table.h"
+#include "source/common/stats/utility.h"
+
+namespace Envoy {
+namespace Stats {
+
+/**
+ * Lazy-initialization wrapper for StatsStructType, intended for deferred instantiation of a block
+ * of stats that might not be needed in a given Envoy process.
+ *
+ * This class is thread-safe -- instantiations can occur on multiple concurrent threads.
+ * This is used when
+ * :ref:`enable_deferred_creation_stats
+ * ` is enabled.
+ */
+template 
+class DeferredStats : public DeferredCreationCompatibleInterface {
+public:
+  // Capture the stat names object and the scope with a ctor, that can be used to instantiate a
+  // StatsStructType object later.
+  // Caller should make sure scope and stat_names outlive this object.
+  DeferredStats(const typename StatsStructType::StatNameType& stat_names,
+                Stats::ScopeSharedPtr scope)
+      : initialized_(
+            // A lambda is used as we need to register the name into the symbol table.
+            // Note: there is no issue to capture a reference of the scope here as this lambda is
+            // only used to initialize the 'initialized_' Gauge.
+            [&scope]() -> Gauge& {
+              Stats::StatNamePool pool(scope->symbolTable());
+              return Stats::Utility::gaugeFromElements(
+                  *scope, {pool.add(StatsStructType::typeName()), pool.add("initialized")},
+                  Stats::Gauge::ImportMode::HiddenAccumulate);
+            }()),
+        ctor_([this, &stat_names, stats_scope = std::move(scope)]() -> StatsStructType* {
+          initialized_.inc();
+          // Reset ctor_ to save some RAM.
+          Cleanup reset_ctor([this] { ctor_ = nullptr; });
+          return new StatsStructType(stat_names, *stats_scope);
+        }) {
+    if (initialized_.value() > 0) {
+      getOrCreateHelper();
+    }
+  }
+  ~DeferredStats() {
+    if (ctor_ == nullptr) {
+      initialized_.dec();
+    }
+  }
+  inline StatsStructType& getOrCreate() override { return getOrCreateHelper(); }
+
+private:
+  // We can't call getOrCreate directly from constructor, otherwise the compiler complains about
+  // bypassing virtual dispatch even though it's fine.
+  inline StatsStructType& getOrCreateHelper() { return *internal_stats_.get(ctor_); }
+
+  // In order to preserve stat value continuity across a config reload, we need to automatically
+  // re-instantiate lazy stats when they are constructed, if there is already a live instantiation
+  // to the same stats. Consider the following alternate scenarios:
+
+  // Scenario 1: a cluster is instantiated but receives no requests, so its traffic-related stats
+  // are never instantiated. When this cluster gets reloaded on a config update, a new lazy-init
+  // block is created, but the stats should again not be instantiated.
+
+  // Scenario 2: a cluster is instantiated and receives traffic, so its traffic-related stats are
+  // instantiated. We must ensure that a new instance for the same cluster gets its lazy-stats
+  // instantiated before the previous cluster of the same name is destructed.
+
+  // To do that we keep an "initialized" gauge in the cluster's scope, which will be associated by
+  // name to the previous generation's cluster's lazy-init block. We use the value in this shared
+  // gauge to determine whether to instantiate the lazy block on construction.
+  Gauge& initialized_;
+  // TODO(#26957): Clean up this ctor_ by moving its ownership to AtomicPtr, and drop
+  // the setter lambda when the nested object is created.
+  std::function ctor_;
+  Thread::AtomicPtr internal_stats_;
+};
+
+// Non-deferred wrapper over StatsStructType. This is used when
+// :ref:`enable_deferred_creation_stats
+// ` is not enabled.
+template 
+class DirectStats : public DeferredCreationCompatibleInterface {
+public:
+  DirectStats(const typename StatsStructType::StatNameType& stat_names, Stats::Scope& scope)
+      : stats_(stat_names, scope) {}
+  inline StatsStructType& getOrCreate() override { return stats_; }
+
+private:
+  StatsStructType stats_;
+};
+
+// Template that lazily initializes a StatsStruct.
+// The bootstrap config :ref:`enable_deferred_creation_stats
+// ` decides if
+// stats lazy initialization is enabled or not.
+template 
+DeferredCreationCompatibleStats
+createDeferredCompatibleStats(Stats::ScopeSharedPtr scope,
+                              const typename StatsStructType::StatNameType& stat_names,
+                              bool deferred_creation) {
+  if (deferred_creation) {
+    return DeferredCreationCompatibleStats(
+        std::make_unique>(stat_names, scope));
+  } else {
+    return DeferredCreationCompatibleStats(
+        std::make_unique>(stat_names, *scope));
+  }
+}
+
+} // namespace Stats
+} // namespace Envoy
diff --git a/source/docs/stats.md b/source/docs/stats.md
index 97e42e5bcd4c..1616558b9afc 100644
--- a/source/docs/stats.md
+++ b/source/docs/stats.md
@@ -295,3 +295,11 @@ from the same symbol table. To facilitate this, a test-only global singleton can
 be instantiated, via either `Stats::TestUtil::TestSymbolTable` or
 `Stats::TestUtil::TestStore`. All such structures use a singleton symbol-table
 whose lifetime is a single test method. This should resolve the assertion.
+
+
+Deferred Initialization of Stats
+================================
+
+When :ref:`enable_deferred_creation_stats `
+is enabled in Bootstrap, for stats that are deferred creation compatible, the actual stats struct creation
+is deferred to first access of any member of that stats, i.e. instantiation only happens when an invocation on operator "*" or "->" happens.
diff --git a/source/server/configuration_impl.cc b/source/server/configuration_impl.cc
index 86ce9a2bb461..254df3e03768 100644
--- a/source/server/configuration_impl.cc
+++ b/source/server/configuration_impl.cc
@@ -60,7 +60,8 @@ void FilterChainUtility::buildUdpFilterChain(
   }
 }
 
-StatsConfigImpl::StatsConfigImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+StatsConfigImpl::StatsConfigImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap)
+    : deferred_stat_options_(bootstrap.deferred_stat_options()) {
   if (bootstrap.has_stats_flush_interval() &&
       bootstrap.stats_flush_case() !=
           envoy::config::bootstrap::v3::Bootstrap::STATS_FLUSH_NOT_SET) {
@@ -87,6 +88,11 @@ void MainImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstr
   // tracing configuration is missing from the bootstrap config.
   initializeTracers(bootstrap.tracing(), server);
 
+  // stats_config_ should be set before creating the ClusterManagers so that it is available
+  // from the ServerFactoryContext when creating the static clusters and stats sinks, where
+  // stats deferred instantiation setting is read.
+  stats_config_ = std::make_unique(bootstrap);
+
   const auto& secrets = bootstrap.static_resources().secrets();
   ENVOY_LOG(info, "loading {} static secret(s)", secrets.size());
   for (ssize_t i = 0; i < secrets.size(); i++) {
@@ -107,8 +113,9 @@ void MainImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstr
       throw EnvoyException(std::string(update_or_error.status().message()));
     }
   }
-
   initializeWatchdogs(bootstrap, server);
+  // This has to happen after ClusterManager initialization, as it depends on config from
+  // ClusterManager.
   initializeStatsConfig(bootstrap, server);
 }
 
@@ -116,10 +123,6 @@ void MainImpl::initializeStatsConfig(const envoy::config::bootstrap::v3::Bootstr
                                      Instance& server) {
   ENVOY_LOG(info, "loading stats configuration");
 
-  // stats_config_ should be set before populating the sinks so that it is available
-  // from the ServerFactoryContext when creating the stats sinks.
-  stats_config_ = std::make_unique(bootstrap);
-
   for (const envoy::config::metrics::v3::StatsSink& sink_object : bootstrap.stats_sinks()) {
     // Generate factory and translate stats sink custom config.
     auto& factory = Config::Utility::getAndCheckFactory(sink_object);
diff --git a/source/server/configuration_impl.h b/source/server/configuration_impl.h
index 503c396ffce3..5b5b81cdbf12 100644
--- a/source/server/configuration_impl.h
+++ b/source/server/configuration_impl.h
@@ -56,11 +56,15 @@ class StatsConfigImpl : public StatsConfig {
   bool flushOnAdmin() const override { return flush_on_admin_; }
 
   void addSink(Stats::SinkPtr sink) { sinks_.emplace_back(std::move(sink)); }
+  bool enableDeferredCreationStats() const override {
+    return deferred_stat_options_.enable_deferred_creation_stats();
+  }
 
 private:
   std::list sinks_;
   std::chrono::milliseconds flush_interval_;
   bool flush_on_admin_{false};
+  const envoy::config::bootstrap::v3::Bootstrap::DeferredStatOptions deferred_stat_options_;
 };
 
 /**
diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD
index 6b8dd9689e62..92883c81c898 100644
--- a/test/common/stats/BUILD
+++ b/test/common/stats/BUILD
@@ -226,6 +226,42 @@ envoy_cc_benchmark_binary(
     ],
 )
 
+envoy_cc_test(
+    name = "deferred_creation_stats_test",
+    srcs = ["deferred_creation_stats_test.cc"],
+    deps = [
+        "//envoy/stats:stats_interface",
+        "//source/common/memory:stats_lib",
+        "//source/common/stats:deferred_creation",
+        "//source/common/stats:thread_local_store_lib",
+        "//test/test_common:utility_lib",
+    ],
+)
+
+envoy_cc_benchmark_binary(
+    name = "deferred_creation_stats_benchmark",
+    srcs = ["deferred_creation_stats_speed_test.cc"],
+    external_deps = [
+        "benchmark",
+    ],
+    deps = [
+        ":real_thread_test_base",
+        "//source/common/common:random_generator_lib",
+        "//source/common/common:utility_lib",
+        "//source/common/runtime:runtime_lib",
+        "//source/common/stats:deferred_creation",
+        "//source/common/stats:isolated_store_lib",
+        "//source/common/stats:symbol_table_lib",
+        "//source/exe:process_wide_lib",
+    ],
+)
+
+envoy_benchmark_test(
+    name = "deferred_creation_stats_benchmark_test",
+    size = "large",
+    benchmark_binary = "deferred_creation_stats_benchmark",
+)
+
 envoy_benchmark_test(
     name = "symbol_table_benchmark_test",
     benchmark_binary = "symbol_table_benchmark",
diff --git a/test/common/stats/deferred_creation_stats_speed_test.cc b/test/common/stats/deferred_creation_stats_speed_test.cc
new file mode 100644
index 000000000000..e3358254e744
--- /dev/null
+++ b/test/common/stats/deferred_creation_stats_speed_test.cc
@@ -0,0 +1,240 @@
+#include "source/common/common/random_generator.h"
+#include "source/common/stats/deferred_creation.h"
+#include "source/common/stats/isolated_store_impl.h"
+#include "source/common/stats/symbol_table.h"
+#include "source/common/stats/thread_local_store.h"
+#include "source/common/thread_local/thread_local_impl.h"
+#include "source/exe/process_wide.h"
+
+#include "test/benchmark/main.h"
+#include "test/common/stats/real_thread_test_base.h"
+#include "test/test_common/real_threads_test_helper.h"
+
+#include "benchmark/benchmark.h"
+
+namespace Envoy {
+namespace Stats {
+
+// Creates a copy of Upstream::ALL_CLUSTER_TRAFFIC_STATS, such that we have a stable
+// set of stats for performance test.
+#define AWESOME_STATS(COUNTER, GAUGE, HISTOGRAM, TEXT_READOUT, STATNAME)                           \
+  COUNTER(bind_errors)                                                                             \
+  COUNTER(original_dst_host_invalid)                                                               \
+  COUNTER(retry_or_shadow_abandoned)                                                               \
+  COUNTER(upstream_cx_close_notify)                                                                \
+  COUNTER(upstream_cx_connect_attempts_exceeded)                                                   \
+  COUNTER(upstream_cx_connect_fail)                                                                \
+  COUNTER(upstream_cx_connect_timeout)                                                             \
+  COUNTER(upstream_cx_connect_with_0_rtt)                                                          \
+  COUNTER(upstream_cx_destroy)                                                                     \
+  COUNTER(upstream_cx_destroy_local)                                                               \
+  COUNTER(upstream_cx_destroy_local_with_active_rq)                                                \
+  COUNTER(upstream_cx_destroy_remote)                                                              \
+  COUNTER(upstream_cx_destroy_remote_with_active_rq)                                               \
+  COUNTER(upstream_cx_destroy_with_active_rq)                                                      \
+  COUNTER(upstream_cx_http1_total)                                                                 \
+  COUNTER(upstream_cx_http2_total)                                                                 \
+  COUNTER(upstream_cx_http3_total)                                                                 \
+  COUNTER(upstream_cx_idle_timeout)                                                                \
+  COUNTER(upstream_cx_max_duration_reached)                                                        \
+  COUNTER(upstream_cx_max_requests)                                                                \
+  COUNTER(upstream_cx_none_healthy)                                                                \
+  COUNTER(upstream_cx_overflow)                                                                    \
+  COUNTER(upstream_cx_pool_overflow)                                                               \
+  COUNTER(upstream_cx_protocol_error)                                                              \
+  COUNTER(upstream_cx_rx_bytes_total)                                                              \
+  COUNTER(upstream_cx_total)                                                                       \
+  COUNTER(upstream_cx_tx_bytes_total)                                                              \
+  COUNTER(upstream_flow_control_backed_up_total)                                                   \
+  COUNTER(upstream_flow_control_drained_total)                                                     \
+  COUNTER(upstream_flow_control_paused_reading_total)                                              \
+  COUNTER(upstream_flow_control_resumed_reading_total)                                             \
+  COUNTER(upstream_internal_redirect_failed_total)                                                 \
+  COUNTER(upstream_internal_redirect_succeeded_total)                                              \
+  COUNTER(upstream_rq_cancelled)                                                                   \
+  COUNTER(upstream_rq_completed)                                                                   \
+  COUNTER(upstream_rq_maintenance_mode)                                                            \
+  COUNTER(upstream_rq_max_duration_reached)                                                        \
+  COUNTER(upstream_rq_pending_failure_eject)                                                       \
+  COUNTER(upstream_rq_pending_overflow)                                                            \
+  COUNTER(upstream_rq_pending_total)                                                               \
+  COUNTER(upstream_rq_0rtt)                                                                        \
+  COUNTER(upstream_rq_per_try_timeout)                                                             \
+  COUNTER(upstream_rq_per_try_idle_timeout)                                                        \
+  COUNTER(upstream_rq_retry)                                                                       \
+  COUNTER(upstream_rq_retry_backoff_exponential)                                                   \
+  COUNTER(upstream_rq_retry_backoff_ratelimited)                                                   \
+  COUNTER(upstream_rq_retry_limit_exceeded)                                                        \
+  COUNTER(upstream_rq_retry_overflow)                                                              \
+  COUNTER(upstream_rq_retry_success)                                                               \
+  COUNTER(upstream_rq_rx_reset)                                                                    \
+  COUNTER(upstream_rq_timeout)                                                                     \
+  COUNTER(upstream_rq_total)                                                                       \
+  COUNTER(upstream_rq_tx_reset)                                                                    \
+  COUNTER(upstream_http3_broken)                                                                   \
+  GAUGE(upstream_cx_active, Accumulate)                                                            \
+  GAUGE(upstream_cx_rx_bytes_buffered, Accumulate)                                                 \
+  GAUGE(upstream_cx_tx_bytes_buffered, Accumulate)                                                 \
+  GAUGE(upstream_rq_active, Accumulate)                                                            \
+  GAUGE(upstream_rq_pending_active, Accumulate)                                                    \
+  HISTOGRAM(upstream_cx_connect_ms, Milliseconds)                                                  \
+  HISTOGRAM(upstream_cx_length_ms, Milliseconds)
+
+MAKE_STAT_NAMES_STRUCT(AwesomeStatNames, AWESOME_STATS);
+MAKE_STATS_STRUCT(AwesomeStats, AwesomeStatNames, AWESOME_STATS);
+
+class DeferredCreationStatsBenchmarkBase {
+public:
+  DeferredCreationStatsBenchmarkBase(bool lazy, const uint64_t n_clusters, Store& s)
+      : deferred_creation_(lazy), num_clusters_(n_clusters), stat_store_(s),
+        stat_names_(stat_store_.symbolTable()) {}
+
+  void createStats(bool defer_init) {
+    for (uint64_t i = 0; i < num_clusters_; ++i) {
+      std::string new_cluster_name = absl::StrCat("cluster_", i);
+      ScopeSharedPtr scope = stat_store_.createScope(new_cluster_name);
+      scopes_.push_back(scope);
+      auto lazy_stat = std::make_shared>(
+          createDeferredCompatibleStats(scope, stat_names_, deferred_creation_));
+      lazy_stats_.push_back(lazy_stat);
+      if (!defer_init) {
+        *(*lazy_stat);
+      }
+    }
+  }
+
+  const bool deferred_creation_;
+  const uint64_t num_clusters_;
+  Store& stat_store_;
+  std::vector scopes_;
+  std::vector>> lazy_stats_;
+  AwesomeStatNames stat_names_;
+};
+
+// Benchmark no-lazy-init on stats, the lazy init version is much faster since no allocation.
+void benchmarkDeferredCreationCreation(::benchmark::State& state) {
+  if (benchmark::skipExpensiveBenchmarks() && state.range(1) > 2000) {
+    state.SkipWithError("Skipping expensive benchmark");
+    return;
+  }
+
+  IsolatedStoreImpl stats_store;
+  DeferredCreationStatsBenchmarkBase base(state.range(0) == 1, state.range(1), stats_store);
+
+  for (auto _ : state) { // NOLINT: Silences warning about dead store
+    base.createStats(/*defer_init=*/true);
+  }
+}
+
+BENCHMARK(benchmarkDeferredCreationCreation)
+    ->ArgsProduct({{0, 1}, {1000, 2000, 5000, 10000, 20000}})
+    ->Unit(::benchmark::kMillisecond);
+
+// Benchmark lazy-init of stats in same thread, mimics main thread creation.
+void benchmarkDeferredCreationCreationInstantiateSameThread(::benchmark::State& state) {
+  if (benchmark::skipExpensiveBenchmarks() && state.range(1) > 2000) {
+    state.SkipWithError("Skipping expensive benchmark");
+    return;
+  }
+
+  IsolatedStoreImpl stats_store;
+  DeferredCreationStatsBenchmarkBase base(state.range(0) == 1, state.range(1), stats_store);
+
+  for (auto _ : state) { // NOLINT: Silences warning about dead store
+    base.createStats(/*defer_init=*/false);
+  }
+}
+
+BENCHMARK(benchmarkDeferredCreationCreationInstantiateSameThread)
+    ->ArgsProduct({{0, 1}, {1000, 2000, 5000, 10000, 20000}})
+    ->Unit(::benchmark::kMillisecond);
+
+class MultiThreadDeferredCreationStatsTest : public ThreadLocalRealThreadsMixin,
+                                             public DeferredCreationStatsBenchmarkBase {
+public:
+  MultiThreadDeferredCreationStatsTest(bool lazy, const uint64_t n_clusters)
+      : ThreadLocalRealThreadsMixin(5),
+        DeferredCreationStatsBenchmarkBase(lazy, n_clusters, *ThreadLocalRealThreadsMixin::store_) {
+  }
+
+  ~MultiThreadDeferredCreationStatsTest() {
+    shutdownThreading();
+    // First, wait for the main-dispatcher to initiate the cross-thread TLS cleanup.
+    mainDispatchBlock();
+
+    // Next, wait for all the worker threads to complete their TLS cleanup.
+    tlsBlock();
+
+    // Finally, wait for the final central-cache cleanup, which occurs on the main thread.
+    mainDispatchBlock();
+  }
+};
+
+// Benchmark lazy-init stats in different worker threads, mimics worker threads creation.
+void benchmarkDeferredCreationCreationInstantiateOnWorkerThreads(::benchmark::State& state) {
+  if (benchmark::skipExpensiveBenchmarks() && state.range(1) > 2000) {
+    state.SkipWithError("Skipping expensive benchmark");
+    return;
+  }
+
+  ProcessWide process_wide_; // Process-wide state setup/teardown (excluding grpc).
+  MultiThreadDeferredCreationStatsTest test(state.range(0) == 1, state.range(1));
+
+  for (auto _ : state) {           // NOLINT: Silences warning about dead store
+    test.runOnMainBlocking([&]() { // Create stats on main-thread.
+      test.createStats(/*defer_init=*/true);
+    });
+
+    std::atomic thread_idx = 0;
+    test.runOnAllWorkersBlocking([&]() {
+      int32_t batch_size = test.num_clusters_ / 5;
+      int t_idx = thread_idx++;
+      uint64_t begin = t_idx * batch_size;
+      uint64_t end = std::min(begin + batch_size, test.num_clusters_);
+      for (uint64_t idx = begin; idx < end; ++idx) {
+        // Instantiate the actual AwesomeStats objects in worker threads, in batches to avoid
+        // possible contention.
+        if (test.deferred_creation_) {
+          // Lazy-init on workers happen when the "index"-th stat instance is not created.
+          *(*test.lazy_stats_[idx]);
+        }
+      }
+    });
+  }
+}
+
+BENCHMARK(benchmarkDeferredCreationCreationInstantiateOnWorkerThreads)
+    ->ArgsProduct({{0, 1}, {1000, 2000, 5000, 10000, 20000}})
+    ->Unit(::benchmark::kMillisecond);
+
+// Benchmark mimics that worker threads inc the stats.
+void benchmarkDeferredCreationStatsAccess(::benchmark::State& state) {
+  if (benchmark::skipExpensiveBenchmarks() && state.range(1) > 2000) {
+    state.SkipWithError("Skipping expensive benchmark");
+    return;
+  }
+
+  ProcessWide process_wide_; // Process-wide state setup/teardown (excluding grpc).
+  MultiThreadDeferredCreationStatsTest test(state.range(0) == 1, state.range(1));
+
+  for (auto _ : state) {           // NOLINT: Silences warning about dead store
+    test.runOnMainBlocking([&]() { // Create stats on main-thread.
+      test.createStats(/*defer_init=*/false);
+    });
+    test.runOnAllWorkersBlocking([&]() {
+      // 50 x num_clusters_ inc() calls.
+      for (uint64_t idx = 0; idx < 10 * test.num_clusters_; ++idx) {
+        AwesomeStats& stats = *(*test.lazy_stats_[idx % test.num_clusters_]);
+        stats.upstream_cx_active_.inc();
+      }
+    });
+  }
+}
+
+BENCHMARK(benchmarkDeferredCreationStatsAccess)
+    ->ArgsProduct({{0, 1}, {1000, 2000, 5000, 10000, 20000}})
+    ->Unit(::benchmark::kMillisecond);
+
+} // namespace Stats
+
+} // namespace Envoy
diff --git a/test/common/stats/deferred_creation_stats_test.cc b/test/common/stats/deferred_creation_stats_test.cc
new file mode 100644
index 000000000000..f067f0ebbb5a
--- /dev/null
+++ b/test/common/stats/deferred_creation_stats_test.cc
@@ -0,0 +1,200 @@
+#include "envoy/stats/scope.h"
+#include "envoy/stats/stats_macros.h"
+
+#include "source/common/common/thread.h"
+#include "source/common/stats/deferred_creation.h"
+#include "source/common/stats/thread_local_store.h"
+
+#include "test/test_common/utility.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace Stats {
+namespace {
+
+#define AWESOME_STATS(COUNTER, GAUGE, HISTOGRAM, TEXT_READOUT, STATNAME) COUNTER(foo)
+
+MAKE_STAT_NAMES_STRUCT(AwesomeStatNames, AWESOME_STATS);
+MAKE_STATS_STRUCT(AwesomeStats, AwesomeStatNames, AWESOME_STATS);
+
+class DeferredCreationStatsTest : public testing::Test {
+public:
+  SymbolTableImpl symbol_table_;
+  AllocatorImpl allocator_{symbol_table_};
+  ThreadLocalStoreImpl store_{allocator_};
+  AwesomeStatNames stats_names_{symbol_table_};
+};
+
+using MyStats = DeferredCreationCompatibleStats;
+
+// Tests that non-lazy stats has no "AwesomeStats.initialized" gauge.
+TEST_F(DeferredCreationStatsTest, NonLazyNoInitializedGauge) {
+  {
+    ScopeSharedPtr scope = store_.createScope("bluh");
+    MyStats non_lazy_y = createDeferredCompatibleStats(scope, stats_names_, false);
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized"), nullptr);
+    EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo")->value(), 0);
+    non_lazy_y->foo_.inc();
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized"), nullptr);
+    EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo")->value(), 1);
+  }
+  // Scope gone, stats deleted.
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized"), nullptr);
+  EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo"), nullptr);
+}
+
+// Tests that "AwesomeStats.initialized" gauge equals the number of initiated MyStats instances.
+TEST_F(DeferredCreationStatsTest, StatsGoneWithScope) {
+  {
+    ScopeSharedPtr scope = store_.createScope("bluh");
+    // No such gauge when there is no lazy init stats instances.
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized"), nullptr);
+    MyStats x = createDeferredCompatibleStats(scope, stats_names_, true);
+    MyStats y = createDeferredCompatibleStats(scope, stats_names_, true);
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+    x->foo_.inc();
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+    EXPECT_EQ(x->foo_.value(), 1);
+    EXPECT_EQ(y->foo_.value(), 1);
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 2);
+  }
+  // Deleted as scope deleted.
+  EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo"), nullptr);
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized"), nullptr);
+  {
+    // Recreate scope "bluh".
+    ScopeSharedPtr scope = store_.createScope("bluh");
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized"), nullptr);
+    MyStats x = createDeferredCompatibleStats(scope, stats_names_, true);
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+    // Previous data is gone, as scope_v2 and scope_1's lifecycle do not overlap.
+    EXPECT_EQ(x->foo_.value(), 0);
+    // Initialized now.
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+  }
+}
+
+// Tests that multiple stats struct instances within the same scope has no issue to keep the
+// stats, with removals.
+TEST_F(DeferredCreationStatsTest, MultipleInstancesSameScopeDynamicallyDestructed) {
+  {
+    ScopeSharedPtr scope_1 = store_.createScope("bluh");
+    auto x = std::make_unique(
+        createDeferredCompatibleStats(scope_1, stats_names_, true));
+    auto y = std::make_unique(
+        createDeferredCompatibleStats(scope_1, stats_names_, true));
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+    // Only instantiate x, and then delete it.
+    (*x)->foo_.inc();
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+    EXPECT_EQ((*x)->foo_.value(), 1);
+    x.reset();
+    // y is not instantiated before x was deleted, no AwesomeStats instance, but stats are not
+    // lost.
+    EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo")->value(), 1);
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+    // Instantiate y now.
+    EXPECT_EQ((*y)->foo_.value(), 1);
+    (*y)->foo_.inc();
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+    EXPECT_EQ((*y)->foo_.value(), 2);
+  }
+  // Deleted as scope deleted.
+  EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo"), nullptr);
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized"), nullptr);
+  {
+    ScopeSharedPtr scope_v2 = store_.createScope("bluh");
+    MyStats x = createDeferredCompatibleStats(scope_v2, stats_names_, true);
+    // Previous data is gone, as scope_v2 and scope_1's lifecycle do not overlap.
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+    EXPECT_EQ(x->foo_.value(), 0);
+    // Initialized now.
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+  }
+  // Deleted as scope deleted.
+  EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo"), nullptr);
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized"), nullptr);
+}
+
+// Tests that as long as scope lives, stats under the scope won't be lost.
+TEST_F(DeferredCreationStatsTest, ScopeOutlivesLazyStats) {
+  ScopeSharedPtr scope_1 = store_.createScope("bluh");
+  {
+    auto x = std::make_unique(
+        createDeferredCompatibleStats(scope_1, stats_names_, true));
+    auto y = std::make_unique(
+        createDeferredCompatibleStats(scope_1, stats_names_, true));
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+    (*x)->foo_.inc();
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+    EXPECT_EQ((*x)->foo_.value(), 1);
+    EXPECT_EQ((*y)->foo_.value(), 1);
+    // x,y instantiated.
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 2);
+    // Only x instantiated.
+    y.reset();
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+    (*x)->foo_.inc();
+    EXPECT_EQ((*x)->foo_.value(), 2);
+  }
+  // Both MyStats deleted.
+  EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo")->value(), 2);
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+  {
+    // scope_1 overlaps with scope_v2.
+    ScopeSharedPtr scope_v2 = store_.createScope("bluh");
+
+    MyStats x_v2 = createDeferredCompatibleStats(scope_v2, stats_names_, true);
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+    // Previous data is NOT gone, as scope_v2 and scope_1's lifecycle overlap.
+    EXPECT_EQ(x_v2->foo_.value(), 2);
+
+    x_v2->foo_.inc();
+    EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo")->value(), 3);
+    EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+  }
+  // scope_v2 is gone, but stat value kept since scope_1 is alive.
+  EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo")->value(), 3);
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+}
+
+// Tests that as two AwesomeStats instances of different scope, as long as the scope life-cycle
+// overlaps, still got data kept when the earlier scope got deleted.
+TEST_F(DeferredCreationStatsTest, WhenScopesOverlapStatsAreAliveAsLongAsThereAre) {
+
+  ScopeSharedPtr scope_v1 = store_.createScope("bluh");
+  auto x = std::make_unique(
+      createDeferredCompatibleStats(scope_v1, stats_names_, true));
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 0);
+  (*x)->foo_.inc();
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+
+  EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo")->value(), 1);
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+
+  // Now scope_v2 gets created, but no action on any stats.
+  ScopeSharedPtr scope_v2 = store_.createScope("bluh");
+  auto y = std::make_unique(
+      createDeferredCompatibleStats(scope_v2, stats_names_, true));
+  // NOTE: since x was instantiated, y is instantiated on creation.
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 2);
+
+  // Now remove scope_v1, stats won't be lost.
+  x.reset();
+  scope_v1.reset();
+  EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo")->value(), 1);
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized")->value(), 1);
+
+  // remove scope_v2, stats will be gone.
+  y.reset();
+  scope_v2.reset();
+
+  EXPECT_EQ(TestUtility::findCounter(store_, "bluh.foo"), nullptr);
+  EXPECT_EQ(TestUtility::findGauge(store_, "bluh.AwesomeStats.initialized"), nullptr);
+}
+
+} // namespace
+} // namespace Stats
+} // namespace Envoy
diff --git a/test/mocks/server/server_factory_context.h b/test/mocks/server/server_factory_context.h
index 77a2610680c0..d7bd11f4719f 100644
--- a/test/mocks/server/server_factory_context.h
+++ b/test/mocks/server/server_factory_context.h
@@ -47,6 +47,7 @@ class MockStatsConfig : public virtual StatsConfig {
   MOCK_METHOD(std::chrono::milliseconds, flushInterval, (), (const));
   MOCK_METHOD(bool, flushOnAdmin, (), (const));
   MOCK_METHOD(const Stats::SinkPredicates*, sinkPredicates, (), (const));
+  MOCK_METHOD(bool, enableDeferredCreationStats, (), (const));
 };
 
 class MockServerFactoryContext : public virtual ServerFactoryContext {
diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt
index 429f6a848cd3..a7a578de18d2 100644
--- a/tools/spelling/spelling_dictionary.txt
+++ b/tools/spelling/spelling_dictionary.txt
@@ -925,6 +925,7 @@ metatable
 microbenchmarks
 midp
 milli
+mimics
 misconfiguration
 misconfigured
 mixin

From 3544614a86b4be0eb01b769214d793cebd1b5320 Mon Sep 17 00:00:00 2001
From: ohadvano <49730675+ohadvano@users.noreply.github.com>
Date: Wed, 21 Jun 2023 01:36:53 +0300
Subject: [PATCH 587/740] stream_info: add uint64 accessor (#28025)

Risk Level: Low
Testing: Unit tests

Signed-off-by: ohadvano 
---
 envoy/stream_info/BUILD                       |  8 +++++
 envoy/stream_info/uint64_accessor.h           | 26 ++++++++++++++
 source/common/stream_info/BUILD               |  8 +++++
 .../common/stream_info/uint64_accessor_impl.h | 31 ++++++++++++++++
 test/common/stream_info/BUILD                 |  8 +++++
 .../stream_info/uint64_accessor_impl_test.cc  | 36 +++++++++++++++++++
 6 files changed, 117 insertions(+)
 create mode 100644 envoy/stream_info/uint64_accessor.h
 create mode 100644 source/common/stream_info/uint64_accessor_impl.h
 create mode 100644 test/common/stream_info/uint64_accessor_impl_test.cc

diff --git a/envoy/stream_info/BUILD b/envoy/stream_info/BUILD
index 10c253696cac..ca692a7ceb45 100644
--- a/envoy/stream_info/BUILD
+++ b/envoy/stream_info/BUILD
@@ -47,6 +47,14 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "uint64_accessor_interface",
+    hdrs = ["uint64_accessor.h"],
+    deps = [
+        ":filter_state_interface",
+    ],
+)
+
 envoy_cc_library(
     name = "stream_id_provider_interface",
     hdrs = ["stream_id_provider.h"],
diff --git a/envoy/stream_info/uint64_accessor.h b/envoy/stream_info/uint64_accessor.h
new file mode 100644
index 000000000000..cdb40701b9ec
--- /dev/null
+++ b/envoy/stream_info/uint64_accessor.h
@@ -0,0 +1,26 @@
+#pragma once
+
+#include "envoy/common/pure.h"
+#include "envoy/stream_info/filter_state.h"
+
+namespace Envoy {
+namespace StreamInfo {
+
+/**
+ * A FilterState object that tracks a single uint64_t value.
+ */
+class UInt64Accessor : public FilterState::Object {
+public:
+  /**
+   * Increments the tracked value by 1.
+   */
+  virtual void increment() PURE;
+
+  /**
+   * @return the tracked value.
+   */
+  virtual uint64_t value() const PURE;
+};
+
+} // namespace StreamInfo
+} // namespace Envoy
diff --git a/source/common/stream_info/BUILD b/source/common/stream_info/BUILD
index c683fcbc91d9..687a50e9ef66 100644
--- a/source/common/stream_info/BUILD
+++ b/source/common/stream_info/BUILD
@@ -54,6 +54,14 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "uint64_accessor_lib",
+    hdrs = ["uint64_accessor_impl.h"],
+    deps = [
+        "//envoy/stream_info:uint64_accessor_interface",
+    ],
+)
+
 envoy_cc_library(
     name = "upstream_address_lib",
     hdrs = ["upstream_address.h"],
diff --git a/source/common/stream_info/uint64_accessor_impl.h b/source/common/stream_info/uint64_accessor_impl.h
new file mode 100644
index 000000000000..5859263135ab
--- /dev/null
+++ b/source/common/stream_info/uint64_accessor_impl.h
@@ -0,0 +1,31 @@
+#pragma once
+
+#include "envoy/stream_info/uint64_accessor.h"
+
+namespace Envoy {
+namespace StreamInfo {
+
+/*
+ * A FilterState object that tracks a single uint64_t value.
+ */
+class UInt64AccessorImpl : public UInt64Accessor {
+public:
+  UInt64AccessorImpl(uint64_t value) : value_(value) {}
+
+  // From FilterState::Object
+  ProtobufTypes::MessagePtr serializeAsProto() const override {
+    auto message = std::make_unique();
+    message->set_value(value_);
+    return message;
+  }
+
+  // From UInt64Accessor.
+  void increment() override { value_++; }
+  uint64_t value() const override { return value_; }
+
+private:
+  uint64_t value_;
+};
+
+} // namespace StreamInfo
+} // namespace Envoy
diff --git a/test/common/stream_info/BUILD b/test/common/stream_info/BUILD
index cc7c42a1c8bd..e34276643ad3 100644
--- a/test/common/stream_info/BUILD
+++ b/test/common/stream_info/BUILD
@@ -78,6 +78,14 @@ envoy_cc_test(
     ],
 )
 
+envoy_cc_test(
+    name = "uint64_accessor_impl_test",
+    srcs = ["uint64_accessor_impl_test.cc"],
+    deps = [
+        "//source/common/stream_info:uint64_accessor_lib",
+    ],
+)
+
 envoy_cc_test(
     name = "bool_accessor_impl_test",
     srcs = ["bool_accessor_impl_test.cc"],
diff --git a/test/common/stream_info/uint64_accessor_impl_test.cc b/test/common/stream_info/uint64_accessor_impl_test.cc
new file mode 100644
index 000000000000..8f143429339e
--- /dev/null
+++ b/test/common/stream_info/uint64_accessor_impl_test.cc
@@ -0,0 +1,36 @@
+#include "source/common/stream_info/uint64_accessor_impl.h"
+
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace StreamInfo {
+namespace {
+
+TEST(UInt64AccessorImplTest, ConstructorInitsValue) {
+  uint64_t init_value = 0xdeadbeefdeadbeef;
+  UInt64AccessorImpl accessor(init_value);
+  EXPECT_EQ(init_value, accessor.value());
+}
+
+TEST(UInt64AccessorImplTest, IncrementValue) {
+  uint64_t init_value = 0xdeadbeefdeadbeef;
+  UInt64AccessorImpl accessor(init_value);
+  accessor.increment();
+
+  EXPECT_EQ(0xdeadbeefdeadbef0, accessor.value());
+}
+
+TEST(UInt64AccessorImplTest, TestProto) {
+  uint64_t init_value = 0xdeadbeefdeadbeef;
+  UInt64AccessorImpl accessor(init_value);
+  auto message = accessor.serializeAsProto();
+  EXPECT_NE(nullptr, message);
+
+  auto* uint64_struct = dynamic_cast(message.get());
+  EXPECT_NE(nullptr, uint64_struct);
+  EXPECT_EQ(init_value, uint64_struct->value());
+}
+
+} // namespace
+} // namespace StreamInfo
+} // namespace Envoy

From b1514466c8e8c543fc3fdf607f23d0f464800e51 Mon Sep 17 00:00:00 2001
From: Kevin Baichoo 
Date: Tue, 20 Jun 2023 18:55:13 -0400
Subject: [PATCH 588/740] Listener: Connections accepted per socket event
 histogram. (#28015)

Signed-off-by: Kevin Baichoo 
---
 changelogs/current.yaml                       |  4 +++-
 docs/root/configuration/listeners/stats.rst   |  1 +
 envoy/network/listener.h                      |  7 +++++++
 source/common/network/tcp_listener_impl.cc    |  1 +
 .../listener_manager/active_tcp_listener.cc   |  4 ++++
 .../listener_manager/active_tcp_listener.h    |  1 +
 source/server/listener_stats.h                |  3 ++-
 test/common/http/codec_client_test.cc         |  1 +
 test/common/network/connection_impl_test.cc   |  7 +++++++
 test/common/network/listener_impl_test.cc     | 15 ++++++++++++++
 .../dns_resolver/cares/dns_impl_test.cc       |  1 +
 ...ected_resource_monitor_integration_test.cc | 13 ++++++++++++
 .../transport_sockets/tls/ssl_socket_test.cc  | 20 +++++++++++++++++++
 test/mocks/network/mocks.h                    |  1 +
 14 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index b8b1cb8b9eae..c97d9784d738 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -238,7 +238,9 @@ new_features:
     that sets the maximum number of new connections to be accepted per socket
     event on a listener. If there are more connections to be accepted beyond
     the maximum, the remaining connections would be processed in later
-    dispatcher loop iterations.
+    dispatcher loop iterations. Added listener histogram
+    ``connections_accepted_per_socket_event`` to allow users to empirically
+    determine an appropriate configuration for their deployment.
 - area: load shed point
   change: |
     added load shed point ``envoy.load_shed_points.http_connection_manager_decode_headers`` that rejects new http streams
diff --git a/docs/root/configuration/listeners/stats.rst b/docs/root/configuration/listeners/stats.rst
index fc961717b68d..42200239aa08 100644
--- a/docs/root/configuration/listeners/stats.rst
+++ b/docs/root/configuration/listeners/stats.rst
@@ -22,6 +22,7 @@ with the following statistics:
    downstream_cx_overflow, Counter, Total connections rejected due to enforcement of listener connection limit
    downstream_cx_overload_reject, Counter, Total connections rejected due to configured overload actions
    downstream_global_cx_overflow, Counter, Total connections rejected due to enforcement of global connection limit
+   connections_accepted_per_socket_event, Histogram, Number of connections accepted per listener socket event
    downstream_pre_cx_timeout, Counter, Sockets that timed out during listener filter processing
    downstream_pre_cx_active, Gauge, Sockets currently undergoing listener filter processing
    extension_config_missing, Counter, Total connections closed due to missing listener filter extension configuration
diff --git a/envoy/network/listener.h b/envoy/network/listener.h
index 5802ca480ae4..7d0454d88025 100644
--- a/envoy/network/listener.h
+++ b/envoy/network/listener.h
@@ -281,6 +281,13 @@ class TcpListenerCallbacks {
    * Called when a new connection is rejected.
    */
   virtual void onReject(RejectCause cause) PURE;
+
+  /**
+   * Called when the listener has finished accepting connections per socket
+   * event.
+   * @param connections_accepted number of connections accepted.
+   */
+  virtual void recordConnectionsAcceptedOnSocketEvent(uint32_t connections_accepted) PURE;
 };
 
 /**
diff --git a/source/common/network/tcp_listener_impl.cc b/source/common/network/tcp_listener_impl.cc
index c9cbe1efa833..0f5c3d5aadda 100644
--- a/source/common/network/tcp_listener_impl.cc
+++ b/source/common/network/tcp_listener_impl.cc
@@ -94,6 +94,7 @@ void TcpListenerImpl::onSocketEvent(short flags) {
 
   ENVOY_LOG_MISC(trace, "TcpListener accepted {} new connections.",
                  connections_accepted_from_kernel_count);
+  cb_.recordConnectionsAcceptedOnSocketEvent(connections_accepted_from_kernel_count);
 }
 
 TcpListenerImpl::TcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random,
diff --git a/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc b/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc
index 879ef13fabc9..6d972a0ff97a 100644
--- a/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc
+++ b/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc
@@ -98,6 +98,10 @@ void ActiveTcpListener::onReject(RejectCause cause) {
   }
 }
 
+void ActiveTcpListener::recordConnectionsAcceptedOnSocketEvent(uint32_t connections_accepted) {
+  stats_.connections_accepted_per_socket_event_.recordValue(connections_accepted);
+}
+
 void ActiveTcpListener::onAcceptWorker(Network::ConnectionSocketPtr&& socket,
                                        bool hand_off_restored_destination_connections,
                                        bool rebalanced) {
diff --git a/source/extensions/listener_managers/listener_manager/active_tcp_listener.h b/source/extensions/listener_managers/listener_manager/active_tcp_listener.h
index 518801744ff4..31bdd34131f8 100644
--- a/source/extensions/listener_managers/listener_manager/active_tcp_listener.h
+++ b/source/extensions/listener_managers/listener_manager/active_tcp_listener.h
@@ -51,6 +51,7 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks,
   // Network::TcpListenerCallbacks
   void onAccept(Network::ConnectionSocketPtr&& socket) override;
   void onReject(RejectCause) override;
+  void recordConnectionsAcceptedOnSocketEvent(uint32_t connections_accepted) override;
 
   // ActiveListenerImplBase
   Network::Listener* listener() override { return listener_.get(); }
diff --git a/source/server/listener_stats.h b/source/server/listener_stats.h
index 2824c438144d..bd6765a55d2b 100644
--- a/source/server/listener_stats.h
+++ b/source/server/listener_stats.h
@@ -20,7 +20,8 @@ namespace Server {
   COUNTER(no_filter_chain_match)                                                                   \
   GAUGE(downstream_cx_active, Accumulate)                                                          \
   GAUGE(downstream_pre_cx_active, Accumulate)                                                      \
-  HISTOGRAM(downstream_cx_length_ms, Milliseconds)
+  HISTOGRAM(downstream_cx_length_ms, Milliseconds)                                                 \
+  HISTOGRAM(connections_accepted_per_socket_event, Unspecified)
 
 /**
  * Wrapper struct for listener stats. @see stats_macros.h
diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc
index b8031d7bc3bb..f42d7ed316d1 100644
--- a/test/common/http/codec_client_test.cc
+++ b/test/common/http/codec_client_test.cc
@@ -518,6 +518,7 @@ class CodecNetworkTest : public Event::TestUsingSimulatedTime,
             dispatcher_->exit();
           }
         }));
+    EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
     EXPECT_CALL(client_callbacks_, onEvent(Network::ConnectionEvent::Connected))
         .WillOnce(InvokeWithoutArgs([&]() -> void {
diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc
index 4208bb5abf8d..12eeef31f26e 100644
--- a/test/common/network/connection_impl_test.cc
+++ b/test/common/network/connection_impl_test.cc
@@ -187,6 +187,7 @@ class ConnectionImplTestBase {
             dispatcher_->exit();
           }
         }));
+    EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
     EXPECT_CALL(client_callbacks_, onEvent(ConnectionEvent::Connected))
         .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
           expected_callbacks--;
@@ -384,6 +385,7 @@ TEST_P(ConnectionImplTest, CloseDuringConnectCallback) {
         server_connection_->addReadFilter(add_and_remove_filter);
         server_connection_->removeReadFilter(add_and_remove_filter);
       }));
+  EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(server_callbacks_, onEvent(ConnectionEvent::RemoteClose))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { dispatcher_->exit(); }));
@@ -414,6 +416,7 @@ TEST_P(ConnectionImplTest, UnregisterRegisterDuringConnectCallback) {
           dispatcher_->exit();
         }
       }));
+  EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
   EXPECT_CALL(client_callbacks_, onEvent(ConnectionEvent::Connected))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
         expected_callbacks--;
@@ -572,6 +575,7 @@ TEST_P(ConnectionImplTest, SocketOptions) {
             socket_->connectionInfoProvider().localAddress(), source_address_,
             Network::Test::createRawBufferSocket(), server_connection_->socketOptions(), nullptr);
       }));
+  EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(server_callbacks_, onEvent(ConnectionEvent::RemoteClose))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
@@ -622,6 +626,7 @@ TEST_P(ConnectionImplTest, SocketOptionsFailureTest) {
             Network::Test::createRawBufferSocket(), server_connection_->socketOptions(), nullptr);
         upstream_connection_->addConnectionCallbacks(upstream_callbacks_);
       }));
+  EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(upstream_callbacks_, onEvent(ConnectionEvent::LocalClose));
 
@@ -717,6 +722,7 @@ TEST_P(ConnectionImplTest, ConnectionStats) {
         server_connection_->addReadFilter(read_filter_);
         EXPECT_EQ("", server_connection_->nextProtocol());
       }));
+  EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
   Sequence s2;
   EXPECT_CALL(server_connection_stats.rx_total_, add(4)).InSequence(s2);
@@ -2937,6 +2943,7 @@ class ReadBufferLimitTest : public ConnectionImplTest {
           EXPECT_EQ("", server_connection_->nextProtocol());
           EXPECT_EQ(read_buffer_limit, server_connection_->bufferLimit());
         }));
+    EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
     uint32_t filter_seen = 0;
 
diff --git a/test/common/network/listener_impl_test.cc b/test/common/network/listener_impl_test.cc
index 11522a18a265..e29da9219d2b 100644
--- a/test/common/network/listener_impl_test.cc
+++ b/test/common/network/listener_impl_test.cc
@@ -118,6 +118,7 @@ TEST_P(TcpListenerImplTest, UseActualDst) {
   EXPECT_CALL(listener_callbacks2, onAccept_(_)).Times(0);
   EXPECT_CALL(listener_callbacks1, onAccept_(_))
       .WillOnce(Invoke([&](Network::ConnectionSocketPtr& accepted_socket) -> void {
+        EXPECT_CALL(listener_callbacks1, recordConnectionsAcceptedOnSocketEvent(_));
         Network::ConnectionPtr conn = dispatcher_->createServerConnection(
             std::move(accepted_socket), Network::Test::createRawBufferSocket(), stream_info);
         EXPECT_EQ(*conn->connectionInfoProvider().localAddress(),
@@ -165,6 +166,7 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitEnforcement) {
   initiate_connections(5);
   EXPECT_CALL(listener_callbacks, onReject(TcpListenerCallbacks::RejectCause::GlobalCxLimit))
       .Times(3);
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(5));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 
   // We expect any server-side connections that get created to populate 'server_connections'.
@@ -175,6 +177,7 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitEnforcement) {
   initiate_connections(5);
   EXPECT_CALL(listener_callbacks, onReject(TcpListenerCallbacks::RejectCause::GlobalCxLimit))
       .Times(4);
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(5));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 
   EXPECT_EQ(3, server_connections.size());
@@ -182,6 +185,7 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitEnforcement) {
   // Clear the limit and verify there's no longer a limit.
   scoped_runtime.mergeValues({{"overload.global_downstream_max_connections", ""}});
   initiate_connections(10);
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(10));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 
   EXPECT_EQ(13, server_connections.size());
@@ -232,6 +236,7 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitListenerOptOut) {
   initiate_connections(2);
   EXPECT_CALL(listener_callbacks, onReject(TcpListenerCallbacks::RejectCause::GlobalCxLimit))
       .Times(0);
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 
   for (const auto& conn : client_connections) {
@@ -274,6 +279,7 @@ TEST_P(TcpListenerImplTest, WildcardListenerUseActualDst) {
         dispatcher_->exit();
       }));
 
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 }
 
@@ -326,6 +332,7 @@ TEST_P(TcpListenerImplTest, WildcardListenerIpv4Compat) {
         dispatcher_->exit();
       }));
 
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 }
 
@@ -370,6 +377,7 @@ TEST_P(TcpListenerImplTest, DisableAndEnableListener) {
         dispatcher_->exit();
       }));
 
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 }
 
@@ -398,6 +406,7 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionZero) {
       Network::Test::createRawBufferSocket(), nullptr, nullptr);
   client_connection->addConnectionCallbacks(connection_callbacks);
   client_connection->connect();
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 
   // Now that we've seen that the connection hasn't been closed by the listener, make sure to close
@@ -438,6 +447,7 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionIntermediate) {
         Network::Test::createRawBufferSocket(), nullptr, nullptr);
     client_connection->addConnectionCallbacks(connection_callbacks);
     client_connection->connect();
+    EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
     dispatcher_->run(Event::Dispatcher::RunType::Block);
   }
 
@@ -461,6 +471,7 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionIntermediate) {
         Network::Test::createRawBufferSocket(), nullptr, nullptr);
     client_connection->addConnectionCallbacks(connection_callbacks);
     client_connection->connect();
+    EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
     dispatcher_->run(Event::Dispatcher::RunType::Block);
 
     EXPECT_CALL(connection_callbacks, onEvent(ConnectionEvent::LocalClose));
@@ -500,6 +511,7 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionAll) {
       Network::Test::createRawBufferSocket(), nullptr, nullptr);
   client_connection->addConnectionCallbacks(connection_callbacks);
   client_connection->connect();
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 }
 
@@ -534,6 +546,7 @@ TEST_P(TcpListenerImplTest, LoadShedPointCanRejectConnection) {
     });
   }
 
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   ClientConnectionPtr client_connection = dispatcher_->createClientConnection(
       socket->connectionInfoProvider().localAddress(), Address::InstanceConstSharedPtr(),
       Network::Test::createRawBufferSocket(), nullptr, nullptr);
@@ -596,6 +609,7 @@ TEST_P(TcpListenerImplTest, EachQueuedConnectionShouldQueryTheLoadShedPoint) {
   client_connection2->addConnectionCallbacks(connection_callbacks2);
   client_connection2->connect();
 
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 
   // Now that we've seen that the connection hasn't been closed by the listener, make sure to
@@ -647,6 +661,7 @@ TEST_P(TcpListenerImplTest, ShouldOnlyAcceptTheMaxNumberOfConnectionsConfiguredP
             }
           }));
   //  Check the logs that they are accepted at different socket events
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(1)).Times(2);
   EXPECT_LOG_CONTAINS_N_TIMES("trace", "accepted 1 new connections", 2,
                               { dispatcher_->run(Event::Dispatcher::RunType::Block); });
 
diff --git a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc
index ac24ce629e9c..d3a9c9af7e43 100644
--- a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc
+++ b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc
@@ -349,6 +349,7 @@ class TestDnsServer : public TcpListenerCallbacks {
   }
 
   void onReject(RejectCause) override { PANIC("not implemented"); }
+  void recordConnectionsAcceptedOnSocketEvent(uint32_t) override {}
 
   void addHosts(const std::string& hostname, const IpList& ip, const RecordType& type) {
     if (type == RecordType::A) {
diff --git a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
index 95909558ea2d..9d41d6e6b899 100644
--- a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
+++ b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
@@ -222,6 +222,19 @@ TEST_P(ListenerMaxConnectionPerSocketEventTest, AcceptsConnectionsUpToTheMaximum
 
   std::for_each(client_codecs.begin(), client_codecs.end(),
                 [](IntegrationCodecClientPtr& client_codec) { client_codec->close(); });
+  const std::string connections_accepted_per_socket_event =
+      (version_ == Network::Address::IpVersion::v4)
+          ? "listener.127.0.0.1_0.connections_accepted_per_socket_event"
+          : "listener.[__1]_0.connections_accepted_per_socket_event";
+  test_server_->waitUntilHistogramHasSamples(connections_accepted_per_socket_event);
+  auto connections_accepted_histogram =
+      test_server_->histogram(connections_accepted_per_socket_event);
+  EXPECT_EQ(TestUtility::readSampleCount(test_server_->server().dispatcher(),
+                                         *connections_accepted_histogram),
+            5);
+  EXPECT_EQ(static_cast(TestUtility::readSampleSum(test_server_->server().dispatcher(),
+                                                        *connections_accepted_histogram)),
+            10);
 }
 
 } // namespace Envoy
diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc
index ffda4d0ab62f..0a1bab07012c 100644
--- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc
+++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc
@@ -393,6 +393,7 @@ void testUtil(const TestUtilOptions& options) {
                                                                std::move(ssl_socket), stream_info);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   if (options.ocspStaplingEnabled()) {
     const SslHandshakerImpl* ssl_socket =
@@ -751,6 +752,7 @@ void testUtilV2(const TestUtilOptionsV2& options) {
             std::move(socket), std::move(transport_socket), stream_info);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   Network::MockConnectionCallbacks client_connection_callbacks;
   client_connection->addConnectionCallbacks(client_connection_callbacks);
@@ -3003,6 +3005,7 @@ TEST_P(SslSocketTest, FlushCloseDuringHandshake) {
         server_connection->write(data, false);
         server_connection->close(Network::ConnectionCloseType::FlushWrite);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::LocalClose));
   EXPECT_CALL(client_connection_callbacks, onEvent(Network::ConnectionEvent::Connected));
@@ -3075,6 +3078,7 @@ TEST_P(SslSocketTest, HalfClose) {
         Buffer::OwnedImpl data("hello");
         server_connection->write(data, true);
       }));
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(*server_read_filter, onNewConnection())
       .WillOnce(Return(Network::FilterStatus::Continue));
@@ -3156,6 +3160,7 @@ TEST_P(SslSocketTest, ShutdownWithCloseNotify) {
         server_connection->addReadFilter(server_read_filter);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   EXPECT_CALL(*server_read_filter, onNewConnection());
   EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::Connected))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
@@ -3245,6 +3250,7 @@ TEST_P(SslSocketTest, ShutdownWithoutCloseNotify) {
         server_connection->addReadFilter(server_read_filter);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(listener_callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::Connected))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
         Buffer::OwnedImpl data("hello");
@@ -3361,6 +3367,7 @@ TEST_P(SslSocketTest, ClientAuthMultipleCAs) {
             stream_info_);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::Connected))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
@@ -3455,6 +3462,7 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1,
         server_connection = dispatcher->createServerConnection(
             std::move(socket), tsf.createDownstreamTransportSocket(), stream_info);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(client_connection_callbacks, onEvent(Network::ConnectionEvent::Connected))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
@@ -3501,6 +3509,7 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1,
             std::move(socket), tsf.createDownstreamTransportSocket(), stream_info2);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   // Different tests have different order of whether client or server gets Connected event
   // first, so always wait until both have happened.
@@ -3603,6 +3612,7 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml
           EXPECT_EQ(SSL_OP_NO_TICKET, (SSL_CTX_get_options(server_ssl_context) & SSL_OP_NO_TICKET));
         }
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(client_connection_callbacks, onEvent(Network::ConnectionEvent::Connected))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
@@ -4205,6 +4215,7 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) {
             std::move(accepted_socket), tsf.createDownstreamTransportSocket(), stream_info_);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::Connected));
   EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::LocalClose));
@@ -4247,6 +4258,7 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) {
             std::move(accepted_socket), tsf.createDownstreamTransportSocket(), stream_info_);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
   EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::RemoteClose));
   EXPECT_CALL(client_connection_callbacks, onEvent(Network::ConnectionEvent::RemoteClose))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { dispatcher_->exit(); }));
@@ -4334,6 +4346,7 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya
             stream_info_);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   const bool expect_tls13 =
       client_ctx_proto.common_tls_context().tls_params().tls_maximum_protocol_version() ==
@@ -4380,6 +4393,7 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya
             stream_info_);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   // The order of "Connected" events depends on the version of the TLS protocol (1.3 or older),
   // and whether or not the session was successfully resumed.
@@ -4565,6 +4579,7 @@ TEST_P(SslSocketTest, SslError) {
             stream_info_);
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::RemoteClose))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
@@ -5146,6 +5161,7 @@ TEST_P(SslSocketTest, SetSignatureAlgorithms) {
 
         server_connection->addConnectionCallbacks(server_connection_callbacks);
       }));
+  EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::Connected))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void {
@@ -5812,6 +5828,7 @@ class SslReadBufferLimitTest : public SslSocketTest {
           EXPECT_EQ("", server_connection_->nextProtocol());
           EXPECT_EQ(read_buffer_limit, server_connection_->bufferLimit());
         }));
+    EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
     EXPECT_CALL(client_callbacks_, onEvent(Network::ConnectionEvent::Connected))
         .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { dispatcher_->exit(); }));
@@ -5888,6 +5905,7 @@ class SslReadBufferLimitTest : public SslSocketTest {
           EXPECT_EQ("", server_connection_->nextProtocol());
           EXPECT_EQ(read_buffer_limit, server_connection_->bufferLimit());
         }));
+    EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
     dispatcher_->run(Event::Dispatcher::RunType::Block);
 
@@ -6008,6 +6026,7 @@ TEST_P(SslReadBufferLimitTest, TestBind) {
         server_connection_->addReadFilter(read_filter_);
         EXPECT_EQ("", server_connection_->nextProtocol());
       }));
+  EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(client_callbacks_, onEvent(Network::ConnectionEvent::Connected))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { dispatcher_->exit(); }));
@@ -6041,6 +6060,7 @@ TEST_P(SslReadBufferLimitTest, SmallReadsIntoSameSlice) {
         EXPECT_EQ("", server_connection_->nextProtocol());
         EXPECT_EQ(read_buffer_limit, server_connection_->bufferLimit());
       }));
+  EXPECT_CALL(listener_callbacks_, recordConnectionsAcceptedOnSocketEvent(_));
 
   EXPECT_CALL(client_callbacks_, onEvent(Network::ConnectionEvent::Connected))
       .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { dispatcher_->exit(); }));
diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h
index 4f680680b0ce..8453b6cdd3b8 100644
--- a/test/mocks/network/mocks.h
+++ b/test/mocks/network/mocks.h
@@ -170,6 +170,7 @@ class MockTcpListenerCallbacks : public TcpListenerCallbacks {
 
   MOCK_METHOD(void, onAccept_, (ConnectionSocketPtr & socket));
   MOCK_METHOD(void, onReject, (RejectCause), (override));
+  MOCK_METHOD(void, recordConnectionsAcceptedOnSocketEvent, (uint32_t), (override));
 };
 
 class MockUdpListenerCallbacks : public UdpListenerCallbacks {

From 8b7f05e4a76097d843127430bf90e250adae1334 Mon Sep 17 00:00:00 2001
From: Jeongseok Son 
Date: Tue, 20 Jun 2023 23:30:12 -0700
Subject: [PATCH 589/740] Update QUICHE from 4506f8d26 to 722cf00a5 (#27970)

* Update QUICHE from 4506f8d26 to 90315e869
https://github.com/google/quiche/compare/4506f8d26..90315e869

```
$ git log 4506f8d26..90315e869 --date=short --no-merges --format="%ad %al %s"

2023-06-08 danzh When a QUIC client probes from a new socket via a new writer or respond to peer's probing on a new socket, check if the socket is write blocked.
2023-06-08 danzh Deprecate flag --gfe2_reloadable_flag_quic_connection_migration_use_new_cid_v2.
2023-06-08 haoyuewang Deprecate --gfe2_reloadable_flag_quic_check_cid_collision_when_issue_new_cid
2023-06-08 vasilvv Make quic::BitMask parametrized by the type of the enum used.
2023-06-07 birenroy Allows the character delimiting URL fragments in request paths.
2023-06-07 vasilvv Notify WebTransport sessions when GOAWAY is received.
2023-06-07 vasilvv Run buildifier as a part of QUICHE Copybara export pipeline
2023-06-07 quiche-dev Add ability to specify preferred groups for TLS handshake.
2023-06-07 bnc Add QUIC_BUGs to debug b/286055504.
2023-06-06 quiche-dev Switch to a constant expression for GetRsaSqrtTwo
2023-06-06 renjietang Do not create multi-port path if the connection has active migration disabled.
2023-06-06 danzh Make QuicConnection to do reverse path validation based on IETF version instead of bool validate_client_addresses_.
2023-06-06 vasilvv Remove WebTransport version header check on the server side.
2023-06-05 quiche-dev Fixes for third party anonymous tokens directory that allow all tests to pass with bazel which will lead to test coverage in the github repo.
2023-06-05 quiche-dev Update phosphor and token spend apis to expose and accept binary public metadata
2023-06-05 martinduke ECN support for several QuicPacketWriter implementations.
2023-06-05 vasilvv Replace uses of std::function inside QUICHE.
2023-06-05 vasilvv Switch uses of std::function to QUICHE callback aliases as appropriate.
2023-06-05 renjietang Add a test on multi-port path receiving stateless reset.
2023-06-02 quiche-dev Remove second QUICHE_EXPORT for SignatureChanger
2023-06-02 danzh Document about deprecation of QUIC connection option RVCM.
2023-06-02 danzh Remove quic connection option `RVCM` everywhere given IETF style connection migration is no longer guarded by it since cl/527045730.
2023-06-02 martinduke QUIC v2 has been published as RFC 9369. Rename all identifiers from "v2 draft 08" to "rfc v2". No behavior change. Also renames the not-enabled flag gfe2_reloadable_flag_quic_enable_version_2_draft_08 to gfe2_reloadable_flag_quic_enable_version_rfcv2.
2023-06-01 quiche-dev Fix test failures in CryptoUtilsTest
2023-06-01 haoyuewang Let RecordingProofVerifier use the underlying verifier_ (if not null) to VerifyCertChain.
2023-05-31 vasilvv Replace uses of std::function with quiche::UnretainedCallback where appropriate
2023-05-31 junyer Update Quiche to RE2 release `2023-06-01`.
2023-05-31 quiche-dev Use ANON_TOKENS_ASSERT_OK_AND_ASSIGN in crypto_utils_test.cc
2023-05-31 vasilvv Fix stand-alone QUICHE build
2023-05-30 quiche-dev Function that DER-encodes the public key of RSA Blind Signature - Probabilistic Signature Scheme algorithm.
2023-05-30 quiche-dev Add `--ignore_errors` option to quic_client.
2023-05-30 vasilvv Create quiche_callbacks.h
2023-05-30 birenroy Adds a basic fuzzer for Http2DecoderAdapter.
2023-05-30 birenroy Adds a basic fuzzer for Http2FrameDecoder.
2023-05-25 danzh Add QUIC connection options `MCS1`, `MCS2` and `MCS3` to adjust GFE allowed max concurrent streams.
2023-05-25 birenroy Validates request paths according to RFC 3986 Section 3.3.
2023-05-25 birenroy Adds HeaderValidator unit tests for valid and invalid path characters.
2023-05-25 birenroy Adds a unit test demonstrating that oghttp2 will accept request paths with space or tab.
```

Signed-off-by: Jeongseok Son 

* Update QUICHE from 90315e869 to c43017f03
https://github.com/google/quiche/compare/90315e869..c43017f03

```
$ git log 90315e869..c43017f03 --date=short --no-merges --format="%ad %al %s"

2023-06-14 martinduke Deprecate gfe2_restart_flags_quic_receive_ecn, replace with quic_receive_ecn2 as canary-only. Also remove flag count that is not actually protected by the flag.
2023-06-14 wub Remove non-initial burst from QUIC PacingSender.
2023-06-13 bnc Add HttpValidationPolicy::require_content_length_if_body_required.
2023-06-13 martinduke Deprecate gfe2_restart_flag_quic_quiche_ecn_sockets.
2023-06-13 wub Remove MockClock from PacketSavingConnection, and use the MockClock in MockQuicConnectionHelper instead.
2023-06-12 quiche-dev Moving functions that use protocol buffers to "testing/proto_utils" from testing/utils. This CL does not add any new code and only touches test and build files.
```

Signed-off-by: Jeongseok Son 

* QUICHE changes into Envoy.

Signed-off-by: Jeongseok Son 

* Update QUICHE from c43017f03 to 722cf00a5
https://github.com/google/quiche/compare/c43017f03..722cf00a5

```
$ git log c43017f03..722cf00a5 --date=short --no-merges --format="%ad %al %s"

2023-06-20 bnc Revert cl/535379163 and cl/538613643.
2023-06-20 quiche-dev Fixing an error message.
2023-06-20 wub Add a QUIC e2e test for QuicConnection::SetMaxPacingRate.
2023-06-20 birenroy Validates HTTP methods according to RFC 9110.
2023-06-20 wub Ignore non-positive RTT samples in QUIC BBR2.
2023-06-20 quiche-dev Remove protocol buffer dependency from crypto/rsa_blinder. Individual clients can still depend on the protocol buffer but crypto library should have minimum dependencies anyway.
2023-06-20 quiche-dev Move protocol buffer dependent functions from crypto_utils to anonymous_tokens_pb_openssl_converters. This CL doesn't add any new code other than a few tests in anonymous_tokens_pb_openssl_converters_test.cc.
2023-06-20 quiche-dev Expose blackhole detector to stop detection when chromium blocks packet writer.
2023-06-20 quiche-dev Factor our openssl object creation from AnonymousTokensRSAPrivateKeyToRSA and AnonymousTokensRSAPublicKeyToRSA so that the creation doesn't always need a proto object in the parameter.
2023-06-20 quiche-dev Adding some test functions that output test keys in a struct rather than parsing keys from a file to a proto. This is done so files such as crypto_utils in this CL can remove their dependency on the proto keys.
2023-06-20 mkruskal IWYS fixes for //base headers
2023-06-19 quiche-dev Remove some unused fields from BalsaFrame.
2023-06-16 vasilvv Implement fetching dependencies in QUICHE depstool
2023-06-16 vasilvv Fix QUICHE standalone build
2023-06-16 wub Automated g4 rollback of changelist 540881082.
2023-06-16 haoyuewang Check if CID is missing in QuicConnection::WriteIfNotBlocked.
2023-06-16 mkruskal IWYS fixes for //base headers
2023-06-16 danzh Internal change
2023-06-16 bnc Automated g4 rollback of changelist 539970480.
2023-06-16 birenroy Fixes the log/early return condition in CallbackVisitor::OnMetadataEndForStream.
2023-06-15 quiche-dev Removing unnecessary validation
2023-06-15 vasilvv Implement sending and receiving DRAIN_WEBTRANSPORT_SESSION
2023-06-15 diannahu Add BalsaHeadersSequence::IsEmpty().
2023-06-15 diannahu Update BalsaHeadersSequence and BalsaVisitorInterface to use std::unique_ptr for interim headers.
2023-06-15 diannahu Update the BalsaHeadersSequence internal data structure from std::list to absl::InlinedVector.
2023-06-15 birenroy Replaces a fatal log message with an early return in CallbackVisitor.
2023-06-15 quiche-dev Internal change
```

Signed-off-by: Jeongseok Son 

* Fix the interface mismatch.

Signed-off-by: Jeongseok Son 

---------

Signed-off-by: Jeongseok Son 
---
 bazel/external/quiche.BUILD                   | 26 +++++++++++++++++++
 bazel/repository_locations.bzl                |  6 ++---
 mobile/library/cc/request_headers_builder.cc  |  2 ++
 .../platform_bridge_cert_validator.cc         |  2 ++
 source/common/http/http1/balsa_parser.cc      |  2 +-
 source/common/http/http1/balsa_parser.h       |  2 +-
 .../quic/envoy_quic_client_connection.cc      |  9 ++-----
 7 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD
index cd3df2b203d3..1b90026a4a7c 100644
--- a/bazel/external/quiche.BUILD
+++ b/bazel/external/quiche.BUILD
@@ -99,6 +99,7 @@ envoy_cc_library(
         ":http2_adapter_http2_visitor_interface",
         ":http2_adapter_nghttp2_include",
         ":http2_adapter_nghttp2_util",
+        ":quiche_common_callbacks",
         ":quiche_common_platform_export",
     ],
 )
@@ -136,6 +137,7 @@ envoy_cc_library(
     copts = quiche_copts,
     repository = "@envoy",
     deps = [
+        ":quiche_common_callbacks",
         ":quiche_common_platform_export",
         ":spdy_core_http2_deframer_lib",
     ],
@@ -469,12 +471,14 @@ envoy_cc_library(
         ":http2_adapter_window_manager",
         ":http2_core_http2_trace_logging_lib",
         ":http2_core_priority_write_scheduler_lib",
+        ":quiche_common_callbacks",
         ":spdy_core_framer_lib",
         ":spdy_core_http2_deframer_lib",
         ":spdy_core_http2_header_block_lib",
         ":spdy_core_protocol_lib",
         ":spdy_header_byte_listener_interface_lib",
         ":spdy_no_op_headers_handler_lib",
+        "@com_google_absl//absl/cleanup",
     ],
 )
 
@@ -619,6 +623,7 @@ envoy_cc_library(
     copts = quiche_copts,
     repository = "@envoy",
     deps = [
+        ":quiche_common_callbacks",
         ":quiche_common_platform",
         ":quiche_common_platform_export",
     ],
@@ -1475,6 +1480,7 @@ envoy_cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":http2_hpack_huffman_hpack_huffman_encoder_lib",
+        ":quiche_common_callbacks",
         ":quiche_common_circular_deque_lib",
         ":quiche_common_platform",
         ":spdy_core_protocol_lib",
@@ -2974,6 +2980,19 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "quiche_common_callbacks",
+    hdrs = ["quiche/common/quiche_callbacks.h"],
+    copts = quiche_copts,
+    repository = "@envoy",
+    tags = ["nofips"],
+    deps = [
+        ":quiche_common_platform_export",
+        "@com_google_absl//absl/functional:any_invocable",
+        "@com_google_absl//absl/functional:function_ref",
+    ],
+)
+
 envoy_cc_library(
     name = "quic_core_data_lib",
     srcs = [
@@ -3614,6 +3633,7 @@ envoy_cc_library(
         ":quic_core_syscall_wrapper_lib",
         ":quic_core_types_lib",
         ":quic_platform",
+        ":quiche_common_callbacks",
     ],
 )
 
@@ -4182,6 +4202,7 @@ envoy_cc_library(
         ":quic_core_version_manager_lib",
         ":quic_platform",
         ":quic_server_session_lib",
+        ":quiche_common_callbacks",
         ":quiche_common_text_utils_lib",
     ],
 )
@@ -4271,6 +4292,7 @@ envoy_cc_library(
         ":quic_platform",
         ":quic_server_crypto_crypto_handshake_lib",
         ":quic_stream_priority_lib",
+        ":quiche_common_callbacks",
         ":quiche_common_structured_headers_lib",
         ":quiche_common_text_utils_lib",
         ":spdy_core_protocol_lib",
@@ -4935,6 +4957,7 @@ envoy_cc_test_library(
         ":quic_test_tools_stream_peer_lib",
         ":quic_test_tools_test_certificates_lib",
         ":quiche_common_buffer_allocator_lib",
+        ":quiche_common_callbacks",
         ":quiche_common_test_tools_test_utils_lib",
         ":spdy_core_framer_lib",
     ],
@@ -5715,6 +5738,7 @@ envoy_cc_library(
     repository = "@envoy",
     tags = ["nofips"],
     deps = [
+        ":quiche_common_callbacks",
         ":quiche_common_platform_export",
         ":quiche_common_platform_lower_case_string",
         "@com_google_absl//absl/strings",
@@ -5791,6 +5815,7 @@ envoy_cc_library(
         ":quiche_balsa_header_api_lib",
         ":quiche_balsa_header_properties_lib",
         ":quiche_balsa_standard_header_map_lib",
+        ":quiche_common_callbacks",
         ":quiche_common_platform_bug_tracker",
         ":quiche_common_platform_export",
         ":quiche_common_platform_header_policy",
@@ -5875,6 +5900,7 @@ envoy_cc_library(
     repository = "@envoy",
     tags = ["nofips"],
     deps = [
+        ":quiche_common_callbacks",
         ":quiche_common_platform_export",
         ":quiche_common_quiche_stream_lib",
         ":spdy_core_http2_header_block_lib",
diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
index 1deaabc0ec69..acfececa4ee3 100644
--- a/bazel/repository_locations.bzl
+++ b/bazel/repository_locations.bzl
@@ -1056,12 +1056,12 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "QUICHE",
         project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols",
         project_url = "https://github.com/google/quiche",
-        version = "4506f8d266853c2b039bec1ee4bc8ed0f22d861c",
-        sha256 = "f532f981381b8e0f817b4426d6dae085c32bdfe75ba01c55ad09a6d9341f91fb",
+        version = "722cf00a5331c425ca5c14f9c6eab5b2f0d12cb3",
+        sha256 = "797d66ddd2d4b20a1309c732b7f62c111a810d608d948f87a01a10e89e35a47a",
         urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"],
         strip_prefix = "quiche-{version}",
         use_category = ["controlplane", "dataplane_core"],
-        release_date = "2023-05-24",
+        release_date = "2023-06-20",
         cpe = "N/A",
         license = "BSD-3-Clause",
         license_url = "https://github.com/google/quiche/blob/{version}/LICENSE",
diff --git a/mobile/library/cc/request_headers_builder.cc b/mobile/library/cc/request_headers_builder.cc
index ac5095ebf881..e0755e2848ee 100644
--- a/mobile/library/cc/request_headers_builder.cc
+++ b/mobile/library/cc/request_headers_builder.cc
@@ -42,8 +42,10 @@ RequestHeadersBuilder::enableRequestCompression(CompressionAlgorithm algorithm)
   switch (algorithm) {
   case CompressionAlgorithm::Gzip:
     value = "gzip";
+    break;
   case CompressionAlgorithm::Brotli:
     value = "brotli";
+    break;
   }
   internalSet("x-envoy-mobile-compression", std::vector{value});
   return *this;
diff --git a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc
index 2918c7824e54..6e09a7f6a8c6 100644
--- a/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc
+++ b/mobile/library/common/extensions/cert_validator/platform_bridge/platform_bridge_cert_validator.cc
@@ -176,9 +176,11 @@ void PlatformBridgeCertValidator::onVerificationComplete(std::thread::id thread_
   case ValidationFailureType::FAIL_VERIFY_ERROR:
     detailed_status = Envoy::Ssl::ClientValidationStatus::Failed;
     stats_.fail_verify_error_.inc();
+    break;
   case ValidationFailureType::FAIL_VERIFY_SAN:
     detailed_status = Envoy::Ssl::ClientValidationStatus::Failed;
     stats_.fail_verify_san_.inc();
+    break;
   }
 
   job.result_callback_->onCertValidationResult(allow_untrusted_certificate_ || success,
diff --git a/source/common/http/http1/balsa_parser.cc b/source/common/http/http1/balsa_parser.cc
index 33082e1b120f..70d968ad0f11 100644
--- a/source/common/http/http1/balsa_parser.cc
+++ b/source/common/http/http1/balsa_parser.cc
@@ -329,7 +329,7 @@ void BalsaParser::OnChunkLength(size_t chunk_length) {
 
 void BalsaParser::OnChunkExtensionInput(absl::string_view /*input*/) {}
 
-void BalsaParser::OnInterimHeaders(BalsaHeaders /*headers*/) {}
+void BalsaParser::OnInterimHeaders(std::unique_ptr /*headers*/) {}
 
 void BalsaParser::HeaderDone() {
   if (status_ == ParserStatus::Error) {
diff --git a/source/common/http/http1/balsa_parser.h b/source/common/http/http1/balsa_parser.h
index 088051612523..90a4d23d59d4 100644
--- a/source/common/http/http1/balsa_parser.h
+++ b/source/common/http/http1/balsa_parser.h
@@ -51,7 +51,7 @@ class BalsaParser : public Parser, public quiche::BalsaVisitorInterface {
                                 absl::string_view reason_input) override;
   void OnChunkLength(size_t chunk_length) override;
   void OnChunkExtensionInput(absl::string_view input) override;
-  void OnInterimHeaders(quiche::BalsaHeaders headers) override;
+  void OnInterimHeaders(std::unique_ptr headers) override;
   void HeaderDone() override;
   void ContinueHeaderDone() override;
   void MessageDone() override;
diff --git a/source/common/quic/envoy_quic_client_connection.cc b/source/common/quic/envoy_quic_client_connection.cc
index 6a7d2159b148..9fd5f64bbe48 100644
--- a/source/common/quic/envoy_quic_client_connection.cc
+++ b/source/common/quic/envoy_quic_client_connection.cc
@@ -109,11 +109,7 @@ void EnvoyQuicClientConnection::switchConnectionSocket(
   // The old socket is not closed in this call, because it could still receive useful packets.
   setConnectionSocket(std::move(connection_socket));
   setUpConnectionSocket(*connectionSocket(), delegate_);
-  if (connection_migration_use_new_cid()) {
-    MigratePath(self_address, peer_address, writer.release(), true);
-  } else {
-    SetQuicPacketWriter(writer.release(), true);
-  }
+  MigratePath(self_address, peer_address, writer.release(), true);
 }
 
 void EnvoyQuicClientConnection::OnPathDegradingDetected() {
@@ -122,8 +118,7 @@ void EnvoyQuicClientConnection::OnPathDegradingDetected() {
 }
 
 void EnvoyQuicClientConnection::maybeMigratePort() {
-  if (!IsHandshakeConfirmed() || !connection_migration_use_new_cid() ||
-      HasPendingPathValidation() || !migrate_port_on_path_degrading_) {
+  if (!IsHandshakeConfirmed() || HasPendingPathValidation() || !migrate_port_on_path_degrading_) {
     return;
   }
 

From e101dcce15827082b1abb7886f4908f142a4284a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:09:27 +0100
Subject: [PATCH 590/740] build(deps): bump google.golang.org/grpc from 1.55.0
 to 1.56.0 in /examples/load-reporting-service (#28009)

build(deps): bump google.golang.org/grpc

Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.55.0 to 1.56.0.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.55.0...v1.56.0)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/load-reporting-service/go.mod |  2 +-
 examples/load-reporting-service/go.sum | 12 ++++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/examples/load-reporting-service/go.mod b/examples/load-reporting-service/go.mod
index a7de6e377294..82b25c6c30cb 100644
--- a/examples/load-reporting-service/go.mod
+++ b/examples/load-reporting-service/go.mod
@@ -5,5 +5,5 @@ go 1.13
 require (
 	github.com/envoyproxy/go-control-plane v0.11.1
 	github.com/golang/protobuf v1.5.3
-	google.golang.org/grpc v1.55.0
+	google.golang.org/grpc v1.56.0
 )
diff --git a/examples/load-reporting-service/go.sum b/examples/load-reporting-service/go.sum
index 3eb080a7e70a..9049678cee91 100644
--- a/examples/load-reporting-service/go.sum
+++ b/examples/load-reporting-service/go.sum
@@ -171,6 +171,7 @@ cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvj
 cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
 cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs=
 cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU=
+cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE=
 cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
@@ -632,8 +633,9 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74 h1:zlUubfBUxApscKFsF4VSvvfhsBNTBu0eF/ddvpo96yk=
 github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -650,12 +652,14 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34=
 github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
+github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q=
 github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM=
 github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo=
 github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w=
 github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
+github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss=
 github.com/envoyproxy/protoc-gen-validate v1.0.1 h1:kt9FtLiooDc0vbwTLhdg3dyNX1K9Qwa1EK9LcD4jVUQ=
 github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
@@ -1015,6 +1019,7 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
 golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
+golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1415,6 +1420,7 @@ google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVix
 google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
 google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
 google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
+google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
 google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY=
 google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw=
 google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk=
@@ -1463,10 +1469,12 @@ google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD
 google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
 google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
 google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
+google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
 google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
 google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
-google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
+google.golang.org/grpc v1.56.0 h1:+y7Bs8rtMd07LeXmL3NxcTLn7mUkbKZqEpPhMNkwJEE=
+google.golang.org/grpc v1.56.0/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

From 5d45ef76e4fa8089768f960196e5f47a0107b4ed Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:09:49 +0100
Subject: [PATCH 591/740] build(deps): bump postgres from `2642aec` to
 `7c0ee16` in /examples/shared/postgres (#28028)

build(deps): bump postgres in /examples/shared/postgres

Bumps postgres from `2642aec` to `7c0ee16`.

---
updated-dependencies:
- dependency-name: postgres
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/postgres/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile
index b4f150cc54f5..ef335b50a226 100644
--- a/examples/shared/postgres/Dockerfile
+++ b/examples/shared/postgres/Dockerfile
@@ -1,3 +1,3 @@
-FROM postgres:latest@sha256:2642aecef7a61b5eafd45138c5b380845db1be3ab9dfaaa5133660341bc203e8
+FROM postgres:latest@sha256:7c0ee16b6a3b4403957ece2c186ff05c57097a557403ae5216ef1286e47c249c
 COPY docker-healthcheck.sh /usr/local/bin/
 HEALTHCHECK CMD ["docker-healthcheck.sh"]

From dc43d016cebf94e30e907a7f5520358ad1460dac Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:10:31 +0100
Subject: [PATCH 592/740] build(deps): bump node from `873d0db` to `4c4d193` in
 /examples/shared/node (#28030)

build(deps): bump node in /examples/shared/node

Bumps node from `873d0db` to `4c4d193`.

---
updated-dependencies:
- dependency-name: node
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/node/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile
index 2f5b1867b05e..15867ba7c493 100644
--- a/examples/shared/node/Dockerfile
+++ b/examples/shared/node/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:20.3-bullseye-slim@sha256:873d0db3312a942fd77d99117d2dbfc7e38c8cf51ab3a2157aa98ec5e9197ad8 as node-base
+FROM node:20.3-bullseye-slim@sha256:4c4d1930c335191ebcf049eec6a4d35571b1fb9468ab0b8a403724c1a6d23f58 as node-base
 
 
 FROM node-base as node-http-auth

From f6af7f69c8e19ac82bae990063b28ae8edfb6f8c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:10:50 +0100
Subject: [PATCH 593/740] build(deps): bump apache/skywalking-ui from `782fb1c`
 to `d973f29` in /examples/skywalking (#28031)

build(deps): bump apache/skywalking-ui in /examples/skywalking

Bumps apache/skywalking-ui from `782fb1c` to `d973f29`.

---
updated-dependencies:
- dependency-name: apache/skywalking-ui
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/skywalking/Dockerfile-skywalking-ui | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/skywalking/Dockerfile-skywalking-ui b/examples/skywalking/Dockerfile-skywalking-ui
index b518d4eb7d37..96bcb8f64d45 100644
--- a/examples/skywalking/Dockerfile-skywalking-ui
+++ b/examples/skywalking/Dockerfile-skywalking-ui
@@ -1 +1 @@
-FROM apache/skywalking-ui:latest@sha256:782fb1c9a854d57b844202c5177e66eacda66e1b15fbb489fd94d089691d32e3
+FROM apache/skywalking-ui:latest@sha256:d973f2938f8f1d6d1028475c0a80190df312d7e38cb81b52fa2f7e98d28bce14

From 65070758c67f0271f5672ad33ff670968487aba9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:11:32 +0100
Subject: [PATCH 594/740] build(deps): bump apache/skywalking-oap-server from
 `8514d57` to `d6e31cf` in /examples/skywalking (#28032)

build(deps): bump apache/skywalking-oap-server in /examples/skywalking

Bumps apache/skywalking-oap-server from `8514d57` to `d6e31cf`.

---
updated-dependencies:
- dependency-name: apache/skywalking-oap-server
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/skywalking/Dockerfile-skywalking-oap | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/skywalking/Dockerfile-skywalking-oap b/examples/skywalking/Dockerfile-skywalking-oap
index f327b80df2e1..5967c86e5275 100644
--- a/examples/skywalking/Dockerfile-skywalking-oap
+++ b/examples/skywalking/Dockerfile-skywalking-oap
@@ -1 +1 @@
-FROM apache/skywalking-oap-server:latest@sha256:8514d57fcca408615a31c7b0fe7ac4aff396e5eb4e0740e0092b1e84865c5df0
+FROM apache/skywalking-oap-server:latest@sha256:d6e31cfa23bd8f0fd2a69e1fa9b80199f28a44eab514076e02b332fed1064f41

From 0ffde243543999ce5af9d0dda4d7db5ff8aeaeee Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:12:13 +0100
Subject: [PATCH 595/740] build(deps): bump golang from `8e927ec` to `a3598b9`
 in /examples/shared/golang (#28034)

build(deps): bump golang in /examples/shared/golang

Bumps golang from `8e927ec` to `a3598b9`.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/golang/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile
index aeeefbcdf60c..d0f1c2fb01f2 100644
--- a/examples/shared/golang/Dockerfile
+++ b/examples/shared/golang/Dockerfile
@@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache
 
 
-FROM golang:1.20.5-bullseye@sha256:8e927ecc8ba530bde779a125b95d2d7f0c9c7ef63df3d492d2e99dd599f69ab8 as golang-base
+FROM golang:1.20.5-bullseye@sha256:a3598b93d32819f1759893c532fa186bc61d58f1ced9aa49c2c77fe13383159a as golang-base
 
 
 FROM golang-base as golang-control-plane-builder

From 9e1ea5245140817f0d0f6589cb95e6cc404c1e47 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:12:38 +0100
Subject: [PATCH 596/740] build(deps): bump python from `03cec81` to `91d194f`
 in /examples/shared/python (#28035)

build(deps): bump python in /examples/shared/python

Bumps python from `03cec81` to `91d194f`.

---
updated-dependencies:
- dependency-name: python
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/python/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/python/Dockerfile b/examples/shared/python/Dockerfile
index cd1cce759148..cf515867e6be 100644
--- a/examples/shared/python/Dockerfile
+++ b/examples/shared/python/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11.4-slim-bullseye@sha256:03cec818d5c99fd643011552ad955952fae5c9bf17cbd9fca5980f0b59662a64 as python-base
+FROM python:3.11.4-slim-bullseye@sha256:91d194f58f50594cda71dcd2e8fdefd90e7ecc57d07823813b67c8521e565dcd as python-base
 RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache
 ARG PYTHON_REQUIREMENTS_FILE=aiohttp/requirements.txt

From 7f16449e7e42feeb4a7b2543ea8b093a5f1d5ded Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:13:14 +0100
Subject: [PATCH 597/740] build(deps): bump yapf from 0.40.0 to 0.40.1 in
 /tools/base (#28044)

Bumps [yapf](https://github.com/google/yapf) from 0.40.0 to 0.40.1.
- [Changelog](https://github.com/google/yapf/blob/main/CHANGELOG)
- [Commits](https://github.com/google/yapf/compare/v0.40.0...v0.40.1)

---
updated-dependencies:
- dependency-name: yapf
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tools/base/requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt
index b7b24775a20c..83eec8c19c4e 100644
--- a/tools/base/requirements.txt
+++ b/tools/base/requirements.txt
@@ -1403,9 +1403,9 @@ yamllint==1.32.0 \
     --hash=sha256:d01dde008c65de5b235188ab3110bebc59d18e5c65fc8a58267cd211cd9df34a \
     --hash=sha256:d97a66e48da820829d96077d76b8dfbe6c6140f106e558dae87e81ac4e6b30b7
     # via envoy-code-check
-yapf==0.40.0 \
-    --hash=sha256:63386557876c2686cb29d8c422b2d1e484f29f7a278821f91515b5c5adf569a7 \
-    --hash=sha256:7eeb8c404e386f16e24cbd785103dbc573f51cbb68e65a35f4392e0233f3d7bc
+yapf==0.40.1 \
+    --hash=sha256:958587eb5c8ec6c860119a9c25d02addf30a44f75aa152a4220d30e56a98037c \
+    --hash=sha256:b8bfc1f280949153e795181768ca14ef43d7312629a06c43e7abd279323af313
     # via
     #   -r requirements.in
     #   envoy-code-check

From 8eca2a0f9845b945165513a996a2568d4d82065a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:14:01 +0100
Subject: [PATCH 598/740] build(deps): bump setuptools from 67.8.0 to 68.0.0 in
 /tools/base (#28043)

Bumps [setuptools](https://github.com/pypa/setuptools) from 67.8.0 to 68.0.0.
- [Release notes](https://github.com/pypa/setuptools/releases)
- [Changelog](https://github.com/pypa/setuptools/blob/main/CHANGES.rst)
- [Commits](https://github.com/pypa/setuptools/compare/v67.8.0...v68.0.0)

---
updated-dependencies:
- dependency-name: setuptools
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tools/base/requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt
index 83eec8c19c4e..cea15224c3ea 100644
--- a/tools/base/requirements.txt
+++ b/tools/base/requirements.txt
@@ -1538,9 +1538,9 @@ zstandard==0.21.0 \
     # via envoy-base-utils
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==67.8.0 \
-    --hash=sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f \
-    --hash=sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102
+setuptools==68.0.0 \
+    --hash=sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f \
+    --hash=sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235
     # via
     #   -r requirements.in
     #   sphinxcontrib-jquery

From 7a0f911571ef8d7de7fe7ab7c7d0779080cd0e22 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:14:48 +0100
Subject: [PATCH 599/740] build(deps): bump jaegertracing/all-in-one from
 `fc197c4` to `9620b1b` in /examples/shared/jaeger (#28058)

build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger

Bumps jaegertracing/all-in-one from `fc197c4` to `9620b1b`.

---
updated-dependencies:
- dependency-name: jaegertracing/all-in-one
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/jaeger/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile
index e2eeeaf1f0cd..7c1bf0dbf411 100644
--- a/examples/shared/jaeger/Dockerfile
+++ b/examples/shared/jaeger/Dockerfile
@@ -1,4 +1,4 @@
-FROM jaegertracing/all-in-one@sha256:fc197c49334b8f82cb4b4aba0557d3b0a4364f78867f3abaa19a1e16b4af6019
+FROM jaegertracing/all-in-one@sha256:9620b1b576e90706a9906d6c5a463fb7c2c59c458c43eeaa2338f09444d4e4e7
 HEALTHCHECK \
     --interval=1s \
     --timeout=1s \

From c54662a555e3afba3464d56c079da017cecbde04 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 09:15:16 +0100
Subject: [PATCH 600/740] build(deps): bump otel/opentelemetry-collector from
 `324e2c7` to `51e6d8e` in /examples/opentelemetry (#28059)

build(deps): bump otel/opentelemetry-collector

Bumps otel/opentelemetry-collector from `324e2c7` to `51e6d8e`.

---
updated-dependencies:
- dependency-name: otel/opentelemetry-collector
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/opentelemetry/Dockerfile-opentelemetry | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry
index 50132701f29f..a1655b42a1f7 100644
--- a/examples/opentelemetry/Dockerfile-opentelemetry
+++ b/examples/opentelemetry/Dockerfile-opentelemetry
@@ -1,7 +1,7 @@
 FROM alpine:3.18@sha256:82d1e9d7ed48a7523bdebc18cf6290bdb97b82302a8a9c27d4fe885949ea94d1 as otelc_curl
 RUN apk --update add curl
 
-FROM otel/opentelemetry-collector:latest@sha256:324e2c7bdd1ecd58c1a1347737174371b4917d71a05cc4403ce3fb83bcddf836
+FROM otel/opentelemetry-collector:latest@sha256:51e6d8edd1ed33d328af83d5b99bd5551a034959c71bd81a973f71d21133bc7e
 
 COPY --from=otelc_curl / /
 

From 500611dfb32f3ca76c9ad518e2978ee161c9fbea Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 11:05:56 +0100
Subject: [PATCH 601/740] build(deps): bump debian from `7606bef` to `924df86`
 in /examples/shared/websocket (#27932)

build(deps): bump debian in /examples/shared/websocket

Bumps debian from `7606bef` to `924df86`.

---
updated-dependencies:
- dependency-name: debian
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/websocket/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/websocket/Dockerfile b/examples/shared/websocket/Dockerfile
index 3e047c58941f..14cb6c938378 100644
--- a/examples/shared/websocket/Dockerfile
+++ b/examples/shared/websocket/Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:bullseye-slim@sha256:7606bef5684b393434f06a50a3d1a09808fee5a0240d37da5d181b1b121e7637 as websocket-base
+FROM debian:bullseye-slim@sha256:924df86f8aad741a0134b2de7d8e70c5c6863f839caadef62609c1be1340daf5 as websocket-base
 ENV DEBIAN_FRONTEND=noninteractive
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \

From f567f66fa68b1533e9daffbdd5c5fd31ee65cba5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 21 Jun 2023 11:06:17 +0100
Subject: [PATCH 602/740] build(deps): bump yapf from 0.33.0 to 0.40.1 in
 /tools/code_format (#28042)

Bumps [yapf](https://github.com/google/yapf) from 0.33.0 to 0.40.1.
- [Changelog](https://github.com/google/yapf/blob/main/CHANGELOG)
- [Commits](https://github.com/google/yapf/compare/v0.33.0...v0.40.1)

---
updated-dependencies:
- dependency-name: yapf
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tools/code_format/requirements.txt | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/tools/code_format/requirements.txt b/tools/code_format/requirements.txt
index c1edabd5334a..9a8e045c4176 100644
--- a/tools/code_format/requirements.txt
+++ b/tools/code_format/requirements.txt
@@ -10,6 +10,10 @@ flake8==4.0.1 \
     # via
     #   -r requirements.in
     #   pep8-naming
+importlib-metadata==6.7.0 \
+    --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \
+    --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5
+    # via yapf
 mccabe==0.6.1 \
     --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
     --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
@@ -18,6 +22,10 @@ pep8-naming==0.13.2 \
     --hash=sha256:59e29e55c478db69cffbe14ab24b5bd2cd615c0413edf790d47d3fb7ba9a4e23 \
     --hash=sha256:93eef62f525fd12a6f8c98f4dcc17fa70baae2f37fa1f73bec00e3e44392fa48
     # via -r requirements.in
+platformdirs==3.7.0 \
+    --hash=sha256:87fbf6473e87c078d536980ba970a472422e94f17b752cfad17024c18876d481 \
+    --hash=sha256:cfd065ba43133ff103ab3bd10aecb095c2a0035fcd1f07217c9376900d94ba07
+    # via yapf
 pycodestyle==2.8.0 \
     --hash=sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20 \
     --hash=sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f
@@ -32,7 +40,11 @@ tomli==2.0.1 \
     --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
     --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
     # via yapf
-yapf==0.33.0 \
-    --hash=sha256:4c2b59bd5ffe46f3a7da48df87596877189148226ce267c16e8b44240e51578d \
-    --hash=sha256:da62bdfea3df3673553351e6246abed26d9fe6780e548a5af9e70f6d2b4f5b9a
+yapf==0.40.1 \
+    --hash=sha256:958587eb5c8ec6c860119a9c25d02addf30a44f75aa152a4220d30e56a98037c \
+    --hash=sha256:b8bfc1f280949153e795181768ca14ef43d7312629a06c43e7abd279323af313
     # via -r requirements.in
+zipp==3.15.0 \
+    --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \
+    --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556
+    # via importlib-metadata

From 3a68249d6b5039f7df5c3f7d740af20a89d97b53 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Wed, 21 Jun 2023 15:47:15 +0100
Subject: [PATCH 603/740] deps: Ignore CVE-2023-32732 in scanner (#28069)

Signed-off-by: Ryan Northey 
---
 tools/dependency/cve.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/dependency/cve.yaml b/tools/dependency/cve.yaml
index 8fff58bc9ed7..b49e2a832f58 100644
--- a/tools/dependency/cve.yaml
+++ b/tools/dependency/cve.yaml
@@ -49,3 +49,6 @@ ignored_cves:
 - CVE-2021-22939
 - CVE-2021-22940
 - CVE-2021-43803
+# This CVE assumes an untrusted client that can affect a shared connection between an intervening proxy and
+# gRPC server, and does not affect Envoy directly
+- CVE-2023-32732

From 12d359e08eba5d44f8d1f02fddb4548dac7fc185 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Wed, 21 Jun 2023 16:44:54 +0100
Subject: [PATCH 604/740] deps: Bump `rules_python` -> 0.23.1 (#28062)

Signed-off-by: Ryan Northey 
---
 bazel/repository_locations.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
index acfececa4ee3..a8d709d840cb 100644
--- a/bazel/repository_locations.bzl
+++ b/bazel/repository_locations.bzl
@@ -884,9 +884,9 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "Python rules for Bazel",
         project_desc = "Bazel rules for the Python language",
         project_url = "https://github.com/bazelbuild/rules_python",
-        version = "0.22.0",
-        sha256 = "863ba0fa944319f7e3d695711427d9ad80ba92c6edd0b7c7443b84e904689539",
-        release_date = "2023-05-25",
+        version = "0.23.1",
+        sha256 = "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841",
+        release_date = "2023-06-13",
         strip_prefix = "rules_python-{version}",
         urls = ["https://github.com/bazelbuild/rules_python/archive/{version}.tar.gz"],
         use_category = ["build"],

From c63546c52c3d43a32f1e69720a1d2c2f8148e938 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Wed, 21 Jun 2023 16:45:14 +0100
Subject: [PATCH 605/740] deps: Bump `rules_license` -> 0.0.7 (#28064)

Signed-off-by: Ryan Northey 
---
 bazel/repository_locations.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
index a8d709d840cb..eca1847b03de 100644
--- a/bazel/repository_locations.bzl
+++ b/bazel/repository_locations.bzl
@@ -1366,11 +1366,11 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "rules_license",
         project_desc = "Bazel rules for checking open source licenses",
         project_url = "https://github.com/bazelbuild/rules_license",
-        version = "0.0.4",
-        sha256 = "6157e1e68378532d0241ecd15d3c45f6e5cfd98fc10846045509fb2a7cc9e381",
+        version = "0.0.7",
+        sha256 = "4531deccb913639c30e5c7512a054d5d875698daeb75d8cf90f284375fe7c360",
         urls = ["https://github.com/bazelbuild/rules_license/releases/download/{version}/rules_license-{version}.tar.gz"],
         use_category = ["build", "dataplane_core", "controlplane"],
-        release_date = "2022-11-02",
+        release_date = "2023-06-16",
         cpe = "N/A",
         license = "Apache-2.0",
         license_url = "https://github.com/bazelbuild/rules_license/blob/{version}/LICENSE",

From 842d1b2ae9b7f73a5055f2117df6f7d086b9e40d Mon Sep 17 00:00:00 2001
From: Greg Greenway 
Date: Wed, 21 Jun 2023 09:08:44 -0700
Subject: [PATCH 606/740] upstream: added new option `weighted_priority_health`
 (#27883)

This computes the health of a priority level by using load balancing weight
instead of the count of healthy hosts.

Signed-off-by: Greg Greenway 
---
 api/envoy/config/endpoint/v3/endpoint.proto   | 12 +++-
 .../endpoint/v3/endpoint_components.proto     |  4 +-
 changelogs/current.yaml                       |  7 ++
 envoy/upstream/upstream.h                     | 13 +++-
 .../common/upstream/cluster_manager_impl.cc   | 17 +++--
 source/common/upstream/cluster_manager_impl.h |  4 +-
 .../upstream/health_discovery_service.cc      |  4 +-
 source/common/upstream/load_balancer_impl.cc  | 47 ++++++++++++--
 source/common/upstream/load_balancer_impl.h   |  8 +--
 source/common/upstream/upstream_impl.cc       | 30 ++++++---
 source/common/upstream/upstream_impl.h        | 17 ++++-
 .../extensions/clusters/aggregate/cluster.cc  |  3 +-
 .../clusters/dynamic_forward_proxy/cluster.cc |  2 +-
 source/extensions/clusters/eds/eds.cc         | 41 ++++++------
 source/extensions/clusters/eds/eds.h          |  5 +-
 .../logical_dns/logical_dns_cluster.cc        |  2 +-
 .../original_dst/original_dst_cluster.cc      |  4 +-
 .../clusters/redis/redis_cluster.cc           |  2 +-
 .../clusters/redis/redis_cluster_lb.h         |  4 +-
 .../clusters/static/static_cluster.cc         |  3 +-
 .../clusters/static/static_cluster.h          |  1 +
 .../clusters/strict_dns/strict_dns_cluster.cc |  4 +-
 .../clusters/strict_dns/strict_dns_cluster.h  |  1 +
 .../maglev/maglev_lb.h                        |  3 +-
 .../ring_hash/ring_hash_lb.h                  |  3 +-
 .../subset/subset_lb.cc                       | 16 +++--
 .../subset/subset_lb.h                        |  5 +-
 .../upstream/cluster_manager_impl_test.cc     | 51 +++++++--------
 .../upstream/load_balancer_impl_test.cc       | 64 +++++++++++++++++++
 test/common/upstream/upstream_impl_test.cc    | 16 +++--
 .../clusters/aggregate/cluster_test.cc        |  4 +-
 .../clusters/aggregate/cluster_update_test.cc |  8 +--
 test/integration/eds_integration_test.cc      | 30 +++++++++
 test/mocks/upstream/host_set.h                |  2 +
 test/mocks/upstream/priority_set.h            |  3 +-
 35 files changed, 322 insertions(+), 118 deletions(-)

diff --git a/api/envoy/config/endpoint/v3/endpoint.proto b/api/envoy/config/endpoint/v3/endpoint.proto
index 7edfb66c9a82..6cdc179d8931 100644
--- a/api/envoy/config/endpoint/v3/endpoint.proto
+++ b/api/envoy/config/endpoint/v3/endpoint.proto
@@ -35,7 +35,7 @@ message ClusterLoadAssignment {
   option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.ClusterLoadAssignment";
 
   // Load balancing policy settings.
-  // [#next-free-field: 6]
+  // [#next-free-field: 7]
   message Policy {
     option (udpa.annotations.versioning).previous_message_type =
         "envoy.api.v2.ClusterLoadAssignment.Policy";
@@ -99,6 +99,16 @@ message ClusterLoadAssignment {
     // are considered stale and should be marked unhealthy.
     // Defaults to 0 which means endpoints never go stale.
     google.protobuf.Duration endpoint_stale_after = 4 [(validate.rules).duration = {gt {}}];
+
+    // If true, use the :ref:`load balancing weight
+    // ` of healthy and unhealthy
+    // hosts to determine the health of the priority level. If false, use the number of healthy and unhealthy hosts
+    // to determine the health of the priority level, or in other words assume each host has a weight of 1 for
+    // this calculation.
+    //
+    // Note: this is not currently implemented for
+    // :ref:`locality weighted load balancing `.
+    bool weighted_priority_health = 6;
   }
 
   // Name of the cluster. This will be the :ref:`service_name
diff --git a/api/envoy/config/endpoint/v3/endpoint_components.proto b/api/envoy/config/endpoint/v3/endpoint_components.proto
index 62598f5d0f31..c9572fd8a11d 100644
--- a/api/envoy/config/endpoint/v3/endpoint_components.proto
+++ b/api/envoy/config/endpoint/v3/endpoint_components.proto
@@ -194,9 +194,9 @@ message LocalityLbEndpoints {
   // default to the highest priority (0).
   //
   // Under usual circumstances, Envoy will only select endpoints for the highest
-  // priority (0). In the event all endpoints for a particular priority are
+  // priority (0). In the event that enough endpoints for a particular priority are
   // unavailable/unhealthy, Envoy will fail over to selecting endpoints for the
-  // next highest priority group.
+  // next highest priority group. Read more at :ref:`priority levels `.
   //
   // Priorities should range from 0 (highest) to N (lowest) without skipping.
   uint32 priority = 5 [(validate.rules).uint32 = {lte: 128}];
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index c97d9784d738..3ada7ca49727 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -354,6 +354,13 @@ new_features:
   change: |
     added new field ``envoy.extensions.filters.http.fault.v3.HTTPFault.filter_metadata`` to aid in logging.
     Metadata will be stored in StreamInfo dynamic metadata under a namespace corresponding to the name of the fault filter.
+- area: load_balancing
+  change: |
+    added new option
+    :ref:`weighted_priority_health `
+    to compute the health of a :ref:`priority level ` by using
+    :ref:`load balancing weight `
+    instead of the count of healthy hosts.
 - area: application_logs
   change: |
     Added bootstrap option
diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h
index 21303beb2b29..348e586510cd 100644
--- a/envoy/upstream/upstream.h
+++ b/envoy/upstream/upstream.h
@@ -452,6 +452,11 @@ class HostSet {
    * @return uint32_t the overprovisioning factor of this host set.
    */
   virtual uint32_t overprovisioningFactor() const PURE;
+
+  /**
+   * @return true to use host weights to calculate the health of a priority.
+   */
+  virtual bool weightedPriorityHealth() const PURE;
 };
 
 using HostSetPtr = std::unique_ptr;
@@ -526,13 +531,15 @@ class PrioritySet {
    * @param locality_weights supplies a map from locality to associated weight.
    * @param hosts_added supplies the hosts added since the last update.
    * @param hosts_removed supplies the hosts removed since the last update.
-   * @param overprovisioning_factor if presents, overwrites the current overprovisioning_factor.
+   * @param weighted_priority_health if present, overwrites the current weighted_priority_health.
+   * @param overprovisioning_factor if present, overwrites the current overprovisioning_factor.
    * @param cross_priority_host_map read only cross-priority host map which is created in the main
    * thread and shared by all the worker threads.
    */
   virtual void updateHosts(uint32_t priority, UpdateHostsParams&& update_hosts_params,
                            LocalityWeightsConstSharedPtr locality_weights,
                            const HostVector& hosts_added, const HostVector& hosts_removed,
+                           absl::optional weighted_priority_health,
                            absl::optional overprovisioning_factor,
                            HostMapConstSharedPtr cross_priority_host_map = nullptr) PURE;
 
@@ -550,11 +557,13 @@ class PrioritySet {
      * @param locality_weights supplies a map from locality to associated weight.
      * @param hosts_added supplies the hosts added since the last update.
      * @param hosts_removed supplies the hosts removed since the last update.
-     * @param overprovisioning_factor if presents, overwrites the current overprovisioning_factor.
+     * @param weighted_priority_health if present, overwrites the current weighted_priority_health.
+     * @param overprovisioning_factor if present, overwrites the current overprovisioning_factor.
      */
     virtual void updateHosts(uint32_t priority, UpdateHostsParams&& update_hosts_params,
                              LocalityWeightsConstSharedPtr locality_weights,
                              const HostVector& hosts_added, const HostVector& hosts_removed,
+                             absl::optional weighted_priority_health,
                              absl::optional overprovisioning_factor) PURE;
   };
 
diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc
index b4bd365e63e6..5229d60b68ee 100644
--- a/source/common/upstream/cluster_manager_impl.cc
+++ b/source/common/upstream/cluster_manager_impl.cc
@@ -1114,6 +1114,7 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_
         cm_cluster.cluster().prioritySet().hostSetsPerPriority()[per_priority.priority_];
     per_priority.update_hosts_params_ = HostSetImpl::updateHostsParams(*host_set);
     per_priority.locality_weights_ = host_set->localityWeights();
+    per_priority.weighted_priority_health_ = host_set->weightedPriorityHealth();
     per_priority.overprovisioning_factor_ = host_set->overprovisioningFactor();
   }
 
@@ -1140,7 +1141,7 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_
       cluster_manager->updateClusterMembership(
           info->name(), per_priority.priority_, per_priority.update_hosts_params_,
           per_priority.locality_weights_, per_priority.hosts_added_, per_priority.hosts_removed_,
-          per_priority.overprovisioning_factor_, map);
+          per_priority.weighted_priority_health_, per_priority.overprovisioning_factor_, map);
     }
 
     if (new_cluster != nullptr) {
@@ -1215,13 +1216,14 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::updateHost
     const std::string& name, uint32_t priority,
     PrioritySet::UpdateHostsParams&& update_hosts_params,
     LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added,
-    const HostVector& hosts_removed, absl::optional overprovisioning_factor,
+    const HostVector& hosts_removed, absl::optional weighted_priority_health,
+    absl::optional overprovisioning_factor,
     HostMapConstSharedPtr cross_priority_host_map) {
   ENVOY_LOG(debug, "membership update for TLS cluster {} added {} removed {}", name,
             hosts_added.size(), hosts_removed.size());
   priority_set_.updateHosts(priority, std::move(update_hosts_params), std::move(locality_weights),
-                            hosts_added, hosts_removed, overprovisioning_factor,
-                            std::move(cross_priority_host_map));
+                            hosts_added, hosts_removed, weighted_priority_health,
+                            overprovisioning_factor, std::move(cross_priority_host_map));
   // If an LB is thread aware, create a new worker local LB on membership changes.
   if (lb_factory_ != nullptr) {
     ENVOY_LOG(debug, "re-creating local LB for TLS cluster {}", name);
@@ -1485,13 +1487,14 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::removeHosts(
 void ClusterManagerImpl::ThreadLocalClusterManagerImpl::updateClusterMembership(
     const std::string& name, uint32_t priority, PrioritySet::UpdateHostsParams update_hosts_params,
     LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added,
-    const HostVector& hosts_removed, uint64_t overprovisioning_factor,
-    HostMapConstSharedPtr cross_priority_host_map) {
+    const HostVector& hosts_removed, bool weighted_priority_health,
+    uint64_t overprovisioning_factor, HostMapConstSharedPtr cross_priority_host_map) {
   ASSERT(thread_local_clusters_.find(name) != thread_local_clusters_.end());
   const auto& cluster_entry = thread_local_clusters_[name];
   cluster_entry->updateHosts(name, priority, std::move(update_hosts_params),
                              std::move(locality_weights), hosts_added, hosts_removed,
-                             overprovisioning_factor, std::move(cross_priority_host_map));
+                             weighted_priority_health, overprovisioning_factor,
+                             std::move(cross_priority_host_map));
 }
 
 void ClusterManagerImpl::ThreadLocalClusterManagerImpl::onHostHealthFailure(
diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h
index 440554a7ea7f..84ea8b79dbd4 100644
--- a/source/common/upstream/cluster_manager_impl.h
+++ b/source/common/upstream/cluster_manager_impl.h
@@ -377,6 +377,7 @@ class ClusterManagerImpl : public ClusterManager,
       LocalityWeightsConstSharedPtr locality_weights_;
       // Keep small members (bools and enums) at the end of class, to reduce alignment overhead.
       const uint32_t priority_;
+      bool weighted_priority_health_;
       uint32_t overprovisioning_factor_;
     };
 
@@ -533,6 +534,7 @@ class ClusterManagerImpl : public ClusterManager,
                        PrioritySet::UpdateHostsParams&& update_hosts_params,
                        LocalityWeightsConstSharedPtr locality_weights,
                        const HostVector& hosts_added, const HostVector& hosts_removed,
+                       absl::optional weighted_priority_health,
                        absl::optional overprovisioning_factor,
                        HostMapConstSharedPtr cross_priority_host_map);
 
@@ -623,7 +625,7 @@ class ClusterManagerImpl : public ClusterManager,
                                  PrioritySet::UpdateHostsParams update_hosts_params,
                                  LocalityWeightsConstSharedPtr locality_weights,
                                  const HostVector& hosts_added, const HostVector& hosts_removed,
-                                 uint64_t overprovisioning_factor,
+                                 bool weighted_priority_health, uint64_t overprovisioning_factor,
                                  HostMapConstSharedPtr cross_priority_host_map);
     void onHostHealthFailure(const HostSharedPtr& host);
 
diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc
index bb6d798ca682..659a701b18b7 100644
--- a/source/common/upstream/health_discovery_service.cc
+++ b/source/common/upstream/health_discovery_service.cc
@@ -521,7 +521,7 @@ void HdsCluster::updateHosts(
   hosts_per_locality_ =
       std::make_shared(std::move(hosts_by_locality), false);
   priority_set_.updateHosts(0, HostSetImpl::partitionHosts(hosts_, hosts_per_locality_), {},
-                            hosts_added, hosts_removed, absl::nullopt);
+                            hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
 }
 
 ClusterSharedPtr HdsCluster::create() { return nullptr; }
@@ -571,7 +571,7 @@ void HdsCluster::initialize(std::function callback) {
     }
     // Use the ungrouped and grouped hosts lists to retain locality structure in the priority set.
     priority_set_.updateHosts(0, HostSetImpl::partitionHosts(hosts_, hosts_per_locality_), {},
-                              *hosts_, {}, absl::nullopt);
+                              *hosts_, {}, absl::nullopt, absl::nullopt);
 
     initialized_ = true;
   }
diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc
index 708578764cb2..25dea7d51b0c 100644
--- a/source/common/upstream/load_balancer_impl.cc
+++ b/source/common/upstream/load_balancer_impl.cc
@@ -196,17 +196,50 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority,
   const auto host_count = host_set.hosts().size() - host_set.excludedHosts().size();
 
   if (host_count > 0) {
-    // Each priority level's health is ratio of healthy hosts to total number of hosts in a priority
-    // multiplied by overprovisioning factor of 1.4 and capped at 100%. It means that if all
-    // hosts are healthy that priority's health is 100%*1.4=140% and is capped at 100% which results
-    // in 100%. If 80% of hosts are healthy, that priority's health is still 100% (80%*1.4=112% and
-    // capped at 100%).
+    uint64_t healthy_weight = 0;
+    uint64_t degraded_weight = 0;
+    uint64_t total_weight = 0;
+    if (host_set.weightedPriorityHealth()) {
+      for (const auto& host : host_set.healthyHosts()) {
+        healthy_weight += host->weight();
+      }
+
+      for (const auto& host : host_set.degradedHosts()) {
+        degraded_weight += host->weight();
+      }
+
+      for (const auto& host : host_set.hosts()) {
+        total_weight += host->weight();
+      }
+
+      uint64_t excluded_weight = 0;
+      for (const auto& host : host_set.excludedHosts()) {
+        excluded_weight += host->weight();
+      }
+      ASSERT(total_weight >= excluded_weight);
+      total_weight -= excluded_weight;
+    } else {
+      healthy_weight = host_set.healthyHosts().size();
+      degraded_weight = host_set.degradedHosts().size();
+      total_weight = host_count;
+    }
+    // Each priority level's health is ratio of healthy hosts to total number of hosts in a
+    // priority multiplied by overprovisioning factor of 1.4 and capped at 100%. It means that if
+    // all hosts are healthy that priority's health is 100%*1.4=140% and is capped at 100% which
+    // results in 100%. If 80% of hosts are healthy, that priority's health is still 100%
+    // (80%*1.4=112% and capped at 100%).
     per_priority_health.get()[priority] = std::min(
-        100, (host_set.overprovisioningFactor() * host_set.healthyHosts().size() / host_count));
+        100, (host_set.overprovisioningFactor() * healthy_weight / total_weight));
 
     // We perform the same computation for degraded hosts.
     per_priority_degraded.get()[priority] = std::min(
-        100, (host_set.overprovisioningFactor() * host_set.degradedHosts().size() / host_count));
+        100, (host_set.overprovisioningFactor() * degraded_weight / total_weight));
+
+    ENVOY_LOG(trace,
+              "recalculated priority state: priority level {}, healthy weight {}, total weight {}, "
+              "overprovision factor {}, healthy result {}, degraded result {}",
+              priority, healthy_weight, total_weight, host_set.overprovisioningFactor(),
+              per_priority_health.get()[priority], per_priority_degraded.get()[priority]);
   }
 
   // Now that we've updated health for the changed priority level, we need to calculate percentage
diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h
index 48b45a7ac4ca..51035fe5b609 100644
--- a/source/common/upstream/load_balancer_impl.h
+++ b/source/common/upstream/load_balancer_impl.h
@@ -82,7 +82,7 @@ class LoadBalancerConfigHelper {
 /**
  * Base class for all LB implementations.
  */
-class LoadBalancerBase : public LoadBalancer {
+class LoadBalancerBase : public LoadBalancer, protected Logger::Loggable {
 public:
   enum class HostAvailability { Healthy, Degraded };
 
@@ -457,8 +457,7 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase {
  * This base class also supports unweighted selection which derived classes can use to customize
  * behavior. Derived classes can also override how host weight is determined when in weighted mode.
  */
-class EdfLoadBalancerBase : public ZoneAwareLoadBalancerBase,
-                            Logger::Loggable {
+class EdfLoadBalancerBase : public ZoneAwareLoadBalancerBase {
 public:
   using SlowStartConfig = envoy::extensions::load_balancing_policies::common::v3::SlowStartConfig;
 
@@ -706,8 +705,7 @@ class LeastRequestLoadBalancer : public EdfLoadBalancerBase {
 /**
  * Random load balancer that picks a random host out of all hosts.
  */
-class RandomLoadBalancer : public ZoneAwareLoadBalancerBase,
-                           Logger::Loggable {
+class RandomLoadBalancer : public ZoneAwareLoadBalancerBase {
 public:
   RandomLoadBalancer(const PrioritySet& priority_set, const PrioritySet* local_priority_set,
                      ClusterLbStats& stats, Runtime::Loader& runtime,
diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc
index 232e32a807b0..0956bfe13252 100644
--- a/source/common/upstream/upstream_impl.cc
+++ b/source/common/upstream/upstream_impl.cc
@@ -579,7 +579,11 @@ std::vector HostsPerLocalityImpl::filter(
 void HostSetImpl::updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_params,
                               LocalityWeightsConstSharedPtr locality_weights,
                               const HostVector& hosts_added, const HostVector& hosts_removed,
+                              absl::optional weighted_priority_health,
                               absl::optional overprovisioning_factor) {
+  if (weighted_priority_health.has_value()) {
+    weighted_priority_health_ = weighted_priority_health.value();
+  }
   if (overprovisioning_factor.has_value()) {
     ASSERT(overprovisioning_factor.value() > 0);
     overprovisioning_factor_ = overprovisioning_factor.value();
@@ -594,6 +598,7 @@ void HostSetImpl::updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_para
   excluded_hosts_per_locality_ = std::move(update_hosts_params.excluded_hosts_per_locality);
   locality_weights_ = std::move(locality_weights);
 
+  // TODO(ggreenway): implement `weighted_priority_health` support in `rebuildLocalityScheduler`.
   rebuildLocalityScheduler(healthy_locality_scheduler_, healthy_locality_entries_,
                            *healthy_hosts_per_locality_, healthy_hosts_->get(), hosts_per_locality_,
                            excluded_hosts_per_locality_, locality_weights_,
@@ -737,10 +742,11 @@ double HostSetImpl::effectiveLocalityWeight(uint32_t index,
 
 const HostSet&
 PrioritySetImpl::getOrCreateHostSet(uint32_t priority,
+                                    absl::optional weighted_priority_health,
                                     absl::optional overprovisioning_factor) {
   if (host_sets_.size() < priority + 1) {
     for (size_t i = host_sets_.size(); i <= priority; ++i) {
-      HostSetImplPtr host_set = createHostSet(i, overprovisioning_factor);
+      HostSetImplPtr host_set = createHostSet(i, weighted_priority_health, overprovisioning_factor);
       host_sets_priority_update_cbs_.push_back(
           host_set->addPriorityUpdateCb([this](uint32_t priority, const HostVector& hosts_added,
                                                const HostVector& hosts_removed) {
@@ -755,6 +761,7 @@ PrioritySetImpl::getOrCreateHostSet(uint32_t priority,
 void PrioritySetImpl::updateHosts(uint32_t priority, UpdateHostsParams&& update_hosts_params,
                                   LocalityWeightsConstSharedPtr locality_weights,
                                   const HostVector& hosts_added, const HostVector& hosts_removed,
+                                  absl::optional weighted_priority_health,
                                   absl::optional overprovisioning_factor,
                                   HostMapConstSharedPtr cross_priority_host_map) {
   // Update cross priority host map first. In this way, when the update callbacks of the priority
@@ -764,10 +771,10 @@ void PrioritySetImpl::updateHosts(uint32_t priority, UpdateHostsParams&& update_
   }
 
   // Ensure that we have a HostSet for the given priority.
-  getOrCreateHostSet(priority, overprovisioning_factor);
+  getOrCreateHostSet(priority, weighted_priority_health, overprovisioning_factor);
   static_cast(host_sets_[priority].get())
       ->updateHosts(std::move(update_hosts_params), std::move(locality_weights), hosts_added,
-                    hosts_removed, overprovisioning_factor);
+                    hosts_removed, weighted_priority_health, overprovisioning_factor);
 
   if (!batch_update_) {
     runUpdateCallbacks(hosts_added, hosts_removed);
@@ -790,7 +797,8 @@ void PrioritySetImpl::batchHostUpdate(BatchUpdateCb& callback) {
 void PrioritySetImpl::BatchUpdateScope::updateHosts(
     uint32_t priority, PrioritySet::UpdateHostsParams&& update_hosts_params,
     LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added,
-    const HostVector& hosts_removed, absl::optional overprovisioning_factor) {
+    const HostVector& hosts_removed, absl::optional weighted_priority_health,
+    absl::optional overprovisioning_factor) {
   // We assume that each call updates a different priority.
   ASSERT(priorities_.find(priority) == priorities_.end());
   priorities_.insert(priority);
@@ -804,13 +812,14 @@ void PrioritySetImpl::BatchUpdateScope::updateHosts(
   }
 
   parent_.updateHosts(priority, std::move(update_hosts_params), locality_weights, hosts_added,
-                      hosts_removed, overprovisioning_factor);
+                      hosts_removed, weighted_priority_health, overprovisioning_factor);
 }
 
 void MainPrioritySetImpl::updateHosts(uint32_t priority, UpdateHostsParams&& update_hosts_params,
                                       LocalityWeightsConstSharedPtr locality_weights,
                                       const HostVector& hosts_added,
                                       const HostVector& hosts_removed,
+                                      absl::optional weighted_priority_health,
                                       absl::optional overprovisioning_factor,
                                       HostMapConstSharedPtr cross_priority_host_map) {
   ASSERT(cross_priority_host_map == nullptr,
@@ -818,7 +827,8 @@ void MainPrioritySetImpl::updateHosts(uint32_t priority, UpdateHostsParams&& upd
   updateCrossPriorityHostMap(hosts_added, hosts_removed);
 
   PrioritySetImpl::updateHosts(priority, std::move(update_hosts_params), locality_weights,
-                               hosts_added, hosts_removed, overprovisioning_factor);
+                               hosts_added, hosts_removed, weighted_priority_health,
+                               overprovisioning_factor);
 }
 
 HostMapConstSharedPtr MainPrioritySetImpl::crossPriorityHostMap() const {
@@ -1618,7 +1628,7 @@ void ClusterImplBase::reloadHealthyHostsHelper(const HostSharedPtr&) {
     HostsPerLocalityConstSharedPtr hosts_per_locality_copy = host_set->hostsPerLocality().clone();
     prioritySet().updateHosts(priority,
                               HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy),
-                              host_set->localityWeights(), {}, {}, absl::nullopt);
+                              host_set->localityWeights(), {}, {}, absl::nullopt, absl::nullopt);
   }
 }
 
@@ -1883,6 +1893,7 @@ void PriorityStateManager::updateClusterPrioritySet(
     const uint32_t priority, HostVectorSharedPtr&& current_hosts,
     const absl::optional& hosts_added, const absl::optional& hosts_removed,
     const absl::optional health_checker_flag,
+    absl::optional weighted_priority_health,
     absl::optional overprovisioning_factor) {
   // If local locality is not defined then skip populating per locality hosts.
   const auto& local_locality = local_info_node_.locality();
@@ -1958,12 +1969,13 @@ void PriorityStateManager::updateClusterPrioritySet(
   if (update_cb_ != nullptr) {
     update_cb_->updateHosts(priority, HostSetImpl::partitionHosts(hosts, per_locality_shared),
                             std::move(locality_weights), hosts_added.value_or(*hosts),
-                            hosts_removed.value_or({}), overprovisioning_factor);
+                            hosts_removed.value_or({}), weighted_priority_health,
+                            overprovisioning_factor);
   } else {
     parent_.prioritySet().updateHosts(
         priority, HostSetImpl::partitionHosts(hosts, per_locality_shared),
         std::move(locality_weights), hosts_added.value_or(*hosts),
-        hosts_removed.value_or({}), overprovisioning_factor);
+        hosts_removed.value_or({}), weighted_priority_health, overprovisioning_factor);
   }
 }
 
diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h
index 276225475197..eae0be2961a0 100644
--- a/source/common/upstream/upstream_impl.h
+++ b/source/common/upstream/upstream_impl.h
@@ -506,10 +506,12 @@ class HostsPerLocalityImpl : public HostsPerLocality {
  */
 class HostSetImpl : public HostSet {
 public:
-  HostSetImpl(uint32_t priority, absl::optional overprovisioning_factor)
+  HostSetImpl(uint32_t priority, absl::optional weighted_priority_health,
+              absl::optional overprovisioning_factor)
       : priority_(priority), overprovisioning_factor_(overprovisioning_factor.has_value()
                                                           ? overprovisioning_factor.value()
                                                           : kDefaultOverProvisioningFactor),
+        weighted_priority_health_(weighted_priority_health.value_or(false)),
         hosts_(new HostVector()), healthy_hosts_(new HealthyHostVector()),
         degraded_hosts_(new DegradedHostVector()), excluded_hosts_(new ExcludedHostVector()) {}
 
@@ -559,6 +561,7 @@ class HostSetImpl : public HostSet {
   absl::optional chooseDegradedLocality() override;
   uint32_t priority() const override { return priority_; }
   uint32_t overprovisioningFactor() const override { return overprovisioning_factor_; }
+  bool weightedPriorityHealth() const override { return weighted_priority_health_; }
 
   static PrioritySet::UpdateHostsParams
   updateHostsParams(HostVectorConstSharedPtr hosts,
@@ -576,6 +579,7 @@ class HostSetImpl : public HostSet {
   void updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_params,
                    LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added,
                    const HostVector& hosts_removed,
+                   absl::optional weighted_priority_health = absl::nullopt,
                    absl::optional overprovisioning_factor = absl::nullopt);
 
 protected:
@@ -595,6 +599,7 @@ class HostSetImpl : public HostSet {
 
   uint32_t priority_;
   uint32_t overprovisioning_factor_;
+  bool weighted_priority_health_;
   HostVectorConstSharedPtr hosts_;
   HealthyHostVectorConstSharedPtr healthy_hosts_;
   DegradedHostVectorConstSharedPtr degraded_hosts_;
@@ -668,11 +673,13 @@ class PrioritySetImpl : public PrioritySet {
   // Get the host set for this priority level, creating it if necessary.
   const HostSet&
   getOrCreateHostSet(uint32_t priority,
+                     absl::optional weighted_priority_health = absl::nullopt,
                      absl::optional overprovisioning_factor = absl::nullopt);
 
   void updateHosts(uint32_t priority, UpdateHostsParams&& update_hosts_params,
                    LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added,
                    const HostVector& hosts_removed,
+                   absl::optional weighted_priority_health = absl::nullopt,
                    absl::optional overprovisioning_factor = absl::nullopt,
                    HostMapConstSharedPtr cross_priority_host_map = nullptr) override;
 
@@ -685,8 +692,10 @@ class PrioritySetImpl : public PrioritySet {
 protected:
   // Allows subclasses of PrioritySetImpl to create their own type of HostSetImpl.
   virtual HostSetImplPtr createHostSet(uint32_t priority,
+                                       absl::optional weighted_priority_health,
                                        absl::optional overprovisioning_factor) {
-    return std::make_unique(priority, overprovisioning_factor);
+    return std::make_unique(priority, weighted_priority_health,
+                                         overprovisioning_factor);
   }
 
 protected:
@@ -728,7 +737,7 @@ class PrioritySetImpl : public PrioritySet {
 
     void updateHosts(uint32_t priority, PrioritySet::UpdateHostsParams&& update_hosts_params,
                      LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added,
-                     const HostVector& hosts_removed,
+                     const HostVector& hosts_removed, absl::optional weighted_priority_health,
                      absl::optional overprovisioning_factor) override;
 
     absl::node_hash_set all_hosts_added_;
@@ -750,6 +759,7 @@ class MainPrioritySetImpl : public PrioritySetImpl, public Logger::Loggable weighted_priority_health = absl::nullopt,
                    absl::optional overprovisioning_factor = absl::nullopt,
                    HostMapConstSharedPtr cross_priority_host_map = nullptr) override;
   HostMapConstSharedPtr crossPriorityHostMap() const override;
@@ -1250,6 +1260,7 @@ class PriorityStateManager : protected Logger::Loggable {
                            const absl::optional& hosts_added,
                            const absl::optional& hosts_removed,
                            const absl::optional health_checker_flag,
+                           absl::optional weighted_priority_health = absl::nullopt,
                            absl::optional overprovisioning_factor = absl::nullopt);
 
   // Returns the saved priority state.
diff --git a/source/extensions/clusters/aggregate/cluster.cc b/source/extensions/clusters/aggregate/cluster.cc
index 29de9f7b3bc0..ca7d40cca6e5 100644
--- a/source/extensions/clusters/aggregate/cluster.cc
+++ b/source/extensions/clusters/aggregate/cluster.cc
@@ -86,7 +86,8 @@ AggregateClusterLoadBalancer::linearizePrioritySet(OptRef exc
       if (!host_set->hosts().empty()) {
         priority_context->priority_set_.updateHosts(
             next_priority_after_linearizing, Upstream::HostSetImpl::updateHostsParams(*host_set),
-            host_set->localityWeights(), host_set->hosts(), {}, host_set->overprovisioningFactor());
+            host_set->localityWeights(), host_set->hosts(), {}, host_set->weightedPriorityHealth(),
+            host_set->overprovisioningFactor());
         priority_context->priority_to_cluster_.emplace_back(
             std::make_pair(priority_in_current_cluster, tlc));
 
diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc
index ace4989c478c..7dd50e893775 100644
--- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc
+++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc
@@ -292,7 +292,7 @@ void Cluster::updatePriorityState(const Upstream::HostVector& hosts_added,
   }
   priority_state_manager.updateClusterPrioritySet(
       0, std::move(priority_state_manager.priorityState()[0].first), hosts_added, hosts_removed,
-      absl::nullopt, absl::nullopt);
+      absl::nullopt, absl::nullopt, absl::nullopt);
 }
 
 void Cluster::onDnsHostRemove(const std::string& host) {
diff --git a/source/extensions/clusters/eds/eds.cc b/source/extensions/clusters/eds/eds.cc
index c9ba84898922..3341992dc8af 100644
--- a/source/extensions/clusters/eds/eds.cc
+++ b/source/extensions/clusters/eds/eds.cc
@@ -77,6 +77,8 @@ void EdsClusterImpl::BatchUpdateHelper::batchUpdate(PrioritySet::HostUpdateCb& h
 
   const uint32_t overprovisioning_factor = PROTOBUF_GET_WRAPPED_OR_DEFAULT(
       cluster_load_assignment_.policy(), overprovisioning_factor, kDefaultOverProvisioningFactor);
+  const bool weighted_priority_health =
+      cluster_load_assignment_.policy().weighted_priority_health();
 
   LocalityWeightsMap empty_locality_map;
 
@@ -88,14 +90,16 @@ void EdsClusterImpl::BatchUpdateHelper::batchUpdate(PrioritySet::HostUpdateCb& h
     }
     if (priority_state[i].first != nullptr) {
       cluster_rebuilt |= parent_.updateHostsPerLocality(
-          i, overprovisioning_factor, *priority_state[i].first, parent_.locality_weights_map_[i],
-          priority_state[i].second, priority_state_manager, *all_hosts, all_new_hosts);
+          i, weighted_priority_health, overprovisioning_factor, *priority_state[i].first,
+          parent_.locality_weights_map_[i], priority_state[i].second, priority_state_manager,
+          *all_hosts, all_new_hosts);
     } else {
       // If the new update contains a priority with no hosts, call the update function with an empty
       // set of hosts.
-      cluster_rebuilt |= parent_.updateHostsPerLocality(
-          i, overprovisioning_factor, {}, parent_.locality_weights_map_[i], empty_locality_map,
-          priority_state_manager, *all_hosts, all_new_hosts);
+      cluster_rebuilt |=
+          parent_.updateHostsPerLocality(i, weighted_priority_health, overprovisioning_factor, {},
+                                         parent_.locality_weights_map_[i], empty_locality_map,
+                                         priority_state_manager, *all_hosts, all_new_hosts);
     }
   }
 
@@ -107,8 +111,8 @@ void EdsClusterImpl::BatchUpdateHelper::batchUpdate(PrioritySet::HostUpdateCb& h
       parent_.locality_weights_map_.resize(i + 1);
     }
     cluster_rebuilt |= parent_.updateHostsPerLocality(
-        i, overprovisioning_factor, {}, parent_.locality_weights_map_[i], empty_locality_map,
-        priority_state_manager, *all_hosts, all_new_hosts);
+        i, weighted_priority_health, overprovisioning_factor, {}, parent_.locality_weights_map_[i],
+        empty_locality_map, priority_state_manager, *all_hosts, all_new_hosts);
   }
 
   if (!cluster_rebuilt) {
@@ -325,17 +329,17 @@ void EdsClusterImpl::reloadHealthyHostsHelper(const HostSharedPtr& host) {
     HostsPerLocalityConstSharedPtr hosts_per_locality_copy = host_set->hostsPerLocality().filter(
         {[&host_to_exclude](const Host& host) { return &host != host_to_exclude.get(); }})[0];
 
-    prioritySet().updateHosts(priority,
-                              HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy),
-                              host_set->localityWeights(), {}, hosts_to_remove, absl::nullopt);
+    prioritySet().updateHosts(
+        priority, HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy),
+        host_set->localityWeights(), {}, hosts_to_remove, absl::nullopt, absl::nullopt);
   }
 }
 
 bool EdsClusterImpl::updateHostsPerLocality(
-    const uint32_t priority, const uint32_t overprovisioning_factor, const HostVector& new_hosts,
-    LocalityWeightsMap& locality_weights_map, LocalityWeightsMap& new_locality_weights_map,
-    PriorityStateManager& priority_state_manager, const HostMap& all_hosts,
-    const absl::flat_hash_set& all_new_hosts) {
+    const uint32_t priority, bool weighted_priority_health, const uint32_t overprovisioning_factor,
+    const HostVector& new_hosts, LocalityWeightsMap& locality_weights_map,
+    LocalityWeightsMap& new_locality_weights_map, PriorityStateManager& priority_state_manager,
+    const HostMap& all_hosts, const absl::flat_hash_set& all_new_hosts) {
   const auto& host_set = priority_set_.getOrCreateHostSet(priority, overprovisioning_factor);
   HostVectorSharedPtr current_hosts_copy(new HostVector(host_set.hosts()));
 
@@ -353,7 +357,8 @@ bool EdsClusterImpl::updateHostsPerLocality(
   // about this. In the future we may need to do better here.
   const bool hosts_updated = updateDynamicHostList(new_hosts, *current_hosts_copy, hosts_added,
                                                    hosts_removed, all_hosts, all_new_hosts);
-  if (hosts_updated || host_set.overprovisioningFactor() != overprovisioning_factor ||
+  if (hosts_updated || host_set.weightedPriorityHealth() != weighted_priority_health ||
+      host_set.overprovisioningFactor() != overprovisioning_factor ||
       locality_weights_map != new_locality_weights_map) {
     ASSERT(std::all_of(current_hosts_copy->begin(), current_hosts_copy->end(),
                        [&](const auto& host) { return host->priority() == priority; }));
@@ -362,9 +367,9 @@ bool EdsClusterImpl::updateHostsPerLocality(
               "EDS hosts or locality weights changed for cluster: {} current hosts {} priority {}",
               info_->name(), host_set.hosts().size(), host_set.priority());
 
-    priority_state_manager.updateClusterPrioritySet(priority, std::move(current_hosts_copy),
-                                                    hosts_added, hosts_removed, absl::nullopt,
-                                                    overprovisioning_factor);
+    priority_state_manager.updateClusterPrioritySet(
+        priority, std::move(current_hosts_copy), hosts_added, hosts_removed, absl::nullopt,
+        weighted_priority_health, overprovisioning_factor);
     return true;
   }
   return false;
diff --git a/source/extensions/clusters/eds/eds.h b/source/extensions/clusters/eds/eds.h
index b362cd07325e..3d4cfbd8de55 100644
--- a/source/extensions/clusters/eds/eds.h
+++ b/source/extensions/clusters/eds/eds.h
@@ -48,8 +48,9 @@ class EdsClusterImpl
                             const EnvoyException* e) override;
   using LocalityWeightsMap = absl::node_hash_map;
-  bool updateHostsPerLocality(const uint32_t priority, const uint32_t overprovisioning_factor,
-                              const HostVector& new_hosts, LocalityWeightsMap& locality_weights_map,
+  bool updateHostsPerLocality(const uint32_t priority, bool weighted_priority_health,
+                              const uint32_t overprovisioning_factor, const HostVector& new_hosts,
+                              LocalityWeightsMap& locality_weights_map,
                               LocalityWeightsMap& new_locality_weights_map,
                               PriorityStateManager& priority_state_manager,
                               const HostMap& all_hosts,
diff --git a/source/extensions/clusters/logical_dns/logical_dns_cluster.cc b/source/extensions/clusters/logical_dns/logical_dns_cluster.cc
index 4bf4e0d71371..a392a0ec8ae8 100644
--- a/source/extensions/clusters/logical_dns/logical_dns_cluster.cc
+++ b/source/extensions/clusters/logical_dns/logical_dns_cluster.cc
@@ -139,7 +139,7 @@ void LogicalDnsCluster::startResolve() {
             const uint32_t priority = locality_lb_endpoint.priority();
             priority_state_manager.updateClusterPrioritySet(
                 priority, std::move(priority_state_manager.priorityState()[priority].first),
-                absl::nullopt, absl::nullopt, absl::nullopt);
+                absl::nullopt, absl::nullopt, absl::nullopt, absl::nullopt, absl::nullopt);
           }
 
           if (!current_resolved_address_ ||
diff --git a/source/extensions/clusters/original_dst/original_dst_cluster.cc b/source/extensions/clusters/original_dst/original_dst_cluster.cc
index e2b91ce1844d..882e728578fa 100644
--- a/source/extensions/clusters/original_dst/original_dst_cluster.cc
+++ b/source/extensions/clusters/original_dst/original_dst_cluster.cc
@@ -188,7 +188,7 @@ void OriginalDstCluster::addHost(HostSharedPtr& host) {
   all_hosts->emplace_back(host);
   priority_set_.updateHosts(0,
                             HostSetImpl::partitionHosts(all_hosts, HostsPerLocalityImpl::empty()),
-                            {}, {std::move(host)}, {}, absl::nullopt);
+                            {}, {std::move(host)}, {}, absl::nullopt, absl::nullopt);
 }
 
 void OriginalDstCluster::cleanup() {
@@ -259,7 +259,7 @@ void OriginalDstCluster::cleanup() {
     setHostMap(new_host_map);
     priority_set_.updateHosts(
         0, HostSetImpl::partitionHosts(keeping_hosts, HostsPerLocalityImpl::empty()), {}, {},
-        to_be_removed, absl::nullopt);
+        to_be_removed, false, absl::nullopt);
   }
 
   cleanup_timer_->enableTimer(cleanup_interval_ms_);
diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc
index 6e261b1221f2..e6d11ff49e28 100644
--- a/source/extensions/clusters/redis/redis_cluster.cc
+++ b/source/extensions/clusters/redis/redis_cluster.cc
@@ -87,7 +87,7 @@ void RedisCluster::updateAllHosts(const Upstream::HostVector& hosts_added,
 
   priority_state_manager.updateClusterPrioritySet(
       current_priority, std::move(priority_state_manager.priorityState()[current_priority].first),
-      hosts_added, hosts_removed, absl::nullopt);
+      hosts_added, hosts_removed, absl::nullopt, absl::nullopt, absl::nullopt);
 }
 
 void RedisCluster::onClusterSlotUpdate(ClusterSlotsSharedPtr&& slots) {
diff --git a/source/extensions/clusters/redis/redis_cluster_lb.h b/source/extensions/clusters/redis/redis_cluster_lb.h
index 8f94b8aa033d..66b00e0897a4 100644
--- a/source/extensions/clusters/redis/redis_cluster_lb.h
+++ b/source/extensions/clusters/redis/redis_cluster_lb.h
@@ -170,8 +170,8 @@ class RedisClusterLoadBalancerFactory : public ClusterSlotUpdateCallBack,
 
   private:
     const Upstream::HostConstSharedPtr primary_;
-    Upstream::HostSetImpl replicas_{0, absl::nullopt};
-    Upstream::HostSetImpl all_hosts_{0, absl::nullopt};
+    Upstream::HostSetImpl replicas_{0, absl::nullopt, absl::nullopt};
+    Upstream::HostSetImpl all_hosts_{0, absl::nullopt, absl::nullopt};
   };
 
   using RedisShardSharedPtr = std::shared_ptr;
diff --git a/source/extensions/clusters/static/static_cluster.cc b/source/extensions/clusters/static/static_cluster.cc
index 025f761c51ce..e53d67ca23f9 100644
--- a/source/extensions/clusters/static/static_cluster.cc
+++ b/source/extensions/clusters/static/static_cluster.cc
@@ -16,6 +16,7 @@ StaticClusterImpl::StaticClusterImpl(const envoy::config::cluster::v3::Cluster&
       cluster.load_assignment();
   overprovisioning_factor_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(
       cluster_load_assignment.policy(), overprovisioning_factor, kDefaultOverProvisioningFactor);
+  weighted_priority_health_ = cluster_load_assignment.policy().weighted_priority_health();
 
   Event::Dispatcher& dispatcher = context.serverFactoryContext().mainThreadDispatcher();
 
@@ -51,7 +52,7 @@ void StaticClusterImpl::startPreInit() {
     }
     priority_state_manager_->updateClusterPrioritySet(
         i, std::move(priority_state[i].first), absl::nullopt, absl::nullopt, health_checker_flag,
-        overprovisioning_factor_);
+        weighted_priority_health_, overprovisioning_factor_);
   }
   priority_state_manager_.reset();
 
diff --git a/source/extensions/clusters/static/static_cluster.h b/source/extensions/clusters/static/static_cluster.h
index e02e0fde8d51..b5a5578cbeba 100644
--- a/source/extensions/clusters/static/static_cluster.h
+++ b/source/extensions/clusters/static/static_cluster.h
@@ -26,6 +26,7 @@ class StaticClusterImpl : public ClusterImplBase {
 
   PriorityStateManagerPtr priority_state_manager_;
   uint32_t overprovisioning_factor_;
+  bool weighted_priority_health_;
 };
 
 /**
diff --git a/source/extensions/clusters/strict_dns/strict_dns_cluster.cc b/source/extensions/clusters/strict_dns/strict_dns_cluster.cc
index 6e44f712c9ad..38dd9c8c3039 100644
--- a/source/extensions/clusters/strict_dns/strict_dns_cluster.cc
+++ b/source/extensions/clusters/strict_dns/strict_dns_cluster.cc
@@ -42,6 +42,7 @@ StrictDnsClusterImpl::StrictDnsClusterImpl(const envoy::config::cluster::v3::Clu
 
   overprovisioning_factor_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(
       load_assignment_.policy(), overprovisioning_factor, kDefaultOverProvisioningFactor);
+  weighted_priority_health_ = load_assignment_.policy().weighted_priority_health();
 }
 
 void StrictDnsClusterImpl::startPreInit() {
@@ -77,7 +78,8 @@ void StrictDnsClusterImpl::updateAllHosts(const HostVector& hosts_added,
   // TODO(dio): Add assertion in here.
   priority_state_manager.updateClusterPrioritySet(
       current_priority, std::move(priority_state_manager.priorityState()[current_priority].first),
-      hosts_added, hosts_removed, absl::nullopt, overprovisioning_factor_);
+      hosts_added, hosts_removed, absl::nullopt, weighted_priority_health_,
+      overprovisioning_factor_);
 }
 
 StrictDnsClusterImpl::ResolveTarget::ResolveTarget(
diff --git a/source/extensions/clusters/strict_dns/strict_dns_cluster.h b/source/extensions/clusters/strict_dns/strict_dns_cluster.h
index c1601713afbe..c635e0dd9052 100644
--- a/source/extensions/clusters/strict_dns/strict_dns_cluster.h
+++ b/source/extensions/clusters/strict_dns/strict_dns_cluster.h
@@ -68,6 +68,7 @@ class StrictDnsClusterImpl : public BaseDynamicClusterImpl {
   const bool respect_dns_ttl_;
   Network::DnsLookupFamily dns_lookup_family_;
   uint32_t overprovisioning_factor_;
+  bool weighted_priority_health_;
 };
 
 /**
diff --git a/source/extensions/load_balancing_policies/maglev/maglev_lb.h b/source/extensions/load_balancing_policies/maglev/maglev_lb.h
index 8efa539fa7e4..01d98c421673 100644
--- a/source/extensions/load_balancing_policies/maglev/maglev_lb.h
+++ b/source/extensions/load_balancing_policies/maglev/maglev_lb.h
@@ -144,8 +144,7 @@ class CompactMaglevTable : public MaglevTable {
 /**
  * Thread aware load balancer implementation for Maglev.
  */
-class MaglevLoadBalancer : public ThreadAwareLoadBalancerBase,
-                           Logger::Loggable {
+class MaglevLoadBalancer : public ThreadAwareLoadBalancerBase {
 public:
   MaglevLoadBalancer(const PrioritySet& priority_set, ClusterLbStats& stats, Stats::Scope& scope,
                      Runtime::Loader& runtime, Random::RandomGenerator& random,
diff --git a/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h b/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h
index 421753b52586..6ad5902f50fe 100644
--- a/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h
+++ b/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h
@@ -39,8 +39,7 @@ struct RingHashLoadBalancerStats {
  * 2) Per-zone rings and optional zone aware routing (not all applications will want this).
  * 3) Max request fallback to support hot shards (not all applications will want this).
  */
-class RingHashLoadBalancer : public ThreadAwareLoadBalancerBase,
-                             Logger::Loggable {
+class RingHashLoadBalancer : public ThreadAwareLoadBalancerBase {
 public:
   RingHashLoadBalancer(const PrioritySet& priority_set, ClusterLbStats& stats, Stats::Scope& scope,
                        Runtime::Loader& runtime, Random::RandomGenerator& random,
diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.cc b/source/extensions/load_balancing_policies/subset/subset_lb.cc
index f9e6e8d863c3..ead745e73020 100644
--- a/source/extensions/load_balancing_policies/subset/subset_lb.cc
+++ b/source/extensions/load_balancing_policies/subset/subset_lb.cc
@@ -828,11 +828,12 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostHashSet& matching_host
   auto excluded_hosts_per_locality =
       original_host_set_.excludedHostsPerLocality().filter({cached_predicate})[0];
 
-  HostSetImpl::updateHosts(
-      HostSetImpl::updateHostsParams(
-          hosts, hosts_per_locality, healthy_hosts, healthy_hosts_per_locality, degraded_hosts,
-          degraded_hosts_per_locality, excluded_hosts, excluded_hosts_per_locality),
-      determineLocalityWeights(*hosts_per_locality), hosts_added, hosts_removed, absl::nullopt);
+  HostSetImpl::updateHosts(HostSetImpl::updateHostsParams(
+                               hosts, hosts_per_locality, healthy_hosts, healthy_hosts_per_locality,
+                               degraded_hosts, degraded_hosts_per_locality, excluded_hosts,
+                               excluded_hosts_per_locality),
+                           determineLocalityWeights(*hosts_per_locality), hosts_added,
+                           hosts_removed, absl::nullopt, absl::nullopt);
 }
 
 LocalityWeightsConstSharedPtr SubsetLoadBalancer::HostSubsetImpl::determineLocalityWeights(
@@ -870,7 +871,8 @@ LocalityWeightsConstSharedPtr SubsetLoadBalancer::HostSubsetImpl::determineLocal
 }
 
 HostSetImplPtr SubsetLoadBalancer::PrioritySubsetImpl::createHostSet(
-    uint32_t priority, absl::optional overprovisioning_factor) {
+    uint32_t priority, absl::optional weighted_priority_health,
+    absl::optional overprovisioning_factor) {
   // Use original hostset's overprovisioning_factor.
   RELEASE_ASSERT(priority < original_priority_set_.hostSetsPerPriority().size(), "");
 
@@ -878,6 +880,8 @@ HostSetImplPtr SubsetLoadBalancer::PrioritySubsetImpl::createHostSet(
 
   ASSERT(!overprovisioning_factor.has_value() ||
          overprovisioning_factor.value() == host_set->overprovisioningFactor());
+  ASSERT(!weighted_priority_health.has_value() ||
+         weighted_priority_health.value() == host_set->weightedPriorityHealth());
   return HostSetImplPtr{
       new HostSubsetImpl(*host_set, locality_weight_aware_, scale_locality_weight_)};
 }
diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.h b/source/extensions/load_balancing_policies/subset/subset_lb.h
index 8d1c0523b464..13a9c073de3a 100644
--- a/source/extensions/load_balancing_policies/subset/subset_lb.h
+++ b/source/extensions/load_balancing_policies/subset/subset_lb.h
@@ -203,7 +203,8 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable weighted_priority_health,
                                  absl::optional overprovisioning_factor) override;
 
   private:
diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc
index 19df7ab84ba7..6316c0aa51cc 100644
--- a/test/common/upstream/cluster_manager_impl_test.cc
+++ b/test/common/upstream/cluster_manager_impl_test.cc
@@ -2100,7 +2100,7 @@ TEST_F(ClusterManagerImplTest, HostsPostedToTlsCluster) {
 
   cluster1->priority_set_.updateHosts(
       0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {},
-      100);
+      true, 100);
 
   auto* tls_cluster = cluster_manager_->getThreadLocalCluster(cluster1->info_->name());
 
@@ -2110,6 +2110,7 @@ TEST_F(ClusterManagerImplTest, HostsPostedToTlsCluster) {
   EXPECT_EQ(1, tls_cluster->prioritySet().hostSetsPerPriority()[0]->healthyHosts().size());
   EXPECT_EQ(host3, tls_cluster->prioritySet().hostSetsPerPriority()[0]->healthyHosts()[0]);
   EXPECT_EQ(3, tls_cluster->prioritySet().hostSetsPerPriority()[0]->hosts().size());
+  EXPECT_TRUE(tls_cluster->prioritySet().hostSetsPerPriority()[0]->weightedPriorityHealth());
   EXPECT_EQ(100, tls_cluster->prioritySet().hostSetsPerPriority()[0]->overprovisioningFactor());
 
   factory_.tls_.shutdownThread();
@@ -3849,7 +3850,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value());
@@ -3860,12 +3861,12 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   cluster.prioritySet().updateHosts(
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value());
@@ -3883,7 +3884,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   EXPECT_EQ(2, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value());
@@ -3896,21 +3897,21 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
 
   (*hosts)[0]->healthFlagSet(Host::HealthFlag::FAILED_EDS_HEALTH);
   cluster.prioritySet().updateHosts(
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
 
   (*hosts)[0]->weight(100);
   cluster.prioritySet().updateHosts(
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
 
   // Updates not delivered yet.
   EXPECT_EQ(2, factory_.stats_.counter("cluster_manager.cluster_updated").value());
@@ -3923,7 +3924,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
 
   EXPECT_EQ(3, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
@@ -3959,7 +3960,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesOutOfWindow) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.update_out_of_merge_window").value());
@@ -3986,7 +3987,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesInsideWindow) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_out_of_merge_window").value());
@@ -4021,7 +4022,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesOutOfWindowDisabled) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_out_of_merge_window").value());
@@ -4091,7 +4092,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value());
@@ -4102,12 +4103,12 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   cluster.prioritySet().updateHosts(
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value());
@@ -4377,7 +4378,7 @@ TEST_F(ClusterManagerImplTest, CrossPriorityHostMapSyncTest) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
 
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
@@ -4394,7 +4395,7 @@ TEST_F(ClusterManagerImplTest, CrossPriorityHostMapSyncTest) {
       0,
       updateHostsParams(hosts, hosts_per_locality,
                         std::make_shared(*hosts), hosts_per_locality),
-      {}, hosts_added, hosts_removed, absl::nullopt);
+      {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
   EXPECT_EQ(2, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value());
@@ -5485,7 +5486,7 @@ TEST_F(ClusterManagerImplTest, DrainConnectionsPredicate) {
   // Sending non-mergeable updates.
   cluster.prioritySet().updateHosts(
       0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {},
-      100);
+      absl::nullopt, 100);
 
   // Using RR LB get a pool for each host.
   EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _))
@@ -5557,7 +5558,7 @@ TEST_F(ClusterManagerImplTest, ConnPoolsDrainedOnHostSetChange) {
   // Sending non-mergeable updates.
   cluster.prioritySet().updateHosts(
       0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {},
-      100);
+      absl::nullopt, 100);
 
   EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value());
   EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value());
@@ -5618,7 +5619,7 @@ TEST_F(ClusterManagerImplTest, ConnPoolsDrainedOnHostSetChange) {
   // This update should drain all connection pools (host1, host2).
   cluster.prioritySet().updateHosts(
       0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, {},
-      hosts_removed, 100);
+      hosts_removed, absl::nullopt, 100);
 
   // Recreate connection pool for host1.
   cp1 = HttpPoolDataPeer::getPool(
@@ -5647,7 +5648,7 @@ TEST_F(ClusterManagerImplTest, ConnPoolsDrainedOnHostSetChange) {
   // Adding host3 should drain connection pool for host1.
   cluster.prioritySet().updateHosts(
       0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr,
-      hosts_added, {}, 100);
+      hosts_added, {}, absl::nullopt, 100);
 }
 
 TEST_F(ClusterManagerImplTest, ConnPoolsNotDrainedOnHostSetChange) {
@@ -5682,7 +5683,7 @@ TEST_F(ClusterManagerImplTest, ConnPoolsNotDrainedOnHostSetChange) {
   // Sending non-mergeable updates.
   cluster.prioritySet().updateHosts(
       0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {},
-      100);
+      absl::nullopt, 100);
 
   EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _))
       .Times(1)
@@ -5716,7 +5717,7 @@ TEST_F(ClusterManagerImplTest, ConnPoolsNotDrainedOnHostSetChange) {
   // No connection pools should be drained.
   cluster.prioritySet().updateHosts(
       0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr,
-      hosts_added, {}, 100);
+      hosts_added, {}, absl::nullopt, 100);
 }
 
 TEST_F(ClusterManagerImplTest, ConnPoolsIdleDeleted) {
@@ -5753,7 +5754,7 @@ TEST_F(ClusterManagerImplTest, ConnPoolsIdleDeleted) {
   // Sending non-mergeable updates.
   cluster.prioritySet().updateHosts(
       0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {},
-      100);
+      absl::nullopt, 100);
 
   {
     auto* cp1 = new NiceMock();
@@ -6055,7 +6056,7 @@ class PreconnectTest : public ClusterManagerImplTest {
     // Sending non-mergeable updates.
     cluster_->prioritySet().updateHosts(
         0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts,
-        {}, 100);
+        {}, absl::nullopt, 100);
   }
 
   Cluster* cluster_{};
diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc
index 2ee53e4b010b..c8cb8231df6f 100644
--- a/test/common/upstream/load_balancer_impl_test.cc
+++ b/test/common/upstream/load_balancer_impl_test.cc
@@ -319,6 +319,70 @@ TEST_P(LoadBalancerBaseTest, OverProvisioningFactor) {
   ASSERT_THAT(getLoadPercentage(), ElementsAre(50, 50));
 }
 
+TEST_P(LoadBalancerBaseTest, WeightedPriorityHealth) {
+  host_set_.weighted_priority_health_ = true;
+  failover_host_set_.weighted_priority_health_ = true;
+
+  // Makes math easier to read.
+  host_set_.setOverprovisioningFactor(100);
+  failover_host_set_.setOverprovisioningFactor(100);
+
+  // Basic healthy/unhealthy test.
+  updateHostSet(host_set_, 4, 2, 0, 0);
+  updateHostSet(failover_host_set_, 1, 1);
+
+  // Total weight is 10, healthy weight is 6.
+  host_set_.hosts_[0]->weight(3); // Healthy
+  host_set_.hosts_[1]->weight(3); // Healthy
+  host_set_.hosts_[2]->weight(2); // Unhealthy
+  host_set_.hosts_[3]->weight(2); // Unhealthy
+  host_set_.runCallbacks({}, {});
+  ASSERT_THAT(getLoadPercentage(), ElementsAre(60, 40));
+}
+
+TEST_P(LoadBalancerBaseTest, WeightedPriorityHealthExcluded) {
+  host_set_.weighted_priority_health_ = true;
+  failover_host_set_.weighted_priority_health_ = true;
+
+  // Makes math easier to read.
+  host_set_.setOverprovisioningFactor(100);
+  failover_host_set_.setOverprovisioningFactor(100);
+
+  updateHostSet(failover_host_set_, 1, 1);
+  updateHostSet(host_set_, 3, 1, 0, 1);
+  host_set_.hosts_[0]->weight(4);  // Healthy
+  host_set_.hosts_[1]->weight(10); // Excluded
+  host_set_.hosts_[2]->weight(6);  // Unhealthy
+  host_set_.runCallbacks({}, {});
+  ASSERT_THAT(getLoadPercentage(), ElementsAre(40, 60));
+}
+
+TEST_P(LoadBalancerBaseTest, WeightedPriorityHealthDegraded) {
+  host_set_.weighted_priority_health_ = true;
+  failover_host_set_.weighted_priority_health_ = true;
+
+  // Makes math easier to read.
+  host_set_.setOverprovisioningFactor(100);
+  failover_host_set_.setOverprovisioningFactor(100);
+
+  updateHostSet(host_set_, 4, 1, 1, 0);
+  host_set_.hosts_[0]->weight(4); // Healthy
+  host_set_.hosts_[1]->weight(3); // Degraded
+  host_set_.hosts_[2]->weight(2); // Unhealthy
+  host_set_.hosts_[3]->weight(1); // Unhealthy
+  host_set_.runCallbacks({}, {});
+
+  updateHostSet(failover_host_set_, 2, 1, 1); // 1 healthy host, 1 degraded.
+  failover_host_set_.hosts_[0]->weight(1);    // Healthy
+  failover_host_set_.hosts_[1]->weight(9);    // Degraded
+  failover_host_set_.runCallbacks({}, {});
+
+  // 40% for healthy priority 0, 10% for healthy priority 1, 30% for degraded priority zero, and the
+  // remaining 20% to degraded priority 1.
+  ASSERT_THAT(getLoadPercentage(), ElementsAre(40, 10));
+  ASSERT_THAT(getDegradedLoadPercentage(), ElementsAre(30, 20));
+}
+
 TEST_P(LoadBalancerBaseTest, GentleFailover) {
   // With 100% of P=0 hosts healthy, P=0 gets all the load.
   // None of the levels is in Panic mode
diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc
index 1b63edb01691..f954fdb3736f 100644
--- a/test/common/upstream/upstream_impl_test.cc
+++ b/test/common/upstream/upstream_impl_test.cc
@@ -762,6 +762,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) {
 
     load_assignment:
       policy:
+        weighted_priority_health: true
         overprovisioning_factor: 100
       endpoints:
       - lb_endpoints:
@@ -840,6 +841,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) {
       ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts())));
   EXPECT_EQ("localhost1", cluster.prioritySet().hostSetsPerPriority()[0]->hosts()[0]->hostname());
   EXPECT_EQ("localhost1", cluster.prioritySet().hostSetsPerPriority()[0]->hosts()[1]->hostname());
+  EXPECT_TRUE(cluster.prioritySet().hostSetsPerPriority()[0]->weightedPriorityHealth());
   EXPECT_EQ(100, cluster.prioritySet().hostSetsPerPriority()[0]->overprovisioningFactor());
   EXPECT_EQ(Host::Health::Degraded,
             cluster.prioritySet().hostSetsPerPriority()[0]->hosts()[0]->coarseHealth());
@@ -857,6 +859,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) {
   EXPECT_THAT(
       std::list({"127.0.0.1:11001", "127.0.0.2:11001"}),
       ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts())));
+  EXPECT_TRUE(cluster.prioritySet().hostSetsPerPriority()[0]->weightedPriorityHealth());
   EXPECT_EQ(100, cluster.prioritySet().hostSetsPerPriority()[0]->overprovisioningFactor());
 
   // Since no change for localhost1, we expect no rebuild.
@@ -870,6 +873,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) {
   EXPECT_THAT(
       std::list({"127.0.0.1:11001", "127.0.0.2:11001"}),
       ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts())));
+  EXPECT_TRUE(cluster.prioritySet().hostSetsPerPriority()[0]->weightedPriorityHealth());
   EXPECT_EQ(100, cluster.prioritySet().hostSetsPerPriority()[0]->overprovisioningFactor());
 
   // Since no change for localhost1, we expect no rebuild.
@@ -1633,6 +1637,7 @@ TEST_F(StaticClusterImplTest, LoadAssignmentEmptyHostname) {
     lb_policy: ROUND_ROBIN
     load_assignment:
       policy:
+        weighted_priority_health: true
         overprovisioning_factor: 100
       endpoints:
       - lb_endpoints:
@@ -1656,6 +1661,7 @@ TEST_F(StaticClusterImplTest, LoadAssignmentEmptyHostname) {
 
   EXPECT_EQ(1UL, cluster.prioritySet().hostSetsPerPriority()[0]->healthyHosts().size());
   EXPECT_EQ("", cluster.prioritySet().hostSetsPerPriority()[0]->hosts()[0]->hostname());
+  EXPECT_TRUE(cluster.prioritySet().hostSetsPerPriority()[0]->weightedPriorityHealth());
   EXPECT_EQ(100, cluster.prioritySet().hostSetsPerPriority()[0]->overprovisioningFactor());
   EXPECT_FALSE(cluster.info()->addedViaApi());
 }
@@ -3215,7 +3221,7 @@ class TestBatchUpdateCb : public PrioritySet::BatchUpdateCb {
           updateHostsParams(hosts_, hosts_per_locality_,
                             std::make_shared(*hosts_),
                             hosts_per_locality_),
-          {}, hosts_added, hosts_removed, absl::nullopt);
+          {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
     }
 
     // Remove the host from P1.
@@ -3228,7 +3234,7 @@ class TestBatchUpdateCb : public PrioritySet::BatchUpdateCb {
           updateHostsParams(empty_hosts, HostsPerLocalityImpl::empty(),
                             std::make_shared(*empty_hosts),
                             HostsPerLocalityImpl::empty()),
-          {}, hosts_added, hosts_removed, absl::nullopt);
+          {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt);
     }
   }
 
@@ -3285,7 +3291,7 @@ TEST(PrioritySet, Extend) {
         1,
         updateHostsParams(hosts, hosts_per_locality,
                           std::make_shared(*hosts), hosts_per_locality),
-        {}, hosts_added, hosts_removed, absl::nullopt, fake_cross_priority_host_map);
+        {}, hosts_added, hosts_removed, absl::nullopt, absl::nullopt, fake_cross_priority_host_map);
   }
   EXPECT_EQ(1, priority_changes);
   EXPECT_EQ(1, membership_changes);
@@ -4885,7 +4891,7 @@ TEST_F(HostsWithLocalityImpl, Filter) {
 class HostSetImplLocalityTest : public Event::TestUsingSimulatedTime, public testing::Test {
 public:
   LocalityWeightsConstSharedPtr locality_weights_;
-  HostSetImpl host_set_{0, kDefaultOverProvisioningFactor};
+  HostSetImpl host_set_{0, false, kDefaultOverProvisioningFactor};
   std::shared_ptr info_{new NiceMock()};
   HostVector hosts_{makeTestHost(info_, "tcp://127.0.0.1:80", simTime()),
                     makeTestHost(info_, "tcp://127.0.0.1:81", simTime()),
@@ -5072,7 +5078,7 @@ TEST_F(HostSetImplLocalityTest, UnhealthyFailover) {
 TEST(OverProvisioningFactorTest, LocalityPickChanges) {
   auto setUpHostSetWithOPFAndTestPicks = [](const uint32_t overprovisioning_factor,
                                             const uint32_t pick_0, const uint32_t pick_1) {
-    HostSetImpl host_set(0, overprovisioning_factor);
+    HostSetImpl host_set(0, false, overprovisioning_factor);
     std::shared_ptr cluster_info{new NiceMock()};
     auto time_source = std::make_unique>();
     HostVector hosts{makeTestHost(cluster_info, "tcp://127.0.0.1:80", *time_source),
diff --git a/test/extensions/clusters/aggregate/cluster_test.cc b/test/extensions/clusters/aggregate/cluster_test.cc
index 60e34275e32b..09fa69ede02d 100644
--- a/test/extensions/clusters/aggregate/cluster_test.cc
+++ b/test/extensions/clusters/aggregate/cluster_test.cc
@@ -74,7 +74,7 @@ class AggregateClusterTest : public Event::TestUsingSimulatedTime, public testin
         priority,
         Upstream::HostSetImpl::partitionHosts(std::make_shared(hosts),
                                               Upstream::HostsPerLocalityImpl::empty()),
-        nullptr, hosts, {}, 100);
+        nullptr, hosts, {}, absl::nullopt, 100);
   }
 
   void setupSecondary(int priority, int healthy_hosts, int degraded_hosts, int unhealthy_hosts) {
@@ -84,7 +84,7 @@ class AggregateClusterTest : public Event::TestUsingSimulatedTime, public testin
         priority,
         Upstream::HostSetImpl::partitionHosts(std::make_shared(hosts),
                                               Upstream::HostsPerLocalityImpl::empty()),
-        nullptr, hosts, {}, 100);
+        nullptr, hosts, {}, absl::nullopt, 100);
   }
 
   void setupPrioritySet() {
diff --git a/test/extensions/clusters/aggregate/cluster_update_test.cc b/test/extensions/clusters/aggregate/cluster_update_test.cc
index 43871d58384d..f82aa0965eb4 100644
--- a/test/extensions/clusters/aggregate/cluster_update_test.cc
+++ b/test/extensions/clusters/aggregate/cluster_update_test.cc
@@ -157,7 +157,7 @@ TEST_F(AggregateClusterUpdateTest, LoadBalancingTest) {
       Upstream::HostSetImpl::partitionHosts(
           std::make_shared(Upstream::HostVector{host1, host2, host3}),
           Upstream::HostsPerLocalityImpl::empty()),
-      nullptr, {host1, host2, host3}, {}, 100);
+      nullptr, {host1, host2, host3}, {}, absl::nullopt, 100);
 
   // Set up the HostSet with 1 healthy, 1 degraded and 1 unhealthy.
   Upstream::HostSharedPtr host4 =
@@ -174,7 +174,7 @@ TEST_F(AggregateClusterUpdateTest, LoadBalancingTest) {
       Upstream::HostSetImpl::partitionHosts(
           std::make_shared(Upstream::HostVector{host4, host5, host6}),
           Upstream::HostsPerLocalityImpl::empty()),
-      nullptr, {host4, host5, host6}, {}, 100);
+      nullptr, {host4, host5, host6}, {}, absl::nullopt, 100);
 
   Upstream::HostConstSharedPtr host;
   for (int i = 0; i < 33; ++i) {
@@ -214,7 +214,7 @@ TEST_F(AggregateClusterUpdateTest, LoadBalancingTest) {
       Upstream::HostSetImpl::partitionHosts(
           std::make_shared(Upstream::HostVector{host7, host8, host9}),
           Upstream::HostsPerLocalityImpl::empty()),
-      nullptr, {host7, host8, host9}, {}, 100);
+      nullptr, {host7, host8, host9}, {}, absl::nullopt, 100);
 
   // Priority set
   //   Priority 0: 1/3 healthy, 1/3 degraded
@@ -301,7 +301,7 @@ TEST_F(AggregateClusterUpdateTest, InitializeAggregateClusterAfterOtherClusters)
       Upstream::HostSetImpl::partitionHosts(
           std::make_shared(Upstream::HostVector{host1, host2, host3}),
           Upstream::HostsPerLocalityImpl::empty()),
-      nullptr, {host1, host2, host3}, {}, 100);
+      nullptr, {host1, host2, host3}, {}, absl::nullopt, 100);
 
   for (int i = 0; i < 50; ++i) {
     EXPECT_CALL(factory_.random_, random()).WillRepeatedly(Return(i));
diff --git a/test/integration/eds_integration_test.cc b/test/integration/eds_integration_test.cc
index d2785c68bf5b..7648a3a5f8a4 100644
--- a/test/integration/eds_integration_test.cc
+++ b/test/integration/eds_integration_test.cc
@@ -61,6 +61,7 @@ class EdsIntegrationTest : public testing::TestWithParam weighted_priority_health = absl::nullopt;
     absl::optional overprovisioning_factor = absl::nullopt;
   };
 
@@ -77,6 +78,10 @@ class EdsIntegrationTest : public testing::TestWithParammutable_overprovisioning_factor()->set_value(
           endpoint_setting.overprovisioning_factor.value());
     }
+    if (endpoint_setting.weighted_priority_health.has_value()) {
+      cluster_load_assignment.mutable_policy()->set_weighted_priority_health(
+          endpoint_setting.weighted_priority_health.value());
+    }
     auto* locality_lb_endpoints = cluster_load_assignment.add_endpoints();
 
     for (uint32_t i = 0; i < endpoint_setting.total_endpoints; ++i) {
@@ -440,6 +445,31 @@ TEST_P(EdsIntegrationTest, OverprovisioningFactorUpdate) {
   get_and_compare(200);
 }
 
+// Validate that weighted_priority_health update are picked up by Envoy.
+TEST_P(EdsIntegrationTest, WeightedPriorityHealthUpdate) {
+  initializeTest(false);
+  // Default overprovisioning factor.
+  EndpointSettingOptions options;
+  options.total_endpoints = 4;
+  options.healthy_endpoints = 4;
+  setEndpoints(options);
+  auto get_and_compare = [this](bool expected) {
+    const auto& cluster_map = test_server_->server().clusterManager().clusters();
+    EXPECT_EQ(1, cluster_map.active_clusters_.size());
+    EXPECT_EQ(1, cluster_map.active_clusters_.count("cluster_0"));
+    const auto& cluster_ref = cluster_map.active_clusters_.find("cluster_0")->second;
+    const auto& hostset_per_priority = cluster_ref.get().prioritySet().hostSetsPerPriority();
+    EXPECT_EQ(1, hostset_per_priority.size());
+    const Envoy::Upstream::HostSetPtr& host_set = hostset_per_priority[0];
+    EXPECT_EQ(expected, host_set->weightedPriorityHealth());
+  };
+  get_and_compare(false);
+
+  options.weighted_priority_health = true;
+  setEndpoints(options);
+  get_and_compare(true);
+}
+
 // Verifies that EDS update only triggers member update callbacks once per update.
 TEST_P(EdsIntegrationTest, BatchMemberUpdateCb) {
   initializeTest(false);
diff --git a/test/mocks/upstream/host_set.h b/test/mocks/upstream/host_set.h
index 7fd18d438f0d..fe48ec1fe62d 100644
--- a/test/mocks/upstream/host_set.h
+++ b/test/mocks/upstream/host_set.h
@@ -50,6 +50,7 @@ class MockHostSet : public HostSet {
   void setOverprovisioningFactor(const uint32_t overprovisioning_factor) {
     overprovisioning_factor_ = overprovisioning_factor;
   }
+  bool weightedPriorityHealth() const override { return weighted_priority_health_; }
 
   HostVector hosts_;
   HostVector healthy_hosts_;
@@ -63,6 +64,7 @@ class MockHostSet : public HostSet {
   Common::CallbackManager member_update_cb_helper_;
   uint32_t priority_{};
   uint32_t overprovisioning_factor_{};
+  bool weighted_priority_health_{false};
   bool run_in_panic_mode_ = false;
 };
 } // namespace Upstream
diff --git a/test/mocks/upstream/priority_set.h b/test/mocks/upstream/priority_set.h
index cda2574854f1..a19e208d1258 100644
--- a/test/mocks/upstream/priority_set.h
+++ b/test/mocks/upstream/priority_set.h
@@ -25,7 +25,8 @@ class MockPrioritySet : public PrioritySet {
   MOCK_METHOD(void, updateHosts,
               (uint32_t priority, UpdateHostsParams&& update_hosts_params,
                LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added,
-               const HostVector& hosts_removed, absl::optional overprovisioning_factor,
+               const HostVector& hosts_removed, absl::optional weighted_priority_health,
+               absl::optional overprovisioning_factor,
                HostMapConstSharedPtr cross_priority_host_map));
   MOCK_METHOD(void, batchHostUpdate, (BatchUpdateCb&));
   MOCK_METHOD(HostMapConstSharedPtr, crossPriorityHostMap, (), (const));

From b34ae828910231350cd7a09434660ab6d6b6d64b Mon Sep 17 00:00:00 2001
From: Tianyu <72890320+tyxia@users.noreply.github.com>
Date: Wed, 21 Jun 2023 12:28:41 -0400
Subject: [PATCH 607/740] cel_matcher: matching on xds route metadata (#28024)

Test CEL matching on xDS attributes -- route metadata

Signed-off-by: tyxia 
---
 .../cel_matcher/cel_matcher_test.cc           | 38 ++++++++++++
 .../cel_matcher/cel_matcher_test.h            | 58 +++++++++++++++++++
 2 files changed, 96 insertions(+)

diff --git a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc
index 0a46dfdc1133..73bcd367cf32 100644
--- a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc
+++ b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.cc
@@ -49,6 +49,7 @@ class CelMatcherTest : public ::testing::Test {
   CelMatcherTest()
       : inject_action_(action_factory_),
         cluster_info_(std::make_shared>()),
+        route_(std::make_shared>()),
         data_(Envoy::Http::Matching::HttpMatchingDataImpl(stream_info_)) {}
 
   void buildCustomHeader(const absl::flat_hash_map& custom_value_pairs,
@@ -125,9 +126,18 @@ class CelMatcherTest : public ::testing::Test {
     stream_info_.setUpstreamClusterInfo(cluster_info_);
   }
 
+  void setUpstreamRouteMetadata(const std::string& namespace_str, const std::string& metadata_key,
+                                const std::string& metadata_value) {
+    Envoy::Config::Metadata::mutableMetadataValue(metadata_, namespace_str, metadata_key)
+        .set_string_value(metadata_value);
+    EXPECT_CALL(*route_, metadata()).WillRepeatedly(testing::ReturnPointee(&metadata_));
+    EXPECT_CALL(stream_info_, route()).WillRepeatedly(testing::ReturnPointee(&route_));
+  }
+
   Matcher::StringActionFactory action_factory_;
   Registry::InjectFactory> inject_action_;
   std::shared_ptr> cluster_info_;
+  std::shared_ptr> route_;
   testing::NiceMock stream_info_;
   absl::string_view context_ = "";
   testing::NiceMock factory_context_;
@@ -206,6 +216,34 @@ TEST_F(CelMatcherTest, CelMatcherClusterMetadataNotMatched) {
   EXPECT_EQ(result.on_match_, absl::nullopt);
 }
 
+TEST_F(CelMatcherTest, CelMatcherRouteMetadataMatched) {
+  setUpstreamRouteMetadata(std::string(kFilterNamespace), std::string(kMetadataKey),
+                           std::string(kMetadataValue));
+  Envoy::Http::Matching::HttpMatchingDataImpl data =
+      Envoy::Http::Matching::HttpMatchingDataImpl(stream_info_);
+  auto matcher_tree = buildMatcherTree(absl::StrFormat(
+      UpstreamRouteMetadataCelString, kFilterNamespace, kMetadataKey, kMetadataValue));
+  const auto result = matcher_tree->match(data_);
+  // The match was complete, match found.
+  EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete);
+  EXPECT_TRUE(result.on_match_.has_value());
+  EXPECT_NE(result.on_match_->action_cb_, nullptr);
+}
+
+TEST_F(CelMatcherTest, CelMatcherRouteMetadataNotMatched) {
+  setUpstreamRouteMetadata(std::string(kFilterNamespace), std::string(kMetadataKey),
+                           "wrong_service");
+  Envoy::Http::Matching::HttpMatchingDataImpl data =
+      Envoy::Http::Matching::HttpMatchingDataImpl(stream_info_);
+  auto matcher_tree = buildMatcherTree(absl::StrFormat(
+      UpstreamClusterMetadataCelString, kFilterNamespace, kMetadataKey, kMetadataValue));
+
+  const auto result = matcher_tree->match(data_);
+  // The match was completed, no match found.
+  EXPECT_EQ(result.match_state_, Matcher::MatchState::MatchComplete);
+  EXPECT_EQ(result.on_match_, absl::nullopt);
+}
+
 TEST_F(CelMatcherTest, CelMatcherRequestHeaderPathMatched) {
   auto matcher_tree = buildMatcherTree(RequestPathCelExprString);
 
diff --git a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h
index 65d6b0626f13..465e2a868381 100644
--- a/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h
+++ b/test/extensions/matching/input_matchers/cel_matcher/cel_matcher_test.h
@@ -607,6 +607,64 @@ inline constexpr absl::string_view UpstreamClusterMetadataCelString = R"pb(
   }
 )pb";
 
+// xds.route_metadata.filter_metadata['cel_matcher']['service_name'] == 'test_service'
+inline constexpr absl::string_view UpstreamRouteMetadataCelString = R"pb(
+  expr {
+    id: 8
+    call_expr {
+      function: "_==_"
+      args {
+        id: 6
+        call_expr {
+          function: "_[_]"
+          args {
+            id: 4
+            call_expr {
+              function: "_[_]"
+              args {
+                id: 3
+                select_expr {
+                  operand {
+                    id: 2
+                    select_expr {
+                      operand {
+                        id: 1
+                        ident_expr {
+                          name: "xds"
+                        }
+                      }
+                      field: "route_metadata"
+                    }
+                  }
+                  field: "filter_metadata"
+                }
+              }
+              args {
+                id: 5
+                const_expr {
+                  string_value: "%s"
+                }
+              }
+            }
+          }
+          args {
+            id: 7
+            const_expr {
+              string_value: "%s"
+            }
+          }
+        }
+      }
+      args {
+        id: 9
+        const_expr {
+          string_value: "%s"
+        }
+      }
+    }
+  }
+)pb";
+
 } // namespace CelMatcher
 } // namespace InputMatchers
 } // namespace Matching

From 20c0278705f999be1225cdc666ce4a4db1fac929 Mon Sep 17 00:00:00 2001
From: RenjieTang 
Date: Wed, 21 Jun 2023 10:58:52 -0700
Subject: [PATCH 608/740] [mobile]Fix CronvoyUrlRequest state transition and
 callback logic (#28052)

Commit Message:Fix CronvoyUrlRequest state transition and callback logic
Additional Description:
Suppose the application sends a Cronvoy request and keeps calling request.read() until the end is reached, and the response is structured in a way such that the endStream signal comes by itself without any data.

Before this change, the behavior at the end of the stream is:
EnvoyMobile calls OnData() and OnComplete() callbacks to Cronvoy. As a result, the application gets an OnReadCompleted() callback with no data read, but no OnSucceeded() because the stream hasn't reached its final state.
The application tries to read again, and OnSucceeded() will be triggered.

After this change, the behavior at the end of the stream is:
EnvoyMobile calls OnData() and OnComplete() callbacks to Cronvoy. In the OnData() callback, the stream state will be updated to reflect that the final read is done. It will not invoke OnReadComplete() because it shouldn't be called when no data is read.
When OnComplete() is called, the stream can successfully move to its final state and invokes OnSucceeded() callback.

Risk Level: Low
Testing: Unit tests
Docs Changes: n/a
Release Notes: n/a
Platform Specific Features: Android only

Signed-off-by: Renjie Tang 
---
 .../chromium/net/impl/CronvoyUrlRequest.java  | 20 ++++++++++++++-----
 .../chromium/net/impl/CronvoyEngineTest.java  | 17 +++++++++++++++-
 .../net/testing/TestUrlRequestCallback.java   |  7 +------
 3 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java
index e38b0840c416..63919abd6041 100644
--- a/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java
+++ b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java
@@ -34,7 +34,6 @@
 import org.chromium.net.CallbackException;
 import org.chromium.net.CronetException;
 import org.chromium.net.InlineExecutionProhibitedException;
-import org.chromium.net.NetworkException;
 import org.chromium.net.RequestFinishedInfo;
 import org.chromium.net.RequestFinishedInfo.Metrics;
 import org.chromium.net.UploadDataProvider;
@@ -541,7 +540,7 @@ private void fireOpenConnection() {
                                  mCurrentUrl, mRequestContext.getBuilder().quicEnabled());
     mCronvoyCallbacks = new CronvoyHttpCallbacks();
     mStream.set(mRequestContext.getEnvoyEngine().startStream(mCronvoyCallbacks,
-                                                             /* explicitFlowCrontrol= */ true,
+                                                             /* explicitFlowCrontrol */ true,
                                                              /* minDeliverySize */ 0));
     mStream.get().sendHeaders(envoyRequestHeaders, mUploadDataStream == null);
     if (mUploadDataStream != null && mUrlChain.size() == 1) {
@@ -864,10 +863,18 @@ public void onData(ByteBuffer data, boolean endStream, EnvoyStreamIntel streamIn
       mEndStream = endStream;
       @State int originalState;
       @State int updatedState;
+
+      // When endStream with no data is read, there will be no more calls to read().
+      boolean isFinalRead = endStream && !data.hasRemaining();
       do {
         originalState = mState.get();
-        updatedState = determineNextState(endStream, originalState, State.AWAITING_READ);
+        updatedState = determineNextState(endStream, originalState,
+                                          isFinalRead ? State.COMPLETE : State.AWAITING_READ);
       } while (!mState.compareAndSet(originalState, updatedState));
+      if (isFinalRead) {
+        // onComplete still needs to be called - this always returns false.
+        mSucceededState.hasReachedFinalState(SucceededState.FINAL_READ_DONE);
+      }
       if (completeAbandonIfAny(originalState, updatedState)) {
         return;
       }
@@ -882,9 +889,12 @@ public void run() {
           try {
             ByteBuffer userBuffer = mUserCurrentReadBuffer;
             mUserCurrentReadBuffer = null; // Avoid the reference to a potentially large buffer.
+            int dataRead = data.remaining();
             userBuffer.put(data); // NPE ==> BUG, BufferOverflowException ==> User not behaving.
-            mWaitingOnRead.set(true);
-            mCallback.onReadCompleted(CronvoyUrlRequest.this, mUrlResponseInfo, userBuffer);
+            if (dataRead > 0 || !endStream) {
+              mWaitingOnRead.set(true);
+              mCallback.onReadCompleted(CronvoyUrlRequest.this, mUrlResponseInfo, userBuffer);
+            }
           } catch (Throwable t) {
             onCallbackException(t);
           }
diff --git a/mobile/test/java/org/chromium/net/impl/CronvoyEngineTest.java b/mobile/test/java/org/chromium/net/impl/CronvoyEngineTest.java
index c4a23a981dc8..160e50eaafa5 100644
--- a/mobile/test/java/org/chromium/net/impl/CronvoyEngineTest.java
+++ b/mobile/test/java/org/chromium/net/impl/CronvoyEngineTest.java
@@ -133,8 +133,19 @@ public void get_withThrottledBodyResponse() throws Exception {
     mockWebServer.start();
     RequestScenario requestScenario = new RequestScenario().addResponseBuffers(13);
 
-    Response response = sendRequest(requestScenario);
+    UrlRequestCallbackTester urlRequestCallbackTester = new UrlRequestCallbackTester<>();
+    UrlRequestCallback testCallback = new UrlRequestCallback(
+        requestScenario.responseBody, urlRequestCallbackTester, requestScenario);
+    ExperimentalUrlRequest.Builder builder = cronvoyEngine.newUrlRequestBuilder(
+        mockWebServer.url(requestScenario.urlPath).toString(),
+        urlRequestCallbackTester.getWrappedUrlRequestCallback(testCallback),
+        Executors.newSingleThreadExecutor());
 
+    Response response = urlRequestCallbackTester.waitForResponse(builder.build());
+
+    // The response will come in 3 chunks. The 4th read is the final read and shouldn't trigger
+    // OnReadComplete() as no data is read.
+    assertThat(testCallback.getNumOnReadCompleteCalled()).isEqualTo(3);
     assertThat(response.getCronetException()).withFailMessage(response.getErrorMessage()).isNull();
     assertThat(response.getBodyAsString()).isEqualTo("hello, world");
     assertThat(response.getNbResponseChunks()).isEqualTo(3); // 5 bytes, 5 bytes, and 2 bytes
@@ -346,6 +357,7 @@ private static class UrlRequestCallback extends UrlRequest.Callback {
     private final AtomicInteger nbChunks = new AtomicInteger(0);
     private final AtomicInteger bufferLastRemaining = new AtomicInteger();
     private final RequestScenario requestScenario;
+    private int numOnReadCompleteCalled = 0;
 
     private UrlRequestCallback(List responseBodyBuffers,
                                UrlRequestCallbackTester urlRequestCallbackTester,
@@ -356,6 +368,8 @@ private UrlRequestCallback(List responseBodyBuffers,
       buffers = new ConcurrentLinkedQueue<>(responseBodyBuffers);
     }
 
+    public int getNumOnReadCompleteCalled() { return numOnReadCompleteCalled; }
+
     @Override
     public void onRedirectReceived(UrlRequest urlRequest, UrlResponseInfo info,
                                    String newLocationUrl) {
@@ -379,6 +393,7 @@ public void onResponseStarted(UrlRequest urlRequest, UrlResponseInfo info) {
     @Override
     public void onReadCompleted(UrlRequest urlRequest, UrlResponseInfo info,
                                 ByteBuffer byteBuffer) {
+      numOnReadCompleteCalled++;
       ByteBuffer buffer = buffers.peek();
       if (byteBuffer != buffer) {
         throw new AssertionError("Can't happen...");
diff --git a/mobile/test/java/org/chromium/net/testing/TestUrlRequestCallback.java b/mobile/test/java/org/chromium/net/testing/TestUrlRequestCallback.java
index 1c94f0e97926..5c88fe2df4fc 100644
--- a/mobile/test/java/org/chromium/net/testing/TestUrlRequestCallback.java
+++ b/mobile/test/java/org/chromium/net/testing/TestUrlRequestCallback.java
@@ -339,12 +339,7 @@ public void startNextRead(UrlRequest request, ByteBuffer buffer) {
     request.read(buffer);
   }
 
-  public boolean isDone() {
-    // It's not mentioned by the Android docs, but block(0) seems to block
-    // indefinitely, so have to block for one millisecond to get state
-    // without blocking.
-    return !mDone.isBlocked();
-  }
+  public boolean isDone() { return !mDone.isBlocked(); }
 
   protected void openDone() { mDone.open(); }
 

From d5e16921e2a9e5476a7e816e76ed8ecc70e373d3 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Wed, 21 Jun 2023 19:23:00 +0100
Subject: [PATCH 609/740] deps: Bump `rules_rust` -> 0.24.1 (#28063)

Signed-off-by: Ryan Northey 
---
 bazel/repository_locations.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
index eca1847b03de..e576c66d4422 100644
--- a/bazel/repository_locations.bzl
+++ b/bazel/repository_locations.bzl
@@ -1325,12 +1325,12 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "Bazel rust rules",
         project_desc = "Bazel rust rules (used by Wasm)",
         project_url = "https://github.com/bazelbuild/rules_rust",
-        version = "0.22.0",
-        sha256 = "50272c39f20a3a3507cb56dcb5c3b348bda697a7d868708449e2fa6fb893444c",
+        version = "0.24.1",
+        sha256 = "190b5aeba104210f8ed9b1ff595d1f459297fe32db70f0a04f5c537a13ee0602",
         urls = ["https://github.com/bazelbuild/rules_rust/releases/download/{version}/rules_rust-v{version}.tar.gz"],
         use_category = ["dataplane_ext"],
         extensions = ["envoy.wasm.runtime.wasmtime"],
-        release_date = "2023-05-22",
+        release_date = "2023-06-20",
         cpe = "N/A",
         license = "Apache-2.0",
         license_url = "https://github.com/bazelbuild/rules_rust/blob/{version}/LICENSE.txt",

From 1090a531f523ff8c6f1647e80d2e57194af8ea4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bence=20B=C3=A9ky?= 
Date: Wed, 21 Jun 2023 16:00:56 -0400
Subject: [PATCH 610/740] [balsa] Do not signal error on POST method with no
 Content-Length. (#28070)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add test to document broken behavior.

* [balsa] Do not signal error on POST method with no body.

This is for consistency with http-parser behavior.

Tracking issue: #21245

* Fix two existing tests.

Also remove newly added test, which turns out to be redundant with
existing tests.

---------

Signed-off-by: Bence Béky 
---
 source/common/http/http1/balsa_parser.cc  |  1 +
 test/common/http/http1/codec_impl_test.cc | 56 +++++++++++------------
 2 files changed, 28 insertions(+), 29 deletions(-)

diff --git a/source/common/http/http1/balsa_parser.cc b/source/common/http/http1/balsa_parser.cc
index 70d968ad0f11..287574eb210e 100644
--- a/source/common/http/http1/balsa_parser.cc
+++ b/source/common/http/http1/balsa_parser.cc
@@ -154,6 +154,7 @@ BalsaParser::BalsaParser(MessageType type, ParserCallbacks* connection, size_t m
   http_validation_policy.disallow_multiple_content_length = false;
   http_validation_policy.disallow_transfer_encoding_with_content_length = false;
   http_validation_policy.validate_transfer_encoding = false;
+  http_validation_policy.require_content_length_if_body_required = false;
   framer_.set_http_validation_policy(http_validation_policy);
 
   framer_.set_balsa_headers(&headers_);
diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc
index 94d848188028..fd2e365b78ed 100644
--- a/test/common/http/http1/codec_impl_test.cc
+++ b/test/common/http/http1/codec_impl_test.cc
@@ -770,12 +770,11 @@ TEST_P(Http1ServerConnectionImplTest, InvalidChunkHeader) {
 
 TEST_P(Http1ServerConnectionImplTest, IdentityAndChunkedBody) {
 #ifdef ENVOY_ENABLE_UHV
-  // TODO(#27377): http-parser will not be used together with UHV and triggers an internal
-  // transfer-encoding check preventing UHV to be called.
-  if (parser_impl_ == Http1ParserImpl::HttpParser) {
-    return;
-  }
+  const bool strict = false;
+#else
+  const bool strict = true;
 #endif
+
   initialize();
 
   InSequence sequence;
@@ -786,16 +785,27 @@ TEST_P(Http1ServerConnectionImplTest, IdentityAndChunkedBody) {
   Buffer::OwnedImpl buffer("POST / HTTP/1.1\r\nHost: host\r\ntransfer-encoding: "
                            "identity,chunked\r\n\r\nb\r\nHello World\r\n0\r\n\r\n");
 
-  if (parser_impl_ == Http1ParserImpl::HttpParser) {
+  if (strict) {
     EXPECT_CALL(decoder, sendLocalReply(Http::Code::NotImplemented, _, _, _,
                                         "http1.invalid_transfer_encoding"));
   } else {
-    // TODO(#27375): Balsa codec produces invalid response in non UHV mode
-    EXPECT_CALL(decoder, sendLocalReply(Http::Code::BadRequest, _, _, _, "http1.codec_error"));
+    if (parser_impl_ == Http1ParserImpl::BalsaParser) {
+      EXPECT_CALL(decoder, decodeHeaders_(_, true));
+    } else {
+      EXPECT_CALL(decoder, decodeHeaders_(_, false));
+      EXPECT_CALL(decoder, decodeData(BufferStringEqual("Hello World"), false));
+      EXPECT_CALL(decoder, decodeData(BufferStringEqual(""), true));
+    }
   }
+
   auto status = codec_->dispatch(buffer);
-  EXPECT_TRUE(isCodecProtocolError(status));
-  EXPECT_THAT(status.message(), StartsWith("http/1.1 protocol error"));
+
+  if (strict) {
+    EXPECT_TRUE(isCodecProtocolError(status));
+    EXPECT_THAT(status.message(), StartsWith("http/1.1 protocol error"));
+  } else {
+    EXPECT_TRUE(status.ok());
+  }
 }
 
 TEST_P(Http1ServerConnectionImplTest, HostWithLWS) {
@@ -4362,8 +4372,8 @@ TEST_P(Http1ClientConnectionImplTest, EOFDuringContentLengthBody) {
   EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_EOF_STATE");
 }
 
-// A request method requiring a body but without a Content-Length (or Transfer-Encoding: chunked)
-// header is an error.
+// Do not signal an error upon receiving a request with a method requiring a
+// body but without a Content-Length (or Transfer-Encoding: chunked) header.
 TEST_P(Http1ServerConnectionImplTest, NoContentLengthRequest) {
   initialize();
   InSequence s;
@@ -4376,27 +4386,15 @@ TEST_P(Http1ServerConnectionImplTest, NoContentLengthRequest) {
         return decoder;
       }));
 
-  if (parser_impl_ == Http1ParserImpl::BalsaParser) {
-    EXPECT_CALL(decoder,
-                sendLocalReply(Http::Code::BadRequest, "Bad Request", _, _, "http1.codec_error"));
-  } else {
-    EXPECT_CALL(decoder, decodeHeaders_(_, true));
-  }
+  EXPECT_CALL(decoder, decodeHeaders_(_, true));
   constexpr absl::string_view kFirstLine = "POST / HTTP/1.1\r\n\r\n";
   constexpr absl::string_view kBody = "foo";
+
   Buffer::OwnedImpl buffer(absl::StrCat(kFirstLine, kBody));
   auto status = codec_->dispatch(buffer);
-
-  if (parser_impl_ == Http1ParserImpl::BalsaParser) {
-    EXPECT_TRUE(isCodecProtocolError(status));
-    EXPECT_EQ(status.message(), "http/1.1 protocol error: REQUIRED_BODY_BUT_NO_CONTENT_LENGTH");
-    EXPECT_EQ("http1.codec_error", response_encoder->getStream().responseDetails());
-    EXPECT_EQ(kFirstLine.length() + kBody.length(), buffer.length());
-  } else {
-    // http-parser actually does not signal an error, but ignores the fraction of the body received.
-    EXPECT_TRUE(status.ok());
-    EXPECT_EQ(kBody.length(), buffer.length());
-  }
+  EXPECT_TRUE(status.ok());
+  // The received body is ignored.
+  EXPECT_EQ(kBody.length(), buffer.length());
 }
 
 // Regression test for #24557: A read of zero bytes can signal the end of response body if there is

From 8bc87d87578fbb7c4e1af7dd564072a153f97289 Mon Sep 17 00:00:00 2001
From: Kevin Baichoo 
Date: Wed, 21 Jun 2023 16:51:06 -0400
Subject: [PATCH 611/740] Docs: Add stats doc for load shed points. (#28078)

Signed-off-by: Kevin Baichoo 
---
 .../operations/overload_manager/overload_manager.rst   | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/root/configuration/operations/overload_manager/overload_manager.rst b/docs/root/configuration/operations/overload_manager/overload_manager.rst
index ac66e556514f..e9ce33d4788e 100644
--- a/docs/root/configuration/operations/overload_manager/overload_manager.rst
+++ b/docs/root/configuration/operations/overload_manager/overload_manager.rst
@@ -353,3 +353,13 @@ with the following statistics:
 
   active, Gauge, "Active state of the action (0=scaling, 1=saturated)"
   scale_percent, Gauge, "Scaled value of the action as a percent (0-99=scaling, 100=saturated)"
+
+Each configured Load Shed Point has a statistics tree rooted at *overload..*
+with the following statistics:
+
+.. csv-table::
+  :header: Name, Type, Description
+  :widths: 1, 1, 2
+
+  scale_percent, Gauge, "Scaled value of the action as a percent (0-99=scaling, 100=saturated)"
+

From 84d0b10196d832b325fdef0be23c438ea5105c26 Mon Sep 17 00:00:00 2001
From: ohadvano <49730675+ohadvano@users.noreply.github.com>
Date: Thu, 22 Jun 2023 00:03:19 +0300
Subject: [PATCH 612/740] Add debug logs to dynamic forward proxy failure paths
 (#28071)

Signed-off-by: ohadvano <49730675+ohadvano@users.noreply.github.com>
---
 source/extensions/clusters/dynamic_forward_proxy/cluster.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc
index 7dd50e893775..86af58ffe5bb 100644
--- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc
+++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc
@@ -357,6 +357,7 @@ Cluster::LoadBalancer::chooseHost(Upstream::LoadBalancerContext* context) {
   std::string host = Common::DynamicForwardProxy::DnsHostInfo::normalizeHostForDfp(raw_host, port);
 
   if (host.empty()) {
+    ENVOY_LOG(debug, "host empty");
     return nullptr;
   }
 
@@ -368,9 +369,11 @@ Cluster::LoadBalancer::chooseHost(Upstream::LoadBalancerContext* context) {
     absl::ReaderMutexLock lock{&cluster_.host_map_lock_};
     const auto host_it = cluster_.host_map_.find(host);
     if (host_it == cluster_.host_map_.end()) {
+      ENVOY_LOG(debug, "host {} not found", host);
       return nullptr;
     } else {
       if (host_it->second.logical_host_->coarseHealth() == Upstream::Host::Health::Unhealthy) {
+        ENVOY_LOG(debug, "host {} is unhealthy", host);
         return nullptr;
       }
       host_it->second.shared_host_info_->touch();

From 4876e6911e439b68c82040a3774a37de0d3e78c0 Mon Sep 17 00:00:00 2001
From: Kevin Baichoo 
Date: Wed, 21 Jun 2023 20:39:27 -0400
Subject: [PATCH 613/740] Clean up: Use contains where count is being used as
 contains. (#28076)

Signed-off-by: Kevin Baichoo 
---
 source/common/common/utility.cc                             | 2 +-
 source/common/common/utility.h                              | 2 +-
 source/common/stream_info/filter_state_impl.cc              | 2 +-
 source/common/upstream/cds_api_impl.cc                      | 2 +-
 source/common/upstream/cluster_manager_impl.cc              | 2 +-
 source/extensions/clusters/eds/eds.cc                       | 2 +-
 source/extensions/config_subscription/grpc/grpc_mux_impl.cc | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc
index c1dc9ed47b45..824558e391e3 100644
--- a/source/common/common/utility.cc
+++ b/source/common/common/utility.cc
@@ -400,7 +400,7 @@ std::string StringUtil::removeTokens(absl::string_view source, absl::string_view
                                      absl::string_view joiner) {
   auto values = Envoy::StringUtil::splitToken(source, delimiters, false, true);
   auto end = std::remove_if(values.begin(), values.end(),
-                            [&](absl::string_view t) { return tokens_to_remove.count(t) != 0; });
+                            [&](absl::string_view t) { return tokens_to_remove.contains(t); });
   return absl::StrJoin(values.begin(), end, joiner);
 }
 
diff --git a/source/common/common/utility.h b/source/common/common/utility.h
index a07a635ddf01..a616c3aebeff 100644
--- a/source/common/common/utility.h
+++ b/source/common/common/utility.h
@@ -828,7 +828,7 @@ class SetUtil {
                             absl::flat_hash_set& result_set) {
     std::copy_if(original_set.begin(), original_set.end(),
                  std::inserter(result_set, result_set.begin()),
-                 [&remove_set](const T& v) -> bool { return remove_set.count(v) == 0; });
+                 [&remove_set](const T& v) -> bool { return !remove_set.contains(v); });
   }
 };
 
diff --git a/source/common/stream_info/filter_state_impl.cc b/source/common/stream_info/filter_state_impl.cc
index 9a997de0ccee..e59e1138cdaa 100644
--- a/source/common/stream_info/filter_state_impl.cc
+++ b/source/common/stream_info/filter_state_impl.cc
@@ -120,7 +120,7 @@ FilterState::ObjectsPtr FilterStateImpl::objectsSharedWithUpstreamConnection() c
 }
 
 bool FilterStateImpl::hasDataWithNameInternally(absl::string_view data_name) const {
-  return data_storage_.count(data_name) > 0;
+  return data_storage_.contains(data_name);
 }
 
 void FilterStateImpl::maybeCreateParent(ParentAccessMode parent_access_mode) {
diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc
index 64c2c5a25cce..ecb1d082a4b7 100644
--- a/source/common/upstream/cds_api_impl.cc
+++ b/source/common/upstream/cds_api_impl.cc
@@ -49,7 +49,7 @@ void CdsApiImpl::onConfigUpdate(const std::vector& r
   for (const auto& [cluster_name, _] : all_existing_clusters.warming_clusters_) {
     UNREFERENCED_PARAMETER(_);
     // Do not add the cluster twice when the cluster is both active and warming.
-    if (all_existing_clusters.active_clusters_.count(cluster_name) == 0) {
+    if (!all_existing_clusters.active_clusters_.contains(cluster_name)) {
       *to_remove_repeated.Add() = cluster_name;
     }
   }
diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc
index 5229d60b68ee..12dbd970cd20 100644
--- a/source/common/upstream/cluster_manager_impl.cc
+++ b/source/common/upstream/cluster_manager_impl.cc
@@ -1126,7 +1126,7 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_
                            OptRef cluster_manager) {
     ThreadLocalClusterManagerImpl::ClusterEntry* new_cluster = nullptr;
     if (add_or_update_cluster) {
-      if (cluster_manager->thread_local_clusters_.count(info->name()) > 0) {
+      if (cluster_manager->thread_local_clusters_.contains(info->name())) {
         ENVOY_LOG(debug, "updating TLS cluster {}", info->name());
       } else {
         ENVOY_LOG(debug, "adding TLS cluster {}", info->name());
diff --git a/source/extensions/clusters/eds/eds.cc b/source/extensions/clusters/eds/eds.cc
index 3341992dc8af..4aaea4dfba38 100644
--- a/source/extensions/clusters/eds/eds.cc
+++ b/source/extensions/clusters/eds/eds.cc
@@ -139,7 +139,7 @@ void EdsClusterImpl::BatchUpdateHelper::updateLocalityEndpoints(
 
   // When the configuration contains duplicate hosts, only the first one will be retained.
   const auto address_as_string = address->asString();
-  if (all_new_hosts.count(address_as_string) > 0) {
+  if (all_new_hosts.contains(address_as_string)) {
     return;
   }
 
diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc
index c5ba997b6f5b..924596483a2c 100644
--- a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc
+++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc
@@ -107,7 +107,7 @@ void GrpcMuxImpl::sendDiscoveryRequest(absl::string_view type_url) {
   absl::node_hash_set resources;
   for (const auto* watch : api_state.watches_) {
     for (const std::string& resource : watch->resources_) {
-      if (resources.count(resource) == 0) {
+      if (!resources.contains(resource)) {
         resources.emplace(resource);
         request.add_resource_names(resource);
       }

From 308627f51e9d8bdd257cbad19bead991c358b393 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Thu, 22 Jun 2023 05:22:09 +0100
Subject: [PATCH 614/740] deps: Bump `com_github_bufbuild_buf` -> 1.21.0
 (#28066)

Fix #27818

Signed-off-by: Ryan Northey 
---
 api/bazel/repository_locations.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl
index 4a95ca496959..ef3af248eb82 100644
--- a/api/bazel/repository_locations.bzl
+++ b/api/bazel/repository_locations.bzl
@@ -131,11 +131,11 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "buf",
         project_desc = "A new way of working with Protocol Buffers.",  # Used for breaking change detection in API protobufs
         project_url = "https://buf.build",
-        version = "1.19.0",
-        sha256 = "ff35aa96b54037d492d30a21dce8d96d47693a761487f98551d750407c27c285",
+        version = "1.21.0",
+        sha256 = "1db51318e49f12095c97866c9b5d939dfec318b50362bba8a3a9545c4cff456b",
         strip_prefix = "buf",
         urls = ["https://github.com/bufbuild/buf/releases/download/v{version}/buf-Linux-x86_64.tar.gz"],
-        release_date = "2023-05-17",
+        release_date = "2023-06-05",
         use_category = ["api"],
         license = "Apache-2.0",
         license_url = "https://github.com/bufbuild/buf/blob/v{version}/LICENSE",

From c530f077630bf22a0b5ecafa1eeda51cc624a3ae Mon Sep 17 00:00:00 2001
From: Ali Beyad 
Date: Thu, 22 Jun 2023 00:47:46 -0400
Subject: [PATCH 615/740] test: Flag-guard the XdsBuilder copy constructor test
 (#28082)

Signed-off-by: Ali Beyad 
---
 mobile/test/cc/unit/envoy_config_test.cc | 52 ++++++++++++------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/mobile/test/cc/unit/envoy_config_test.cc b/mobile/test/cc/unit/envoy_config_test.cc
index 6e215c7e0de5..d2063656bb85 100644
--- a/mobile/test/cc/unit/envoy_config_test.cc
+++ b/mobile/test/cc/unit/envoy_config_test.cc
@@ -88,32 +88,6 @@ TEST(TestConfig, MultiFlag) {
   EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_true\" value { bool_value: false }"));
 }
 
-TEST(TestConfig, CopyConstructor) {
-  EngineBuilder engine_builder;
-  engine_builder.setRuntimeGuard("test_feature_false", true).enableGzipDecompression(false);
-
-  std::unique_ptr bootstrap = engine_builder.generateBootstrap();
-  std::string bootstrap_str = bootstrap->ShortDebugString();
-  EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_false\" value { bool_value: true }"));
-  EXPECT_THAT(bootstrap_str, Not(HasSubstr("envoy.filters.http.decompressor")));
-
-  EngineBuilder engine_builder_copy(engine_builder);
-  engine_builder_copy.enableGzipDecompression(true);
-  XdsBuilder xdsBuilder("FAKE_XDS_SERVER", 0);
-  xdsBuilder.addClusterDiscoveryService();
-  engine_builder_copy.setXds(xdsBuilder);
-  bootstrap_str = engine_builder_copy.generateBootstrap()->ShortDebugString();
-  EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_false\" value { bool_value: true }"));
-  EXPECT_THAT(bootstrap_str, HasSubstr("envoy.filters.http.decompressor"));
-  EXPECT_THAT(bootstrap_str, HasSubstr("FAKE_XDS_SERVER"));
-
-  EngineBuilder engine_builder_copy2(engine_builder_copy);
-  bootstrap_str = engine_builder_copy2.generateBootstrap()->ShortDebugString();
-  EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_false\" value { bool_value: true }"));
-  EXPECT_THAT(bootstrap_str, HasSubstr("envoy.filters.http.decompressor"));
-  EXPECT_THAT(bootstrap_str, HasSubstr("FAKE_XDS_SERVER"));
-}
-
 TEST(TestConfig, ConfigIsValid) {
   EngineBuilder engine_builder;
   std::unique_ptr bootstrap = engine_builder.generateBootstrap();
@@ -340,6 +314,32 @@ TEST(TestConfig, XdsConfig) {
                 .token_lifetime_seconds(),
             500);
 }
+
+TEST(TestConfig, CopyConstructor) {
+  EngineBuilder engine_builder;
+  engine_builder.setRuntimeGuard("test_feature_false", true).enableGzipDecompression(false);
+
+  std::unique_ptr bootstrap = engine_builder.generateBootstrap();
+  std::string bootstrap_str = bootstrap->ShortDebugString();
+  EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_false\" value { bool_value: true }"));
+  EXPECT_THAT(bootstrap_str, Not(HasSubstr("envoy.filters.http.decompressor")));
+
+  EngineBuilder engine_builder_copy(engine_builder);
+  engine_builder_copy.enableGzipDecompression(true);
+  XdsBuilder xdsBuilder("FAKE_XDS_SERVER", 0);
+  xdsBuilder.addClusterDiscoveryService();
+  engine_builder_copy.setXds(xdsBuilder);
+  bootstrap_str = engine_builder_copy.generateBootstrap()->ShortDebugString();
+  EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_false\" value { bool_value: true }"));
+  EXPECT_THAT(bootstrap_str, HasSubstr("envoy.filters.http.decompressor"));
+  EXPECT_THAT(bootstrap_str, HasSubstr("FAKE_XDS_SERVER"));
+
+  EngineBuilder engine_builder_copy2(engine_builder_copy);
+  bootstrap_str = engine_builder_copy2.generateBootstrap()->ShortDebugString();
+  EXPECT_THAT(bootstrap_str, HasSubstr("\"test_feature_false\" value { bool_value: true }"));
+  EXPECT_THAT(bootstrap_str, HasSubstr("envoy.filters.http.decompressor"));
+  EXPECT_THAT(bootstrap_str, HasSubstr("FAKE_XDS_SERVER"));
+}
 #endif
 
 TEST(TestConfig, EnablePlatformCertificatesValidation) {

From 1e1fd8e21dcf5b1c8945f1b0df89f5ced14f57fd Mon Sep 17 00:00:00 2001
From: phlax 
Date: Thu, 22 Jun 2023 05:50:59 +0100
Subject: [PATCH 616/740] deps: Bump `python` -> 3.11.3 (#28068)

Signed-off-by: Ryan Northey 
---
 bazel/repositories_extra.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl
index 877392ebf473..40d348073fa4 100644
--- a/bazel/repositories_extra.bzl
+++ b/bazel/repositories_extra.bzl
@@ -8,7 +8,7 @@ def _python_minor_version(python_version):
     return "_".join(python_version.split(".")[:-1])
 
 # Python version for `rules_python`
-PYTHON_VERSION = "3.11.1"
+PYTHON_VERSION = "3.11.3"
 PYTHON_MINOR_VERSION = _python_minor_version(PYTHON_VERSION)
 
 # Envoy deps that rely on a first stage of dependency loading in envoy_dependencies().

From a94cf4e062c8a1b873225ec42d3e019e160384b2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 22 Jun 2023 09:19:18 +0100
Subject: [PATCH 617/740] build(deps): bump gsutil from 5.24 to 5.25 in
 /tools/base (#28086)

Bumps [gsutil](https://github.com/GoogleCloudPlatform/gsutil) from 5.24 to 5.25.
- [Changelog](https://github.com/GoogleCloudPlatform/gsutil/blob/master/CHANGES.md)
- [Commits](https://github.com/GoogleCloudPlatform/gsutil/commits)

---
updated-dependencies:
- dependency-name: gsutil
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tools/base/requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt
index cea15224c3ea..2274f8d64a05 100644
--- a/tools/base/requirements.txt
+++ b/tools/base/requirements.txt
@@ -749,8 +749,8 @@ googleapis-common-protos==1.59.0 \
     --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \
     --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f
     # via google-api-core
-gsutil==5.24 \
-    --hash=sha256:1f841645cda40fcc817e9ca84d285cdf541cc015fd38a5862017b085756729a0
+gsutil==5.25 \
+    --hash=sha256:7e4cb7fa9a332c401e4b7f5fef1da3e9ef21e3e4885de6d007b07a11b5d0524a
     # via -r requirements.in
 httplib2==0.20.4 \
     --hash=sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585 \

From 225415475e55cbd79c6b7eefa99c0cc503183986 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 22 Jun 2023 09:20:11 +0100
Subject: [PATCH 618/740] build(deps): bump google.golang.org/grpc from 1.56.0
 to 1.56.1 in /examples/ext_authz/auth/grpc-service (#28087)

build(deps): bump google.golang.org/grpc

Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.56.0 to 1.56.1.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.56.0...v1.56.1)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/ext_authz/auth/grpc-service/go.mod | 2 +-
 examples/ext_authz/auth/grpc-service/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/ext_authz/auth/grpc-service/go.mod b/examples/ext_authz/auth/grpc-service/go.mod
index fbe4b7c6cf53..ccb670f309ef 100644
--- a/examples/ext_authz/auth/grpc-service/go.mod
+++ b/examples/ext_authz/auth/grpc-service/go.mod
@@ -6,5 +6,5 @@ require (
 	github.com/envoyproxy/go-control-plane v0.11.1
 	github.com/golang/protobuf v1.5.3
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e
-	google.golang.org/grpc v1.56.0
+	google.golang.org/grpc v1.56.1
 )
diff --git a/examples/ext_authz/auth/grpc-service/go.sum b/examples/ext_authz/auth/grpc-service/go.sum
index 9049678cee91..9c7e78a51bfd 100644
--- a/examples/ext_authz/auth/grpc-service/go.sum
+++ b/examples/ext_authz/auth/grpc-service/go.sum
@@ -1473,8 +1473,8 @@ google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v
 google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
 google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
-google.golang.org/grpc v1.56.0 h1:+y7Bs8rtMd07LeXmL3NxcTLn7mUkbKZqEpPhMNkwJEE=
-google.golang.org/grpc v1.56.0/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
+google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

From 0a24253a5fee34c47f81e77d277e9fdb09864701 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 22 Jun 2023 09:20:44 +0100
Subject: [PATCH 619/740] build(deps): bump google.golang.org/grpc from 1.56.0
 to 1.56.1 in /examples/load-reporting-service (#28088)

build(deps): bump google.golang.org/grpc

Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.56.0 to 1.56.1.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.56.0...v1.56.1)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/load-reporting-service/go.mod | 2 +-
 examples/load-reporting-service/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/load-reporting-service/go.mod b/examples/load-reporting-service/go.mod
index 82b25c6c30cb..d5c2d1cb35e0 100644
--- a/examples/load-reporting-service/go.mod
+++ b/examples/load-reporting-service/go.mod
@@ -5,5 +5,5 @@ go 1.13
 require (
 	github.com/envoyproxy/go-control-plane v0.11.1
 	github.com/golang/protobuf v1.5.3
-	google.golang.org/grpc v1.56.0
+	google.golang.org/grpc v1.56.1
 )
diff --git a/examples/load-reporting-service/go.sum b/examples/load-reporting-service/go.sum
index 9049678cee91..9c7e78a51bfd 100644
--- a/examples/load-reporting-service/go.sum
+++ b/examples/load-reporting-service/go.sum
@@ -1473,8 +1473,8 @@ google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v
 google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
 google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
-google.golang.org/grpc v1.56.0 h1:+y7Bs8rtMd07LeXmL3NxcTLn7mUkbKZqEpPhMNkwJEE=
-google.golang.org/grpc v1.56.0/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
+google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

From 7499f6d98b24305c43d45f173515ac41f38a6588 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 22 Jun 2023 09:21:44 +0100
Subject: [PATCH 620/740] build(deps): bump node from `4c4d193` to `a70c22c` in
 /examples/shared/node (#28089)

build(deps): bump node in /examples/shared/node

Bumps node from `4c4d193` to `a70c22c`.

---
updated-dependencies:
- dependency-name: node
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/node/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile
index 15867ba7c493..a6a5c5358e03 100644
--- a/examples/shared/node/Dockerfile
+++ b/examples/shared/node/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:20.3-bullseye-slim@sha256:4c4d1930c335191ebcf049eec6a4d35571b1fb9468ab0b8a403724c1a6d23f58 as node-base
+FROM node:20.3-bullseye-slim@sha256:a70c22cb6ef7c6d809970b2889e5e556337fda8bfaa439b30c035efaef8fc3a1 as node-base
 
 
 FROM node-base as node-http-auth

From 28fb348d9f25deaedf3b3145305786d2e8579235 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 22 Jun 2023 09:22:15 +0100
Subject: [PATCH 621/740] build(deps): bump elasticsearch from 8.8.0 to 8.8.1
 in /examples/skywalking (#28090)

build(deps): bump elasticsearch in /examples/skywalking

Bumps elasticsearch from 8.8.0 to 8.8.1.

---
updated-dependencies:
- dependency-name: elasticsearch
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/skywalking/Dockerfile-elasticsearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/skywalking/Dockerfile-elasticsearch b/examples/skywalking/Dockerfile-elasticsearch
index b721cc086bfa..2db48fc12c78 100644
--- a/examples/skywalking/Dockerfile-elasticsearch
+++ b/examples/skywalking/Dockerfile-elasticsearch
@@ -1 +1 @@
-FROM elasticsearch:8.8.0@sha256:5c28849be5e91610761fcd4a49c2561dfae72be9ac0a3e7b5c42c9576aa9157b
+FROM elasticsearch:8.8.1@sha256:e31753d052509353f99be86892bf5c65a070805aadbb295d1b3128163061ea68

From 2a0514626d0d2a2b59821b410efde94454fd5a6c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 22 Jun 2023 09:23:02 +0100
Subject: [PATCH 622/740] build(deps): bump golang from `a3598b9` to `4875242`
 in /examples/shared/golang (#28091)

build(deps): bump golang in /examples/shared/golang

Bumps golang from `a3598b9` to `4875242`.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/golang/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile
index d0f1c2fb01f2..55a59f6cae82 100644
--- a/examples/shared/golang/Dockerfile
+++ b/examples/shared/golang/Dockerfile
@@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache
 
 
-FROM golang:1.20.5-bullseye@sha256:a3598b93d32819f1759893c532fa186bc61d58f1ced9aa49c2c77fe13383159a as golang-base
+FROM golang:1.20.5-bullseye@sha256:48752423122c351658c15f2575e05cd82a7c629e6182635db8fdc74e99370b1d as golang-base
 
 
 FROM golang-base as golang-control-plane-builder

From e781a3fa208f4eba77d94d7a39aa6c09e0c064e8 Mon Sep 17 00:00:00 2001
From: Tianyu <72890320+tyxia@users.noreply.github.com>
Date: Thu, 22 Jun 2023 09:56:12 -0400
Subject: [PATCH 623/740] Composite filter: Switch the order to create from
 factory context first (#28021)

* tweak the order of filter factory creation function

Signed-off-by: tyxia 
---
 .../filters/http/composite/action.cc          | 38 +++++++++++++++++
 .../filters/http/composite/action.h           | 41 +------------------
 2 files changed, 40 insertions(+), 39 deletions(-)

diff --git a/source/extensions/filters/http/composite/action.cc b/source/extensions/filters/http/composite/action.cc
index 743430b5fad3..40fdbe1bf45f 100644
--- a/source/extensions/filters/http/composite/action.cc
+++ b/source/extensions/filters/http/composite/action.cc
@@ -7,6 +7,44 @@ namespace Composite {
 void ExecuteFilterAction::createFilters(Http::FilterChainFactoryCallbacks& callbacks) const {
   cb_(callbacks);
 }
+
+Matcher::ActionFactoryCb ExecuteFilterActionFactory::createActionFactoryCb(
+    const Protobuf::Message& config, Http::Matching::HttpFilterActionContext& context,
+    ProtobufMessage::ValidationVisitor& validation_visitor) {
+  const auto& composite_action = MessageUtil::downcastAndValidate<
+      const envoy::extensions::filters::http::composite::v3::ExecuteFilterAction&>(
+      config, validation_visitor);
+
+  auto& factory =
+      Config::Utility::getAndCheckFactory(
+          composite_action.typed_config());
+  ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig(
+      composite_action.typed_config().typed_config(), validation_visitor, factory);
+
+  Envoy::Http::FilterFactoryCb callback = nullptr;
+
+  // First, try to create the filter factory creation function from factory context (if exists).
+  if (context.factory_context_.has_value()) {
+    callback = factory.createFilterFactoryFromProto(*message, context.stat_prefix_,
+                                                    context.factory_context_.value());
+  }
+
+  // If above failed, try to create the filter factory creation function from server factory
+  // context (if exists).
+  if (callback == nullptr && context.server_factory_context_.has_value()) {
+    callback = factory.createFilterFactoryFromProtoWithServerContext(
+        *message, context.stat_prefix_, context.server_factory_context_.value());
+  }
+
+  if (callback == nullptr) {
+    throw EnvoyException("Failed to get filter factory creation function");
+  }
+
+  return [cb = std::move(callback)]() -> Matcher::ActionPtr {
+    return std::make_unique(cb);
+  };
+}
+
 REGISTER_FACTORY(ExecuteFilterActionFactory,
                  Matcher::ActionFactory);
 } // namespace Composite
diff --git a/source/extensions/filters/http/composite/action.h b/source/extensions/filters/http/composite/action.h
index 9457614a9b84..725eadada76d 100644
--- a/source/extensions/filters/http/composite/action.h
+++ b/source/extensions/filters/http/composite/action.h
@@ -27,49 +27,12 @@ class ExecuteFilterActionFactory
       public Matcher::ActionFactory {
 public:
   std::string name() const override { return "composite-action"; }
+
   Matcher::ActionFactoryCb
   createActionFactoryCb(const Protobuf::Message& config,
                         Http::Matching::HttpFilterActionContext& context,
-                        ProtobufMessage::ValidationVisitor& validation_visitor) override {
-    const auto& composite_action = MessageUtil::downcastAndValidate<
-        const envoy::extensions::filters::http::composite::v3::ExecuteFilterAction&>(
-        config, validation_visitor);
-
-    auto& factory =
-        Config::Utility::getAndCheckFactory(
-            composite_action.typed_config());
-    ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig(
-        composite_action.typed_config().typed_config(), validation_visitor, factory);
-
-    Envoy::Http::FilterFactoryCb callback = nullptr;
+                        ProtobufMessage::ValidationVisitor& validation_visitor) override;
 
-    // TODO(tyxia) Update the logic later to create the filter from the `factoryContext` first if it
-    // is present.
-    TRY_NEEDS_AUDIT {
-      if (context.server_factory_context_.has_value()) {
-        callback = factory.createFilterFactoryFromProtoWithServerContext(
-            *message, context.stat_prefix_, context.server_factory_context_.value());
-      }
-    }
-    END_TRY CATCH(EnvoyException & e, {
-      // First, we try to create the delegated filter creation callback from server factory context.
-      // If it failed (i.e., the corresponding filter doesn't support this method), we log this
-      // message and fallback to creating the filter from factory context.
-      ENVOY_LOG(trace,
-                absl::StrCat(e.what(), ", fallback to creating the filter from factory context."));
-    });
-
-    if (callback == nullptr) {
-      RELEASE_ASSERT(context.factory_context_.has_value(),
-                     "The factory context must exist here to create the delegated filter");
-      callback = factory.createFilterFactoryFromProto(*message, context.stat_prefix_,
-                                                      context.factory_context_.value());
-    }
-
-    return [cb = std::move(callback)]() -> Matcher::ActionPtr {
-      return std::make_unique(cb);
-    };
-  }
   ProtobufTypes::MessagePtr createEmptyConfigProto() override {
     return std::make_unique();
   }

From 9c6e75062ebdd8c8382c671662fb096569d9eaa9 Mon Sep 17 00:00:00 2001
From: yanjunxiang-google
 <78807980+yanjunxiang-google@users.noreply.github.com>
Date: Thu, 22 Jun 2023 10:03:24 -0400
Subject: [PATCH 624/740] Fix ext_proc filter can not send non-utf8 character
 by gRPC (#27865)

* Fix ext_proc filter can not send non-utf8 character by gRPC

Signed-off-by: Yanjun Xiang 
---
 api/envoy/config/core/v3/base.proto           | 12 ++-
 .../ext_proc/v3/external_processor.proto      | 24 +++++
 changelogs/current.yaml                       |  9 ++
 source/common/runtime/runtime_features.cc     |  1 +
 .../filters/http/ext_proc/mutation_utils.cc   | 31 +++++--
 test/extensions/filters/http/ext_proc/BUILD   |  2 +
 .../ext_proc/ext_proc_integration_test.cc     | 88 ++++++++++++++++++-
 .../filters/http/ext_proc/filter_test.cc      |  4 +
 .../http/ext_proc/mutation_utils_test.cc      |  7 ++
 .../ext_proc/streaming_integration_test.cc    |  4 +
 .../extensions/filters/http/ext_proc/utils.cc |  6 +-
 11 files changed, 179 insertions(+), 9 deletions(-)

diff --git a/api/envoy/config/core/v3/base.proto b/api/envoy/config/core/v3/base.proto
index 383b3d96398a..2a6c944665c8 100644
--- a/api/envoy/config/core/v3/base.proto
+++ b/api/envoy/config/core/v3/base.proto
@@ -324,8 +324,18 @@ message HeaderValue {
   // The same :ref:`format specifier ` as used for
   // :ref:`HTTP access logging ` applies here, however
   // unknown header values are replaced with the empty string instead of ``-``.
+  // Header value is encoded as string. This does not work for non-utf8 characters.
+  // Only one of ``value`` or ``value_bytes`` can be set.
   string value = 2 [
-    (validate.rules).string = {max_bytes: 16384 well_known_regex: HTTP_HEADER_VALUE strict: false}
+    (validate.rules).string = {max_bytes: 16384 well_known_regex: HTTP_HEADER_VALUE strict: false},
+    (udpa.annotations.field_migrate).oneof_promotion = "value_type"
+  ];
+
+  // Header value is encoded as bytes which can support non-utf8 characters.
+  // Only one of ``value`` or ``value_bytes`` can be set.
+  bytes value_bytes = 3 [
+    (validate.rules).bytes = {min_len: 0 max_len: 16384},
+    (udpa.annotations.field_migrate).oneof_promotion = "value_type"
   ];
 }
 
diff --git a/api/envoy/service/ext_proc/v3/external_processor.proto b/api/envoy/service/ext_proc/v3/external_processor.proto
index 21ae8c032cd0..9d33ffb3d051 100644
--- a/api/envoy/service/ext_proc/v3/external_processor.proto
+++ b/api/envoy/service/ext_proc/v3/external_processor.proto
@@ -199,6 +199,12 @@ message ProcessingResponse {
 message HttpHeaders {
   // The HTTP request headers. All header keys will be
   // lower-cased, because HTTP header keys are case-insensitive.
+  // The ``headers`` encoding is based on the runtime guard
+  // envoy_reloadable_features_send_header_value_in_bytes setting.
+  // When it is true, the header value is encoded in the
+  // :ref:`value_bytes ` field.
+  // When it is false, the header value is encoded in the
+  // :ref:`value ` field.
   config.core.v3.HeaderMap headers = 1;
 
   // [#not-implemented-hide:]
@@ -223,6 +229,12 @@ message HttpBody {
 
 // This message contains the trailers.
 message HttpTrailers {
+  // The ``trailers`` encoding is based on the runtime guard
+  // envoy_reloadable_features_send_header_value_in_bytes setting.
+  // When it is true, the header value is encoded in the
+  // :ref:`value_bytes ` field.
+  // When it is false, the header value is encoded in the
+  // :ref:`value ` field.
   config.core.v3.HeaderMap trailers = 1;
 }
 
@@ -290,6 +302,12 @@ message CommonResponse {
   // Add new trailers to the message. This may be used when responding to either a
   // HttpHeaders or HttpBody message, but only if this message is returned
   // along with the CONTINUE_AND_REPLACE status.
+  // The ``trailers`` encoding is based on the runtime guard
+  // envoy_reloadable_features_send_header_value_in_bytes setting.
+  // When it is true, the header value is encoded in the
+  // :ref:`value_bytes ` field.
+  // When it is false, the header value is encoded in the
+  // :ref:`value ` field.
   config.core.v3.HeaderMap trailers = 4;
 
   // Clear the route cache for the current client request. This is necessary
@@ -337,6 +355,12 @@ message HeaderMutation {
   // Add or replace HTTP headers. Attempts to set the value of
   // any ``x-envoy`` header, and attempts to set the ``:method``,
   // ``:authority``, ``:scheme``, or ``host`` headers will be ignored.
+  // The ``set_headers`` encoding is based on the runtime guard
+  // envoy_reloadable_features_send_header_value_in_bytes setting.
+  // When it is true, the header value is encoded in the
+  // :ref:`value_bytes ` field.
+  // When it is false, the header value is encoded in the
+  // :ref:`value ` field.
   repeated config.core.v3.HeaderValueOption set_headers = 1;
 
   // Remove these HTTP headers. Attempts to remove system headers --
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 3ada7ca49727..1c6d195cba49 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -15,6 +15,15 @@ behavior_changes:
     When ``append_x_forwarded_host`` is enabled for a given route action it is now only appended iff it is different from the last
     value in the list. This resolves issues where a retry caused the same value to be appended multiple times. This
     behavioral change can be temporarily reverted by setting runtime guard ``envoy_reloadable_features_append_xfh_idempotent`` to false.
+- area: ext_proc
+  change: |
+    The proto field :ref:`value ` type is string.
+    This make it unable to support enconding non-utf8 characters in the HeaderValue message.
+    To support sending header value with non-utf8 characters, a new proto field is added in the HeaderValue message:
+    :ref:`value_bytes `.
+    The header values are now encoded in this ``value_bytes`` field when Envoy ext_proc filter sending
+    and receiving messages from the ext_proc server. This behavioral change can be temporarily
+    reverted by setting the runtime guard ``envoy_reloadable_features_send_header_value_in_bytes`` to false.
 - area: ext_proc
   change: |
     Apply header mutation rules from the ext_proc config to the ImmediateResponse. This behavior change can be temporarily
diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc
index 05dca2e7c39c..214a6ee911a0 100644
--- a/source/common/runtime/runtime_features.cc
+++ b/source/common/runtime/runtime_features.cc
@@ -65,6 +65,7 @@ RUNTIME_GUARD(envoy_reloadable_features_prohibit_route_refresh_after_response_he
 RUNTIME_GUARD(envoy_reloadable_features_quic_defer_logging_to_ack_listener);
 RUNTIME_GUARD(envoy_reloadable_features_reject_require_client_certificate_with_quic);
 RUNTIME_GUARD(envoy_reloadable_features_sanitize_original_path);
+RUNTIME_GUARD(envoy_reloadable_features_send_header_value_in_bytes);
 RUNTIME_GUARD(envoy_reloadable_features_service_sanitize_non_utf8_strings);
 RUNTIME_GUARD(envoy_reloadable_features_skip_dns_lookup_for_proxied_requests);
 RUNTIME_GUARD(envoy_reloadable_features_stateful_session_encode_ttl_in_cookie);
diff --git a/source/extensions/filters/http/ext_proc/mutation_utils.cc b/source/extensions/filters/http/ext_proc/mutation_utils.cc
index 390e6b139e78..91c3a4b82214 100644
--- a/source/extensions/filters/http/ext_proc/mutation_utils.cc
+++ b/source/extensions/filters/http/ext_proc/mutation_utils.cc
@@ -38,7 +38,12 @@ void MutationUtils::headersToProto(const Http::HeaderMap& headers_in,
     if (header_matchers.empty() || headerInAllowList(e.key().getStringView(), header_matchers)) {
       auto* new_header = proto_out.add_headers();
       new_header->set_key(std::string(e.key().getStringView()));
-      new_header->set_value(MessageUtil::sanitizeUtf8String(e.value().getStringView()));
+      // Setting up value or value_bytes field based on the runtime flag.
+      if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.send_header_value_in_bytes")) {
+        new_header->set_value_bytes(std::string(e.value().getStringView()));
+      } else {
+        new_header->set_value(MessageUtil::sanitizeUtf8String(e.value().getStringView()));
+      }
     }
     return Http::HeaderMap::Iterate::Continue;
   });
@@ -76,8 +81,24 @@ absl::Status MutationUtils::applyHeaderMutations(const HeaderMutation& mutation,
     if (!sh.has_header()) {
       continue;
     }
+
+    // Only one of value or value_bytes in the HeaderValue message should be set.
+    if (!sh.header().value().empty() && !sh.header().value_bytes().empty()) {
+      ENVOY_LOG(debug, "Only one of value or value_bytes in the HeaderValue message should be set, "
+                       "may not be append.");
+      rejected_mutations.inc();
+      return absl::InvalidArgumentError(
+          "Only one of value or value_bytes in the HeaderValue message should be set.");
+    }
+
+    absl::string_view header_value;
+    if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.send_header_value_in_bytes")) {
+      header_value = sh.header().value_bytes();
+    } else {
+      header_value = sh.header().value();
+    }
     if (!Http::HeaderUtility::headerNameIsValid(sh.header().key()) ||
-        !Http::HeaderUtility::headerValueIsValid(sh.header().value())) {
+        !Http::HeaderUtility::headerValueIsValid(header_value)) {
       ENVOY_LOG(debug,
                 "set_headers contain invalid character in key or value, may not be appended.");
       rejected_mutations.inc();
@@ -87,7 +108,7 @@ absl::Status MutationUtils::applyHeaderMutations(const HeaderMutation& mutation,
     const bool append = PROTOBUF_GET_WRAPPED_OR_DEFAULT(sh, append, false);
     const auto check_op = (append && !headers.get(header_name).empty()) ? CheckOperation::APPEND
                                                                         : CheckOperation::SET;
-    auto check_result = checker.check(check_op, header_name, sh.header().value());
+    auto check_result = checker.check(check_op, header_name, header_value);
     if (replacing_message && header_name == Http::Headers::get().Method) {
       // Special handling to allow changing ":method" when the
       // CONTINUE_AND_REPLACE option is selected, to stay compatible.
@@ -97,9 +118,9 @@ absl::Status MutationUtils::applyHeaderMutations(const HeaderMutation& mutation,
     case CheckResult::OK:
       ENVOY_LOG(trace, "Setting header {} append = {}", sh.header().key(), append);
       if (append) {
-        headers.addCopy(header_name, sh.header().value());
+        headers.addCopy(header_name, header_value);
       } else {
-        headers.setCopy(header_name, sh.header().value());
+        headers.setCopy(header_name, header_value);
       }
       break;
     case CheckResult::IGNORE:
diff --git a/test/extensions/filters/http/ext_proc/BUILD b/test/extensions/filters/http/ext_proc/BUILD
index 58c52ad997bc..4f6af31bc0f7 100644
--- a/test/extensions/filters/http/ext_proc/BUILD
+++ b/test/extensions/filters/http/ext_proc/BUILD
@@ -107,6 +107,7 @@ envoy_extension_cc_test(
         "//source/extensions/filters/common/mutation_rules:mutation_rules_lib",
         "//source/extensions/filters/http/ext_proc:mutation_utils_lib",
         "//test/mocks/stats:stats_mocks",
+        "//test/test_common:test_runtime_lib",
         "//test/test_common:utility_lib",
         "@envoy_api//envoy/config/common/mutation_rules/v3:pkg_cc_proto",
     ],
@@ -148,6 +149,7 @@ envoy_extension_cc_test(
         "//source/extensions/filters/http/ext_proc:config",
         "//test/common/http:common_lib",
         "//test/integration:http_integration_lib",
+        "//test/test_common:test_runtime_lib",
         "//test/test_common:utility_lib",
         "@com_google_absl//absl/strings:str_format",
         "@envoy_api//envoy/extensions/filters/http/ext_proc/v3:pkg_cc_proto",
diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc
index 726c69c14a48..44b977451684 100644
--- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc
+++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc
@@ -69,7 +69,11 @@ class ExtProcIntegrationTest : public HttpIntegrationTest,
 
   void initializeConfig() {
     scoped_runtime_.mergeValues(
-        {{"envoy.reloadable_features.send_header_value_in_bytes", filter_mutation_rule_}});
+        {{"envoy.reloadable_features.send_header_value_in_bytes", header_value_bytes_}});
+    scoped_runtime_.mergeValues(
+        {{"envoy_reloadable_features_immediate_response_use_filter_mutation_rule",
+          filter_mutation_rule_}});
+
     config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
       // Ensure "HTTP2 with no prior knowledge." Necessary for gRPC and for headers
       ConfigHelper::setHttp2(
@@ -385,6 +389,7 @@ class ExtProcIntegrationTest : public HttpIntegrationTest,
   FakeHttpConnectionPtr processor_connection_;
   FakeStreamPtr processor_stream_;
   TestScopedRuntime scoped_runtime_;
+  std::string header_value_bytes_{"false"};
   std::string filter_mutation_rule_{"false"};
 };
 
@@ -560,7 +565,7 @@ TEST_P(ExtProcIntegrationTest, GetAndSetHeaders) {
   verifyDownstreamResponse(*response, 200);
 }
 
-TEST_P(ExtProcIntegrationTest, GetAndSetHeadersNonUtf8) {
+TEST_P(ExtProcIntegrationTest, GetAndSetHeadersNonUtf8WithValueInString) {
   initializeConfig();
   HttpIntegrationTest::initialize();
   auto response = sendDownstreamRequest([](Http::HeaderMap& headers) {
@@ -582,6 +587,8 @@ TEST_P(ExtProcIntegrationTest, GetAndSetHeadersNonUtf8) {
             {"x-bad-utf8", "valid_prefix!(valid_suffix"},
             {"x-forwarded-proto", "http"}};
         for (const auto& header : headers.headers().headers()) {
+          EXPECT_TRUE(!header.value().empty());
+          EXPECT_TRUE(header.value_bytes().empty());
           ENVOY_LOG_MISC(critical, "{}", header.value());
         }
         EXPECT_THAT(headers.headers(), HeaderProtosEqual(expected_request_headers));
@@ -610,6 +617,83 @@ TEST_P(ExtProcIntegrationTest, GetAndSetHeadersNonUtf8) {
   verifyDownstreamResponse(*response, 200);
 }
 
+TEST_P(ExtProcIntegrationTest, GetAndSetHeadersNonUtf8WithValueInBytes) {
+  proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP);
+  // Set up runtime flag to have header value encoded in value_bytes.
+  header_value_bytes_ = "true";
+  initializeConfig();
+  HttpIntegrationTest::initialize();
+  auto response = sendDownstreamRequest([](Http::HeaderMap& headers) {
+    std::string invalid_unicode("valid_prefix");
+    invalid_unicode.append(1, char(0xc3));
+    invalid_unicode.append(1, char(0x28));
+    invalid_unicode.append("valid_suffix");
+
+    headers.addCopy(LowerCaseString("x-bad-utf8"), invalid_unicode);
+  });
+
+  // Verify the encoded non-utf8 character is received by the server as it is. Then send back a
+  // response with non-utf8 character in the header value, and verify it is received by Envoy as it
+  // is.
+  processRequestHeadersMessage(
+      *grpc_upstreams_[0], true, [](const HttpHeaders& headers, HeadersResponse& headers_resp) {
+        Http::TestRequestHeaderMapImpl expected_request_headers{
+            {":scheme", "http"},
+            {":method", "GET"},
+            {"host", "host"},
+            {":path", "/"},
+            {"x-bad-utf8", "valid_prefix\303(valid_suffix"},
+            {"x-forwarded-proto", "http"}};
+        for (const auto& header : headers.headers().headers()) {
+          EXPECT_TRUE(header.value().empty());
+          EXPECT_TRUE(!header.value_bytes().empty());
+          ENVOY_LOG_MISC(critical, "{}", header.value_bytes());
+        }
+        EXPECT_THAT(headers.headers(), HeaderProtosEqual(expected_request_headers));
+
+        auto response_header_mutation = headers_resp.mutable_response()->mutable_header_mutation();
+        response_header_mutation->add_remove_headers("x-bad-utf8");
+        auto* mut1 = response_header_mutation->add_set_headers();
+        mut1->mutable_header()->set_key("x-new-utf8");
+        // Construct a non-utf8 header value and send back to Envoy.
+        std::string invalid_unicode("valid_prefix");
+        invalid_unicode.append(1, char(0xc3));
+        invalid_unicode.append(1, char(0x28));
+        invalid_unicode.append("valid_suffix");
+        mut1->mutable_header()->set_value_bytes(invalid_unicode);
+        return true;
+      });
+
+  ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_));
+  ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_));
+  ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_));
+  EXPECT_THAT(upstream_request_->headers(), HasNoHeader("x-bad-utf8"));
+  EXPECT_THAT(upstream_request_->headers(),
+              SingleHeaderValueIs("x-new-utf8", "valid_prefix\303(valid_suffix"));
+  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true);
+  verifyDownstreamResponse(*response, 200);
+}
+
+TEST_P(ExtProcIntegrationTest, BothValueAndValueBytesAreSetInHeaderValueWrong) {
+  proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP);
+  // Set up runtime flag to have header value encoded in value_bytes.
+  header_value_bytes_ = "true";
+  initializeConfig();
+  HttpIntegrationTest::initialize();
+  auto response = sendDownstreamRequest(absl::nullopt);
+
+  processRequestHeadersMessage(
+      *grpc_upstreams_[0], true, [](const HttpHeaders&, HeadersResponse& headers_resp) {
+        auto response_header_mutation = headers_resp.mutable_response()->mutable_header_mutation();
+        auto* mut1 = response_header_mutation->add_set_headers();
+        mut1->mutable_header()->set_key("x-new-header");
+        mut1->mutable_header()->set_value("foo");
+        mut1->mutable_header()->set_value_bytes("bar");
+        return true;
+      });
+  verifyDownstreamResponse(*response, 500);
+}
+
 // Test the filter with body buffering turned on, but sending a GET
 // and a response that both have no body.
 TEST_P(ExtProcIntegrationTest, GetBufferedButNoBodies) {
diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc
index 3c77b8c99957..64c019e4b264 100644
--- a/test/extensions/filters/http/ext_proc/filter_test.cc
+++ b/test/extensions/filters/http/ext_proc/filter_test.cc
@@ -29,6 +29,7 @@
 #include "test/mocks/tracing/mocks.h"
 #include "test/mocks/upstream/cluster_manager.h"
 #include "test/test_common/printers.h"
+#include "test/test_common/test_runtime.h"
 #include "test/test_common/utility.h"
 
 #include "gmock/gmock.h"
@@ -74,6 +75,8 @@ static const std::string filter_config_name = "scooby.dooby.doo";
 class HttpFilterTest : public testing::Test {
 protected:
   void initialize(std::string&& yaml) {
+    scoped_runtime_.mergeValues(
+        {{"envoy.reloadable_features.send_header_value_in_bytes", "false"}});
     client_ = std::make_unique();
     route_ = std::make_shared>();
     EXPECT_CALL(*client_, start(_, _, _)).WillOnce(Invoke(this, &HttpFilterTest::doStart));
@@ -333,6 +336,7 @@ class HttpFilterTest : public testing::Test {
   Http::TestRequestTrailerMapImpl request_trailers_;
   Http::TestResponseTrailerMapImpl response_trailers_;
   std::vector timers_;
+  TestScopedRuntime scoped_runtime_;
 };
 
 // Using the default configuration, test the filter with a processor that
diff --git a/test/extensions/filters/http/ext_proc/mutation_utils_test.cc b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc
index bc8d36d9d77e..711d73e9cd95 100644
--- a/test/extensions/filters/http/ext_proc/mutation_utils_test.cc
+++ b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc
@@ -5,6 +5,7 @@
 
 #include "test/extensions/filters/http/ext_proc/utils.h"
 #include "test/mocks/stats/mocks.h"
+#include "test/test_common/test_runtime.h"
 #include "test/test_common/utility.h"
 
 #include "gtest/gtest.h"
@@ -47,6 +48,8 @@ TEST(MutationUtils, TestBuildHeaders) {
 }
 
 TEST(MutationUtils, TestApplyMutations) {
+  TestScopedRuntime scoped_runtime;
+  scoped_runtime.mergeValues({{"envoy.reloadable_features.send_header_value_in_bytes", "false"}});
   Http::TestRequestHeaderMapImpl headers{
       {":scheme", "https"},
       {":method", "GET"},
@@ -154,6 +157,8 @@ TEST(MutationUtils, TestApplyMutations) {
 }
 
 TEST(MutationUtils, TestNonAppendableHeaders) {
+  TestScopedRuntime scoped_runtime;
+  scoped_runtime.mergeValues({{"envoy.reloadable_features.send_header_value_in_bytes", "false"}});
   Http::TestRequestHeaderMapImpl headers;
   envoy::service::ext_proc::v3::HeaderMutation mutation;
   auto* s = mutation.add_set_headers();
@@ -190,6 +195,8 @@ TEST(MutationUtils, TestNonAppendableHeaders) {
 }
 
 TEST(MutationUtils, TestSetHeaderWithInvalidCharacter) {
+  TestScopedRuntime scoped_runtime;
+  scoped_runtime.mergeValues({{"envoy.reloadable_features.send_header_value_in_bytes", "false"}});
   Http::TestRequestHeaderMapImpl headers{
       {":method", "GET"},
       {"host", "localhost:1000"},
diff --git a/test/extensions/filters/http/ext_proc/streaming_integration_test.cc b/test/extensions/filters/http/ext_proc/streaming_integration_test.cc
index 5b6d46413e21..7108ebccea6f 100644
--- a/test/extensions/filters/http/ext_proc/streaming_integration_test.cc
+++ b/test/extensions/filters/http/ext_proc/streaming_integration_test.cc
@@ -8,6 +8,7 @@
 #include "test/extensions/filters/http/ext_proc/test_processor.h"
 #include "test/extensions/filters/http/ext_proc/utils.h"
 #include "test/integration/http_integration.h"
+#include "test/test_common/test_runtime.h"
 #include "test/test_common/utility.h"
 
 #include "absl/strings/str_format.h"
@@ -42,6 +43,8 @@ class StreamingIntegrationTest : public HttpIntegrationTest,
   }
 
   void initializeConfig() {
+    scoped_runtime_.mergeValues(
+        {{"envoy.reloadable_features.send_header_value_in_bytes", "false"}});
     // This enables a built-in automatic upstream server.
     autonomous_upstream_ = true;
     proto_config_.set_allow_mode_override(true);
@@ -139,6 +142,7 @@ class StreamingIntegrationTest : public HttpIntegrationTest,
   IntegrationStreamDecoderPtr client_response_;
   std::atomic processor_request_hash_;
   std::atomic processor_response_hash_;
+  TestScopedRuntime scoped_runtime_;
 };
 
 // Ensure that the test suite is run with all combinations the Envoy and Google gRPC clients.
diff --git a/test/extensions/filters/http/ext_proc/utils.cc b/test/extensions/filters/http/ext_proc/utils.cc
index 4bc448909b18..abf062146659 100644
--- a/test/extensions/filters/http/ext_proc/utils.cc
+++ b/test/extensions/filters/http/ext_proc/utils.cc
@@ -19,7 +19,11 @@ bool ExtProcTestUtility::headerProtosEqualIgnoreOrder(
   Http::TestRequestHeaderMapImpl actual_headers;
   for (const auto& header : actual.headers()) {
     if (!ignoredHeaders().contains(header.key())) {
-      actual_headers.addCopy(header.key(), header.value());
+      if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.send_header_value_in_bytes")) {
+        actual_headers.addCopy(header.key(), header.value_bytes());
+      } else {
+        actual_headers.addCopy(header.key(), header.value());
+      }
     }
   }
   return TestUtility::headerMapEqualIgnoreOrder(expected, actual_headers);

From b4f37553d6887447f942a1aedbc8c2dacae45537 Mon Sep 17 00:00:00 2001
From: Jeongseok Son 
Date: Thu, 22 Jun 2023 07:06:38 -0700
Subject: [PATCH 625/740] http: add CONNECT-UDP support (#27714)

Commit Message:
This commit adds CONNECT-UDP (RFC 9298) support. UdpConnPool is added to create a UDP socket for a new CONNECT-UDP request, and UDPUpstream is added to maintain the socket and other relevant data associated with UDP upstreams.

We added an integration test for the terminating CONNECT-UDP proxy, but not the forwarding proxy in this commit. We are going to add test cases to cover the forwarding proxy scenario in a subsequent commit.

Additional Description:
Risk Level: Medium, the feature can only be enabled by the new configuration added in this commit.
Testing: Integration test
Runtime guard: envoy.reloadable_features.enable_connect_udp_support
Release Notes: added support for CONNECT-UDP (RFC 9298). Can be disabled by setting runtime feature envoy.reloadable_features.enable_connect_udp_support to false.

Signed-off-by: Jeongseok Son 
Co-authored-by: asingh-g 
---
 api/BUILD                                     |   1 +
 .../extensions/upstreams/http/udp/v3/BUILD    |   9 +
 .../http/udp/v3/udp_connection_pool.proto     |  18 ++
 api/versioning/BUILD                          |   1 +
 bazel/external/quiche.BUILD                   |  15 ++
 changelogs/current.yaml                       |   4 +
 envoy/network/connection.h                    |   2 +-
 envoy/router/router.h                         |   8 +-
 envoy/stream_info/stream_info.h               |   2 +
 source/common/http/conn_manager_impl.cc       |   9 +
 source/common/http/header_utility.cc          |  49 ++++
 source/common/http/header_utility.h           |  11 +
 source/common/http/headers.h                  |   2 +
 source/common/network/connection_impl_base.h  |   2 +-
 .../network/multi_connection_base_impl.cc     |   2 +-
 .../network/multi_connection_base_impl.h      |   2 +-
 .../common/quic/envoy_quic_client_session.h   |   8 +-
 .../common/quic/envoy_quic_client_stream.cc   |   7 +
 source/common/quic/envoy_quic_client_stream.h |  13 +-
 .../common/quic/envoy_quic_server_session.h   |   6 +-
 .../common/quic/envoy_quic_server_stream.cc   |  13 +
 source/common/quic/envoy_quic_server_stream.h |  14 +-
 source/common/quic/http_datagram_handler.cc   |  11 +-
 .../quic_filter_manager_connection_impl.h     |   2 +-
 source/common/router/config_impl.cc           |   8 +-
 source/common/router/router.cc                |  19 +-
 source/common/runtime/runtime_features.cc     |   1 +
 source/extensions/extensions_build_config.bzl |   1 +
 source/extensions/extensions_metadata.yaml    |   5 +
 .../extensions/upstreams/http/generic/BUILD   |   1 +
 .../upstreams/http/generic/config.cc          |  30 ++-
 .../upstreams/http/generic/config.h           |   3 +-
 .../extensions/upstreams/http/http/config.cc  |   8 +-
 .../extensions/upstreams/http/http/config.h   |   3 +-
 .../upstreams/http/http/upstream_request.h    |   3 +-
 .../extensions/upstreams/http/tcp/config.cc   |  10 +-
 source/extensions/upstreams/http/tcp/config.h |   3 +-
 .../upstreams/http/tcp/upstream_request.cc    |   2 +-
 .../upstreams/http/tcp/upstream_request.h     |   5 +-
 source/extensions/upstreams/http/udp/BUILD    |  56 ++++
 .../extensions/upstreams/http/udp/config.cc   |  25 ++
 source/extensions/upstreams/http/udp/config.h |  37 +++
 .../upstreams/http/udp/upstream_request.cc    | 159 +++++++++++
 .../upstreams/http/udp/upstream_request.h     | 116 ++++++++
 source/server/api_listener_impl.h             |   2 +-
 test/common/http/hcm_router_fuzz_test.cc      |  12 +-
 test/common/http/header_utility_test.cc       |  48 ++++
 .../quic/envoy_quic_client_stream_test.cc     |   1 -
 .../quic/envoy_quic_server_stream_test.cc     |   2 -
 test/config/utility.cc                        |  22 ++
 test/config/utility.h                         |   4 +
 test/extensions/upstreams/http/generic/BUILD  |  20 ++
 .../upstreams/http/generic/config_test.cc     |  58 ++++
 .../http/tcp/upstream_request_test.cc         |   3 +-
 test/extensions/upstreams/http/udp/BUILD      |  45 ++++
 .../upstreams/http/udp/config_test.cc         |  49 ++++
 .../http/udp/upstream_request_test.cc         | 192 ++++++++++++++
 test/integration/BUILD                        |  21 ++
 .../udp_tunneling_integration_test.cc         | 251 ++++++++++++++++++
 .../upstreams/per_host_upstream_config.h      |  13 +-
 test/mocks/network/connection.h               |   2 +-
 test/per_file_coverage.sh                     |   1 +
 62 files changed, 1371 insertions(+), 81 deletions(-)
 create mode 100644 api/envoy/extensions/upstreams/http/udp/v3/BUILD
 create mode 100644 api/envoy/extensions/upstreams/http/udp/v3/udp_connection_pool.proto
 create mode 100644 source/extensions/upstreams/http/udp/BUILD
 create mode 100644 source/extensions/upstreams/http/udp/config.cc
 create mode 100644 source/extensions/upstreams/http/udp/config.h
 create mode 100644 source/extensions/upstreams/http/udp/upstream_request.cc
 create mode 100644 source/extensions/upstreams/http/udp/upstream_request.h
 create mode 100644 test/extensions/upstreams/http/generic/BUILD
 create mode 100644 test/extensions/upstreams/http/generic/config_test.cc
 create mode 100644 test/extensions/upstreams/http/udp/BUILD
 create mode 100644 test/extensions/upstreams/http/udp/config_test.cc
 create mode 100644 test/extensions/upstreams/http/udp/upstream_request_test.cc
 create mode 100644 test/integration/udp_tunneling_integration_test.cc

diff --git a/api/BUILD b/api/BUILD
index 0a9934e0207b..915c7bce63f7 100644
--- a/api/BUILD
+++ b/api/BUILD
@@ -309,6 +309,7 @@ proto_library(
         "//envoy/extensions/upstreams/http/generic/v3:pkg",
         "//envoy/extensions/upstreams/http/http/v3:pkg",
         "//envoy/extensions/upstreams/http/tcp/v3:pkg",
+        "//envoy/extensions/upstreams/http/udp/v3:pkg",
         "//envoy/extensions/upstreams/http/v3:pkg",
         "//envoy/extensions/upstreams/tcp/generic/v3:pkg",
         "//envoy/extensions/upstreams/tcp/v3:pkg",
diff --git a/api/envoy/extensions/upstreams/http/udp/v3/BUILD b/api/envoy/extensions/upstreams/http/udp/v3/BUILD
new file mode 100644
index 000000000000..ee92fb652582
--- /dev/null
+++ b/api/envoy/extensions/upstreams/http/udp/v3/BUILD
@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+)
diff --git a/api/envoy/extensions/upstreams/http/udp/v3/udp_connection_pool.proto b/api/envoy/extensions/upstreams/http/udp/v3/udp_connection_pool.proto
new file mode 100644
index 000000000000..07d637d322f9
--- /dev/null
+++ b/api/envoy/extensions/upstreams/http/udp/v3/udp_connection_pool.proto
@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+package envoy.extensions.upstreams.http.udp.v3;
+
+import "udpa/annotations/status.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.upstreams.http.udp.v3";
+option java_outer_classname = "UdpConnectionPoolProtoOuterClass";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/udp/v3;udpv3";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Udp Connection Pool]
+
+// A connection pool which forwards downstream HTTP as UDP to upstream,
+// [#extension: envoy.upstreams.http.udp]
+message UdpConnectionPoolProto {
+}
diff --git a/api/versioning/BUILD b/api/versioning/BUILD
index a256f1a4cada..58405a717c4d 100644
--- a/api/versioning/BUILD
+++ b/api/versioning/BUILD
@@ -248,6 +248,7 @@ proto_library(
         "//envoy/extensions/upstreams/http/generic/v3:pkg",
         "//envoy/extensions/upstreams/http/http/v3:pkg",
         "//envoy/extensions/upstreams/http/tcp/v3:pkg",
+        "//envoy/extensions/upstreams/http/udp/v3:pkg",
         "//envoy/extensions/upstreams/http/v3:pkg",
         "//envoy/extensions/upstreams/tcp/generic/v3:pkg",
         "//envoy/extensions/upstreams/tcp/v3:pkg",
diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD
index 1b90026a4a7c..e7492a72efa7 100644
--- a/bazel/external/quiche.BUILD
+++ b/bazel/external/quiche.BUILD
@@ -3177,6 +3177,21 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "quiche_common_connect_udp_datagram_payload_lib",
+    srcs = ["quiche/common/masque/connect_udp_datagram_payload.cc"],
+    hdrs = ["quiche/common/masque/connect_udp_datagram_payload.h"],
+    copts = quiche_copts,
+    repository = "@envoy",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":quic_core_data_lib",
+        ":quiche_common_platform_bug_tracker",
+        ":quiche_common_platform_logging",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 envoy_cc_library(
     name = "quiche_common_quiche_stream_lib",
     srcs = [],
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 1c6d195cba49..7afc3e73b97d 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -240,6 +240,10 @@ new_features:
     added Runtime feature ``envoy.reloadable_features.max_request_headers_size_kb`` to override the default value of
     :ref:`max request headers size
     `.
+- area: http
+  change: |
+    added support for CONNECT-UDP (RFC 9298). Can be disabled by setting runtime feature
+    ``envoy.reloadable_features.enable_connect_udp_support`` to false.
 - area: listeners
   change: |
     added :ref:`max_connections_to_accept_per_socket_event
diff --git a/envoy/network/connection.h b/envoy/network/connection.h
index 9bc3b4fe1b28..301366c4e54f 100644
--- a/envoy/network/connection.h
+++ b/envoy/network/connection.h
@@ -149,7 +149,7 @@ class Connection : public Event::DeferredDeletable,
   /**
    * @return Event::Dispatcher& the dispatcher backing this connection.
    */
-  virtual Event::Dispatcher& dispatcher() PURE;
+  virtual Event::Dispatcher& dispatcher() const PURE;
 
   /**
    * @return uint64_t the unique local ID of this connection.
diff --git a/envoy/router/router.h b/envoy/router/router.h
index 7cb57a8d02b9..7f9326160ba5 100644
--- a/envoy/router/router.h
+++ b/envoy/router/router.h
@@ -1523,6 +1523,11 @@ using GenericConnPoolPtr = std::unique_ptr;
  */
 class GenericConnPoolFactory : public Envoy::Config::TypedFactory {
 public:
+  /*
+   * Protocol used by the upstream sockets.
+   */
+  enum class UpstreamProtocol { HTTP, TCP, UDP };
+
   ~GenericConnPoolFactory() override = default;
 
   /*
@@ -1530,7 +1535,8 @@ class GenericConnPoolFactory : public Envoy::Config::TypedFactory {
    * @return may be null
    */
   virtual GenericConnPoolPtr
-  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
+                        GenericConnPoolFactory::UpstreamProtocol upstream_protocol,
                         const RouteEntry& route_entry,
                         absl::optional downstream_protocol,
                         Upstream::LoadBalancerContext* ctx) const PURE;
diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h
index 198cd6255759..b5fbd9108fd3 100644
--- a/envoy/stream_info/stream_info.h
+++ b/envoy/stream_info/stream_info.h
@@ -135,6 +135,8 @@ struct ResponseCodeDetailValues {
   const std::string InvalidEnvoyRequestHeaders = "request_headers_failed_strict_check";
   // The request was rejected due to a missing Path or :path header field.
   const std::string MissingPath = "missing_path_rejected";
+  // The request was rejected due to an invalid Path or :path header field.
+  const std::string InvalidPath = "invalid_path";
   // The request was rejected due to using an absolute path on a route not supporting them.
   const std::string AbsolutePath = "absolute_path_rejected";
   // The request was rejected because path normalization was configured on and failed, probably due
diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc
index 31fcaeec15cd..8dacc437d0b4 100644
--- a/source/common/http/conn_manager_impl.cc
+++ b/source/common/http/conn_manager_impl.cc
@@ -1177,6 +1177,15 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt
     return;
   }
 
+  // Rewrites the host of CONNECT-UDP requests.
+  if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") &&
+      HeaderUtility::isConnectUdp(*request_headers_) &&
+      !HeaderUtility::rewriteAuthorityForConnectUdp(*request_headers_)) {
+    sendLocalReply(Code::NotFound, "The path is incorrect for CONNECT-UDP", nullptr, absl::nullopt,
+                   StreamInfo::ResponseCodeDetails::get().InvalidPath);
+    return;
+  }
+
   // Currently we only support relative paths at the application layer.
   if (!request_headers_->getPathValue().empty() && request_headers_->getPathValue()[0] != '/') {
     connection_manager_.stats_.named_.downstream_rq_non_relative_path_.inc();
diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc
index 71f1942eb788..b5ce5a118b68 100644
--- a/source/common/http/header_utility.cc
+++ b/source/common/http/header_utility.cc
@@ -243,6 +243,11 @@ bool HeaderUtility::isConnect(const RequestHeaderMap& headers) {
   return headers.Method() && headers.Method()->value() == Http::Headers::get().MethodValues.Connect;
 }
 
+bool HeaderUtility::isConnectUdp(const RequestHeaderMap& headers) {
+  return headers.Upgrade() &&
+         headers.Upgrade()->value() == Http::Headers::get().UpgradeValues.ConnectUdp;
+}
+
 bool HeaderUtility::isConnectResponse(const RequestHeaderMap* request_headers,
                                       const ResponseHeaderMap& response_headers) {
   return request_headers && isConnect(*request_headers) &&
@@ -250,6 +255,50 @@ bool HeaderUtility::isConnectResponse(const RequestHeaderMap* request_headers,
              Http::Code::OK;
 }
 
+bool HeaderUtility::rewriteAuthorityForConnectUdp(RequestHeaderMap& headers) {
+  // Per RFC 9298, the URI template must only contain ASCII characters in the range 0x21-0x7E.
+  absl::string_view path = headers.getPathValue();
+  for (char c : path) {
+    unsigned char ascii_code = static_cast(c);
+    if (ascii_code < 0x21 || ascii_code > 0x7e) {
+      ENVOY_LOG_MISC(warn, "CONNECT-UDP request with a bad character in the path {}", path);
+      return false;
+    }
+  }
+
+  // Extract target host and port from path using default template.
+  if (!absl::StartsWith(path, "/.well-known/masque/udp/")) {
+    ENVOY_LOG_MISC(warn, "CONNECT-UDP request path is not a well-known URI: {}", path);
+    return false;
+  }
+
+  std::vector path_split = absl::StrSplit(path, '/');
+  if (path_split.size() != 7 || path_split[4].empty() || path_split[5].empty() ||
+      !path_split[6].empty()) {
+    ENVOY_LOG_MISC(warn, "CONNECT-UDP request with a malformed URI template in the path {}", path);
+    return false;
+  }
+
+  // Utility::PercentEncoding::decode never returns an empty string if the input argument is not
+  // empty.
+  std::string target_host = Utility::PercentEncoding::decode(path_split[4]);
+  // Per RFC 9298, IPv6 Zone ID is not supported.
+  if (target_host.find('%') != std::string::npos) {
+    ENVOY_LOG_MISC(warn, "CONNECT-UDP request with a non-escpaed char (%) in the path {}", path);
+    return false;
+  }
+  std::string target_port = Utility::PercentEncoding::decode(path_split[5]);
+
+  // If the host is an IPv6 address, surround the address with square brackets.
+  in6_addr sin6_addr;
+  bool is_ipv6 = (inet_pton(AF_INET6, target_host.c_str(), &sin6_addr) == 1);
+  std::string new_host =
+      absl::StrCat((is_ipv6 ? absl::StrCat("[", target_host, "]") : target_host), ":", target_port);
+  headers.setHost(new_host);
+
+  return true;
+}
+
 #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
 bool HeaderUtility::isCapsuleProtocol(const RequestOrResponseHeaderMap& headers) {
   Http::HeaderMap::GetResult capsule_protocol =
diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h
index bd63fe4c0bbe..4022a7059636 100644
--- a/source/common/http/header_utility.h
+++ b/source/common/http/header_utility.h
@@ -171,12 +171,23 @@ class HeaderUtility {
    */
   static bool isConnect(const RequestHeaderMap& headers);
 
+  /**
+   * @brief a helper function to determine if the headers represent a CONNECT-UDP request.
+   */
+  static bool isConnectUdp(const RequestHeaderMap& headers);
+
   /**
    * @brief a helper function to determine if the headers represent an accepted CONNECT response.
    */
   static bool isConnectResponse(const RequestHeaderMap* request_headers,
                                 const ResponseHeaderMap& response_headers);
 
+  /**
+   * @brief Rewrites the authority header field by parsing the path using the default CONNECT-UDP
+   * URI template. Returns true if the parsing was successful, otherwise returns false.
+   */
+  static bool rewriteAuthorityForConnectUdp(RequestHeaderMap& headers);
+
 #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
   /**
    * @brief Returns true if the Capsule-Protocol header field (RFC 9297) is set to true. If the
diff --git a/source/common/http/headers.h b/source/common/http/headers.h
index 1cda4d521eea..5c1252a8e2c6 100644
--- a/source/common/http/headers.h
+++ b/source/common/http/headers.h
@@ -142,6 +142,7 @@ class HeaderValues {
 
   const LowerCaseString ProxyAuthenticate{"proxy-authenticate"};
   const LowerCaseString ProxyAuthorization{"proxy-authorization"};
+  const LowerCaseString CapsuleProtocol{"capsule-protocol"};
   const LowerCaseString ClientTraceId{"x-client-trace-id"};
   const LowerCaseString Connection{"connection"};
   const LowerCaseString ContentLength{"content-length"};
@@ -246,6 +247,7 @@ class HeaderValues {
   struct {
     const std::string H2c{"h2c"};
     const std::string WebSocket{"websocket"};
+    const std::string ConnectUdp{"connect-udp"};
   } UpgradeValues;
 
   struct {
diff --git a/source/common/network/connection_impl_base.h b/source/common/network/connection_impl_base.h
index 4d01684ee34e..21ee7530a45d 100644
--- a/source/common/network/connection_impl_base.h
+++ b/source/common/network/connection_impl_base.h
@@ -23,7 +23,7 @@ class ConnectionImplBase : public FilterManagerConnection,
   // Network::Connection
   void addConnectionCallbacks(ConnectionCallbacks& cb) override;
   void removeConnectionCallbacks(ConnectionCallbacks& cb) override;
-  Event::Dispatcher& dispatcher() override { return dispatcher_; }
+  Event::Dispatcher& dispatcher() const override { return dispatcher_; }
   uint64_t id() const override { return id_; }
   void hashKey(std::vector& hash) const override;
   void setConnectionStats(const ConnectionStats& stats) override;
diff --git a/source/common/network/multi_connection_base_impl.cc b/source/common/network/multi_connection_base_impl.cc
index 96c232992fd2..839ad3af4cad 100644
--- a/source/common/network/multi_connection_base_impl.cc
+++ b/source/common/network/multi_connection_base_impl.cc
@@ -346,7 +346,7 @@ void MultiConnectionBaseImpl::close(ConnectionCloseType type, absl::string_view
   connections_[0]->close(type, details);
 }
 
-Event::Dispatcher& MultiConnectionBaseImpl::dispatcher() {
+Event::Dispatcher& MultiConnectionBaseImpl::dispatcher() const {
   ASSERT(&dispatcher_ == &connections_[0]->dispatcher());
   return connections_[0]->dispatcher();
 }
diff --git a/source/common/network/multi_connection_base_impl.h b/source/common/network/multi_connection_base_impl.h
index c5574fe3334f..a6521d0a0254 100644
--- a/source/common/network/multi_connection_base_impl.h
+++ b/source/common/network/multi_connection_base_impl.h
@@ -124,7 +124,7 @@ class MultiConnectionBaseImpl : public ClientConnection,
 
   // Methods implemented largely by this class itself.
   uint64_t id() const override;
-  Event::Dispatcher& dispatcher() override;
+  Event::Dispatcher& dispatcher() const override;
   void close(ConnectionCloseType type) override { close(type, ""); }
   void close(ConnectionCloseType type, absl::string_view details) override;
   bool readEnabled() const override;
diff --git a/source/common/quic/envoy_quic_client_session.h b/source/common/quic/envoy_quic_client_session.h
index e3582d4c333c..a5dff63fb883 100644
--- a/source/common/quic/envoy_quic_client_session.h
+++ b/source/common/quic/envoy_quic_client_session.h
@@ -65,9 +65,11 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl,
                                    std::unique_ptr encrypter) override;
 
   quic::HttpDatagramSupport LocalHttpDatagramSupport() override {
-    // TODO(https://github.com/envoyproxy/envoy/issues/23564): Http3 Datagram support should be
-    // turned on by returning quic::HttpDatagramSupport::kRfc once the CONNECT-UDP support work is
-    // completed.
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+    if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support")) {
+      return quic::HttpDatagramSupport::kRfc;
+    }
+#endif
     return quic::HttpDatagramSupport::kNone;
   }
 
diff --git a/source/common/quic/envoy_quic_client_stream.cc b/source/common/quic/envoy_quic_client_stream.cc
index e62805e28b2e..dca38b6518cc 100644
--- a/source/common/quic/envoy_quic_client_stream.cc
+++ b/source/common/quic/envoy_quic_client_stream.cc
@@ -87,6 +87,13 @@ Http::Status EnvoyQuicClientStream::encodeHeaders(const Http::RequestHeaderMap&
   if (headers.Method()->value() == "HEAD") {
     sent_head_request_ = true;
   }
+#endif
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+  if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") &&
+      (Http::HeaderUtility::isCapsuleProtocol(headers) ||
+       Http::HeaderUtility::isConnectUdp(headers))) {
+    useCapsuleProtocol();
+  }
 #endif
   {
     IncrementalBytesSentTracker tracker(*this, *mutableBytesMeter(), true);
diff --git a/source/common/quic/envoy_quic_client_stream.h b/source/common/quic/envoy_quic_client_stream.h
index bb55bd680155..7c40341b422f 100644
--- a/source/common/quic/envoy_quic_client_stream.h
+++ b/source/common/quic/envoy_quic_client_stream.h
@@ -51,12 +51,6 @@ class EnvoyQuicClientStream : public quic::QuicSpdyClientStream,
   void OnConnectionClosed(quic::QuicErrorCode error, quic::ConnectionCloseSource source) override;
 
   void clearWatermarkBuffer();
-#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
-  // Makes the QUIC stream use Capsule Protocol. Once this method is called, any calls to encodeData
-  // are expected to contain capsules which will be sent along as HTTP Datagrams. Also, the stream
-  // starts to receive HTTP/3 Datagrams and decode into Capsules.
-  void useCapsuleProtocol();
-#endif
 
 protected:
   // EnvoyQuicStream
@@ -83,6 +77,13 @@ class EnvoyQuicClientStream : public quic::QuicSpdyClientStream,
   // Deliver awaiting trailers if body has been delivered.
   void maybeDecodeTrailers();
 
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+  // Makes the QUIC stream use Capsule Protocol. Once this method is called, any calls to encodeData
+  // are expected to contain capsules which will be sent along as HTTP Datagrams. Also, the stream
+  // starts to receive HTTP/3 Datagrams and decode into Capsules.
+  void useCapsuleProtocol();
+#endif
+
   Http::ResponseDecoder* response_decoder_{nullptr};
   bool decoded_1xx_{false};
 #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
diff --git a/source/common/quic/envoy_quic_server_session.h b/source/common/quic/envoy_quic_server_session.h
index 1858c4e78833..94e4f145f47a 100644
--- a/source/common/quic/envoy_quic_server_session.h
+++ b/source/common/quic/envoy_quic_server_session.h
@@ -115,9 +115,11 @@ class EnvoyQuicServerSession : public quic::QuicServerSessionBase,
   quic::QuicSpdyStream* CreateOutgoingUnidirectionalStream() override;
 
   quic::HttpDatagramSupport LocalHttpDatagramSupport() override {
-    // TODO(jeongseokson): Http3 Datagram support should be turned on by returning
-    // quic::HttpDatagramSupport::kRfc once the CONNECT-UDP support work is completed.
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+    return quic::HttpDatagramSupport::kRfc;
+#else
     return quic::HttpDatagramSupport::kNone;
+#endif
   }
 
   // QuicFilterManagerConnectionImpl
diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc
index 123ac025aa14..7a12c93e7b08 100644
--- a/source/common/quic/envoy_quic_server_stream.cc
+++ b/source/common/quic/envoy_quic_server_stream.cc
@@ -41,6 +41,11 @@ EnvoyQuicServerStream::EnvoyQuicServerStream(
 
   stats_gatherer_ = new QuicStatsGatherer(&filterManagerConnection()->dispatcher().timeSource());
   set_ack_listener(stats_gatherer_);
+  // TODO(https://github.com/envoyproxy/envoy/issues/23564): Remove this line when the QUICHE is
+  // updated with a more reasonable default expiry time for QUIC Datagrams.
+  if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support")) {
+    SetMaxDatagramTimeInQueue(::quic::QuicTime::Delta::FromMilliseconds(100));
+  }
 }
 
 void EnvoyQuicServerStream::encode1xxHeaders(const Http::ResponseHeaderMap& headers) {
@@ -261,6 +266,14 @@ void EnvoyQuicServerStream::OnInitialHeadersComplete(bool fin, size_t frame_len,
   }
 #endif
 
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+  if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") &&
+      (Http::HeaderUtility::isCapsuleProtocol(*headers) ||
+       Http::HeaderUtility::isConnectUdp(*headers))) {
+    useCapsuleProtocol();
+  }
+#endif
+
   request_decoder_->decodeHeaders(std::move(headers), /*end_stream=*/fin);
   ConsumeHeaderList();
 }
diff --git a/source/common/quic/envoy_quic_server_stream.h b/source/common/quic/envoy_quic_server_stream.h
index a262dc15af27..ea9b4bf07ad8 100644
--- a/source/common/quic/envoy_quic_server_stream.h
+++ b/source/common/quic/envoy_quic_server_stream.h
@@ -79,13 +79,6 @@ class EnvoyQuicServerStream : public quic::QuicSpdyServerStreamBase,
   Http::HeaderUtility::HeaderValidationResult
   validateHeader(absl::string_view header_name, absl::string_view header_value) override;
 
-#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
-  // Makes the QUIC stream use Capsule Protocol. Once this method is called, any calls to encodeData
-  // are expected to contain capsules which will be sent along as HTTP Datagrams. Also, the stream
-  // starts to receive HTTP/3 Datagrams and decode into Capsules.
-  void useCapsuleProtocol();
-#endif
-
 protected:
   // EnvoyQuicStream
   void switchStreamBlockState() override;
@@ -113,6 +106,13 @@ class EnvoyQuicServerStream : public quic::QuicSpdyServerStreamBase,
   // Deliver awaiting trailers if body has been delivered.
   void maybeDecodeTrailers();
 
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+  // Makes the QUIC stream use Capsule Protocol. Once this method is called, any calls to encodeData
+  // are expected to contain capsules which will be sent along as HTTP Datagrams. Also, the stream
+  // starts to receive HTTP/3 Datagrams and decode into Capsules.
+  void useCapsuleProtocol();
+#endif
+
   Http::RequestDecoder* request_decoder_{nullptr};
   envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction
       headers_with_underscores_action_;
diff --git a/source/common/quic/http_datagram_handler.cc b/source/common/quic/http_datagram_handler.cc
index 804ed5c08fd3..1af50e90a510 100644
--- a/source/common/quic/http_datagram_handler.cc
+++ b/source/common/quic/http_datagram_handler.cc
@@ -51,9 +51,14 @@ bool HttpDatagramHandler::OnCapsule(const quiche::Capsule& capsule) {
   if (status == quic::MessageStatus::MESSAGE_STATUS_SUCCESS) {
     return true;
   }
-  // Drops the Datagram and move on without reporting a failure in the following statuses.
-  if (status == quic::MessageStatus::MESSAGE_STATUS_BLOCKED ||
-      status == quic::MessageStatus::MESSAGE_STATUS_TOO_LARGE) {
+  // When SendHttp3Datagram cannot send a datagram immediately, it puts it into the queue and
+  // returns MESSAGE_STATUS_BLOCKED.
+  if (status == quic::MessageStatus::MESSAGE_STATUS_BLOCKED) {
+    ENVOY_LOG(trace, fmt::format("SendHttpH3Datagram failed: status = {}, buffers the Datagram.",
+                                 quic::MessageStatusToString(status)));
+    return true;
+  }
+  if (status == quic::MessageStatus::MESSAGE_STATUS_TOO_LARGE) {
     ENVOY_LOG(warn, fmt::format("SendHttpH3Datagram failed: status = {}, drops the Datagram.",
                                 quic::MessageStatusToString(status)));
     return true;
diff --git a/source/common/quic/quic_filter_manager_connection_impl.h b/source/common/quic/quic_filter_manager_connection_impl.h
index 0b8a6ec92d15..1520e01400e4 100644
--- a/source/common/quic/quic_filter_manager_connection_impl.h
+++ b/source/common/quic/quic_filter_manager_connection_impl.h
@@ -59,7 +59,7 @@ class QuicFilterManagerConnectionImpl : public Network::ConnectionImplBase,
     }
     close(type);
   }
-  Event::Dispatcher& dispatcher() override { return dispatcher_; }
+  Event::Dispatcher& dispatcher() const override { return dispatcher_; }
   std::string nextProtocol() const override { return EMPTY_STRING; }
   // No-op. TCP_NODELAY doesn't apply to UDP.
   void noDelay(bool /*enable*/) override {}
diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc
index 29d631dc1f37..891af09a8e37 100644
--- a/source/common/router/config_impl.cc
+++ b/source/common/router/config_impl.cc
@@ -649,7 +649,9 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost,
     if (!success) {
       throw EnvoyException(absl::StrCat("Duplicate upgrade ", upgrade_config.upgrade_type()));
     }
-    if (upgrade_config.upgrade_type() == Http::Headers::get().MethodValues.Connect) {
+    if (upgrade_config.upgrade_type() == Http::Headers::get().MethodValues.Connect ||
+        (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") &&
+         upgrade_config.upgrade_type() == Http::Headers::get().UpgradeValues.ConnectUdp)) {
       connect_config_ = std::make_unique(upgrade_config.connect_config());
     } else if (upgrade_config.has_connect_config()) {
       throw EnvoyException(absl::StrCat("Non-CONNECT upgrade type ", upgrade_config.upgrade_type(),
@@ -1600,7 +1602,9 @@ ConnectRouteEntryImpl::currentUrlPathAfterRewrite(const Http::RequestHeaderMap&
 RouteConstSharedPtr ConnectRouteEntryImpl::matches(const Http::RequestHeaderMap& headers,
                                                    const StreamInfo::StreamInfo& stream_info,
                                                    uint64_t random_value) const {
-  if (Http::HeaderUtility::isConnect(headers) &&
+  if ((Http::HeaderUtility::isConnect(headers) ||
+       (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") &&
+        Http::HeaderUtility::isConnectUdp(headers))) &&
       RouteEntryImplBase::matchRoute(headers, stream_info, random_value)) {
     return clusterEntry(headers, random_value);
   }
diff --git a/source/common/router/router.cc b/source/common/router/router.cc
index 6e50cd72c999..db2e0b440723 100644
--- a/source/common/router/router.cc
+++ b/source/common/router/router.cc
@@ -769,18 +769,21 @@ Filter::createConnPool(Upstream::ThreadLocalCluster& thread_local_cluster) {
     factory = &config_.router_context_.genericConnPoolFactory();
   }
 
-  bool should_tcp_proxy = false;
-
+  using UpstreamProtocol = Envoy::Router::GenericConnPoolFactory::UpstreamProtocol;
+  UpstreamProtocol upstream_protocol = UpstreamProtocol::HTTP;
   if (route_entry_->connectConfig().has_value()) {
     auto method = downstream_headers_->getMethodValue();
-    should_tcp_proxy = (method == Http::Headers::get().MethodValues.Connect);
-
-    // Allow POST for proxying raw TCP if it is configured.
-    if (!should_tcp_proxy && route_entry_->connectConfig()->allow_post()) {
-      should_tcp_proxy = (method == Http::Headers::get().MethodValues.Post);
+    if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") &&
+        Http::HeaderUtility::isConnectUdp(*downstream_headers_)) {
+      upstream_protocol = UpstreamProtocol::UDP;
+    } else if (method == Http::Headers::get().MethodValues.Connect ||
+               (route_entry_->connectConfig()->allow_post() &&
+                method == Http::Headers::get().MethodValues.Post)) {
+      // Allow POST for proxying raw TCP if it is configured.
+      upstream_protocol = UpstreamProtocol::TCP;
     }
   }
-  return factory->createGenericConnPool(thread_local_cluster, should_tcp_proxy, *route_entry_,
+  return factory->createGenericConnPool(thread_local_cluster, upstream_protocol, *route_entry_,
                                         callbacks_->streamInfo().protocol(), this);
 }
 
diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc
index 214a6ee911a0..b4fe9e99cbef 100644
--- a/source/common/runtime/runtime_features.cc
+++ b/source/common/runtime/runtime_features.cc
@@ -38,6 +38,7 @@ RUNTIME_GUARD(envoy_reloadable_features_count_unused_mapped_pages_as_free);
 RUNTIME_GUARD(envoy_reloadable_features_dfp_mixed_scheme);
 RUNTIME_GUARD(envoy_reloadable_features_enable_aws_credentials_file);
 RUNTIME_GUARD(envoy_reloadable_features_enable_compression_bomb_protection);
+RUNTIME_GUARD(envoy_reloadable_features_enable_connect_udp_support);
 RUNTIME_GUARD(envoy_reloadable_features_enable_intermediate_ca);
 RUNTIME_GUARD(envoy_reloadable_features_enable_update_listener_socket_options);
 RUNTIME_GUARD(envoy_reloadable_features_expand_agnostic_stream_lifetime);
diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl
index 49f995c7bc32..1b777ecfb4c7 100644
--- a/source/extensions/extensions_build_config.bzl
+++ b/source/extensions/extensions_build_config.bzl
@@ -299,6 +299,7 @@ EXTENSIONS = {
 
     "envoy.upstreams.http.http":                        "//source/extensions/upstreams/http/http:config",
     "envoy.upstreams.http.tcp":                         "//source/extensions/upstreams/http/tcp:config",
+    "envoy.upstreams.http.udp":                         "//source/extensions/upstreams/http/udp:config",
 
     #
     # Watchdog actions
diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml
index 9dc8d4f64040..92dd4929745b 100644
--- a/source/extensions/extensions_metadata.yaml
+++ b/source/extensions/extensions_metadata.yaml
@@ -1221,6 +1221,11 @@ envoy.upstreams.http.tcp:
   - envoy.upstreams
   security_posture: robust_to_untrusted_downstream
   status: stable
+envoy.upstreams.http.udp:
+  categories:
+  - envoy.upstreams
+  security_posture: robust_to_untrusted_downstream
+  status: alpha
 envoy.upstreams.tcp.generic:
   categories:
   - envoy.upstreams
diff --git a/source/extensions/upstreams/http/generic/BUILD b/source/extensions/upstreams/http/generic/BUILD
index 759f4626f205..a15d7ac05eed 100644
--- a/source/extensions/upstreams/http/generic/BUILD
+++ b/source/extensions/upstreams/http/generic/BUILD
@@ -20,6 +20,7 @@ envoy_cc_extension(
     deps = [
         "//source/extensions/upstreams/http/http:upstream_request_lib",
         "//source/extensions/upstreams/http/tcp:upstream_request_lib",
+        "//source/extensions/upstreams/http/udp:upstream_request_lib",
         "@envoy_api//envoy/extensions/upstreams/http/generic/v3:pkg_cc_proto",
     ],
 )
diff --git a/source/extensions/upstreams/http/generic/config.cc b/source/extensions/upstreams/http/generic/config.cc
index b2b02e28ac56..9b09233137c1 100644
--- a/source/extensions/upstreams/http/generic/config.cc
+++ b/source/extensions/upstreams/http/generic/config.cc
@@ -2,6 +2,7 @@
 
 #include "source/extensions/upstreams/http/http/upstream_request.h"
 #include "source/extensions/upstreams/http/tcp/upstream_request.h"
+#include "source/extensions/upstreams/http/udp/upstream_request.h"
 
 namespace Envoy {
 namespace Extensions {
@@ -9,19 +10,32 @@ namespace Upstreams {
 namespace Http {
 namespace Generic {
 
+using UpstreamProtocol = Envoy::Router::GenericConnPoolFactory::UpstreamProtocol;
+
 Router::GenericConnPoolPtr GenericGenericConnPoolFactory::createGenericConnPool(
-    Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+    Upstream::ThreadLocalCluster& thread_local_cluster, UpstreamProtocol upstream_protocol,
     const Router::RouteEntry& route_entry,
     absl::optional downstream_protocol,
     Upstream::LoadBalancerContext* ctx) const {
-  if (is_connect) {
-    auto ret = std::make_unique(
-        thread_local_cluster, is_connect, route_entry, downstream_protocol, ctx);
-    return (ret->valid() ? std::move(ret) : nullptr);
+  switch (upstream_protocol) {
+  case UpstreamProtocol::HTTP: {
+    auto http_conn_pool = std::make_unique(
+        thread_local_cluster, route_entry, downstream_protocol, ctx);
+    return (http_conn_pool->valid() ? std::move(http_conn_pool) : nullptr);
+  }
+  case UpstreamProtocol::TCP: {
+    auto tcp_conn_pool =
+        std::make_unique(thread_local_cluster, route_entry, ctx);
+    return (tcp_conn_pool->valid() ? std::move(tcp_conn_pool) : nullptr);
+  }
+  case UpstreamProtocol::UDP: {
+    auto udp_conn_pool =
+        std::make_unique(thread_local_cluster, ctx);
+    return (udp_conn_pool->valid() ? std::move(udp_conn_pool) : nullptr);
   }
-  auto ret = std::make_unique(
-      thread_local_cluster, is_connect, route_entry, downstream_protocol, ctx);
-  return (ret->valid() ? std::move(ret) : nullptr);
+  }
+
+  return nullptr;
 }
 
 REGISTER_FACTORY(GenericGenericConnPoolFactory, Router::GenericConnPoolFactory);
diff --git a/source/extensions/upstreams/http/generic/config.h b/source/extensions/upstreams/http/generic/config.h
index 62f9d72602c7..524f3f22444d 100644
--- a/source/extensions/upstreams/http/generic/config.h
+++ b/source/extensions/upstreams/http/generic/config.h
@@ -18,7 +18,8 @@ class GenericGenericConnPoolFactory : public Router::GenericConnPoolFactory {
   std::string name() const override { return "envoy.filters.connection_pools.http.generic"; }
   std::string category() const override { return "envoy.upstreams"; }
   Router::GenericConnPoolPtr
-  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
+                        Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol,
                         const Router::RouteEntry& route_entry,
                         absl::optional downstream_protocol,
                         Upstream::LoadBalancerContext* ctx) const override;
diff --git a/source/extensions/upstreams/http/http/config.cc b/source/extensions/upstreams/http/http/config.cc
index 620915a11d66..7004c49caa9c 100644
--- a/source/extensions/upstreams/http/http/config.cc
+++ b/source/extensions/upstreams/http/http/config.cc
@@ -8,13 +8,15 @@ namespace Upstreams {
 namespace Http {
 namespace Http {
 
+using UpstreamProtocol = Envoy::Router::GenericConnPoolFactory::UpstreamProtocol;
+
 Router::GenericConnPoolPtr HttpGenericConnPoolFactory::createGenericConnPool(
-    Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+    Upstream::ThreadLocalCluster& thread_local_cluster, UpstreamProtocol,
     const Router::RouteEntry& route_entry,
     absl::optional downstream_protocol,
     Upstream::LoadBalancerContext* ctx) const {
-  auto ret = std::make_unique(thread_local_cluster, is_connect, route_entry,
-                                            downstream_protocol, ctx);
+  auto ret =
+      std::make_unique(thread_local_cluster, route_entry, downstream_protocol, ctx);
   return (ret->valid() ? std::move(ret) : nullptr);
 }
 
diff --git a/source/extensions/upstreams/http/http/config.h b/source/extensions/upstreams/http/http/config.h
index e438999e7dac..717d028e4d5e 100644
--- a/source/extensions/upstreams/http/http/config.h
+++ b/source/extensions/upstreams/http/http/config.h
@@ -18,7 +18,8 @@ class HttpGenericConnPoolFactory : public Router::GenericConnPoolFactory {
   std::string name() const override { return "envoy.filters.connection_pools.http.http"; }
   std::string category() const override { return "envoy.upstreams"; }
   Router::GenericConnPoolPtr
-  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
+                        Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol,
                         const Router::RouteEntry& route_entry,
                         absl::optional downstream_protocol,
                         Upstream::LoadBalancerContext* ctx) const override;
diff --git a/source/extensions/upstreams/http/http/upstream_request.h b/source/extensions/upstreams/http/http/upstream_request.h
index b1e82f06ca22..88c437c8e3f7 100644
--- a/source/extensions/upstreams/http/http/upstream_request.h
+++ b/source/extensions/upstreams/http/http/upstream_request.h
@@ -21,11 +21,10 @@ namespace Http {
 class HttpConnPool : public Router::GenericConnPool, public Envoy::Http::ConnectionPool::Callbacks {
 public:
   // GenericConnPool
-  HttpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+  HttpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
                const Router::RouteEntry& route_entry,
                absl::optional downstream_protocol,
                Upstream::LoadBalancerContext* ctx) {
-    ASSERT(!is_connect);
     pool_data_ =
         thread_local_cluster.httpConnPool(route_entry.priority(), downstream_protocol, ctx);
   }
diff --git a/source/extensions/upstreams/http/tcp/config.cc b/source/extensions/upstreams/http/tcp/config.cc
index b0773c0a9535..8cd71bbb21ba 100644
--- a/source/extensions/upstreams/http/tcp/config.cc
+++ b/source/extensions/upstreams/http/tcp/config.cc
@@ -9,12 +9,10 @@ namespace Http {
 namespace Tcp {
 
 Router::GenericConnPoolPtr TcpGenericConnPoolFactory::createGenericConnPool(
-    Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
-    const Router::RouteEntry& route_entry,
-    absl::optional downstream_protocol,
-    Upstream::LoadBalancerContext* ctx) const {
-  auto ret = std::make_unique(thread_local_cluster, is_connect, route_entry,
-                                           downstream_protocol, ctx);
+    Upstream::ThreadLocalCluster& thread_local_cluster,
+    Router::GenericConnPoolFactory::UpstreamProtocol, const Router::RouteEntry& route_entry,
+    absl::optional, Upstream::LoadBalancerContext* ctx) const {
+  auto ret = std::make_unique(thread_local_cluster, route_entry, ctx);
   return (ret->valid() ? std::move(ret) : nullptr);
 }
 
diff --git a/source/extensions/upstreams/http/tcp/config.h b/source/extensions/upstreams/http/tcp/config.h
index 784e8f15b502..bf92975f9180 100644
--- a/source/extensions/upstreams/http/tcp/config.h
+++ b/source/extensions/upstreams/http/tcp/config.h
@@ -18,7 +18,8 @@ class TcpGenericConnPoolFactory : public Router::GenericConnPoolFactory {
   std::string name() const override { return "envoy.filters.connection_pools.http.tcp"; }
   std::string category() const override { return "envoy.upstreams"; }
   Router::GenericConnPoolPtr
-  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
+                        Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol,
                         const Router::RouteEntry& route_entry,
                         absl::optional downstream_protocol,
                         Upstream::LoadBalancerContext* ctx) const override;
diff --git a/source/extensions/upstreams/http/tcp/upstream_request.cc b/source/extensions/upstreams/http/tcp/upstream_request.cc
index aa96aa176d8d..2e9a3eb1357b 100644
--- a/source/extensions/upstreams/http/tcp/upstream_request.cc
+++ b/source/extensions/upstreams/http/tcp/upstream_request.cc
@@ -71,7 +71,7 @@ Envoy::Http::Status TcpUpstream::encodeHeaders(const Envoy::Http::RequestHeaderM
   Envoy::Http::ResponseHeaderMapPtr headers{
       Envoy::Http::createHeaderMap(
           {{Envoy::Http::Headers::get().Status, "200"}})};
-  upstream_request_->decodeHeaders(std::move(headers), false);
+  upstream_request_->decodeHeaders(std::move(headers), /*end_stream=*/false);
   return Envoy::Http::okStatus();
 }
 
diff --git a/source/extensions/upstreams/http/tcp/upstream_request.h b/source/extensions/upstreams/http/tcp/upstream_request.h
index e397e8b8c3e9..e2f5fa861d20 100644
--- a/source/extensions/upstreams/http/tcp/upstream_request.h
+++ b/source/extensions/upstreams/http/tcp/upstream_request.h
@@ -22,9 +22,8 @@ namespace Tcp {
 
 class TcpConnPool : public Router::GenericConnPool, public Envoy::Tcp::ConnectionPool::Callbacks {
 public:
-  TcpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, bool,
-              const Router::RouteEntry& route_entry, absl::optional,
-              Upstream::LoadBalancerContext* ctx) {
+  TcpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
+              const Router::RouteEntry& route_entry, Upstream::LoadBalancerContext* ctx) {
     conn_pool_data_ = thread_local_cluster.tcpConnPool(route_entry.priority(), ctx);
   }
   void newStream(Router::GenericConnectionPoolCallbacks* callbacks) override {
diff --git a/source/extensions/upstreams/http/udp/BUILD b/source/extensions/upstreams/http/udp/BUILD
new file mode 100644
index 000000000000..1967e17642e2
--- /dev/null
+++ b/source/extensions/upstreams/http/udp/BUILD
@@ -0,0 +1,56 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_extension",
+    "envoy_cc_library",
+    "envoy_extension_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_extension_package()
+
+envoy_cc_extension(
+    name = "config",
+    srcs = [
+        "config.cc",
+    ],
+    hdrs = [
+        "config.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":upstream_request_lib",
+        "@envoy_api//envoy/extensions/upstreams/http/udp/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_cc_library(
+    name = "upstream_request_lib",
+    srcs = [
+        "upstream_request.cc",
+    ],
+    hdrs = [
+        "upstream_request.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//envoy/http:codes_interface",
+        "//envoy/http:filter_interface",
+        "//envoy/upstream:upstream_interface",
+        "//source/common/common:assert_lib",
+        "//source/common/common:minimal_logger_lib",
+        "//source/common/common:utility_lib",
+        "//source/common/http:codes_lib",
+        "//source/common/http:header_map_lib",
+        "//source/common/http:headers_lib",
+        "//source/common/http:message_lib",
+        "//source/common/network:application_protocol_lib",
+        "//source/common/network:transport_socket_options_lib",
+        "//source/common/router:router_lib",
+        "//source/common/upstream:load_balancer_lib",
+        "//source/extensions/common/proxy_protocol:proxy_protocol_header_lib",
+        "@com_github_google_quiche//:quic_core_http_spdy_session_lib",
+        "@com_github_google_quiche//:quic_core_types_lib",
+        "@com_github_google_quiche//:quiche_common_connect_udp_datagram_payload_lib",
+    ],
+)
diff --git a/source/extensions/upstreams/http/udp/config.cc b/source/extensions/upstreams/http/udp/config.cc
new file mode 100644
index 000000000000..54ea9e1e4c19
--- /dev/null
+++ b/source/extensions/upstreams/http/udp/config.cc
@@ -0,0 +1,25 @@
+#include "source/extensions/upstreams/http/udp/config.h"
+
+#include "source/extensions/upstreams/http/udp/upstream_request.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Upstreams {
+namespace Http {
+namespace Udp {
+
+Router::GenericConnPoolPtr UdpGenericConnPoolFactory::createGenericConnPool(
+    Upstream::ThreadLocalCluster& thread_local_cluster,
+    Router::GenericConnPoolFactory::UpstreamProtocol, const Router::RouteEntry&,
+    absl::optional, Upstream::LoadBalancerContext* ctx) const {
+  auto ret = std::make_unique(thread_local_cluster, ctx);
+  return (ret->valid() ? std::move(ret) : nullptr);
+}
+
+REGISTER_FACTORY(UdpGenericConnPoolFactory, Router::GenericConnPoolFactory);
+
+} // namespace Udp
+} // namespace Http
+} // namespace Upstreams
+} // namespace Extensions
+} // namespace Envoy
diff --git a/source/extensions/upstreams/http/udp/config.h b/source/extensions/upstreams/http/udp/config.h
new file mode 100644
index 000000000000..7bc42d649dd8
--- /dev/null
+++ b/source/extensions/upstreams/http/udp/config.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include "envoy/extensions/upstreams/http/udp/v3/udp_connection_pool.pb.h"
+#include "envoy/registry/registry.h"
+#include "envoy/router/router.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Upstreams {
+namespace Http {
+namespace Udp {
+
+/**
+ * Config registration for the UdpConnPool. @see Router::GenericConnPoolFactory
+ */
+class UdpGenericConnPoolFactory : public Router::GenericConnPoolFactory {
+public:
+  std::string name() const override { return "envoy.filters.connection_pools.http.udp"; }
+  std::string category() const override { return "envoy.upstreams"; }
+  Router::GenericConnPoolPtr
+  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
+                        Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol,
+                        const Router::RouteEntry& route_entry,
+                        absl::optional downstream_protocol,
+                        Upstream::LoadBalancerContext* ctx) const override;
+  ProtobufTypes::MessagePtr createEmptyConfigProto() override {
+    return std::make_unique();
+  }
+};
+
+DECLARE_FACTORY(UdpGenericConnPoolFactory);
+
+} // namespace Udp
+} // namespace Http
+} // namespace Upstreams
+} // namespace Extensions
+} // namespace Envoy
diff --git a/source/extensions/upstreams/http/udp/upstream_request.cc b/source/extensions/upstreams/http/udp/upstream_request.cc
new file mode 100644
index 000000000000..5a11cc0f5ff7
--- /dev/null
+++ b/source/extensions/upstreams/http/udp/upstream_request.cc
@@ -0,0 +1,159 @@
+#include "source/extensions/upstreams/http/udp/upstream_request.h"
+
+#include 
+#include 
+
+#include "envoy/upstream/upstream.h"
+
+#include "source/common/common/assert.h"
+#include "source/common/common/logger.h"
+#include "source/common/common/utility.h"
+#include "source/common/http/codes.h"
+#include "source/common/http/header_map_impl.h"
+#include "source/common/http/headers.h"
+#include "source/common/http/message_impl.h"
+#include "source/common/network/transport_socket_options_impl.h"
+#include "source/common/router/router.h"
+#include "source/extensions/common/proxy_protocol/proxy_protocol_header.h"
+
+#include "quiche/common/masque/connect_udp_datagram_payload.h"
+#include "quiche/common/simple_buffer_allocator.h"
+#include "quiche/quic/core/http/quic_spdy_stream.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Upstreams {
+namespace Http {
+namespace Udp {
+
+void UdpConnPool::newStream(Router::GenericConnectionPoolCallbacks* callbacks) {
+  Router::UpstreamToDownstream& upstream_to_downstream = callbacks->upstreamToDownstream();
+  Network::SocketPtr socket = createSocket(host_);
+  const Network::ConnectionInfoProvider& connection_info_provider =
+      socket->connectionInfoProvider();
+  ASSERT(upstream_to_downstream.connection().has_value());
+  Event::Dispatcher& dispatcher = upstream_to_downstream.connection()->dispatcher();
+  auto upstream =
+      std::make_unique(&upstream_to_downstream, std::move(socket), host_, dispatcher);
+  StreamInfo::StreamInfoImpl stream_info(dispatcher.timeSource(), nullptr);
+  callbacks->onPoolReady(std::move(upstream), host_, connection_info_provider, stream_info, {});
+}
+
+UdpUpstream::UdpUpstream(Router::UpstreamToDownstream* upstream_to_downstream,
+                         Network::SocketPtr socket, Upstream::HostConstSharedPtr host,
+                         Event::Dispatcher& dispatcher)
+    : upstream_to_downstream_(upstream_to_downstream), socket_(std::move(socket)), host_(host),
+      dispatcher_(dispatcher) {
+  socket_->ioHandle().initializeFileEvent(
+      dispatcher_, [this](uint32_t) { onSocketReadReady(); }, Event::PlatformDefaultTriggerType,
+      Event::FileReadyType::Read);
+}
+
+void UdpUpstream::encodeData(Buffer::Instance& data, bool end_stream) {
+  for (const Buffer::RawSlice& slice : data.getRawSlices()) {
+    absl::string_view mem_slice(static_cast(slice.mem_), slice.len_);
+    if (!capsule_parser_.IngestCapsuleFragment(mem_slice)) {
+      ENVOY_LOG_MISC(error, "Capsule ingestion error occured: slice = {}", mem_slice);
+      break;
+    }
+  }
+  if (end_stream) {
+    capsule_parser_.ErrorIfThereIsRemainingBufferedData();
+  }
+}
+
+Envoy::Http::Status UdpUpstream::encodeHeaders(const Envoy::Http::RequestHeaderMap& /*headers*/,
+                                               bool end_stream) {
+  // For successful CONNECT-UDP handshakes, synthesizes the 200 response headers downstream.
+  Envoy::Http::ResponseHeaderMapPtr response_headers{
+      Envoy::Http::createHeaderMap(
+          {{Envoy::Http::Headers::get().Status, "200"},
+           {Envoy::Http::Headers::get().CapsuleProtocol, "?1"}})};
+  if (end_stream) {
+    // If the request header is the end of the stream, responds with 400 Bad Request. Does not
+    // return an error code to avoid replying with 503 Service Unavailable.
+    response_headers->setStatus("400");
+    response_headers->remove(Envoy::Http::Headers::get().CapsuleProtocol);
+    upstream_to_downstream_->onResetStream(Envoy::Http::StreamResetReason::ConnectError, "");
+  } else {
+    Api::SysCallIntResult rc = socket_->connect(host_->address());
+    if (SOCKET_FAILURE(rc.return_value_)) {
+      return absl::InternalError("Upstream socket connect failure.");
+    }
+  }
+  // Indicates the end of stream for the subsequent filters in the chain.
+  upstream_to_downstream_->decodeHeaders(std::move(response_headers), end_stream);
+  return Envoy::Http::okStatus();
+}
+
+void UdpUpstream::resetStream() {
+  upstream_to_downstream_ = nullptr;
+  socket_->close();
+}
+
+void UdpUpstream::onSocketReadReady() {
+  uint32_t packets_dropped = 0;
+  const Api::IoErrorPtr result = Network::Utility::readPacketsFromSocket(
+      socket_->ioHandle(), *socket_->connectionInfoProvider().localAddress(), *this,
+      dispatcher_.timeSource(), /*prefer_gro=*/true, packets_dropped);
+  if (result == nullptr) {
+    socket_->ioHandle().activateFileEvents(Event::FileReadyType::Read);
+    return;
+  }
+}
+
+// The local and peer addresses are not used in this method since the socket is already bound and
+// connected to the upstream server in the encodeHeaders method.
+void UdpUpstream::processPacket(Network::Address::InstanceConstSharedPtr /*local_address*/,
+                                Network::Address::InstanceConstSharedPtr /*peer_address*/,
+                                Buffer::InstancePtr buffer, MonotonicTime /*receive_time*/) {
+  std::string data = buffer->toString();
+  quiche::ConnectUdpDatagramUdpPacketPayload payload(data);
+  quiche::QuicheBuffer serialized_capsule =
+      SerializeCapsule(quiche::Capsule::Datagram(payload.Serialize()), &capsule_buffer_allocator_);
+
+  Buffer::InstancePtr capsule_data = std::make_unique();
+  capsule_data->add(serialized_capsule.AsStringView());
+  bytes_meter_->addWireBytesReceived(capsule_data->length());
+  upstream_to_downstream_->decodeData(*capsule_data, /*end_stream=*/false);
+}
+
+bool UdpUpstream::OnCapsule(const quiche::Capsule& capsule) {
+  quiche::CapsuleType capsule_type = capsule.capsule_type();
+  if (capsule_type != quiche::CapsuleType::DATAGRAM) {
+    // Silently drops capsules with an unknown type.
+    return true;
+  }
+
+  std::unique_ptr connect_udp_datagram_payload =
+      quiche::ConnectUdpDatagramPayload::Parse(capsule.datagram_capsule().http_datagram_payload);
+  if (!connect_udp_datagram_payload) {
+    // Indicates parsing failure to reset the data stream.
+    return false;
+  }
+
+  if (connect_udp_datagram_payload->GetType() !=
+      quiche::ConnectUdpDatagramPayload::Type::kUdpPacket) {
+    // Silently drops Datagrams with an unknown Context ID.
+    return true;
+  }
+
+  Buffer::InstancePtr buffer = std::make_unique();
+  buffer->add(connect_udp_datagram_payload->GetUdpProxyingPayload());
+  bytes_meter_->addWireBytesSent(buffer->length());
+  Api::IoCallUint64Result rc = Network::Utility::writeToSocket(
+      socket_->ioHandle(), *buffer, /*local_ip=*/nullptr, *host_->address());
+  // TODO(https://github.com/envoyproxy/envoy/issues/23564): Handle some socket errors here.
+  return true;
+}
+
+void UdpUpstream::OnCapsuleParseFailure(absl::string_view error_message) {
+  upstream_to_downstream_->onResetStream(Envoy::Http::StreamResetReason::ProtocolError,
+                                         error_message);
+}
+
+} // namespace Udp
+} // namespace Http
+} // namespace Upstreams
+} // namespace Extensions
+} // namespace Envoy
diff --git a/source/extensions/upstreams/http/udp/upstream_request.h b/source/extensions/upstreams/http/udp/upstream_request.h
new file mode 100644
index 000000000000..ce363e57ae17
--- /dev/null
+++ b/source/extensions/upstreams/http/udp/upstream_request.h
@@ -0,0 +1,116 @@
+#pragma once
+
+#include 
+#include 
+
+#include "envoy/http/codec.h"
+#include "envoy/tcp/conn_pool.h"
+#include "envoy/upstream/thread_local_cluster.h"
+
+#include "source/common/buffer/watermark_buffer.h"
+#include "source/common/common/cleanup.h"
+#include "source/common/common/logger.h"
+#include "source/common/config/well_known_names.h"
+#include "source/common/network/utility.h"
+#include "source/common/router/upstream_request.h"
+#include "source/common/stream_info/stream_info_impl.h"
+
+#include "quiche/common/simple_buffer_allocator.h"
+#include "quiche/quic/core/http/quic_spdy_stream.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Upstreams {
+namespace Http {
+namespace Udp {
+
+// Creates a UDP socket for a UDP upstream connection. When a new UDP upstream is requested by the
+// UpstreamRequest of Router, creates a UDPUpstream object and hands over the created socket to it.
+class UdpConnPool : public Router::GenericConnPool {
+public:
+  UdpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
+              Upstream::LoadBalancerContext* ctx)
+      : host_(thread_local_cluster.loadBalancer().chooseHost(ctx)) {}
+
+  // Creates a UDPUpstream object for a new stream.
+  void newStream(Router::GenericConnectionPoolCallbacks* callbacks) override;
+
+  bool cancelAnyPendingStream() override {
+    // Unlike TCP, UDP Upstreams do not have any pending streams because the upstream connection is
+    // created immediately without a handshake.
+    return false;
+  }
+
+  Upstream::HostDescriptionConstSharedPtr host() const override { return host_; }
+
+  Network::SocketPtr createSocket(const Upstream::HostConstSharedPtr& host) {
+    return std::make_unique(Network::Socket::Type::Datagram, host->address(),
+                                                 /*remote_address=*/nullptr,
+                                                 Network::SocketCreationOptions{});
+  }
+
+  bool valid() { return host_ != nullptr; }
+
+private:
+  Upstream::HostConstSharedPtr host_;
+};
+
+// Maintains data relevant to a UDP upstream connection including the socket for the upstream.
+// When a CONNECT-UDP request comes in, connects the socket to a node in the upstream cluster.
+// Also, adds appropriate header entries to the CONNECT-UDP response.
+class UdpUpstream : public Router::GenericUpstream,
+                    public Network::UdpPacketProcessor,
+                    public quiche::CapsuleParser::Visitor {
+public:
+  UdpUpstream(Router::UpstreamToDownstream* upstream_to_downstream, Network::SocketPtr socket,
+              Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher);
+
+  // GenericUpstream
+  void encodeData(Buffer::Instance& data, bool end_stream) override;
+  void encodeMetadata(const Envoy::Http::MetadataMapVector&) override {}
+  Envoy::Http::Status encodeHeaders(const Envoy::Http::RequestHeaderMap&, bool end_stream) override;
+  void encodeTrailers(const Envoy::Http::RequestTrailerMap&) override {}
+  void readDisable(bool) override {}
+  void resetStream() override;
+  void setAccount(Buffer::BufferMemoryAccountSharedPtr) override {}
+  const StreamInfo::BytesMeterSharedPtr& bytesMeter() override { return bytes_meter_; }
+
+  // Network::UdpPacketProcessor
+  // Handles data received from the UDP Upstream.
+  void processPacket(Network::Address::InstanceConstSharedPtr local_address,
+                     Network::Address::InstanceConstSharedPtr peer_address,
+                     Buffer::InstancePtr buffer, MonotonicTime receive_time) override;
+  uint64_t maxDatagramSize() const override { return Network::DEFAULT_UDP_MAX_DATAGRAM_SIZE; }
+  void onDatagramsDropped(uint32_t dropped) override {
+    // TODO(https://github.com/envoyproxy/envoy/issues/23564): Add statistics for CONNECT-UDP
+    // upstreams.
+    ENVOY_LOG_MISC(warn, "{} UDP datagrams were dropped.", dropped);
+    datagrams_dropped_ += dropped;
+  }
+  size_t numPacketsExpectedPerEventLoop() const override {
+    return Network::MAX_NUM_PACKETS_PER_EVENT_LOOP;
+  }
+  uint32_t numOfDroppedDatagrams() { return datagrams_dropped_; }
+
+  // quiche::CapsuleParser::Visitor
+  bool OnCapsule(const quiche::Capsule& capsule) override;
+  void OnCapsuleParseFailure(absl::string_view error_message) override;
+
+private:
+  void onSocketReadReady();
+
+  Router::UpstreamToDownstream* upstream_to_downstream_;
+  const Network::SocketPtr socket_;
+  Upstream::HostConstSharedPtr host_;
+  Event::Dispatcher& dispatcher_;
+  StreamInfo::BytesMeterSharedPtr bytes_meter_{std::make_shared()};
+  quiche::CapsuleParser capsule_parser_{this};
+  quiche::SimpleBufferAllocator capsule_buffer_allocator_;
+  uint32_t datagrams_dropped_ = 0;
+};
+
+} // namespace Udp
+} // namespace Http
+} // namespace Upstreams
+} // namespace Extensions
+} // namespace Envoy
diff --git a/source/server/api_listener_impl.h b/source/server/api_listener_impl.h
index 1f18ffe1efac..8508c852e7b1 100644
--- a/source/server/api_listener_impl.h
+++ b/source/server/api_listener_impl.h
@@ -122,7 +122,7 @@ class ApiListenerImplBase : public ApiListener,
       }
       void close(Network::ConnectionCloseType) override {}
       void close(Network::ConnectionCloseType, absl::string_view) override {}
-      Event::Dispatcher& dispatcher() override { return dispatcher_; }
+      Event::Dispatcher& dispatcher() const override { return dispatcher_; }
       uint64_t id() const override { return 12345; }
       void hashKey(std::vector&) const override {}
       std::string nextProtocol() const override { return EMPTY_STRING; }
diff --git a/test/common/http/hcm_router_fuzz_test.cc b/test/common/http/hcm_router_fuzz_test.cc
index b21618137d15..cc8a618d62a7 100644
--- a/test/common/http/hcm_router_fuzz_test.cc
+++ b/test/common/http/hcm_router_fuzz_test.cc
@@ -396,11 +396,13 @@ class FuzzGenericConnPoolFactory : public Router::GenericConnPoolFactory {
       : cluster_manager_(cluster_manager) {}
   std::string name() const override { return "envoy.filters.connection_pools.http.generic"; }
   std::string category() const override { return "envoy.upstreams"; }
-  Router::GenericConnPoolPtr createGenericConnPool(Upstream::ThreadLocalCluster&, bool is_connect,
-                                                   const Router::RouteEntry& route_entry,
-                                                   absl::optional protocol,
-                                                   Upstream::LoadBalancerContext*) const override {
-    if (is_connect) {
+  Router::GenericConnPoolPtr
+  createGenericConnPool(Upstream::ThreadLocalCluster&,
+                        Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol,
+                        const Router::RouteEntry& route_entry,
+                        absl::optional protocol,
+                        Upstream::LoadBalancerContext*) const override {
+    if (upstream_protocol != UpstreamProtocol::HTTP) {
       return nullptr;
     }
     FuzzCluster* cluster = cluster_manager_.selectClusterByName(route_entry.clusterName());
diff --git a/test/common/http/header_utility_test.cc b/test/common/http/header_utility_test.cc
index f8ad125c256f..a5e3f92af29b 100644
--- a/test/common/http/header_utility_test.cc
+++ b/test/common/http/header_utility_test.cc
@@ -1148,6 +1148,15 @@ TEST(HeaderIsValidTest, IsConnect) {
   EXPECT_FALSE(HeaderUtility::isConnect(Http::TestRequestHeaderMapImpl{}));
 }
 
+TEST(HeaderIsValidTest, IsConnectUdp) {
+  EXPECT_TRUE(
+      HeaderUtility::isConnectUdp(Http::TestRequestHeaderMapImpl{{"upgrade", "connect-udp"}}));
+  // Extended CONNECT requests should be normalized to HTTP/1.1.
+  EXPECT_FALSE(HeaderUtility::isConnectUdp(
+      Http::TestRequestHeaderMapImpl{{":method", "CONNECT"}, {":protocol", "connect-udp"}}));
+  EXPECT_FALSE(HeaderUtility::isConnectUdp(Http::TestRequestHeaderMapImpl{}));
+}
+
 TEST(HeaderIsValidTest, IsConnectResponse) {
   RequestHeaderMapPtr connect_request{new TestRequestHeaderMapImpl{{":method", "CONNECT"}}};
   RequestHeaderMapPtr get_request{new TestRequestHeaderMapImpl{{":method", "GET"}}};
@@ -1160,6 +1169,45 @@ TEST(HeaderIsValidTest, IsConnectResponse) {
   EXPECT_FALSE(HeaderUtility::isConnectResponse(get_request.get(), success_response));
 }
 
+TEST(HeaderIsValidTest, RewriteAuthorityForConnectUdp) {
+  TestRequestHeaderMapImpl connect_udp_request{
+      {":path", "/.well-known/masque/udp/foo.lyft.com/80/"}, {":authority", "example.org"}};
+  EXPECT_TRUE(HeaderUtility::rewriteAuthorityForConnectUdp(connect_udp_request));
+  EXPECT_EQ(connect_udp_request.getHostValue(), "foo.lyft.com:80");
+
+  TestRequestHeaderMapImpl connect_udp_request_ipv6{
+      {":path", "/.well-known/masque/udp/2001%3A0db8%3A85a3%3A%3A8a2e%3A0370%3A7334/80/"},
+      {":authority", "example.org"}};
+  EXPECT_TRUE(HeaderUtility::rewriteAuthorityForConnectUdp(connect_udp_request_ipv6));
+  EXPECT_EQ(connect_udp_request_ipv6.getHostValue(), "[2001:0db8:85a3::8a2e:0370:7334]:80");
+
+  TestRequestHeaderMapImpl connect_udp_request_ipv6_not_escaped{
+      {":path", "/.well-known/masque/udp/2001:0db8:85a3::8a2e:0370:7334/80"},
+      {":authority", "example.org"}};
+  EXPECT_FALSE(HeaderUtility::rewriteAuthorityForConnectUdp(connect_udp_request_ipv6_not_escaped));
+
+  TestRequestHeaderMapImpl connect_udp_request_ipv6_zoneid{
+      {":path", "/.well-known/masque/udp/fe80::a%25ee1/80/"}, {":authority", "example.org"}};
+  EXPECT_FALSE(HeaderUtility::rewriteAuthorityForConnectUdp(connect_udp_request_ipv6_zoneid));
+
+  TestRequestHeaderMapImpl connect_udp_malformed1{
+      {":path", "/well-known/masque/udp/foo.lyft.com/80/"}};
+  EXPECT_FALSE(HeaderUtility::rewriteAuthorityForConnectUdp(connect_udp_malformed1));
+
+  TestRequestHeaderMapImpl connect_udp_malformed2{{":path", "/masque/udp/foo.lyft.com/80/"}};
+  EXPECT_FALSE(HeaderUtility::rewriteAuthorityForConnectUdp(connect_udp_malformed2));
+
+  TestRequestHeaderMapImpl connect_udp_no_path{{":authority", "example.org"}};
+  EXPECT_FALSE(HeaderUtility::rewriteAuthorityForConnectUdp(connect_udp_no_path));
+
+  TestRequestHeaderMapImpl connect_udp_empty_host{{":path", "/.well-known/masque/udp//80/"}};
+  EXPECT_FALSE(HeaderUtility::rewriteAuthorityForConnectUdp(connect_udp_empty_host));
+
+  TestRequestHeaderMapImpl connect_udp_empty_port{
+      {":path", "/.well-known/masque/udp/foo.lyft.com//"}};
+  EXPECT_FALSE(HeaderUtility::rewriteAuthorityForConnectUdp(connect_udp_empty_port));
+}
+
 #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
 TEST(HeaderIsValidTest, IsCapsuleProtocol) {
   EXPECT_TRUE(
diff --git a/test/common/quic/envoy_quic_client_stream_test.cc b/test/common/quic/envoy_quic_client_stream_test.cc
index e548e237909d..ef4caa13d87e 100644
--- a/test/common/quic/envoy_quic_client_stream_test.cc
+++ b/test/common/quic/envoy_quic_client_stream_test.cc
@@ -134,7 +134,6 @@ class EnvoyQuicClientStreamTest : public testing::Test {
 #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
   void setUpCapsuleProtocol(bool close_send_stream, bool close_recv_stream) {
     EXPECT_TRUE(quic_session_.OnSetting(quic::SETTINGS_H3_DATAGRAM, 1));
-    quic_stream_->useCapsuleProtocol();
 
     // Encodes a CONNECT-UDP request.
     Http::TestRequestHeaderMapImpl request_headers = {
diff --git a/test/common/quic/envoy_quic_server_stream_test.cc b/test/common/quic/envoy_quic_server_stream_test.cc
index 7b2f5f9e05ef..d56e63ad0f76 100644
--- a/test/common/quic/envoy_quic_server_stream_test.cc
+++ b/test/common/quic/envoy_quic_server_stream_test.cc
@@ -118,8 +118,6 @@ class EnvoyQuicServerStreamTest : public testing::Test {
 
 #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
   void setUpCapsuleProtocol(bool close_send_stream, bool close_recv_stream) {
-    quic_stream_->useCapsuleProtocol();
-
     // Decodes a CONNECT-UDP request.
     EXPECT_CALL(stream_decoder_, decodeHeaders_(_, _))
         .WillOnce(Invoke([](const Http::RequestHeaderMapSharedPtr& headers, bool) {
diff --git a/test/config/utility.cc b/test/config/utility.cc
index 21781be3852e..65525dfead0b 100644
--- a/test/config/utility.cc
+++ b/test/config/utility.cc
@@ -852,6 +852,28 @@ void ConfigHelper::setConnectConfig(
   }
 }
 
+void ConfigHelper::setConnectUdpConfig(
+    envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm,
+    bool terminate_connect, bool http3) {
+  auto* route_config = hcm.mutable_route_config();
+  ASSERT_EQ(1, route_config->virtual_hosts_size());
+  auto* route = route_config->mutable_virtual_hosts(0)->mutable_routes(0);
+  auto* match = route->mutable_match();
+  match->Clear();
+  match->mutable_connect_matcher();
+
+  if (terminate_connect) {
+    auto* upgrade = route->mutable_route()->add_upgrade_configs();
+    upgrade->set_upgrade_type("connect-udp");
+  }
+
+  hcm.add_upgrade_configs()->set_upgrade_type("connect-udp");
+  hcm.mutable_http2_protocol_options()->set_allow_connect(true);
+  if (http3) {
+    hcm.mutable_http3_protocol_options()->set_allow_extended_connect(true);
+  }
+}
+
 void ConfigHelper::applyConfigModifiers() {
   for (const auto& config_modifier : config_modifiers_) {
     config_modifier(bootstrap_);
diff --git a/test/config/utility.h b/test/config/utility.h
index 800446c20395..ce3cb3c04cd4 100644
--- a/test/config/utility.h
+++ b/test/config/utility.h
@@ -405,6 +405,10 @@ class ConfigHelper {
                                bool http3 = false,
                                absl::optional
                                    proxy_protocol_version = absl::nullopt);
+  // Given an HCM with the default config, set the matcher to be a connect matcher and enable
+  // CONNECT-UDP requests.
+  static void setConnectUdpConfig(HttpConnectionManager& hcm, bool terminate_connect,
+                                  bool http3 = false);
 
   void setLocalReply(
       const envoy::extensions::filters::network::http_connection_manager::v3::LocalReplyConfig&
diff --git a/test/extensions/upstreams/http/generic/BUILD b/test/extensions/upstreams/http/generic/BUILD
new file mode 100644
index 000000000000..17cb97dc79a7
--- /dev/null
+++ b/test/extensions/upstreams/http/generic/BUILD
@@ -0,0 +1,20 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_test",
+    "envoy_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+envoy_cc_test(
+    name = "config_test",
+    srcs = ["config_test.cc"],
+    deps = [
+        "//source/extensions/upstreams/http/generic:config",
+        "//test/mocks:common_lib",
+        "//test/mocks/router:router_mocks",
+        "//test/mocks/upstream:upstream_mocks",
+    ],
+)
diff --git a/test/extensions/upstreams/http/generic/config_test.cc b/test/extensions/upstreams/http/generic/config_test.cc
new file mode 100644
index 000000000000..3c51f53b782d
--- /dev/null
+++ b/test/extensions/upstreams/http/generic/config_test.cc
@@ -0,0 +1,58 @@
+#include "source/extensions/upstreams/http/generic/config.h"
+
+#include "test/mocks/router/mocks.h"
+#include "test/mocks/upstream/thread_local_cluster.h"
+#include "test/test_common/utility.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Upstreams {
+namespace Http {
+namespace Generic {
+
+using ::testing::NiceMock;
+
+class GenericGenericConnPoolFactoryTest : public ::testing::Test {
+public:
+  GenericGenericConnPoolFactoryTest() = default;
+
+protected:
+  NiceMock thread_local_cluster_;
+  NiceMock route_entry_;
+  Upstream::HostConstSharedPtr host_;
+  GenericGenericConnPoolFactory factory_;
+};
+
+TEST_F(GenericGenericConnPoolFactoryTest, CreateValidHttpConnPool) {
+  EXPECT_TRUE(factory_.createGenericConnPool(thread_local_cluster_,
+                                             Router::GenericConnPoolFactory::UpstreamProtocol::HTTP,
+                                             route_entry_, Envoy::Http::Protocol::Http2, nullptr));
+}
+
+TEST_F(GenericGenericConnPoolFactoryTest, CreateValidTcpConnPool) {
+  EXPECT_TRUE(factory_.createGenericConnPool(thread_local_cluster_,
+                                             Router::GenericConnPoolFactory::UpstreamProtocol::TCP,
+                                             route_entry_, Envoy::Http::Protocol::Http2, nullptr));
+}
+
+TEST_F(GenericGenericConnPoolFactoryTest, CreateValidUdpConnPool) {
+  EXPECT_TRUE(factory_.createGenericConnPool(thread_local_cluster_,
+                                             Router::GenericConnPoolFactory::UpstreamProtocol::UDP,
+                                             route_entry_, Envoy::Http::Protocol::Http2, nullptr));
+}
+
+TEST_F(GenericGenericConnPoolFactoryTest, InvalidConnPool) {
+  // Passes an invalid UpstreamProtocol and check a nullptr is returned.
+  EXPECT_FALSE(factory_.createGenericConnPool(
+      thread_local_cluster_, static_cast(0xff),
+      route_entry_, Envoy::Http::Protocol::Http2, nullptr));
+}
+
+} // namespace Generic
+} // namespace Http
+} // namespace Upstreams
+} // namespace Extensions
+} // namespace Envoy
diff --git a/test/extensions/upstreams/http/tcp/upstream_request_test.cc b/test/extensions/upstreams/http/tcp/upstream_request_test.cc
index aa51aa515cbc..d2745bdc529e 100644
--- a/test/extensions/upstreams/http/tcp/upstream_request_test.cc
+++ b/test/extensions/upstreams/http/tcp/upstream_request_test.cc
@@ -41,8 +41,7 @@ class TcpConnPoolTest : public ::testing::Test {
     cm.initializeThreadLocalClusters({"fake_cluster"});
     EXPECT_CALL(cm.thread_local_cluster_, tcpConnPool(_, _))
         .WillOnce(Return(Upstream::TcpPoolData([]() {}, &mock_pool_)));
-    conn_pool_ = std::make_unique(cm.thread_local_cluster_, true, route_entry,
-                                               Envoy::Http::Protocol::Http11, nullptr);
+    conn_pool_ = std::make_unique(cm.thread_local_cluster_, route_entry, nullptr);
   }
 
   std::unique_ptr conn_pool_;
diff --git a/test/extensions/upstreams/http/udp/BUILD b/test/extensions/upstreams/http/udp/BUILD
new file mode 100644
index 000000000000..e94d6b501009
--- /dev/null
+++ b/test/extensions/upstreams/http/udp/BUILD
@@ -0,0 +1,45 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_test",
+    "envoy_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+envoy_cc_test(
+    name = "upstream_request_test",
+    srcs = ["upstream_request_test.cc"],
+    deps = [
+        "//source/common/buffer:buffer_lib",
+        "//source/common/network:address_lib",
+        "//source/common/router:router_lib",
+        "//source/common/upstream:upstream_includes",
+        "//source/common/upstream:upstream_lib",
+        "//source/extensions/upstreams/http/udp:upstream_request_lib",
+        "//test/common/http:common_lib",
+        "//test/mocks:common_lib",
+        "//test/mocks/network:network_mocks",
+        "//test/mocks/router:router_filter_interface",
+        "//test/mocks/router:router_mocks",
+        "//test/mocks/server:factory_context_mocks",
+        "//test/mocks/server:instance_mocks",
+        "//test/mocks/upstream:upstream_mocks",
+        "//test/test_common:environment_lib",
+        "//test/test_common:simulated_time_system_lib",
+        "//test/test_common:utility_lib",
+    ],
+)
+
+envoy_cc_test(
+    name = "config_test",
+    srcs = ["config_test.cc"],
+    deps = [
+        "//source/extensions/upstreams/http/udp:config",
+        "//source/extensions/upstreams/http/udp:upstream_request_lib",
+        "//test/mocks:common_lib",
+        "//test/mocks/router:router_mocks",
+        "//test/mocks/upstream:upstream_mocks",
+    ],
+)
diff --git a/test/extensions/upstreams/http/udp/config_test.cc b/test/extensions/upstreams/http/udp/config_test.cc
new file mode 100644
index 000000000000..258940e111bf
--- /dev/null
+++ b/test/extensions/upstreams/http/udp/config_test.cc
@@ -0,0 +1,49 @@
+#include "source/extensions/upstreams/http/udp/config.h"
+
+#include "test/mocks/router/mocks.h"
+#include "test/mocks/upstream/thread_local_cluster.h"
+#include "test/test_common/utility.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Upstreams {
+namespace Http {
+namespace Udp {
+
+using ::testing::NiceMock;
+using ::testing::Return;
+
+class UdpGenericConnPoolFactoryTest : public ::testing::Test {
+public:
+  UdpGenericConnPoolFactoryTest() = default;
+
+protected:
+  NiceMock thread_local_cluster_;
+  NiceMock route_entry_;
+  Upstream::HostConstSharedPtr host_;
+  UdpGenericConnPoolFactory factory_;
+};
+
+TEST_F(UdpGenericConnPoolFactoryTest, CreateValidUdpConnPool) {
+  auto host = std::make_shared();
+  EXPECT_CALL(thread_local_cluster_.lb_, chooseHost).WillOnce(Return(host));
+  EXPECT_TRUE(factory_.createGenericConnPool(thread_local_cluster_,
+                                             Router::GenericConnPoolFactory::UpstreamProtocol::UDP,
+                                             route_entry_, Envoy::Http::Protocol::Http2, nullptr));
+}
+
+TEST_F(UdpGenericConnPoolFactoryTest, CreateInvalidUdpConnPool) {
+  EXPECT_CALL(thread_local_cluster_.lb_, chooseHost).WillOnce(Return(nullptr));
+  EXPECT_FALSE(factory_.createGenericConnPool(thread_local_cluster_,
+                                              Router::GenericConnPoolFactory::UpstreamProtocol::UDP,
+                                              route_entry_, Envoy::Http::Protocol::Http2, nullptr));
+}
+
+} // namespace Udp
+} // namespace Http
+} // namespace Upstreams
+} // namespace Extensions
+} // namespace Envoy
diff --git a/test/extensions/upstreams/http/udp/upstream_request_test.cc b/test/extensions/upstreams/http/udp/upstream_request_test.cc
new file mode 100644
index 000000000000..3f89801321e2
--- /dev/null
+++ b/test/extensions/upstreams/http/udp/upstream_request_test.cc
@@ -0,0 +1,192 @@
+#include "envoy/http/header_map.h"
+
+#include "source/common/buffer/buffer_impl.h"
+#include "source/common/network/address_impl.h"
+#include "source/common/network/utility.h"
+#include "source/common/router/config_impl.h"
+#include "source/common/router/router.h"
+#include "source/common/router/upstream_codec_filter.h"
+#include "source/common/router/upstream_request.h"
+#include "source/extensions/common/proxy_protocol/proxy_protocol_header.h"
+#include "source/extensions/upstreams/http/udp/upstream_request.h"
+
+#include "test/common/http/common.h"
+#include "test/mocks/common.h"
+#include "test/mocks/router/mocks.h"
+#include "test/mocks/router/router_filter_interface.h"
+#include "test/mocks/server/factory_context.h"
+#include "test/mocks/server/instance.h"
+#include "test/test_common/utility.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Upstreams {
+namespace Http {
+namespace Udp {
+
+using ::testing::NiceMock;
+using ::testing::Return;
+
+class UdpUpstreamTest : public ::testing::Test {
+public:
+  UdpUpstreamTest() {
+    auto mock_socket = std::make_unique>();
+    mock_socket_ = mock_socket.get();
+    EXPECT_CALL(*mock_socket_->io_handle_, createFileEvent_);
+    auto mock_host = std::make_shared>();
+    mock_host_ = mock_host.get();
+    ON_CALL(*mock_host_, address)
+        .WillByDefault(
+            Return(Network::Utility::parseInternetAddressAndPortNoThrow("127.0.0.1:80", false)));
+    udp_upstream_ =
+        std::make_unique(&mock_upstream_to_downstream_, std::move(mock_socket),
+                                      std::move(mock_host), mock_dispatcher_);
+  }
+
+protected:
+  ::Envoy::Http::TestRequestHeaderMapImpl connect_udp_headers_{
+      {":path", "/.well-known/masque/udp/foo.lyft.com/80/"},
+      {"upgrade", "connect-udp"},
+      {"connection", "upgrade"},
+      {":authority", "example.org"}};
+
+  NiceMock mock_upstream_to_downstream_;
+  NiceMock* mock_socket_;
+  NiceMock mock_dispatcher_;
+  NiceMock* mock_host_;
+  std::unique_ptr udp_upstream_;
+};
+
+TEST_F(UdpUpstreamTest, ExchangeCapsules) {
+  // Swallow the request headers and generate response headers.
+  EXPECT_CALL(mock_upstream_to_downstream_, decodeHeaders)
+      .WillOnce([](Envoy::Http::ResponseHeaderMapPtr&& headers, bool end_stream) {
+        EXPECT_EQ(headers->getStatusValue(), "200");
+        EXPECT_FALSE(end_stream);
+      });
+  EXPECT_TRUE(udp_upstream_->encodeHeaders(connect_udp_headers_, false).ok());
+
+  // Swallow read disable.
+  udp_upstream_->readDisable(false);
+
+  // Sends a capsule to upstream.
+  const std::string sent_capsule_fragment =
+      absl::HexStringToBytes("00"             // DATAGRAM Capsule Type
+                             "08"             // Capsule Length
+                             "00"             // Context ID
+                             "a1a2a3a4a5a6a7" // UDP Proxying Payload
+      );
+  Buffer::OwnedImpl sent_capsule(sent_capsule_fragment);
+  EXPECT_CALL(*mock_socket_->io_handle_, sendmsg(_, _, _, _, _))
+      .WillOnce([](const Buffer::RawSlice* slices, uint64_t num_slice, int /*flags*/,
+                   const Network::Address::Ip* /*self_ip*/,
+                   const Network::Address::Instance& /*peer_address*/) {
+        Buffer::OwnedImpl buffer(absl::HexStringToBytes("a1a2a3a4a5a6a7"));
+        EXPECT_TRUE(TestUtility::rawSlicesEqual(buffer.getRawSlices().data(), slices, num_slice));
+        return Api::ioCallUint64ResultNoError();
+      });
+  udp_upstream_->encodeData(sent_capsule, false);
+
+  // Receives data from upstream and converts it to capsule.
+  const std::string decoded_capsule_fragment =
+      absl::HexStringToBytes("00"             // DATAGRAM Capsule Type
+                             "08"             // Capsule Length
+                             "00"             // Context ID
+                             "b1b2b3b4b5b6b7" // UDP Proxying Payload
+      );
+  Buffer::InstancePtr received_data =
+      std::make_unique(absl::HexStringToBytes("b1b2b3b4b5b6b7"));
+  EXPECT_CALL(mock_upstream_to_downstream_,
+              decodeData(BufferStringEqual(decoded_capsule_fragment), false));
+  Envoy::MonotonicTime timestamp;
+  udp_upstream_->processPacket(nullptr, nullptr, std::move(received_data), timestamp);
+}
+
+TEST_F(UdpUpstreamTest, HeaderOnlyRequest) {
+  EXPECT_CALL(mock_upstream_to_downstream_, decodeHeaders)
+      .WillOnce([](Envoy::Http::ResponseHeaderMapPtr&& headers, bool end_stream) {
+        EXPECT_EQ(headers->getStatusValue(), "400");
+        EXPECT_TRUE(end_stream);
+      });
+  EXPECT_TRUE(udp_upstream_->encodeHeaders(connect_udp_headers_, true).ok());
+}
+
+TEST_F(UdpUpstreamTest, SwallowMetadata) {
+  Envoy::Http::MetadataMapVector metadata_map_vector;
+  udp_upstream_->encodeMetadata(metadata_map_vector);
+  EXPECT_CALL(*mock_socket_->io_handle_, sendmsg).Times(0);
+}
+
+TEST_F(UdpUpstreamTest, SwallowTrailers) {
+  Envoy::Http::TestRequestTrailerMapImpl trailers{{"foo", "bar"}};
+  udp_upstream_->encodeTrailers(trailers);
+  EXPECT_CALL(*mock_socket_->io_handle_, sendmsg).Times(0);
+}
+
+TEST_F(UdpUpstreamTest, DatagramsDropped) {
+  udp_upstream_->onDatagramsDropped(1);
+  EXPECT_EQ(udp_upstream_->numOfDroppedDatagrams(), 1);
+  udp_upstream_->onDatagramsDropped(3);
+  EXPECT_EQ(udp_upstream_->numOfDroppedDatagrams(), 4);
+}
+
+TEST_F(UdpUpstreamTest, InvalidCapsule) {
+  EXPECT_TRUE(udp_upstream_->encodeHeaders(connect_udp_headers_, false).ok());
+  // Sends an invalid capsule.
+  const std::string invalid_capsule_fragment =
+      absl::HexStringToBytes("0x1eca6a00" // DATAGRAM Capsule Type
+                             "01"         // Capsule Length
+                             "c0"         // Invalid VarInt62
+      );
+  Buffer::OwnedImpl invalid_capsule(invalid_capsule_fragment);
+  EXPECT_CALL(mock_upstream_to_downstream_, onResetStream);
+  udp_upstream_->encodeData(invalid_capsule, true);
+}
+
+TEST_F(UdpUpstreamTest, MalformedContextIdDatagram) {
+  EXPECT_TRUE(udp_upstream_->encodeHeaders(connect_udp_headers_, false).ok());
+  // Sends a capsule with an invalid variable length integer.
+  const std::string invalid_context_id_fragment =
+      absl::HexStringToBytes("00" // DATAGRAM Capsule Type
+                             "01" // Capsule Length
+                             "c0" // Context ID (Invalid VarInt62)
+      );
+  Buffer::OwnedImpl invalid_context_id_capsule(invalid_context_id_fragment);
+  EXPECT_CALL(mock_upstream_to_downstream_, onResetStream);
+  udp_upstream_->encodeData(invalid_context_id_capsule, true);
+}
+
+TEST_F(UdpUpstreamTest, RemainingDataWhenStreamEnded) {
+  EXPECT_CALL(mock_upstream_to_downstream_, decodeHeaders)
+      .WillOnce([](Envoy::Http::ResponseHeaderMapPtr&& headers, bool end_stream) {
+        EXPECT_EQ(headers->getStatusValue(), "200");
+        EXPECT_FALSE(end_stream);
+      });
+  EXPECT_TRUE(udp_upstream_->encodeHeaders(connect_udp_headers_, false).ok());
+
+  // Sends a capsule to upstream with a large length value.
+  const std::string sent_capsule_fragment =
+      absl::HexStringToBytes("00"             // DATAGRAM Capsule Type
+                             "ff"             // Capsule Length
+                             "00"             // Context ID
+                             "a1a2a3a4a5a6a7" // UDP Proxying Payload
+      );
+  Buffer::OwnedImpl sent_capsule(sent_capsule_fragment);
+  EXPECT_CALL(mock_upstream_to_downstream_, onResetStream);
+  udp_upstream_->encodeData(sent_capsule, true);
+}
+
+TEST_F(UdpUpstreamTest, SocketConnectError) {
+  EXPECT_CALL(mock_upstream_to_downstream_, decodeHeaders).Times(0);
+  EXPECT_CALL(*mock_socket_, connect(_)).WillOnce(Return(Api::SysCallIntResult{-1, EADDRINUSE}));
+  EXPECT_FALSE(udp_upstream_->encodeHeaders(connect_udp_headers_, false).ok());
+}
+
+} // namespace Udp
+} // namespace Http
+} // namespace Upstreams
+} // namespace Extensions
+} // namespace Envoy
diff --git a/test/integration/BUILD b/test/integration/BUILD
index 2f53109884c5..451b949851b4 100644
--- a/test/integration/BUILD
+++ b/test/integration/BUILD
@@ -1739,6 +1739,27 @@ envoy_cc_test(
     ],
 )
 
+envoy_cc_test(
+    name = "udp_tunneling_integration_test",
+    size = "large",
+    srcs = [
+        "udp_tunneling_integration_test.cc",
+    ],
+    data = [
+        "//test/config/integration/certs",
+    ],
+    shard_count = 16,
+    deps = [
+        ":http_integration_lib",
+        ":http_protocol_integration_lib",
+        "//source/extensions/upstreams/http/udp:config",
+        "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/access_loggers/file/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/upstreams/http/udp/v3:pkg_cc_proto",
+    ],
+)
+
 envoy_cc_test(
     name = "tcp_conn_pool_integration_test",
     size = "large",
diff --git a/test/integration/udp_tunneling_integration_test.cc b/test/integration/udp_tunneling_integration_test.cc
new file mode 100644
index 000000000000..44e240ec45ae
--- /dev/null
+++ b/test/integration/udp_tunneling_integration_test.cc
@@ -0,0 +1,251 @@
+#include 
+
+#include "envoy/config/bootstrap/v3/bootstrap.pb.h"
+#include "envoy/config/core/v3/proxy_protocol.pb.h"
+#include "envoy/extensions/access_loggers/file/v3/file.pb.h"
+#include "envoy/extensions/upstreams/http/udp/v3/udp_connection_pool.pb.h"
+
+#include "test/integration/http_integration.h"
+#include "test/integration/http_protocol_integration.h"
+
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace {
+
+// Terminates CONNECT-UDP and sends raw UDP datagrams upstream.
+class ConnectUdpTerminationIntegrationTest : public HttpProtocolIntegrationTest {
+public:
+  ConnectUdpTerminationIntegrationTest() = default;
+
+  ~ConnectUdpTerminationIntegrationTest() override {
+    // Since the upstream is a UDP server, there is nothing to check on the upstream side. Simply
+    // make sure that the connection is closed to avoid TSAN error.
+    if (codec_client_) {
+      codec_client_->close();
+    }
+  }
+
+  void initialize() override {
+    config_helper_.addConfigModifier(
+        [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager&
+                hcm) {
+          hcm.mutable_delayed_close_timeout()->set_seconds(1);
+          if (enable_timeout_) {
+            hcm.mutable_stream_idle_timeout()->set_seconds(0);
+            hcm.mutable_stream_idle_timeout()->set_nanos(200 * 1000 * 1000);
+          }
+          if (!host_to_match_.empty()) {
+            auto* route_config = hcm.mutable_route_config();
+            ASSERT_EQ(1, route_config->virtual_hosts_size());
+            route_config->mutable_virtual_hosts(0)->clear_domains();
+            route_config->mutable_virtual_hosts(0)->add_domains(host_to_match_);
+          }
+          ConfigHelper::setConnectUdpConfig(hcm, true,
+                                            downstream_protocol_ == Http::CodecType::HTTP3);
+        });
+    setUdpFakeUpstream(FakeUpstreamConfig::UdpConfig());
+    HttpIntegrationTest::initialize();
+  }
+
+  void setUpConnection() {
+    codec_client_ = makeHttpConnection(lookupPort("http"));
+    auto encoder_decoder = codec_client_->startRequest(connect_udp_headers_);
+    request_encoder_ = &encoder_decoder.first;
+    response_ = std::move(encoder_decoder.second);
+    response_->waitForHeaders();
+  }
+
+  void sendBidirectionalData(const std::string downstream_send_data = "hello",
+                             const std::string upstream_received_data = "hello",
+                             const std::string upstream_send_data = "there!",
+                             const std::string downstream_received_data = "there!") {
+    // Send some data upstream.
+    codec_client_->sendData(*request_encoder_, downstream_send_data, false);
+    Network::UdpRecvData request_datagram;
+    ASSERT_TRUE(fake_upstreams_[0]->waitForUdpDatagram(request_datagram));
+    EXPECT_EQ(upstream_received_data, request_datagram.buffer_->toString());
+
+    // Send some data downstream.
+    fake_upstreams_[0]->sendUdpDatagram(upstream_send_data, request_datagram.addresses_.peer_);
+    response_->waitForBodyData(downstream_received_data.length());
+    EXPECT_EQ(downstream_received_data, response_->body());
+  }
+
+  void exchangeValidCapsules() {
+    const std::string sent_capsule_fragment =
+        absl::HexStringToBytes("00"             // DATAGRAM Capsule Type
+                               "08"             // Capsule Length
+                               "00"             // Context ID
+                               "a1a2a3a4a5a6a7" // UDP Proxying Payload
+        );
+    const std::string received_capsule_fragment =
+        absl::HexStringToBytes("00"             // DATAGRAM Capsule Type
+                               "08"             // Capsule Length
+                               "00"             // Context ID
+                               "b1b2b3b4b5b6b7" // UDP Proxying Payload
+        );
+
+    sendBidirectionalData(sent_capsule_fragment, absl::HexStringToBytes("a1a2a3a4a5a6a7"),
+                          absl::HexStringToBytes("b1b2b3b4b5b6b7"), received_capsule_fragment);
+  }
+
+  // The Envoy HTTP/2 and HTTP/3 clients expect the request header map to be in the form of HTTP/1
+  // upgrade to issue an extended CONNECT request.
+  Http::TestRequestHeaderMapImpl connect_udp_headers_{
+      {":method", "GET"},         {":path", "/.well-known/masque/udp/foo.lyft.com/80/"},
+      {"upgrade", "connect-udp"}, {"connection", "upgrade"},
+      {":scheme", "https"},       {":authority", "example.org"},
+      {"capsule-protocol", "?1"}};
+
+  IntegrationStreamDecoderPtr response_;
+  bool enable_timeout_{};
+  std::string host_to_match_{};
+};
+
+TEST_P(ConnectUdpTerminationIntegrationTest, ExchangeCapsules) {
+  initialize();
+  setUpConnection();
+  exchangeValidCapsules();
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, ExchangeCapsulesWithHostMatch) {
+  host_to_match_ = "foo.lyft.com:80";
+  initialize();
+  setUpConnection();
+  exchangeValidCapsules();
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, IncorrectHostMatch) {
+  host_to_match_ = "foo.lyft.com:80";
+  connect_udp_headers_.setPath("/.well-known/masque/udp/bar.lyft.com/80/");
+  initialize();
+  setUpConnection();
+  EXPECT_EQ("404", response_->headers().getStatusValue());
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, IncorrectPortMatch) {
+  host_to_match_ = "foo.lyft.com:80";
+  connect_udp_headers_.setPath("/.well-known/masque/udp/foo.lyft.com/8080/");
+  initialize();
+  setUpConnection();
+  EXPECT_EQ("404", response_->headers().getStatusValue());
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, IPv4HostMatch) {
+  host_to_match_ = "179.0.112.43:80";
+  connect_udp_headers_.setPath("/.well-known/masque/udp/179.0.112.43/80/");
+  initialize();
+  setUpConnection();
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, IPv6HostMatch) {
+  host_to_match_ = "[2001:0db8:85a3::8a2e:0370:7334]:80";
+  connect_udp_headers_.setPath("/.well-known/masque/udp/2001:0db8:85a3::8a2e:0370:7334/80/");
+  initialize();
+  setUpConnection();
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, IPv6WithZoneIdHostMatch) {
+  host_to_match_ = "[fe80::a%ee1]:80";
+  connect_udp_headers_.setPath("/.well-known/masque/udp/fe80::a%25ee1/80/");
+  initialize();
+  setUpConnection();
+  EXPECT_EQ("404", response_->headers().getStatusValue());
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, ExchangeCapsulesWithoutCapsuleProtocolHeader) {
+  initialize();
+  connect_udp_headers_.remove(Envoy::Http::Headers::get().CapsuleProtocol);
+  setUpConnection();
+  exchangeValidCapsules();
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, StreamIdleTimeout) {
+  enable_timeout_ = true;
+  initialize();
+  setUpConnection();
+
+  // Wait for the timeout to close the connection.
+  ASSERT_TRUE(response_->waitForReset());
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, MaxStreamDuration) {
+  setUpstreamProtocol(upstreamProtocol());
+  config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+    ConfigHelper::HttpProtocolOptions protocol_options;
+    protocol_options.mutable_common_http_protocol_options()
+        ->mutable_max_stream_duration()
+        ->MergeFrom(ProtobufUtil::TimeUtil::MillisecondsToDuration(1000));
+    ConfigHelper::setProtocolOptions(*bootstrap.mutable_static_resources()->mutable_clusters(0),
+                                     protocol_options);
+  });
+
+  initialize();
+  setUpConnection();
+  exchangeValidCapsules();
+
+  test_server_->waitForCounterGe("cluster.cluster_0.upstream_rq_max_duration_reached", 1);
+
+  if (downstream_protocol_ == Http::CodecType::HTTP1) {
+    ASSERT_TRUE(codec_client_->waitForDisconnect());
+  } else {
+    ASSERT_TRUE(response_->waitForReset());
+  }
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, PathWithInvalidUriTemplate) {
+  initialize();
+  connect_udp_headers_.setPath("/masque/udp/foo.lyft.com/80/");
+  setUpConnection();
+  EXPECT_EQ("404", response_->headers().getStatusValue());
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, PathWithEmptyHost) {
+  initialize();
+  connect_udp_headers_.setPath("/.well-known/masque/udp//80/");
+  setUpConnection();
+  EXPECT_EQ("404", response_->headers().getStatusValue());
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, PathWithEmptyPort) {
+  initialize();
+  connect_udp_headers_.setPath("/.well-known/masque/udp/foo.lyft.com//");
+  setUpConnection();
+  EXPECT_EQ("404", response_->headers().getStatusValue());
+}
+
+TEST_P(ConnectUdpTerminationIntegrationTest, DropUnknownCapsules) {
+  initialize();
+  setUpConnection();
+  Network::UdpRecvData request_datagram;
+  const std::string unknown_capsule_fragment =
+      absl::HexStringToBytes("01"             // DATAGRAM Capsule Type
+                             "08"             // Capsule Length
+                             "00"             // Context ID
+                             "a1a2a3a4a5a6a7" // UDP Proxying Payload
+      );
+  codec_client_->sendData(*request_encoder_, unknown_capsule_fragment, false);
+  ASSERT_FALSE(
+      fake_upstreams_[0]->waitForUdpDatagram(request_datagram, std::chrono::milliseconds(1)));
+
+  const std::string unknown_context_id =
+      absl::HexStringToBytes("00"             // DATAGRAM Capsule Type
+                             "08"             // Capsule Length
+                             "01"             // Context ID
+                             "a1a2a3a4a5a6a7" // UDP Proxying Payload
+      );
+  codec_client_->sendData(*request_encoder_, unknown_context_id, false);
+  ASSERT_FALSE(
+      fake_upstreams_[0]->waitForUdpDatagram(request_datagram, std::chrono::milliseconds(1)));
+}
+
+INSTANTIATE_TEST_SUITE_P(Protocols, ConnectUdpTerminationIntegrationTest,
+                         testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams(
+                             {Http::CodecType::HTTP1, Http::CodecType::HTTP2,
+                              Http::CodecType::HTTP3},
+                             {Http::CodecType::HTTP1})),
+                         HttpProtocolIntegrationTest::protocolTestParamsToString);
+
+} // namespace
+} // namespace Envoy
diff --git a/test/integration/upstreams/per_host_upstream_config.h b/test/integration/upstreams/per_host_upstream_config.h
index 829d4b871bae..c19569ac1c5f 100644
--- a/test/integration/upstreams/per_host_upstream_config.h
+++ b/test/integration/upstreams/per_host_upstream_config.h
@@ -70,11 +70,11 @@ class PerHostHttpUpstream : public Extensions::Upstreams::Http::Http::HttpUpstre
 
 class PerHostHttpConnPool : public Extensions::Upstreams::Http::Http::HttpConnPool {
 public:
-  PerHostHttpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+  PerHostHttpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
                       const Router::RouteEntry& route_entry,
                       absl::optional downstream_protocol,
                       Upstream::LoadBalancerContext* ctx)
-      : HttpConnPool(thread_local_cluster, is_connect, route_entry, downstream_protocol, ctx) {}
+      : HttpConnPool(thread_local_cluster, route_entry, downstream_protocol, ctx) {}
 
   void onPoolReady(Envoy::Http::RequestEncoder& callbacks_encoder,
                    Upstream::HostDescriptionConstSharedPtr host, StreamInfo::StreamInfo& info,
@@ -95,16 +95,17 @@ class PerHostGenericConnPoolFactory : public Router::GenericConnPoolFactory {
   std::string name() const override { return "envoy.filters.connection_pools.http.per_host"; }
   std::string category() const override { return "envoy.upstreams"; }
   Router::GenericConnPoolPtr
-  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
+                        Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol,
                         const Router::RouteEntry& route_entry,
                         absl::optional downstream_protocol,
                         Upstream::LoadBalancerContext* ctx) const override {
-    if (is_connect) {
-      // This example factory doesn't support terminating CONNECT stream.
+    if (upstream_protocol != UpstreamProtocol::HTTP) {
+      // This example factory doesn't support terminating CONNECT/CONNECT-UDP stream.
       return nullptr;
     }
     auto upstream_http_conn_pool = std::make_unique(
-        thread_local_cluster, is_connect, route_entry, downstream_protocol, ctx);
+        thread_local_cluster, route_entry, downstream_protocol, ctx);
     return (upstream_http_conn_pool->valid() ? std::move(upstream_http_conn_pool) : nullptr);
   }
 
diff --git a/test/mocks/network/connection.h b/test/mocks/network/connection.h
index 93213544a47c..99ecdd17a207 100644
--- a/test/mocks/network/connection.h
+++ b/test/mocks/network/connection.h
@@ -58,7 +58,7 @@ class MockConnectionBase {
   MOCK_METHOD(bool, isHalfCloseEnabled, (), (const));                                              \
   MOCK_METHOD(void, close, (ConnectionCloseType type));                                            \
   MOCK_METHOD(void, close, (ConnectionCloseType type, absl::string_view details));                 \
-  MOCK_METHOD(Event::Dispatcher&, dispatcher, ());                                                 \
+  MOCK_METHOD(Event::Dispatcher&, dispatcher, (), (const));                                        \
   MOCK_METHOD(uint64_t, id, (), (const));                                                          \
   MOCK_METHOD(void, hashKey, (std::vector&), (const));                                    \
   MOCK_METHOD(bool, initializeReadFilters, ());                                                    \
diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh
index a9c1113fcc2f..da38e06f8c44 100755
--- a/test/per_file_coverage.sh
+++ b/test/per_file_coverage.sh
@@ -62,6 +62,7 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/transport_sockets/tls:95.0"
 "source/extensions/transport_sockets/tls/cert_validator:95.2"
 "source/extensions/transport_sockets/tls/private_key:88.9"
+"source/extensions/upstreams/http/generic:85.0" # Braces in switch statements are considered uncovered
 "source/extensions/wasm_runtime/wamr:0.0" # Not enabled in coverage build
 "source/extensions/wasm_runtime/wasmtime:0.0" # Not enabled in coverage build
 "source/extensions/wasm_runtime/wavm:0.0" # Not enabled in coverage build

From 5cd5061817dc069caee95d8050ee131c8b181205 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Thu, 22 Jun 2023 18:11:52 +0100
Subject: [PATCH 626/740] mobile/ci: Use async mode for BEP uploads (#28084)

Signed-off-by: Ryan Northey 
---
 mobile/.bazelrc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mobile/.bazelrc b/mobile/.bazelrc
index 60b815c66132..f520875f3187 100644
--- a/mobile/.bazelrc
+++ b/mobile/.bazelrc
@@ -153,6 +153,7 @@ build:remote-ci-common --spawn_strategy=remote,sandboxed,local
 build:remote-ci-common --grpc_keepalive_time=30s
 build:remote-ci-common --remote_timeout=3600s
 build:remote-ci-common --bes_timeout=3600s
+build:remote-ci-common --bes_upload_mode=fully_async
 #############################################################################
 # remote-ci-linux: These options are linux-only using GCC by default
 #############################################################################

From c27c885a08318506915b0e90f0bb214e76505e3d Mon Sep 17 00:00:00 2001
From: phlax 
Date: Thu, 22 Jun 2023 18:17:12 +0100
Subject: [PATCH 627/740] deps: Bump `bazel_gazelle` -> 0.31.1 (#28092)

Signed-off-by: Ryan Northey 
---
 bazel/repository_locations.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
index e576c66d4422..9b04f572ba79 100644
--- a/bazel/repository_locations.bzl
+++ b/bazel/repository_locations.bzl
@@ -33,10 +33,10 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "Gazelle",
         project_desc = "Bazel BUILD file generator for Go projects",
         project_url = "https://github.com/bazelbuild/bazel-gazelle",
-        version = "0.31.0",
-        sha256 = "29d5dafc2a5582995488c6735115d1d366fcd6a0fc2e2a153f02988706349825",
+        version = "0.31.1",
+        sha256 = "b8b6d75de6e4bf7c41b7737b183523085f56283f6db929b86c5e7e1f09cf59c9",
         urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v{version}/bazel-gazelle-v{version}.tar.gz"],
-        release_date = "2023-05-27",
+        release_date = "2023-06-13",
         use_category = ["build"],
         license = "Apache-2.0",
         license_url = "https://github.com/bazelbuild/bazel-gazelle/blob/v{version}/LICENSE",

From 1ef7992acc2259b208b1c8c5ea84dcd33f2b072d Mon Sep 17 00:00:00 2001
From: phlax 
Date: Thu, 22 Jun 2023 18:44:37 +0100
Subject: [PATCH 628/740] ci: Update Ubuntu host agents (#28095)

Signed-off-by: Ryan Northey 
---
 .azure-pipelines/env.yml                          |  8 ++++----
 .azure-pipelines/pipelines.yml                    |  3 +++
 .azure-pipelines/stage/checks.yml                 |  4 ++--
 .azure-pipelines/stage/linux.yml                  |  2 +-
 .azure-pipelines/stage/macos.yml                  |  2 +-
 .azure-pipelines/stage/prechecks.yml              |  6 +++---
 .azure-pipelines/stage/publish.yml                | 12 ++++++------
 .azure-pipelines/stage/verify.yml                 |  4 ++--
 .azure-pipelines/stage/windows.yml                |  2 +-
 .github/workflows/check-deps.yml                  |  2 +-
 .github/workflows/depsreview.yml                  |  2 +-
 .github/workflows/env.yml                         |  4 +++-
 .github/workflows/envoy-sync.yml                  |  2 +-
 .github/workflows/envoy-verify.yml                |  2 +-
 .github/workflows/mobile-android_build.yml        |  2 +-
 .github/workflows/mobile-android_tests.yml        |  2 +-
 .github/workflows/mobile-asan.yml                 |  2 +-
 .github/workflows/mobile-cc_tests.yml             |  2 +-
 .github/workflows/mobile-compile_time_options.yml |  2 +-
 .github/workflows/mobile-core.yml                 |  2 +-
 .github/workflows/mobile-coverage.yml             |  2 +-
 .github/workflows/mobile-docs.yml                 |  2 +-
 .github/workflows/mobile-format.yml               |  4 ++--
 .github/workflows/mobile-perf.yml                 |  6 +++---
 .github/workflows/mobile-tsan.yml                 |  2 +-
 .github/workflows/mobile_release.yml              |  2 +-
 .github/workflows/pr_notifier.yml                 |  2 +-
 .github/workflows/release_branch.yml              |  4 ++--
 .github/workflows/stale.yml                       |  2 +-
 .github/workflows/workflow-complete.yml           |  2 +-
 .github/workflows/workflow-start.yml              |  2 +-
 31 files changed, 51 insertions(+), 46 deletions(-)

diff --git a/.azure-pipelines/env.yml b/.azure-pipelines/env.yml
index 9511877caf0a..e3f0fba5825a 100644
--- a/.azure-pipelines/env.yml
+++ b/.azure-pipelines/env.yml
@@ -38,7 +38,7 @@ jobs:
 - job: cache
   displayName: Cache
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - template: cached.yml
     parameters:
@@ -48,7 +48,7 @@ jobs:
   dependsOn: []
   displayName: Cache (arm64)
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - template: cached.yml
     parameters:
@@ -60,7 +60,7 @@ jobs:
   dependsOn: []
   displayName: Repository
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - checkout: self
     fetchDepth: 0
@@ -259,7 +259,7 @@ jobs:
   displayName: Test artifacts
   condition: ne(variables['Build.DefinitionName'], 'envoy-postsubmit')
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - script: $(Build.SourcesDirectory)/.azure-pipelines/gpg/generate-test-key.sh
     displayName: "Generate snakeoil GPG key for testing"
diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml
index e7bbb8c78070..aef608c585bb 100644
--- a/.azure-pipelines/pipelines.yml
+++ b/.azure-pipelines/pipelines.yml
@@ -13,6 +13,9 @@ pr: none
 
 
 variables:
+- name: agentUbuntu
+  value: ubuntu-20.04
+
 - name: isDev
   # This must be checked/set in a `step` from the VERSION.txt file, in order to be useful
   value: true
diff --git a/.azure-pipelines/stage/checks.yml b/.azure-pipelines/stage/checks.yml
index 3c2ef0e9a434..471904313e9e 100644
--- a/.azure-pipelines/stage/checks.yml
+++ b/.azure-pipelines/stage/checks.yml
@@ -68,7 +68,7 @@ jobs:
         CI_TARGET: "bazel.api"
   timeoutInMinutes: 180
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - template: ../bazel.yml
     parameters:
@@ -132,7 +132,7 @@ jobs:
   displayName: "Checks complete"
   dependsOn: ["bazel", "coverage"]
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   # This condition ensures that this (required) check passes if all of
   # the preceding checks either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/linux.yml b/.azure-pipelines/stage/linux.yml
index d92861610105..f581456b9596 100644
--- a/.azure-pipelines/stage/linux.yml
+++ b/.azure-pipelines/stage/linux.yml
@@ -48,7 +48,7 @@ jobs:
   displayName: Complete
   dependsOn: ["release"]
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   # This condition ensures that this (required) job passes if all of
   # the preceeding jobs either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/macos.yml b/.azure-pipelines/stage/macos.yml
index cb3ecfd38825..bc16cacd7d4f 100644
--- a/.azure-pipelines/stage/macos.yml
+++ b/.azure-pipelines/stage/macos.yml
@@ -43,7 +43,7 @@ jobs:
   displayName: Complete
   dependsOn: ["test"]
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   # This condition ensures that this (required) job passes if all of
   # the preceeding jobs either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/prechecks.yml b/.azure-pipelines/stage/prechecks.yml
index 869b438bc7d5..abaee79a2e9e 100644
--- a/.azure-pipelines/stage/prechecks.yml
+++ b/.azure-pipelines/stage/prechecks.yml
@@ -39,7 +39,7 @@ jobs:
   displayName: Precheck
   timeoutInMinutes: 30
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   variables:
     CI_TARGET: ""
   strategy:
@@ -165,7 +165,7 @@ jobs:
   displayName: Precheck dependencies
   timeoutInMinutes: 20
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   condition: |
     and(not(canceled()),
         eq(${{ parameters.checkDeps }}, 'true'))
@@ -183,7 +183,7 @@ jobs:
   displayName: Prechecked
   dependsOn: ["prechecks", "dependencies"]
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   # This condition ensures that this (required) job passes if all of
   # the preceeding jobs either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml
index 17101a5bc84f..6b47eeae5134 100644
--- a/.azure-pipelines/stage/publish.yml
+++ b/.azure-pipelines/stage/publish.yml
@@ -104,7 +104,7 @@ jobs:
         eq(${{ parameters.runDocker }}, 'true'))
   timeoutInMinutes: 120
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - task: DownloadBuildArtifacts@0
     inputs:
@@ -201,7 +201,7 @@ jobs:
         eq(${{ parameters.runPackaging }}, 'true'))
   timeoutInMinutes: 120
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - task: DownloadBuildArtifacts@0
     inputs:
@@ -268,7 +268,7 @@ jobs:
     and(not(canceled()),
         eq(${{ parameters.publishDocs }}, 'true'))
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - template: ../bazel.yml
     parameters:
@@ -348,7 +348,7 @@ jobs:
         eq(${{ parameters.runPackaging }}, 'true'))
   timeoutInMinutes: 120
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - task: DownloadBuildArtifacts@0
     inputs:
@@ -377,7 +377,7 @@ jobs:
   dependsOn: ["docker", "docs", "signed_release"]
   displayName: Success (linux artefacts)
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   # This condition ensures that this (required) check passes if all of
   # the preceding checks either pass or are skipped
   # adapted from:
@@ -400,7 +400,7 @@ jobs:
         in(dependencies.success.result, 'Succeeded', 'SucceededWithIssues'),
         eq(${{ parameters.publishGithubRelease }}, 'true'))
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - task: DownloadBuildArtifacts@0
     inputs:
diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml
index 127a2148ac8f..1ae438a85dcc 100644
--- a/.azure-pipelines/stage/verify.yml
+++ b/.azure-pipelines/stage/verify.yml
@@ -13,7 +13,7 @@ jobs:
   condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
   timeoutInMinutes: 120
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   steps:
   - task: DownloadBuildArtifacts@0
     inputs:
@@ -61,7 +61,7 @@ jobs:
   displayName: Verification complete
   dependsOn: ["packages_x64", "packages_arm64"]
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   # This condition ensures that this (required) check passes if all of
   # the preceding checks either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/windows.yml b/.azure-pipelines/stage/windows.yml
index 6b1062f714b9..b2d81d766198 100644
--- a/.azure-pipelines/stage/windows.yml
+++ b/.azure-pipelines/stage/windows.yml
@@ -107,7 +107,7 @@ jobs:
   displayName: Complete
   dependsOn: ["release", "docker"]
   pool:
-    vmImage: "ubuntu-20.04"
+    vmImage: $(agentUbuntu)
   # This condition ensures that this (required) job passes if all of
   # the preceeding jobs either pass or are skipped
   # adapted from:
diff --git a/.github/workflows/check-deps.yml b/.github/workflows/check-deps.yml
index bb98ec2f36d7..a1b96c89ce71 100644
--- a/.github/workflows/check-deps.yml
+++ b/.github/workflows/check-deps.yml
@@ -9,7 +9,7 @@ permissions: read-all
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: |
       ${{
           github.repository == 'envoyproxy/envoy'
diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml
index 542888585103..e53564b0bb9d 100644
--- a/.github/workflows/depsreview.yml
+++ b/.github/workflows/depsreview.yml
@@ -7,7 +7,7 @@ concurrency:
 
 jobs:
   dependency-review:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: github.repository == 'envoyproxy/envoy'
     steps:
     - name: 'Checkout Repository'
diff --git a/.github/workflows/env.yml b/.github/workflows/env.yml
index 27149302a3d6..cd2acfb374bb 100644
--- a/.github/workflows/env.yml
+++ b/.github/workflows/env.yml
@@ -3,6 +3,8 @@ name: Environment
 on:
   workflow_call:
     outputs:
+      agent_ubuntu:
+        value: ubuntu-22.04
       build_image_ubuntu:
         value: ${{ jobs.repo.outputs.build_image_ubuntu }}
       build_image_ubuntu_mobile:
@@ -41,7 +43,7 @@ concurrency:
 jobs:
   repo:
     if: github.repository == 'envoyproxy/envoy'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     outputs:
       build_image_ubuntu: ${{ steps.build_image.outputs.build_image_ubuntu }}
       build_image_ubuntu_mobile: ${{ steps.build_image.outputs.build_image_ubuntu_mobile }}
diff --git a/.github/workflows/envoy-sync.yml b/.github/workflows/envoy-sync.yml
index 1b7f4fe27d4a..a55e20bcdb2a 100644
--- a/.github/workflows/envoy-sync.yml
+++ b/.github/workflows/envoy-sync.yml
@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   sync:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: github.repository == 'envoyproxy/envoy'
     strategy:
       fail-fast: false
diff --git a/.github/workflows/envoy-verify.yml b/.github/workflows/envoy-verify.yml
index 5460797233b5..e72707047e90 100644
--- a/.github/workflows/envoy-verify.yml
+++ b/.github/workflows/envoy-verify.yml
@@ -35,7 +35,7 @@ jobs:
 
   # Runs untrusted code
   verify-examples:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: check
     steps:
     - run: |
diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml
index 81814c64ada9..a2f6e928e82c 100644
--- a/.github/workflows/mobile-android_build.yml
+++ b/.github/workflows/mobile-android_build.yml
@@ -20,7 +20,7 @@ jobs:
     if: ${{ needs.env.outputs.mobile_android_build == 'true' }}
     needs: env
     name: android_build
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 90
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu_mobile }}
diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml
index 93129f5b9e86..280ab61c24c9 100644
--- a/.github/workflows/mobile-android_tests.yml
+++ b/.github/workflows/mobile-android_tests.yml
@@ -80,7 +80,7 @@ jobs:
     # Only kotlin tests are executed since with linux:
     # https://github.com/envoyproxy/envoy-mobile/issues/1418.
     name: kotlin_tests_linux
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 90
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu_mobile }}
diff --git a/.github/workflows/mobile-asan.yml b/.github/workflows/mobile-asan.yml
index e6e72fcb3967..97db9e2dc314 100644
--- a/.github/workflows/mobile-asan.yml
+++ b/.github/workflows/mobile-asan.yml
@@ -20,7 +20,7 @@ jobs:
     if: ${{ needs.env.outputs.mobile_asan == 'true' }}
     needs: env
     name: asan
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 180
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu_mobile }}
diff --git a/.github/workflows/mobile-cc_tests.yml b/.github/workflows/mobile-cc_tests.yml
index 99e1951fde1b..3a2f16c81f14 100644
--- a/.github/workflows/mobile-cc_tests.yml
+++ b/.github/workflows/mobile-cc_tests.yml
@@ -20,7 +20,7 @@ jobs:
     if: ${{ needs.env.outputs.mobile_cc_tests == 'true' }}
     needs: env
     name: cc_tests
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 120
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu }}
diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml
index 196921a57e4c..497f60e5e106 100644
--- a/.github/workflows/mobile-compile_time_options.yml
+++ b/.github/workflows/mobile-compile_time_options.yml
@@ -19,7 +19,7 @@ jobs:
   cc_test:
     needs: env
     name: cc_test
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 120
     container:
       image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33
diff --git a/.github/workflows/mobile-core.yml b/.github/workflows/mobile-core.yml
index 31383adf4752..b6b942d428af 100644
--- a/.github/workflows/mobile-core.yml
+++ b/.github/workflows/mobile-core.yml
@@ -20,7 +20,7 @@ jobs:
     if: ${{ github.repository == 'envoyproxy/envoy' }}
     needs: env
     name: unit_tests
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 120
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu }}
diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml
index f628f2d62134..3c9725400394 100644
--- a/.github/workflows/mobile-coverage.yml
+++ b/.github/workflows/mobile-coverage.yml
@@ -20,7 +20,7 @@ jobs:
     if: ${{ needs.env.outputs.mobile_coverage == 'true' }}
     needs: env
     name: coverage
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 120
     defaults:
       run:
diff --git a/.github/workflows/mobile-docs.yml b/.github/workflows/mobile-docs.yml
index 31554d189288..9bf360c4703f 100644
--- a/.github/workflows/mobile-docs.yml
+++ b/.github/workflows/mobile-docs.yml
@@ -19,7 +19,7 @@ jobs:
   docs:
     if: ${{ github.repository == 'envoyproxy/envoy' }}
     needs: env
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 20
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu }}
diff --git a/.github/workflows/mobile-format.yml b/.github/workflows/mobile-format.yml
index 54b57cddbcf4..3a601745a251 100644
--- a/.github/workflows/mobile-format.yml
+++ b/.github/workflows/mobile-format.yml
@@ -20,7 +20,7 @@ jobs:
     if: ${{ needs.env.outputs.mobile_formatting == 'true' }}
     needs: env
     name: format_all
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 45
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu }}
@@ -51,7 +51,7 @@ jobs:
     if: ${{ needs.env.outputs.mobile_formatting == 'true' }}
     needs: env
     name: swift_lint
-    runs-on: ubuntu-latest
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 5
     container:
       image: ghcr.io/realm/swiftlint:0.50.3
diff --git a/.github/workflows/mobile-perf.yml b/.github/workflows/mobile-perf.yml
index 9da97dbb0016..754097c2b0aa 100644
--- a/.github/workflows/mobile-perf.yml
+++ b/.github/workflows/mobile-perf.yml
@@ -14,7 +14,7 @@ jobs:
   sizecurrent:
     if: ${{ github.repository == 'envoyproxy/envoy' }}
     name: size_current
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 120
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu }}
@@ -41,7 +41,7 @@ jobs:
   sizemain:
     if: ${{ github.repository == 'envoyproxy/envoy' }}
     name: size_main
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 90
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu }}
@@ -73,7 +73,7 @@ jobs:
     - sizecurrent
     - sizemain
     name: size_compare
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 30
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu }}
diff --git a/.github/workflows/mobile-tsan.yml b/.github/workflows/mobile-tsan.yml
index 713e9a27d720..1a6d9a8c7d41 100644
--- a/.github/workflows/mobile-tsan.yml
+++ b/.github/workflows/mobile-tsan.yml
@@ -20,7 +20,7 @@ jobs:
     if: ${{ needs.env.outputs.mobile_tsan == 'true' }}
     needs: env
     name: tsan
-    runs-on: ubuntu-20.04
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
     timeout-minutes: 90
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu_mobile }}
diff --git a/.github/workflows/mobile_release.yml b/.github/workflows/mobile_release.yml
index e2f6f7b947b6..aaecefb138b6 100644
--- a/.github/workflows/mobile_release.yml
+++ b/.github/workflows/mobile_release.yml
@@ -15,7 +15,7 @@ jobs:
               || !contains(github.actor, '[bot]'))
       }}
     name: android_release_artifacts
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 120
     container:
       image: ${{ needs.env.outputs.build_image_ubuntu_mobile }}
diff --git a/.github/workflows/pr_notifier.yml b/.github/workflows/pr_notifier.yml
index ac913f34f2e1..21f0bc622b11 100644
--- a/.github/workflows/pr_notifier.yml
+++ b/.github/workflows/pr_notifier.yml
@@ -13,7 +13,7 @@ jobs:
       statuses: read  # for pr_notifier.py
       pull-requests: read  # for pr_notifier.py
     name: PR Notifier
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: |
       ${{
           github.repository == 'envoyproxy/envoy'
diff --git a/.github/workflows/release_branch.yml b/.github/workflows/release_branch.yml
index 5162d224612e..ed9d42b43f94 100644
--- a/.github/workflows/release_branch.yml
+++ b/.github/workflows/release_branch.yml
@@ -11,7 +11,7 @@ permissions: read-all
 
 jobs:
   fork_release_branch:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: github.repository == 'envoyproxy/envoy'
     permissions:
       contents: write
@@ -27,7 +27,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   reopen_branch:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: github.repository == 'envoyproxy/envoy'
     permissions:
       contents: write
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index faec7e043697..aa6d198d0744 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -9,7 +9,7 @@ jobs:
       issues: write  # for actions/stale to close stale issues
       pull-requests: write  # for actions/stale to close stale PRs
     name: Prune Stale
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: |
       ${{
           github.repository == 'envoyproxy/envoy'
diff --git a/.github/workflows/workflow-complete.yml b/.github/workflows/workflow-complete.yml
index 12a7363fe127..bed4f808122a 100644
--- a/.github/workflows/workflow-complete.yml
+++ b/.github/workflows/workflow-complete.yml
@@ -15,7 +15,7 @@ permissions:
 
 jobs:
   complete:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     permissions:
       statuses: write
     steps:
diff --git a/.github/workflows/workflow-start.yml b/.github/workflows/workflow-start.yml
index 1e832b6261f8..66017c0cf0c2 100644
--- a/.github/workflows/workflow-start.yml
+++ b/.github/workflows/workflow-start.yml
@@ -14,7 +14,7 @@ permissions:
 
 jobs:
   start:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     permissions:
       statuses: write
     steps:

From 4d36e443425ad150591a79ae1f1d7ed35a0835b4 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Fri, 23 Jun 2023 09:12:18 +0100
Subject: [PATCH 629/740] ci/actions: Fix sync dispatch action (#28113)

Signed-off-by: Ryan Northey 
---
 .github/workflows/envoy-sync.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/envoy-sync.yml b/.github/workflows/envoy-sync.yml
index a55e20bcdb2a..7cf644bb2e26 100644
--- a/.github/workflows/envoy-sync.yml
+++ b/.github/workflows/envoy-sync.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
     - main
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}
@@ -12,7 +13,12 @@ concurrency:
 jobs:
   sync:
     runs-on: ubuntu-22.04
-    if: github.repository == 'envoyproxy/envoy'
+    if: |
+      ${{
+          github.repository == 'envoyproxy/envoy'
+          && (github.event.push
+              || !contains(github.actor, '[bot]'))
+      }}
     strategy:
       fail-fast: false
       matrix:
@@ -21,7 +27,7 @@ jobs:
         - envoy-filter-example
         - data-plane-api
     steps:
-    - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.0.2
+    - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.0.3
       with:
         repository: "envoyproxy/${{ matrix.downstream }}"
         ref: main

From 8d20966d248377c87a457405a1bab453954b66d7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 23 Jun 2023 09:16:33 +0100
Subject: [PATCH 630/740] build(deps): bump grpcio-tools from 1.55.0 to 1.56.0
 in /examples/grpc-bridge/client (#28106)

build(deps): bump grpcio-tools in /examples/grpc-bridge/client

Bumps [grpcio-tools](https://github.com/grpc/grpc) from 1.55.0 to 1.56.0.
- [Release notes](https://github.com/grpc/grpc/releases)
- [Changelog](https://github.com/grpc/grpc/blob/master/doc/grpc_release_schedule.md)
- [Commits](https://github.com/grpc/grpc/compare/v1.55.0...v1.56.0)

---
updated-dependencies:
- dependency-name: grpcio-tools
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/grpc-bridge/client/requirements.txt | 184 +++++++++----------
 1 file changed, 92 insertions(+), 92 deletions(-)

diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt
index ec9135051317..3c50e391ffdc 100644
--- a/examples/grpc-bridge/client/requirements.txt
+++ b/examples/grpc-bridge/client/requirements.txt
@@ -12,101 +12,101 @@ charset-normalizer==2.0.6 \
     --hash=sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6 \
     --hash=sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f
     # via requests
-grpcio==1.55.0 \
-    --hash=sha256:054b7164b25712ec71339e139875a66708a2ab09be36ac75e73b2d337ab2dc1b \
-    --hash=sha256:0d3d5c644d523dee82ffcc44ad50cd66e3bf66e7fa60ad3cdb1eb868228e4ab0 \
-    --hash=sha256:1041cad23f00943d8889ad15427d87bbdacbbe2df5cec951c314f2f3967d4691 \
-    --hash=sha256:10af4774da9c0665a1bf519333694ac40d72d83cb514534b99db0a5e3d5c3593 \
-    --hash=sha256:1173a05117798aca4834d3edd504e6adc25ae9967df0f44b91a612884fb2707a \
-    --hash=sha256:157f5615c7b5d0968727472f6394dee01555ef4246d2f2cfb6555be857936d74 \
-    --hash=sha256:1982c99c7091d1b7e3e78b1173097f705feef233e253a27e99746b11815ac897 \
-    --hash=sha256:2663741acc117370fd53336267cfb24c965e9d3ea1e4933a3e4411712d3091fb \
-    --hash=sha256:29ab0e879b1585be41cfbb02faed67913700ced8015da4763f1f0bdd7dfb4ab7 \
-    --hash=sha256:2d25d7fcb528a40578b3d0428d401745fd5c0eeeda81f35ce2f40a10d79afd19 \
-    --hash=sha256:322d4ebc37cbc8d8596b1da6055e3e81e8cfd36816ab4b285c1163c3042e6067 \
-    --hash=sha256:3ab9bf80c19c91847f45ff32af94c85d282545a62db39d797838244d57831d78 \
-    --hash=sha256:4370d2cca37301bcc69453d3dd3c1576d06d6b3e337bfec55b3aab2fe106b25c \
-    --hash=sha256:48f6088d898e1e987d761d58dc4cd724e7457a7a86d11561fa95c3b826d025dc \
-    --hash=sha256:51b7a27a129f743d68394f94029f88ef3da090fc13776b9dfa3c79c5f4b30525 \
-    --hash=sha256:56631cc0bdf86d15ea1599b9697ace65e6b52c6b136d3666bf7769d3d6d087a8 \
-    --hash=sha256:60efab181c32e029e0960f238508396dd001ba2064168f8148e6356db093967c \
-    --hash=sha256:67c4fda71f92225c5e74fa15bffa6be022c07111f674fe1f234c1ef4c1bb7927 \
-    --hash=sha256:6b8dbb151b116825c10f01e5b7b75e14edd0e60736a65311d0d98a4cd0489303 \
-    --hash=sha256:70de2b73cf22241173cb21d308786ba4ea443e4c88441a2ce445829aa638dda8 \
-    --hash=sha256:74780f570c76feb8e62a8c019b495fea435b60218682fce513ff2c71262c346c \
-    --hash=sha256:7b38e028a7bbc97a9ae5e418712452f298618b9d0493390770bf2de785251ae7 \
-    --hash=sha256:7b8665da31b5bd701b338a581de7b9631d50b4b7ee67125c2d1dc2228cc119d8 \
-    --hash=sha256:7c00263d792a244bef67a8d3b357ccbcdae6341c5961dbee494d8f967f9aee69 \
-    --hash=sha256:7c32f87bec58a8a0d4f4d5387bd61a383bd32b2caffb2de3cd579e47490b7e19 \
-    --hash=sha256:89107071b5f14af6bbb855183d338a0fa94136bbeb3989c9773c6184e51a95e9 \
-    --hash=sha256:8a910fa9b95a286f4bc1879dcf8d5ccb95b5e33bb63323fc4414d157f23afef1 \
-    --hash=sha256:8b440ccc434c1ad5874465bfae40c0a27f562ae5f7c5b468b6689bc55e8bf1c1 \
-    --hash=sha256:8bd4f4932ef63ed32a725065aebb8585e4118a523d923db896e85c09429a36e6 \
-    --hash=sha256:9a11b1dd4b1572e85fba5911309c15980a1ff77c555fad0ecdbe3711ef741908 \
-    --hash=sha256:a202dcf0c512292fd7a2154e4044c70400212eaa726685ebf8af105e25693c5a \
-    --hash=sha256:a82283d6e0403d3e2e7eebb99cb0d2783e20b6791c8c94bd8d4a4233b58b1ea0 \
-    --hash=sha256:ab784204d9923368e0e5877d7795584b9606a51b128ee199ad8b5888d0c66592 \
-    --hash=sha256:b1e2b705d524e780998218cf429d30b6ffc54cb6e54812c9597bc5df12dbcb5b \
-    --hash=sha256:b2a3b837d5837b9069783026b57aa0ff12e34d3218fdeda3f9c06d3950266d8e \
-    --hash=sha256:ba32a8e9bc3eecc6bab6824b905f04c3fdc31659c3e6e06841b774e7cb4410af \
-    --hash=sha256:c33dbeecc14f1a413e8af8ae1208cb383b063fa2ff2e1f309b4d3d7739b0927e \
-    --hash=sha256:c97cfae0b7a17dc1a0a3e4333f4f46daa114d85f950a67f39cc141b5425182e4 \
-    --hash=sha256:ce82d06cdfb8a9292fb857f00bee11a2430e4ac2742e07b46c1a3072d683256a \
-    --hash=sha256:d0209fb3cb55c5288a1dec72dcaae2c1b501edceca10d22c0f0baa5e60e2b22c \
-    --hash=sha256:d396ec4d520b58f43142958cff071e5ad1c50ac87d29d086a9c6a990a09ea536 \
-    --hash=sha256:dad999423b33ad5409e986587593b6062a8260b74ae8fc8162ce231c6b7a929e \
-    --hash=sha256:dd15027a171ff93c97f9c704fa120bc5d0691dc7e71ae450e2ecade1a2799b53 \
-    --hash=sha256:ee0de9cb6813704969e53743e0969fd95225ff24bd686c89ed12a18147f6566c \
-    --hash=sha256:fe78365c64b2c7470d31c4941e10c6654042bcbb53897b9b1e2c96d6d0da9ef9
+grpcio==1.56.0 \
+    --hash=sha256:008767c0aed4899e657b50f2e0beacbabccab51359eba547f860e7c55f2be6ba \
+    --hash=sha256:03a80451530fd3b8b155e0c4480434f6be669daf7ecba56f73ef98f94222ee01 \
+    --hash=sha256:0409de787ebbf08c9d2bca2bcc7762c1efe72eada164af78b50567a8dfc7253c \
+    --hash=sha256:14e70b4dda3183abea94c72d41d5930c333b21f8561c1904a372d80370592ef3 \
+    --hash=sha256:17f47aeb9be0da5337f9ff33ebb8795899021e6c0741ee68bd69774a7804ca86 \
+    --hash=sha256:187b8f71bad7d41eea15e0c9812aaa2b87adfb343895fffb704fb040ca731863 \
+    --hash=sha256:1eadd6de258901929223f422ffed7f8b310c0323324caf59227f9899ea1b1674 \
+    --hash=sha256:38fdf5bd0a1c754ce6bf9311a3c2c7ebe56e88b8763593316b69e0e9a56af1de \
+    --hash=sha256:4241a1c2c76e748023c834995cd916570e7180ee478969c2d79a60ce007bc837 \
+    --hash=sha256:437af5a7673bca89c4bc0a993382200592d104dd7bf55eddcd141cef91f40bab \
+    --hash=sha256:43c50d810cc26349b093bf2cfe86756ab3e9aba3e7e681d360930c1268e1399a \
+    --hash=sha256:4c08ee21b3d10315b8dc26f6c13917b20ed574cdbed2d2d80c53d5508fdcc0f2 \
+    --hash=sha256:4f84a6fd4482e5fe73b297d4874b62a535bc75dc6aec8e9fe0dc88106cd40397 \
+    --hash=sha256:4feee75565d1b5ab09cb3a5da672b84ca7f6dd80ee07a50f5537207a9af543a4 \
+    --hash=sha256:50f4daa698835accbbcc60e61e0bc29636c0156ddcafb3891c987e533a0031ba \
+    --hash=sha256:59c4e606993a47146fbeaf304b9e78c447f5b9ee5641cae013028c4cca784617 \
+    --hash=sha256:5d2fc471668a7222e213f86ef76933b18cdda6a51ea1322034478df8c6519959 \
+    --hash=sha256:64bd3abcf9fb4a9fa4ede8d0d34686314a7075f62a1502217b227991d9ca4245 \
+    --hash=sha256:66f0369d27f4c105cd21059d635860bb2ea81bd593061c45fb64875103f40e4a \
+    --hash=sha256:6b5ce42a5ebe3e04796246ba50357f1813c44a6efe17a37f8dc7a5c470377312 \
+    --hash=sha256:72836b5a1d4f508ffbcfe35033d027859cc737972f9dddbe33fb75d687421e2e \
+    --hash=sha256:76b6e6e1ee9bda32e6e933efd61c512e9a9f377d7c580977f090d1a9c78cca44 \
+    --hash=sha256:79d4c5911d12a7aa671e5eb40cbb50a830396525014d2d6f254ea2ba180ce637 \
+    --hash=sha256:7beb84ebd0a3f732625124b73969d12b7350c5d9d64ddf81ae739bbc63d5b1ed \
+    --hash=sha256:8219f17baf069fe8e42bd8ca0b312b875595e43a70cabf397be4fda488e2f27d \
+    --hash=sha256:83ec714bbbe9b9502177c842417fde39f7a267031e01fa3cd83f1ca49688f537 \
+    --hash=sha256:8674fdbd28266d8efbcddacf4ec3643f76fe6376f73283fd63a8374c14b0ef7c \
+    --hash=sha256:881575f240eb5db72ddca4dc5602898c29bc082e0d94599bf20588fb7d1ee6a0 \
+    --hash=sha256:8b3b2c7b5feef90bc9a5fa1c7f97637e55ec3e76460c6d16c3013952ee479cd9 \
+    --hash=sha256:991224fd485e088d3cb5e34366053691a4848a6b7112b8f5625a411305c26691 \
+    --hash=sha256:aa08affbf672d051cd3da62303901aeb7042a2c188c03b2c2a2d346fc5e81c14 \
+    --hash=sha256:b1f4b6f25a87d80b28dd6d02e87d63fe1577fe6d04a60a17454e3f8077a38279 \
+    --hash=sha256:b4638a796778329cc8e142e4f57c705adb286b3ba64e00b0fa91eeb919611be8 \
+    --hash=sha256:bd55f743e654fb050c665968d7ec2c33f03578a4bbb163cfce38024775ff54cc \
+    --hash=sha256:c0bc9dda550785d23f4f025be614b7faa8d0293e10811f0f8536cf50435b7a30 \
+    --hash=sha256:c2148170e01d464d41011a878088444c13413264418b557f0bdcd1bf1b674a0e \
+    --hash=sha256:c243b158dd7585021d16c50498c4b2ec0a64a6119967440c5ff2d8c89e72330e \
+    --hash=sha256:c63bc5ac6c7e646c296fed9139097ae0f0e63f36f0864d7ce431cce61fe0118a \
+    --hash=sha256:c6f36621aabecbaff3e70c4d1d924c76c8e6a7ffec60c331893640a4af0a8037 \
+    --hash=sha256:d596408bab632ec7b947761e83ce6b3e7632e26b76d64c239ba66b554b7ee286 \
+    --hash=sha256:defdd14b518e6e468466f799aaa69db0355bca8d3a5ea75fb912d28ba6f8af31 \
+    --hash=sha256:e2db108b4c8e29c145e95b0226973a66d73ae3e3e7fae00329294af4e27f1c42 \
+    --hash=sha256:f92a99ab0c7772fb6859bf2e4f44ad30088d18f7c67b83205297bfb229e0d2cf \
+    --hash=sha256:fb34ace11419f1ae321c36ccaa18d81cd3f20728cd191250be42949d6845bb2d \
+    --hash=sha256:fdc3a895791af4addbb826808d4c9c35917c59bb5c430d729f44224e51c92d61
     # via
     #   -r requirements.in
     #   grpcio-tools
-grpcio-tools==1.55.0 \
-    --hash=sha256:053bbdfb74f76511db47e1e18a1962432468ae9f356cc00f15d1f1353eaf32a1 \
-    --hash=sha256:07c23ed940e046c9dd471bc870eb5db4d93e518f90011cf9aebf8bfda6cd68a5 \
-    --hash=sha256:0dead7fb37bfe7c7eb8294143015645297f4affa683783b8bbf2cd4d7f7036d4 \
-    --hash=sha256:2ba87592f2cd689e127cd4fce76ec23b19562e230fa41ea089af8b15120aea78 \
-    --hash=sha256:2eacb0b1e8e5cfd0b40e12e62bd5adebbbae8c73cdf6e04fad9ddd37e32d98a4 \
-    --hash=sha256:350303ef3a2b25ed1b90e42764923e40b664d9f10840f7a0f06117c4dc414aff \
-    --hash=sha256:36745762689df18f83273a9a004848897793f63a10a30acd18acb2d170c663a9 \
-    --hash=sha256:3724e48c3db499b2d212c5a89d7cc4b49ccd476dc26bf8a9b855d59b6cc00796 \
-    --hash=sha256:41005002cbfa0ad39972486bde8116b2a042804119e5b998086a4dc26e625d6a \
-    --hash=sha256:416a8b61ed4223715755b4519858419e1f4653d64572a28029f2ac63e677e3d2 \
-    --hash=sha256:4580df5a9867f7bcbb828a5485c030ca232c1578e615caf751333c7a7980d838 \
-    --hash=sha256:4a41130c97775bb0dfaf87e34b492f2eca448d02d213410005544c534f3f7c26 \
-    --hash=sha256:4a6db1494955d2a5531575b5fcdc08094ea4a331a94b9cdf864d78e801c5fa23 \
-    --hash=sha256:4dea66623548b52429fb03495f2c76f4c993bf9a56267c6b3d0fb62573dd52c2 \
-    --hash=sha256:51a1ccab6f67edd1a3768a75ac495907fe0cd6d6617af2f9f2033400b5858a11 \
-    --hash=sha256:52e34e9b6496f4c1e3289ada7bc41d759e4a8ec5f2679e187067cab8532ffbf4 \
-    --hash=sha256:632364ffbd4fb0338cb03c590a2ddc258d9cd59bff0bf4199c02e3e581f802d7 \
-    --hash=sha256:734ede84d613b044f72e7d9c190bd2388ebb83e85bcd3aa75afa9f30c096dbc7 \
-    --hash=sha256:73ef9e0e0ee8ab055a621e7b42e5fb32753b0b6607900887dba6d55df5947be8 \
-    --hash=sha256:7f084cd619cf66d8620a99f8586018f19b918ffb2ddb92d3e5943a06038bead8 \
-    --hash=sha256:87152893c7c3bef58a6a9b548db290aa318cc314c700ae7d7f2970aa567f875e \
-    --hash=sha256:87dbc98528f88faa3f8f56a47d41dc6fda382928abbdb5537b5444eb8bb1ac1b \
-    --hash=sha256:89f6ed47415a22568bbf4a62336bfde7cafb53492a5a9f33a22243411b00f443 \
-    --hash=sha256:8e59fd4a58688117cb5128d7785909d45a6e5f8212efeb65b6fd74bb9b8b9512 \
-    --hash=sha256:9395c4fdee6b22137e878ebd461444854a3cd9c6c260c43f4a4c4a4023700129 \
-    --hash=sha256:946266cbd639847548c9f97e38da0682746c2eadea790ceb4320b1f85387bd6d \
-    --hash=sha256:95428be2db12412ff23f0969386fc51d2aa6de38a57cc54c57363352f1d7a832 \
-    --hash=sha256:98ff3129ff7134a95f4d2857663625771f6838ac44b7799c34259b7ea87ebe5c \
-    --hash=sha256:9933a1f18f780c42214b126ef27e273b54c9c28de3fae5b1887b413ceb374c4c \
-    --hash=sha256:a61567f27661ab9327dc060615dc22d2bde80c56731f1e856008f1fd8ee83311 \
-    --hash=sha256:ab64f9d6f5e3636ae6298e2d795225daa83aacb057105943728ed50a8a582237 \
-    --hash=sha256:b00a67a1230968c1a0424915922d17983d824ed45e8db06f9f17be6d5571faee \
-    --hash=sha256:b131b2bbf25198d9e508dfa588cb215580629b514e293d5609eeee98c8941dbc \
-    --hash=sha256:b197de69ca0431b718ffa47b32a733703fa5503da49f49dd315c866842b6cfbd \
-    --hash=sha256:b674de79571357c5381bc5fa12e3b89fefef74c164ab9077ed22158c3529aa8e \
-    --hash=sha256:bc23034b1959d6cda27347b2207fee0fb0fb0aff242da228a6b7c1a18fce4116 \
-    --hash=sha256:bcf5e1858137cbe13ef10a7931a7edc745c77f8b39f032f52072443f0dd681e1 \
-    --hash=sha256:c3c7b7eb89f963b87922ecc0c0ab2485fff05997ada66dffd53597b507a83bc8 \
-    --hash=sha256:c7a18bd5f994b7911d3e70e0abb05bea9f1b084a1725d404a8e231bf9727613b \
-    --hash=sha256:cfc82c11ce51de6ed5836fbafbc188d9eac0737abc116978f151c40271783817 \
-    --hash=sha256:d796f5d7cea260ef2afed12d13ec34b13e09dd74d7f292d7428c506fa8c17a74 \
-    --hash=sha256:e76f35e5e65600a75c3547855e8c9ab935c55c473f5409e7746cca8f1f7c8f4a \
-    --hash=sha256:f87d99aa3826aa20c3b89493984cf278f4c9d20b3418534a46239c804fee506c \
-    --hash=sha256:f900bce944b5777effecb9078e5fd3a224e42b1ca33c7546c3d043f9ef9eb4e8 \
-    --hash=sha256:fbbe2bee4af93c03ba064d40199dbf38067d2aa6ae98dfa0687a08ee980ebfd5
+grpcio-tools==1.56.0 \
+    --hash=sha256:02b23a12b91287ebea14b3685735d1d675e77c3cd365ec1771c3e9afbeba1ec6 \
+    --hash=sha256:0a8767e4de0f573c678313c5de075ac0e163a192bb135018e45015a22f234387 \
+    --hash=sha256:11cdd9cbf0c09c3a761c6f59dfd7128104be7cd393334efe386d4fc3f990ee1a \
+    --hash=sha256:128bb13fe9a2681eeb08175f5fbc8e2d8953d7d0dd240e96f9244b9d2547a1aa \
+    --hash=sha256:142530b9fdfabe04f0c7e5dacd45b6c419d39704fa439cc0aabf73ea0d8f916d \
+    --hash=sha256:168940a4a955b6c65da978dbf62e1c36e3a311bb27f649fd201a228e2583a6d4 \
+    --hash=sha256:1bd361fcc967c21672ba855fc77ea0e7afa51664033a746df96545f84edc4670 \
+    --hash=sha256:21cf32ccffd4f1800b0dcdf58aa1fc7f626795c9da784c3d817c944edcf2d3ae \
+    --hash=sha256:23e2ef1dc6a9bf766f091e2c52a68e54d0aff3548f94562e61fb0ac3874d514a \
+    --hash=sha256:282176066fb082ad21c403b84f9d6b440a20482e6f52b83bb2adf54d6fdcae9f \
+    --hash=sha256:2b7a4eb5003a29eecd71707589f93ae7e8fa2e681366a811b3f86695055d8666 \
+    --hash=sha256:2c1c43d185ebf904c3deec23c36ca2ba4e95db999cf00fc8f85eda4551622a26 \
+    --hash=sha256:2d1ee9e13ce135a6ed451b428ef14af131dc7df2551a5344ff4f8aee2d9fab99 \
+    --hash=sha256:39f5877cea514b3da9f2683dfb3ffb45ef47b05f4ff39c287d7d61c5057f48b8 \
+    --hash=sha256:3a4b06169493f9454a7f2516c5d41b566d9734e553bbc505f2a7837f7f4a2df1 \
+    --hash=sha256:3de6c08b545920a39b31ed13305f946c00b19ac1b13d26119f111b6360f22ccf \
+    --hash=sha256:4acdc7b957abfd76581717f0ac8e4408e0a85b7d0ac8d2cdf4d964f16926b897 \
+    --hash=sha256:4d59009ed52220eb2d62f5cefa4e58dec930fb92fab27bb390c4cf1d360ac7e1 \
+    --hash=sha256:5f5c416b88d76fbdb548cfee0486928748816b700ece6e591006e5b1dc67598f \
+    --hash=sha256:5fd4c005a4afec16578849bc522ddf3298d6d499b3d37bf51314b086c714cdd5 \
+    --hash=sha256:781cf09e4d5c9288708f6ec9c3eae64d9d5a0f4c46c7ebe70ebb7ab4f6384789 \
+    --hash=sha256:79291bfb1fe5f21d99f4839f43d3c5d44c5402c830a24dbb2811d785dd21264b \
+    --hash=sha256:7e6bcb194b81e372411494d8ed69fab89aa3452b7275fce4f7917fbe7b04fb72 \
+    --hash=sha256:7f063443870650e55012fdb3a58ff4ce5f4042b81dad6b749333ee8146157511 \
+    --hash=sha256:80d75856f8ec949847386ad2f56a460f21c63bf82ce99ca5b6aa512c0b875fb1 \
+    --hash=sha256:8115b416ea2cad8a87dc3aadfaf26da684e003c3770b12e7219b462505bb5b85 \
+    --hash=sha256:8870ab60f8a76b4a7e43184ee03d28112b976d83c43d41cec821f47b3a297da2 \
+    --hash=sha256:8989d363ac1996238fee61c8f5663f15a8fc362cb1e758c4a686b76cb457cd70 \
+    --hash=sha256:96fe2f7f5805d88cb7f2e3e3502550b2883dfab0f9efcf3cbd444942cf2ee1da \
+    --hash=sha256:9cffff0b4af80285fa49637d69b69d640eb775dc74b23635e4de5faad9e7e744 \
+    --hash=sha256:ac33fd2d02d24101ea389be8e05b928acb58be56403d4ebc3aecfab473fa4a25 \
+    --hash=sha256:accf713f51da74b1a18aa4b31df0ab135510704661f735a938081777b79a4c25 \
+    --hash=sha256:b12bb8c1d408ae40e4c806a3a8ebda2d107310e46696e1da13d0dc3f91fbd19d \
+    --hash=sha256:b309659534b5d930f9ab6d521670c2dd86cb6ef7f47f37f73f96557e2ec13a49 \
+    --hash=sha256:b57f7f01eafbfe3a293f2efffb675774dbe4074c4627975ec4dc4aa5766801fb \
+    --hash=sha256:c43b4fe8c8df4c52d3106bba2cf427f0e46bbebb80e127fbbc3134db0fead7be \
+    --hash=sha256:c62f07452dee3f1ed23aeaef821797c5e516f79535e97fe6a6b0a0ee8db1cc91 \
+    --hash=sha256:cd69107705794e815a8b262722c6fea995911cb1dfc1310abf63b476165335d6 \
+    --hash=sha256:cdbae7312e6d132d38ec2c1611b8cafb783e0416cc5c6deae04efde5f16fb190 \
+    --hash=sha256:d9b8d1c42854d3433c058795f52b1418b53dd8c1e9811fecb1312202e803a2c5 \
+    --hash=sha256:e4cb62a521efbca4cb1ad50233aa400574b3daaf6eb26707d661a0afe8191d92 \
+    --hash=sha256:e59ab6c0bf4a8bb975553ad578d4425bd192775ae384f9406d77d31ad00f6efe \
+    --hash=sha256:f3ab1a9fad636302f7307d143f64a9fbd11bc041652bf53bb016006e9a5ca820 \
+    --hash=sha256:f7302acaa07cf4966c926fcd6a60c8d30a697f730c38168bf83e1519b464115b \
+    --hash=sha256:fa6d9bdd75d3625dae38372b43696e159c10aa98719b4302b1e94f1ff7878d47
     # via -r requirements.in
 idna==3.2 \
     --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \

From 398e0404a04165566d9882f059ccf40cf3d8e203 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 23 Jun 2023 09:18:03 +0100
Subject: [PATCH 631/740] build(deps): bump pygithub from 1.58.2 to 1.59.0 in
 /tools/base (#28109)

Bumps [pygithub](https://github.com/pygithub/pygithub) from 1.58.2 to 1.59.0.
- [Release notes](https://github.com/pygithub/pygithub/releases)
- [Changelog](https://github.com/PyGithub/PyGithub/blob/main/doc/changes.rst)
- [Commits](https://github.com/pygithub/pygithub/compare/v1.58.2...v1.59.0)

---
updated-dependencies:
- dependency-name: pygithub
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tools/base/requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt
index 2274f8d64a05..5a4ddb0456b1 100644
--- a/tools/base/requirements.txt
+++ b/tools/base/requirements.txt
@@ -1056,9 +1056,9 @@ pyflakes==3.0.1 \
     --hash=sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf \
     --hash=sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd
     # via flake8
-pygithub==1.58.2 \
-    --hash=sha256:1e6b1b7afe31f75151fb81f7ab6b984a7188a852bdb123dbb9ae90023c3ce60f \
-    --hash=sha256:f435884af617c6debaa76cbc355372d1027445a56fbc39972a3b9ed4968badc8
+pygithub==1.59.0 \
+    --hash=sha256:126bdbae72087d8d038b113aab6b059b4553cb59348e3024bb1a1cae406ace9e \
+    --hash=sha256:6e05ff49bac3caa7d1d6177a10c6e55a3e20c85b92424cc198571fd0cf786690
     # via -r requirements.in
 pygments==2.15.1 \
     --hash=sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c \

From ba733237b3a84f417035eb66f516558be8148c70 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 23 Jun 2023 09:18:15 +0100
Subject: [PATCH 632/740] build(deps): bump pygithub from 1.58.2 to 1.59.0 in
 /.github/actions/pr_notifier (#28108)

build(deps): bump pygithub in /.github/actions/pr_notifier

Bumps [pygithub](https://github.com/pygithub/pygithub) from 1.58.2 to 1.59.0.
- [Release notes](https://github.com/pygithub/pygithub/releases)
- [Changelog](https://github.com/PyGithub/PyGithub/blob/main/doc/changes.rst)
- [Commits](https://github.com/pygithub/pygithub/compare/v1.58.2...v1.59.0)

---
updated-dependencies:
- dependency-name: pygithub
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/actions/pr_notifier/requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/actions/pr_notifier/requirements.txt b/.github/actions/pr_notifier/requirements.txt
index 0ffc7c83a529..8315df03ec0a 100644
--- a/.github/actions/pr_notifier/requirements.txt
+++ b/.github/actions/pr_notifier/requirements.txt
@@ -171,9 +171,9 @@ pycparser==2.20 \
     --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \
     --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705
     # via cffi
-pygithub==1.58.2 \
-    --hash=sha256:1e6b1b7afe31f75151fb81f7ab6b984a7188a852bdb123dbb9ae90023c3ce60f \
-    --hash=sha256:f435884af617c6debaa76cbc355372d1027445a56fbc39972a3b9ed4968badc8
+pygithub==1.59.0 \
+    --hash=sha256:126bdbae72087d8d038b113aab6b059b4553cb59348e3024bb1a1cae406ace9e \
+    --hash=sha256:6e05ff49bac3caa7d1d6177a10c6e55a3e20c85b92424cc198571fd0cf786690
     # via -r requirements.in
 pyjwt[crypto]==2.4.0 \
     --hash=sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf \

From 86d9d07b5dfdfa100e218734bf4da4169d7e0af3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 23 Jun 2023 09:24:20 +0100
Subject: [PATCH 633/740] build(deps): bump node from `a70c22c` to `c92280d` in
 /examples/shared/node (#28110)

build(deps): bump node in /examples/shared/node

Bumps node from `a70c22c` to `c92280d`.

---
updated-dependencies:
- dependency-name: node
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/node/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile
index a6a5c5358e03..8f530552c0f1 100644
--- a/examples/shared/node/Dockerfile
+++ b/examples/shared/node/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:20.3-bullseye-slim@sha256:a70c22cb6ef7c6d809970b2889e5e556337fda8bfaa439b30c035efaef8fc3a1 as node-base
+FROM node:20.3-bullseye-slim@sha256:c92280d8fb6e7ca07f258c45e9f18cb643ea798a5441855a05e982cfd2b90789 as node-base
 
 
 FROM node-base as node-http-auth

From aa408dccd6a7b76cd0f7101162b5ac20bb7bbd3e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 23 Jun 2023 09:24:39 +0100
Subject: [PATCH 634/740] build(deps): bump golang from `4875242` to `4c94dfe`
 in /examples/shared/golang (#28111)

build(deps): bump golang in /examples/shared/golang

Bumps golang from `4875242` to `4c94dfe`.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/golang/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile
index 55a59f6cae82..7a6b7f2f35ab 100644
--- a/examples/shared/golang/Dockerfile
+++ b/examples/shared/golang/Dockerfile
@@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache
 
 
-FROM golang:1.20.5-bullseye@sha256:48752423122c351658c15f2575e05cd82a7c629e6182635db8fdc74e99370b1d as golang-base
+FROM golang:1.20.5-bullseye@sha256:4c94dfe9567ee66a0ebc35a28f9f939e9ca3a71cb2ab942359887c584da1b28e as golang-base
 
 
 FROM golang-base as golang-control-plane-builder

From 8df16ca90db93379670a2f15b77245ada1180c8a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 23 Jun 2023 09:24:58 +0100
Subject: [PATCH 635/740] build(deps): bump jaegertracing/all-in-one from
 `9620b1b` to `aa450ca` in /examples/shared/jaeger (#28112)

build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger

Bumps jaegertracing/all-in-one from `9620b1b` to `aa450ca`.

---
updated-dependencies:
- dependency-name: jaegertracing/all-in-one
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/jaeger/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile
index 7c1bf0dbf411..8d998fb780a8 100644
--- a/examples/shared/jaeger/Dockerfile
+++ b/examples/shared/jaeger/Dockerfile
@@ -1,4 +1,4 @@
-FROM jaegertracing/all-in-one@sha256:9620b1b576e90706a9906d6c5a463fb7c2c59c458c43eeaa2338f09444d4e4e7
+FROM jaegertracing/all-in-one@sha256:aa450ca8ac0afd03baf23cdf973dafc0d766c2f82e61749e00399442daf5f58c
 HEALTHCHECK \
     --interval=1s \
     --timeout=1s \

From e95b889597e536c6cbfcebea78b9b3a96584e077 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Fri, 23 Jun 2023 14:12:34 +0100
Subject: [PATCH 636/740] docker/ci: Fix debug images (#28116)

Signed-off-by: Ryan Northey 
---
 ci/Dockerfile-envoy                 | 5 ++++-
 ci/docker_ci.sh                     | 2 +-
 ci/test/docker/linux/dev/main       | 4 ++--
 ci/test/docker/linux/dev/other      | 4 ++--
 ci/test/docker/linux/dev/release    | 4 ++--
 ci/test/docker/linux/dev/tag        | 4 ++--
 ci/test/docker/linux/nondev/main    | 4 ++--
 ci/test/docker/linux/nondev/other   | 4 ++--
 ci/test/docker/linux/nondev/release | 4 ++--
 ci/test/docker/linux/nondev/tag     | 4 ++--
 10 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/ci/Dockerfile-envoy b/ci/Dockerfile-envoy
index 973278f82fa2..9ea1d1a06cce 100644
--- a/ci/Dockerfile-envoy
+++ b/ci/Dockerfile-envoy
@@ -41,8 +41,11 @@ COPY --from=binary --chown=0:0 --chmod=755 \
 COPY --from=binary --chown=0:0 --chmod=755 \
     /usr/local/bin/utils/su-exec /usr/local/bin/
 ARG ENVOY_BINARY=envoy
+ARG ENVOY_BINARY_PREFIX=
 COPY --from=binary --chown=0:0 --chmod=755 \
-    "/usr/local/bin/${ENVOY_BINARY}" /usr/local/bin/envoy
+    "/usr/local/bin/${ENVOY_BINARY_PREFIX}${ENVOY_BINARY}" /usr/local/bin/envoy
+COPY --from=binary --chown=0:0 --chmod=755 \
+    /usr/local/bin/${ENVOY_BINARY_PREFIX}${ENVOY_BINARY}\.* /usr/local/bin/
 
 
 # STAGE: envoy-tools
diff --git a/ci/docker_ci.sh b/ci/docker_ci.sh
index 136676cf0955..3845486acf07 100755
--- a/ci/docker_ci.sh
+++ b/ci/docker_ci.sh
@@ -172,7 +172,7 @@ build_args() {
     fi
 
     if [[ "${build_type}" == *-debug ]]; then
-        printf ' --build-arg ENVOY_BINARY_SUFFIX='
+        printf ' --build-arg ENVOY_BINARY_PREFIX=dbg/'
     fi
 }
 
diff --git a/ci/test/docker/linux/dev/main b/ci/test/docker/linux/dev/main
index 96169beb06c0..011ce8eca498 100644
--- a/ci/test/docker/linux/dev/main
+++ b/ci/test/docker/linux/dev/main
@@ -18,7 +18,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD+PUSH: mocktest/repo-debug-dev:MOCKSHA
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false --push -t mocktest/repo-debug-dev:MOCKSHA .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false --push -t mocktest/repo-debug-dev:MOCKSHA .
 >> TAG: mocktest/repo-debug-dev:MOCKSHA -> mocktest/repo:debug-dev-MOCKSHA
 > docker buildx imagetools create docker.io/mocktest/repo-debug-dev:MOCKSHA --tag docker.io/mocktest/repo:debug-dev-MOCKSHA
 >> TAG: mocktest/repo-debug-dev:MOCKSHA -> mocktest/repo-debug-dev:latest
@@ -40,7 +40,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD+PUSH: mocktest/repo-contrib-debug-dev:MOCKSHA
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false --push -t mocktest/repo-contrib-debug-dev:MOCKSHA .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false --push -t mocktest/repo-contrib-debug-dev:MOCKSHA .
 >> TAG: mocktest/repo-contrib-debug-dev:MOCKSHA -> mocktest/repo:contrib-debug-dev-MOCKSHA
 > docker buildx imagetools create docker.io/mocktest/repo-contrib-debug-dev:MOCKSHA --tag docker.io/mocktest/repo:contrib-debug-dev-MOCKSHA
 >> TAG: mocktest/repo-contrib-debug-dev:MOCKSHA -> mocktest/repo-contrib-debug-dev:latest
diff --git a/ci/test/docker/linux/dev/other b/ci/test/docker/linux/dev/other
index 05ab440e6df9..c5b2f33b032a 100644
--- a/ci/test/docker/linux/dev/other
+++ b/ci/test/docker/linux/dev/other
@@ -9,7 +9,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-debug-dev:MOCKSHA
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-debug-dev:MOCKSHA .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-debug-dev:MOCKSHA .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib-dev:MOCKSHA
@@ -17,7 +17,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib-debug-dev:MOCKSHA
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-contrib-debug-dev:MOCKSHA .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-contrib-debug-dev:MOCKSHA .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-distroless-dev:MOCKSHA
diff --git a/ci/test/docker/linux/dev/release b/ci/test/docker/linux/dev/release
index 05ab440e6df9..c5b2f33b032a 100644
--- a/ci/test/docker/linux/dev/release
+++ b/ci/test/docker/linux/dev/release
@@ -9,7 +9,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-debug-dev:MOCKSHA
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-debug-dev:MOCKSHA .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-debug-dev:MOCKSHA .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib-dev:MOCKSHA
@@ -17,7 +17,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib-debug-dev:MOCKSHA
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-contrib-debug-dev:MOCKSHA .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-contrib-debug-dev:MOCKSHA .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-distroless-dev:MOCKSHA
diff --git a/ci/test/docker/linux/dev/tag b/ci/test/docker/linux/dev/tag
index 05ab440e6df9..c5b2f33b032a 100644
--- a/ci/test/docker/linux/dev/tag
+++ b/ci/test/docker/linux/dev/tag
@@ -9,7 +9,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-debug-dev:MOCKSHA
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-debug-dev:MOCKSHA .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-debug-dev:MOCKSHA .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib-dev:MOCKSHA
@@ -17,7 +17,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib-debug-dev:MOCKSHA
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-contrib-debug-dev:MOCKSHA .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-contrib-debug-dev:MOCKSHA .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-distroless-dev:MOCKSHA
diff --git a/ci/test/docker/linux/nondev/main b/ci/test/docker/linux/nondev/main
index 8d4a6d685491..84e08062a9ad 100644
--- a/ci/test/docker/linux/nondev/main
+++ b/ci/test/docker/linux/nondev/main
@@ -14,7 +14,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD+PUSH: mocktest/repo-debug:v1.73.0
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false --push -t mocktest/repo-debug:v1.73.0 .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false --push -t mocktest/repo-debug:v1.73.0 .
 >> TAG: mocktest/repo-debug:v1.73.0 -> mocktest/repo:debug-v1.73.0
 > docker buildx imagetools create docker.io/mocktest/repo-debug:v1.73.0 --tag docker.io/mocktest/repo:debug-v1.73.0
 >> TAG: mocktest/repo-debug:v1.73.0 -> mocktest/repo-debug:v1.73-latest
@@ -36,7 +36,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD+PUSH: mocktest/repo-contrib-debug:v1.73.0
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false --push -t mocktest/repo-contrib-debug:v1.73.0 .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false --push -t mocktest/repo-contrib-debug:v1.73.0 .
 >> TAG: mocktest/repo-contrib-debug:v1.73.0 -> mocktest/repo:contrib-debug-v1.73.0
 > docker buildx imagetools create docker.io/mocktest/repo-contrib-debug:v1.73.0 --tag docker.io/mocktest/repo:contrib-debug-v1.73.0
 >> TAG: mocktest/repo-contrib-debug:v1.73.0 -> mocktest/repo-contrib-debug:v1.73-latest
diff --git a/ci/test/docker/linux/nondev/other b/ci/test/docker/linux/nondev/other
index 9ff61285bf21..71a775ce4671 100644
--- a/ci/test/docker/linux/nondev/other
+++ b/ci/test/docker/linux/nondev/other
@@ -9,7 +9,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-debug:v1.73.3
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-debug:v1.73.3 .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-debug:v1.73.3 .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib:v1.73.3
@@ -17,7 +17,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib-debug:v1.73.3
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-contrib-debug:v1.73.3 .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-contrib-debug:v1.73.3 .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-distroless:v1.73.3
diff --git a/ci/test/docker/linux/nondev/release b/ci/test/docker/linux/nondev/release
index ba48dce78418..b3482396fbfe 100644
--- a/ci/test/docker/linux/nondev/release
+++ b/ci/test/docker/linux/nondev/release
@@ -14,7 +14,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD+PUSH: mocktest/repo-debug:v1.73.3
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false --push -t mocktest/repo-debug:v1.73.3 .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false --push -t mocktest/repo-debug:v1.73.3 .
 >> TAG: mocktest/repo-debug:v1.73.3 -> mocktest/repo:debug-v1.73.3
 > docker buildx imagetools create docker.io/mocktest/repo-debug:v1.73.3 --tag docker.io/mocktest/repo:debug-v1.73.3
 >> TAG: mocktest/repo-debug:v1.73.3 -> mocktest/repo-debug:v1.73-latest
@@ -36,7 +36,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD+PUSH: mocktest/repo-contrib-debug:v1.73.3
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false --push -t mocktest/repo-contrib-debug:v1.73.3 .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false --push -t mocktest/repo-contrib-debug:v1.73.3 .
 >> TAG: mocktest/repo-contrib-debug:v1.73.3 -> mocktest/repo:contrib-debug-v1.73.3
 > docker buildx imagetools create docker.io/mocktest/repo-contrib-debug:v1.73.3 --tag docker.io/mocktest/repo:contrib-debug-v1.73.3
 >> TAG: mocktest/repo-contrib-debug:v1.73.3 -> mocktest/repo-contrib-debug:v1.73-latest
diff --git a/ci/test/docker/linux/nondev/tag b/ci/test/docker/linux/nondev/tag
index 9ff61285bf21..71a775ce4671 100644
--- a/ci/test/docker/linux/nondev/tag
+++ b/ci/test/docker/linux/nondev/tag
@@ -9,7 +9,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-debug:v1.73.3
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-debug:v1.73.3 .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-debug:v1.73.3 .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib:v1.73.3
@@ -17,7 +17,7 @@
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-contrib-debug:v1.73.3
-> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_SUFFIX= --sbom=false --provenance=false -t mocktest/repo-contrib-debug:v1.73.3 .
+> docker buildx build --platform linux/arm64,linux/amd64 -f ci/Dockerfile-envoy --target envoy --build-arg ENVOY_BINARY=envoy-contrib --build-arg ENVOY_BINARY_PREFIX=dbg/ --sbom=false --provenance=false -t mocktest/repo-contrib-debug:v1.73.3 .
 >> BUILDX: use multi-builder
 > docker buildx use multi-builder
 >> BUILD: mocktest/repo-distroless:v1.73.3

From 2ec61d05bd5c8e98f072e98954d998726556f503 Mon Sep 17 00:00:00 2001
From: Kuat 
Date: Fri, 23 Jun 2023 07:07:12 -0700
Subject: [PATCH 637/740] original_dst: fix timer lifetime (#28102)

* original_dst: fix timer lifetime

Signed-off-by: Kuat Yessenov 
---
 .../clusters/original_dst/original_dst_cluster.h       |  2 ++
 .../clusters/original_dst/original_dst_cluster_test.cc | 10 ++++++++++
 2 files changed, 12 insertions(+)

diff --git a/source/extensions/clusters/original_dst/original_dst_cluster.h b/source/extensions/clusters/original_dst/original_dst_cluster.h
index 47ab51199759..e84da53e78fd 100644
--- a/source/extensions/clusters/original_dst/original_dst_cluster.h
+++ b/source/extensions/clusters/original_dst/original_dst_cluster.h
@@ -49,6 +49,8 @@ class OriginalDstCluster : public ClusterImplBase {
   OriginalDstCluster(const envoy::config::cluster::v3::Cluster& config,
                      ClusterFactoryContext& context);
 
+  ~OriginalDstCluster() override { cleanup_timer_->disableTimer(); }
+
   // Upstream::Cluster
   InitializePhase initializePhase() const override { return InitializePhase::Primary; }
 
diff --git a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc
index f94cec7385da..af336ec5b48b 100644
--- a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc
+++ b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc
@@ -79,9 +79,18 @@ class OriginalDstClusterTest : public Event::TestUsingSimulatedTime, public test
         [&](uint32_t, const HostVector&, const HostVector&) -> void {
           membership_updated_.ready();
         });
+    ON_CALL(initialized_, ready()).WillByDefault(testing::Invoke([this] {
+      init_complete_ = true;
+    }));
     cluster_->initialize([&]() -> void { initialized_.ready(); });
   }
 
+  void TearDown() override {
+    if (init_complete_) {
+      EXPECT_CALL(*cleanup_timer_, disableTimer());
+    }
+  }
+
   NiceMock server_context_;
   Stats::TestUtil::TestStore& stats_store_ = server_context_.store_;
   Ssl::MockContextManager ssl_context_manager_;
@@ -91,6 +100,7 @@ class OriginalDstClusterTest : public Event::TestUsingSimulatedTime, public test
   ReadyWatcher initialized_;
   Event::MockTimer* cleanup_timer_;
   Common::CallbackHandlePtr priority_update_cb_;
+  bool init_complete_{false};
 };
 
 TEST(OriginalDstClusterConfigTest, GoodConfig) {

From 1f405f13a131f5157da0d94446c9695a06eaf046 Mon Sep 17 00:00:00 2001
From: Ali Beyad 
Date: Fri, 23 Jun 2023 10:45:31 -0400
Subject: [PATCH 638/740] ci: Add test targets to the mobile compile options
 workflow (#28083)

All C++ test targets are added to the cc_test job in the workflow.

This can help us catch test build failures when the standard build
options are disabled as they would often be in a release build.

This update caught a bug in the sds_integration_test.cc, which
wasn't guarded with envoy_select_google_grpc in the BUILD
file, so that problem has been fixed in this commit as well.

Signed-off-by: Ali Beyad 
---
 .../workflows/mobile-compile_time_options.yml | 46 +++++++++++++++----
 mobile/test/common/integration/BUILD          |  8 ++--
 2 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml
index 497f60e5e106..5ef97418bfdc 100644
--- a/.github/workflows/mobile-compile_time_options.yml
+++ b/.github/workflows/mobile-compile_time_options.yml
@@ -16,10 +16,10 @@ jobs:
     uses: ./.github/workflows/env.yml
     secrets: inherit
 
-  cc_test:
+  cc_test_no_yaml:
     needs: env
-    name: cc_test
-    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
+    name: cc_test_no_yaml
+    runs-on: ubuntu-20.04
     timeout-minutes: 120
     container:
       image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33
@@ -27,17 +27,44 @@ jobs:
     - uses: actions/checkout@v3
     - name: Add safe directory
       run: git config --global --add safe.directory /__w/envoy/envoy
-    - name: 'Building C++ library'
+    - name: 'Running C++ test with YAML disabled'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Envoy Mobile build which verifies that the build configuration where YAML is disabled.
       run: |
-        cd mobile && ./bazelw test \
+        cd mobile
+        ./bazelw test \
             $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-linux") \
             --config=ci \
             --define=envoy_yaml=disabled \
             --test_env=ENVOY_IP_TEST_VERSIONS=v4only \
             //test/common/integration:client_integration_test --test_output=all
+  cc_test:
+    needs: env
+    name: cc_test
+    runs-on: ${{ needs.env.outputs.agent_ubuntu }}
+    timeout-minutes: 120
+    container:
+      image: envoyproxy/envoy-build-ubuntu:41c5a05d708972d703661b702a63ef5060125c33
+    steps:
+    - uses: actions/checkout@v3
+    - name: Add safe directory
+      run: git config --global --add safe.directory /__w/envoy/envoy
+    - name: 'Running C++ tests'
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        cd mobile
+        ./bazelw test \
+            --test_output=all \
+            $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-linux") \
+            --config=ci \
+            --define=signal_trace=disabled \
+            --define=envoy_mobile_request_compression=disabled \
+            --define=envoy_enable_http_datagrams=disabled \
+            --define=google_grpc=disabled \
+            --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \
+            $(bazel query //test/cc/... + //test/common/... except //test/common/integration:client_integration_test)
   swift_build:
     if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }}
     needs: env
@@ -52,8 +79,9 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        cd mobile && ./bazelw shutdown
-          ./bazelw build \
+        cd mobile
+        ./bazelw shutdown
+        ./bazelw build \
             --config=ios \
             $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \
             --define=signal_trace=disabled \
@@ -85,8 +113,8 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        cd mobile \
-        && ./bazelw build \
+        cd mobile
+        ./bazelw build \
             $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \
             --fat_apk_cpu=x86_64 \
             --define=signal_trace=disabled \
diff --git a/mobile/test/common/integration/BUILD b/mobile/test/common/integration/BUILD
index 049a6b9548b1..474c845b080e 100644
--- a/mobile/test/common/integration/BUILD
+++ b/mobile/test/common/integration/BUILD
@@ -73,9 +73,11 @@ envoy_cc_test(
 
 envoy_cc_test(
     name = "sds_integration_test",
-    srcs = [
-        "sds_integration_test.cc",
-    ],
+    # The test relies on the Google gRPC library.
+    srcs = envoy_select_google_grpc(
+        ["sds_integration_test.cc"],
+        "@envoy",
+    ),
     data = [
         "@envoy//test/config/integration/certs",
     ],

From 2209e12cd5b747df333344553642b1374ea009f0 Mon Sep 17 00:00:00 2001
From: Ali Beyad 
Date: Fri, 23 Jun 2023 11:26:03 -0400
Subject: [PATCH 639/740] ci: Increase timeout in Swift integration tests
 (#28121)

Some work was done previously in
https://github.com/envoyproxy/envoy/pull/24892 to increase the Swift
integration test timeout to 10 seconds to match the Kotlin test
counterparts and reduce test flakiness.

However, there are still some places where the timeout was low and it
caused flakey tests, see
https://github.com/envoyproxy/envoy/issues/28115.

This change increases the timeout to 10 seconds in the other Swift tests
as well.

Signed-off-by: Ali Beyad 
---
 .../test/swift/integration/ResetConnectivityStateTest.swift   | 2 +-
 mobile/test/swift/integration/SetLoggerTest.swift             | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/mobile/test/swift/integration/ResetConnectivityStateTest.swift b/mobile/test/swift/integration/ResetConnectivityStateTest.swift
index 8cd5a4f6d807..41bc8977e6cb 100644
--- a/mobile/test/swift/integration/ResetConnectivityStateTest.swift
+++ b/mobile/test/swift/integration/ResetConnectivityStateTest.swift
@@ -84,7 +84,7 @@ static_resources:
       .start()
       .sendHeaders(requestHeaders, endStream: true)
 
-    XCTAssertEqual(XCTWaiter.wait(for: [expectation1], timeout: 1), .completed)
+    XCTAssertEqual(XCTWaiter.wait(for: [expectation1], timeout: 10), .completed)
 
     engine.resetConnectivityState()
 
diff --git a/mobile/test/swift/integration/SetLoggerTest.swift b/mobile/test/swift/integration/SetLoggerTest.swift
index c94d3fc643b2..0c6fa988c6e0 100644
--- a/mobile/test/swift/integration/SetLoggerTest.swift
+++ b/mobile/test/swift/integration/SetLoggerTest.swift
@@ -86,8 +86,8 @@ static_resources:
       }
       .build()
 
-    XCTAssertEqual(XCTWaiter.wait(for: [engineExpectation], timeout: 1), .completed)
-    XCTAssertEqual(XCTWaiter.wait(for: [loggingExpectation], timeout: 1), .completed)
+    XCTAssertEqual(XCTWaiter.wait(for: [engineExpectation], timeout: 10), .completed)
+    XCTAssertEqual(XCTWaiter.wait(for: [loggingExpectation], timeout: 10), .completed)
 
     // Send a request to trigger the test filter which should log an event.
     let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "https",

From b64e1f6d1e2f1d868d77e0d0f6f517ea0101c312 Mon Sep 17 00:00:00 2001
From: Kuat 
Date: Fri, 23 Jun 2023 13:27:14 -0700
Subject: [PATCH 640/740] docs: add changelog to #28102 (#28125)

Signed-off-by: Kuat Yessenov 
---
 changelogs/current.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 7afc3e73b97d..222607d5385f 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -182,6 +182,10 @@ bug_fixes:
     Fixes a bug where route properties such as key_formatter,
     prefix and remove_prefix do not take effect when configured for :ref:`catch_all_route
     `.
+- area: original_dst
+  change: |
+    Fixes an issue with the ORIGINAL_DST cluster cleanup timer lifetime, which
+    can occur if the cluster is removed while the timer is armed.
 
 removed_config_or_runtime:
 # *Normally occurs at the end of the* :ref:`deprecation period `

From 63d4fb0c442d6362bcf5101fc2a5279aeaef31bb Mon Sep 17 00:00:00 2001
From: antJack <52443884+antJack@users.noreply.github.com>
Date: Mon, 26 Jun 2023 11:54:42 +0800
Subject: [PATCH 641/740] golang: L4 network extension (#26173)

Signed-off-by: yongjie.yyj 
---
 .github/dependabot.yml                        |   6 +
 api/BUILD                                     |   1 +
 .../filters/http/golang/v3alpha/golang.proto  |   8 +-
 .../filters/network/golang/v3alpha/BUILD      |  12 +
 .../network/golang/v3alpha/golang.proto       |  54 ++++
 api/versioning/BUILD                          |   1 +
 changelogs/current.yaml                       |   3 +
 contrib/contrib_build_config.bzl              |   1 +
 contrib/extensions_metadata.yaml              |   5 +
 contrib/golang/common/dso/api.h               |   2 +-
 contrib/golang/common/dso/dso.cc              |  85 ++++++
 contrib/golang/common/dso/dso.h               |  87 ++++++
 contrib/golang/common/dso/libgolang.h         |  66 +++++
 contrib/golang/common/dso/test/dso_test.cc    |  67 +++--
 contrib/golang/common/dso/test/mocks.h        |  23 ++
 .../common/dso/test/test_data/simple.go       |  38 +++
 .../source/go/pkg => common/go}/api/BUILD     |   3 +-
 .../source/go/pkg => common/go}/api/api.h     |  14 +
 .../go/pkg/http => common/go/api}/capi.go     |  42 ++-
 .../go/pkg => common/go}/api/cgocheck.go      |   0
 .../source/go/pkg => common/go}/api/filter.go |  33 +++
 .../source/go/pkg => common/go}/api/type.go   | 118 ++++++++
 .../filters/http/source/go/pkg/http/BUILD     |   3 +-
 .../filters/http/source/go/pkg/http/api.h     |   2 +-
 .../http/source/go/pkg/http/capi_impl.go      |  13 +-
 .../filters/http/source/go/pkg/http/config.go |   2 +-
 .../filters/http/source/go/pkg/http/filter.go |  18 +-
 .../http/source/go/pkg/http/filtermanager.go  |   2 +-
 .../http/source/go/pkg/http/passthrough.go    |   2 +-
 .../filters/http/source/go/pkg/http/shim.go   |   2 +-
 .../filters/http/source/go/pkg/http/type.go   |   2 +-
 .../filters/http/test/test_data/basic/BUILD   |   2 +-
 .../http/test/test_data/basic/config.go       |   2 +-
 .../http/test/test_data/basic/filter.go       |   2 +-
 .../filters/http/test/test_data/echo/BUILD    |   2 +-
 .../http/test/test_data/echo/config.go        |   5 +-
 .../http/test/test_data/echo/filter.go        |   2 +-
 .../http/test/test_data/passthrough/BUILD     |   2 +-
 .../http/test/test_data/routeconfig/BUILD     |   2 +-
 .../http/test/test_data/routeconfig/config.go |   5 +-
 .../http/test/test_data/routeconfig/filter.go |   2 +-
 contrib/golang/filters/network/source/BUILD   | 105 +++++++
 contrib/golang/filters/network/source/cgo.cc  | 176 ++++++++++++
 .../golang/filters/network/source/config.cc   |  47 +++
 .../golang/filters/network/source/config.h    |  48 ++++
 .../network/source/go/pkg/network/BUILD       |  38 +++
 .../network/source/go/pkg/network/api.h       |   1 +
 .../network/source/go/pkg/network/capi.go     |  89 ++++++
 .../network/source/go/pkg/network/factory.go  |  76 +++++
 .../network/source/go/pkg/network/filter.go   |  53 ++++
 .../network/source/go/pkg/network/shim.go     | 268 ++++++++++++++++++
 .../golang/filters/network/source/golang.cc   | 121 ++++++++
 .../golang/filters/network/source/golang.h    | 118 ++++++++
 .../golang/filters/network/source/upstream.cc | 176 ++++++++++++
 .../golang/filters/network/source/upstream.h  | 128 +++++++++
 contrib/golang/filters/network/test/BUILD     |  58 ++++
 .../filters/network/test/config_test.cc       | 115 ++++++++
 .../filters/network/test/filter_test.cc       | 112 ++++++++
 .../filters/network/test/test_data/BUILD      |  21 ++
 .../filters/network/test/test_data/filter.go  |  45 +++
 .../filters/network/test/test_data/go.mod     |   9 +
 .../filters/network/test/upstream_test.cc     | 126 ++++++++
 docs/root/_configs/go/network.yaml            |  29 ++
 .../http/http_filters/golang_filter.rst       |   4 +-
 .../network_filters/golang_filter.rst         |  63 ++++
 .../network_filters/network_filters.rst       |   9 +-
 examples/golang/simple/config.go              |   5 +-
 examples/golang/simple/filter.go              |   2 +-
 source/common/common/logger.h                 |   3 +-
 69 files changed, 2702 insertions(+), 84 deletions(-)
 create mode 100644 api/contrib/envoy/extensions/filters/network/golang/v3alpha/BUILD
 create mode 100644 api/contrib/envoy/extensions/filters/network/golang/v3alpha/golang.proto
 rename contrib/golang/{filters/http/source/go/pkg => common/go}/api/BUILD (77%)
 rename contrib/golang/{filters/http/source/go/pkg => common/go}/api/api.h (78%)
 rename contrib/golang/{filters/http/source/go/pkg/http => common/go/api}/capi.go (58%)
 rename contrib/golang/{filters/http/source/go/pkg => common/go}/api/cgocheck.go (100%)
 rename contrib/golang/{filters/http/source/go/pkg => common/go}/api/filter.go (77%)
 rename contrib/golang/{filters/http/source/go/pkg => common/go}/api/type.go (74%)
 create mode 100644 contrib/golang/filters/network/source/BUILD
 create mode 100644 contrib/golang/filters/network/source/cgo.cc
 create mode 100644 contrib/golang/filters/network/source/config.cc
 create mode 100644 contrib/golang/filters/network/source/config.h
 create mode 100644 contrib/golang/filters/network/source/go/pkg/network/BUILD
 create mode 120000 contrib/golang/filters/network/source/go/pkg/network/api.h
 create mode 100644 contrib/golang/filters/network/source/go/pkg/network/capi.go
 create mode 100644 contrib/golang/filters/network/source/go/pkg/network/factory.go
 create mode 100644 contrib/golang/filters/network/source/go/pkg/network/filter.go
 create mode 100644 contrib/golang/filters/network/source/go/pkg/network/shim.go
 create mode 100644 contrib/golang/filters/network/source/golang.cc
 create mode 100644 contrib/golang/filters/network/source/golang.h
 create mode 100644 contrib/golang/filters/network/source/upstream.cc
 create mode 100644 contrib/golang/filters/network/source/upstream.h
 create mode 100644 contrib/golang/filters/network/test/BUILD
 create mode 100644 contrib/golang/filters/network/test/config_test.cc
 create mode 100644 contrib/golang/filters/network/test/filter_test.cc
 create mode 100644 contrib/golang/filters/network/test/test_data/BUILD
 create mode 100644 contrib/golang/filters/network/test/test_data/filter.go
 create mode 100644 contrib/golang/filters/network/test/test_data/go.mod
 create mode 100644 contrib/golang/filters/network/test/upstream_test.cc
 create mode 100644 docs/root/_configs/go/network.yaml
 create mode 100644 docs/root/configuration/listeners/network_filters/golang_filter.rst

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index fa90aaa0a12e..86c794a72e67 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -238,6 +238,12 @@ updates:
     interval: daily
     time: "06:00"
 
+- package-ecosystem: "gomod"
+  directory: "/contrib/golang/filters/network/test/test_data"
+  schedule:
+    interval: daily
+    time: "06:00"
+
 - package-ecosystem: "gomod"
   directory: "/examples/ext_authz/auth/grpc-service"
   schedule:
diff --git a/api/BUILD b/api/BUILD
index 915c7bce63f7..d384bf90b044 100644
--- a/api/BUILD
+++ b/api/BUILD
@@ -83,6 +83,7 @@ proto_library(
         "//contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3:pkg",
         "//contrib/envoy/extensions/filters/network/generic_proxy/router/v3:pkg",
         "//contrib/envoy/extensions/filters/network/generic_proxy/v3:pkg",
+        "//contrib/envoy/extensions/filters/network/golang/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/network/kafka_broker/v3:pkg",
         "//contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/network/mysql_proxy/v3:pkg",
diff --git a/api/contrib/envoy/extensions/filters/http/golang/v3alpha/golang.proto b/api/contrib/envoy/extensions/filters/http/golang/v3alpha/golang.proto
index 01aa4cecbd1e..ee1268a25683 100644
--- a/api/contrib/envoy/extensions/filters/http/golang/v3alpha/golang.proto
+++ b/api/contrib/envoy/extensions/filters/http/golang/v3alpha/golang.proto
@@ -16,9 +16,9 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/fil
 option (udpa.annotations.file_status).package_version_status = ACTIVE;
 option (xds.annotations.v3.file_status).work_in_progress = true;
 
-// [#protodoc-title: Golang]
+// [#protodoc-title: Golang HTTP filter]
 //
-// For an overview of the Golang filter please see the :ref:`configuration reference documentation `.
+// For an overview of the Golang HTTP filter please see the :ref:`configuration reference documentation `.
 // [#extension: envoy.filters.http.golang]
 
 // [#next-free-field: 6]
@@ -40,7 +40,7 @@ message Config {
   string library_id = 1 [(validate.rules).string = {min_len: 1}];
 
   // Path to a dynamic library implementing the
-  // :repo:`StreamFilter API `
+  // :repo:`StreamFilter API `
   // interface.
   // [#comment:TODO(wangfakang): Support for downloading libraries from remote repositories.]
   string library_path = 2 [(validate.rules).string = {min_len: 1}];
@@ -59,7 +59,7 @@ message Config {
   //     This configuration is only parsed in the go plugin, and is therefore not validated
   //     by Envoy.
   //
-  //     See the :repo:`StreamFilter API `
+  //     See the :repo:`StreamFilter API `
   //     for more information about how the plugin's configuration data can be accessed.
   //
   google.protobuf.Any plugin_config = 4;
diff --git a/api/contrib/envoy/extensions/filters/network/golang/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/network/golang/v3alpha/BUILD
new file mode 100644
index 000000000000..ec1e778e06e5
--- /dev/null
+++ b/api/contrib/envoy/extensions/filters/network/golang/v3alpha/BUILD
@@ -0,0 +1,12 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = [
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+        "@com_github_cncf_udpa//xds/annotations/v3:pkg",
+    ],
+)
diff --git a/api/contrib/envoy/extensions/filters/network/golang/v3alpha/golang.proto b/api/contrib/envoy/extensions/filters/network/golang/v3alpha/golang.proto
new file mode 100644
index 000000000000..f1051bc1aff3
--- /dev/null
+++ b/api/contrib/envoy/extensions/filters/network/golang/v3alpha/golang.proto
@@ -0,0 +1,54 @@
+syntax = "proto3";
+
+package envoy.extensions.filters.network.golang.v3alpha;
+
+import "google/protobuf/any.proto";
+
+import "xds/annotations/v3/status.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.filters.network.golang.v3alpha";
+option java_outer_classname = "GolangProto";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/golang/v3alpha";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+option (xds.annotations.v3.file_status).work_in_progress = true;
+
+// [#protodoc-title: Golang network filter]
+//
+// Golang network filter :ref:`configuration overview `.
+// [#extension: envoy.filters.network.golang]
+
+// [#next-free-field: 6]
+message Config {
+  // Bool ``true`` if this filter must be the last filter in a filter chain, ``false`` otherwise.
+  bool is_terminal_filter = 1;
+
+  // Globally unique ID for a dynamic library file.
+  string library_id = 2 [(validate.rules).string = {min_len: 1}];
+
+  // Path to a dynamic library implementing the
+  // :repo:`DownstreamFilter API `
+  // interface.
+  // [#comment:TODO(wangfakang): Support for downloading libraries from remote repositories.]
+  string library_path = 3 [(validate.rules).string = {min_len: 1}];
+
+  // Globally unique name of the Go plugin.
+  //
+  // This name **must** be consistent with the name registered in ``network::RegisterNetworkFilterConfigFactory``
+  //
+  string plugin_name = 4 [(validate.rules).string = {min_len: 1}];
+
+  // Configuration for the Go plugin.
+  //
+  // .. note::
+  //     This configuration is only parsed in the go plugin, and is therefore not validated
+  //     by Envoy.
+  //
+  //     See the :repo:`DownstreamFilter API `
+  //     for more information about how the plugin's configuration data can be accessed.
+  //
+  google.protobuf.Any plugin_config = 5;
+}
diff --git a/api/versioning/BUILD b/api/versioning/BUILD
index 58405a717c4d..250fb3bff4f7 100644
--- a/api/versioning/BUILD
+++ b/api/versioning/BUILD
@@ -21,6 +21,7 @@ proto_library(
         "//contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3:pkg",
         "//contrib/envoy/extensions/filters/network/generic_proxy/router/v3:pkg",
         "//contrib/envoy/extensions/filters/network/generic_proxy/v3:pkg",
+        "//contrib/envoy/extensions/filters/network/golang/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/network/kafka_broker/v3:pkg",
         "//contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/network/mysql_proxy/v3:pkg",
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 222607d5385f..3a4271d184b5 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -224,6 +224,9 @@ removed_config_or_runtime:
     removed runtime key ``envoy.reloadable_features.http_strip_fragment_from_path_unsafe_if_disabled`` and legacy code paths.
 
 new_features:
+- area: golang
+  change: |
+    added new :ref:`l4 golang network filter `.
 - area: access_log
   change: |
     added %ACCESS_LOG_TYPE% substitution string, to help distinguishing between access log records and when they are being
diff --git a/contrib/contrib_build_config.bzl b/contrib/contrib_build_config.bzl
index 51294a6d1805..7ec170da1aa3 100644
--- a/contrib/contrib_build_config.bzl
+++ b/contrib/contrib_build_config.bzl
@@ -21,6 +21,7 @@ CONTRIB_EXTENSIONS = {
     "envoy.filters.network.postgres_proxy":                     "//contrib/postgres_proxy/filters/network/source:config",
     "envoy.filters.network.rocketmq_proxy":                     "//contrib/rocketmq_proxy/filters/network/source:config",
     "envoy.filters.network.generic_proxy":                      "//contrib/generic_proxy/filters/network/source:config",
+    "envoy.filters.network.golang":                             "//contrib/golang/filters/network/source:config",
 
     #
     # Sip proxy
diff --git a/contrib/extensions_metadata.yaml b/contrib/extensions_metadata.yaml
index e81a440122a2..1e6a8f3dc0b3 100644
--- a/contrib/extensions_metadata.yaml
+++ b/contrib/extensions_metadata.yaml
@@ -33,6 +33,11 @@ envoy.filters.network.kafka_mesh:
   - envoy.filters.network
   security_posture: requires_trusted_downstream_and_upstream
   status: wip
+envoy.filters.network.golang:
+  categories:
+  - envoy.filters.network
+  security_posture: requires_trusted_downstream_and_upstream
+  status: alpha
 envoy.filters.network.rocketmq_proxy:
   categories:
   - envoy.filters.network
diff --git a/contrib/golang/common/dso/api.h b/contrib/golang/common/dso/api.h
index a1fc898675b8..5d14bbd40fd4 120000
--- a/contrib/golang/common/dso/api.h
+++ b/contrib/golang/common/dso/api.h
@@ -1 +1 @@
-../../filters/http/source/go/pkg/api/api.h
\ No newline at end of file
+../go/api/api.h
\ No newline at end of file
diff --git a/contrib/golang/common/dso/dso.cc b/contrib/golang/common/dso/dso.cc
index 18b8018af197..66c1f375bf42 100644
--- a/contrib/golang/common/dso/dso.cc
+++ b/contrib/golang/common/dso/dso.cc
@@ -124,5 +124,90 @@ GoInt64 ClusterSpecifierDsoImpl::envoyGoOnClusterSpecify(GoUint64 plugin_ptr, Go
   return envoy_go_on_cluster_specify_(plugin_ptr, header_ptr, plugin_id, buffer_ptr, buffer_len);
 }
 
+NetworkFilterDsoImpl::NetworkFilterDsoImpl(const std::string dso_name)
+    : NetworkFilterDso(dso_name) {
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_on_network_filter_config_, handler_, dso_name,
+      "envoyGoFilterOnNetworkFilterConfig");
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_on_downstream_connection_, handler_, dso_name,
+      "envoyGoFilterOnDownstreamConnection");
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_on_downstream_data_, handler_, dso_name, "envoyGoFilterOnDownstreamData");
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_on_downstream_event_, handler_, dso_name, "envoyGoFilterOnDownstreamEvent");
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_on_downstream_write_, handler_, dso_name, "envoyGoFilterOnDownstreamWrite");
+
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_on_upstream_connection_ready_, handler_, dso_name,
+      "envoyGoFilterOnUpstreamConnectionReady");
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_on_upstream_connection_failure_, handler_, dso_name,
+      "envoyGoFilterOnUpstreamConnectionFailure");
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_on_upstream_data_, handler_, dso_name, "envoyGoFilterOnUpstreamData");
+  loaded_ &= dlsymInternal(
+      envoy_go_filter_on_upstream_event_, handler_, dso_name, "envoyGoFilterOnUpstreamEvent");
+}
+
+GoUint64 NetworkFilterDsoImpl::envoyGoFilterOnNetworkFilterConfig(GoUint64 library_id_ptr,
+                                                                  GoUint64 library_id_len,
+                                                                  GoUint64 config_ptr,
+                                                                  GoUint64 config_len) {
+  ASSERT(envoy_go_filter_on_network_filter_config_ != nullptr);
+  return envoy_go_filter_on_network_filter_config_(library_id_ptr, library_id_len, config_ptr,
+                                                   config_len);
+}
+
+GoUint64 NetworkFilterDsoImpl::envoyGoFilterOnDownstreamConnection(void* w,
+                                                                   GoUint64 plugin_name_ptr,
+                                                                   GoUint64 plugin_name_len,
+                                                                   GoUint64 config_id) {
+  ASSERT(envoy_go_filter_on_downstream_connection_ != nullptr);
+  return envoy_go_filter_on_downstream_connection_(w, plugin_name_ptr, plugin_name_len, config_id);
+}
+
+GoUint64 NetworkFilterDsoImpl::envoyGoFilterOnDownstreamData(void* w, GoUint64 data_size,
+                                                             GoUint64 data_ptr, GoInt slice_num,
+                                                             GoInt end_of_stream) {
+  ASSERT(envoy_go_filter_on_downstream_data_ != nullptr);
+  return envoy_go_filter_on_downstream_data_(w, data_size, data_ptr, slice_num, end_of_stream);
+}
+
+void NetworkFilterDsoImpl::envoyGoFilterOnDownstreamEvent(void* w, GoInt event) {
+  ASSERT(envoy_go_filter_on_downstream_event_ != nullptr);
+  envoy_go_filter_on_downstream_event_(w, event);
+}
+
+GoUint64 NetworkFilterDsoImpl::envoyGoFilterOnDownstreamWrite(void* w, GoUint64 data_size,
+                                                              GoUint64 data_ptr, GoInt slice_num,
+                                                              GoInt end_of_stream) {
+  ASSERT(envoy_go_filter_on_downstream_write_ != nullptr);
+  return envoy_go_filter_on_downstream_write_(w, data_size, data_ptr, slice_num, end_of_stream);
+}
+
+void NetworkFilterDsoImpl::envoyGoFilterOnUpstreamConnectionReady(void* w) {
+  ASSERT(envoy_go_filter_on_upstream_connection_ready_ != nullptr);
+  envoy_go_filter_on_upstream_connection_ready_(w);
+}
+
+void NetworkFilterDsoImpl::envoyGoFilterOnUpstreamConnectionFailure(void* w, GoInt reason) {
+  ASSERT(envoy_go_filter_on_upstream_connection_failure_ != nullptr);
+  envoy_go_filter_on_upstream_connection_failure_(w, reason);
+}
+
+void NetworkFilterDsoImpl::envoyGoFilterOnUpstreamData(void* w, GoUint64 data_size,
+                                                       GoUint64 data_ptr, GoInt slice_num,
+                                                       GoInt end_of_stream) {
+  ASSERT(envoy_go_filter_on_upstream_data_ != nullptr);
+  envoy_go_filter_on_upstream_data_(w, data_size, data_ptr, slice_num, end_of_stream);
+}
+
+void NetworkFilterDsoImpl::envoyGoFilterOnUpstreamEvent(void* w, GoInt event) {
+  ASSERT(envoy_go_filter_on_upstream_event_ != nullptr);
+  envoy_go_filter_on_upstream_event_(w, event);
+}
+
 } // namespace Dso
 } // namespace Envoy
diff --git a/contrib/golang/common/dso/dso.h b/contrib/golang/common/dso/dso.h
index 540d4b97aa3b..438dd773df0b 100644
--- a/contrib/golang/common/dso/dso.h
+++ b/contrib/golang/common/dso/dso.h
@@ -15,6 +15,7 @@ namespace Dso {
 
 class Dso {
 public:
+  Dso() = default;
   Dso(const std::string dso_name);
   ~Dso();
   bool loaded() { return loaded_; }
@@ -104,6 +105,77 @@ class ClusterSpecifierDsoImpl : public ClusterSpecifierDso {
 using HttpFilterDsoPtr = std::shared_ptr;
 using ClusterSpecifierDsoPtr = std::shared_ptr;
 
+class NetworkFilterDso : public Dso {
+public:
+  NetworkFilterDso() = default;
+  NetworkFilterDso(const std::string dso_name) : Dso(dso_name){};
+  virtual ~NetworkFilterDso() = default;
+
+  virtual GoUint64 envoyGoFilterOnNetworkFilterConfig(GoUint64 library_id_ptr,
+                                                      GoUint64 library_id_len, GoUint64 config_ptr,
+                                                      GoUint64 config_len) PURE;
+  virtual GoUint64 envoyGoFilterOnDownstreamConnection(void* w, GoUint64 plugin_name_ptr,
+                                                       GoUint64 plugin_name_len,
+                                                       GoUint64 config_id) PURE;
+  virtual GoUint64 envoyGoFilterOnDownstreamData(void* w, GoUint64 data_size, GoUint64 data_ptr,
+                                                 GoInt slice_num, GoInt end_of_stream) PURE;
+  virtual void envoyGoFilterOnDownstreamEvent(void* w, GoInt event) PURE;
+  virtual GoUint64 envoyGoFilterOnDownstreamWrite(void* w, GoUint64 data_size, GoUint64 data_ptr,
+                                                  GoInt slice_num, GoInt end_of_stream) PURE;
+
+  virtual void envoyGoFilterOnUpstreamConnectionReady(void* w) PURE;
+  virtual void envoyGoFilterOnUpstreamConnectionFailure(void* w, GoInt reason) PURE;
+  virtual void envoyGoFilterOnUpstreamData(void* w, GoUint64 data_size, GoUint64 data_ptr,
+                                           GoInt slice_num, GoInt end_of_stream) PURE;
+  virtual void envoyGoFilterOnUpstreamEvent(void* w, GoInt event) PURE;
+};
+
+class NetworkFilterDsoImpl : public NetworkFilterDso {
+public:
+  NetworkFilterDsoImpl(const std::string dso_name);
+  ~NetworkFilterDsoImpl() override = default;
+
+  GoUint64 envoyGoFilterOnNetworkFilterConfig(GoUint64 library_id_ptr, GoUint64 library_id_len,
+                                              GoUint64 config_ptr, GoUint64 config_len) override;
+  GoUint64 envoyGoFilterOnDownstreamConnection(void* w, GoUint64 plugin_name_ptr,
+                                               GoUint64 plugin_name_len,
+                                               GoUint64 config_id) override;
+  GoUint64 envoyGoFilterOnDownstreamData(void* w, GoUint64 data_size, GoUint64 data_ptr,
+                                         GoInt slice_num, GoInt end_of_stream) override;
+  void envoyGoFilterOnDownstreamEvent(void* w, GoInt event) override;
+  GoUint64 envoyGoFilterOnDownstreamWrite(void* w, GoUint64 data_size, GoUint64 data_ptr,
+                                          GoInt slice_num, GoInt end_of_stream) override;
+
+  void envoyGoFilterOnUpstreamConnectionReady(void* w) override;
+  void envoyGoFilterOnUpstreamConnectionFailure(void* w, GoInt reason) override;
+  void envoyGoFilterOnUpstreamData(void* w, GoUint64 data_size, GoUint64 data_ptr, GoInt slice_num,
+                                   GoInt end_of_stream) override;
+  void envoyGoFilterOnUpstreamEvent(void* w, GoInt event) override;
+
+private:
+  GoUint64 (*envoy_go_filter_on_network_filter_config_)(GoUint64 library_id_ptr,
+                                                        GoUint64 library_id_len,
+                                                        GoUint64 config_ptr,
+                                                        GoUint64 config_len) = {nullptr};
+  GoUint64 (*envoy_go_filter_on_downstream_connection_)(void* w, GoUint64 plugin_name_ptr,
+                                                        GoUint64 plugin_name_len,
+                                                        GoUint64 config_id) = {nullptr};
+  GoUint64 (*envoy_go_filter_on_downstream_data_)(void* w, GoUint64 data_size, GoUint64 data_ptr,
+                                                  GoInt slice_num, GoInt end_of_stream) = {nullptr};
+  void (*envoy_go_filter_on_downstream_event_)(void* w, GoInt event) = {nullptr};
+  GoUint64 (*envoy_go_filter_on_downstream_write_)(void* w, GoUint64 data_size, GoUint64 data_ptr,
+                                                   GoInt slice_num,
+                                                   GoInt end_of_stream) = {nullptr};
+
+  void (*envoy_go_filter_on_upstream_connection_ready_)(void* w) = {nullptr};
+  void (*envoy_go_filter_on_upstream_connection_failure_)(void* w, GoInt reason) = {nullptr};
+  void (*envoy_go_filter_on_upstream_data_)(void* w, GoUint64 data_size, GoUint64 data_ptr,
+                                            GoInt slice_num, GoInt end_of_stream) = {nullptr};
+  void (*envoy_go_filter_on_upstream_event_)(void* w, GoInt event) = {nullptr};
+};
+
+using NetworkFilterDsoPtr = std::shared_ptr;
+
 /*
  * We do not unload a dynamic library once it is loaded. This is because
  * Go shared library could not be unload by dlclose yet, see:
@@ -154,6 +226,21 @@ template  class DsoManager {
     return dso;
   };
 
+  /**
+   * Get the go plugin dynamic library.
+   * @param dso_id is unique ID for dynamic library.
+   * @return nullptr if get failed. Otherwise, return the DSO instance.
+   */
+  static std::shared_ptr getDsoByID(std::string dso_id) {
+    DsoStoreType& dsoStore = getDsoStore();
+    absl::ReaderMutexLock lock(&dsoStore.mutex_);
+    auto it = dsoStore.id_to_dso_.find(dso_id);
+    if (it != dsoStore.id_to_dso_.end()) {
+      return it->second;
+    }
+    return nullptr;
+  };
+
   /**
    * Get the go plugin dynamic library by plugin name.
    * @param plugin_name is unique ID for a plugin, one DSO may contains multiple plugins.
diff --git a/contrib/golang/common/dso/libgolang.h b/contrib/golang/common/dso/libgolang.h
index 66a8d27ba43b..d9be431248ba 100644
--- a/contrib/golang/common/dso/libgolang.h
+++ b/contrib/golang/common/dso/libgolang.h
@@ -150,6 +150,72 @@ extern GoUint64
 envoyGoClusterSpecifierNewPlugin(GoUint64 configPtr,  // NOLINT(readability-identifier-naming)
                                  GoUint64 configLen); // NOLINT(readability-identifier-naming)
 
+// go:linkname envoyGoFilterOnNetworkFilterConfig
+// github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnNetworkFilterConfig
+extern GoUint64
+envoyGoFilterOnNetworkFilterConfig(GoUint64 libraryIDPtr, // NOLINT(readability-identifier-naming)
+                                   GoUint64 libraryIDLen, // NOLINT(readability-identifier-naming)
+                                   GoUint64 configPtr,    // NOLINT(readability-identifier-naming)
+                                   GoUint64 configLen);   // NOLINT(readability-identifier-naming)
+
+// go:linkname envoyGoFilterOnDownstreamConnection
+// github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnDownstreamConnection
+extern GoUint64
+envoyGoFilterOnDownstreamConnection(void* f,
+                                    GoUint64 pluginNamePtr, // NOLINT(readability-identifier-naming)
+                                    GoUint64 pluginNameLen, // NOLINT(readability-identifier-naming)
+                                    GoUint64 configID);     // NOLINT(readability-identifier-naming)
+
+// go:linkname envoyGoFilterOnDownstreamData
+// github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnDownstreamData
+extern GoUint64
+envoyGoFilterOnDownstreamData(void* f,
+                              GoUint64 dataSize,  // NOLINT(readability-identifier-naming)
+                              GoUint64 dataPtr,   // NOLINT(readability-identifier-naming)
+                              GoInt sliceNum,     // NOLINT(readability-identifier-naming)
+                              GoInt endOfStream); // NOLINT(readability-identifier-naming)
+
+// go:linkname envoyGoFilterOnDownstreamWrite
+// github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnDownstreamWrite
+extern GoUint64
+envoyGoFilterOnDownstreamWrite(void* f,
+                               GoUint64 dataSize,  // NOLINT(readability-identifier-naming)
+                               GoUint64 dataPtr,   // NOLINT(readability-identifier-naming)
+                               GoInt sliceNum,     // NOLINT(readability-identifier-naming)
+                               GoInt endOfStream); // NOLINT(readability-identifier-naming)
+
+// go:linkname envoyGoFilterOnDownstreamEvent
+// github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnDownstreamEvent
+extern void envoyGoFilterOnDownstreamEvent(void* f,
+                                           GoInt event); // NOLINT(readability-identifier-naming)
+
+// go:linkname envoyGoFilterOnUpstreamConnectionReady
+// github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnUpstreamConnectionReady
+extern void envoyGoFilterOnUpstreamConnectionReady(
+    void* f,
+    GoUint64 envoyConnID, // NOLINT(readability-identifier-naming)
+    GoUint64 configID);   // NOLINT(readability-identifier-naming)
+
+// go:linkname envoyGoFilterOnUpstreamConnectionFailure
+// github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnUpstreamConnectionFailure
+extern void
+envoyGoFilterOnUpstreamConnectionFailure(void* f,
+                                         GoInt reason); // NOLINT(readability-identifier-naming)
+
+// go:linkname envoyGoFilterOnUpstreamData
+// github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnUpstreamData
+extern GoUint64
+envoyGoFilterOnUpstreamData(void* f,
+                            GoUint64 dataSize,  // NOLINT(readability-identifier-naming)
+                            GoUint64 dataPtr,   // NOLINT(readability-identifier-naming)
+                            GoInt sliceNum,     // NOLINT(readability-identifier-naming)
+                            GoInt endOfStream); // NOLINT(readability-identifier-naming)
+
+// go:linkname envoyGoFilterOnUpstreamEvent
+// github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnUpstreamEvent
+extern void envoyGoFilterOnUpstreamEvent(void* f,
+                                         GoInt event); // NOLINT(readability-identifier-naming)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/contrib/golang/common/dso/test/dso_test.cc b/contrib/golang/common/dso/test/dso_test.cc
index 95912d855b60..6471b62ac46e 100644
--- a/contrib/golang/common/dso/test/dso_test.cc
+++ b/contrib/golang/common/dso/test/dso_test.cc
@@ -29,28 +29,51 @@ TEST(DsoManagerTest, Pub) {
   auto plugin_name = "example";
   auto path = genSoPath(id);
 
-  // get before load http filter dso
-  auto dso = DsoManager::getDsoByPluginName(plugin_name);
-  EXPECT_EQ(dso, nullptr);
-
-  // first time load http filter dso
-  dso = DsoManager::load(id, path, plugin_name);
-  EXPECT_NE(dso, nullptr);
-
-  // get after load http filter dso
-  dso = DsoManager::getDsoByPluginName(plugin_name);
-  EXPECT_NE(dso, nullptr);
-  EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 200), 200);
-
-  // second time load http filter dso
-  dso = DsoManager::load(id, path, plugin_name);
-  EXPECT_NE(dso, nullptr);
-
-  // first time load cluster specifier dso
-  auto cluster_dso = DsoManager::load(id, path);
-  EXPECT_NE(dso, nullptr);
-
-  EXPECT_EQ(cluster_dso->envoyGoClusterSpecifierNewPlugin(0, 0), 200);
+  {
+    // get before load http filter dso
+    auto dso = DsoManager::getDsoByPluginName(plugin_name);
+    EXPECT_EQ(dso, nullptr);
+
+    // first time load http filter dso
+    dso = DsoManager::load(id, path, plugin_name);
+    EXPECT_NE(dso, nullptr);
+
+    // get after load http filter dso
+    dso = DsoManager::getDsoByPluginName(plugin_name);
+    EXPECT_NE(dso, nullptr);
+    EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(0, 0, 0, 200), 200);
+
+    // second time load http filter dso
+    dso = DsoManager::load(id, path, plugin_name);
+    EXPECT_NE(dso, nullptr);
+  }
+
+  {
+    // first time load cluster specifier dso
+    auto cluster_dso = DsoManager::load(id, path);
+    EXPECT_NE(cluster_dso, nullptr);
+
+    EXPECT_EQ(cluster_dso->envoyGoClusterSpecifierNewPlugin(0, 0), 200);
+  }
+
+  {
+    // get before load network filter dso
+    auto dso = DsoManager::getDsoByID(id);
+    EXPECT_EQ(dso, nullptr);
+
+    // first time load network filter dso
+    auto res = DsoManager::load(id, path);
+    EXPECT_NE(res, nullptr);
+
+    // get after load network filter dso
+    dso = DsoManager::getDsoByID(id);
+    EXPECT_NE(dso, nullptr);
+    EXPECT_EQ(dso->envoyGoFilterOnNetworkFilterConfig(0, 0, 0, 0), 100);
+
+    // second time load network filter dso
+    res = DsoManager::load(id, path);
+    EXPECT_NE(dso, nullptr);
+  }
 }
 
 // missing a symbol
diff --git a/contrib/golang/common/dso/test/mocks.h b/contrib/golang/common/dso/test/mocks.h
index 5cc346ea7385..4c2944e2f350 100644
--- a/contrib/golang/common/dso/test/mocks.h
+++ b/contrib/golang/common/dso/test/mocks.h
@@ -24,5 +24,28 @@ class MockHttpFilterDsoImpl : public HttpFilterDso {
   MOCK_METHOD(void, envoyGoRequestSemaDec, (httpRequest * p0));
 };
 
+class MockNetworkFilterDsoImpl : public NetworkFilterDso {
+public:
+  MockNetworkFilterDsoImpl() = default;
+  ~MockNetworkFilterDsoImpl() override = default;
+
+  MOCK_METHOD(GoUint64, envoyGoFilterOnNetworkFilterConfig,
+              (GoUint64 libraryIDPtr, GoUint64 libraryIDLen, GoUint64 configPtr,
+               GoUint64 configLen));
+  MOCK_METHOD(GoUint64, envoyGoFilterOnDownstreamConnection,
+              (void* w, GoUint64 pluginNamePtr, GoUint64 pluginNameLen, GoUint64 configID));
+  MOCK_METHOD(GoUint64, envoyGoFilterOnDownstreamData,
+              (void* w, GoUint64 dataSize, GoUint64 dataPtr, GoInt sliceNum, GoInt endOfStream));
+  MOCK_METHOD(void, envoyGoFilterOnDownstreamEvent, (void* w, GoInt event));
+  MOCK_METHOD(GoUint64, envoyGoFilterOnDownstreamWrite,
+              (void* w, GoUint64 dataSize, GoUint64 dataPtr, GoInt sliceNum, GoInt endOfStream));
+
+  MOCK_METHOD(void, envoyGoFilterOnUpstreamConnectionReady, (void* w));
+  MOCK_METHOD(void, envoyGoFilterOnUpstreamConnectionFailure, (void* w, GoInt reason));
+  MOCK_METHOD(void, envoyGoFilterOnUpstreamData,
+              (void* w, GoUint64 dataSize, GoUint64 dataPtr, GoInt sliceNum, GoInt endOfStream));
+  MOCK_METHOD(void, envoyGoFilterOnUpstreamEvent, (void* w, GoInt event));
+};
+
 } // namespace Dso
 } // namespace Envoy
diff --git a/contrib/golang/common/dso/test/test_data/simple.go b/contrib/golang/common/dso/test/test_data/simple.go
index adcff7b0a9bc..ff9638b8e756 100644
--- a/contrib/golang/common/dso/test/test_data/simple.go
+++ b/contrib/golang/common/dso/test/test_data/simple.go
@@ -6,6 +6,7 @@ typedef struct {
 } httpRequest;
 */
 import "C"
+import "unsafe"
 
 import (
 	"sync"
@@ -58,6 +59,43 @@ func envoyGoOnClusterSpecify(pluginPtr uint64, headerPtr uint64, pluginId uint64
 	return 0
 }
 
+//export envoyGoFilterOnNetworkFilterConfig
+func envoyGoFilterOnNetworkFilterConfig(libraryIDPtr uint64, libraryIDLen uint64, configPtr uint64, configLen uint64) uint64 {
+	return 100
+}
+
+//export envoyGoFilterOnDownstreamConnection
+func envoyGoFilterOnDownstreamConnection(wrapper unsafe.Pointer, pluginNamePtr uint64, pluginNameLen uint64,
+	configID uint64) uint64 {
+	return 0
+}
+
+//export envoyGoFilterOnDownstreamData
+func envoyGoFilterOnDownstreamData(wrapper unsafe.Pointer, dataSize uint64, dataPtr uint64, sliceNum int, endOfStream int) uint64 {
+	return 0
+}
+
+//export envoyGoFilterOnDownstreamEvent
+func envoyGoFilterOnDownstreamEvent(wrapper unsafe.Pointer, event int) {}
+
+//export envoyGoFilterOnDownstreamWrite
+func envoyGoFilterOnDownstreamWrite(wrapper unsafe.Pointer, dataSize uint64, dataPtr uint64, sliceNum int, endOfStream int) uint64 {
+	return 0
+}
+
+//export envoyGoFilterOnUpstreamConnectionReady
+func envoyGoFilterOnUpstreamConnectionReady(wrapper unsafe.Pointer) {}
+
+//export envoyGoFilterOnUpstreamConnectionFailure
+func envoyGoFilterOnUpstreamConnectionFailure(wrapper unsafe.Pointer, reason int) {}
+
+//export envoyGoFilterOnUpstreamData
+func envoyGoFilterOnUpstreamData(wrapper unsafe.Pointer, dataSize uint64, dataPtr uint64, sliceNum int, endOfStream int) {
+}
+
+//export envoyGoFilterOnUpstreamEvent
+func envoyGoFilterOnUpstreamEvent(wrapper unsafe.Pointer, event int) {}
+
 //export envoyGoRequestSemaDec
 func envoyGoRequestSemaDec(r *C.httpRequest) {
 }
diff --git a/contrib/golang/filters/http/source/go/pkg/api/BUILD b/contrib/golang/common/go/api/BUILD
similarity index 77%
rename from contrib/golang/filters/http/source/go/pkg/api/BUILD
rename to contrib/golang/common/go/api/BUILD
index bbce799a1516..f77587290be1 100644
--- a/contrib/golang/filters/http/source/go/pkg/api/BUILD
+++ b/contrib/golang/common/go/api/BUILD
@@ -5,11 +5,12 @@ licenses(["notice"])  # Apache 2
 go_library(
     name = "api",
     srcs = [
+        "capi.go",
         "cgocheck.go",
         "filter.go",
         "type.go",
     ],
-    importpath = "github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api",
+    importpath = "github.com/envoyproxy/envoy/contrib/golang/common/go/api",
     visibility = ["//visibility:public"],
     deps = [
         "@org_golang_google_protobuf//types/known/anypb",
diff --git a/contrib/golang/filters/http/source/go/pkg/api/api.h b/contrib/golang/common/go/api/api.h
similarity index 78%
rename from contrib/golang/filters/http/source/go/pkg/api/api.h
rename to contrib/golang/common/go/api/api.h
index 34127323724f..85857ef06dc4 100644
--- a/contrib/golang/filters/http/source/go/pkg/api/api.h
+++ b/contrib/golang/common/go/api/api.h
@@ -74,6 +74,20 @@ CAPIStatus envoyGoFilterHttpSetStringFilterState(void* r, void* key, void* value
                                                  int life_span, int stream_sharing);
 CAPIStatus envoyGoFilterHttpGetStringFilterState(void* r, void* key, void* value);
 
+// downstream
+CAPIStatus envoyGoFilterDownstreamClose(void* wrapper, int closeType);
+CAPIStatus envoyGoFilterDownstreamWrite(void* wrapper, void* buffers, int buffersNum,
+                                        int endStream);
+void envoyGoFilterDownstreamFinalize(void* wrapper, int reason);
+CAPIStatus envoyGoFilterDownstreamInfo(void* wrapper, int t, void* ret);
+
+// upstream
+void* envoyGoFilterUpstreamConnect(void* libraryID, void* addr);
+CAPIStatus envoyGoFilterUpstreamWrite(void* wrapper, void* buffers, int buffersNum, int endStream);
+CAPIStatus envoyGoFilterUpstreamClose(void* wrapper, int closeType);
+void envoyGoFilterUpstreamFinalize(void* wrapper, int reason);
+CAPIStatus envoyGoFilterUpstreamInfo(void* wrapper, int t, void* ret);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif
diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi.go b/contrib/golang/common/go/api/capi.go
similarity index 58%
rename from contrib/golang/filters/http/source/go/pkg/http/capi.go
rename to contrib/golang/common/go/api/capi.go
index 40238541b8a5..eb9c6db5ab21 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/capi.go
+++ b/contrib/golang/common/go/api/capi.go
@@ -15,13 +15,9 @@
  * limitations under the License.
  */
 
-package http
+package api
 
-import (
-	"unsafe"
-
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
-)
+import "unsafe"
 
 type HttpCAPI interface {
 	HttpContinue(r unsafe.Pointer, status uint64)
@@ -37,23 +33,45 @@ type HttpCAPI interface {
 	HttpRemoveHeader(r unsafe.Pointer, key *string)
 
 	HttpGetBuffer(r unsafe.Pointer, bufferPtr uint64, value *string, length uint64)
-	HttpSetBufferHelper(r unsafe.Pointer, bufferPtr uint64, value string, action api.BufferAction)
+	HttpSetBufferHelper(r unsafe.Pointer, bufferPtr uint64, value string, action BufferAction)
 
 	HttpCopyTrailers(r unsafe.Pointer, num uint64, bytes uint64) map[string][]string
 	HttpSetTrailer(r unsafe.Pointer, key *string, value *string, add bool)
 	HttpRemoveTrailer(r unsafe.Pointer, key *string)
 
-	HttpGetStringValue(r *httpRequest, id int) (string, bool)
+	HttpGetStringValue(r unsafe.Pointer, id int) (string, bool)
 	HttpGetIntegerValue(r unsafe.Pointer, id int) (uint64, bool)
 
 	// TODO: HttpGetDynamicMetadata(r unsafe.Pointer, filterName string) map[string]interface{}
 	HttpSetDynamicMetadata(r unsafe.Pointer, filterName string, key string, value interface{})
 
-	HttpLog(level api.LogType, message string)
-	HttpLogLevel() api.LogType
+	HttpLog(level LogType, message string)
+	HttpLogLevel() LogType
 
 	HttpFinalize(r unsafe.Pointer, reason int)
 
-	HttpSetStringFilterState(r unsafe.Pointer, key string, value string, stateType api.StateType, lifeSpan api.LifeSpan, streamSharing api.StreamSharing)
-	HttpGetStringFilterState(r *httpRequest, key string) string
+	HttpSetStringFilterState(r unsafe.Pointer, key string, value string, stateType StateType, lifeSpan LifeSpan, streamSharing StreamSharing)
+	HttpGetStringFilterState(r unsafe.Pointer, key string) string
+}
+
+type NetworkCAPI interface {
+	// DownstreamWrite writes buffer data into downstream connection.
+	DownstreamWrite(f unsafe.Pointer, bufferPtr unsafe.Pointer, bufferLen int, endStream int)
+	// DownstreamClose closes the downstream connection
+	DownstreamClose(f unsafe.Pointer, closeType int)
+	// DownstreamFinalize cleans up the resource of downstream connection, should be called only by runtime.SetFinalizer
+	DownstreamFinalize(f unsafe.Pointer, reason int)
+	// DownstreamInfo gets the downstream connection info of infoType
+	DownstreamInfo(f unsafe.Pointer, infoType int) string
+
+	// UpstreamConnect creates an envoy upstream connection to address
+	UpstreamConnect(libraryID string, addr string) unsafe.Pointer
+	// UpstreamWrite writes buffer data into upstream connection.
+	UpstreamWrite(f unsafe.Pointer, bufferPtr unsafe.Pointer, bufferLen int, endStream int)
+	// UpstreamClose closes the upstream connection
+	UpstreamClose(f unsafe.Pointer, closeType int)
+	// UpstreamFinalize cleans up the resource of upstream connection, should be called only by runtime.SetFinalizer
+	UpstreamFinalize(f unsafe.Pointer, reason int)
+	// UpstreamInfo gets the upstream connection info of infoType
+	UpstreamInfo(f unsafe.Pointer, infoType int) string
 }
diff --git a/contrib/golang/filters/http/source/go/pkg/api/cgocheck.go b/contrib/golang/common/go/api/cgocheck.go
similarity index 100%
rename from contrib/golang/filters/http/source/go/pkg/api/cgocheck.go
rename to contrib/golang/common/go/api/cgocheck.go
diff --git a/contrib/golang/filters/http/source/go/pkg/api/filter.go b/contrib/golang/common/go/api/filter.go
similarity index 77%
rename from contrib/golang/filters/http/source/go/pkg/api/filter.go
rename to contrib/golang/common/go/api/filter.go
index 2f4fa70d9829..850227730654 100644
--- a/contrib/golang/filters/http/source/go/pkg/api/filter.go
+++ b/contrib/golang/common/go/api/filter.go
@@ -103,6 +103,39 @@ type DynamicMetadata interface {
 	Set(filterName string, key string, value interface{})
 }
 
+type DownstreamFilter interface {
+	// Called when a connection is first established.
+	OnNewConnection() FilterStatus
+	// Called when data is read on the connection.
+	OnData(buffer []byte, endOfStream bool) FilterStatus
+	// Callback for connection events.
+	OnEvent(event ConnectionEvent)
+	// Called when data is to be written on the connection.
+	OnWrite(buffer []byte, endOfStream bool) FilterStatus
+}
+
+type UpstreamFilter interface {
+	// Called when a connection is available to process a request/response.
+	OnPoolReady(cb ConnectionCallback)
+	// Called when a pool error occurred and no connection could be acquired for making the request.
+	OnPoolFailure(poolFailureReason PoolFailureReason, transportFailureReason string)
+	// Invoked when data is delivered from the upstream connection.
+	OnData(buffer []byte, endOfStream bool)
+	// Callback for connection events.
+	OnEvent(event ConnectionEvent)
+}
+
+type ConnectionCallback interface {
+	// Return the local address of the connection.
+	LocalAddr() string
+	// Return the remote address of the connection.
+	RemoteAddr() string
+	// Write data to the connection.
+	Write(buffer []byte, endStream bool)
+	// Close the connection.
+	Close(closeType ConnectionCloseType)
+}
+
 type StateType int
 
 const (
diff --git a/contrib/golang/filters/http/source/go/pkg/api/type.go b/contrib/golang/common/go/api/type.go
similarity index 74%
rename from contrib/golang/filters/http/source/go/pkg/api/type.go
rename to contrib/golang/common/go/api/type.go
index 002cfd80a28d..1c055ebdfe36 100644
--- a/contrib/golang/filters/http/source/go/pkg/api/type.go
+++ b/contrib/golang/common/go/api/type.go
@@ -287,3 +287,121 @@ func (e EnvoyRequestPhase) String() string {
 	}
 	return "unknown phase"
 }
+
+// Status codes returned by filters that can cause future filters to not get iterated to.
+type FilterStatus int
+
+const (
+	// Continue to further filters.
+	NetworkFilterContinue FilterStatus = 0
+	// Stop executing further filters.
+	NetworkFilterStopIteration FilterStatus = 1
+)
+
+func (s FilterStatus) String() string {
+	switch s {
+	case NetworkFilterContinue:
+		return "Continue"
+	case NetworkFilterStopIteration:
+		return "StopIteration"
+	}
+	return "unknown"
+}
+
+// Events that occur on a connection.
+type ConnectionEvent int
+
+const (
+	RemoteClose      ConnectionEvent = 0
+	LocalClose       ConnectionEvent = 1
+	Connected        ConnectionEvent = 2
+	ConnectedZeroRtt ConnectionEvent = 3
+)
+
+func (e ConnectionEvent) String() string {
+	switch e {
+	case RemoteClose:
+		return "RemoteClose"
+	case LocalClose:
+		return "LocalClose"
+	case Connected:
+		return "Connected"
+	case ConnectedZeroRtt:
+		return "ConnectedZeroRtt"
+	}
+	return "unknown"
+}
+
+// Type of connection close to perform.
+type ConnectionCloseType int
+
+const (
+	// Flush pending write data before raising ConnectionEvent::LocalClose
+	FlushWrite ConnectionCloseType = 0
+	// Do not flush any pending data. Write the pending data to buffer and then immediately
+	// raise ConnectionEvent::LocalClose
+	NoFlush ConnectionCloseType = 1
+	// Flush pending write data and delay raising a ConnectionEvent::LocalClose
+	// until the delayed_close_timeout expires
+	FlushWriteAndDelay ConnectionCloseType = 2
+	// Do not write/flush any pending data and immediately raise ConnectionEvent::LocalClose
+	Abort ConnectionCloseType = 3
+)
+
+func (t ConnectionCloseType) String() string {
+	switch t {
+	case FlushWrite:
+		return "FlushWrite"
+	case NoFlush:
+		return "NoFlush"
+	case FlushWriteAndDelay:
+		return "FlushWriteAndDelay"
+	case Abort:
+		return "Abort"
+	}
+	return "unknown"
+}
+
+type PoolFailureReason int
+
+const (
+	// A resource overflowed and policy prevented a new connection from being created.
+	Overflow PoolFailureReason = 0
+	// A local connection failure took place while creating a new connection.
+	LocalConnectionFailure PoolFailureReason = 1
+	// A remote connection failure took place while creating a new connection.
+	RemoteConnectionFailure PoolFailureReason = 2
+	// A timeout occurred while creating a new connection.
+	Timeout PoolFailureReason = 3
+)
+
+func (r PoolFailureReason) String() string {
+	switch r {
+	case Overflow:
+		return "Overflow"
+	case LocalConnectionFailure:
+		return "LocalConnectionFailure"
+	case RemoteConnectionFailure:
+		return "RemoteConnectionFailure"
+	case Timeout:
+		return "Timeout"
+	}
+	return "unknown"
+}
+
+type ConnectionInfoType int
+
+const (
+	ConnectionInfoLocalAddr  ConnectionInfoType = 0
+	ConnectionInfoRemoteAddr ConnectionInfoType = 1
+)
+
+func (t ConnectionInfoType) String() string {
+	switch t {
+	case ConnectionInfoLocalAddr:
+		return "ConnectionInfoLocalAddr"
+	case ConnectionInfoRemoteAddr:
+		return "ConnectionInfoRemoteAddr"
+	}
+	return "unknown"
+}
diff --git a/contrib/golang/filters/http/source/go/pkg/http/BUILD b/contrib/golang/filters/http/source/go/pkg/http/BUILD
index 6cfca08915ce..0c1b726ce318 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/BUILD
+++ b/contrib/golang/filters/http/source/go/pkg/http/BUILD
@@ -6,7 +6,6 @@ go_library(
     name = "http",
     srcs = [
         "api.h",
-        "capi.go",
         "capi_impl.go",
         "config.go",
         "filter.go",
@@ -34,8 +33,8 @@ go_library(
     importpath = "github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http",
     visibility = ["//visibility:public"],
     deps = [
+        "//contrib/golang/common/go/api",
         "//contrib/golang/common/go/utils",
-        "//contrib/golang/filters/http/source/go/pkg/api",
         "@org_golang_google_protobuf//proto",
         "@org_golang_google_protobuf//types/known/anypb",
         "@org_golang_google_protobuf//types/known/structpb",
diff --git a/contrib/golang/filters/http/source/go/pkg/http/api.h b/contrib/golang/filters/http/source/go/pkg/http/api.h
index 7ccbc18e959f..6c0942de6865 120000
--- a/contrib/golang/filters/http/source/go/pkg/http/api.h
+++ b/contrib/golang/filters/http/source/go/pkg/http/api.h
@@ -1 +1 @@
-../api/api.h
\ No newline at end of file
+../../../../../../common/go/api/api.h
\ No newline at end of file
diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
index 1ba63f71c238..4b9d58592671 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
@@ -38,9 +38,10 @@ import (
 	"sync/atomic"
 	"unsafe"
 
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/structpb"
+
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 const (
@@ -216,7 +217,8 @@ func (c *httpCApiImpl) HttpRemoveTrailer(r unsafe.Pointer, key *string) {
 	handleCApiStatus(res)
 }
 
-func (c *httpCApiImpl) HttpGetStringValue(r *httpRequest, id int) (string, bool) {
+func (c *httpCApiImpl) HttpGetStringValue(rr unsafe.Pointer, id int) (string, bool) {
+	r := (*httpRequest)(rr)
 	var value string
 	// add a lock to protect filter->req_->strValue field in the Envoy side, from being writing concurrency,
 	// since there might be multiple concurrency goroutines invoking this API on the Go side.
@@ -266,10 +268,10 @@ func (c *httpCApiImpl) HttpFinalize(r unsafe.Pointer, reason int) {
 	C.envoyGoFilterHttpFinalize(r, C.int(reason))
 }
 
-var cAPI HttpCAPI = &httpCApiImpl{}
+var cAPI api.HttpCAPI = &httpCApiImpl{}
 
 // SetHttpCAPI for mock cAPI
-func SetHttpCAPI(api HttpCAPI) {
+func SetHttpCAPI(api api.HttpCAPI) {
 	cAPI = api
 }
 
@@ -278,7 +280,8 @@ func (c *httpCApiImpl) HttpSetStringFilterState(r unsafe.Pointer, key string, va
 	handleCApiStatus(res)
 }
 
-func (c *httpCApiImpl) HttpGetStringFilterState(r *httpRequest, key string) string {
+func (c *httpCApiImpl) HttpGetStringFilterState(rr unsafe.Pointer, key string) string {
+	r := (*httpRequest)(rr)
 	var value string
 	r.sema.Add(1)
 	res := C.envoyGoFilterHttpGetStringFilterState(unsafe.Pointer(r.req), unsafe.Pointer(&key), unsafe.Pointer(&value))
diff --git a/contrib/golang/filters/http/source/go/pkg/http/config.go b/contrib/golang/filters/http/source/go/pkg/http/config.go
index eb3dd284f625..1727370f6910 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/config.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/config.go
@@ -36,10 +36,10 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 	"github.com/envoyproxy/envoy/contrib/golang/common/go/utils"
 )
 
diff --git a/contrib/golang/filters/http/source/go/pkg/http/filter.go b/contrib/golang/filters/http/source/go/pkg/http/filter.go
index 74def40b8996..afb8d26fa751 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/filter.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/filter.go
@@ -36,7 +36,7 @@ import (
 	"sync"
 	"unsafe"
 
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 const (
@@ -138,12 +138,12 @@ type streamInfo struct {
 }
 
 func (s *streamInfo) GetRouteName() string {
-	name, _ := cAPI.HttpGetStringValue(s.request, ValueRouteName)
+	name, _ := cAPI.HttpGetStringValue(unsafe.Pointer(s.request), ValueRouteName)
 	return name
 }
 
 func (s *streamInfo) FilterChainName() string {
-	name, _ := cAPI.HttpGetStringValue(s.request, ValueFilterChainName)
+	name, _ := cAPI.HttpGetStringValue(unsafe.Pointer(s.request), ValueFilterChainName)
 	return name
 }
 
@@ -165,7 +165,7 @@ func (s *streamInfo) ResponseCode() (uint32, bool) {
 }
 
 func (s *streamInfo) ResponseCodeDetails() (string, bool) {
-	return cAPI.HttpGetStringValue(s.request, ValueResponseCodeDetails)
+	return cAPI.HttpGetStringValue(unsafe.Pointer(s.request), ValueResponseCodeDetails)
 }
 
 func (s *streamInfo) AttemptCount() uint32 {
@@ -188,21 +188,21 @@ func (d *dynamicMetadata) Set(filterName string, key string, value interface{})
 }
 
 func (s *streamInfo) DownstreamLocalAddress() string {
-	address, _ := cAPI.HttpGetStringValue(s.request, ValueDownstreamLocalAddress)
+	address, _ := cAPI.HttpGetStringValue(unsafe.Pointer(s.request), ValueDownstreamLocalAddress)
 	return address
 }
 
 func (s *streamInfo) DownstreamRemoteAddress() string {
-	address, _ := cAPI.HttpGetStringValue(s.request, ValueDownstreamRemoteAddress)
+	address, _ := cAPI.HttpGetStringValue(unsafe.Pointer(s.request), ValueDownstreamRemoteAddress)
 	return address
 }
 
 func (s *streamInfo) UpstreamHostAddress() (string, bool) {
-	return cAPI.HttpGetStringValue(s.request, ValueUpstreamHostAddress)
+	return cAPI.HttpGetStringValue(unsafe.Pointer(s.request), ValueUpstreamHostAddress)
 }
 
 func (s *streamInfo) UpstreamClusterName() (string, bool) {
-	return cAPI.HttpGetStringValue(s.request, ValueUpstreamClusterName)
+	return cAPI.HttpGetStringValue(unsafe.Pointer(s.request), ValueUpstreamClusterName)
 }
 
 type filterState struct {
@@ -220,5 +220,5 @@ func (f *filterState) SetString(key, value string, stateType api.StateType, life
 }
 
 func (f *filterState) GetString(key string) string {
-	return cAPI.HttpGetStringFilterState(f.request, key)
+	return cAPI.HttpGetStringFilterState(unsafe.Pointer(f.request), key)
 }
diff --git a/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go b/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go
index 15f8ba5b7c3e..a790a9007042 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go
@@ -21,7 +21,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 var httpFilterConfigFactoryAndParser = sync.Map{}
diff --git a/contrib/golang/filters/http/source/go/pkg/http/passthrough.go b/contrib/golang/filters/http/source/go/pkg/http/passthrough.go
index bae0ce9c4df8..0fa1cc40b71e 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/passthrough.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/passthrough.go
@@ -18,7 +18,7 @@
 package http
 
 import (
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 type passThroughFilter struct {
diff --git a/contrib/golang/filters/http/source/go/pkg/http/shim.go b/contrib/golang/filters/http/source/go/pkg/http/shim.go
index 53e3499ac5bd..aa91732e47bc 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/shim.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/shim.go
@@ -39,7 +39,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 var ErrDupRequestKey = errors.New("dup request key")
diff --git a/contrib/golang/filters/http/source/go/pkg/http/type.go b/contrib/golang/filters/http/source/go/pkg/http/type.go
index 2011190c7534..33b70cbac909 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/type.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/type.go
@@ -22,7 +22,7 @@ import (
 	"sync"
 	"unsafe"
 
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 // panic error messages when C API return not ok
diff --git a/contrib/golang/filters/http/test/test_data/basic/BUILD b/contrib/golang/filters/http/test/test_data/basic/BUILD
index 8faddf5f67be..f12124da2423 100644
--- a/contrib/golang/filters/http/test/test_data/basic/BUILD
+++ b/contrib/golang/filters/http/test/test_data/basic/BUILD
@@ -14,7 +14,7 @@ go_binary(
     linkmode = "c-shared",
     visibility = ["//visibility:public"],
     deps = [
-        "//contrib/golang/filters/http/source/go/pkg/api",
+        "//contrib/golang/common/go/api",
         "//contrib/golang/filters/http/source/go/pkg/http",
     ],
 )
diff --git a/contrib/golang/filters/http/test/test_data/basic/config.go b/contrib/golang/filters/http/test/test_data/basic/config.go
index 05e2d7eb8bd1..c55011d5054d 100644
--- a/contrib/golang/filters/http/test/test_data/basic/config.go
+++ b/contrib/golang/filters/http/test/test_data/basic/config.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http"
 )
 
diff --git a/contrib/golang/filters/http/test/test_data/basic/filter.go b/contrib/golang/filters/http/test/test_data/basic/filter.go
index 7181f36034be..4774078ebf67 100644
--- a/contrib/golang/filters/http/test/test_data/basic/filter.go
+++ b/contrib/golang/filters/http/test/test_data/basic/filter.go
@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 type filter struct {
diff --git a/contrib/golang/filters/http/test/test_data/echo/BUILD b/contrib/golang/filters/http/test/test_data/echo/BUILD
index dc9baef0c5f2..1764f4ea0d17 100644
--- a/contrib/golang/filters/http/test/test_data/echo/BUILD
+++ b/contrib/golang/filters/http/test/test_data/echo/BUILD
@@ -14,7 +14,7 @@ go_binary(
     linkmode = "c-shared",
     visibility = ["//visibility:public"],
     deps = [
-        "//contrib/golang/filters/http/source/go/pkg/api",
+        "//contrib/golang/common/go/api",
         "//contrib/golang/filters/http/source/go/pkg/http",
         "@com_github_cncf_xds_go//xds/type/v3:type",
         "@org_golang_google_protobuf//types/known/anypb",
diff --git a/contrib/golang/filters/http/test/test_data/echo/config.go b/contrib/golang/filters/http/test/test_data/echo/config.go
index 481175951ab5..18c3b65bc2bd 100644
--- a/contrib/golang/filters/http/test/test_data/echo/config.go
+++ b/contrib/golang/filters/http/test/test_data/echo/config.go
@@ -2,9 +2,10 @@ package main
 
 import (
 	xds "github.com/cncf/xds/go/xds/type/v3"
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http"
 	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
+	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http"
 )
 
 const Name = "echo"
diff --git a/contrib/golang/filters/http/test/test_data/echo/filter.go b/contrib/golang/filters/http/test/test_data/echo/filter.go
index 77c8473654a6..a93b0aecf4ae 100644
--- a/contrib/golang/filters/http/test/test_data/echo/filter.go
+++ b/contrib/golang/filters/http/test/test_data/echo/filter.go
@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"runtime"
 
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 type filter struct {
diff --git a/contrib/golang/filters/http/test/test_data/passthrough/BUILD b/contrib/golang/filters/http/test/test_data/passthrough/BUILD
index 158311129abd..a9dd87316c58 100644
--- a/contrib/golang/filters/http/test/test_data/passthrough/BUILD
+++ b/contrib/golang/filters/http/test/test_data/passthrough/BUILD
@@ -13,7 +13,7 @@ go_binary(
     linkmode = "c-shared",
     visibility = ["//visibility:public"],
     deps = [
-        "//contrib/golang/filters/http/source/go/pkg/api",
+        "//contrib/golang/common/go/api",
         "//contrib/golang/filters/http/source/go/pkg/http",
         "@com_github_cncf_xds_go//udpa/type/v1:type",
         "@org_golang_google_protobuf//types/known/anypb",
diff --git a/contrib/golang/filters/http/test/test_data/routeconfig/BUILD b/contrib/golang/filters/http/test/test_data/routeconfig/BUILD
index 6a7b2aa38c5b..a477ad3ddab3 100644
--- a/contrib/golang/filters/http/test/test_data/routeconfig/BUILD
+++ b/contrib/golang/filters/http/test/test_data/routeconfig/BUILD
@@ -14,7 +14,7 @@ go_binary(
     linkmode = "c-shared",
     visibility = ["//visibility:public"],
     deps = [
-        "//contrib/golang/filters/http/source/go/pkg/api",
+        "//contrib/golang/common/go/api",
         "//contrib/golang/filters/http/source/go/pkg/http",
         "@com_github_cncf_xds_go//xds/type/v3:type",
         "@org_golang_google_protobuf//types/known/anypb",
diff --git a/contrib/golang/filters/http/test/test_data/routeconfig/config.go b/contrib/golang/filters/http/test/test_data/routeconfig/config.go
index 5af08bf8a41e..2c8779ceb8ad 100644
--- a/contrib/golang/filters/http/test/test_data/routeconfig/config.go
+++ b/contrib/golang/filters/http/test/test_data/routeconfig/config.go
@@ -4,9 +4,10 @@ import (
 	"errors"
 
 	xds "github.com/cncf/xds/go/xds/type/v3"
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http"
 	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
+	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http"
 )
 
 const Name = "routeconfig"
diff --git a/contrib/golang/filters/http/test/test_data/routeconfig/filter.go b/contrib/golang/filters/http/test/test_data/routeconfig/filter.go
index 2ade3a1d8d19..6733c8e0e316 100644
--- a/contrib/golang/filters/http/test/test_data/routeconfig/filter.go
+++ b/contrib/golang/filters/http/test/test_data/routeconfig/filter.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 type filter struct {
diff --git a/contrib/golang/filters/network/source/BUILD b/contrib/golang/filters/network/source/BUILD
new file mode 100644
index 000000000000..e4a1151e8a9c
--- /dev/null
+++ b/contrib/golang/filters/network/source/BUILD
@@ -0,0 +1,105 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_contrib_extension",
+    "envoy_cc_library",
+    "envoy_contrib_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+# Golang L4 network filter.
+
+envoy_contrib_package()
+
+envoy_cc_library(
+    name = "golang",
+    srcs = ["golang.cc"],
+    hdrs = ["golang.h"],
+    deps = [
+        ":cgo",
+        ":upstream",
+        "//contrib/golang/common/dso:dso_lib",
+        "@envoy_api//contrib/envoy/extensions/filters/network/golang/v3alpha:pkg_cc_proto",
+    ],
+)
+
+envoy_cc_library(
+    name = "upstream",
+    srcs = ["upstream.cc"],
+    hdrs = [
+        "upstream.h",
+    ],
+    deps = [
+        "//contrib/golang/common/dso:dso_lib",
+        "//envoy/buffer:buffer_interface",
+        "//envoy/event:dispatcher_interface",
+        "//envoy/network:connection_interface",
+        "//envoy/network:filter_interface",
+        "//envoy/tcp:conn_pool_interface",
+        "//envoy/upstream:cluster_manager_interface",
+        "//envoy/upstream:load_balancer_interface",
+        "//envoy/upstream:thread_local_cluster_interface",
+        "//source/common/buffer:buffer_lib",
+        "//source/common/common:assert_lib",
+        "//source/common/common:minimal_logger_lib",
+        "//source/common/common:thread_lib",
+        "//source/common/http:header_map_lib",
+        "//source/common/http:headers_lib",
+        "//source/common/memory:utils_lib",
+        "//source/common/network:connection_lib",
+        "//source/common/tcp:conn_pool_lib",
+        "//source/common/upstream:load_balancer_lib",
+        "//source/extensions/filters/network/common:factory_base_lib",
+        "@envoy_api//contrib/envoy/extensions/filters/network/golang/v3alpha:pkg_cc_proto",
+    ],
+)
+
+envoy_cc_contrib_extension(
+    name = "config",
+    srcs = ["config.cc"],
+    hdrs = [
+        "config.h",
+    ],
+    deps = [
+        ":golang",
+        ":upstream",
+        "//envoy/registry",
+        "//envoy/server:filter_config_interface",
+        "//envoy/server:lifecycle_notifier_interface",
+        "//source/extensions/filters/network:well_known_names",
+        "//source/extensions/filters/network/common:factory_base_lib",
+        "@envoy_api//contrib/envoy/extensions/filters/network/golang/v3alpha:pkg_cc_proto",
+    ],
+)
+
+envoy_cc_contrib_extension(
+    name = "cgo",
+    srcs = ["cgo.cc"],
+    hdrs = [
+        "golang.h",
+        "upstream.h",
+    ],
+    deps = [
+        "//contrib/golang/common/dso:dso_lib",
+        "//envoy/buffer:buffer_interface",
+        "//envoy/event:dispatcher_interface",
+        "//envoy/network:connection_interface",
+        "//envoy/network:filter_interface",
+        "//envoy/registry",
+        "//envoy/server:filter_config_interface",
+        "//envoy/upstream:cluster_manager_interface",
+        "//envoy/upstream:load_balancer_interface",
+        "//envoy/upstream:thread_local_cluster_interface",
+        "//source/common/buffer:buffer_lib",
+        "//source/common/common:assert_lib",
+        "//source/common/common:minimal_logger_lib",
+        "//source/common/http:header_map_lib",
+        "//source/common/http:headers_lib",
+        "//source/common/memory:utils_lib",
+        "//source/common/network:connection_lib",
+        "//source/common/upstream:load_balancer_lib",
+        "//source/extensions/filters/network:well_known_names",
+        "//source/extensions/filters/network/common:factory_base_lib",
+        "@envoy_api//contrib/envoy/extensions/filters/network/golang/v3alpha:pkg_cc_proto",
+    ],
+)
diff --git a/contrib/golang/filters/network/source/cgo.cc b/contrib/golang/filters/network/source/cgo.cc
new file mode 100644
index 000000000000..1f2e1d92fac7
--- /dev/null
+++ b/contrib/golang/filters/network/source/cgo.cc
@@ -0,0 +1,176 @@
+#include "contrib/golang/filters/network/source/golang.h"
+#include "contrib/golang/filters/network/source/upstream.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+
+//
+// These functions may be invoked in another go thread,
+// which means may introduce race between go thread and envoy thread.
+// So we use the envoy's dispatcher in the filter to post it, and make it only executes in the envoy
+// thread.
+//
+
+// Deep copy GoString into std::string, including the string content,
+// it's safe to use it after the current cgo call returns.
+std::string copyGoString(void* str) {
+  if (str == nullptr) {
+    return "";
+  }
+  auto go_str = reinterpret_cast(str);
+  return std::string{go_str->p, size_t(go_str->n)};
+}
+
+enum class ConnectionInfoType {
+  LocalAddr,
+  RemoteAddr,
+};
+
+extern "C" {
+
+//
+// Downstream
+//
+
+CAPIStatus envoyGoFilterDownstreamClose(void* f, int close_type) {
+  auto* wrapper = reinterpret_cast(f);
+  FilterWeakPtr& weak_ptr = wrapper->filter_ptr_;
+  if (FilterSharedPtr f = weak_ptr.lock()) {
+    f->dispatcher()->post([weak_ptr, close_type] {
+      if (FilterSharedPtr filter = weak_ptr.lock()) {
+        filter->close(static_cast(close_type));
+      }
+    });
+    return CAPIStatus::CAPIOK;
+  }
+  return CAPIStatus::CAPIFilterIsGone;
+}
+
+CAPIStatus envoyGoFilterDownstreamWrite(void* f, void* buffer_ptr, int buffer_len, int end_stream) {
+  auto* wrapper = reinterpret_cast(f);
+  FilterWeakPtr& weak_ptr = wrapper->filter_ptr_;
+  if (FilterSharedPtr f = weak_ptr.lock()) {
+    // should do the copy right now, because the 'data' pointer still point at the go's heap
+    Buffer::InstancePtr buffer =
+        std::make_unique(reinterpret_cast(buffer_ptr), buffer_len);
+    // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+    f->dispatcher()->post([weak_ptr, end_stream, buf_raw_ptr = buffer.release()] {
+      Buffer::InstancePtr buf = absl::WrapUnique(buf_raw_ptr);
+      if (FilterSharedPtr filter = weak_ptr.lock()) {
+        filter->write(*buf.get(), end_stream);
+      }
+    });
+    return CAPIStatus::CAPIOK;
+  }
+  return CAPIStatus::CAPIFilterIsGone;
+}
+
+void envoyGoFilterDownstreamFinalize(void* f, int reason) {
+  UNREFERENCED_PARAMETER(reason);
+  auto* wrapper = reinterpret_cast(f);
+  FilterWeakPtr& weak_ptr = wrapper->filter_ptr_;
+  if (FilterSharedPtr filter = weak_ptr.lock()) {
+    // make sure that the deconstructor is also executed by envoy wrk thread.
+    filter->dispatcher()->post([wrapper] { delete wrapper; });
+  } else {
+    // the Filter is already deleted, just release the wrapper.
+    delete wrapper;
+  }
+}
+
+CAPIStatus envoyGoFilterDownstreamInfo(void* f, int info_type, void* ret) {
+  auto* wrapper = reinterpret_cast(f);
+  FilterWeakPtr& weak_ptr = wrapper->filter_ptr_;
+  if (FilterSharedPtr filter = weak_ptr.lock()) {
+    auto* goStr = reinterpret_cast(ret);
+    switch (static_cast(info_type)) {
+    case ConnectionInfoType::LocalAddr:
+      wrapper->str_value_ = filter->getLocalAddrStr();
+      break;
+    case ConnectionInfoType::RemoteAddr:
+      wrapper->str_value_ = filter->getRemoteAddrStr();
+      break;
+    default:
+      PANIC_DUE_TO_CORRUPT_ENUM;
+    }
+    goStr->p = wrapper->str_value_.data();
+    goStr->n = wrapper->str_value_.length();
+    return CAPIStatus::CAPIOK;
+  }
+  return CAPIStatus::CAPIFilterIsGone;
+}
+
+//
+// Upstream
+//
+
+void* envoyGoFilterUpstreamConnect(void* library_id, void* addr) {
+  std::string id = copyGoString(library_id);
+  auto dynamic_lib = Dso::DsoManager::getDsoByID(id);
+  UpstreamConnPtr conn_ptr = std::make_shared(copyGoString(addr), dynamic_lib);
+  // the upstream connect wrapper will be deleted by envoyGoFilterUpstreamFinalize
+  UpstreamConnWrapper* wrapper = new UpstreamConnWrapper(conn_ptr);
+  conn_ptr->setWrapper(wrapper);
+
+  conn_ptr->dispatcher()->post([conn_ptr] { conn_ptr->connect(); });
+
+  return static_cast(wrapper);
+}
+
+CAPIStatus envoyGoFilterUpstreamWrite(void* u, void* buffer_ptr, int buffer_len, int end_stream) {
+  auto* wrapper = reinterpret_cast(u);
+  UpstreamConnPtr& conn_ptr = wrapper->conn_ptr_;
+  // should do the copy right now, because the 'data' pointer still point at the go's heap
+  Buffer::InstancePtr buffer =
+      std::make_unique(reinterpret_cast(buffer_ptr), buffer_len);
+  // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+  conn_ptr->dispatcher()->post([conn_ptr, end_stream, buf_raw_ptr = buffer.release()] {
+    Buffer::InstancePtr buf = absl::WrapUnique(buf_raw_ptr);
+    conn_ptr->write(*buf.get(), end_stream);
+  });
+  return CAPIOK;
+}
+
+CAPIStatus envoyGoFilterUpstreamClose(void* u, int close_type) {
+  auto* wrapper = reinterpret_cast(u);
+  UpstreamConnPtr& conn_ptr = wrapper->conn_ptr_;
+  conn_ptr->dispatcher()->post([conn_ptr, close_type] {
+    conn_ptr->close(static_cast(close_type));
+  });
+  return CAPIOK;
+}
+
+void envoyGoFilterUpstreamFinalize(void* u, int reason) {
+  UNREFERENCED_PARAMETER(reason);
+  auto* wrapper = reinterpret_cast(u);
+  UpstreamConnPtr& conn_ptr = wrapper->conn_ptr_;
+  // make sure that the deconstructor is also executed by envoy wrk thread
+  conn_ptr->dispatcher()->post([wrapper] { delete wrapper; });
+}
+
+CAPIStatus envoyGoFilterUpstreamInfo(void* u, int info_type, void* ret) {
+  auto* wrapper = reinterpret_cast(u);
+  UpstreamConnPtr& conn_ptr = wrapper->conn_ptr_;
+  auto* goStr = reinterpret_cast(ret);
+  switch (static_cast(info_type)) {
+  case ConnectionInfoType::LocalAddr:
+    wrapper->str_value_ = conn_ptr->getLocalAddrStr();
+    break;
+  case ConnectionInfoType::RemoteAddr:
+    wrapper->str_value_ = conn_ptr->getRemoteAddrStr();
+    break;
+  default:
+    PANIC_DUE_TO_CORRUPT_ENUM;
+  }
+  goStr->p = wrapper->str_value_.data();
+  goStr->n = wrapper->str_value_.length();
+  return CAPIStatus::CAPIOK;
+}
+
+} // extern "C"
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/contrib/golang/filters/network/source/config.cc b/contrib/golang/filters/network/source/config.cc
new file mode 100644
index 000000000000..6e87b438dca1
--- /dev/null
+++ b/contrib/golang/filters/network/source/config.cc
@@ -0,0 +1,47 @@
+#include "contrib/golang/filters/network/source/config.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+
+/**
+ * Static registration for the golang filter. @see RegisterFactory.
+ */
+REGISTER_FACTORY(GolangConfigFactory,
+                 Envoy::Server::Configuration::NamedNetworkFilterConfigFactory);
+
+Network::FilterFactoryCb GolangConfigFactory::createFilterFactoryFromProtoTyped(
+    const envoy::extensions::filters::network::golang::v3alpha::Config& proto_config,
+    Server::Configuration::FactoryContext& context) {
+  is_terminal_filter_ = proto_config.is_terminal_filter();
+
+  UpstreamConn::initThreadLocalStorage(context, context.threadLocal());
+
+  FilterConfigSharedPtr config = std::make_shared(proto_config);
+  std::string config_str;
+  auto res = config->pluginConfig().SerializeToString(&config_str);
+  ASSERT(res, "SerializeToString should always successful");
+  std::string library_id = config->libraryID();
+
+  // TODO(antJack): should check the return value and fast fail if we cannot load the dso.
+  Dso::DsoManager::load(library_id, config->libraryPath());
+
+  uint64_t config_id = 0;
+  auto dlib = Dso::DsoManager::getDsoByID(library_id);
+  if (dlib) {
+    config_id = dlib->envoyGoFilterOnNetworkFilterConfig(
+        reinterpret_cast(library_id.data()), library_id.length(),
+        reinterpret_cast(config_str.data()), config_str.length());
+    ASSERT(config_id, "config id should not be zero");
+  }
+
+  return [config, config_id, &context, dlib](Network::FilterManager& filter_manager) -> void {
+    filter_manager.addFilter(std::make_shared(context, config, config_id, dlib));
+  };
+}
+
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/contrib/golang/filters/network/source/config.h b/contrib/golang/filters/network/source/config.h
new file mode 100644
index 000000000000..5652825914a4
--- /dev/null
+++ b/contrib/golang/filters/network/source/config.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#include 
+
+#include "envoy/registry/registry.h"
+#include "envoy/server/filter_config.h"
+#include "envoy/server/lifecycle_notifier.h"
+
+#include "source/extensions/filters/network/common/factory_base.h"
+
+#include "contrib/envoy/extensions/filters/network/golang/v3alpha/golang.pb.h"
+#include "contrib/envoy/extensions/filters/network/golang/v3alpha/golang.pb.validate.h"
+#include "contrib/golang/common/dso/dso.h"
+#include "contrib/golang/filters/network/source/golang.h"
+#include "contrib/golang/filters/network/source/upstream.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+
+constexpr char CanonicalName[] = "envoy.filters.network.golang";
+
+/**
+ * Config registration for the golang filter. @see NamedNetworkFilterConfigFactory.
+ */
+class GolangConfigFactory
+    : public Common::FactoryBase {
+public:
+  GolangConfigFactory() : FactoryBase(CanonicalName) {}
+
+  bool isTerminalFilterByProto(const Protobuf::Message&,
+                               Server::Configuration::ServerFactoryContext&) override {
+    return is_terminal_filter_;
+  }
+
+private:
+  Network::FilterFactoryCb createFilterFactoryFromProtoTyped(
+      const envoy::extensions::filters::network::golang::v3alpha::Config& proto_config,
+      Server::Configuration::FactoryContext& context) override;
+
+  bool is_terminal_filter_{true};
+};
+
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/contrib/golang/filters/network/source/go/pkg/network/BUILD b/contrib/golang/filters/network/source/go/pkg/network/BUILD
new file mode 100644
index 000000000000..182ea4deddfb
--- /dev/null
+++ b/contrib/golang/filters/network/source/go/pkg/network/BUILD
@@ -0,0 +1,38 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+licenses(["notice"])  # Apache 2
+
+go_library(
+    name = "network",
+    srcs = [
+        "api.h",
+        "capi.go",
+        "factory.go",
+        "filter.go",
+        "shim.go",
+    ],
+    cgo = True,
+    clinkopts = select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "-Wl,-unresolved-symbols=ignore-all",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "-Wl,-undefined,dynamic_lookup",
+        ],
+        "@io_bazel_rules_go//go/platform:ios": [
+            "-Wl,-undefined,dynamic_lookup",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "-Wl,-unresolved-symbols=ignore-all",
+        ],
+        "//conditions:default": [],
+    }),
+    importpath = "github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//contrib/golang/common/go/api",
+        "//contrib/golang/common/go/utils",
+        "@org_golang_google_protobuf//proto",
+        "@org_golang_google_protobuf//types/known/anypb",
+    ],
+)
diff --git a/contrib/golang/filters/network/source/go/pkg/network/api.h b/contrib/golang/filters/network/source/go/pkg/network/api.h
new file mode 120000
index 000000000000..6c0942de6865
--- /dev/null
+++ b/contrib/golang/filters/network/source/go/pkg/network/api.h
@@ -0,0 +1 @@
+../../../../../../common/go/api/api.h
\ No newline at end of file
diff --git a/contrib/golang/filters/network/source/go/pkg/network/capi.go b/contrib/golang/filters/network/source/go/pkg/network/capi.go
new file mode 100644
index 000000000000..cea41545e1c6
--- /dev/null
+++ b/contrib/golang/filters/network/source/go/pkg/network/capi.go
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package network
+
+/*
+// ref https://github.com/golang/go/issues/25832
+
+#cgo CFLAGS: -I../api
+#cgo linux LDFLAGS: -Wl,-unresolved-symbols=ignore-all
+#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup
+
+#include 
+#include 
+
+#include "api.h"
+
+*/
+import "C"
+import (
+	"strings"
+	"unsafe"
+
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
+)
+
+var cgoAPI api.NetworkCAPI = &cgoApiImpl{}
+
+func SetCgoAPI(apiImpl api.NetworkCAPI) {
+	if apiImpl != nil {
+		cgoAPI = apiImpl
+	}
+}
+
+type cgoApiImpl struct{}
+
+func (c *cgoApiImpl) DownstreamWrite(f unsafe.Pointer, bufferPtr unsafe.Pointer, bufferLen int, endStream int) {
+	C.envoyGoFilterDownstreamWrite(f, bufferPtr, C.int(bufferLen), C.int(endStream))
+}
+
+func (c *cgoApiImpl) DownstreamClose(f unsafe.Pointer, closeType int) {
+	C.envoyGoFilterDownstreamClose(f, C.int(closeType))
+}
+
+func (c *cgoApiImpl) DownstreamFinalize(f unsafe.Pointer, reason int) {
+	C.envoyGoFilterDownstreamFinalize(f, C.int(reason))
+}
+
+func (c *cgoApiImpl) DownstreamInfo(f unsafe.Pointer, infoType int) string {
+	var info string
+	C.envoyGoFilterDownstreamInfo(f, C.int(infoType), unsafe.Pointer(&info))
+	return strings.Clone(info)
+}
+
+func (c *cgoApiImpl) UpstreamConnect(libraryID string, addr string) unsafe.Pointer {
+	return unsafe.Pointer(C.envoyGoFilterUpstreamConnect(unsafe.Pointer(&libraryID), unsafe.Pointer(&addr)))
+}
+
+func (c *cgoApiImpl) UpstreamWrite(f unsafe.Pointer, bufferPtr unsafe.Pointer, bufferLen int, endStream int) {
+	C.envoyGoFilterUpstreamWrite(f, bufferPtr, C.int(bufferLen), C.int(endStream))
+}
+
+func (c *cgoApiImpl) UpstreamClose(f unsafe.Pointer, closeType int) {
+	C.envoyGoFilterUpstreamClose(f, C.int(closeType))
+}
+
+func (c *cgoApiImpl) UpstreamFinalize(f unsafe.Pointer, reason int) {
+	C.envoyGoFilterUpstreamFinalize(f, C.int(reason))
+}
+
+func (c *cgoApiImpl) UpstreamInfo(f unsafe.Pointer, infoType int) string {
+	var info string
+	C.envoyGoFilterUpstreamInfo(f, C.int(infoType), unsafe.Pointer(&info))
+	return strings.Clone(info)
+}
diff --git a/contrib/golang/filters/network/source/go/pkg/network/factory.go b/contrib/golang/filters/network/source/go/pkg/network/factory.go
new file mode 100644
index 000000000000..8ec48b064685
--- /dev/null
+++ b/contrib/golang/filters/network/source/go/pkg/network/factory.go
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package network
+
+import (
+	"sync"
+
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
+)
+
+type ConfigFactory interface {
+	CreateFactoryFromConfig(config interface{}) FilterFactory
+}
+
+type FilterFactory interface {
+	CreateFilter(cb api.ConnectionCallback) api.DownstreamFilter
+}
+
+type ConfigParser interface {
+	// TODO: should return error when the config is invalid.
+	ParseConfig(any *anypb.Any) interface{}
+}
+
+var (
+	networkFilterConfigFactoryMap = &sync.Map{} // pluginName -> ConfigFactory
+)
+
+func RegisterNetworkFilterConfigFactory(name string, factory ConfigFactory) {
+	if factory != nil {
+		networkFilterConfigFactoryMap.Store(name, factory)
+	}
+}
+
+func GetNetworkFilterConfigFactory(name string) ConfigFactory {
+	if v, ok := networkFilterConfigFactoryMap.Load(name); ok {
+		return v.(ConfigFactory)
+	}
+	return nil
+}
+
+var networkFilterConfigParser ConfigParser = &noopConfigParser{}
+
+func RegisterNetworkFilterConfigParser(parser ConfigParser) {
+	if parser != nil {
+		networkFilterConfigParser = parser
+	}
+}
+
+func GetNetworkFilterConfigParser() ConfigParser {
+	return networkFilterConfigParser
+}
+
+type noopConfigParser struct{}
+
+var _ ConfigParser = (*noopConfigParser)(nil)
+
+func (n *noopConfigParser) ParseConfig(any *anypb.Any) interface{} {
+	return any
+}
diff --git a/contrib/golang/filters/network/source/go/pkg/network/filter.go b/contrib/golang/filters/network/source/go/pkg/network/filter.go
new file mode 100644
index 000000000000..f31f76199d98
--- /dev/null
+++ b/contrib/golang/filters/network/source/go/pkg/network/filter.go
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package network
+
+import (
+	"unsafe"
+
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
+)
+
+type connectionCallback struct {
+	wrapper   unsafe.Pointer
+	writeFunc func(envoyFilter unsafe.Pointer, buffers unsafe.Pointer, buffersNum int, endStream int)
+	closeFunc func(envoyFilter unsafe.Pointer, closeType int)
+	infoFunc  func(envoyFilter unsafe.Pointer, infoType int) string
+}
+
+var _ api.ConnectionCallback = (*connectionCallback)(nil)
+
+func (n *connectionCallback) LocalAddr() string {
+	return n.infoFunc(n.wrapper, int(api.ConnectionInfoLocalAddr))
+}
+
+func (n *connectionCallback) RemoteAddr() string {
+	return n.infoFunc(n.wrapper, int(api.ConnectionInfoRemoteAddr))
+}
+
+func (n *connectionCallback) Write(buffer []byte, endStream bool) {
+	var endStreamInt int
+	if endStream {
+		endStreamInt = 1
+	}
+	n.writeFunc(n.wrapper, unsafe.Pointer(&buffer[0]), len(buffer), endStreamInt)
+}
+
+func (n *connectionCallback) Close(closeType api.ConnectionCloseType) {
+	n.closeFunc(n.wrapper, int(closeType))
+}
diff --git a/contrib/golang/filters/network/source/go/pkg/network/shim.go b/contrib/golang/filters/network/source/go/pkg/network/shim.go
new file mode 100644
index 000000000000..a5caef9b0174
--- /dev/null
+++ b/contrib/golang/filters/network/source/go/pkg/network/shim.go
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package network
+
+/*
+// ref https://github.com/golang/go/issues/25832
+
+#cgo CFLAGS: -I../api
+#cgo linux LDFLAGS: -Wl,-unresolved-symbols=ignore-all
+#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup
+
+#include 
+#include 
+
+#include "api.h"
+*/
+import "C"
+import (
+	"errors"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"unsafe"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/utils"
+)
+
+var (
+	// ref: https://golang.org/cmd/cgo/
+	// The size of any C type T is available as C.sizeof_T, as in C.sizeof_struct_stat.
+	CULLSize uintptr = C.sizeof_ulonglong
+
+	ErrDupRequestKey = errors.New("dup request key")
+
+	DownstreamFilters = &DownstreamFilterMap{}
+	UpstreamFilters   = &UpstreamFilterMap{}
+
+	configIDGenerator uint64
+	configCache       = &sync.Map{} // uint64 -> *anypb.Any
+
+	libraryID string
+)
+
+func CreateUpstreamConn(addr string, filter api.UpstreamFilter) {
+	h := uint64(uintptr(cgoAPI.UpstreamConnect(libraryID, addr)))
+
+	// NP: make sure filter will be deleted.
+	runtime.SetFinalizer(filter, func(f api.UpstreamFilter) {
+		cgoAPI.UpstreamFinalize(unsafe.Pointer(uintptr(h)), api.NormalFinalize)
+	})
+
+	// TODO: handle error
+	_ = UpstreamFilters.StoreFilter(h, filter)
+}
+
+//export envoyGoFilterOnNetworkFilterConfig
+func envoyGoFilterOnNetworkFilterConfig(libraryIDPtr uint64, libraryIDLen uint64, configPtr uint64, configLen uint64) uint64 {
+	buf := utils.BytesToSlice(configPtr, configLen)
+	var any anypb.Any
+	proto.Unmarshal(buf, &any)
+
+	libraryID = strings.Clone(utils.BytesToString(libraryIDPtr, libraryIDLen))
+	configID := atomic.AddUint64(&configIDGenerator, 1)
+	configCache.Store(configID, GetNetworkFilterConfigParser().ParseConfig(&any))
+
+	return configID
+}
+
+//export envoyGoFilterOnDownstreamConnection
+func envoyGoFilterOnDownstreamConnection(wrapper unsafe.Pointer, pluginNamePtr uint64, pluginNameLen uint64,
+	configID uint64) uint64 {
+	filterFactoryConfig, ok := configCache.Load(configID)
+	if !ok {
+		// TODO: panic
+		return uint64(api.NetworkFilterStopIteration)
+	}
+	pluginName := strings.Clone(utils.BytesToString(pluginNamePtr, pluginNameLen))
+	filterFactory := GetNetworkFilterConfigFactory(pluginName).CreateFactoryFromConfig(filterFactoryConfig)
+
+	cb := &connectionCallback{
+		wrapper:   wrapper,
+		writeFunc: cgoAPI.DownstreamWrite,
+		closeFunc: cgoAPI.DownstreamClose,
+		infoFunc:  cgoAPI.DownstreamInfo,
+	}
+	filter := filterFactory.CreateFilter(cb)
+
+	// NP: make sure filter will be deleted.
+	runtime.SetFinalizer(filter, func(ff api.DownstreamFilter) {
+		cgoAPI.DownstreamFinalize(unsafe.Pointer(uintptr(wrapper)), api.NormalFinalize)
+	})
+
+	// TODO: handle error
+	_ = DownstreamFilters.StoreFilter(uint64(uintptr(wrapper)), filter)
+
+	return uint64(filter.OnNewConnection())
+}
+
+//export envoyGoFilterOnDownstreamData
+func envoyGoFilterOnDownstreamData(wrapper unsafe.Pointer, dataSize uint64, dataPtr uint64, sliceNum int, endOfStream int) uint64 {
+	filter := DownstreamFilters.GetFilter(uint64(uintptr(wrapper)))
+
+	var buf []byte
+
+	for i := 0; i < sliceNum; i++ {
+		slicePtr := dataPtr + uint64(i)*uint64(CULLSize+CULLSize)
+		sliceData := *((*uint64)(unsafe.Pointer(uintptr(slicePtr))))
+		sliceLen := *((*uint64)(unsafe.Pointer(uintptr(slicePtr) + CULLSize)))
+
+		data := utils.BytesToSlice(sliceData, sliceLen)
+		buf = append(buf, data...)
+	}
+
+	return uint64(filter.OnData(buf, endOfStream == 1))
+}
+
+//export envoyGoFilterOnDownstreamEvent
+func envoyGoFilterOnDownstreamEvent(wrapper unsafe.Pointer, event int) {
+	filter := DownstreamFilters.GetFilter(uint64(uintptr(wrapper)))
+	e := api.ConnectionEvent(event)
+	filter.OnEvent(e)
+	if e == api.LocalClose || e == api.RemoteClose {
+		DownstreamFilters.DeleteFilter(uint64(uintptr(wrapper)))
+	}
+}
+
+//export envoyGoFilterOnDownstreamWrite
+func envoyGoFilterOnDownstreamWrite(wrapper unsafe.Pointer, dataSize uint64, dataPtr uint64, sliceNum int, endOfStream int) uint64 {
+	filter := DownstreamFilters.GetFilter(uint64(uintptr(wrapper)))
+
+	var buf []byte
+
+	for i := 0; i < sliceNum; i++ {
+		slicePtr := dataPtr + uint64(i)*uint64(CULLSize+CULLSize)
+		sliceData := *((*uint64)(unsafe.Pointer(uintptr(slicePtr))))
+		sliceLen := *((*uint64)(unsafe.Pointer(uintptr(slicePtr) + CULLSize)))
+
+		data := utils.BytesToSlice(sliceData, sliceLen)
+		buf = append(buf, data...)
+	}
+
+	return uint64(filter.OnWrite(buf, endOfStream == 1))
+}
+
+//export envoyGoFilterOnUpstreamConnectionReady
+func envoyGoFilterOnUpstreamConnectionReady(wrapper unsafe.Pointer) {
+	cb := &connectionCallback{
+		wrapper:   wrapper,
+		writeFunc: cgoAPI.UpstreamWrite,
+		closeFunc: cgoAPI.UpstreamClose,
+		infoFunc:  cgoAPI.UpstreamInfo,
+	}
+	filter := UpstreamFilters.GetFilter(uint64(uintptr(wrapper)))
+	filter.OnPoolReady(cb)
+}
+
+//export envoyGoFilterOnUpstreamConnectionFailure
+func envoyGoFilterOnUpstreamConnectionFailure(wrapper unsafe.Pointer, reason int) {
+	filter := UpstreamFilters.GetFilter(uint64(uintptr(wrapper)))
+	filter.OnPoolFailure(api.PoolFailureReason(reason), "")
+	UpstreamFilters.DeleteFilter(uint64(uintptr(wrapper)))
+}
+
+//export envoyGoFilterOnUpstreamData
+func envoyGoFilterOnUpstreamData(wrapper unsafe.Pointer, dataSize uint64, dataPtr uint64, sliceNum int, endOfStream int) {
+	filter := UpstreamFilters.GetFilter(uint64(uintptr(wrapper)))
+
+	var buf []byte
+
+	for i := 0; i < sliceNum; i++ {
+		slicePtr := dataPtr + uint64(i)*uint64(CULLSize+CULLSize)
+		sliceData := *((*uint64)(unsafe.Pointer(uintptr(slicePtr))))
+		sliceLen := *((*uint64)(unsafe.Pointer(uintptr(slicePtr) + CULLSize)))
+
+		data := utils.BytesToSlice(sliceData, sliceLen)
+		buf = append(buf, data...)
+	}
+
+	filter.OnData(buf, endOfStream == 1)
+}
+
+//export envoyGoFilterOnUpstreamEvent
+func envoyGoFilterOnUpstreamEvent(wrapper unsafe.Pointer, event int) {
+	filter := UpstreamFilters.GetFilter(uint64(uintptr(wrapper)))
+	e := api.ConnectionEvent(event)
+	filter.OnEvent(e)
+	if e == api.LocalClose || e == api.RemoteClose {
+		UpstreamFilters.DeleteFilter(uint64(uintptr(wrapper)))
+	}
+}
+
+type DownstreamFilterMap struct {
+	m sync.Map // uint64 -> DownstreamFilter
+}
+
+func (f *DownstreamFilterMap) StoreFilter(key uint64, filter api.DownstreamFilter) error {
+	if _, loaded := f.m.LoadOrStore(key, filter); loaded {
+		return ErrDupRequestKey
+	}
+	return nil
+}
+
+func (f *DownstreamFilterMap) GetFilter(key uint64) api.DownstreamFilter {
+	if v, ok := f.m.Load(key); ok {
+		return v.(api.DownstreamFilter)
+	}
+	return nil
+}
+
+func (f *DownstreamFilterMap) DeleteFilter(key uint64) {
+	f.m.Delete(key)
+}
+
+func (f *DownstreamFilterMap) Clear() {
+	f.m.Range(func(key, _ interface{}) bool {
+		f.m.Delete(key)
+		return true
+	})
+}
+
+type UpstreamFilterMap struct {
+	m sync.Map // uint64 -> UpstreamFilter
+}
+
+func (f *UpstreamFilterMap) StoreFilter(key uint64, filter api.UpstreamFilter) error {
+	if _, loaded := f.m.LoadOrStore(key, filter); loaded {
+		return ErrDupRequestKey
+	}
+	return nil
+}
+
+func (f *UpstreamFilterMap) GetFilter(key uint64) api.UpstreamFilter {
+	if v, ok := f.m.Load(key); ok {
+		return v.(api.UpstreamFilter)
+	}
+	return nil
+}
+
+func (f *UpstreamFilterMap) DeleteFilter(key uint64) {
+	f.m.Delete(key)
+}
+
+func (f *UpstreamFilterMap) Clear() {
+	f.m.Range(func(key, _ interface{}) bool {
+		f.m.Delete(key)
+		return true
+	})
+}
diff --git a/contrib/golang/filters/network/source/golang.cc b/contrib/golang/filters/network/source/golang.cc
new file mode 100644
index 000000000000..59d90cc504e5
--- /dev/null
+++ b/contrib/golang/filters/network/source/golang.cc
@@ -0,0 +1,121 @@
+#include "contrib/golang/filters/network/source/golang.h"
+
+#include 
+
+#include "envoy/network/connection.h"
+
+#include "source/common/common/assert.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+
+FilterConfig::FilterConfig(
+    const envoy::extensions::filters::network::golang::v3alpha::Config& proto_config)
+    : library_id_(proto_config.library_id()), library_path_(proto_config.library_path()),
+      plugin_name_(proto_config.plugin_name()), plugin_config_(proto_config.plugin_config()) {}
+
+void Filter::initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) {
+  read_callbacks_ = &callbacks;
+  dispatcher_ = &read_callbacks_->connection().dispatcher();
+  read_callbacks_->connection().addConnectionCallbacks(*this);
+
+  local_addr_ = callbacks.connection().connectionInfoProvider().localAddress()->asString();
+  addr_ = callbacks.connection().connectionInfoProvider().directRemoteAddress()->asString();
+}
+
+void Filter::close(Network::ConnectionCloseType close_type) {
+  if (closed_) {
+    ENVOY_CONN_LOG(warn, "connection has closed, addr: {}", read_callbacks_->connection(), addr_);
+    return;
+  }
+  ENVOY_CONN_LOG(debug, "close addr: {}, type: {}", read_callbacks_->connection(), addr_,
+                 static_cast(close_type));
+  read_callbacks_->connection().close(close_type);
+}
+
+void Filter::write(Buffer::Instance& buf, bool end_stream) {
+  if (closed_) {
+    ENVOY_CONN_LOG(warn, "connection has closed, addr: {}", read_callbacks_->connection(), addr_);
+    return;
+  }
+  ENVOY_CONN_LOG(debug, "write to addr: {}, len: {}, end: {}", read_callbacks_->connection(), addr_,
+                 buf.length(), end_stream);
+  read_callbacks_->connection().write(buf, end_stream);
+}
+
+Network::FilterStatus Filter::onNewConnection() {
+  ENVOY_CONN_LOG(debug, "onNewConnection, addr: {}, localAddr: {}", read_callbacks_->connection(),
+                 addr_, local_addr_);
+  wrapper_ = new FilterWrapper(weak_from_this());
+
+  return Network::FilterStatus(dynamic_lib_->envoyGoFilterOnDownstreamConnection(
+      wrapper_, reinterpret_cast(plugin_name_.data()), plugin_name_.length(),
+      config_id_));
+}
+
+void Filter::onEvent(Network::ConnectionEvent event) {
+  ENVOY_CONN_LOG(debug, "onEvent addr: {}, event: {}", read_callbacks_->connection(), addr_,
+                 static_cast(event));
+
+  if (event == Network::ConnectionEvent::LocalClose ||
+      event == Network::ConnectionEvent::RemoteClose) {
+    closed_ = true;
+  }
+
+  dynamic_lib_->envoyGoFilterOnDownstreamEvent(wrapper_, static_cast(event));
+}
+
+Network::FilterStatus Filter::onData(Buffer::Instance& data, bool end_stream) {
+  ENVOY_CONN_LOG(debug, "onData, addr: {}, len: {}, end: {}", read_callbacks_->connection(), addr_,
+                 data.length(), end_stream);
+
+  Buffer::RawSliceVector slice_vector = data.getRawSlices();
+  int slice_num = slice_vector.size();
+  unsigned long long* slices = new unsigned long long[2 * slice_num];
+  for (int i = 0; i < slice_num; i++) {
+    const Buffer::RawSlice& s = slice_vector[i];
+    slices[2 * i] = reinterpret_cast(s.mem_);
+    slices[2 * i + 1] = s.len_;
+  }
+
+  auto ret = dynamic_lib_->envoyGoFilterOnDownstreamData(
+      wrapper_, data.length(), reinterpret_cast(slices), slice_num, end_stream);
+
+  // TODO: do not drain buffer by default
+  data.drain(data.length());
+
+  delete[] slices;
+
+  return Network::FilterStatus(ret);
+}
+
+Network::FilterStatus Filter::onWrite(Buffer::Instance& data, bool end_stream) {
+  ENVOY_CONN_LOG(debug, "onWrite, addr: {}, len: {}, end: {}", read_callbacks_->connection(), addr_,
+                 data.length(), end_stream);
+
+  Buffer::RawSliceVector slice_vector = data.getRawSlices();
+  int slice_num = slice_vector.size();
+  unsigned long long* slices = new unsigned long long[2 * slice_num];
+  for (int i = 0; i < slice_num; i++) {
+    const Buffer::RawSlice& s = slice_vector[i];
+    slices[2 * i] = reinterpret_cast(s.mem_);
+    slices[2 * i + 1] = s.len_;
+  }
+
+  auto ret = dynamic_lib_->envoyGoFilterOnDownstreamWrite(
+      wrapper_, data.length(), reinterpret_cast(slices), slice_num, end_stream);
+
+  // TODO: do not drain buffer by default
+  data.drain(data.length());
+
+  delete[] slices;
+
+  return Network::FilterStatus(ret);
+}
+
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/contrib/golang/filters/network/source/golang.h b/contrib/golang/filters/network/source/golang.h
new file mode 100644
index 000000000000..12e30a9940d2
--- /dev/null
+++ b/contrib/golang/filters/network/source/golang.h
@@ -0,0 +1,118 @@
+#pragma once
+
+#include "envoy/buffer/buffer.h"
+#include "envoy/event/dispatcher.h"
+#include "envoy/http/header_map.h"
+#include "envoy/network/connection.h"
+#include "envoy/network/filter.h"
+#include "envoy/ssl/connection.h"
+#include "envoy/tcp/conn_pool.h"
+#include "envoy/upstream/cluster_manager.h"
+
+#include "source/common/buffer/buffer_impl.h"
+#include "source/common/common/logger.h"
+#include "source/common/http/header_map_impl.h"
+#include "source/common/http/headers.h"
+#include "source/common/network/connection_impl.h"
+#include "source/common/upstream/load_balancer_impl.h"
+#include "source/extensions/filters/network/common/factory_base.h"
+
+#include "contrib/envoy/extensions/filters/network/golang/v3alpha/golang.pb.h"
+#include "contrib/golang/common/dso/dso.h"
+#include "contrib/golang/filters/network/source/upstream.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+
+/**
+ * Configuration for the HTTP golang extension filter.
+ */
+class FilterConfig {
+public:
+  FilterConfig(const envoy::extensions::filters::network::golang::v3alpha::Config& proto_config);
+
+  const std::string& libraryID() const { return library_id_; }
+  const std::string& libraryPath() const { return library_path_; }
+  const std::string& pluginName() const { return plugin_name_; }
+  const ProtobufWkt::Any& pluginConfig() const { return plugin_config_; }
+
+private:
+  const std::string library_id_;
+  const std::string library_path_;
+  const std::string plugin_name_;
+  const ProtobufWkt::Any plugin_config_;
+};
+
+using FilterConfigSharedPtr = std::shared_ptr;
+
+struct FilterWrapper;
+
+/**
+ * Implementation of a basic golang filter.
+ */
+class Filter : public Network::Filter,
+               public Network::ConnectionCallbacks,
+               public std::enable_shared_from_this,
+               Logger::Loggable {
+public:
+  explicit Filter(Server::Configuration::FactoryContext& context, FilterConfigSharedPtr config,
+                  uint64_t config_id, Dso::NetworkFilterDsoPtr dynamic_lib)
+      : context_(context), config_(config),
+        config_id_(config_id), plugin_name_{config->pluginName()}, dynamic_lib_(dynamic_lib) {}
+
+  // Network::ReadFilter
+  Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override;
+  Network::FilterStatus onNewConnection() override;
+  void initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) override;
+
+  // Network::WriteFilter
+  Network::FilterStatus onWrite(Buffer::Instance& data, bool end_stream) override;
+
+  // Network::ConnectionCallbacks
+  void onEvent(Network::ConnectionEvent event) override;
+  void onAboveWriteBufferHighWatermark() override {}
+  void onBelowWriteBufferLowWatermark() override {}
+
+  void write(Buffer::Instance& buf, bool end_stream);
+  void close(Network::ConnectionCloseType close_type);
+
+  Event::Dispatcher* dispatcher() { return dispatcher_; }
+  Upstream::ClusterManager& clusterManager() { return context_.clusterManager(); }
+
+  std::string getLocalAddrStr() const { return local_addr_; }
+  std::string getRemoteAddrStr() const { return addr_; };
+
+private:
+  Server::Configuration::FactoryContext& context_;
+  const FilterConfigSharedPtr config_;
+  const uint64_t config_id_;
+  std::string plugin_name_{};
+  Dso::NetworkFilterDsoPtr dynamic_lib_{nullptr};
+  FilterWrapper* wrapper_{nullptr};
+  Event::Dispatcher* dispatcher_{nullptr};
+
+  bool closed_{false};
+  Network::ReadFilterCallbacks* read_callbacks_{};
+  std::string local_addr_{};
+  std::string addr_{};
+};
+
+using FilterSharedPtr = std::shared_ptr;
+using FilterWeakPtr = std::weak_ptr;
+
+struct FilterWrapper {
+public:
+  FilterWrapper(FilterWeakPtr ptr) : filter_ptr_(ptr) {}
+  ~FilterWrapper() = default;
+
+  FilterWeakPtr filter_ptr_{};
+  // anchor a string temporarily, make sure it won't be freed before copied to Go.
+  std::string str_value_;
+};
+
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/contrib/golang/filters/network/source/upstream.cc b/contrib/golang/filters/network/source/upstream.cc
new file mode 100644
index 000000000000..500c287c579e
--- /dev/null
+++ b/contrib/golang/filters/network/source/upstream.cc
@@ -0,0 +1,176 @@
+#include "contrib/golang/filters/network/source/upstream.h"
+
+#include 
+
+#include "envoy/network/connection.h"
+#include "envoy/tcp/conn_pool.h"
+
+#include "source/common/common/assert.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+
+// init function registers each works' dispatcher for load balance when creating upstream conn
+void UpstreamConn::initThreadLocalStorage(Server::Configuration::FactoryContext& context,
+                                          ThreadLocal::SlotAllocator& tls) {
+  // dispatchers array should be init only once.
+  DispatcherStore& store = dispatcherStore();
+  std::call_once(store.init_once_, [&context, &tls, &store]() {
+    // should be the singleton for use by the entire server.
+    ClusterManagerContainer& cluster_manager_container = clusterManagerContainer();
+    cluster_manager_container.cluster_manager_ = &context.clusterManager();
+
+    SlotPtrContainer& slot_ptr_container = slotPtrContainer();
+    slot_ptr_container.slot_ = tls.allocateSlot();
+
+    Thread::ThreadId main_thread_id = context.api().threadFactory().currentThreadId();
+    slot_ptr_container.slot_->set(
+        [&context, main_thread_id,
+         &store](Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr {
+          if (context.api().threadFactory().currentThreadId() == main_thread_id) {
+            return nullptr;
+          }
+
+          {
+            Thread::LockGuard guard(store.lock_);
+            store.dispatchers_.push_back(dispatcher);
+          }
+
+          return nullptr;
+        });
+  });
+}
+
+UpstreamConn::UpstreamConn(std::string addr, Dso::NetworkFilterDsoPtr dynamic_lib,
+                           Event::Dispatcher* dispatcher)
+    : dynamic_lib_(dynamic_lib), dispatcher_(dispatcher), addr_(addr) {
+  if (dispatcher_ == nullptr) {
+    DispatcherStore& store = dispatcherStore();
+    Thread::LockGuard guard(store.lock_);
+    // load balance among each workers' dispatcher
+    ASSERT(!store.dispatchers_.empty());
+    dispatcher_ = &store.dispatchers_[store.dispatcher_idx_++ % store.dispatchers_.size()].get();
+  }
+  header_map_ = Http::createHeaderMap(
+      {{Http::Headers::get().EnvoyOriginalDstHost, addr}});
+}
+
+void UpstreamConn::connect() {
+  ENVOY_LOG(debug, "do connect addr: {}", addr_);
+
+  // TODO(antJack): add support for upstream TLS cluster.
+  static const std::string upstream_cluster = "plainText";
+  ClusterManagerContainer& cluster_manager_container = clusterManagerContainer();
+  Upstream::ThreadLocalCluster* cluster =
+      cluster_manager_container.cluster_manager_->getThreadLocalCluster(upstream_cluster);
+  if (!cluster) {
+    ENVOY_LOG(error, "cluster not found");
+    onPoolFailure(Tcp::ConnectionPool::PoolFailureReason::LocalConnectionFailure,
+                  absl::string_view("cluster not found"), nullptr);
+    return;
+  }
+
+  auto conn_pool = cluster->tcpConnPool(Upstream::ResourcePriority::Default, this);
+  if (!conn_pool) {
+    ENVOY_LOG(error, "no host available for cluster");
+    // prevent golang from blocking on connection result channel
+    onPoolFailure(Tcp::ConnectionPool::PoolFailureReason::LocalConnectionFailure,
+                  absl::string_view("no host available"), nullptr);
+    return;
+  }
+
+  Tcp::ConnectionPool::Cancellable* cancellable = conn_pool.value().newConnection(*this);
+  if (cancellable) {
+    handler_ = cancellable;
+  }
+}
+
+void UpstreamConn::write(Buffer::Instance& buf, bool end_stream) {
+  if (closed_) {
+    ENVOY_LOG(warn, "connection has closed, addr: {}", addr_);
+    return;
+  }
+  ENVOY_CONN_LOG(debug, "write to addr: {}, len: {}, end: {}", conn_->connection(), addr_,
+                 buf.length(), end_stream);
+  ASSERT(conn_ != nullptr);
+  conn_->connection().write(buf, end_stream);
+}
+
+void UpstreamConn::close(Network::ConnectionCloseType close_type) {
+  if (closed_) {
+    ENVOY_LOG(warn, "connection has closed, addr: {}", addr_);
+    return;
+  }
+  ENVOY_CONN_LOG(debug, "close addr: {}, type: {}", conn_->connection(), addr_,
+                 static_cast(close_type));
+  ASSERT(conn_ != nullptr);
+  conn_->connection().close(close_type);
+}
+
+void UpstreamConn::onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr&& conn,
+                               Upstream::HostDescriptionConstSharedPtr host) {
+  ENVOY_CONN_LOG(debug, "onPoolReady, addr: {}", conn->connection(), addr_);
+  if (handler_) {
+    handler_ = nullptr;
+  }
+
+  conn_ = std::move(conn);
+  host_ = host;
+  conn_->addUpstreamCallbacks(*this);
+  remote_addr_ = conn_->connection().connectionInfoProvider().directRemoteAddress()->asString();
+
+  dynamic_lib_->envoyGoFilterOnUpstreamConnectionReady(wrapper_);
+}
+
+void UpstreamConn::onPoolFailure(Tcp::ConnectionPool::PoolFailureReason reason,
+                                 absl::string_view transport_failure_reason,
+                                 Upstream::HostDescriptionConstSharedPtr) {
+  ENVOY_LOG(error, "onPoolFailure, addr: {}, reason: {}, {}", addr_, int(reason),
+            std::string(transport_failure_reason));
+  if (handler_) {
+    handler_ = nullptr;
+  }
+
+  dynamic_lib_->envoyGoFilterOnUpstreamConnectionFailure(wrapper_, static_cast(reason));
+}
+
+void UpstreamConn::onEvent(Network::ConnectionEvent event) {
+  ENVOY_CONN_LOG(debug, "onEvent addr: {}, event: {}", conn_->connection(), addr_,
+                 static_cast(event));
+  if (event == Network::ConnectionEvent::LocalClose ||
+      event == Network::ConnectionEvent::RemoteClose) {
+    closed_ = true;
+    conn_ = nullptr;
+  }
+
+  dynamic_lib_->envoyGoFilterOnUpstreamEvent(wrapper_, static_cast(event));
+}
+
+void UpstreamConn::onUpstreamData(Buffer::Instance& data, bool end_stream) {
+  ENVOY_CONN_LOG(debug, "onData, addr: {}, len: {}, end: {}", conn_->connection(), addr_,
+                 data.length(), end_stream);
+
+  Buffer::RawSliceVector slice_vector = data.getRawSlices();
+  int slice_num = slice_vector.size();
+  unsigned long long* slices = new unsigned long long[2 * slice_num];
+  for (int i = 0; i < slice_num; i++) {
+    const Buffer::RawSlice& s = slice_vector[i];
+    slices[2 * i] = reinterpret_cast(s.mem_);
+    slices[2 * i + 1] = s.len_;
+  }
+
+  dynamic_lib_->envoyGoFilterOnUpstreamData(
+      wrapper_, data.length(), reinterpret_cast(slices), slice_num, end_stream);
+
+  // TODO: do not drain buffer by default
+  data.drain(data.length());
+
+  delete[] slices;
+}
+
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/contrib/golang/filters/network/source/upstream.h b/contrib/golang/filters/network/source/upstream.h
new file mode 100644
index 000000000000..8573f83b9906
--- /dev/null
+++ b/contrib/golang/filters/network/source/upstream.h
@@ -0,0 +1,128 @@
+#pragma once
+
+#include 
+#include 
+
+#include "envoy/buffer/buffer.h"
+#include "envoy/event/dispatcher.h"
+#include "envoy/http/header_map.h"
+#include "envoy/network/connection.h"
+#include "envoy/network/filter.h"
+#include "envoy/tcp/conn_pool.h"
+#include "envoy/upstream/cluster_manager.h"
+
+#include "source/common/buffer/buffer_impl.h"
+#include "source/common/common/logger.h"
+#include "source/common/common/thread.h"
+#include "source/common/http/header_map_impl.h"
+#include "source/common/http/headers.h"
+#include "source/common/memory/utils.h"
+#include "source/common/network/connection_impl.h"
+#include "source/common/upstream/load_balancer_impl.h"
+#include "source/extensions/filters/network/common/factory_base.h"
+
+#include "contrib/envoy/extensions/filters/network/golang/v3alpha/golang.pb.h"
+#include "contrib/golang/common/dso/dso.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+
+struct UpstreamConnWrapper;
+
+class UpstreamConn : public Tcp::ConnectionPool::Callbacks,
+                     public Upstream::LoadBalancerContextBase,
+                     public Tcp::ConnectionPool::UpstreamCallbacks,
+                     public std::enable_shared_from_this,
+                     Logger::Loggable {
+public:
+  UpstreamConn(std::string addr, Dso::NetworkFilterDsoPtr dynamic_lib,
+               Event::Dispatcher* dispatcher = nullptr);
+  ~UpstreamConn() override {
+    if (handler_) {
+      handler_->cancel(Tcp::ConnectionPool::CancelPolicy::Default);
+    }
+  }
+
+  static void initThreadLocalStorage(Server::Configuration::FactoryContext& context,
+                                     ThreadLocal::SlotAllocator& tls);
+
+  // Tcp::ConnectionPool::Callbacks
+  void onPoolFailure(Tcp::ConnectionPool::PoolFailureReason reason,
+                     absl::string_view transport_failure_reason,
+                     Upstream::HostDescriptionConstSharedPtr host) override;
+
+  void onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr&& conn,
+                   Upstream::HostDescriptionConstSharedPtr host) override;
+
+  // Tcp::ConnectionPool::UpstreamCallbacks
+  void onUpstreamData(Buffer::Instance& data, bool end_stream) override;
+  void onAboveWriteBufferHighWatermark() override {}
+  void onBelowWriteBufferLowWatermark() override {}
+  void onEvent(Network::ConnectionEvent event) override;
+
+  // Upstream::LoadBalancerContextBase
+  const Http::RequestHeaderMap* downstreamHeaders() const override { return header_map_.get(); };
+
+  void connect();
+  void write(Buffer::Instance& buf, bool end_stream);
+  void close(Network::ConnectionCloseType close_type);
+
+  Event::Dispatcher* dispatcher() { return dispatcher_; }
+  void setWrapper(UpstreamConnWrapper* wrapper) { wrapper_ = wrapper; };
+
+  std::string getLocalAddrStr() const { return addr_; }
+  std::string getRemoteAddrStr() const { return remote_addr_; };
+
+private:
+  struct DispatcherStore {
+    std::vector> dispatchers_ ABSL_GUARDED_BY(lock_){};
+    int dispatcher_idx_ ABSL_GUARDED_BY(lock_){0};
+    Thread::MutexBasicLockable lock_{};
+    std::once_flag init_once_{};
+  };
+  static DispatcherStore& dispatcherStore() { MUTABLE_CONSTRUCT_ON_FIRST_USE(DispatcherStore); }
+
+  struct SlotPtrContainer {
+    ThreadLocal::SlotPtr slot_{nullptr};
+  };
+  static SlotPtrContainer& slotPtrContainer() { MUTABLE_CONSTRUCT_ON_FIRST_USE(SlotPtrContainer); }
+
+  struct ClusterManagerContainer {
+    Upstream::ClusterManager* cluster_manager_{nullptr};
+  };
+  static ClusterManagerContainer& clusterManagerContainer() {
+    MUTABLE_CONSTRUCT_ON_FIRST_USE(ClusterManagerContainer);
+  }
+
+  Dso::NetworkFilterDsoPtr dynamic_lib_{nullptr};
+  UpstreamConnWrapper* wrapper_{nullptr};
+  Event::Dispatcher* dispatcher_{nullptr};
+  std::unique_ptr header_map_{nullptr};
+  Tcp::ConnectionPool::ConnectionDataPtr conn_{nullptr};
+  Upstream::HostDescriptionConstSharedPtr host_{nullptr};
+  Tcp::ConnectionPool::Cancellable* handler_{nullptr};
+  bool closed_{false};
+  std::string addr_{};
+  std::string remote_addr_{};
+};
+
+using UpstreamConnPtr = std::shared_ptr;
+
+struct UpstreamConnWrapper {
+public:
+  UpstreamConnWrapper(UpstreamConnPtr ptr) : conn_ptr_(ptr) {}
+  ~UpstreamConnWrapper() = default;
+
+  // Must be strong shared_ptr, otherwise the UpstreamConn will be released immediately since we do
+  // not have any other place to keep strong reference of the UpstreamConn.
+  UpstreamConnPtr conn_ptr_{};
+  // anchor a string temporarily, make sure it won't be freed before copied to Go.
+  std::string str_value_;
+};
+
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/contrib/golang/filters/network/test/BUILD b/contrib/golang/filters/network/test/BUILD
new file mode 100644
index 000000000000..16d212c1bfdb
--- /dev/null
+++ b/contrib/golang/filters/network/test/BUILD
@@ -0,0 +1,58 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_test",
+    "envoy_contrib_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_contrib_package()
+
+envoy_cc_test(
+    name = "config_test",
+    srcs = ["config_test.cc"],
+    data = [
+        "//contrib/golang/filters/network/test/test_data:filter.so",
+    ],
+    deps = [
+        "//contrib/golang/filters/network/source:config",
+        "//test/mocks/server:factory_context_mocks",
+        "//test/test_common:utility_lib",
+    ],
+)
+
+envoy_cc_test(
+    name = "filter_test",
+    srcs = ["filter_test.cc"],
+    deps = [
+        "//contrib/golang/common/dso/test:dso_mocks",
+        "//contrib/golang/filters/network/source:golang",
+        "//test/mocks/api:api_mocks",
+        "//test/mocks/network:network_mocks",
+        "//test/mocks/server:factory_context_mocks",
+        "//test/mocks/ssl:ssl_mocks",
+        "//test/mocks/thread_local:thread_local_mocks",
+        "//test/mocks/upstream:cluster_manager_mocks",
+        "//test/test_common:logging_lib",
+        "//test/test_common:test_runtime_lib",
+        "//test/test_common:utility_lib",
+    ],
+)
+
+envoy_cc_test(
+    name = "upstream_test",
+    srcs = ["upstream_test.cc"],
+    deps = [
+        "//contrib/golang/common/dso/test:dso_mocks",
+        "//contrib/golang/filters/network/source:upstream",
+        "//test/mocks/api:api_mocks",
+        "//test/mocks/network:network_mocks",
+        "//test/mocks/server:factory_context_mocks",
+        "//test/mocks/ssl:ssl_mocks",
+        "//test/mocks/thread_local:thread_local_mocks",
+        "//test/mocks/upstream:cluster_manager_mocks",
+        "//test/test_common:logging_lib",
+        "//test/test_common:test_runtime_lib",
+        "//test/test_common:utility_lib",
+    ],
+)
diff --git a/contrib/golang/filters/network/test/config_test.cc b/contrib/golang/filters/network/test/config_test.cc
new file mode 100644
index 000000000000..e60d86f4a550
--- /dev/null
+++ b/contrib/golang/filters/network/test/config_test.cc
@@ -0,0 +1,115 @@
+#include 
+
+#include "envoy/registry/registry.h"
+
+#include "test/mocks/server/factory_context.h"
+#include "test/test_common/environment.h"
+#include "test/test_common/utility.h"
+
+#include "absl/strings/str_format.h"
+#include "contrib/golang/filters/network/source/config.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using testing::_;
+using testing::ReturnRef;
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+namespace {
+
+class MockThreadFactory : public Thread::ThreadFactory {
+public:
+  MOCK_METHOD(Thread::ThreadPtr, createThread, (std::function, Thread::OptionsOptConstRef));
+  MOCK_METHOD(Thread::ThreadId, currentThreadId, ());
+};
+
+class GolangFilterConfigTestBase {
+public:
+  void testConfig(envoy::extensions::filters::network::golang::v3alpha::Config& config) {
+    EXPECT_CALL(slot_allocator_, allocateSlot())
+        .WillRepeatedly(Invoke(&slot_allocator_, &ThreadLocal::MockInstance::allocateSlotMock));
+    ON_CALL(context_, threadLocal()).WillByDefault(ReturnRef(slot_allocator_));
+    ON_CALL(context_.api_, threadFactory()).WillByDefault(ReturnRef(thread_factory_));
+
+    Network::FilterFactoryCb cb;
+    EXPECT_NO_THROW({ cb = factory_.createFilterFactoryFromProto(config, context_); });
+    Network::MockConnection connection;
+    EXPECT_CALL(connection, addFilter(_));
+    cb(connection);
+  }
+
+  NiceMock context_;
+  NiceMock thread_factory_;
+  ThreadLocal::MockInstance slot_allocator_;
+  GolangConfigFactory factory_;
+};
+
+class GolangFilterConfigTest : public GolangFilterConfigTestBase, public testing::Test {
+public:
+  ~GolangFilterConfigTest() override = default;
+};
+
+TEST(GolangConfigFactoryTest, InvalidateEmptyConfig) {
+  NiceMock context;
+  EXPECT_THROW_WITH_REGEX(
+      GolangConfigFactory().createFilterFactoryFromProto(
+          envoy::extensions::filters::network::golang::v3alpha::Config(), context),
+      Envoy::ProtoValidationException,
+      "ConfigValidationError.LibraryId: value length must be at least 1 characters");
+}
+
+TEST_F(GolangFilterConfigTest, GolangFilterWithValidConfig) {
+  const auto yaml_fmt = R"EOF(
+  library_id: %s
+  library_path: %s
+  is_terminal_filter: true
+  plugin_name: xxx
+  plugin_config:
+    "@type": type.googleapis.com/udpa.type.v1.TypedStruct
+    type_url: typexx
+    value:
+        key: value
+        int: 10
+  )EOF";
+
+  auto yaml_string = absl::StrFormat(
+      yaml_fmt, "test",
+      TestEnvironment::substitute(
+          "{{ test_rundir }}/contrib/golang/filters/network/test/test_data/filter.so"));
+  envoy::extensions::filters::network::golang::v3alpha::Config proto_config;
+  TestUtility::loadFromYaml(yaml_string, proto_config);
+  auto plugin_config = proto_config.plugin_config();
+  std::string str;
+  EXPECT_TRUE(plugin_config.SerializeToString(&str));
+
+  testConfig(proto_config);
+}
+
+TEST_F(GolangFilterConfigTest, GolangFilterWithNilPluginConfig) {
+  const auto yaml_fmt = R"EOF(
+  library_id: %s
+  library_path: %s
+  plugin_name: xxx
+  )EOF";
+
+  auto yaml_string = absl::StrFormat(
+      yaml_fmt, "test",
+      TestEnvironment::substitute(
+          "{{ test_rundir }}/contrib/golang/filters/network/test/test_data/filter.so"));
+  envoy::extensions::filters::network::golang::v3alpha::Config proto_config;
+  TestUtility::loadFromYaml(yaml_string, proto_config);
+  auto plugin_config = proto_config.plugin_config();
+  std::string str;
+  EXPECT_TRUE(plugin_config.SerializeToString(&str));
+
+  testConfig(proto_config);
+}
+
+} // namespace
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/contrib/golang/filters/network/test/filter_test.cc b/contrib/golang/filters/network/test/filter_test.cc
new file mode 100644
index 000000000000..57f85caa1ec4
--- /dev/null
+++ b/contrib/golang/filters/network/test/filter_test.cc
@@ -0,0 +1,112 @@
+#include 
+
+#include "envoy/registry/registry.h"
+
+#include "test/mocks/server/factory_context.h"
+#include "test/test_common/environment.h"
+#include "test/test_common/utility.h"
+
+#include "absl/strings/str_format.h"
+#include "contrib/golang/common/dso/test/mocks.h"
+#include "contrib/golang/filters/network/source/golang.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using testing::_;
+using testing::Return;
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+namespace {
+
+class FilterTest : public testing::Test {
+public:
+  FilterTest() { ENVOY_LOG_MISC(info, "test"); }
+  ~FilterTest() override = default;
+
+  void initialize() {
+    proto_config_ = prepareProtoConfig();
+    config_ = std::make_shared(proto_config_);
+    dso_ = std::make_shared();
+    filter_ = std::make_shared(context_, config_, 1, dso_);
+    filter_->initializeReadFilterCallbacks(filter_callbacks_);
+  }
+
+  envoy::extensions::filters::network::golang::v3alpha::Config prepareProtoConfig() {
+    const auto yaml_fmt = R"EOF(
+  library_id: %s
+  library_path: %s
+  is_terminal_filter: true
+  plugin_name: xxx
+  plugin_config:
+    "@type": type.googleapis.com/udpa.type.v1.TypedStruct
+    type_url: typexx
+    value:
+        key: value
+        int: 10
+  )EOF";
+
+    auto yaml_string = absl::StrFormat(
+        yaml_fmt, "test",
+        TestEnvironment::substitute(
+            "{{ test_rundir }}/contrib/golang/filters/network/test/test_data/filter.so"));
+    envoy::extensions::filters::network::golang::v3alpha::Config proto_config;
+    TestUtility::loadFromYaml(yaml_string, proto_config);
+    return proto_config;
+  }
+
+  FilterConfigSharedPtr config_;
+  envoy::extensions::filters::network::golang::v3alpha::Config proto_config_;
+  NiceMock context_;
+  std::shared_ptr dso_;
+  NiceMock filter_callbacks_;
+  FilterSharedPtr filter_;
+};
+
+TEST_F(FilterTest, InvokeDsoOnEventOrData) {
+  initialize();
+
+  EXPECT_CALL(*dso_.get(), envoyGoFilterOnDownstreamConnection(_, _, _, _));
+  filter_->onNewConnection();
+
+  EXPECT_CALL(*dso_.get(),
+              envoyGoFilterOnDownstreamEvent(_, GoInt(Network::ConnectionEvent::Connected)));
+  filter_->onEvent(Network::ConnectionEvent::Connected);
+
+  Buffer::OwnedImpl someData("123");
+  EXPECT_CALL(*dso_.get(), envoyGoFilterOnDownstreamData(_, someData.length(), _, _, _))
+      .WillOnce(Return(GoUint64(Network::FilterStatus::Continue)));
+  EXPECT_EQ(Network::FilterStatus::Continue, filter_->onData(someData, false));
+
+  EXPECT_CALL(*dso_.get(), envoyGoFilterOnDownstreamWrite(_, someData.length(), _, _, _))
+      .WillOnce(Return(GoUint64(Network::FilterStatus::Continue)));
+  EXPECT_EQ(Network::FilterStatus::Continue, filter_->onWrite(someData, false));
+}
+
+TEST_F(FilterTest, WriteAndClose) {
+  initialize();
+
+  Buffer::OwnedImpl someData("123");
+  EXPECT_CALL(filter_callbacks_.connection_, write(_, false));
+  filter_->write(someData, false);
+
+  EXPECT_CALL(filter_callbacks_.connection_, close(_));
+  EXPECT_CALL(*dso_.get(), envoyGoFilterOnDownstreamEvent(_, _));
+  filter_->close(Network::ConnectionCloseType::NoFlush);
+
+  // once filter got closed, should not write any more
+  EXPECT_CALL(filter_callbacks_.connection_, write(_, _)).Times(0);
+  filter_->write(someData, false);
+
+  // once filter got closed, should not close any more
+  EXPECT_CALL(filter_callbacks_.connection_, close(_)).Times(0);
+  filter_->close(Network::ConnectionCloseType::NoFlush);
+}
+
+} // namespace
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/contrib/golang/filters/network/test/test_data/BUILD b/contrib/golang/filters/network/test/test_data/BUILD
new file mode 100644
index 000000000000..d21b0925ef86
--- /dev/null
+++ b/contrib/golang/filters/network/test/test_data/BUILD
@@ -0,0 +1,21 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary")
+
+licenses(["notice"])  # Apache 2
+
+go_binary(
+    name = "filter.so",
+    srcs = [
+        "filter.go",
+    ],
+    out = "filter.so",
+    cgo = True,
+    importpath = "github.com/envoyproxy/envoy/contrib/golang/filters/network/test/test_data",
+    linkmode = "c-shared",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//contrib/golang/common/go/api",
+        "//contrib/golang/filters/network/source/go/pkg/network",
+        "@com_github_cncf_xds_go//udpa/type/v1:type",
+        "@org_golang_google_protobuf//types/known/anypb",
+    ],
+)
diff --git a/contrib/golang/filters/network/test/test_data/filter.go b/contrib/golang/filters/network/test/test_data/filter.go
new file mode 100644
index 000000000000..76eb8e885ee7
--- /dev/null
+++ b/contrib/golang/filters/network/test/test_data/filter.go
@@ -0,0 +1,45 @@
+package main
+
+import (
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
+	"github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network"
+)
+
+func init() {
+	network.RegisterNetworkFilterConfigFactory("", simpleConfigFactory)
+}
+
+var simpleConfigFactory = &SimpleConfigFactory{}
+
+type SimpleConfigFactory struct{}
+
+func (f *SimpleConfigFactory) CreateFactoryFromConfig(config interface{}) network.FilterFactory {
+	return &SimpleFilterFactory{}
+}
+
+type SimpleFilterFactory struct{}
+
+func (f *SimpleFilterFactory) CreateFilter(cb api.ConnectionCallback) api.DownstreamFilter {
+	return &SimpleFilter{}
+}
+
+type SimpleFilter struct{}
+
+func (f *SimpleFilter) OnNewConnection() api.FilterStatus {
+	panic("implement me")
+}
+
+func (f *SimpleFilter) OnData(buffer []byte, endOfStream bool) api.FilterStatus {
+	panic("implement me")
+}
+
+func (f *SimpleFilter) OnEvent(event api.ConnectionEvent) {
+	panic("implement me")
+}
+
+func (f *SimpleFilter) OnWrite(buffer []byte, endOfStream bool) api.FilterStatus {
+	panic("implement me")
+}
+
+func main() {
+}
diff --git a/contrib/golang/filters/network/test/test_data/go.mod b/contrib/golang/filters/network/test/test_data/go.mod
new file mode 100644
index 000000000000..8c20e9bd14f6
--- /dev/null
+++ b/contrib/golang/filters/network/test/test_data/go.mod
@@ -0,0 +1,9 @@
+module github.com/envoyproxy/envoy/contrib/golang/filters/network/test/test_data
+
+go 1.18
+
+require github.com/envoyproxy/envoy v1.24.0
+
+require google.golang.org/protobuf v1.30.0 // indirect
+
+replace github.com/envoyproxy/envoy => ../../../../../../
diff --git a/contrib/golang/filters/network/test/upstream_test.cc b/contrib/golang/filters/network/test/upstream_test.cc
new file mode 100644
index 000000000000..ecb5583c672c
--- /dev/null
+++ b/contrib/golang/filters/network/test/upstream_test.cc
@@ -0,0 +1,126 @@
+#include 
+
+#include "envoy/registry/registry.h"
+
+#include "test/mocks/server/factory_context.h"
+#include "test/test_common/environment.h"
+#include "test/test_common/utility.h"
+
+#include "absl/strings/str_format.h"
+#include "contrib/golang/common/dso/test/mocks.h"
+#include "contrib/golang/filters/network/source/upstream.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using testing::_;
+using testing::ReturnRef;
+
+namespace Envoy {
+namespace Extensions {
+namespace NetworkFilters {
+namespace Golang {
+namespace {
+
+class MockThreadFactory : public Thread::ThreadFactory {
+public:
+  MOCK_METHOD(Thread::ThreadPtr, createThread, (std::function, Thread::OptionsOptConstRef));
+  MOCK_METHOD(Thread::ThreadId, currentThreadId, ());
+};
+
+class UpstreamConnTest : public testing::Test {
+public:
+  UpstreamConnTest() { ENVOY_LOG_MISC(info, "test"); }
+  ~UpstreamConnTest() override = default;
+
+  void initialize() {
+    EXPECT_CALL(slot_allocator_, allocateSlot())
+        .WillRepeatedly(Invoke(&slot_allocator_, &ThreadLocal::MockInstance::allocateSlotMock));
+    context_.cluster_manager_.initializeClusters({"plainText"}, {});
+    context_.cluster_manager_.initializeThreadLocalClusters({"plainText"});
+    ON_CALL(context_.api_, threadFactory()).WillByDefault(ReturnRef(thread_factory_));
+    UpstreamConn::initThreadLocalStorage(context_, slot_allocator_);
+    dso_ = std::make_shared();
+    upConn_ = std::make_shared(addr_, dso_, &dispatcher_);
+  }
+
+  ThreadLocal::MockInstance slot_allocator_;
+  NiceMock thread_factory_;
+  NiceMock context_;
+  std::shared_ptr dso_;
+  NiceMock dispatcher_;
+  const std::string addr_{"127.0.0.1:8080"};
+  UpstreamConnPtr upConn_;
+
+  NiceMock upstream_connection_;
+};
+
+TEST_F(UpstreamConnTest, ConnectUpstream) {
+  initialize();
+
+  EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_))
+      .WillOnce(
+          Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* {
+            context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb);
+            context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady(
+                upstream_connection_);
+            return nullptr;
+          }));
+  EXPECT_CALL(*dso_.get(), envoyGoFilterOnUpstreamConnectionReady(_));
+  upConn_->connect();
+
+  EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_))
+      .WillOnce(
+          Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* {
+            context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb);
+            context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure(
+                ConnectionPool::PoolFailureReason::RemoteConnectionFailure, true);
+            return nullptr;
+          }));
+  EXPECT_CALL(*dso_.get(),
+              envoyGoFilterOnUpstreamConnectionFailure(
+                  _, GoInt(ConnectionPool::PoolFailureReason::RemoteConnectionFailure)));
+  upConn_->connect();
+}
+
+TEST_F(UpstreamConnTest, InvokeDsoOnEventOrData) {
+  initialize();
+  EXPECT_CALL(*dso_.get(),
+              envoyGoFilterOnUpstreamEvent(_, GoInt(Network::ConnectionEvent::Connected)));
+  upConn_->onEvent(Network::ConnectionEvent::Connected);
+
+  Buffer::OwnedImpl someData("123");
+  EXPECT_CALL(*dso_.get(), envoyGoFilterOnUpstreamData(_, someData.length(), _, _, _));
+  upConn_->onUpstreamData(someData, false);
+}
+
+TEST_F(UpstreamConnTest, WriteAndClose) {
+  initialize();
+
+  EXPECT_CALL(*dso_.get(), envoyGoFilterOnUpstreamConnectionReady(_));
+  auto data = std::make_unique>();
+  EXPECT_CALL(*data, connection()).WillRepeatedly(ReturnRef(upstream_connection_));
+  upConn_->onPoolReady(std::move(data), nullptr);
+
+  Buffer::OwnedImpl someData("123");
+  EXPECT_CALL(upstream_connection_, write(_, false));
+  upConn_->write(someData, false);
+
+  EXPECT_CALL(upstream_connection_, close(_));
+  EXPECT_CALL(*dso_.get(), envoyGoFilterOnUpstreamEvent(_, _));
+  upConn_->close(Network::ConnectionCloseType::NoFlush);
+  upConn_->onEvent(Network::ConnectionEvent::RemoteClose);
+
+  // once upstream conn got closed, should not write any more
+  EXPECT_CALL(upstream_connection_, write(_, _)).Times(0);
+  upConn_->write(someData, false);
+
+  // once upstream conn got closed, should not close any more
+  EXPECT_CALL(upstream_connection_, close(_)).Times(0);
+  upConn_->close(Network::ConnectionCloseType::NoFlush);
+}
+
+} // namespace
+} // namespace Golang
+} // namespace NetworkFilters
+} // namespace Extensions
+} // namespace Envoy
diff --git a/docs/root/_configs/go/network.yaml b/docs/root/_configs/go/network.yaml
new file mode 100644
index 000000000000..4a8884ccaa41
--- /dev/null
+++ b/docs/root/_configs/go/network.yaml
@@ -0,0 +1,29 @@
+# envoy demo with golang extension enabled
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 10000
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.golang
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.golang.v3alpha.Config
+          is_terminal_filter: true
+          library_id: simple
+          library_path: "/lib/simple.so"
+          plugin_name: simple
+          plugin_config:
+            "@type": type.googleapis.com/xds.type.v3.TypedStruct
+            value:
+              echo_server_addr: echo_service
+  clusters:
+  - name: plainText
+    type: ORIGINAL_DST
+    lb_policy: CLUSTER_PROVIDED
+    connect_timeout: 10s
+    original_dst_lb_config:
+      use_http_header: true
+    per_connection_buffer_limit_bytes: 10485760
diff --git a/docs/root/configuration/http/http_filters/golang_filter.rst b/docs/root/configuration/http/http_filters/golang_filter.rst
index 9a1106d8bfcd..a5faf13565c7 100644
--- a/docs/root/configuration/http/http_filters/golang_filter.rst
+++ b/docs/root/configuration/http/http_filters/golang_filter.rst
@@ -22,7 +22,7 @@ for more details on the filter's implementation.
 Developing a Go plugin
 ----------------------
 
-Envoy's Go plugins must implement the :repo:`StreamFilter API `.
+Envoy's Go plugins must implement the :repo:`StreamFilter API `.
 
 Building a Go plugin
 ~~~~~~~~~~~~~~~~~~~~
@@ -93,7 +93,7 @@ Below is a very simple example of how such a plugin might be configured in Envoy
    :emphasize-lines: 7-10
    :caption: :download:`golang-with-config.yaml `
 
-See the :repo:`StreamFilter API `
+See the :repo:`StreamFilter API `
 for more information about how the plugin's configuration data can be accessed.
 
 Per-route plugin configuration
diff --git a/docs/root/configuration/listeners/network_filters/golang_filter.rst b/docs/root/configuration/listeners/network_filters/golang_filter.rst
new file mode 100644
index 000000000000..9fd2153535a0
--- /dev/null
+++ b/docs/root/configuration/listeners/network_filters/golang_filter.rst
@@ -0,0 +1,63 @@
+.. _config_network_filters_golang:
+
+Golang
+======
+
+The Golang network filter allows `Golang `_ to be run during both the downstream
+and upstream tcp data flows and makes it easier to extend Envoy.
+
+Go plugins used by this filter can be recompiled independently of Envoy.
+
+See the `Envoy's Golang extension proposal documentation
+`_
+for more details on the filter's implementation.
+
+.. warning::
+  The Envoy Golang filter is designed to be run with the ``GODEBUG=cgocheck=0`` environment variable set.
+
+  This disables the cgo pointer check.
+
+  Failure to set this environment variable will cause Envoy to crash!
+
+Developing a Go plugin
+----------------------
+
+Envoy's Go plugins must implement the :repo:`DownstreamFilter/UpstreamFilter API `.
+
+Building a Go plugin
+~~~~~~~~~~~~~~~~~~~~
+
+.. attention::
+  When building a Go plugin dynamic library, you **must** use a Go version consistent
+  with Envoy's version of glibc.
+
+One way to ensure a compatible Go version is to use the Go binary provided by Envoy's bazel setup:
+
+.. code-block:: console
+
+   $ bazel run @go_sdk//:bin/go -- version
+   ...
+   go version goX.YZ linux/amd64
+
+For example, to build the ``.so`` for a ``foo`` plugin, you might run:
+
+.. code-block:: console
+
+   $ bazel run @go_sdk//:bin/go build --buildmode=c-shared  -v -o path/to/output/libfoo.so path/to/src/foo
+
+Configuration
+-------------
+
+.. tip::
+   This filter should be configured with the type URL
+   :ref:`type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config `.
+
+A prebuilt Golang network filter  ``my_plugin.so`` might be configured as follows:
+
+.. literalinclude:: /_configs/go/network.yaml
+   :language: yaml
+   :linenos:
+   :lines: 11-21
+   :lineno-start: 10
+   :emphasize-lines: 2-11
+   :caption: :download:`golang.yaml `
diff --git a/docs/root/configuration/listeners/network_filters/network_filters.rst b/docs/root/configuration/listeners/network_filters/network_filters.rst
index d4dcc5e86c97..6e0f71443527 100644
--- a/docs/root/configuration/listeners/network_filters/network_filters.rst
+++ b/docs/root/configuration/listeners/network_filters/network_filters.rst
@@ -10,12 +10,13 @@ filters.
 .. toctree::
   :maxdepth: 2
 
-  dubbo_proxy_filter
   client_ssl_auth_filter
   connection_limit_filter
-  echo_filter
   direct_response_filter
+  dubbo_proxy_filter
+  echo_filter
   ext_authz_filter
+  golang_filter
   kafka_broker_filter
   kafka_mesh_filter
   local_rate_limit_filter
@@ -26,9 +27,9 @@ filters.
   rbac_filter
   redis_proxy_filter
   rocketmq_proxy_filter
-  tcp_proxy_filter
-  thrift_proxy_filter
   sni_cluster_filter
   sni_dynamic_forward_proxy_filter
+  tcp_proxy_filter
+  thrift_proxy_filter
   wasm_filter
   zookeeper_proxy_filter
diff --git a/examples/golang/simple/config.go b/examples/golang/simple/config.go
index 588f90745672..be7257a6f2a6 100644
--- a/examples/golang/simple/config.go
+++ b/examples/golang/simple/config.go
@@ -5,9 +5,10 @@ import (
 	"fmt"
 
 	xds "github.com/cncf/xds/go/xds/type/v3"
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http"
 	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
+	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http"
 )
 
 const Name = "simple"
diff --git a/examples/golang/simple/filter.go b/examples/golang/simple/filter.go
index 731bf99aa2b8..fd19bde1707e 100644
--- a/examples/golang/simple/filter.go
+++ b/examples/golang/simple/filter.go
@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strconv"
 
-	"github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/api"
+	"github.com/envoyproxy/envoy/contrib/golang/common/go/api"
 )
 
 var UpdateUpstreamBody = "upstream response body updated by the simple plugin"
diff --git a/source/common/common/logger.h b/source/common/common/logger.h
index 45ded0852277..b44130a136df 100644
--- a/source/common/common/logger.h
+++ b/source/common/common/logger.h
@@ -91,7 +91,8 @@ const static bool should_log = true;
   FUNCTION(upstream)                                                                               \
   FUNCTION(udp)                                                                                    \
   FUNCTION(wasm)                                                                                   \
-  FUNCTION(websocket)
+  FUNCTION(websocket)                                                                              \
+  FUNCTION(golang)
 
 // clang-format off
 enum class Id {

From f68cab5c5c181960a3f71d1b1667cb4db7e53c7c Mon Sep 17 00:00:00 2001
From: River <6375745+RiverPhillips@users.noreply.github.com>
Date: Mon, 26 Jun 2023 07:28:00 +0100
Subject: [PATCH 642/740] dep: bump buf => 1.22.0 (#28135)

Signed-off-by: River Phillips 
---
 api/bazel/repository_locations.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl
index ef3af248eb82..cbaa4d3c3960 100644
--- a/api/bazel/repository_locations.bzl
+++ b/api/bazel/repository_locations.bzl
@@ -131,11 +131,11 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "buf",
         project_desc = "A new way of working with Protocol Buffers.",  # Used for breaking change detection in API protobufs
         project_url = "https://buf.build",
-        version = "1.21.0",
-        sha256 = "1db51318e49f12095c97866c9b5d939dfec318b50362bba8a3a9545c4cff456b",
+        version = "1.22.0",
+        sha256 = "d068c4c620d79f5576cfdae7e1c2989abd1dd3cb0f475b2cc5461dd7e786a8e4",
         strip_prefix = "buf",
         urls = ["https://github.com/bufbuild/buf/releases/download/v{version}/buf-Linux-x86_64.tar.gz"],
-        release_date = "2023-06-05",
+        release_date = "2023-06-23",
         use_category = ["api"],
         license = "Apache-2.0",
         license_url = "https://github.com/bufbuild/buf/blob/v{version}/LICENSE",

From bc8baa71dbde397928f9c3abeb08bcc70713a616 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Mon, 26 Jun 2023 07:31:19 +0100
Subject: [PATCH 643/740] ci: Use self-hosted agents for (quick) confirmation
 jobs (#28077)

Signed-off-by: Ryan Northey 
---
 .azure-pipelines/stage/checks.yml    | 3 +--
 .azure-pipelines/stage/linux.yml     | 3 +--
 .azure-pipelines/stage/macos.yml     | 3 +--
 .azure-pipelines/stage/prechecks.yml | 3 +--
 .azure-pipelines/stage/publish.yml   | 3 +--
 .azure-pipelines/stage/verify.yml    | 3 +--
 .azure-pipelines/stage/windows.yml   | 3 +--
 7 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/.azure-pipelines/stage/checks.yml b/.azure-pipelines/stage/checks.yml
index 471904313e9e..ddea706c45a5 100644
--- a/.azure-pipelines/stage/checks.yml
+++ b/.azure-pipelines/stage/checks.yml
@@ -131,8 +131,7 @@ jobs:
 - job: complete
   displayName: "Checks complete"
   dependsOn: ["bazel", "coverage"]
-  pool:
-    vmImage: $(agentUbuntu)
+  pool: x64-nano
   # This condition ensures that this (required) check passes if all of
   # the preceding checks either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/linux.yml b/.azure-pipelines/stage/linux.yml
index f581456b9596..80bfe1a0f549 100644
--- a/.azure-pipelines/stage/linux.yml
+++ b/.azure-pipelines/stage/linux.yml
@@ -47,8 +47,7 @@ jobs:
 - job: released
   displayName: Complete
   dependsOn: ["release"]
-  pool:
-    vmImage: $(agentUbuntu)
+  pool: x64-nano
   # This condition ensures that this (required) job passes if all of
   # the preceeding jobs either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/macos.yml b/.azure-pipelines/stage/macos.yml
index bc16cacd7d4f..960680dc4dfd 100644
--- a/.azure-pipelines/stage/macos.yml
+++ b/.azure-pipelines/stage/macos.yml
@@ -42,8 +42,7 @@ jobs:
 - job: tested
   displayName: Complete
   dependsOn: ["test"]
-  pool:
-    vmImage: $(agentUbuntu)
+  pool: x64-nano
   # This condition ensures that this (required) job passes if all of
   # the preceeding jobs either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/prechecks.yml b/.azure-pipelines/stage/prechecks.yml
index abaee79a2e9e..fb39b6631c8e 100644
--- a/.azure-pipelines/stage/prechecks.yml
+++ b/.azure-pipelines/stage/prechecks.yml
@@ -182,8 +182,7 @@ jobs:
 - job: prechecked
   displayName: Prechecked
   dependsOn: ["prechecks", "dependencies"]
-  pool:
-    vmImage: $(agentUbuntu)
+  pool: x64-nano
   # This condition ensures that this (required) job passes if all of
   # the preceeding jobs either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml
index 6b47eeae5134..70cc8beb3dd9 100644
--- a/.azure-pipelines/stage/publish.yml
+++ b/.azure-pipelines/stage/publish.yml
@@ -376,8 +376,7 @@ jobs:
 - job: success
   dependsOn: ["docker", "docs", "signed_release"]
   displayName: Success (linux artefacts)
-  pool:
-    vmImage: $(agentUbuntu)
+  pool: x64-nano
   # This condition ensures that this (required) check passes if all of
   # the preceding checks either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml
index 1ae438a85dcc..ae2632fbe7f0 100644
--- a/.azure-pipelines/stage/verify.yml
+++ b/.azure-pipelines/stage/verify.yml
@@ -60,8 +60,7 @@ jobs:
 - job: verified
   displayName: Verification complete
   dependsOn: ["packages_x64", "packages_arm64"]
-  pool:
-    vmImage: $(agentUbuntu)
+  pool: x64-nano
   # This condition ensures that this (required) check passes if all of
   # the preceding checks either pass or are skipped
   # adapted from:
diff --git a/.azure-pipelines/stage/windows.yml b/.azure-pipelines/stage/windows.yml
index b2d81d766198..62898bcc5fa7 100644
--- a/.azure-pipelines/stage/windows.yml
+++ b/.azure-pipelines/stage/windows.yml
@@ -106,8 +106,7 @@ jobs:
 - job: released
   displayName: Complete
   dependsOn: ["release", "docker"]
-  pool:
-    vmImage: $(agentUbuntu)
+  pool: x64-nano
   # This condition ensures that this (required) job passes if all of
   # the preceeding jobs either pass or are skipped
   # adapted from:

From 4feddca2199b98dc1197f6b483447e7e40dba8be Mon Sep 17 00:00:00 2001
From: phlax 
Date: Mon, 26 Jun 2023 07:34:50 +0100
Subject: [PATCH 644/740] deps: Bump `opentelemetry_proto` -> 0.20.0 (#28093)

Signed-off-by: Ryan Northey 
---
 api/bazel/repository_locations.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl
index cbaa4d3c3960..97da4496ef50 100644
--- a/api/bazel/repository_locations.bzl
+++ b/api/bazel/repository_locations.bzl
@@ -118,9 +118,9 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "OpenTelemetry Proto",
         project_desc = "Language Independent Interface Types For OpenTelemetry",
         project_url = "https://github.com/open-telemetry/opentelemetry-proto",
-        version = "0.19.0",
-        sha256 = "464bc2b348e674a1a03142e403cbccb01be8655b6de0f8bfe733ea31fcd421be",
-        release_date = "2022-08-03",
+        version = "0.20.0",
+        sha256 = "6ab267cf82832ed60ad075d574c78da736193eecb9693e8a8d02f65d6d3f3520",
+        release_date = "2023-06-06",
         strip_prefix = "opentelemetry-proto-{version}",
         urls = ["https://github.com/open-telemetry/opentelemetry-proto/archive/v{version}.tar.gz"],
         use_category = ["api"],

From d68186c631bb2f9d582d87ab96b397768f14ae2c Mon Sep 17 00:00:00 2001
From: phlax 
Date: Mon, 26 Jun 2023 07:35:26 +0100
Subject: [PATCH 645/740] deps: Bump `aspect_bazel_lib` -> 1.32.1 (#28065)

Signed-off-by: Ryan Northey 
---
 bazel/repository_locations.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
index 9b04f572ba79..3b1a805e80e2 100644
--- a/bazel/repository_locations.bzl
+++ b/bazel/repository_locations.bzl
@@ -137,12 +137,12 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "Aspect Bazel helpers",
         project_desc = "Base Starlark libraries and basic Bazel rules which are useful for constructing rulesets and BUILD files",
         project_url = "https://github.com/aspect-build/bazel-lib",
-        version = "1.32.0",
-        sha256 = "f1c181b910f821072f38ee45bb87db6b56275458c7f832c54c54ba6334119eca",
+        version = "1.32.1",
+        sha256 = "e3151d87910f69cf1fc88755392d7c878034a69d6499b287bcfc00b1cf9bb415",
         strip_prefix = "bazel-lib-{version}",
         urls = ["https://github.com/aspect-build/bazel-lib/archive/v{version}.tar.gz"],
         use_category = ["build"],
-        release_date = "2023-05-16",
+        release_date = "2023-05-18",
         cpe = "N/A",
         license = "Apache-2.0",
         license_url = "https://github.com/aspect-build/bazel-lib/blob/v{version}/LICENSE",

From f97e3d8e7e772aa0fd58c7a8f1df98e4e049d706 Mon Sep 17 00:00:00 2001
From: doujiang24 
Date: Mon, 26 Jun 2023 21:22:19 +0800
Subject: [PATCH 646/740] doc: add API compatibility section for golang filter.
 (#28138)

Signed-off-by: doujiang24 
---
 .../http/http_filters/golang_filter.rst              | 12 +++++++++++-
 examples/golang/simple/go.mod                        |  2 ++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/docs/root/configuration/http/http_filters/golang_filter.rst b/docs/root/configuration/http/http_filters/golang_filter.rst
index a5faf13565c7..96ed8c87c6e9 100644
--- a/docs/root/configuration/http/http_filters/golang_filter.rst
+++ b/docs/root/configuration/http/http_filters/golang_filter.rst
@@ -24,8 +24,18 @@ Developing a Go plugin
 
 Envoy's Go plugins must implement the :repo:`StreamFilter API `.
 
+.. attention::
+  The Go plugin API is not yet stable, you are **strongly** recommended to use the same version of Go plugin SDK and Envoy.
+
+When you are using a release version of Envoy, i.e. 1.26.x,
+you should use ``github.com/envoyproxy/envoy v1.26.x`` in the go.mod file.
+
+When you are not using a release, i.e. the latest main branch of Envoy,
+you could use ``go get -u github.com/envoyproxy/envoy@SHA`` to get the same version of Go plugin SDK,
+the SHA is the latest commit sha.
+
 Building a Go plugin
-~~~~~~~~~~~~~~~~~~~~
+--------------------
 
 .. attention::
   When building a Go plugin dynamic library, you **must** use a Go version consistent
diff --git a/examples/golang/simple/go.mod b/examples/golang/simple/go.mod
index 39a1f6653b29..311aabd1aad7 100644
--- a/examples/golang/simple/go.mod
+++ b/examples/golang/simple/go.mod
@@ -21,4 +21,6 @@ require (
 )
 
 // NOTICE: it's just for testing, please remove it.
+// And check the "API compatibility" section in doc:
+// https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/golang_filter#developing-a-go-plugin
 replace github.com/envoyproxy/envoy => ../../..

From 0738682f90db658be778d250ff401603ce5c87cf Mon Sep 17 00:00:00 2001
From: "Dr. Andre Vehreschild" <101638173+vehre-x41@users.noreply.github.com>
Date: Mon, 26 Jun 2023 16:43:54 +0200
Subject: [PATCH 647/740] [fuzz] Keep config part of input for
 network_readfilter (#27580)

Signed-off-by: Andre Vehreschild 
---
 .../filters/network/common/fuzz/BUILD         |  1 +
 .../common/fuzz/network_readfilter_fuzz.proto |  5 +
 .../fuzz/network_readfilter_fuzz_test.cc      | 94 ++++++++++++++-----
 3 files changed, 76 insertions(+), 24 deletions(-)

diff --git a/test/extensions/filters/network/common/fuzz/BUILD b/test/extensions/filters/network/common/fuzz/BUILD
index edddb378e109..199d1a8c50d6 100644
--- a/test/extensions/filters/network/common/fuzz/BUILD
+++ b/test/extensions/filters/network/common/fuzz/BUILD
@@ -95,6 +95,7 @@ envoy_cc_fuzz_test(
         "//source/common/config:utility_lib",
         "//test/config:utility_lib",
         "//test/test_common:test_runtime_lib",
+        "@com_github_google_libprotobuf_mutator//:libprotobuf_mutator",
     ] + envoy_filters_from_selected(READFILTER_FUZZ_FILTERS),
 )
 
diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz.proto b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz.proto
index e8205658d25e..7c8ea50ea4ec 100644
--- a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz.proto
+++ b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz.proto
@@ -27,6 +27,11 @@ message Action {
   }
 }
 
+// Helper to mutate actions in FilterFuzzTestCase, because mutate() does not apply to repeated members.
+message FuzzHelperForActions {
+  repeated Action actions = 1;
+}
+
 message FilterFuzzTestCase {
   // This is actually a protobuf type for the config of network filters.
   envoy.config.listener.v3.Filter config = 1;
diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc
index bed56c089f78..1f9ecc3250c5 100644
--- a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc
+++ b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc
@@ -9,11 +9,79 @@
 #include "test/fuzz/fuzz_runner.h"
 #include "test/test_common/test_runtime.h"
 
+#include "libprotobuf_mutator/src/libfuzzer/libfuzzer_macro.h"
+
+// TODO: Needed for protobuf_mutator::ParseTextMessage() and ::SaveMessageAsText(). Replace by
+// envoy-style input parsing methods.
+#include "src/text_format.h" // from libprotobuf_mutator/
+
 namespace Envoy {
 namespace Extensions {
 namespace NetworkFilters {
 
-DEFINE_PROTO_FUZZER(const test::extensions::filters::network::FilterFuzzTestCase& input) {
+void ensuredValidFilter(unsigned int random_number, envoy::config::listener::v3::Filter* config) {
+  // TODO(jianwendong): After extending to cover all the filters, we can use
+  // `Registry::FactoryRegistry<
+  // Server::Configuration::NamedNetworkFilterConfigFactory>::registeredNames()`
+  // to get all the filter names instead of calling `UberFilterFuzzer::filter_names()`.
+  static const auto filter_names = UberFilterFuzzer::filterNames();
+  static const auto factories = Registry::FactoryRegistry<
+      Server::Configuration::NamedNetworkFilterConfigFactory>::factories();
+
+  // Choose a valid filter name.
+  if (std::find(filter_names.begin(), filter_names.end(), config->name()) ==
+      std::end(filter_names)) {
+    absl::string_view filter_name = filter_names[random_number % filter_names.size()];
+    if (filter_name != config->name()) {
+      // Clear old config, or unpacking non-suitable value may crash.
+      config->clear_typed_config();
+      config->set_name(std::string(filter_name));
+    }
+  }
+  // Set the corresponding type_url for Any.
+  auto& factory = factories.at(config->name());
+  config->mutable_typed_config()->set_type_url(absl::StrCat(
+      "type.googleapis.com/", factory->createEmptyConfigProto()->GetDescriptor()->full_name()));
+}
+
+static void TestOneProtoInput(const test::extensions::filters::network::FilterFuzzTestCase&);
+using FuzzerProtoType = test::extensions::filters::network::FilterFuzzTestCase;
+
+// NOLINTNEXTLINE - suppress clang-tidy, because llvm's lib depends on this identifier.
+extern "C" size_t LLVMFuzzerCustomMutator(uint8_t* data, size_t size, size_t max_size,
+                                          unsigned int seed) {
+  // mutate the config part of the fuzzer only with a probability of
+  static const unsigned config_mutation_probability = 1 /* / 100 */;
+  static protobuf_mutator::Mutator mutator = [seed] {
+    protobuf_mutator::Mutator mutator;
+    mutator.Seed(seed);
+    return mutator;
+  }();
+  static Random::PsuedoRandomGenerator64 random;
+  ABSL_ATTRIBUTE_UNUSED static bool _random_inited = [seed] {
+    random.initializeSeed(seed);
+    return true;
+  }();
+
+  FuzzerProtoType input;
+  protobuf_mutator::ParseTextMessage(data, size, &input);
+  if (random.random() % 100 < config_mutation_probability) {
+    test::extensions::filters::network::FuzzHelperForActions actions;
+    *actions.mutable_actions() = std::move(*input.mutable_actions());
+    mutator.Mutate(&actions, max_size);
+    *input.mutable_actions() = std::move(*actions.mutable_actions());
+  } else {
+    mutator.Mutate(input.mutable_config(), max_size);
+    ensuredValidFilter(random.random(), input.mutable_config());
+  }
+
+  return protobuf_mutator::SaveMessageAsText(input, data, max_size);
+}
+
+DEFINE_CUSTOM_PROTO_CROSSOVER_IMPL(false, FuzzerProtoType)
+DEFINE_TEST_ONE_PROTO_INPUT_IMPL(false, FuzzerProtoType)
+DEFINE_POST_PROCESS_PROTO_MUTATION_IMPL(FuzzerProtoType)
+static void TestOneProtoInput(const test::extensions::filters::network::FilterFuzzTestCase& input) {
   TestDeprecatedV2Api _deprecated_v2_api;
   ABSL_ATTRIBUTE_UNUSED static PostProcessorRegistration reg = {
       [](test::extensions::filters::network::FilterFuzzTestCase* input, unsigned int seed) {
@@ -21,29 +89,7 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::network::FilterFuzzTestCase
         // calls mutate on an input, and *not* during fuzz target execution.
         // Replaying a corpus through the fuzzer will not be affected by the
         // post-processor mutation.
-
-        // TODO(jianwendong): After extending to cover all the filters, we can use
-        // `Registry::FactoryRegistry<
-        // Server::Configuration::NamedNetworkFilterConfigFactory>::registeredNames()`
-        // to get all the filter names instead of calling `UberFilterFuzzer::filter_names()`.
-        static const auto filter_names = UberFilterFuzzer::filterNames();
-        static const auto factories = Registry::FactoryRegistry<
-            Server::Configuration::NamedNetworkFilterConfigFactory>::factories();
-        // Choose a valid filter name.
-        if (std::find(filter_names.begin(), filter_names.end(), input->config().name()) ==
-            std::end(filter_names)) {
-          absl::string_view filter_name = filter_names[seed % filter_names.size()];
-          if (filter_name != input->config().name()) {
-            // Clear old config, or unpacking non-suitable value may crash.
-            input->mutable_config()->clear_typed_config();
-            input->mutable_config()->set_name(std::string(filter_name));
-          }
-        }
-        // Set the corresponding type_url for Any.
-        auto& factory = factories.at(input->config().name());
-        input->mutable_config()->mutable_typed_config()->set_type_url(
-            absl::StrCat("type.googleapis.com/",
-                         factory->createEmptyConfigProto()->GetDescriptor()->full_name()));
+        ensuredValidFilter(seed, input->mutable_config());
 
         ProtobufMessage::ValidatedInputGenerator generator(
             seed, ProtobufMessage::composeFiltersAnyMap(), 20);

From 4e5031013746a0768e9a3065dbab08b70eaf3c05 Mon Sep 17 00:00:00 2001
From: yanjunxiang-google
 <78807980+yanjunxiang-google@users.noreply.github.com>
Date: Mon, 26 Jun 2023 13:41:22 -0400
Subject: [PATCH 648/740] Turn ext_proc into API stable. (#28101)

* Turn ext_proc into API stable.

Signed-off-by: Yanjun Xiang 
---
 api/envoy/extensions/filters/http/ext_proc/v3/BUILD            | 1 -
 api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto   | 3 ---
 .../extensions/filters/http/ext_proc/v3/processing_mode.proto  | 3 ---
 api/envoy/service/ext_proc/v3/BUILD                            | 1 -
 api/envoy/service/ext_proc/v3/external_processor.proto         | 3 ---
 5 files changed, 11 deletions(-)

diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/BUILD b/api/envoy/extensions/filters/http/ext_proc/v3/BUILD
index e5fde2bafa2d..18cc12771da3 100644
--- a/api/envoy/extensions/filters/http/ext_proc/v3/BUILD
+++ b/api/envoy/extensions/filters/http/ext_proc/v3/BUILD
@@ -10,6 +10,5 @@ api_proto_package(
         "//envoy/config/core/v3:pkg",
         "//envoy/type/matcher/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
-        "@com_github_cncf_udpa//xds/annotations/v3:pkg",
     ],
 )
diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto
index 2198a2d6f25e..97aab336f26c 100644
--- a/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto
+++ b/api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto
@@ -10,8 +10,6 @@ import "envoy/type/matcher/v3/string.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 
-import "xds/annotations/v3/status.proto";
-
 import "udpa/annotations/status.proto";
 import "validate/validate.proto";
 
@@ -20,7 +18,6 @@ option java_outer_classname = "ExtProcProto";
 option java_multiple_files = true;
 option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3;ext_procv3";
 option (udpa.annotations.file_status).package_version_status = ACTIVE;
-option (xds.annotations.v3.file_status).work_in_progress = true;
 
 // [#protodoc-title: External Processing Filter]
 // External Processing Filter
diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto b/api/envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto
index 9c692a17d581..eafdb1eabc65 100644
--- a/api/envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto
+++ b/api/envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto
@@ -2,8 +2,6 @@ syntax = "proto3";
 
 package envoy.extensions.filters.http.ext_proc.v3;
 
-import "xds/annotations/v3/status.proto";
-
 import "udpa/annotations/status.proto";
 import "validate/validate.proto";
 
@@ -12,7 +10,6 @@ option java_outer_classname = "ProcessingModeProto";
 option java_multiple_files = true;
 option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3;ext_procv3";
 option (udpa.annotations.file_status).package_version_status = ACTIVE;
-option (xds.annotations.v3.file_status).work_in_progress = true;
 
 // [#protodoc-title: External Processing Filter]
 // External Processing Filter Processing Mode
diff --git a/api/envoy/service/ext_proc/v3/BUILD b/api/envoy/service/ext_proc/v3/BUILD
index d4506b16ed5d..62a33c34631b 100644
--- a/api/envoy/service/ext_proc/v3/BUILD
+++ b/api/envoy/service/ext_proc/v3/BUILD
@@ -11,6 +11,5 @@ api_proto_package(
         "//envoy/extensions/filters/http/ext_proc/v3:pkg",
         "//envoy/type/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
-        "@com_github_cncf_udpa//xds/annotations/v3:pkg",
     ],
 )
diff --git a/api/envoy/service/ext_proc/v3/external_processor.proto b/api/envoy/service/ext_proc/v3/external_processor.proto
index 9d33ffb3d051..ae28733aec95 100644
--- a/api/envoy/service/ext_proc/v3/external_processor.proto
+++ b/api/envoy/service/ext_proc/v3/external_processor.proto
@@ -9,8 +9,6 @@ import "envoy/type/v3/http_status.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 
-import "xds/annotations/v3/status.proto";
-
 import "udpa/annotations/status.proto";
 import "validate/validate.proto";
 
@@ -19,7 +17,6 @@ option java_outer_classname = "ExternalProcessorProto";
 option java_multiple_files = true;
 option go_package = "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3;ext_procv3";
 option (udpa.annotations.file_status).package_version_status = ACTIVE;
-option (xds.annotations.v3.file_status).work_in_progress = true;
 
 // [#protodoc-title: External processing service]
 

From fa99a034d08f8dca71c43a8b982bb59c458327f6 Mon Sep 17 00:00:00 2001
From: yanavlasov 
Date: Mon, 26 Jun 2023 13:43:11 -0400
Subject: [PATCH 649/740] Address flakiness of http2 flood test in CPU heavy
 builds (#28074)

Address flakiness of htttp2 flood test in CPU heavy builds

Signed-off-by: Yan Avlasov 
---
 test/integration/http2_flood_integration_test.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/integration/http2_flood_integration_test.cc b/test/integration/http2_flood_integration_test.cc
index 6f508bf35bdf..10f72aed5f40 100644
--- a/test/integration/http2_flood_integration_test.cc
+++ b/test/integration/http2_flood_integration_test.cc
@@ -957,13 +957,12 @@ TEST_P(Http2FloodMitigationTest, RstStreamOnStreamIdleTimeoutAfterResponseHeader
 // timer. The test verifies protocol constraint violation handling in the
 // Http2::ConnectionImpl::sendKeepalive() method.
 TEST_P(Http2FloodMitigationTest, KeepAliveTimeeTriggersFloodProtection) {
-  DISABLE_UNDER_COVERAGE; // https://github.com/envoyproxy/envoy/issues/21019
   config_helper_.addConfigModifier(
       [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager&
              hcm) {
         auto* keep_alive = hcm.mutable_http2_protocol_options()->mutable_connection_keepalive();
         keep_alive->mutable_interval()->set_nanos(500 * 1000 * 1000);
-        keep_alive->mutable_timeout()->set_seconds(1);
+        keep_alive->mutable_timeout()->set_seconds(1 * TSAN_TIMEOUT_FACTOR);
       });
 
   prefillOutboundDownstreamQueue(AllFrameFloodLimit - 1);

From e907eca221c8bf8ed6175dd3ed8d25ff7db4b626 Mon Sep 17 00:00:00 2001
From: Atul Thosar 
Date: Mon, 26 Jun 2023 23:17:41 +0530
Subject: [PATCH 650/740] test coverage added for LowerCaseString class
 (#28141)

This PR seeks to improve this issue: #1963

Signed-off-by: Atul Thosar 
---
 test/common/http/header_map_impl_test.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/common/http/header_map_impl_test.cc b/test/common/http/header_map_impl_test.cc
index 5d14645e82bc..50fa5d1600a8 100644
--- a/test/common/http/header_map_impl_test.cc
+++ b/test/common/http/header_map_impl_test.cc
@@ -46,6 +46,14 @@ TEST(HeaderStringTest, All) {
     EXPECT_EQ("goodbye", hello_string.get());
   }
 
+  // assignment operator with const rhs
+  {
+    LowerCaseString present_value("present_value");
+    const LowerCaseString new_value("new_value");
+    present_value = new_value;
+    EXPECT_EQ("new_value", present_value.get());
+  }
+
   // Move constructor for normal UnionString.
   {
     UnionString to_move;

From 08cf7e4f208506e96481994e06f1644c05d3d818 Mon Sep 17 00:00:00 2001
From: Yuichiro Ueno 
Date: Tue, 27 Jun 2023 02:53:10 +0900
Subject: [PATCH 651/740] maglev: fix maglev stability by sorting host_weights
 beforehand (#28055)

* maglev: fix maglev stability by sorting host_weights beforehand

Signed-off-by: Yuichiro Ueno 
---
 changelogs/current.yaml                       |  3 ++
 .../maglev/maglev_lb.cc                       | 24 ++++++++--
 .../maglev/maglev_lb_test.cc                  | 45 +++++++++++++++++++
 3 files changed, 68 insertions(+), 4 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 3a4271d184b5..b269c082b38b 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -186,6 +186,9 @@ bug_fixes:
   change: |
     Fixes an issue with the ORIGINAL_DST cluster cleanup timer lifetime, which
     can occur if the cluster is removed while the timer is armed.
+- area: maglev loadbalancer
+  change: |
+    Fixes maglev stability problem. Previously, maglev returns slightly different backend assignment from the same backends and keys.
 
 removed_config_or_runtime:
 # *Normally occurs at the end of the* :ref:`deprecation period `
diff --git a/source/extensions/load_balancing_policies/maglev/maglev_lb.cc b/source/extensions/load_balancing_policies/maglev/maglev_lb.cc
index 03332ec6c376..170da42e33fe 100644
--- a/source/extensions/load_balancing_policies/maglev/maglev_lb.cc
+++ b/source/extensions/load_balancing_policies/maglev/maglev_lb.cc
@@ -81,16 +81,32 @@ void MaglevTable::constructMaglevTableInternal(
     return;
   }
 
-  // Implementation of pseudocode listing 1 in the paper (see header file for more info).
-  std::vector table_build_entries;
-  table_build_entries.reserve(normalized_host_weights.size());
+  // Prepare stable (sorted) vector of host_weight.
+  // Maglev requires stable order of table_build_entries because the hash table will be filled in
+  // the order. Unstable table_build_entries results the change of backend assignment.
+  std::vector> sorted_host_weights;
+  sorted_host_weights.reserve(normalized_host_weights.size());
   for (const auto& host_weight : normalized_host_weights) {
     const auto& host = host_weight.first;
     const absl::string_view key_to_hash = hashKey(host, use_hostname_for_hashing);
     ASSERT(!key_to_hash.empty());
+
+    sorted_host_weights.emplace_back(key_to_hash, host_weight.first, host_weight.second);
+  }
+
+  std::sort(sorted_host_weights.begin(), sorted_host_weights.end());
+
+  // Implementation of pseudocode listing 1 in the paper (see header file for more info).
+  std::vector table_build_entries;
+  table_build_entries.reserve(normalized_host_weights.size());
+  for (const auto& sorted_host_weight : sorted_host_weights) {
+    const auto& key_to_hash = std::get<0>(sorted_host_weight);
+    const auto& host = std::get<1>(sorted_host_weight);
+    const auto& weight = std::get<2>(sorted_host_weight);
+
     table_build_entries.emplace_back(host, HashUtil::xxHash64(key_to_hash) % table_size_,
                                      (HashUtil::xxHash64(key_to_hash, 1) % (table_size_ - 1)) + 1,
-                                     host_weight.second);
+                                     weight);
   }
 
   constructImplementationInternals(table_build_entries, max_normalized_weight);
diff --git a/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc
index 857e6ea9fd17..5de22a6d181e 100644
--- a/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc
+++ b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc
@@ -285,6 +285,51 @@ TEST_P(MaglevLoadBalancerTest, BasicWithRetryHostPredicate) {
   }
 }
 
+// Basic stability test.
+TEST_P(MaglevLoadBalancerTest, BasicStability) {
+  host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime()),
+                      makeTestHost(info_, "tcp://127.0.0.1:91", simTime()),
+                      makeTestHost(info_, "tcp://127.0.0.1:92", simTime()),
+                      makeTestHost(info_, "tcp://127.0.0.1:93", simTime()),
+                      makeTestHost(info_, "tcp://127.0.0.1:94", simTime()),
+                      makeTestHost(info_, "tcp://127.0.0.1:95", simTime())};
+  host_set_.healthy_hosts_ = host_set_.hosts_;
+  host_set_.runCallbacks({}, {});
+  init(7);
+
+  EXPECT_EQ("maglev_lb.min_entries_per_host", lb_->stats().min_entries_per_host_.name());
+  EXPECT_EQ("maglev_lb.max_entries_per_host", lb_->stats().max_entries_per_host_.name());
+  EXPECT_EQ(1, lb_->stats().min_entries_per_host_.value());
+  EXPECT_EQ(2, lb_->stats().max_entries_per_host_.value());
+
+  // maglev: i=0 host=127.0.0.1:92
+  // maglev: i=1 host=127.0.0.1:94
+  // maglev: i=2 host=127.0.0.1:90
+  // maglev: i=3 host=127.0.0.1:91
+  // maglev: i=4 host=127.0.0.1:95
+  // maglev: i=5 host=127.0.0.1:90
+  // maglev: i=6 host=127.0.0.1:93
+  LoadBalancerPtr lb = lb_->factory()->create(lb_params_);
+  const std::vector expected_assignments{2, 4, 0, 1, 5, 0, 3};
+  for (uint32_t i = 0; i < 3 * expected_assignments.size(); ++i) {
+    TestLoadBalancerContext context(i);
+    EXPECT_EQ(host_set_.hosts_[expected_assignments[i % expected_assignments.size()]],
+              lb->chooseHost(&context));
+  }
+
+  // Shuffle healthy_hosts_ to check stability of assignments
+  std::shuffle(host_set_.healthy_hosts_.begin(), host_set_.healthy_hosts_.end(),
+               std::default_random_engine());
+  host_set_.runCallbacks({}, {});
+  lb = lb_->factory()->create(lb_params_);
+
+  for (uint32_t i = 0; i < 3 * expected_assignments.size(); ++i) {
+    TestLoadBalancerContext context(i);
+    EXPECT_EQ(host_set_.hosts_[expected_assignments[i % expected_assignments.size()]],
+              lb->chooseHost(&context));
+  }
+}
+
 // Weighted sanity test.
 TEST_P(MaglevLoadBalancerTest, Weighted) {
   host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), 1),

From 8a3286e1f0f5af021b40fbf187e6f3bd6b0263b6 Mon Sep 17 00:00:00 2001
From: ohadvano <49730675+ohadvano@users.noreply.github.com>
Date: Mon, 26 Jun 2023 21:03:20 +0300
Subject: [PATCH 652/740] application logs: add ENVOY_TAGGED_CONN_LOG and
 ENVOY_TAGGED_STREAM_LOG (#28117)

Signed-off-by: ohadvano 
---
 source/common/common/logger.h         |  38 ++++++
 test/common/common/BUILD              |   2 +
 test/common/common/log_macros_test.cc |   6 +
 test/common/common/logger_test.cc     | 182 +++++++++++++++++++++++++-
 4 files changed, 221 insertions(+), 7 deletions(-)

diff --git a/source/common/common/logger.h b/source/common/common/logger.h
index b44130a136df..380d2e6a731c 100644
--- a/source/common/common/logger.h
+++ b/source/common/common/logger.h
@@ -574,6 +574,28 @@ class ExtractedMessage : public spdlog::custom_flag_formatter {
     }                                                                                              \
   } while (0)
 
+#define ENVOY_TAGGED_CONN_LOG_TO_LOGGER(LOGGER, LEVEL, TAGS, CONNECTION, FORMAT, ...)              \
+  do {                                                                                             \
+    if (ENVOY_LOG_COMP_LEVEL(LOGGER, LEVEL)) {                                                     \
+      TAGS.emplace("ConnectionId", std::to_string((CONNECTION).id()));                             \
+      ENVOY_LOG_TO_LOGGER(LOGGER, LEVEL,                                                           \
+                          ::Envoy::Logger::Utility::serializeLogTags(TAGS) + FORMAT,               \
+                          ##__VA_ARGS__);                                                          \
+    }                                                                                              \
+  } while (0)
+
+#define ENVOY_TAGGED_STREAM_LOG_TO_LOGGER(LOGGER, LEVEL, TAGS, STREAM, FORMAT, ...)                \
+  do {                                                                                             \
+    if (ENVOY_LOG_COMP_LEVEL(LOGGER, LEVEL)) {                                                     \
+      TAGS.emplace("ConnectionId",                                                                 \
+                   (STREAM).connection() ? std::to_string((STREAM).connection()->id()) : "0");     \
+      TAGS.emplace("StreamId", std::to_string((STREAM).streamId()));                               \
+      ENVOY_LOG_TO_LOGGER(LOGGER, LEVEL,                                                           \
+                          ::Envoy::Logger::Utility::serializeLogTags(TAGS) + FORMAT,               \
+                          ##__VA_ARGS__);                                                          \
+    }                                                                                              \
+  } while (0)
+
 /**
  * Log with tags which are a map of key and value strings. When ENVOY_TAGGED_LOG is used, the tags
  * are serialized and prepended to the log message.
@@ -584,6 +606,22 @@ class ExtractedMessage : public spdlog::custom_flag_formatter {
 #define ENVOY_TAGGED_LOG(LEVEL, TAGS, FORMAT, ...)                                                 \
   ENVOY_TAGGED_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, TAGS, FORMAT, ##__VA_ARGS__)
 
+/**
+ * Log with tags which are a map of key and value strings. Has the same operation as
+ * ENVOY_TAGGED_LOG, only this macro will also add the connection ID to the tags, if the provided
+ * tags do not already have a ConnectionId key existing.
+ */
+#define ENVOY_TAGGED_CONN_LOG(LEVEL, TAGS, CONNECTION, FORMAT, ...)                                \
+  ENVOY_TAGGED_CONN_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, TAGS, CONNECTION, FORMAT, ##__VA_ARGS__)
+
+/**
+ * Log with tags which are a map of key and value strings. Has the same operation as
+ * ENVOY_TAGGED_LOG, only this macro will also add the connection and stream ID to the tags,
+ * if the provided tags do not already have a ConnectionId or StreamId keys existing.
+ */
+#define ENVOY_TAGGED_STREAM_LOG(LEVEL, TAGS, STREAM, FORMAT, ...)                                  \
+  ENVOY_TAGGED_STREAM_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, TAGS, STREAM, FORMAT, ##__VA_ARGS__)
+
 /**
  * Log with a stable event name. This allows emitting a log line with a stable name in addition to
  * the standard log line. The stable log line is passed to custom sinks that may want to intercept
diff --git a/test/common/common/BUILD b/test/common/common/BUILD
index 74f86504f154..6ef06e492d2f 100644
--- a/test/common/common/BUILD
+++ b/test/common/common/BUILD
@@ -212,6 +212,8 @@ envoy_cc_test(
     srcs = ["logger_test.cc"],
     deps = [
         "//source/common/common:minimal_logger_lib",
+        "//test/mocks/http:http_mocks",
+        "//test/mocks/network:network_mocks",
     ],
 )
 
diff --git a/test/common/common/log_macros_test.cc b/test/common/common/log_macros_test.cc
index aaa9dce4154a..cb9e86515ca2 100644
--- a/test/common/common/log_macros_test.cc
+++ b/test/common/common/log_macros_test.cc
@@ -35,6 +35,12 @@ class TestFilterLog : public Logger::Loggable {
     ENVOY_TAGGED_LOG(info, tags_, "fake message {}", "val");
     ENVOY_TAGGED_LOG(info, (std::map{{"key", "val"}}), "fake message {}",
                      "val");
+    ENVOY_TAGGED_CONN_LOG(info, tags_, connection_, "fake message {}", "val");
+    ENVOY_TAGGED_CONN_LOG(info, (std::map{{"key", "val"}}), connection_,
+                          "fake message {}", "val");
+    ENVOY_TAGGED_STREAM_LOG(info, tags_, stream_, "fake message {}", "val");
+    ENVOY_TAGGED_STREAM_LOG(info, (std::map{{"key", "val"}}), stream_,
+                            "fake message {}", "val");
   }
 
   void logMessageEscapeSequences() { ENVOY_LOG_MISC(info, "line 1 \n line 2 \t tab \\r test"); }
diff --git a/test/common/common/logger_test.cc b/test/common/common/logger_test.cc
index 86a4e9803176..6a3c8ae1882f 100644
--- a/test/common/common/logger_test.cc
+++ b/test/common/common/logger_test.cc
@@ -5,6 +5,8 @@
 #include "source/common/common/logger.h"
 
 #include "test/mocks/common.h"
+#include "test/mocks/http/mocks.h"
+#include "test/mocks/network/mocks.h"
 #include "test/test_common/environment.h"
 
 #include "gmock/gmock.h"
@@ -13,6 +15,7 @@
 using testing::_;
 using testing::HasSubstr;
 using testing::Invoke;
+using testing::Return;
 
 namespace Envoy {
 namespace Logger {
@@ -448,6 +451,13 @@ TEST(LoggerUtilityTest, TestSerializeLogTags) {
 
 class ClassForTaggedLog : public Envoy::Logger::Loggable {
 public:
+  ClassForTaggedLog() {
+    EXPECT_CALL(connection_, id()).WillRepeatedly(Return(200));
+    EXPECT_CALL(stream_, streamId()).WillRepeatedly(Return(300));
+    EXPECT_CALL(stream_, connection())
+        .WillRepeatedly(Return(makeOptRef(dynamic_cast(connection_))));
+  }
+
   void logMessageWithPreCreatedTags() { ENVOY_TAGGED_LOG(info, tags_, "fake message {}", "val"); }
 
   void logMessageWithInlineTags() {
@@ -459,15 +469,29 @@ class ClassForTaggedLog : public Envoy::Logger::Loggable& tags) {
+    ENVOY_TAGGED_CONN_LOG(info, tags, connection_, "fake message");
+  }
+
+  void logMessageWithStream(std::map& tags) {
+    ENVOY_TAGGED_STREAM_LOG(info, tags, stream_, "fake message");
+  }
+
 private:
   std::map tags_{{"key", "val"}};
   std::map tags_with_escaping_{{"key", "val"}, {"ke\"y", "v\"al"}};
+  NiceMock connection_;
+  NiceMock stream_;
 };
 
 TEST(TaggedLogTest, TestTaggedLog) {
   Envoy::Logger::Registry::setLogFormat("%v");
   MockLogSink sink(Envoy::Logger::Registry::getSink());
   EXPECT_CALL(sink, log(_, _))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        // Using the mock connection write a log, skipping it.
+        EXPECT_THAT(msg, HasSubstr("TestRandomGenerator"));
+      }))
       .WillOnce(Invoke([](auto msg, auto&) {
         EXPECT_THAT(msg, HasSubstr("[Tags: \"key\":\"val\"] fake message val"));
       }))
@@ -485,6 +509,67 @@ TEST(TaggedLogTest, TestTaggedLog) {
   object.logMessageWithEscaping();
 }
 
+TEST(TaggedLogTest, TestTaggedConnLog) {
+  Envoy::Logger::Registry::setLogFormat("%v");
+  MockLogSink sink(Envoy::Logger::Registry::getSink());
+  EXPECT_CALL(sink, log(_, _))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        // Using the mock connection write a log, skipping it.
+        EXPECT_THAT(msg, HasSubstr("TestRandomGenerator"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_THAT(msg, HasSubstr("[Tags: \"ConnectionId\":\"200\"] fake message"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_THAT(msg,
+                    HasSubstr("[Tags: \"ConnectionId\":\"200\",\"key\":\"val\"] fake message"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_THAT(msg, HasSubstr("[Tags: \"ConnectionId\":\"105\"] fake message"));
+      }));
+
+  std::map empty_tags;
+  std::map tags{{"key", "val"}};
+  std::map tags_with_conn_id{{"ConnectionId", "105"}};
+
+  ClassForTaggedLog object;
+  object.logMessageWithConnection(empty_tags);
+  object.logMessageWithConnection(tags);
+  object.logMessageWithConnection(tags_with_conn_id);
+}
+
+TEST(TaggedLogTest, TestTaggedStreamLog) {
+  Envoy::Logger::Registry::setLogFormat("%v");
+  MockLogSink sink(Envoy::Logger::Registry::getSink());
+  EXPECT_CALL(sink, log(_, _))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        // Using the mock connection write a log, skipping it.
+        EXPECT_THAT(msg, HasSubstr("TestRandomGenerator"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_THAT(
+            msg, HasSubstr("[Tags: \"ConnectionId\":\"200\",\"StreamId\":\"300\"] fake message"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_THAT(
+            msg, HasSubstr("[Tags: \"ConnectionId\":\"200\",\"StreamId\":\"300\",\"key\":\"val\"] "
+                           "fake message"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_THAT(
+            msg, HasSubstr("[Tags: \"ConnectionId\":\"200\",\"StreamId\":\"405\"] fake message"));
+      }));
+
+  std::map empty_tags;
+  std::map tags{{"key", "val"}};
+  std::map tags_with_stream_id{{"StreamId", "405"}};
+
+  ClassForTaggedLog object;
+  object.logMessageWithStream(empty_tags);
+  object.logMessageWithStream(tags);
+  object.logMessageWithStream(tags_with_stream_id);
+}
+
 TEST(TaggedLogTest, TestTaggedLogWithJsonFormat) {
   ProtobufWkt::Struct log_struct;
   (*log_struct.mutable_fields())["Level"].set_string_value("%l");
@@ -495,6 +580,10 @@ TEST(TaggedLogTest, TestTaggedLogWithJsonFormat) {
 
   MockLogSink sink(Envoy::Logger::Registry::getSink());
   EXPECT_CALL(sink, log(_, _))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        // Using the mock connection write a log, skipping it.
+        EXPECT_THAT(msg, HasSubstr("TestRandomGenerator"));
+      }))
       .WillOnce(Invoke([](auto msg, auto&) {
         EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
         EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
@@ -514,6 +603,80 @@ TEST(TaggedLogTest, TestTaggedLogWithJsonFormat) {
   object.logMessageWithEscaping();
 }
 
+TEST(TaggedLogTest, TestTaggedConnLogWithJsonFormat) {
+  ProtobufWkt::Struct log_struct;
+  (*log_struct.mutable_fields())["Level"].set_string_value("%l");
+  (*log_struct.mutable_fields())["Message"].set_string_value("%j");
+  Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
+  EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok());
+  EXPECT_TRUE(Envoy::Logger::Registry::jsonLogFormatSet());
+
+  MockLogSink sink(Envoy::Logger::Registry::getSink());
+  EXPECT_CALL(sink, log(_, _))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        // Using the mock connection write a log, skipping it.
+        EXPECT_THAT(msg, HasSubstr("TestRandomGenerator"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
+        EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
+        EXPECT_THAT(msg, HasSubstr("\"Message\":\"fake message\""));
+        EXPECT_THAT(msg, HasSubstr("\"ConnectionId\":\"200\""));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
+        EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
+        EXPECT_THAT(msg, HasSubstr("\"Message\":\"fake message\""));
+        EXPECT_THAT(msg, HasSubstr("\"key\":\"val\""));
+        EXPECT_THAT(msg, HasSubstr("\"ConnectionId\":\"200\""));
+      }));
+
+  std::map empty_tags;
+  std::map tags{{"key", "val"}};
+
+  ClassForTaggedLog object;
+  object.logMessageWithConnection(empty_tags);
+  object.logMessageWithConnection(tags);
+}
+
+TEST(TaggedLogTest, TestTaggedStreamLogWithJsonFormat) {
+  ProtobufWkt::Struct log_struct;
+  (*log_struct.mutable_fields())["Level"].set_string_value("%l");
+  (*log_struct.mutable_fields())["Message"].set_string_value("%j");
+  Envoy::Logger::Registry::setLogLevel(spdlog::level::info);
+  EXPECT_TRUE(Envoy::Logger::Registry::setJsonLogFormat(log_struct).ok());
+  EXPECT_TRUE(Envoy::Logger::Registry::jsonLogFormatSet());
+
+  MockLogSink sink(Envoy::Logger::Registry::getSink());
+  EXPECT_CALL(sink, log(_, _))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        // Using the mock connection write a log, skipping it.
+        EXPECT_THAT(msg, HasSubstr("TestRandomGenerator"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
+        EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
+        EXPECT_THAT(msg, HasSubstr("\"Message\":\"fake message\""));
+        EXPECT_THAT(msg, HasSubstr("\"StreamId\":\"300\""));
+        EXPECT_THAT(msg, HasSubstr("\"ConnectionId\":\"200\""));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
+        EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
+        EXPECT_THAT(msg, HasSubstr("\"Message\":\"fake message\""));
+        EXPECT_THAT(msg, HasSubstr("\"key\":\"val\""));
+        EXPECT_THAT(msg, HasSubstr("\"StreamId\":\"300\""));
+        EXPECT_THAT(msg, HasSubstr("\"ConnectionId\":\"200\""));
+      }));
+
+  std::map empty_tags;
+  std::map tags{{"key", "val"}};
+
+  ClassForTaggedLog object;
+  object.logMessageWithStream(empty_tags);
+  object.logMessageWithStream(tags);
+}
+
 TEST(TaggedLogTest, TestTaggedLogWithJsonFormatMultipleJFlags) {
   ProtobufWkt::Struct log_struct;
   (*log_struct.mutable_fields())["Level"].set_string_value("%l");
@@ -524,13 +687,18 @@ TEST(TaggedLogTest, TestTaggedLogWithJsonFormatMultipleJFlags) {
   EXPECT_TRUE(Envoy::Logger::Registry::jsonLogFormatSet());
 
   MockLogSink sink(Envoy::Logger::Registry::getSink());
-  EXPECT_CALL(sink, log(_, _)).WillOnce(Invoke([](auto msg, auto&) {
-    EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
-    EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
-    EXPECT_THAT(msg, HasSubstr("\"Message1\":\"fake message val\""));
-    EXPECT_THAT(msg, HasSubstr("\"Message2\":\"fake message val\""));
-    EXPECT_THAT(msg, HasSubstr("\"key\":\"val\""));
-  }));
+  EXPECT_CALL(sink, log(_, _))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        // Using the mock connection write a log, skipping it.
+        EXPECT_THAT(msg, HasSubstr("TestRandomGenerator"));
+      }))
+      .WillOnce(Invoke([](auto msg, auto&) {
+        EXPECT_NO_THROW(Json::Factory::loadFromString(std::string(msg)));
+        EXPECT_THAT(msg, HasSubstr("\"Level\":\"info\""));
+        EXPECT_THAT(msg, HasSubstr("\"Message1\":\"fake message val\""));
+        EXPECT_THAT(msg, HasSubstr("\"Message2\":\"fake message val\""));
+        EXPECT_THAT(msg, HasSubstr("\"key\":\"val\""));
+      }));
 
   ClassForTaggedLog object;
   object.logMessageWithPreCreatedTags();

From dd557fd6a7db5d5f7c23b8bdea725bae0458bc73 Mon Sep 17 00:00:00 2001
From: deveshkandpal1224 <100540598+deveshkandpal1224@users.noreply.github.com>
Date: Mon, 26 Jun 2023 11:19:38 -0700
Subject: [PATCH 653/740] http: Add metric for active outbound frames (#28072)

* http: Add metric for active outbound frames

Signed-off-by: deveshkandpal1224 
---
 changelogs/current.yaml                              | 4 ++++
 docs/root/configuration/http/http_conn_man/stats.rst | 2 ++
 source/common/http/http2/codec_stats.h               | 5 +++--
 source/common/http/http2/protocol_constraints.cc     | 4 ++++
 test/common/http/http2/protocol_constraints_test.cc  | 8 ++++++++
 test/integration/http2_flood_integration_test.cc     | 2 +-
 6 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index b269c082b38b..cd3933a47d81 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -436,6 +436,10 @@ new_features:
 - area: lds
   change: |
     pause SRDS when LDS is updated.
+- area: http
+  change: |
+    added :ref:`outbound_control_frames_active ` and :ref:`outbound_frames_active `
+    statistic.
 
 deprecated:
 - area: access_log
diff --git a/docs/root/configuration/http/http_conn_man/stats.rst b/docs/root/configuration/http/http_conn_man/stats.rst
index 4f7ee8dd5b79..2605414255a1 100644
--- a/docs/root/configuration/http/http_conn_man/stats.rst
+++ b/docs/root/configuration/http/http_conn_man/stats.rst
@@ -169,7 +169,9 @@ On the upstream side all http2 statistics are rooted at ``cluster..http2.`
    ``inbound_window_update_frames_flood``, Counter, Total number of connections terminated for exceeding the limit on inbound frames of type WINDOW_UPDATE. The limit is configured by setting the :ref:`max_inbound_window_updateframes_per_data_frame_sent config setting `.
    ``keepalive_timeout``, Counter, Total number of connections closed due to :ref:`keepalive timeout `
    ``metadata_empty_frames``, Counter, Total number of metadata frames that were received and contained empty maps.
+   ``outbound_control_frames_active``, Gauge, "Total outbound control frames that are active."
    ``outbound_control_flood``, Counter, "Total number of connections terminated for exceeding the limit on outbound frames of types PING, SETTINGS and RST_STREAM. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `."
+   ``outbound_frames_active``, Gauge, "Total outbound frames that are active."
    ``outbound_flood``, Counter, Total number of connections terminated for exceeding the limit on outbound frames of all types. The limit is configured by setting the :ref:`max_outbound_frames config setting `.
    ``requests_rejected_with_underscores_in_headers``, Counter, Total numbers of rejected requests due to header names containing underscores. This action is configured by setting the :ref:`headers_with_underscores_action config setting `.
    ``rx_messaging_error``, Counter, Total number of invalid received frames that violated `section 8 `_ of the HTTP/2 spec. This will result in a ``tx_reset``
diff --git a/source/common/http/http2/codec_stats.h b/source/common/http/http2/codec_stats.h
index fb403bea301d..2912cd05b1b5 100644
--- a/source/common/http/http2/codec_stats.h
+++ b/source/common/http/http2/codec_stats.h
@@ -34,8 +34,9 @@ namespace Http2 {
   COUNTER(tx_reset)                                                                                \
   GAUGE(streams_active, Accumulate)                                                                \
   GAUGE(pending_send_bytes, Accumulate)                                                            \
-  GAUGE(deferred_stream_close, Accumulate)
-
+  GAUGE(deferred_stream_close, Accumulate)                                                         \
+  GAUGE(outbound_frames_active, Accumulate)                                                        \
+  GAUGE(outbound_control_frames_active, Accumulate)
 /**
  * Wrapper struct for the HTTP/2 codec stats. @see stats_macros.h
  */
diff --git a/source/common/http/http2/protocol_constraints.cc b/source/common/http/http2/protocol_constraints.cc
index 3948ff3a969b..cc4b7c49dd9c 100644
--- a/source/common/http/http2/protocol_constraints.cc
+++ b/source/common/http/http2/protocol_constraints.cc
@@ -23,8 +23,10 @@ ProtocolConstraints::ProtocolConstraints(
 ProtocolConstraints::ReleasorProc
 ProtocolConstraints::incrementOutboundFrameCount(bool is_outbound_flood_monitored_control_frame) {
   ++outbound_frames_;
+  stats_.outbound_frames_active_.set(outbound_frames_);
   if (is_outbound_flood_monitored_control_frame) {
     ++outbound_control_frames_;
+    stats_.outbound_control_frames_active_.set(outbound_control_frames_);
   }
   return is_outbound_flood_monitored_control_frame ? control_frame_buffer_releasor_
                                                    : frame_buffer_releasor_;
@@ -33,11 +35,13 @@ ProtocolConstraints::incrementOutboundFrameCount(bool is_outbound_flood_monitore
 void ProtocolConstraints::releaseOutboundFrame() {
   ASSERT(outbound_frames_ >= 1);
   --outbound_frames_;
+  stats_.outbound_frames_active_.set(outbound_frames_);
 }
 
 void ProtocolConstraints::releaseOutboundControlFrame() {
   ASSERT(outbound_control_frames_ >= 1);
   --outbound_control_frames_;
+  stats_.outbound_control_frames_active_.set(outbound_control_frames_);
   releaseOutboundFrame();
 }
 
diff --git a/test/common/http/http2/protocol_constraints_test.cc b/test/common/http/http2/protocol_constraints_test.cc
index 291a9d1723b4..3d715c7db52c 100644
--- a/test/common/http/http2/protocol_constraints_test.cc
+++ b/test/common/http/http2/protocol_constraints_test.cc
@@ -2,6 +2,7 @@
 #include "source/common/http/http2/protocol_constraints.h"
 
 #include "test/common/stats/stat_test_utility.h"
+#include "test/test_common/utility.h"
 
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
@@ -40,6 +41,10 @@ TEST_F(ProtocolConstraintsTest, OutboundControlFrameFlood) {
   EXPECT_TRUE(isBufferFloodError(constraints.status()));
   EXPECT_EQ("Too many control frames in the outbound queue.", constraints.status().message());
   EXPECT_EQ(1, stats_store_.counter("http2.outbound_control_flood").value());
+  EXPECT_EQ(3,
+            stats_store_
+                .gauge("http2.outbound_control_frames_active", Stats::Gauge::ImportMode::Accumulate)
+                .value());
 }
 
 TEST_F(ProtocolConstraintsTest, OutboundFrameFlood) {
@@ -57,6 +62,9 @@ TEST_F(ProtocolConstraintsTest, OutboundFrameFlood) {
   EXPECT_TRUE(isBufferFloodError(constraints.status()));
   EXPECT_EQ("Too many frames in the outbound queue.", constraints.status().message());
   EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value());
+  EXPECT_EQ(6,
+            stats_store_.gauge("http2.outbound_frames_active", Stats::Gauge::ImportMode::Accumulate)
+                .value());
 }
 
 // Verify that the `status()` method reflects the first violation and is not modified by subsequent
diff --git a/test/integration/http2_flood_integration_test.cc b/test/integration/http2_flood_integration_test.cc
index 10f72aed5f40..b53e6f8cdacd 100644
--- a/test/integration/http2_flood_integration_test.cc
+++ b/test/integration/http2_flood_integration_test.cc
@@ -241,6 +241,7 @@ void Http2FloodMitigationTest::prefillOutboundDownstreamQueue(uint32_t data_fram
   // that the first request has completed.
   test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_rx_bytes_total", 10000);
   test_server_->waitForGaugeEq("cluster.cluster_0.upstream_rq_active", 0);
+  test_server_->waitForGaugeEq("http2.outbound_frames_active", data_frame_count + 1);
   // Verify that pre-fill did not trigger flood protection
   EXPECT_EQ(0, test_server_->counter("http2.outbound_flood")->value());
 }
@@ -361,7 +362,6 @@ TEST_P(Http2FloodMitigationTest, Data) {
   ASSERT_EQ(0, test_server_->counter("cluster.cluster_0.upstream_cx_destroy")->value());
   // Verify that the flood check was triggered
   EXPECT_EQ(1, test_server_->counter("http2.outbound_flood")->value());
-
   // The factory will be used to create 4 buffers: the input and output buffers for request and
   // response pipelines.
   EXPECT_EQ(8, buffer_factory->numBuffersCreated());

From 020de5c9838729acc007b4748ecae63331182c5b Mon Sep 17 00:00:00 2001
From: Robert Femmer <114982872+robertfemmer@users.noreply.github.com>
Date: Mon, 26 Jun 2023 21:02:39 +0200
Subject: [PATCH 654/740] hcm router fuzzer: call correct fromHeaders
 implementation (#28096)

Signed-off-by: Robert Femmer 
---
 test/common/http/hcm_router_fuzz_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/common/http/hcm_router_fuzz_test.cc b/test/common/http/hcm_router_fuzz_test.cc
index cc8a618d62a7..6560fe2208a5 100644
--- a/test/common/http/hcm_router_fuzz_test.cc
+++ b/test/common/http/hcm_router_fuzz_test.cc
@@ -259,7 +259,7 @@ class FuzzCluster {
     FuzzUpstream* s = select(stream);
     if (s) {
       auto trailers = std::make_unique(
-          Fuzz::fromHeaders(response_trailers));
+          fromHeaders(response_trailers));
       s->sendTrailers(std::move(trailers));
     }
   }

From 10468b320421cb14d7911b4e6d139cc18780fb1a Mon Sep 17 00:00:00 2001
From: chabster4g <59761227+chabster4g@users.noreply.github.com>
Date: Mon, 26 Jun 2023 20:08:49 -0700
Subject: [PATCH 655/740] Add dynamic metadata support for ORIGINAL_DST cluster
 to pick a host (#28054)

One can specify a MetadataKey with a path selector to pick up a host
from the dynamic metadata of the request or downstream. Selected
value can either be a string or a list with at least a single
element of string type. Request metadata is considered first.

Signed-off-by: Andrii Chabykin 
---
 api/envoy/config/cluster/v3/BUILD             |   1 +
 api/envoy/config/cluster/v3/cluster.proto     |   5 +
 changelogs/current.yaml                       |   6 +-
 envoy/upstream/load_balancer.h                |   7 +
 source/common/router/router.h                 |   4 +
 source/common/upstream/load_balancer_impl.h   |   3 +
 .../original_dst/original_dst_cluster.cc      |  76 +++++--
 .../original_dst/original_dst_cluster.h       |   8 +-
 .../subset/subset_lb.h                        |   4 +
 .../original_dst/original_dst_cluster_test.cc | 185 +++++++++++++++++-
 test/mocks/upstream/load_balancer_context.h   |   2 +
 11 files changed, 286 insertions(+), 15 deletions(-)

diff --git a/api/envoy/config/cluster/v3/BUILD b/api/envoy/config/cluster/v3/BUILD
index 21528bbc7bfb..a98acddff0e4 100644
--- a/api/envoy/config/cluster/v3/BUILD
+++ b/api/envoy/config/cluster/v3/BUILD
@@ -9,6 +9,7 @@ api_proto_package(
         "//envoy/annotations:pkg",
         "//envoy/config/core/v3:pkg",
         "//envoy/config/endpoint/v3:pkg",
+        "//envoy/type/metadata/v3:pkg",
         "//envoy/type/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
         "@com_github_cncf_udpa//xds/core/v3:pkg",
diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto
index 7ecf42425791..91535b9ee1aa 100644
--- a/api/envoy/config/cluster/v3/cluster.proto
+++ b/api/envoy/config/cluster/v3/cluster.proto
@@ -13,6 +13,7 @@ import "envoy/config/core/v3/health_check.proto";
 import "envoy/config/core/v3/protocol.proto";
 import "envoy/config/core/v3/resolver.proto";
 import "envoy/config/endpoint/v3/endpoint.proto";
+import "envoy/type/metadata/v3/metadata.proto";
 import "envoy/type/v3/percent.proto";
 
 import "google/protobuf/any.proto";
@@ -551,6 +552,10 @@ message Cluster {
     // The port to override for the original dst address. This port
     // will take precedence over filter state and header override ports
     google.protobuf.UInt32Value upstream_port_override = 3 [(validate.rules).uint32 = {lte: 65535}];
+
+    // The dynamic metadata key to override destination address.
+    // First the request metadata is considered, then the connection one.
+    type.metadata.v3.MetadataKey metadata_key = 4;
   }
 
   // Common configuration for all load balancer implementations.
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index cd3933a47d81..84df92e3278d 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -440,7 +440,11 @@ new_features:
   change: |
     added :ref:`outbound_control_frames_active ` and :ref:`outbound_frames_active `
     statistic.
-
+- area: original_dst
+  change: |
+    Filter state is pulled from request context first (if available), then falls back to connection context. Added ability to pick host
+    from dynamic metadata using :ref:`metadata_key `.
+    Same behavior - looks in request context first (if available), falls back to connection context.
 deprecated:
 - area: access_log
   change: |
diff --git a/envoy/upstream/load_balancer.h b/envoy/upstream/load_balancer.h
index de07149356b5..8c1bf0c42a6a 100644
--- a/envoy/upstream/load_balancer.h
+++ b/envoy/upstream/load_balancer.h
@@ -6,6 +6,7 @@
 #include "envoy/common/pure.h"
 #include "envoy/network/transport_socket.h"
 #include "envoy/router/router.h"
+#include "envoy/stream_info/stream_info.h"
 #include "envoy/upstream/types.h"
 #include "envoy/upstream/upstream.h"
 
@@ -45,6 +46,12 @@ class LoadBalancerContext {
    */
   virtual const Network::Connection* downstreamConnection() const PURE;
 
+  /**
+   * @return const StreamInfo* the incoming request stream info or nullptr to use during load
+   * balancing.
+   */
+  virtual const StreamInfo::StreamInfo* requestStreamInfo() const PURE;
+
   /**
    * @return const Http::HeaderMap* the incoming headers or nullptr to use during load
    * balancing.
diff --git a/source/common/router/router.h b/source/common/router/router.h
index 7d86381d312d..ddbcfad71d0a 100644
--- a/source/common/router/router.h
+++ b/source/common/router/router.h
@@ -22,6 +22,7 @@
 #include "envoy/server/filter_config.h"
 #include "envoy/stats/scope.h"
 #include "envoy/stats/stats_macros.h"
+#include "envoy/stream_info/stream_info.h"
 #include "envoy/upstream/cluster_manager.h"
 
 #include "source/common/access_log/access_log_impl.h"
@@ -447,6 +448,9 @@ class Filter : Logger::Loggable,
   const Network::Connection* downstreamConnection() const override {
     return callbacks_->connection().ptr();
   }
+  const StreamInfo::StreamInfo* requestStreamInfo() const override {
+    return &callbacks_->streamInfo();
+  }
   const Http::RequestHeaderMap* downstreamHeaders() const override { return downstream_headers_; }
 
   bool shouldSelectAnotherHost(const Upstream::Host& host) override {
diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h
index 51035fe5b609..72597743cc68 100644
--- a/source/common/upstream/load_balancer_impl.h
+++ b/source/common/upstream/load_balancer_impl.h
@@ -19,6 +19,7 @@
 #include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.h"
 #include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.validate.h"
 #include "envoy/runtime/runtime.h"
+#include "envoy/stream_info/stream_info.h"
 #include "envoy/upstream/load_balancer.h"
 #include "envoy/upstream/upstream.h"
 
@@ -221,6 +222,8 @@ class LoadBalancerContextBase : public LoadBalancerContext {
 
   const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return nullptr; }
 
+  const StreamInfo::StreamInfo* requestStreamInfo() const override { return nullptr; }
+
   const Http::RequestHeaderMap* downstreamHeaders() const override { return nullptr; }
 
   const HealthyAndDegradedLoad&
diff --git a/source/extensions/clusters/original_dst/original_dst_cluster.cc b/source/extensions/clusters/original_dst/original_dst_cluster.cc
index 882e728578fa..56d01b66de2e 100644
--- a/source/extensions/clusters/original_dst/original_dst_cluster.cc
+++ b/source/extensions/clusters/original_dst/original_dst_cluster.cc
@@ -24,9 +24,14 @@ namespace Upstream {
 
 HostConstSharedPtr OriginalDstCluster::LoadBalancer::chooseHost(LoadBalancerContext* context) {
   if (context) {
-    // Check if filter state override is present, if yes use it before headers and local address.
+    // Check if filter state override is present, if yes use it before anything else.
     Network::Address::InstanceConstSharedPtr dst_host = filterStateOverrideHost(context);
 
+    // Check if override host metadata is present, if yes use it otherwise check the header.
+    if (dst_host == nullptr) {
+      dst_host = metadataOverrideHost(context);
+    }
+
     // Check if override host header is present, if yes use it otherwise check local address.
     if (dst_host == nullptr) {
       dst_host = requestOverrideHost(context);
@@ -90,17 +95,21 @@ HostConstSharedPtr OriginalDstCluster::LoadBalancer::chooseHost(LoadBalancerCont
 
 Network::Address::InstanceConstSharedPtr
 OriginalDstCluster::LoadBalancer::filterStateOverrideHost(LoadBalancerContext* context) {
-  const auto* conn = context->downstreamConnection();
-  if (!conn) {
-    return nullptr;
-  }
-  const auto* dst_address =
-      conn->streamInfo().filterState().getDataReadOnly(
-          Network::DestinationAddress::key());
-  if (!dst_address) {
-    return nullptr;
+  const auto streamInfos = {
+      context->requestStreamInfo(),
+      context->downstreamConnection() ? &context->downstreamConnection()->streamInfo() : nullptr};
+  for (const auto streamInfo : streamInfos) {
+    if (streamInfo == nullptr) {
+      continue;
+    }
+    const auto* dst_address =
+        streamInfo->filterState().getDataReadOnly(
+            Network::DestinationAddress::key());
+    if (dst_address) {
+      return dst_address->address();
+    }
   }
-  return dst_address->address();
+  return nullptr;
 }
 
 Network::Address::InstanceConstSharedPtr
@@ -131,6 +140,48 @@ OriginalDstCluster::LoadBalancer::requestOverrideHost(LoadBalancerContext* conte
   return request_host;
 }
 
+Network::Address::InstanceConstSharedPtr
+OriginalDstCluster::LoadBalancer::metadataOverrideHost(LoadBalancerContext* context) {
+  if (!metadata_key_.has_value()) {
+    return nullptr;
+  }
+  const auto streamInfos = {
+      context->requestStreamInfo(),
+      context->downstreamConnection() ? &context->downstreamConnection()->streamInfo() : nullptr};
+  const ProtobufWkt::Value* value = nullptr;
+  for (const auto streamInfo : streamInfos) {
+    if (streamInfo == nullptr) {
+      continue;
+    }
+    const auto& metadata = streamInfo->dynamicMetadata();
+    value = &Config::Metadata::metadataValue(&metadata, metadata_key_.value());
+    // Path can refer to a list, in which case we extract the first element.
+    if (value->kind_case() == ProtobufWkt::Value::kListValue) {
+      const auto& values = value->list_value().values();
+      if (!values.empty()) {
+        value = &(values[0]);
+      }
+    }
+    if (value->kind_case() == ProtobufWkt::Value::kStringValue) {
+      break;
+    }
+  }
+  if (value == nullptr || value->kind_case() != ProtobufWkt::Value::kStringValue) {
+    return nullptr;
+  }
+  const std::string& metadata_override_host = value->string_value();
+  Network::Address::InstanceConstSharedPtr metadata_host =
+      Network::Utility::parseInternetAddressAndPortNoThrow(metadata_override_host, false);
+  if (metadata_host == nullptr) {
+    ENVOY_LOG(debug, "original_dst_load_balancer: invalid override metadata value. {}",
+              metadata_override_host);
+    parent_->info()->trafficStats()->original_dst_host_invalid_.inc();
+    return nullptr;
+  }
+  ENVOY_LOG(debug, "Using metadata override host {}.", metadata_override_host);
+  return metadata_host;
+}
+
 OriginalDstCluster::OriginalDstCluster(const envoy::config::cluster::v3::Cluster& config,
                                        ClusterFactoryContext& context)
     : ClusterImplBase(config, context),
@@ -152,6 +203,9 @@ OriginalDstCluster::OriginalDstCluster(const envoy::config::cluster::v3::Cluster
             config_opt->http_header_name()));
       }
     }
+    if (config_opt->has_metadata_key()) {
+      metadata_key_ = Config::MetadataKey(config_opt->metadata_key());
+    }
     if (config_opt->has_upstream_port_override()) {
       port_override_ = config_opt->upstream_port_override().value();
     }
diff --git a/source/extensions/clusters/original_dst/original_dst_cluster.h b/source/extensions/clusters/original_dst/original_dst_cluster.h
index e84da53e78fd..ba4d1f6ee5fd 100644
--- a/source/extensions/clusters/original_dst/original_dst_cluster.h
+++ b/source/extensions/clusters/original_dst/original_dst_cluster.h
@@ -12,6 +12,7 @@
 
 #include "source/common/common/empty_string.h"
 #include "source/common/common/logger.h"
+#include "source/common/config/metadata.h"
 #include "source/common/upstream/cluster_factory_impl.h"
 #include "source/common/upstream/upstream_impl.h"
 
@@ -69,7 +70,8 @@ class OriginalDstCluster : public ClusterImplBase {
   public:
     LoadBalancer(const std::shared_ptr& parent)
         : parent_(parent), http_header_name_(parent->httpHeaderName()),
-          port_override_(parent->portOverride()), host_map_(parent->getCurrentHostMap()) {}
+          metadata_key_(parent->metadataKey()), port_override_(parent->portOverride()),
+          host_map_(parent->getCurrentHostMap()) {}
 
     // Upstream::LoadBalancer
     HostConstSharedPtr chooseHost(LoadBalancerContext* context) override;
@@ -89,16 +91,19 @@ class OriginalDstCluster : public ClusterImplBase {
 
     Network::Address::InstanceConstSharedPtr filterStateOverrideHost(LoadBalancerContext* context);
     Network::Address::InstanceConstSharedPtr requestOverrideHost(LoadBalancerContext* context);
+    Network::Address::InstanceConstSharedPtr metadataOverrideHost(LoadBalancerContext* context);
 
   private:
     const std::shared_ptr parent_;
     // The optional original host provider that extracts the address from HTTP header map.
     const absl::optional& http_header_name_;
+    const absl::optional& metadata_key_;
     const absl::optional port_override_;
     HostMultiMapConstSharedPtr host_map_;
   };
 
   const absl::optional& httpHeaderName() { return http_header_name_; }
+  const absl::optional& metadataKey() { return metadata_key_; }
   const absl::optional portOverride() { return port_override_; }
 
 private:
@@ -149,6 +154,7 @@ class OriginalDstCluster : public ClusterImplBase {
   absl::Mutex host_map_lock_;
   HostMultiMapConstSharedPtr host_map_ ABSL_GUARDED_BY(host_map_lock_);
   absl::optional http_header_name_;
+  absl::optional metadata_key_;
   absl::optional port_override_;
   friend class OriginalDstClusterFactory;
 };
diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.h b/source/extensions/load_balancing_policies/subset/subset_lb.h
index 13a9c073de3a..664cc902e93b 100644
--- a/source/extensions/load_balancing_policies/subset/subset_lb.h
+++ b/source/extensions/load_balancing_policies/subset/subset_lb.h
@@ -13,6 +13,7 @@
 #include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.validate.h"
 #include "envoy/runtime/runtime.h"
 #include "envoy/stats/scope.h"
+#include "envoy/stream_info/stream_info.h"
 #include "envoy/upstream/load_balancer.h"
 
 #include "source/common/common/macros.h"
@@ -291,6 +292,9 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::LoggabledownstreamConnection();
     }
+    const StreamInfo::StreamInfo* requestStreamInfo() const override {
+      return wrapped_->requestStreamInfo();
+    }
     const Http::RequestHeaderMap* downstreamHeaders() const override {
       return wrapped_->downstreamHeaders();
     }
diff --git a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc
index af336ec5b48b..7940a393eb27 100644
--- a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc
+++ b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc
@@ -34,6 +34,7 @@
 using testing::_;
 using testing::NiceMock;
 using testing::Return;
+using testing::ReturnRef;
 
 namespace Envoy {
 namespace Upstream {
@@ -41,10 +42,14 @@ namespace {
 
 class TestLoadBalancerContext : public LoadBalancerContextBase {
 public:
-  TestLoadBalancerContext(const Network::Connection* connection) : connection_(connection) {}
+  TestLoadBalancerContext(const Network::Connection* connection)
+      : TestLoadBalancerContext(connection, nullptr) {}
+  TestLoadBalancerContext(const Network::Connection* connection,
+                          const StreamInfo::StreamInfo* request_stream_info)
+      : connection_(connection), request_stream_info_(request_stream_info) {}
   TestLoadBalancerContext(const Network::Connection* connection, const std::string& key,
                           const std::string& value)
-      : connection_(connection) {
+      : TestLoadBalancerContext(connection) {
     downstream_headers_ =
         Http::RequestHeaderMapPtr{new Http::TestRequestHeaderMapImpl{{key, value}}};
   }
@@ -52,12 +57,14 @@ class TestLoadBalancerContext : public LoadBalancerContextBase {
   // Upstream::LoadBalancerContext
   absl::optional computeHashKey() override { return 0; }
   const Network::Connection* downstreamConnection() const override { return connection_; }
+  const StreamInfo::StreamInfo* requestStreamInfo() const override { return request_stream_info_; }
   const Http::RequestHeaderMap* downstreamHeaders() const override {
     return downstream_headers_.get();
   }
 
   absl::optional hash_key_;
   const Network::Connection* connection_;
+  const StreamInfo::StreamInfo* request_stream_info_;
   Http::RequestHeaderMapPtr downstream_headers_;
 };
 
@@ -871,6 +878,180 @@ TEST_F(OriginalDstClusterTest, UseHttpHeaderDisabled) {
   EXPECT_EQ(host3, nullptr);
 }
 
+TEST_F(OriginalDstClusterTest, UseMetadataKeyWithRequest) {
+  std::string yaml = R"EOF(
+    name: name
+    connect_timeout: 1.250s
+    type: ORIGINAL_DST
+    lb_policy: CLUSTER_PROVIDED
+    original_dst_lb_config:
+      metadata_key:
+        key: xxx
+        path:
+        - key: a
+        - key: b
+  )EOF";
+
+  EXPECT_CALL(initialized_, ready());
+  EXPECT_CALL(*cleanup_timer_, enableTimer(_, _));
+  setupFromYaml(yaml);
+
+  OriginalDstCluster::LoadBalancer lb(cluster_);
+  Event::PostCb post_cb;
+
+  NiceMock connection;
+  NiceMock request_stream_info;
+  TestLoadBalancerContext lb_context(&connection, &request_stream_info);
+
+  envoy::config::core::v3::Metadata req_dynamic_metadata;
+  (*(*(*req_dynamic_metadata.mutable_filter_metadata())["xxx"].mutable_fields())["a"]
+        .mutable_struct_value()
+        ->mutable_fields())["b"]
+      .set_string_value("127.0.0.1:6666");
+  envoy::config::core::v3::Metadata conn_dynamic_metadata;
+  (*(*(*conn_dynamic_metadata.mutable_filter_metadata())["xxx"].mutable_fields())["a"]
+        .mutable_struct_value()
+        ->mutable_fields())["b"]
+      .set_string_value("127.0.0.1:7777");
+
+  EXPECT_CALL(Const(request_stream_info), dynamicMetadata())
+      .WillRepeatedly(ReturnRef(req_dynamic_metadata));
+  EXPECT_CALL(Const(connection.stream_info_), dynamicMetadata())
+      .WillRepeatedly(ReturnRef(conn_dynamic_metadata));
+
+  EXPECT_CALL(membership_updated_, ready());
+  EXPECT_CALL(server_context_.dispatcher_, post(_)).WillOnce([&post_cb](Event::PostCb cb) {
+    post_cb = std::move(cb);
+  });
+  HostConstSharedPtr host = lb.chooseHost(&lb_context);
+  post_cb();
+  ASSERT_NE(host, nullptr);
+  EXPECT_EQ("127.0.0.1:6666", host->address()->asString());
+}
+
+TEST_F(OriginalDstClusterTest, UseMetadataKeyNoRequest) {
+  std::string yaml = R"EOF(
+    name: name
+    connect_timeout: 1.250s
+    type: ORIGINAL_DST
+    lb_policy: CLUSTER_PROVIDED
+    original_dst_lb_config:
+      metadata_key:
+        key: xxx
+        path:
+        - key: a
+        - key: b
+  )EOF";
+
+  EXPECT_CALL(initialized_, ready());
+  EXPECT_CALL(*cleanup_timer_, enableTimer(_, _));
+  setupFromYaml(yaml);
+
+  OriginalDstCluster::LoadBalancer lb(cluster_);
+  Event::PostCb post_cb;
+
+  NiceMock connection;
+  TestLoadBalancerContext lb_context(&connection);
+
+  envoy::config::core::v3::Metadata dynamic_metadata;
+  (*(*(*dynamic_metadata.mutable_filter_metadata())["xxx"].mutable_fields())["a"]
+        .mutable_struct_value()
+        ->mutable_fields())["b"]
+      .set_string_value("127.0.0.1:6666");
+
+  EXPECT_CALL(Const(connection.stream_info_), dynamicMetadata())
+      .WillRepeatedly(ReturnRef(dynamic_metadata));
+
+  EXPECT_CALL(membership_updated_, ready());
+  EXPECT_CALL(server_context_.dispatcher_, post(_)).WillOnce([&post_cb](Event::PostCb cb) {
+    post_cb = std::move(cb);
+  });
+  HostConstSharedPtr host = lb.chooseHost(&lb_context);
+  post_cb();
+  ASSERT_NE(host, nullptr);
+  EXPECT_EQ("127.0.0.1:6666", host->address()->asString());
+}
+
+TEST_F(OriginalDstClusterTest, UseMetadataKeyInvalidValue) {
+  std::string yaml = R"EOF(
+    name: name
+    connect_timeout: 1.250s
+    type: ORIGINAL_DST
+    lb_policy: CLUSTER_PROVIDED
+    original_dst_lb_config:
+      metadata_key:
+        key: xxx
+        path:
+        - key: a
+  )EOF";
+
+  EXPECT_CALL(initialized_, ready());
+  EXPECT_CALL(*cleanup_timer_, enableTimer(_, _));
+  setupFromYaml(yaml);
+
+  OriginalDstCluster::LoadBalancer lb(cluster_);
+  Event::PostCb post_cb;
+
+  NiceMock connection;
+  TestLoadBalancerContext lb_context(&connection);
+
+  envoy::config::core::v3::Metadata dynamic_metadata;
+  (*(*dynamic_metadata.mutable_filter_metadata())["xxx"].mutable_fields())["a"].set_string_value(
+      "$IP$");
+
+  EXPECT_CALL(Const(connection.stream_info_), dynamicMetadata())
+      .WillRepeatedly(ReturnRef(dynamic_metadata));
+
+  EXPECT_CALL(membership_updated_, ready()).Times(0);
+  EXPECT_CALL(server_context_.dispatcher_, post(_)).Times(0);
+  HostConstSharedPtr host = lb.chooseHost(&lb_context);
+  EXPECT_EQ(host, nullptr);
+  EXPECT_EQ(
+      1, TestUtility::findCounter(stats_store_, "cluster.name.original_dst_host_invalid")->value());
+}
+
+TEST_F(OriginalDstClusterTest, UseMetadataKeyListValue) {
+  std::string yaml = R"EOF(
+    name: name
+    connect_timeout: 1.250s
+    type: ORIGINAL_DST
+    lb_policy: CLUSTER_PROVIDED
+    original_dst_lb_config:
+      metadata_key:
+        key: xxx
+        path:
+        - key: a
+  )EOF";
+
+  EXPECT_CALL(initialized_, ready());
+  EXPECT_CALL(*cleanup_timer_, enableTimer(_, _));
+  setupFromYaml(yaml);
+
+  OriginalDstCluster::LoadBalancer lb(cluster_);
+  Event::PostCb post_cb;
+
+  NiceMock connection;
+  TestLoadBalancerContext lb_context(&connection);
+
+  envoy::config::core::v3::Metadata dynamic_metadata;
+  (*(*dynamic_metadata.mutable_filter_metadata())["xxx"].mutable_fields())["a"]
+      .mutable_list_value()
+      ->add_values()
+      ->set_string_value("127.0.0.1:6666");
+
+  EXPECT_CALL(Const(connection.stream_info_), dynamicMetadata())
+      .WillRepeatedly(ReturnRef(dynamic_metadata));
+
+  EXPECT_CALL(membership_updated_, ready());
+  EXPECT_CALL(server_context_.dispatcher_, post(_)).WillOnce([&post_cb](Event::PostCb cb) {
+    post_cb = std::move(cb);
+  });
+  HostConstSharedPtr host = lb.chooseHost(&lb_context);
+  post_cb();
+  ASSERT_NE(host, nullptr);
+  EXPECT_EQ("127.0.0.1:6666", host->address()->asString());
+}
+
 TEST_F(OriginalDstClusterTest, UseFilterState) {
   std::string yaml = R"EOF(
     name: name
diff --git a/test/mocks/upstream/load_balancer_context.h b/test/mocks/upstream/load_balancer_context.h
index 8d0bc05a857a..ef0fccbe90be 100644
--- a/test/mocks/upstream/load_balancer_context.h
+++ b/test/mocks/upstream/load_balancer_context.h
@@ -1,4 +1,5 @@
 #pragma once
+#include "envoy/stream_info/stream_info.h"
 #include "envoy/upstream/load_balancer.h"
 
 #include "gmock/gmock.h"
@@ -14,6 +15,7 @@ class MockLoadBalancerContext : public LoadBalancerContext {
   MOCK_METHOD(absl::optional, computeHashKey, ());
   MOCK_METHOD(Router::MetadataMatchCriteria*, metadataMatchCriteria, ());
   MOCK_METHOD(const Network::Connection*, downstreamConnection, (), (const));
+  MOCK_METHOD(const StreamInfo::StreamInfo*, requestStreamInfo, (), (const));
   MOCK_METHOD(const Http::RequestHeaderMap*, downstreamHeaders, (), (const));
   MOCK_METHOD(const HealthyAndDegradedLoad&, determinePriorityLoad,
               (const PrioritySet&, const HealthyAndDegradedLoad&,

From 10fc0796c09146ffa6bf0b015604ac3f90596bed Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 27 Jun 2023 09:49:10 +0100
Subject: [PATCH 656/740] build(deps): bump google-cloud-storage from 2.9.0 to
 2.10.0 in /tools/base (#28151)

build(deps): bump google-cloud-storage in /tools/base

Bumps [google-cloud-storage](https://github.com/googleapis/python-storage) from 2.9.0 to 2.10.0.
- [Release notes](https://github.com/googleapis/python-storage/releases)
- [Changelog](https://github.com/googleapis/python-storage/blob/main/CHANGELOG.md)
- [Commits](https://github.com/googleapis/python-storage/compare/v2.9.0...v2.10.0)

---
updated-dependencies:
- dependency-name: google-cloud-storage
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tools/base/requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt
index 5a4ddb0456b1..3cce6b9b608b 100644
--- a/tools/base/requirements.txt
+++ b/tools/base/requirements.txt
@@ -661,9 +661,9 @@ google-cloud-core==2.3.2 \
     --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \
     --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a
     # via google-cloud-storage
-google-cloud-storage==2.9.0 \
-    --hash=sha256:83a90447f23d5edd045e0037982c270302e3aeb45fc1288d2c2ca713d27bad94 \
-    --hash=sha256:9b6ae7b509fc294bdacb84d0f3ea8e20e2c54a8b4bbe39c5707635fec214eff3
+google-cloud-storage==2.10.0 \
+    --hash=sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7 \
+    --hash=sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7
     # via -r requirements.in
 google-crc32c==1.5.0 \
     --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \

From cf4e480771d5bc1bfe3dafc1a60ecfa6e135b1b6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 27 Jun 2023 09:50:14 +0100
Subject: [PATCH 657/740] build(deps): bump google.golang.org/protobuf from
 1.30.0 to 1.31.0 in /contrib/golang/filters/http/test/test_data/echo (#28154)

build(deps): bump google.golang.org/protobuf

Bumps google.golang.org/protobuf from 1.30.0 to 1.31.0.

---
updated-dependencies:
- dependency-name: google.golang.org/protobuf
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 contrib/golang/filters/http/test/test_data/echo/go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/golang/filters/http/test/test_data/echo/go.mod b/contrib/golang/filters/http/test/test_data/echo/go.mod
index 030155522ee3..a69a4acf2b67 100644
--- a/contrib/golang/filters/http/test/test_data/echo/go.mod
+++ b/contrib/golang/filters/http/test/test_data/echo/go.mod
@@ -15,7 +15,7 @@ require (
 	golang.org/x/text v0.7.0 // indirect
 	google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect
 	google.golang.org/grpc v1.25.1 // indirect
-	google.golang.org/protobuf v1.30.0
+	google.golang.org/protobuf v1.31.0
 )
 
 replace github.com/envoyproxy/envoy => ../../../../../../../

From 5ccdb01dc687583c1107d2314d1a32996c325960 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 27 Jun 2023 09:50:24 +0100
Subject: [PATCH 658/740] build(deps): bump google.golang.org/protobuf from
 1.30.0 to 1.31.0 in /examples/golang/simple (#28155)

build(deps): bump google.golang.org/protobuf in /examples/golang/simple

Bumps google.golang.org/protobuf from 1.30.0 to 1.31.0.

---
updated-dependencies:
- dependency-name: google.golang.org/protobuf
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/golang/simple/go.mod | 2 +-
 examples/golang/simple/go.sum | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/examples/golang/simple/go.mod b/examples/golang/simple/go.mod
index 311aabd1aad7..d4831c9133c3 100644
--- a/examples/golang/simple/go.mod
+++ b/examples/golang/simple/go.mod
@@ -7,7 +7,7 @@ go 1.18
 require (
 	github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195
 	github.com/envoyproxy/envoy v1.24.0
-	google.golang.org/protobuf v1.30.0
+	google.golang.org/protobuf v1.31.0
 )
 
 require (
diff --git a/examples/golang/simple/go.sum b/examples/golang/simple/go.sum
index ab4004b7c937..12d44a0649ce 100644
--- a/examples/golang/simple/go.sum
+++ b/examples/golang/simple/go.sum
@@ -4,8 +4,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 h1:58f1tJ1ra+zFINPlwLWvQsR9CzAKt2e+EWV2yX9oXQ4=
 github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go v0.0.0-20230324062628-466cf15f1790 h1:0gwaogAlbAdFSlmhaJ5wZTz7OyZ+FJ0jnU+C/eUKwNs=
-github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go v0.0.0-20230324062628-466cf15f1790/go.mod h1:fhJpuCrm8CxKONqp/fBZM3IeMxnK2L9edMVb170sfzA=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@@ -58,7 +56,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

From 2a145762586824998d14a1ccabafd3580df46305 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 27 Jun 2023 09:50:53 +0100
Subject: [PATCH 659/740] build(deps): bump google.golang.org/protobuf from
 1.30.0 to 1.31.0 in /contrib/golang/filters/http/test/test_data/routeconfig
 (#28152)

build(deps): bump google.golang.org/protobuf

Bumps google.golang.org/protobuf from 1.30.0 to 1.31.0.

---
updated-dependencies:
- dependency-name: google.golang.org/protobuf
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 contrib/golang/filters/http/test/test_data/routeconfig/go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/golang/filters/http/test/test_data/routeconfig/go.mod b/contrib/golang/filters/http/test/test_data/routeconfig/go.mod
index be10ae3ff0e6..b0096b62676a 100644
--- a/contrib/golang/filters/http/test/test_data/routeconfig/go.mod
+++ b/contrib/golang/filters/http/test/test_data/routeconfig/go.mod
@@ -15,7 +15,7 @@ require (
 	golang.org/x/text v0.7.0 // indirect
 	google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect
 	google.golang.org/grpc v1.25.1 // indirect
-	google.golang.org/protobuf v1.30.0
+	google.golang.org/protobuf v1.31.0
 )
 
 replace github.com/envoyproxy/envoy => ../../../../../../../

From b066af3149a3c67a30c7ad8677c3a2b98f278f31 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 27 Jun 2023 11:36:45 +0000
Subject: [PATCH 660/740] build(deps): bump google.golang.org/protobuf from
 1.30.0 to 1.31.0 (#28153)

Bumps google.golang.org/protobuf from 1.30.0 to 1.31.0.

---
updated-dependencies:
- dependency-name: google.golang.org/protobuf
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 59ea33ddcbfb..61b62bbe8bfe 100644
--- a/go.mod
+++ b/go.mod
@@ -2,4 +2,4 @@ module github.com/envoyproxy/envoy
 
 go 1.18
 
-require google.golang.org/protobuf v1.30.0
+require google.golang.org/protobuf v1.31.0
diff --git a/go.sum b/go.sum
index 1838366909dd..9ea5597b8346 100644
--- a/go.sum
+++ b/go.sum
@@ -4,5 +4,5 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=

From 68cbb10b4ea1093211fb39ce17874ee122f132c3 Mon Sep 17 00:00:00 2001
From: Tianyu <72890320+tyxia@users.noreply.github.com>
Date: Tue, 27 Jun 2023 09:13:20 -0400
Subject: [PATCH 661/740] matcher: add handler if input proto is corrupted
 (#28003)

Signed-off-by: tyxia 
---
 source/common/matcher/matcher.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/common/matcher/matcher.h b/source/common/matcher/matcher.h
index 173ed906714b..ee29f4e26aea 100644
--- a/source/common/matcher/matcher.h
+++ b/source/common/matcher/matcher.h
@@ -166,7 +166,7 @@ class MatchTreeFactory : public OnMatchFactory {
     case MatcherType::MATCHER_TYPE_NOT_SET:
       return createAnyMatcher(config);
     }
-    return nullptr;
+    PANIC_DUE_TO_CORRUPT_ENUM;
   }
 
   absl::optional>

From 5c56aec3f7c5ee71eb6f312d0158d56f5abd10af Mon Sep 17 00:00:00 2001
From: "Adi (Suissa) Peleg" 
Date: Tue, 27 Jun 2023 09:19:32 -0400
Subject: [PATCH 662/740] active-hc: preserve active health check status when
 the locality is updated (#27900)

Signed-off-by: Adi Suissa-Peleg 
---
 changelogs/current.yaml                   |   5 +
 envoy/upstream/upstream.h                 |  12 ++
 source/common/runtime/runtime_features.cc |   1 +
 source/common/upstream/upstream_impl.cc   |  44 +++-
 source/common/upstream/upstream_impl.h    |   2 +
 test/extensions/clusters/eds/eds_test.cc  | 160 ++++++++++++++
 test/integration/BUILD                    |   1 +
 test/integration/eds_integration_test.cc  | 249 ++++++++++++++++++++++
 test/mocks/upstream/host.h                |   2 +
 9 files changed, 471 insertions(+), 5 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 84df92e3278d..c3451e461f29 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -28,6 +28,11 @@ behavior_changes:
   change: |
     Apply header mutation rules from the ext_proc config to the ImmediateResponse. This behavior change can be temporarily
     reverted by setting the runtime guard ``envoy_reloadable_features_immediate_response_use_filter_mutation_rule`` to false.
+- area: active health check
+  change: |
+    Preserve the active-health check status of a host after a cluster/assignment update. This is now preserved in cases
+    where the assignment updates a host's locality. This behavioral change can be temporarily reverted by setting the
+    runtime flag ``envoy.reloadable_features.keep_endpoint_active_hc_status_on_locality_update`` to false.
 
 minor_behavior_changes:
 # *Changes that may cause incompatibilities for some users, but should not for most*
diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h
index 348e586510cd..29fa641d4396 100644
--- a/envoy/upstream/upstream.h
+++ b/envoy/upstream/upstream.h
@@ -175,6 +175,18 @@ class Host : virtual public HostDescription {
    */
   virtual void healthFlagSet(HealthFlag flag) PURE;
 
+  /**
+   * Atomically get multiple health flags that are set for a host. Flags are specified
+   * as a bitset of HealthFlags.
+   */
+  virtual uint32_t healthFlagsGetAll() const PURE;
+
+  /**
+   * Atomically set the health flag for a host. Flags are specified as a bitset
+   * of HealthFlags.
+   */
+  virtual void healthFlagsSetAll(uint32_t bits) PURE;
+
   enum class Health {
     /**
      * Host is unhealthy and is not able to serve traffic. A host may be marked as unhealthy either
diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc
index b4fe9e99cbef..d534a385ce25 100644
--- a/source/common/runtime/runtime_features.cc
+++ b/source/common/runtime/runtime_features.cc
@@ -54,6 +54,7 @@ RUNTIME_GUARD(envoy_reloadable_features_http_reject_path_with_fragment);
 RUNTIME_GUARD(envoy_reloadable_features_ignore_optional_option_from_hcm_for_route_config);
 RUNTIME_GUARD(envoy_reloadable_features_immediate_response_use_filter_mutation_rule);
 RUNTIME_GUARD(envoy_reloadable_features_initialize_upstream_filters);
+RUNTIME_GUARD(envoy_reloadable_features_keep_endpoint_active_hc_status_on_locality_update);
 RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name);
 RUNTIME_GUARD(envoy_reloadable_features_no_full_scan_certs_on_sni_mismatch);
 RUNTIME_GUARD(envoy_reloadable_features_oauth_header_passthrough_fix);
diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc
index 0956bfe13252..a48063e24e5b 100644
--- a/source/common/upstream/upstream_impl.cc
+++ b/source/common/upstream/upstream_impl.cc
@@ -2111,13 +2111,47 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(
       }
 
       // If we are depending on a health checker, we initialize to unhealthy.
+      // If there's an existing host with the same health checker, the
+      // active health-status is kept.
       if (health_checker_ != nullptr && !host->disableActiveHealthCheck()) {
-        host->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC);
+        if (Runtime::runtimeFeatureEnabled(
+                "envoy.reloadable_features.keep_endpoint_active_hc_status_on_locality_update")) {
+          if (existing_host_found && !health_check_address_changed &&
+              !active_health_check_flag_changed) {
+            // If there's an existing host, use the same active health-status.
+            // The existing host can be marked PENDING_ACTIVE_HC or
+            // ACTIVE_HC_TIMEOUT if it is also marked with FAILED_ACTIVE_HC.
+            ASSERT(!existing_host->second->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC) ||
+                   existing_host->second->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC));
+            ASSERT(!existing_host->second->healthFlagGet(Host::HealthFlag::ACTIVE_HC_TIMEOUT) ||
+                   existing_host->second->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC));
+
+            constexpr uint32_t active_hc_statuses_mask =
+                enumToInt(Host::HealthFlag::FAILED_ACTIVE_HC) |
+                enumToInt(Host::HealthFlag::DEGRADED_ACTIVE_HC) |
+                enumToInt(Host::HealthFlag::PENDING_ACTIVE_HC) |
+                enumToInt(Host::HealthFlag::ACTIVE_HC_TIMEOUT);
+
+            const uint32_t existing_host_statuses = existing_host->second->healthFlagsGetAll();
+            host->healthFlagsSetAll(existing_host_statuses & active_hc_statuses_mask);
+          } else {
+            // No previous known host, mark it as failed active HC.
+            host->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC);
+
+            // If we want to exclude hosts until they have been health checked, mark them with
+            // a flag to indicate that they have not been health checked yet.
+            if (info_->warmHosts()) {
+              host->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC);
+            }
+          }
+        } else {
+          host->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC);
 
-        // If we want to exclude hosts until they have been health checked, mark them with
-        // a flag to indicate that they have not been health checked yet.
-        if (info_->warmHosts()) {
-          host->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC);
+          // If we want to exclude hosts until they have been health checked, mark them with
+          // a flag to indicate that they have not been health checked yet.
+          if (info_->warmHosts()) {
+            host->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC);
+          }
         }
       }
 
diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h
index eae0be2961a0..ccd00a653c29 100644
--- a/source/common/upstream/upstream_impl.h
+++ b/source/common/upstream/upstream_impl.h
@@ -379,6 +379,8 @@ class HostImpl : public HostDescriptionImpl,
   void healthFlagClear(HealthFlag flag) override { health_flags_ &= ~enumToInt(flag); }
   bool healthFlagGet(HealthFlag flag) const override { return health_flags_ & enumToInt(flag); }
   void healthFlagSet(HealthFlag flag) final { health_flags_ |= enumToInt(flag); }
+  uint32_t healthFlagsGetAll() const override { return health_flags_; }
+  void healthFlagsSetAll(uint32_t bits) override { health_flags_ |= bits; }
 
   void setHealthChecker(HealthCheckHostMonitorPtr&& health_checker) override {
     setHealthCheckerImpl(std::move(health_checker));
diff --git a/test/extensions/clusters/eds/eds_test.cc b/test/extensions/clusters/eds/eds_test.cc
index df77453512a5..ba70e9a711fd 100644
--- a/test/extensions/clusters/eds/eds_test.cc
+++ b/test/extensions/clusters/eds/eds_test.cc
@@ -1578,6 +1578,166 @@ TEST_F(EdsTest, EndpointMovedToNewPriorityWithHealthAddressChange) {
   }
 }
 
+// Verifies that if an endpoint's locality is updated, the active hc flags are preserved,
+// unless the health checker is updated.
+TEST_F(EdsTest, ActiveHealthCheckFlagsEndpointMovedToNewLocality) {
+  envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment;
+  cluster_load_assignment.set_cluster_name("fare");
+  resetCluster();
+
+  auto health_checker = std::make_shared();
+  EXPECT_CALL(*health_checker, start());
+  EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)).Times(2);
+  cluster_->setHealthChecker(health_checker);
+
+  auto* endpoints = cluster_load_assignment.add_endpoints();
+  auto* endpoint = endpoints->add_lb_endpoints()->mutable_endpoint();
+  auto* socket_address = endpoint->mutable_address()->mutable_socket_address();
+  socket_address->set_address("1.2.3.4");
+  socket_address->set_port_value(80);
+
+  // Start with an endpoint in locality zone1, it should be marked FAILED_ACTIVE_HC.
+  endpoints->mutable_locality()->set_zone("zone1");
+  doOnConfigUpdateVerifyNoThrow(cluster_load_assignment);
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    EXPECT_EQ(hosts.size(), 1);
+    EXPECT_EQ(hosts[0]->locality().zone(), "zone1");
+
+    // When active-HC is used, the state is initialized to FAILED_ACTIVE_HC.
+    EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC));
+  }
+
+  // Verify that the host is added (to a new locality) and removed (from its
+  // current locality) as part of an update.
+  auto member_update_cb =
+      cluster_->prioritySet().addMemberUpdateCb([&](const auto& added, const auto& removed) {
+        EXPECT_EQ(1, added.size());
+        EXPECT_EQ(1, removed.size());
+      });
+
+  // Validate that moving an healthy endpoint to another locality keeps
+  // it as healthy.
+  {
+    // Set the endpoint in healthy status, and update its zone.
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    // Mark the host as healthy.
+    hosts[0]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC);
+  }
+  endpoints->mutable_locality()->set_zone("zone2");
+  doOnConfigUpdateVerifyNoThrow(cluster_load_assignment);
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    EXPECT_EQ(hosts[0]->locality().zone(), "zone2");
+
+    // The endpoint was healthy in the previous locality, so moving it
+    // to another locality should preserve that.
+    EXPECT_FALSE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC));
+  }
+
+  // Validate that moving an unhealthy endpoint to another locality keeps
+  // it as unhealthy.
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    hosts[0]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC);
+  }
+  endpoints->mutable_locality()->set_zone("zone3");
+  doOnConfigUpdateVerifyNoThrow(cluster_load_assignment);
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    EXPECT_EQ(hosts.size(), 1);
+    EXPECT_EQ(hosts[0]->locality().zone(), "zone3");
+
+    // The endpoint was in failed status in the previous priority, so moving it
+    // to another locality should preserve that.
+    EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC));
+  }
+
+  // Validate that moving a degraded endpoint to another locality keeps
+  // it as degraded.
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    hosts[0]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC);
+    hosts[0]->healthFlagSet(Host::HealthFlag::DEGRADED_ACTIVE_HC);
+  }
+  endpoints->mutable_locality()->set_zone("zone4");
+  doOnConfigUpdateVerifyNoThrow(cluster_load_assignment);
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    EXPECT_EQ(hosts.size(), 1);
+    EXPECT_EQ(hosts[0]->locality().zone(), "zone4");
+
+    // The endpoint was in degraded status in the original locality, so moving it
+    // to another locality should preserve that.
+    EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::DEGRADED_ACTIVE_HC));
+  }
+
+  // Validate that moving a endpoint marked as timeout active health check to
+  // another locality keeps that flag.
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    // The code requires that ACTIVE_HC_TIMEOUT is set only if FAILED_ACTIVE_HC
+    // is also set.
+    hosts[0]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC);
+    hosts[0]->healthFlagSet(Host::HealthFlag::ACTIVE_HC_TIMEOUT);
+  }
+  endpoints->mutable_locality()->set_zone("zone5");
+  doOnConfigUpdateVerifyNoThrow(cluster_load_assignment);
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    EXPECT_EQ(hosts.size(), 1);
+    EXPECT_EQ(hosts[0]->locality().zone(), "zone5");
+
+    // The endpoint was in timeout status in the original locality, so moving it
+    // to another locality keeps the flag..
+    EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::ACTIVE_HC_TIMEOUT));
+    EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC));
+  }
+
+  // Validate that moving a endpoint marked as pending active health check to
+  // another locality keeps that flag.
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    // The code requires that PENDING_ACTIVE_HC is set only if FAILED_ACTIVE_HC
+    // is also set.
+    hosts[0]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC);
+    hosts[0]->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC);
+    hosts[0]->healthFlagClear(Host::HealthFlag::ACTIVE_HC_TIMEOUT);
+  }
+  endpoints->mutable_locality()->set_zone("zone6");
+  doOnConfigUpdateVerifyNoThrow(cluster_load_assignment);
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    EXPECT_EQ(hosts.size(), 1);
+    EXPECT_EQ(hosts[0]->locality().zone(), "zone6");
+
+    // The endpoint was in timeout status in the original priority, but its
+    // health checker host changes, so now it is initialized to unhealthy.
+    EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC));
+    EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC));
+  }
+
+  // Validate that updating the locality and the health checker of a healthy
+  // endpoint, marks it as failing active HC.
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    hosts[0]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC);
+    hosts[0]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC);
+  }
+  endpoints->mutable_locality()->set_zone("zone7");
+  endpoint->mutable_health_check_config()->set_port_value(90);
+  doOnConfigUpdateVerifyNoThrow(cluster_load_assignment);
+  {
+    auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts();
+    EXPECT_EQ(hosts.size(), 1);
+    EXPECT_EQ(hosts[0]->locality().zone(), "zone7");
+
+    // The endpoint was in healthy status in the original priority, but its
+    // health checker host changes, so now it is initialized to unhealthy.
+    EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC));
+  }
+}
+
 // Validates that we correctly update the host list when a new overprovisioning factor is set.
 TEST_F(EdsTest, EndpointAddedWithNewOverprovisioningFactor) {
   envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment;
diff --git a/test/integration/BUILD b/test/integration/BUILD
index 451b949851b4..3f60a1f16ed9 100644
--- a/test/integration/BUILD
+++ b/test/integration/BUILD
@@ -162,6 +162,7 @@ envoy_cc_test(
         "//test/config:utility_lib",
         "//test/integration/filters:eds_ready_filter_config_lib",
         "//test/test_common:network_utility_lib",
+        "//test/test_common:test_runtime_lib",
         "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
diff --git a/test/integration/eds_integration_test.cc b/test/integration/eds_integration_test.cc
index 7648a3a5f8a4..9b6e142a67f4 100644
--- a/test/integration/eds_integration_test.cc
+++ b/test/integration/eds_integration_test.cc
@@ -10,6 +10,7 @@
 #include "test/config/utility.h"
 #include "test/integration/http_integration.h"
 #include "test/test_common/network_utility.h"
+#include "test/test_common/test_runtime.h"
 
 #include "gtest/gtest.h"
 
@@ -56,6 +57,16 @@ class EdsIntegrationTest : public testing::TestWithParamgauge("cluster.cluster_0.membership_total")->value());
 }
 
+// Validates that updating a locality of an actively checked endpoint, reuses
+// the previously determined health status.
+TEST_P(EdsIntegrationTest, LocalityUpdateActiveHealthStatusReuse) {
+  // setup with active-HC.
+  initializeTest(true, [](envoy::config::cluster::v3::Cluster& cluster) {
+    // Disable the healthy panic threshold, preventing using a non-healthy host.
+    cluster.mutable_common_lb_config()->mutable_healthy_panic_threshold();
+    cluster.mutable_health_checks(0)->mutable_interval()->CopyFrom(
+        Protobuf::util::TimeUtil::MillisecondsToDuration(10000));
+    cluster.mutable_health_checks(0)->mutable_no_traffic_interval()->CopyFrom(
+        Protobuf::util::TimeUtil::MillisecondsToDuration(10000));
+    cluster.set_ignore_health_on_host_removal(true);
+  });
+  envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment;
+
+  // Create an EDS message with a duplicated endpoint in 2 localities.
+  TestUtility::loadFromYaml(R"EOF(
+    cluster_name: "cluster_0"
+    endpoints:
+    - lb_endpoints:
+      - endpoint:
+      priority: 0
+      locality:
+        sub_zone: xx
+)EOF",
+                            cluster_load_assignment);
+  auto* endpoint_A = cluster_load_assignment.mutable_endpoints(0)->mutable_lb_endpoints(0);
+  setUpstreamAddress(0, *endpoint_A);
+
+  // Send the first EDS response.
+  sendPlainEds(cluster_load_assignment);
+
+  // Wait for the first HC to arrive to the upstream.
+  waitForNextUpstreamRequest();
+  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true);
+  test_server_->waitForCounterEq("cluster.cluster_0.health_check.attempt", 1);
+  test_server_->waitForCounterGe("cluster.cluster_0.health_check.success", 1);
+  cleanupUpstreamAndDownstream();
+
+  // After the first hc there should be no upstream healthy host, and any
+  // downstream request will fail, until the update-merge-window ends.
+  EXPECT_EQ(0, test_server_->counter("cluster.cluster_0.upstream_cx_none_healthy")->value());
+  BufferingStreamDecoderPtr response = IntegrationUtil::makeSingleRequest(
+      lookupPort("http"), "GET", "/cluster_0", "", downstream_protocol_, version_, "foo.com");
+  ASSERT_TRUE(response->complete());
+  EXPECT_EQ("503", response->headers().getStatusValue());
+  EXPECT_EQ(1, test_server_->counter("cluster.cluster_0.upstream_cx_none_healthy")->value());
+  cleanupUpstreamAndDownstream();
+  response.reset();
+
+  // Wait for scheduled updates (update-merge-window).
+  test_server_->waitForCounterEq("cluster_manager.cluster_updated_via_merge", 1);
+
+  // Now the upstream should be healthy.
+  // There should only be 1 endpoint, and it should be healthy (according to EDS).
+  EXPECT_EQ(0, test_server_->counter("cluster.cluster_0.health_check.failure")->value());
+  EXPECT_EQ(1, test_server_->counter("cluster.cluster_0.membership_change")->value());
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value());
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_healthy")->value());
+
+  // Now a downstream request can be passed to the upstream.
+  registerTestServerPorts({"http"});
+  testRouterHeaderOnlyRequestAndResponse();
+  cleanupUpstreamAndDownstream();
+
+  // 2nd change: Connection is open, send an EDS response changing the order of the
+  // priorities in the message, but doesn't impact localities. This should have no
+  // impact on the host's health status, and cluster availability.
+  cluster_load_assignment.mutable_endpoints(0)->set_priority(1);
+  sendPlainEds(cluster_load_assignment, true);
+
+  // No locality update, so the previous pool can be used, and a request will be
+  // served.
+  testRouterHeaderOnlyRequestAndResponse();
+  cleanupUpstreamAndDownstream();
+
+  // There should be 3 membership changes (+2 compared to the previous, as the
+  // memberships of priority 0 and 1 have changed).
+  test_server_->waitForCounterEq("cluster.cluster_0.membership_change", 3);
+  // There should still be be 1 endpoint, and it should stay healthy.
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value());
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_healthy")->value());
+
+  // 3rd change: Connection is open, send an EDS response changing the localities in the
+  // message. This will cause recreation of the LB, resending the HC to the "new"
+  // endpoint. This will impact cluster availability.
+  cluster_load_assignment.mutable_endpoints(0)->mutable_locality()->set_sub_zone("xx2");
+  sendPlainEds(cluster_load_assignment, true);
+
+  // Wait for first HC for the tcp proxy after EDS update with localities.
+  waitForNextUpstreamRequest();
+  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true);
+  cleanupUpstreamAndDownstream();
+
+  // Although the host's locality was updated, it will use the previously
+  // detected active health-status, so the upstream should be healthy, and a
+  // request will be served.
+  testRouterHeaderOnlyRequestAndResponse();
+  cleanupUpstreamAndDownstream();
+  // There should be 3 membership changes (+1 compared to the previous, as the
+  // membership of the first locality has changed).
+  test_server_->waitForCounterEq("cluster.cluster_0.membership_change", 4);
+  // There should still be be 1 endpoint, and it should stay healthy.
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value());
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_healthy")->value());
+}
+
+// Validates that updating a locality of an actively checked endpoint, does
+// not reuse the previously determined health status.
+// The test should be removed once
+// envoy.reloadable_features.keep_endpoint_active_hc_status_on_locality_update
+// is deprecated.
+TEST_P(EdsIntegrationTest, LocalityUpdateActiveHealthStatusNoReuse) {
+  TestScopedRuntime scoped_runtime;
+  scoped_runtime.mergeValues(
+      {{"envoy.reloadable_features.keep_endpoint_active_hc_status_on_locality_update", "false"}});
+  // setup with active-HC.
+  initializeTest(true, [](envoy::config::cluster::v3::Cluster& cluster) {
+    // Disable the healthy panic threshold, preventing using a non-healthy host.
+    cluster.mutable_common_lb_config()->mutable_healthy_panic_threshold();
+    cluster.mutable_health_checks(0)->mutable_interval()->CopyFrom(
+        Protobuf::util::TimeUtil::MillisecondsToDuration(10000));
+    cluster.mutable_health_checks(0)->mutable_no_traffic_interval()->CopyFrom(
+        Protobuf::util::TimeUtil::MillisecondsToDuration(10000));
+    cluster.set_ignore_health_on_host_removal(true);
+  });
+  envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment;
+
+  // Create an EDS message with a duplicated endpoint in 2 localities.
+  TestUtility::loadFromYaml(R"EOF(
+    cluster_name: "cluster_0"
+    endpoints:
+    - lb_endpoints:
+      - endpoint:
+      priority: 0
+      locality:
+        sub_zone: xx
+)EOF",
+                            cluster_load_assignment);
+  auto* endpoint_A = cluster_load_assignment.mutable_endpoints(0)->mutable_lb_endpoints(0);
+  setUpstreamAddress(0, *endpoint_A);
+
+  // Send the first EDS response.
+  sendPlainEds(cluster_load_assignment);
+
+  // Wait for the first HC to arrive to the upstream.
+  waitForNextUpstreamRequest();
+  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true);
+  test_server_->waitForCounterEq("cluster.cluster_0.health_check.attempt", 1);
+  test_server_->waitForCounterGe("cluster.cluster_0.health_check.success", 1);
+  cleanupUpstreamAndDownstream();
+
+  // After the first hc there should be no upstream healthy host, and any
+  // downstream request will fail, until the update-merge-window ends.
+  EXPECT_EQ(0, test_server_->counter("cluster.cluster_0.upstream_cx_none_healthy")->value());
+  BufferingStreamDecoderPtr response = IntegrationUtil::makeSingleRequest(
+      lookupPort("http"), "GET", "/cluster_0", "", downstream_protocol_, version_, "foo.com");
+  ASSERT_TRUE(response->complete());
+  EXPECT_EQ("503", response->headers().getStatusValue());
+  EXPECT_EQ(1, test_server_->counter("cluster.cluster_0.upstream_cx_none_healthy")->value());
+  cleanupUpstreamAndDownstream();
+  response.reset();
+
+  // Wait for scheduled updates (update-merge-window).
+  test_server_->waitForCounterEq("cluster_manager.cluster_updated_via_merge", 1);
+
+  // Now the upstream should be healthy.
+  // There should only be 1 endpoint, and it should be healthy (according to EDS).
+  EXPECT_EQ(0, test_server_->counter("cluster.cluster_0.health_check.failure")->value());
+  EXPECT_EQ(1, test_server_->counter("cluster.cluster_0.membership_change")->value());
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value());
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_healthy")->value());
+
+  // Now a downstream request can be passed to the upstream.
+  registerTestServerPorts({"http"});
+  testRouterHeaderOnlyRequestAndResponse();
+  cleanupUpstreamAndDownstream();
+
+  // 2nd change: Connection is open, send an EDS response changing the order of the
+  // priorities in the message, but doesn't impact localities. This should have no
+  // impact on the host's health status, and cluster availability.
+  cluster_load_assignment.mutable_endpoints(0)->set_priority(1);
+  sendPlainEds(cluster_load_assignment, true);
+
+  // No locality update, so the previous pool can be used, and a request will be
+  // served.
+  testRouterHeaderOnlyRequestAndResponse();
+  cleanupUpstreamAndDownstream();
+
+  // There should be 3 membership changes (+2 compared to the previous, as the
+  // memberships of priority 0 and 1 have changed).
+  test_server_->waitForCounterEq("cluster.cluster_0.membership_change", 3);
+  // There should still be be 1 endpoint, and it should stay healthy.
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value());
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_healthy")->value());
+
+  // 3rd change: Connection is open, send an EDS response changing the localities in the
+  // message. This will cause recreation of the LB, resending the HC to the "new"
+  // endpoint. This will impact cluster availability.
+  cluster_load_assignment.mutable_endpoints(0)->mutable_locality()->set_sub_zone("xx2");
+  sendPlainEds(cluster_load_assignment, true);
+
+  // Wait for first HC for the tcp proxy after EDS update with localities.
+  waitForNextUpstreamRequest();
+  upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true);
+  cleanupUpstreamAndDownstream();
+
+  // After the first hc there should be no upstream healthy host, and any
+  // downstream request will fail, until the update-merge-window ends.
+  //
+  // when reusing health-status, Envoy will be able to connect to the upstream
+  // even before receiving the HC response, because the endpoint was already
+  // set to healthy.
+  {
+    EXPECT_EQ(1, test_server_->counter("cluster.cluster_0.upstream_cx_none_healthy")->value());
+    response = IntegrationUtil::makeSingleRequest(lookupPort("http"), "GET", "/cluster_0", "",
+                                                  downstream_protocol_, version_, "foo.com");
+    ASSERT_TRUE(response->complete());
+    EXPECT_EQ("503", response->headers().getStatusValue());
+    cleanupUpstreamAndDownstream();
+    response.reset();
+    EXPECT_EQ(2, test_server_->counter("cluster.cluster_0.upstream_cx_none_healthy")->value());
+
+    // Wait for scheduled updates (update-merge-window).
+    test_server_->waitForCounterEq("cluster_manager.cluster_updated_via_merge", 3);
+  }
+
+  // Now the upstream should be healthy, and a request will be served.
+  testRouterHeaderOnlyRequestAndResponse();
+  cleanupUpstreamAndDownstream();
+  // There should be 3 membership changes (+1 compared to the previous, as the
+  // membership of the first locality has changed).
+  test_server_->waitForCounterEq("cluster.cluster_0.membership_change", 4);
+  // There should still be be 1 endpoint, and it should stay healthy.
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value());
+  EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_healthy")->value());
+}
+
 // Verifies that cluster warming proceeds even if a host is deleted before health checks complete.
 // This is a regression test for https://github.com/envoyproxy/envoy/issues/17836.
 TEST_P(EdsIntegrationTest, FinishWarmingIgnoreHealthCheck) {
diff --git a/test/mocks/upstream/host.h b/test/mocks/upstream/host.h
index 70ff984ec30f..757a64f549f1 100644
--- a/test/mocks/upstream/host.h
+++ b/test/mocks/upstream/host.h
@@ -194,6 +194,8 @@ class MockHost : public Host {
   MOCK_METHOD(void, healthFlagClear, (HealthFlag flag));
   MOCK_METHOD(bool, healthFlagGet, (HealthFlag flag), (const));
   MOCK_METHOD(void, healthFlagSet, (HealthFlag flag));
+  MOCK_METHOD(uint32_t, healthFlagsGetAll, (), (const));
+  MOCK_METHOD(void, healthFlagsSetAll, (uint32_t));
   MOCK_METHOD(Host::Health, coarseHealth, (), (const));
   MOCK_METHOD(Host::HealthStatus, healthStatus, (), (const));
   MOCK_METHOD(const std::string&, hostnameForHealthChecks, (), (const));

From 66c7caaa96874d1a9beb9d69aba6c1858772bf75 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Tue, 27 Jun 2023 14:36:32 +0100
Subject: [PATCH 663/740] deps: Bump `bazel_skylib` -> 1.4.2 (#28067)

Fix #27747

Signed-off-by: Ryan Northey 
---
 api/bazel/repository_locations.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl
index 97da4496ef50..fbfe8fd17fc3 100644
--- a/api/bazel/repository_locations.bzl
+++ b/api/bazel/repository_locations.bzl
@@ -4,9 +4,9 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "bazel-skylib",
         project_desc = "Common useful functions and rules for Bazel",
         project_url = "https://github.com/bazelbuild/bazel-skylib",
-        version = "1.4.1",
-        sha256 = "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7",
-        release_date = "2023-02-09",
+        version = "1.4.2",
+        sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa",
+        release_date = "2023-05-31",
         urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz"],
         use_category = ["api"],
         license = "Apache-2.0",

From d93f3182c01265559d2d528ddaf31c88fdeecc6d Mon Sep 17 00:00:00 2001
From: "Adi (Suissa) Peleg" 
Date: Tue, 27 Jun 2023 10:20:32 -0400
Subject: [PATCH 664/740] fuzz: protobuf mutator utility functions cleanup
 (#28143)

fuzz: utility functions cleanup

Signed-off-by: Adi Suissa-Peleg 
---
 .../fuzz/network_readfilter_fuzz_test.cc      | 22 ++++++++++++++-----
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc
index 1f9ecc3250c5..afe5c1451425 100644
--- a/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc
+++ b/test/extensions/filters/network/common/fuzz/network_readfilter_fuzz_test.cc
@@ -11,10 +11,6 @@
 
 #include "libprotobuf_mutator/src/libfuzzer/libfuzzer_macro.h"
 
-// TODO: Needed for protobuf_mutator::ParseTextMessage() and ::SaveMessageAsText(). Replace by
-// envoy-style input parsing methods.
-#include "src/text_format.h" // from libprotobuf_mutator/
-
 namespace Envoy {
 namespace Extensions {
 namespace NetworkFilters {
@@ -63,8 +59,14 @@ extern "C" size_t LLVMFuzzerCustomMutator(uint8_t* data, size_t size, size_t max
     return true;
   }();
 
+  // Attempt reading the input string as a text proto.
   FuzzerProtoType input;
-  protobuf_mutator::ParseTextMessage(data, size, &input);
+  if (!input.ParseFromString(std::string(reinterpret_cast(data), size))) {
+    return 0;
+  }
+
+  // Mutate the config part of the test case with a low probability, and
+  // the actions part with high probability.
   if (random.random() % 100 < config_mutation_probability) {
     test::extensions::filters::network::FuzzHelperForActions actions;
     *actions.mutable_actions() = std::move(*input.mutable_actions());
@@ -75,7 +77,15 @@ extern "C" size_t LLVMFuzzerCustomMutator(uint8_t* data, size_t size, size_t max
     ensuredValidFilter(random.random(), input.mutable_config());
   }
 
-  return protobuf_mutator::SaveMessageAsText(input, data, max_size);
+  // Convert the proto back to a string and validate its length.
+  std::string input_str;
+  if (!Protobuf::TextFormat::PrintToString(input, &input_str) || (input_str.size() > max_size)) {
+    return 0;
+  }
+  // Copy the input string back to data. data is at least max_size bytes,
+  // so it is safe to copy input_text because of the validation above.
+  safeMemcpyUnsafeDst(data, input_str.data());
+  return input_str.size();
 }
 
 DEFINE_CUSTOM_PROTO_CROSSOVER_IMPL(false, FuzzerProtoType)

From 83d633d9ff0f0a9ea550ffda2fef20f7f9f3ede1 Mon Sep 17 00:00:00 2001
From: Kuo-Chung Hsu 
Date: Tue, 27 Jun 2023 08:04:05 -0700
Subject: [PATCH 665/740] Support generic value getter for json utility
 (#27926)

Commit Message:
We need a generic value getter for #27645
However, exception handling is expensive if we tried every type.

Additional Description:
Risk Level: low
Testing: unit

Signed-off-by: kuochunghsu 
---
 envoy/json/json_object.h             | 11 +++++++++++
 source/common/json/json_internal.cc  | 22 ++++++++++++++++++++++
 test/common/json/json_loader_test.cc | 15 +++++++++++++++
 3 files changed, 48 insertions(+)

diff --git a/envoy/json/json_object.h b/envoy/json/json_object.h
index 79ce0d9c9765..486954a1d3b8 100644
--- a/envoy/json/json_object.h
+++ b/envoy/json/json_object.h
@@ -9,6 +9,8 @@
 #include "envoy/common/exception.h"
 #include "envoy/common/pure.h"
 
+#include "absl/types/variant.h"
+
 namespace Envoy {
 namespace Json {
 class Object;
@@ -26,6 +28,8 @@ class Exception : public EnvoyException {
   Exception(const std::string& message) : EnvoyException(message) {}
 };
 
+using ValueType = absl::variant;
+
 /**
  * Wraps an individual JSON node.
  */
@@ -40,6 +44,13 @@ class Object {
    */
   virtual std::vector asObjectArray() const PURE;
 
+  /**
+   * Get a bool, integer, double or string value by name.
+   * @param name supplies the key name.
+   * @return bool the value.
+   */
+  virtual ValueType getValue(const std::string& name) const PURE;
+
   /**
    * Get a boolean value by name.
    * @param name supplies the key name.
diff --git a/source/common/json/json_internal.cc b/source/common/json/json_internal.cc
index 6eaf5d8fa658..ad775d2adf35 100644
--- a/source/common/json/json_internal.cc
+++ b/source/common/json/json_internal.cc
@@ -60,6 +60,7 @@ class Field : public Object {
 
   uint64_t hash() const override;
 
+  ValueType getValue(const std::string& name) const override;
   bool getBoolean(const std::string& name) const override;
   bool getBoolean(const std::string& name, bool default_value) const override;
   double getDouble(const std::string& name) const override;
@@ -383,6 +384,27 @@ nlohmann::json Field::asJsonDocument() const {
 
 uint64_t Field::hash() const { return HashUtil::xxHash64(asJsonString()); }
 
+ValueType Field::getValue(const std::string& name) const {
+  auto value_itr = value_.object_value_.find(name);
+  if (value_itr == value_.object_value_.end()) {
+    throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_,
+                                line_number_end_));
+  }
+  switch (value_itr->second->type_) {
+  case Type::Boolean:
+    return value_itr->second->booleanValue();
+  case Type::Double:
+    return value_itr->second->doubleValue();
+  case Type::Integer:
+    return value_itr->second->integerValue();
+  case Type::String:
+    return value_itr->second->stringValue();
+  default:
+    throw Exception(fmt::format("key '{}' not a value type from lines {}-{}", name,
+                                line_number_start_, line_number_end_));
+  }
+}
+
 bool Field::getBoolean(const std::string& name) const {
   checkType(Type::Object);
   auto value_itr = value_.object_value_.find(name);
diff --git a/test/common/json/json_loader_test.cc b/test/common/json/json_loader_test.cc
index a95341847a8c..552c01916396 100644
--- a/test/common/json/json_loader_test.cc
+++ b/test/common/json/json_loader_test.cc
@@ -48,6 +48,9 @@ TEST_F(JsonLoaderTest, Basic) {
     EXPECT_TRUE(json->getBoolean("hello"));
     EXPECT_TRUE(json->getBoolean("hello", false));
     EXPECT_FALSE(json->getBoolean("world", false));
+
+    EXPECT_TRUE(absl::get(json->getValue("hello")));
+    EXPECT_THROW(json->getValue("world"), Exception);
   }
 
   {
@@ -60,6 +63,8 @@ TEST_F(JsonLoaderTest, Basic) {
     ObjectSharedPtr json = Factory::loadFromString("{\"hello\":123}");
     EXPECT_EQ(123, json->getInteger("hello", 456));
     EXPECT_EQ(456, json->getInteger("world", 456));
+
+    EXPECT_EQ(123, absl::get(json->getValue("hello")));
   }
 
   {
@@ -96,8 +101,10 @@ TEST_F(JsonLoaderTest, Basic) {
 
       if (key == "1") {
         EXPECT_EQ("111", value.getString("11"));
+        EXPECT_EQ("111", absl::get(value.getValue("11")));
       } else {
         EXPECT_EQ("222", value.getString("22"));
+        EXPECT_EQ("222", absl::get(value.getValue("22")));
       }
 
       pos++;
@@ -152,6 +159,9 @@ TEST_F(JsonLoaderTest, Basic) {
     ObjectSharedPtr config = Factory::loadFromString(json);
     std::vector array = config->getObjectArray("descriptors");
     EXPECT_THROW(array[0]->asObjectArray(), Exception);
+
+    // Object Array is not supported as an value.
+    EXPECT_THROW(config->getValue("descriptors"), Exception);
   }
 
   {
@@ -268,11 +278,16 @@ TEST_F(JsonLoaderTest, Double) {
     ObjectSharedPtr json = Factory::loadFromString("{\"value1\": 10.5, \"value2\": -12.3}");
     EXPECT_EQ(10.5, json->getDouble("value1"));
     EXPECT_EQ(-12.3, json->getDouble("value2"));
+
+    EXPECT_EQ(10.5, absl::get(json->getValue("value1")));
+    EXPECT_EQ(-12.3, absl::get(json->getValue("value2")));
   }
   {
     ObjectSharedPtr json = Factory::loadFromString("{\"foo\": 13.22}");
     EXPECT_EQ(13.22, json->getDouble("foo", 0));
     EXPECT_EQ(0, json->getDouble("bar", 0));
+
+    EXPECT_EQ(13.22, absl::get(json->getValue("foo")));
   }
   {
     ObjectSharedPtr json = Factory::loadFromString("{\"foo\": \"bar\"}");

From 23d6164e4405c9389fb1f6403f274942e9dd2d9e Mon Sep 17 00:00:00 2001
From: Xin 
Date: Tue, 27 Jun 2023 16:38:23 -0400
Subject: [PATCH 666/740] [stats] lazyinit ClusterInfo::trafficStats() 
 (#23921)

Commit Message: LazyInit ClusterInfo::trafficStats().

Additional Description:
3rd PR for https://github.com/envoyproxy/envoy/issues/23575, this is stack upon 2nd PR https://github.com/envoyproxy/envoy/pull/23907

With 100K clusters, we are seeing ~1.5 GBs less RAM usage with this PR.
Kudos to @jmarantz for the data:


|                      | Clean client     | in-MB       | Defferred | Diff       | Diff %           | Defferred-Fulfilled | Diff-in-MB (fulfilled - Clean) |
|----------------------|------------|-------------|-------------------|------------|-----------------|-----------------------------|-----------------------------|
| allocated            | 4561550208 | 4350.233276 | 2886860656        | 1674689552 | 36.71316714    | 4565167632                  | 3.44984436                  |
| heap_size            | 5303697408 | 656         | 3443523584        | 1860173824 | 35.07315144    | 5146411008                  | -150                        |
| pageheap_unmapped    | 687865856  |             | 501219328         | 186646528  | 27.13414634    | 524288000                   | -156                        |
| pageheap_free        | 22921216   | 21.859375   | 23109632          | -188416    | -00.8220157255 | 22257664                    | -0.6328125                  |
| total_thread_cache   | 1718288    | 1.638687134 | 4197032           | -2478744   | -144.2566089    | 4833576                     | 2.970970154                 |
| total_physical_bytes | 4647192158 | 4431.907804 | 2965242430        | 1681949728 | 36.19281645    | 4653479518                  | 5.99609375                  |


To reproduce, use this script to add 100K clusters to the bootstrap tmpl:
```
$ cat large_bootstrap_maker.sh
#!/bin/bash

set -u
set -e
limit="$1"

for i in $(seq 1 $limit); do
  service="zzzzzzz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_$i"
  echo "  - name: $service"
  echo "    connect_timeout: 0.25s"
  echo "    type: LOGICAL_DNS"
  echo "    dns_lookup_family: \"v6_only\""
  echo "    lb_policy: ROUND_ROBIN"
  echo "    load_assignment:"
  echo "      cluster_name: $service"
  echo "      endpoints:"
  echo "      - lb_endpoints:"
  echo "        - endpoint:"
  echo "            address: {socket_address: {address: google.com, port_value: 443}}"
done


```
base.tmpl:

```$ cat base.yaml
admin:
  access_log:
  - name: envoy.access_loggers.file
    typed_config:
      "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
      path: "/dev/null"
  address:
    socket_address:
      address: "::"
      port_value: 8080
layered_runtime:
  layers:
  - name: admin
    admin_layer: {}

static_resources:
  listeners:
  - name: listener_0
    address:
      socket_address:
        address: "::"
        port_value: 0
    filter_chains:
    - filters:
      - name: http
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          stat_prefix: ingress_http
          codec_type: AUTO
          route_config:
            name: local_route
            virtual_hosts:
            - name: local_service
              domains: ["*"]
              routes:
              - match: {prefix: "/"}
                route: {host_rewrite_literal: 127.0.0.1, cluster: yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy-wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww}
  clusters:
```

Run the following commands to generate the bootstrap.
```
bash large_bootstrap_maker.sh 100000 >> base.yaml
mv base.yaml test/config/integration/100k_clusters.yaml
```
and modify the test/config/main_common_test.cc to load from test/config/integration/100k_clusters.yaml instead. Then you could run the test and observe the RAM uasage.

Risk Level: Medium (changes how cluster_info::trafficStats() are created).

Testing: existing unit tests.
Docs Changes: done
Release Notes: incluided
Platform Specific Features: n/a

Signed-off-by: Xin Zhuang 
---
 api/envoy/config/bootstrap/v3/bootstrap.proto |   1 -
 changelogs/current.yaml                       |   5 +
 envoy/upstream/upstream.h                     |  11 +-
 source/common/stats/deferred_creation.h       |   4 +-
 source/common/upstream/BUILD                  |  12 +-
 source/common/upstream/upstream_impl.cc       |  14 +-
 source/common/upstream/upstream_impl.h        |  11 +-
 .../clusters/aggregate/cluster_test.cc        |   6 +-
 .../proxy_filter_test.cc                      |   5 +-
 test/integration/cds_integration_test.cc      | 209 ++++++++++++++++--
 test/mocks/upstream/BUILD                     |   1 +
 test/mocks/upstream/cluster_info.cc           |   2 +-
 test/mocks/upstream/cluster_info.h            |   4 +-
 13 files changed, 230 insertions(+), 55 deletions(-)

diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto
index 30d05dac50c8..a5fcbef104b3 100644
--- a/api/envoy/config/bootstrap/v3/bootstrap.proto
+++ b/api/envoy/config/bootstrap/v3/bootstrap.proto
@@ -199,7 +199,6 @@ message Bootstrap {
   repeated metrics.v3.StatsSink stats_sinks = 6;
 
   // Options to control behaviors of deferred creation compatible stats.
-  // [#not-implemented-hide:]
   DeferredStatOptions deferred_stat_options = 39;
 
   // Configuration for internal processing of stats.
diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index c3451e461f29..03e2fcfdcb38 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -459,3 +459,8 @@ deprecated:
   change: |
     deprecated the :ref:`HealthCheck event_log_path ` in favor of
     :ref:`HealthCheck event_logger extension `.
+- area: stats
+  change: |
+    added :ref:`enable_deferred_creation_stats
+    `.
+    support for ClusterTrafficStats.
diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h
index 29fa641d4396..37a8adf34884 100644
--- a/envoy/upstream/upstream.h
+++ b/envoy/upstream/upstream.h
@@ -789,12 +789,8 @@ MAKE_STATS_STRUCT(ClusterLbStats, ClusterLbStatNames, ALL_CLUSTER_LB_STATS);
  */
 MAKE_STAT_NAMES_STRUCT(ClusterTrafficStatNames, ALL_CLUSTER_TRAFFIC_STATS);
 MAKE_STATS_STRUCT(ClusterTrafficStats, ClusterTrafficStatNames, ALL_CLUSTER_TRAFFIC_STATS);
-/*
- * NOTE: LazyClusterTrafficStats for now is an alias of "std::unique_ptr",
- * this is to make way for future lazy-init on trafficStats(). See
- * https://github.com/envoyproxy/envoy/pull/23921#issuecomment-1335239116 for more context.
- */
-using LazyClusterTrafficStats = std::unique_ptr;
+using DeferredCreationCompatibleClusterTrafficStats =
+    Stats::DeferredCreationCompatibleStats;
 
 MAKE_STAT_NAMES_STRUCT(ClusterLoadReportStatNames, ALL_CLUSTER_LOAD_REPORT_STATS);
 MAKE_STATS_STRUCT(ClusterLoadReportStats, ClusterLoadReportStatNames,
@@ -1097,8 +1093,7 @@ class ClusterInfo : public Http::FilterChainFactory {
   /**
    * @return  all traffic related stats for this cluster.
    */
-  virtual LazyClusterTrafficStats& trafficStats() const PURE;
-
+  virtual DeferredCreationCompatibleClusterTrafficStats& trafficStats() const PURE;
   /**
    * @return the stats scope that contains all cluster stats. This can be used to produce dynamic
    *         stats that will be freed when the cluster is removed.
diff --git a/source/common/stats/deferred_creation.h b/source/common/stats/deferred_creation.h
index 4e2732728355..b2999666e7a0 100644
--- a/source/common/stats/deferred_creation.h
+++ b/source/common/stats/deferred_creation.h
@@ -105,8 +105,8 @@ template 
 DeferredCreationCompatibleStats
 createDeferredCompatibleStats(Stats::ScopeSharedPtr scope,
                               const typename StatsStructType::StatNameType& stat_names,
-                              bool deferred_creation) {
-  if (deferred_creation) {
+                              bool defer_creation) {
+  if (defer_creation) {
     return DeferredCreationCompatibleStats(
         std::make_unique>(stat_names, scope));
   } else {
diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD
index 86daa4417864..60ba2173cb7f 100644
--- a/source/common/upstream/BUILD
+++ b/source/common/upstream/BUILD
@@ -79,14 +79,12 @@ envoy_cc_library(
     srcs = ["cluster_manager_impl.cc"],
     hdrs = ["cluster_manager_impl.h"],
     deps = [
-        "//source/extensions/filters/network/http_connection_manager:config",
-        "//source/common/config:null_grpc_mux_lib",
         ":cds_api_lib",
         ":cluster_discovery_manager_lib",
+        ":host_utility_lib",
         ":load_balancer_lib",
         ":load_stats_reporter_lib",
         ":od_cds_api_lib",
-        ":host_utility_lib",
         "//envoy/api:api_interface",
         "//envoy/config:xds_resources_delegate_interface",
         "//envoy/event:dispatcher_interface",
@@ -102,6 +100,7 @@ envoy_cc_library(
         "//source/common/common:enum_to_int",
         "//source/common/common:utility_lib",
         "//source/common/config:custom_config_validators_lib",
+        "//source/common/config:null_grpc_mux_lib",
         "//source/common/config:subscription_factory_lib",
         "//source/common/config:utility_lib",
         "//source/common/config:xds_resource_lib",
@@ -114,14 +113,15 @@ envoy_cc_library(
         "//source/common/network:resolver_lib",
         "//source/common/network:utility_lib",
         "//source/common/protobuf:utility_lib",
+        "//source/common/quic:quic_stat_names_lib",
         "//source/common/router:context_lib",
         "//source/common/router:shadow_writer_lib",
         "//source/common/shared_pool:shared_pool_lib",
-        "//source/common/tcp:conn_pool_lib",
         "//source/common/tcp:async_tcp_client_lib",
+        "//source/common/tcp:conn_pool_lib",
         "//source/common/upstream:priority_conn_pool_map_impl_lib",
         "//source/common/upstream:upstream_lib",
-        "//source/common/quic:quic_stat_names_lib",
+        "//source/extensions/filters/network/http_connection_manager:config",
         "//source/server:factory_context_base_impl_lib",
         "@envoy_api//envoy/admin/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
@@ -399,6 +399,7 @@ envoy_cc_library(
         ":transport_socket_match_lib",
         "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+        "//source/common/stats:deferred_creation",
         "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/http/upstream_codec/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/transport_sockets/raw_buffer/v3:pkg_cc_proto",
@@ -479,6 +480,7 @@ envoy_cc_library(
         "//source/common/http/http3:codec_stats_lib",
         "//source/common/init:manager_lib",
         "//source/common/shared_pool:shared_pool_lib",
+        "//source/common/stats:deferred_creation",
         "//source/common/stats:isolated_store_lib",
         "//source/common/stats:stats_lib",
         "//source/extensions/filters/network/http_connection_manager:config",
diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc
index a48063e24e5b..36ecd2d6242c 100644
--- a/source/common/upstream/upstream_impl.cc
+++ b/source/common/upstream/upstream_impl.cc
@@ -48,6 +48,7 @@
 #include "source/common/router/config_utility.h"
 #include "source/common/runtime/runtime_features.h"
 #include "source/common/runtime/runtime_impl.h"
+#include "source/common/stats/deferred_creation.h"
 #include "source/common/upstream/cluster_factory_impl.h"
 #include "source/common/upstream/health_checker_impl.h"
 #include "source/extensions/filters/network/http_connection_manager/config.h"
@@ -863,9 +864,11 @@ void MainPrioritySetImpl::updateCrossPriorityHostMap(const HostVector& hosts_add
   }
 }
 
-LazyClusterTrafficStats ClusterInfoImpl::generateStats(Stats::Scope& scope,
-                                                       const ClusterTrafficStatNames& stat_names) {
-  return std::make_unique(stat_names, scope);
+DeferredCreationCompatibleClusterTrafficStats
+ClusterInfoImpl::generateStats(Stats::ScopeSharedPtr scope,
+                               const ClusterTrafficStatNames& stat_names, bool defer_creation) {
+  return Stats::createDeferredCompatibleStats(scope, stat_names,
+                                                                   defer_creation);
 }
 
 ClusterRequestResponseSizeStats ClusterInfoImpl::generateRequestResponseSizeStats(
@@ -1042,8 +1045,9 @@ ClusterInfoImpl::ClusterInfoImpl(
       peekahead_ratio_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.preconnect_policy(),
                                                        predictive_preconnect_ratio, 0)),
       socket_matcher_(std::move(socket_matcher)), stats_scope_(std::move(stats_scope)),
-      traffic_stats_(
-          generateStats(*stats_scope_, factory_context.clusterManager().clusterStatNames())),
+      traffic_stats_(generateStats(stats_scope_,
+                                   factory_context.clusterManager().clusterStatNames(),
+                                   server_context.statsConfig().enableDeferredCreationStats())),
       config_update_stats_(factory_context.clusterManager().clusterConfigUpdateStatNames(),
                            *stats_scope_),
       lb_stats_(factory_context.clusterManager().clusterLbStatNames(), *stats_scope_),
diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h
index ccd00a653c29..ff6831081bfe 100644
--- a/source/common/upstream/upstream_impl.h
+++ b/source/common/upstream/upstream_impl.h
@@ -789,8 +789,9 @@ class ClusterInfoImpl : public ClusterInfo,
                   Stats::ScopeSharedPtr&& stats_scope, bool added_via_api,
                   Server::Configuration::TransportSocketFactoryContext&);
 
-  static LazyClusterTrafficStats generateStats(Stats::Scope& scope,
-                                               const ClusterTrafficStatNames& cluster_stat_names);
+  static DeferredCreationCompatibleClusterTrafficStats
+  generateStats(Stats::ScopeSharedPtr scope, const ClusterTrafficStatNames& cluster_stat_names,
+                bool defer_creation);
   static ClusterLoadReportStats
   generateLoadReportStats(Stats::Scope& scope, const ClusterLoadReportStatNames& stat_names);
   static ClusterCircuitBreakersStats
@@ -911,7 +912,9 @@ class ClusterInfoImpl : public ClusterInfo,
   }
   ResourceManager& resourceManager(ResourcePriority priority) const override;
   TransportSocketMatcher& transportSocketMatcher() const override { return *socket_matcher_; }
-  LazyClusterTrafficStats& trafficStats() const override { return traffic_stats_; }
+  DeferredCreationCompatibleClusterTrafficStats& trafficStats() const override {
+    return traffic_stats_;
+  }
   ClusterConfigUpdateStats& configUpdateStats() const override { return config_update_stats_; }
   ClusterLbStats& lbStats() const override { return lb_stats_; }
   ClusterEndpointStats& endpointStats() const override { return endpoint_stats_; }
@@ -1056,7 +1059,7 @@ class ClusterInfoImpl : public ClusterInfo,
   const float peekahead_ratio_;
   TransportSocketMatcherPtr socket_matcher_;
   Stats::ScopeSharedPtr stats_scope_;
-  mutable LazyClusterTrafficStats traffic_stats_;
+  mutable DeferredCreationCompatibleClusterTrafficStats traffic_stats_;
   mutable ClusterConfigUpdateStats config_update_stats_;
   mutable ClusterLbStats lb_stats_;
   mutable ClusterEndpointStats endpoint_stats_;
diff --git a/test/extensions/clusters/aggregate/cluster_test.cc b/test/extensions/clusters/aggregate/cluster_test.cc
index 09fa69ede02d..f78ccb511a33 100644
--- a/test/extensions/clusters/aggregate/cluster_test.cc
+++ b/test/extensions/clusters/aggregate/cluster_test.cc
@@ -34,8 +34,8 @@ class AggregateClusterTest : public Event::TestUsingSimulatedTime, public testin
 public:
   AggregateClusterTest()
       : stat_names_(server_context_.store_.symbolTable()),
-        stats_(Upstream::ClusterInfoImpl::generateStats(*server_context_.store_.rootScope(),
-                                                        stat_names_)) {
+        traffic_stats_(Upstream::ClusterInfoImpl::generateStats(server_context_.store_.rootScope(),
+                                                                stat_names_, false)) {
     ON_CALL(*primary_info_, name()).WillByDefault(ReturnRef(primary_name));
     ON_CALL(*secondary_info_, name()).WillByDefault(ReturnRef(secondary_name));
 
@@ -138,7 +138,7 @@ class AggregateClusterTest : public Event::TestUsingSimulatedTime, public testin
   Upstream::LoadBalancerFactorySharedPtr lb_factory_;
   Upstream::LoadBalancerPtr lb_;
   Upstream::ClusterTrafficStatNames stat_names_;
-  Upstream::LazyClusterTrafficStats stats_;
+  Upstream::DeferredCreationCompatibleClusterTrafficStats traffic_stats_;
   std::shared_ptr primary_info_{
       new NiceMock()};
   std::shared_ptr secondary_info_{
diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc
index fe546b87e102..512f660e6d8a 100644
--- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc
+++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc
@@ -280,8 +280,9 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflowWithDnsCacheResourceManager) {
             filter2->decodeHeaders(request_headers_, false));
 
   // Cluster circuit breaker overflow counter won't be incremented.
-  EXPECT_EQ(0, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_
-                   ->traffic_stats_->upstream_rq_pending_overflow_.value());
+  EXPECT_EQ(0,
+            factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->trafficStats()
+                ->upstream_rq_pending_overflow_.value());
   filter2->onDestroy();
   EXPECT_CALL(*handle, onDestroy());
   filter_->onDestroy();
diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc
index f3b0db0ce4b4..3da6ca2b3c08 100644
--- a/test/integration/cds_integration_test.cc
+++ b/test/integration/cds_integration_test.cc
@@ -97,8 +97,7 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht
 
     // Do the initial compareDiscoveryRequest / sendDiscoveryResponse for cluster_1.
     EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true));
-    sendDiscoveryResponse(Config::TypeUrl::get().Cluster,
-                                                               {cluster1_}, {cluster1_}, {}, "55");
+    sendClusterDiscoveryResponse({cluster1_}, {cluster1_}, {}, "55");
 
     // We can continue the test once we're sure that Envoy's ClusterManager has made use of
     // the DiscoveryResponse describing cluster_1 that we sent.
@@ -111,6 +110,14 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht
     registerTestServerPorts({"http"});
   }
 
+  void sendClusterDiscoveryResponse(
+      const std::vector& state_of_the_world,
+      const std::vector& added_or_updated,
+      const std::vector& removed, const std::string& version) {
+    sendDiscoveryResponse(
+        Config::TypeUrl::get().Cluster, state_of_the_world, added_or_updated, removed, version);
+  }
+
   // Regression test to catch the code declaring a gRPC service method for {SotW,delta}
   // when the user's bootstrap config asks for the other type.
   void verifyGrpcServiceMethod() {
@@ -162,8 +169,7 @@ TEST_P(CdsIntegrationTest, CdsClusterUpDownUp) {
 
   // Tell Envoy that cluster_1 is gone.
   EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {}));
-  sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {}, {},
-                                                             {ClusterName1}, "42");
+  sendClusterDiscoveryResponse({}, {}, {ClusterName1}, "42");
   // We can continue the test once we're sure that Envoy's ClusterManager has made use of
   // the DiscoveryResponse that says cluster_1 is gone.
   test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1);
@@ -181,8 +187,7 @@ TEST_P(CdsIntegrationTest, CdsClusterUpDownUp) {
 
   // Tell Envoy that cluster_1 is back.
   EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {}));
-  sendDiscoveryResponse(Config::TypeUrl::get().Cluster,
-                                                             {cluster1_}, {cluster1_}, {}, "413");
+  sendClusterDiscoveryResponse({cluster1_}, {cluster1_}, {}, "413");
 
   // We can continue the test once we're sure that Envoy's ClusterManager has made use of
   // the DiscoveryResponse describing cluster_1 that we sent. Again, 2 includes CDS server.
@@ -213,8 +218,7 @@ TEST_P(CdsIntegrationTest, CdsClusterTeardownWhileConnecting) {
 
   // Tell Envoy that cluster_1 is gone.
   EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {}));
-  sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {}, {},
-                                                             {ClusterName1}, "42");
+  sendClusterDiscoveryResponse({}, {}, {ClusterName1}, "42");
   // We can continue the test once we're sure that Envoy's ClusterManager has made use of
   // the DiscoveryResponse that says cluster_1 is gone.
   test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1);
@@ -225,6 +229,170 @@ TEST_P(CdsIntegrationTest, CdsClusterTeardownWhileConnecting) {
   EXPECT_LE(cx_counter->value(), 1);
 }
 
+class DeferredCreationClusterStatsTest : public CdsIntegrationTest {
+public:
+  void initializeDeferredCreationTest(bool enable_deferred_creation_stats) {
+    use_real_stats_ = true;
+    config_helper_.addConfigModifier(
+        [enable_deferred_creation_stats](::envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+          bootstrap.mutable_deferred_stat_options()->set_enable_deferred_creation_stats(
+              enable_deferred_creation_stats);
+        });
+    CdsIntegrationTest::initialize();
+    test_server_->waitForCounterGe("cluster_manager.cluster_added", 1);
+  }
+
+  void sendRequestToClusterAndWaitForResponse() {
+    BufferingStreamDecoderPtr response = IntegrationUtil::makeSingleRequest(
+        lookupPort("http"), "GET", "/cluster1", "", downstream_protocol_, version_, "foo.com");
+    ASSERT_TRUE(response->complete());
+    cleanupUpstreamAndDownstream();
+  };
+
+  void updateCluster() {
+    envoy::config::cluster::v3::Cluster cluster1_updated = cluster_creator_(
+        ClusterName1, fake_upstreams_[UpstreamIndex2]->localAddress()->ip()->port(),
+        Network::Test::getLoopbackAddressString(ipVersion()),
+        envoy::config::cluster::v3::Cluster::ROUND_ROBIN);
+
+    sendDiscoveryResponse(
+        Config::TypeUrl::get().Cluster, {cluster1_updated}, {cluster1_updated}, {}, "42");
+  }
+
+  void removeClusters(const std::vector& removed) {
+    uint64_t cluster_removed = test_server_->counter("cluster_manager.cluster_removed")->value();
+    sendClusterDiscoveryResponse({}, {}, removed, "42");
+    test_server_->waitForCounterGe("cluster_manager.cluster_removed",
+                                   cluster_removed + removed.size());
+  }
+};
+
+INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, DeferredCreationClusterStatsTest,
+                         DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS);
+
+// Test that DeferredCreationTrafficStats gets created and updated correctly.
+TEST_P(DeferredCreationClusterStatsTest,
+       DeferredCreationTrafficStatsWithClusterCreateUpdateDelete) {
+  initializeDeferredCreationTest(/*enable_deferred_creation_stats=*/true);
+
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized")->value(), 0);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr);
+
+  sendRequestToClusterAndWaitForResponse();
+  // cluster_1 trafficStats updated.
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized")->value(), 1);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1);
+
+  updateCluster();
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 2);
+
+  // cluster_1 traffic stats not lost.
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1);
+  removeClusters({ClusterName1});
+  // update_success is 3: initialize(), update cluster1. and remove cluster1.
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 3);
+  sendClusterDiscoveryResponse({cluster2_}, {cluster2_}, {}, "43");
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 4);
+  EXPECT_EQ(test_server_->counter("cluster_manager.cluster_added")->value(), 3);
+  // Now the cluster_1 stats are gone, as well as the lazy init wrapper.
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized"), nullptr);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr);
+
+  // No cluster_2 traffic stats.
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_2.ClusterTrafficStats.initialized")->value(), 0);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_2.upstream_cx_total"), nullptr);
+}
+
+// Test that Non-DeferredCreationTrafficStats gets created and updated correctly.
+TEST_P(DeferredCreationClusterStatsTest,
+       NonDeferredCreationTrafficStatsWithClusterCreateUpdateDelete) {
+  initializeDeferredCreationTest(/*enable_deferred_creation_stats=*/false);
+  // cluster_1 trafficStats created by CDS push.
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 0);
+
+  sendRequestToClusterAndWaitForResponse();
+  // cluster_1 trafficStats updated.
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1);
+
+  updateCluster();
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 2);
+
+  // cluster_1 traffic stats not lost.
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1);
+  removeClusters({ClusterName1});
+  // update_success is 3: initialize(), update cluster1. and remove cluster1.
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 3);
+  sendClusterDiscoveryResponse({cluster2_}, {cluster2_}, {}, "43");
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 4);
+  EXPECT_EQ(test_server_->counter("cluster_manager.cluster_added")->value(), 3);
+  // Now the cluster_1 stats are gone.
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr);
+  // cluster_2 traffic stats stays.
+  EXPECT_EQ(test_server_->counter("cluster.cluster_2.upstream_cx_total")->value(), 0);
+}
+
+// Test that DeferredCreationTrafficStats with cluster_1 create-remove-create sequence.
+TEST_P(DeferredCreationClusterStatsTest,
+       DeferredCreationTrafficStatsWithClusterCreateDeleteRecrete) {
+  initializeDeferredCreationTest(/*enable_deferred_creation_stats=*/true);
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized")->value(), 0);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr);
+
+  sendRequestToClusterAndWaitForResponse();
+  // cluster_1 trafficStats updated.
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized")->value(), 1);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1);
+  removeClusters({ClusterName1});
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 2);
+  sendClusterDiscoveryResponse({cluster2_}, {cluster2_}, {}, "43");
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 3);
+  EXPECT_EQ(test_server_->counter("cluster_manager.cluster_added")->value(), 3);
+  // No cluster_2 traffic stats.
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_2.ClusterTrafficStats.initialized")->value(), 0);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_2.upstream_cx_total"), nullptr);
+  // Now the cluster_1 stats are gone, as well as the lazy init wrapper.
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized"), nullptr);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr);
+  // Now add cluster1 back.
+  updateCluster();
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 4);
+  // Now the cluster_1.ClusterTrafficStats.initialized gauge is 0, since it didn't see previous
+  // stats.
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized")->value(), 0);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr);
+
+  // cluster_1 traffic stats created, due to the above http request.
+  sendRequestToClusterAndWaitForResponse();
+  EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized")->value(), 1);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1);
+}
+
+// Test that Non-DeferredCreationTrafficStats with cluster_1 create-remove-create sequence.
+TEST_P(DeferredCreationClusterStatsTest,
+       NonDeferredCreationTrafficStatsWithClusterCreateDeleteRecrete) {
+  initializeDeferredCreationTest(/*enable_deferred_creation_stats=*/false);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 0);
+
+  sendRequestToClusterAndWaitForResponse();
+  // cluster_1 trafficStats updated.
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1);
+  removeClusters({ClusterName1});
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 2);
+  sendClusterDiscoveryResponse({cluster2_}, {cluster2_}, {}, "43");
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 3);
+  EXPECT_EQ(test_server_->counter("cluster_manager.cluster_added")->value(), 3);
+  // cluster_2 traffic stats created.
+  EXPECT_EQ(test_server_->counter("cluster.cluster_2.upstream_cx_total")->value(), 0);
+  // Now the cluster_1 stats are gone.
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr);
+  // Now add cluster1 back.
+  updateCluster();
+  test_server_->waitForCounterGe("cluster_manager.cds.update_success", 4);
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 0);
+  sendRequestToClusterAndWaitForResponse();
+  EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1);
+}
+
 // Test the fast addition and removal of clusters when they use ThreadAwareLb.
 TEST_P(CdsIntegrationTest, CdsClusterWithThreadAwareLbCycleUpDownUp) {
   // Calls our initialize(), which includes establishing a listener, route, and cluster.
@@ -233,10 +401,9 @@ TEST_P(CdsIntegrationTest, CdsClusterWithThreadAwareLbCycleUpDownUp) {
 
   // Tell Envoy that cluster_1 is gone.
   EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {}));
-  sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {}, {},
-                                                             {ClusterName1}, "42");
-  // Make sure that Envoy's ClusterManager has made use of the DiscoveryResponse that says cluster_1
-  // is gone.
+  sendClusterDiscoveryResponse({}, {}, {ClusterName1}, "42");
+  // Make sure that Envoy's ClusterManager has made use of the DiscoveryResponse that says
+  // cluster_1 is gone.
   test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1);
 
   // Update cluster1_ to use MAGLEV load balancer policy.
@@ -282,8 +449,7 @@ TEST_P(CdsIntegrationTest, TwoClusters) {
 
   // Tell Envoy that cluster_1 is gone.
   EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {}));
-  sendDiscoveryResponse(Config::TypeUrl::get().Cluster,
-                                                             {cluster2_}, {}, {ClusterName1}, "43");
+  sendClusterDiscoveryResponse({cluster2_}, {}, {ClusterName1}, "43");
   // We can continue the test once we're sure that Envoy's ClusterManager has made use of
   // the DiscoveryResponse that says cluster_1 is gone.
   test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1);
@@ -328,8 +494,7 @@ TEST_P(CdsIntegrationTest, TwoClustersAndRedirects) {
   // The '3' includes the fake CDS server.
   test_server_->waitForGaugeGe("cluster_manager.active_clusters", 3);
   // Tell Envoy that cluster_1 is gone.
-  sendDiscoveryResponse(Config::TypeUrl::get().Cluster,
-                                                             {cluster2_}, {}, {ClusterName1}, "43");
+  sendClusterDiscoveryResponse({cluster2_}, {}, {ClusterName1}, "43");
   test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1);
 
   codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http"))));
@@ -355,7 +520,8 @@ TEST_P(CdsIntegrationTest, TwoClustersAndRedirects) {
 }
 
 // Tests that when Envoy's delta xDS stream dis/reconnects, Envoy can inform the server of the
-// resources it already has: the reconnected stream need not start with a state-of-the-world update.
+// resources it already has: the reconnected stream need not start with a state-of-the-world
+// update.
 TEST_P(CdsIntegrationTest, VersionsRememberedAfterReconnect) {
   SKIP_IF_XDS_IS(Grpc::SotwOrDelta::Sotw);
   SKIP_IF_XDS_IS(Grpc::SotwOrDelta::UnifiedSotw);
@@ -470,8 +636,7 @@ TEST_P(CdsIntegrationTest, CdsClusterDownWithLotsOfIdleConnections) {
 
   // Tell Envoy that cluster_1 is gone. Envoy will try to close all idle connections
   EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {}));
-  sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {}, {},
-                                                             {ClusterName1}, "42");
+  sendClusterDiscoveryResponse({}, {}, {ClusterName1}, "42");
   // We can continue the test once we're sure that Envoy's ClusterManager has made use of
   // the DiscoveryResponse that says cluster_1 is gone.
   test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1);
@@ -490,7 +655,8 @@ TEST_P(CdsIntegrationTest, CdsClusterDownWithLotsOfIdleConnections) {
 
 // This test verifies that Envoy can delete a cluster with a lot of connections in the connecting
 // state and associated pending requests. The recursion guard in the
-// ConnPoolImplBase::closeIdleConnectionsForDrainingPool() would fire if it was called recursively.
+// ConnPoolImplBase::closeIdleConnectionsForDrainingPool() would fire if it was called
+// recursively.
 //
 // Test is currently disabled as there is presently no reliable way of making upstream connections
 // hang in connecting state.
@@ -541,8 +707,7 @@ TEST_P(CdsIntegrationTest, DISABLED_CdsClusterDownWithLotsOfConnectingConnection
 
   // Tell Envoy that cluster_1 is gone. Envoy will try to close all pending connections
   EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {}));
-  sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {}, {},
-                                                             {ClusterName1}, "42");
+  sendClusterDiscoveryResponse({}, {}, {ClusterName1}, "42");
   // We can continue the test once we're sure that Envoy's ClusterManager has made use of
   // the DiscoveryResponse that says cluster_1 is gone.
   test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1);
diff --git a/test/mocks/upstream/BUILD b/test/mocks/upstream/BUILD
index 8b4258c4f668..52e6e6571cac 100644
--- a/test/mocks/upstream/BUILD
+++ b/test/mocks/upstream/BUILD
@@ -23,6 +23,7 @@ envoy_cc_mock(
         "//source/common/http/http2:codec_stats_lib",
         "//source/common/network:raw_buffer_socket_lib",
         "//source/common/router:upstream_codec_filter_lib",
+        "//source/common/stats:deferred_creation",
         "//source/common/upstream:upstream_includes",
         "//source/common/upstream:upstream_lib",
         "//test/mocks/runtime:runtime_mocks",
diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc
index 108df1c3c0cb..72f448576d25 100644
--- a/test/mocks/upstream/cluster_info.cc
+++ b/test/mocks/upstream/cluster_info.cc
@@ -63,7 +63,7 @@ MockClusterInfo::MockClusterInfo()
       cluster_request_response_size_stat_names_(stats_store_.symbolTable()),
       cluster_timeout_budget_stat_names_(stats_store_.symbolTable()),
       traffic_stats_(
-          std::make_unique(traffic_stat_names_, *stats_store_.rootScope())),
+          ClusterInfoImpl::generateStats(stats_store_.rootScope(), traffic_stat_names_, false)),
       config_update_stats_(config_update_stats_names_, *stats_store_.rootScope()),
       lb_stats_(lb_stat_names_, *stats_store_.rootScope()),
       endpoint_stats_(endpoint_stat_names_, *stats_store_.rootScope()),
diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h
index 1245ea7bdf3f..4a50c2328127 100644
--- a/test/mocks/upstream/cluster_info.h
+++ b/test/mocks/upstream/cluster_info.h
@@ -152,7 +152,7 @@ class MockClusterInfo : public ClusterInfo {
   MOCK_METHOD(const std::string&, observabilityName, (), (const));
   MOCK_METHOD(ResourceManager&, resourceManager, (ResourcePriority priority), (const));
   MOCK_METHOD(TransportSocketMatcher&, transportSocketMatcher, (), (const));
-  MOCK_METHOD(LazyClusterTrafficStats&, trafficStats, (), (const));
+  MOCK_METHOD(DeferredCreationCompatibleClusterTrafficStats&, trafficStats, (), (const));
   MOCK_METHOD(ClusterLbStats&, lbStats, (), (const));
   MOCK_METHOD(ClusterEndpointStats&, endpointStats, (), (const));
   MOCK_METHOD(ClusterConfigUpdateStats&, configUpdateStats, (), (const));
@@ -213,7 +213,7 @@ class MockClusterInfo : public ClusterInfo {
   ClusterCircuitBreakersStatNames cluster_circuit_breakers_stat_names_;
   ClusterRequestResponseSizeStatNames cluster_request_response_size_stat_names_;
   ClusterTimeoutBudgetStatNames cluster_timeout_budget_stat_names_;
-  LazyClusterTrafficStats traffic_stats_;
+  mutable DeferredCreationCompatibleClusterTrafficStats traffic_stats_;
   ClusterConfigUpdateStats config_update_stats_;
   ClusterLbStats lb_stats_;
   ClusterEndpointStats endpoint_stats_;

From 1e2df3d4efa4b9197f2804582b1047deb00a4e62 Mon Sep 17 00:00:00 2001
From: Kevin Baichoo 
Date: Wed, 28 Jun 2023 09:20:18 -0400
Subject: [PATCH 667/740] Efficiency: Load Balancer prefer absl::flat_hash_map
 (#28163)

Use absl::flat_hash_map since we don't need pointer stability of
elements and kv are small.

Signed-off-by: Kevin Baichoo 
---
 source/common/upstream/load_balancer_impl.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h
index 72597743cc68..4f632905ae80 100644
--- a/source/common/upstream/load_balancer_impl.h
+++ b/source/common/upstream/load_balancer_impl.h
@@ -310,7 +310,7 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase {
 
   struct HostsSourceHash {
     size_t operator()(const HostsSource& hs) const {
-      // This is only used for absl::node_hash_map keys, so we don't need a deterministic hash.
+      // This is only used for absl::flat_hash_map keys, so we don't need a deterministic hash.
       size_t hash = std::hash()(hs.priority_);
       hash = 37 * hash + std::hash()(static_cast(hs.source_type_));
       hash = 37 * hash + std::hash()(hs.locality_index_);
@@ -512,7 +512,7 @@ class EdfLoadBalancerBase : public ZoneAwareLoadBalancerBase {
                                                 const HostsSource& source) PURE;
 
   // Scheduler for each valid HostsSource.
-  absl::node_hash_map scheduler_;
+  absl::flat_hash_map scheduler_;
   Common::CallbackHandlePtr priority_update_cb_;
   Common::CallbackHandlePtr member_update_cb_;
 
@@ -602,7 +602,7 @@ class RoundRobinLoadBalancer : public EdfLoadBalancerBase {
   }
 
   uint64_t peekahead_index_{};
-  absl::node_hash_map rr_indexes_;
+  absl::flat_hash_map rr_indexes_;
 };
 
 /**

From b9cf1e6a195316b269862b5bfa3e47e739efdcd4 Mon Sep 17 00:00:00 2001
From: "Adi (Suissa) Peleg" 
Date: Wed, 28 Jun 2023 09:57:07 -0400
Subject: [PATCH 668/740] lb: reserve space for vectors on hosts filtering
 (#28164)

Signed-off-by: Adi Suissa-Peleg 
---
 source/common/upstream/upstream_impl.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc
index 36ecd2d6242c..701b0c050414 100644
--- a/source/common/upstream/upstream_impl.cc
+++ b/source/common/upstream/upstream_impl.cc
@@ -550,6 +550,8 @@ std::vector HostsPerLocalityImpl::filter(
   std::vector> mutable_clones;
   std::vector filtered_clones;
 
+  mutable_clones.reserve(predicates.size());
+  filtered_clones.reserve(predicates.size());
   for (size_t i = 0; i < predicates.size(); ++i) {
     mutable_clones.emplace_back(std::make_shared());
     filtered_clones.emplace_back(mutable_clones.back());

From bcad5b15a93c0f674fa66c3f3a9b2c134e2fe076 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 28 Jun 2023 15:28:36 +0100
Subject: [PATCH 669/740] build(deps): bump jaegertracing/all-in-one from
 `aa450ca` to `9c75fc2` in /examples/shared/jaeger (#28167)

build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger

Bumps jaegertracing/all-in-one from `aa450ca` to `9c75fc2`.

---
updated-dependencies:
- dependency-name: jaegertracing/all-in-one
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/jaeger/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile
index 8d998fb780a8..315c0bc1fff1 100644
--- a/examples/shared/jaeger/Dockerfile
+++ b/examples/shared/jaeger/Dockerfile
@@ -1,4 +1,4 @@
-FROM jaegertracing/all-in-one@sha256:aa450ca8ac0afd03baf23cdf973dafc0d766c2f82e61749e00399442daf5f58c
+FROM jaegertracing/all-in-one@sha256:9c75fc21ad3691e5bb882672e38c3bd3eacd9b97b8ad8239e60a6fc578bef66a
 HEALTHCHECK \
     --interval=1s \
     --timeout=1s \

From 057c80b8215dccc4a7d76e14cb3bb59c91208d6b Mon Sep 17 00:00:00 2001
From: "Adi (Suissa) Peleg" 
Date: Wed, 28 Jun 2023 11:58:15 -0400
Subject: [PATCH 670/740] eds: introducing EDS resources cache (#28079)

Signed-off-by: Adi Suissa-Peleg 
---
 envoy/config/BUILD                            |   8 +
 envoy/config/eds_resources_cache.h            | 114 +++++++
 .../extensions/config_subscription/grpc/BUILD |  11 +
 .../grpc/eds_resources_cache_impl.cc          |  86 +++++
 .../grpc/eds_resources_cache_impl.h           |  48 +++
 .../extensions/config_subscription/grpc/BUILD |  11 +
 .../grpc/eds_resources_cache_impl_test.cc     | 323 ++++++++++++++++++
 7 files changed, 601 insertions(+)
 create mode 100644 envoy/config/eds_resources_cache.h
 create mode 100644 source/extensions/config_subscription/grpc/eds_resources_cache_impl.cc
 create mode 100644 source/extensions/config_subscription/grpc/eds_resources_cache_impl.h
 create mode 100644 test/extensions/config_subscription/grpc/eds_resources_cache_impl_test.cc

diff --git a/envoy/config/BUILD b/envoy/config/BUILD
index a660c39c59c7..3d0faec811a0 100644
--- a/envoy/config/BUILD
+++ b/envoy/config/BUILD
@@ -139,3 +139,11 @@ envoy_cc_library(
         "@com_google_googleapis//google/rpc:status_cc_proto",
     ],
 )
+
+envoy_cc_library(
+    name = "eds_resources_cache_interface",
+    hdrs = ["eds_resources_cache.h"],
+    deps = [
+        "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto",
+    ],
+)
diff --git a/envoy/config/eds_resources_cache.h b/envoy/config/eds_resources_cache.h
new file mode 100644
index 000000000000..472c927f31bd
--- /dev/null
+++ b/envoy/config/eds_resources_cache.h
@@ -0,0 +1,114 @@
+#pragma once
+
+#include "envoy/common/optref.h"
+#include "envoy/common/pure.h"
+#include "envoy/config/endpoint/v3/endpoint.pb.h"
+
+#include "absl/strings/string_view.h"
+
+namespace Envoy {
+namespace Config {
+
+// An interface for cached resource removed callback.
+class EdsResourceRemovalCallback {
+public:
+  virtual ~EdsResourceRemovalCallback() = default;
+
+  // Invoked when a cached resource is removed from the cache.
+  virtual void onCachedResourceRemoved(absl::string_view resource_name) PURE;
+};
+
+// Represents an xDS resources cache for EDS resources, and currently supports
+// a single config-source (ADS). The motivation is that clusters that are
+// updated (not added) during a CDS response will be able to use the current EDS
+// configuration, thus avoiding the need of the xDS server to send an additional
+// EDS response that is identical to what was already sent.
+// However, using the cached EDS config is not always desired, for example when
+// the cluster changes from non-TLS to TLS (see discussion in
+// https://github.com/envoyproxy/envoy/issues/5168). Thus, this cache allows
+// the EDS subscription to decide whether to use the cache or not.
+//
+// This cache will be instantiated once and owned by the ADS Mux, and passed to
+// the EDS subscriptions.
+//
+// Resources lifetime in the cache is determined by the gRPC mux that adds/updates a
+// resource when it receives its contents, and removes a resource when there is
+// no longer interest in that resource.
+// An EDS subscription may fetch a resource from the cache, and optionally
+// install a callback to be triggered if the resource is removed from the cache.
+// In addition, a resource in the cache may have an expiration timer if
+// "endpoint_stale_after" (TTL) is set for that resource. Once the timer
+// expires, the callbacks will be triggered to remove the resource.
+class EdsResourcesCache {
+public:
+  virtual ~EdsResourcesCache() = default;
+
+  /**
+   * Adds or updates a given resource name with its resource.
+   * Any callback that was previously assigned to the resource will be removed
+   * without any notification.
+   * @param resource_name the name of the resource to add/update.
+   * @param resource the contents of the resource.
+   */
+  virtual void setResource(absl::string_view resource_name,
+                           const envoy::config::endpoint::v3::ClusterLoadAssignment& resource) PURE;
+
+  /**
+   * Removes a resource from the resource cache given the resource name.
+   * The callbacks for the resource will be invoked, notifying that the resource
+   * is removed.
+   * @param resource_name the name of the resource that will be removed from
+   *        the cache.
+   */
+  virtual void removeResource(absl::string_view resource_name) PURE;
+
+  /**
+   * Retrieves a resource from the cache, and adds the given callback (if any)
+   * to the resource's removal list. if the resource is removed, all callbacks
+   * for that resource will be invoked.
+   * @param resource_name the name of the resource to fetch.
+   * @param removal_cb an optional callback that will be invoked if the resource is removed
+   *        in the future. Note that updating the resource (`setResource()`) will also
+   *        remove the callback. The caller of this function can also call
+   *        `removeCallback()` to explicitly remove the callback. The callback
+   *        is owned by the caller as it is part of the EDS subscription.
+   * @return A reference to the cluster load assignment resource, or nullopt if the
+   *         resource doesn't exist.
+   */
+  virtual OptRef
+  getResource(absl::string_view resource_name, EdsResourceRemovalCallback* removal_cb) PURE;
+
+  /**
+   * Removes a callback for a given resource name (if it was previously added).
+   * @param resource_name the name of the resource for which the callback should be removed.
+   * @param removal_cb a pointer to the callback that needs to be removed.
+   */
+  virtual void removeCallback(absl::string_view resource_name,
+                              EdsResourceRemovalCallback* removal_cb) PURE;
+
+  /**
+   * Sets an expiry timer for the given resource_name after the given ms milliseconds.
+   * Once the timer expires, the callbacks for that resource (if any) will be
+   * @param resource_name the name of the resource for which the timer should be added.
+   * @param ms the number of milliseconds until expiration.
+   */
+  virtual void setExpiryTimer(absl::string_view resource_name, std::chrono::milliseconds ms) PURE;
+
+  /**
+   * Disables the expiration timer for the given resource_name.
+   * @param resource_name the name of the resource for which the timer should be disabled.
+   */
+  virtual void disableExpiryTimer(absl::string_view resource_name) PURE;
+
+  /**
+   * Returns the number of items in the cache. Only used in tests.
+   * @return the number of items in the cache.
+   */
+  virtual uint32_t cacheSizeForTest() const PURE;
+};
+
+using EdsResourcesCachePtr = std::unique_ptr;
+using EdsResourcesCacheOptRef = OptRef;
+
+} // namespace Config
+} // namespace Envoy
diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD
index 09f73cbc92e4..a3e3fce0d05e 100644
--- a/source/extensions/config_subscription/grpc/BUILD
+++ b/source/extensions/config_subscription/grpc/BUILD
@@ -216,3 +216,14 @@ envoy_cc_library(
         "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto",
     ],
 )
+
+envoy_cc_library(
+    name = "eds_resources_cache_lib",
+    srcs = ["eds_resources_cache_impl.cc"],
+    hdrs = ["eds_resources_cache_impl.h"],
+    deps = [
+        "//envoy/config:eds_resources_cache_interface",
+        "//envoy/event:dispatcher_interface",
+        "//source/common/common:minimal_logger_lib",
+    ],
+)
diff --git a/source/extensions/config_subscription/grpc/eds_resources_cache_impl.cc b/source/extensions/config_subscription/grpc/eds_resources_cache_impl.cc
new file mode 100644
index 000000000000..b527c2b6b404
--- /dev/null
+++ b/source/extensions/config_subscription/grpc/eds_resources_cache_impl.cc
@@ -0,0 +1,86 @@
+#include "source/extensions/config_subscription/grpc/eds_resources_cache_impl.h"
+
+#include "source/common/common/logger.h"
+
+namespace Envoy {
+namespace Config {
+
+void EdsResourcesCacheImpl::setResource(
+    absl::string_view resource_name,
+    const envoy::config::endpoint::v3::ClusterLoadAssignment& resource) {
+  resources_map_.insert_or_assign(resource_name, ResourceData(resource));
+}
+
+void EdsResourcesCacheImpl::removeResource(absl::string_view resource_name) {
+  if (const auto& resource_it = resources_map_.find(resource_name);
+      resource_it != resources_map_.end()) {
+    // Invoke the callbacks, and remove all watchers.
+    for (auto& removal_cb : resource_it->second.removal_cbs_) {
+      removal_cb->onCachedResourceRemoved(resource_name);
+    }
+    resource_it->second.removal_cbs_.clear();
+
+    // Remove the resource entry from the cache.
+    resources_map_.erase(resource_it);
+  }
+  // Remove the expiration timers (if any) as there's no longer interest in the resource.
+  expiry_timers_.erase(resource_name);
+}
+
+OptRef
+EdsResourcesCacheImpl::getResource(absl::string_view resource_name,
+                                   EdsResourceRemovalCallback* removal_cb) {
+  if (const auto& resource_it = resources_map_.find(resource_name);
+      resource_it != resources_map_.end()) {
+    ENVOY_LOG_MISC(trace, "Returning resource {} from the xDS resource cache", resource_name);
+    // Add the removal callback to the list associated with the resource.
+    if (removal_cb != nullptr) {
+      resource_it->second.removal_cbs_.push_back(removal_cb);
+    }
+    return resource_it->second.resource_;
+  }
+  // The resource doesn't exist in the resource map.
+  return {};
+}
+
+void EdsResourcesCacheImpl::removeCallback(absl::string_view resource_name,
+                                           EdsResourceRemovalCallback* removal_cb) {
+  if (const auto& resource_it = resources_map_.find(resource_name);
+      resource_it != resources_map_.end()) {
+    ENVOY_LOG_MISC(trace, "Removing callback for resource {} from the xDS resource cache",
+                   resource_name);
+    resource_it->second.removal_cbs_.erase(std::remove(resource_it->second.removal_cbs_.begin(),
+                                                       resource_it->second.removal_cbs_.end(),
+                                                       removal_cb));
+  }
+}
+
+uint32_t EdsResourcesCacheImpl::cacheSizeForTest() const { return resources_map_.size(); }
+
+void EdsResourcesCacheImpl::setExpiryTimer(absl::string_view resource_name,
+                                           std::chrono::milliseconds ms) {
+  auto it = expiry_timers_.find(resource_name);
+  if (it == expiry_timers_.end()) {
+    // No timer for this resource, create one, and create a copy of resource_name that will outlive
+    // this function.
+    Event::TimerPtr resource_timeout =
+        dispatcher_.createTimer([this, str_resource_name = std::string(resource_name)]() -> void {
+          // On expiration the resource is removed (from the cache and from the watchers).
+          removeResource(str_resource_name);
+        });
+    it = expiry_timers_.emplace(resource_name, std::move(resource_timeout)).first;
+  }
+  (it->second)->enableTimer(ms);
+}
+
+void EdsResourcesCacheImpl::disableExpiryTimer(absl::string_view resource_name) {
+  auto it = expiry_timers_.find(resource_name);
+  if (it != expiry_timers_.end()) {
+    (it->second)->disableTimer();
+    // Remove the timer as it is no longer needed.
+    expiry_timers_.erase(it);
+  }
+}
+
+} // namespace Config
+} // namespace Envoy
diff --git a/source/extensions/config_subscription/grpc/eds_resources_cache_impl.h b/source/extensions/config_subscription/grpc/eds_resources_cache_impl.h
new file mode 100644
index 000000000000..0e96e7ff1d0f
--- /dev/null
+++ b/source/extensions/config_subscription/grpc/eds_resources_cache_impl.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#include 
+
+#include "envoy/config/eds_resources_cache.h"
+#include "envoy/event/dispatcher.h"
+#include "envoy/event/timer.h"
+
+#include "absl/container/flat_hash_map.h"
+
+namespace Envoy {
+namespace Config {
+
+class EdsResourcesCacheImpl : public EdsResourcesCache {
+public:
+  EdsResourcesCacheImpl(Event::Dispatcher& main_thread_dispatcher)
+      : dispatcher_(main_thread_dispatcher) {}
+
+  // EdsResourcesCache
+  void setResource(absl::string_view resource_name,
+                   const envoy::config::endpoint::v3::ClusterLoadAssignment& resource) override;
+  void removeResource(absl::string_view resource_name) override;
+  OptRef
+  getResource(absl::string_view resource_name, EdsResourceRemovalCallback* removal_cb) override;
+  void removeCallback(absl::string_view resource_name,
+                      EdsResourceRemovalCallback* removal_cb) override;
+  uint32_t cacheSizeForTest() const override;
+  void setExpiryTimer(absl::string_view resource_name, std::chrono::milliseconds ms) override;
+  void disableExpiryTimer(absl::string_view resource_name) override;
+
+private:
+  // The value of the map, holds the resource and the removal callbacks.
+  struct ResourceData {
+    envoy::config::endpoint::v3::ClusterLoadAssignment resource_;
+    std::vector removal_cbs_;
+
+    ResourceData(const envoy::config::endpoint::v3::ClusterLoadAssignment& resource)
+        : resource_(resource) {}
+  };
+  // A map between a resource name and its ResourceData.
+  absl::flat_hash_map resources_map_;
+  // The per-resource timeout timer to track when the resource should be removed.
+  absl::flat_hash_map expiry_timers_;
+  Event::Dispatcher& dispatcher_;
+};
+
+} // namespace Config
+} // namespace Envoy
diff --git a/test/extensions/config_subscription/grpc/BUILD b/test/extensions/config_subscription/grpc/BUILD
index b352ca281b13..46e75f764c04 100644
--- a/test/extensions/config_subscription/grpc/BUILD
+++ b/test/extensions/config_subscription/grpc/BUILD
@@ -241,3 +241,14 @@ envoy_cc_test(
         "//source/extensions/config_subscription/grpc:xds_source_id_lib",
     ],
 )
+
+envoy_cc_test(
+    name = "eds_resources_cache_impl_test",
+    srcs = ["eds_resources_cache_impl_test.cc"],
+    deps = [
+        "//source/extensions/config_subscription/grpc:eds_resources_cache_lib",
+        "//test/mocks/event:event_mocks",
+        "//test/test_common:simulated_time_system_lib",
+        "//test/test_common:utility_lib",
+    ],
+)
diff --git a/test/extensions/config_subscription/grpc/eds_resources_cache_impl_test.cc b/test/extensions/config_subscription/grpc/eds_resources_cache_impl_test.cc
new file mode 100644
index 000000000000..4a769c8b25f0
--- /dev/null
+++ b/test/extensions/config_subscription/grpc/eds_resources_cache_impl_test.cc
@@ -0,0 +1,323 @@
+#include "source/extensions/config_subscription/grpc/eds_resources_cache_impl.h"
+
+#include "test/mocks/event/mocks.h"
+#include "test/test_common/simulated_time_system.h"
+#include "test/test_common/utility.h"
+
+#include "gtest/gtest.h"
+
+namespace Envoy {
+namespace Config {
+
+class EdsResourcesCacheImplTest : public testing::Test {
+public:
+  using ClusterLoadAssignment = envoy::config::endpoint::v3::ClusterLoadAssignment;
+
+  EdsResourcesCacheImplTest()
+      : api_(Api::createApiForTest(time_system_)),
+        dispatcher_(api_->allocateDispatcher("test_thread")), resources_cache_(*dispatcher_.get()) {
+  }
+
+  // An implementation of EdsResourceRemovalCallback that tracks calls with using a counter.
+  struct ResourceRemovalCallbackCounter : public EdsResourceRemovalCallback {
+    void onCachedResourceRemoved(absl::string_view resource_name) override {
+      auto& resource_calls_counter = calls_counter_map_[resource_name];
+      resource_calls_counter++;
+    }
+
+    absl::flat_hash_map calls_counter_map_;
+  };
+
+  Event::SimulatedTimeSystemHelper time_system_;
+  Api::ApiPtr api_;
+  Event::DispatcherPtr dispatcher_;
+  EdsResourcesCacheImpl resources_cache_;
+};
+
+// Validates that simple addition works.
+TEST_F(EdsResourcesCacheImplTest, AddSuccess) {
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+
+  // Add a resource.
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+}
+
+// Validates that simple addition and update works.
+TEST_F(EdsResourcesCacheImplTest, AddAndSetSuccess) {
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+
+  // Add a resource.
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Add a new resource.
+  ClusterLoadAssignment new_resource;
+  new_resource.set_cluster_name("new_foo");
+  resources_cache_.setResource("new_foo_cla", new_resource);
+  EXPECT_EQ(2, resources_cache_.cacheSizeForTest());
+
+  // Update the first resource.
+  ClusterLoadAssignment resource_update;
+  resource_update.set_cluster_name("foo_update");
+  resources_cache_.setResource("foo_cla", resource_update);
+  EXPECT_EQ(2, resources_cache_.cacheSizeForTest());
+}
+
+// Validates simple addition and fetching works.
+TEST_F(EdsResourcesCacheImplTest, AddAndFetchSuccess) {
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", nullptr);
+  EXPECT_TRUE(fetched_resource.has_value());
+  EXPECT_EQ("foo", fetched_resource->cluster_name());
+}
+
+// Validates simple addition, update and fetching works.
+TEST_F(EdsResourcesCacheImplTest, AddUpdateAndFetchSuccess) {
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", nullptr);
+  EXPECT_TRUE(fetched_resource.has_value());
+  EXPECT_EQ("foo", fetched_resource->cluster_name());
+
+  // Update the resource.
+  ClusterLoadAssignment resource_update;
+  resource_update.set_cluster_name("foo_update");
+  resources_cache_.setResource("foo_cla", resource_update);
+
+  const auto& fetched_resource2 = resources_cache_.getResource("foo_cla", nullptr);
+  EXPECT_TRUE(fetched_resource2.has_value());
+  EXPECT_EQ("foo_update", fetched_resource2->cluster_name());
+}
+
+// Validates that adding and fetching a different resource returns nullopt.
+TEST_F(EdsResourcesCacheImplTest, AddAndFetchNonExistent) {
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Fetch non-existent resource name.
+  const auto& fetched_resource = resources_cache_.getResource("non_existent", nullptr);
+  EXPECT_FALSE(fetched_resource.has_value());
+}
+
+// Validates that fetching from an empty map returns nullopt.
+TEST_F(EdsResourcesCacheImplTest, FetchEmpty) {
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", nullptr);
+  EXPECT_FALSE(fetched_resource.has_value());
+}
+
+// Validates that adding and removing works.
+TEST_F(EdsResourcesCacheImplTest, AddRemoveThenFetchEmpty) {
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Remove the resource.
+  resources_cache_.removeResource("foo_cla");
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+
+  // Fetch the removed resource name.
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", nullptr);
+  EXPECT_FALSE(fetched_resource.has_value());
+}
+
+// Validates that adding and removing a non-existent resource works.
+TEST_F(EdsResourcesCacheImplTest, AddAndRemoveNonExistent) {
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Remove the resource.
+  resources_cache_.removeResource("non_existent");
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+}
+
+// Validates that the removal from an empty map works.
+TEST_F(EdsResourcesCacheImplTest, RemoveEmpty) {
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+  resources_cache_.removeResource("non_existent");
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+}
+
+// Validate removal callback gets notified.
+TEST_F(EdsResourcesCacheImplTest, RemoveCallbackCalled) {
+  ResourceRemovalCallbackCounter callback;
+
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Fetch the resource and register a callback.
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", &callback);
+  EXPECT_TRUE(fetched_resource.has_value());
+  EXPECT_EQ("foo", fetched_resource->cluster_name());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo_cla"]);
+
+  // Remove the resource.
+  resources_cache_.removeResource("foo_cla");
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+  EXPECT_EQ(1, callback.calls_counter_map_["foo_cla"]);
+}
+
+// Validate removal callback isn't invoked after update.
+TEST_F(EdsResourcesCacheImplTest, RemoveCallbackNotCalledAfterUpdate) {
+  ResourceRemovalCallbackCounter callback;
+
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Fetch the resource and register a callback.
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", &callback);
+  EXPECT_TRUE(fetched_resource.has_value());
+  EXPECT_EQ("foo", fetched_resource->cluster_name());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo_cla"]);
+
+  // Update the resource.
+  ClusterLoadAssignment resource_update;
+  resource_update.set_cluster_name("foo_update");
+  resources_cache_.setResource("foo_cla", resource_update);
+
+  // Remove the resource.
+  resources_cache_.removeResource("foo_cla");
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo_cla"]);
+}
+
+// Validate explicit callback removal does not get notification.
+TEST_F(EdsResourcesCacheImplTest, ExplicitRemoveCallback) {
+  ResourceRemovalCallbackCounter callback;
+
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Fetch the resource and register a callback.
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", &callback);
+  EXPECT_TRUE(fetched_resource.has_value());
+  EXPECT_EQ("foo", fetched_resource->cluster_name());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo_cla"]);
+
+  // Remove the callback.
+  resources_cache_.removeCallback("foo_cla", &callback);
+
+  // Remove the resource, callback not invoked.
+  resources_cache_.removeResource("foo_cla");
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo_cla"]);
+}
+
+// Validate correct removal callback gets notified when multiple resources are used.
+TEST_F(EdsResourcesCacheImplTest, MultipleResourcesRemoveCallbackCalled) {
+  ResourceRemovalCallbackCounter callback;
+
+  // Add first resource.
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Fetch the resource and register a callback.
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", &callback);
+  EXPECT_TRUE(fetched_resource.has_value());
+  EXPECT_EQ("foo", fetched_resource->cluster_name());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo_cla"]);
+
+  // Add second resource.
+  ClusterLoadAssignment resource2;
+  resource2.set_cluster_name("foo2");
+  resources_cache_.setResource("foo2_cla", resource2);
+  EXPECT_EQ(2, resources_cache_.cacheSizeForTest());
+
+  // Fetch the resource and register a callback.
+  const auto& fetched_resource2 = resources_cache_.getResource("foo2_cla", &callback);
+  EXPECT_TRUE(fetched_resource2.has_value());
+  EXPECT_EQ("foo2", fetched_resource2->cluster_name());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo2_cla"]);
+
+  // Remove the first resource.
+  resources_cache_.removeResource("foo_cla");
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+  EXPECT_EQ(1, callback.calls_counter_map_["foo_cla"]);
+  EXPECT_EQ(0, callback.calls_counter_map_["foo2_cla"]);
+
+  // Remove the second resource.
+  resources_cache_.removeResource("foo2_cla");
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+  EXPECT_EQ(1, callback.calls_counter_map_["foo_cla"]);
+  EXPECT_EQ(1, callback.calls_counter_map_["foo2_cla"]);
+}
+
+// Validate that adding a timer and expiring it invokes a notification.
+TEST_F(EdsResourcesCacheImplTest, ExpiredCacheResource) {
+  ResourceRemovalCallbackCounter callback;
+
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Set an expiration timer for the resource.
+  resources_cache_.setExpiryTimer("foo_cla", std::chrono::seconds(1));
+
+  // Fetch the resource and register a callback.
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", &callback);
+  EXPECT_TRUE(fetched_resource.has_value());
+  EXPECT_EQ("foo", fetched_resource->cluster_name());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo_cla"]);
+
+  // Make sure the expiration is removing the resource and invoking the callback.
+  time_system_.advanceTimeAndRun(std::chrono::seconds(1), *dispatcher_,
+                                 Envoy::Event::Dispatcher::RunType::Block);
+  EXPECT_EQ(0, resources_cache_.cacheSizeForTest());
+  EXPECT_EQ(1, callback.calls_counter_map_["foo_cla"]);
+}
+
+// Validate that adding a timer with an expiration timer, and disabling the timer
+// prevents invoking a notification.
+TEST_F(EdsResourcesCacheImplTest, DisableExpiredCacheResource) {
+  ResourceRemovalCallbackCounter callback;
+
+  ClusterLoadAssignment resource;
+  resource.set_cluster_name("foo");
+  resources_cache_.setResource("foo_cla", resource);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+
+  // Set an expiration timer for the resource.
+  resources_cache_.setExpiryTimer("foo_cla", std::chrono::seconds(1));
+
+  // Fetch the resource and register a callback.
+  const auto& fetched_resource = resources_cache_.getResource("foo_cla", &callback);
+  EXPECT_TRUE(fetched_resource.has_value());
+  EXPECT_EQ("foo", fetched_resource->cluster_name());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo_cla"]);
+
+  // Disable the timer
+  resources_cache_.disableExpiryTimer("foo_cla");
+
+  // Make sure the expiration isn't invoking the callback.
+  time_system_.advanceTimeAndRun(std::chrono::seconds(1), *dispatcher_,
+                                 Envoy::Event::Dispatcher::RunType::Block);
+  EXPECT_EQ(1, resources_cache_.cacheSizeForTest());
+  EXPECT_EQ(0, callback.calls_counter_map_["foo_cla"]);
+}
+
+} // namespace Config
+} // namespace Envoy

From 164187e552cc65fde27e7f546199f93ce6763011 Mon Sep 17 00:00:00 2001
From: Haiyue Wang 
Date: Thu, 29 Jun 2023 01:00:57 +0800
Subject: [PATCH 671/740] docs: Update the Traffic tapping configuration
 (#28158)

The option 'envoy.config.tap.v3.TapConfig.match_config' has been marked
deprecated by the commit 520389e677cd ("tap: factor out the TAP filter
matcher for later reuse in other filters (#12429)").

Update the option to 'envoy.config.tap.v3.TapConfig.match'. And copy the
configuration out of rst and into yaml.

Signed-off-by: Haiyue Wang 
---
 .../_include/traffic_tapping_plain_text.yaml  | 60 +++++++++++++++++++
 .../_include/traffic_tapping_ssl.yaml         | 60 +++++++++++++++++++
 docs/root/operations/traffic_tapping.rst      | 49 ++++-----------
 3 files changed, 132 insertions(+), 37 deletions(-)
 create mode 100644 docs/root/operations/_include/traffic_tapping_plain_text.yaml
 create mode 100644 docs/root/operations/_include/traffic_tapping_ssl.yaml

diff --git a/docs/root/operations/_include/traffic_tapping_plain_text.yaml b/docs/root/operations/_include/traffic_tapping_plain_text.yaml
new file mode 100644
index 000000000000..2fa80b39f0b2
--- /dev/null
+++ b/docs/root/operations/_include/traffic_tapping_plain_text.yaml
@@ -0,0 +1,60 @@
+static_resources:
+  listeners:
+  - address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 8000
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          codec_type: AUTO
+          stat_prefix: ingress_http
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: app
+              domains:
+              - "*"
+              routes:
+              - match:
+                  prefix: "/"
+                route:
+                  cluster: service-http
+          http_filters:
+          - name: envoy.filters.http.router
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+      transport_socket:
+        name: envoy.transport_sockets.tap
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tap.v3.Tap
+          common_config:
+            static_config:
+              match:
+                any_match: true
+              output_config:
+                sinks:
+                - format: PROTO_BINARY
+                  file_per_tap:
+                    path_prefix: /some/tap/path
+          transport_socket:
+            name: envoy.transport_sockets.raw_buffer
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer
+
+  clusters:
+  - name: service-http
+    type: STATIC
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: service-http
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: 127.0.0.1
+                port_value: 80
+                protocol: TCP
diff --git a/docs/root/operations/_include/traffic_tapping_ssl.yaml b/docs/root/operations/_include/traffic_tapping_ssl.yaml
new file mode 100644
index 000000000000..0a82d54a6b17
--- /dev/null
+++ b/docs/root/operations/_include/traffic_tapping_ssl.yaml
@@ -0,0 +1,60 @@
+static_resources:
+  listeners:
+  - address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 8000
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          codec_type: AUTO
+          stat_prefix: ingress_http
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: app
+              domains:
+              - "*"
+              routes:
+              - match:
+                  prefix: "/"
+                route:
+                  cluster: service-https
+          http_filters:
+          - name: envoy.filters.http.router
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+
+  clusters:
+  - name: service-https
+    type: STATIC
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: service-https
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: 127.0.0.1
+                port_value: 8080
+                protocol: TCP
+    transport_socket:
+      name: envoy.transport_sockets.tap
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tap.v3.Tap
+        common_config:
+          static_config:
+            match:
+              any_match: true
+            output_config:
+              sinks:
+              - format: PROTO_BINARY
+                file_per_tap:
+                  path_prefix: /some/tap/path
+        transport_socket:
+          name: envoy.transport_sockets.tls
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
diff --git a/docs/root/operations/traffic_tapping.rst b/docs/root/operations/traffic_tapping.rst
index fafb86d32ad7..3722890a2682 100644
--- a/docs/root/operations/traffic_tapping.rst
+++ b/docs/root/operations/traffic_tapping.rst
@@ -30,46 +30,21 @@ To configure traffic tapping, add an ``envoy.transport_sockets.tap`` transport s
 :ref:`configuration ` to the listener
 or cluster. For a plain text socket this might look like:
 
-.. code-block:: yaml
-
-  transport_socket:
-    name: envoy.transport_sockets.tap
-    typed_config:
-      "@type": type.googleapis.com/envoy.extensions.transport_sockets.tap.v3.Tap
-      common_config:
-        static_config:
-          match_config:
-            any_match: true
-          output_config:
-            sinks:
-              - format: PROTO_BINARY
-                file_per_tap:
-                  path_prefix: /some/tap/path
-      transport_socket:
-        name: envoy.transport_sockets.raw_buffer
-        typed_config:
-          "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer
+.. literalinclude:: _include/traffic_tapping_plain_text.yaml
+   :language: yaml
+   :lines: 29-45
+   :linenos:
+   :lineno-start: 29
+   :caption: :download:`traffic_tapping_plain_text.yaml <_include/traffic_tapping_plain_text.yaml>`
 
 For a TLS socket, this will be:
 
-.. code-block:: yaml
-
-  transport_socket:
-    name: envoy.transport_sockets.tap
-    typed_config:
-      "@type": type.googleapis.com/envoy.extensions.transport_sockets.tap.v3.Tap
-      common_config:
-        static_config:
-          match_config:
-            any_match: true
-          output_config:
-            sinks:
-              - format: PROTO_BINARY
-                file_per_tap:
-                  path_prefix: /some/tap/path
-      transport_socket:
-        name: envoy.transport_sockets.tls
-        typed_config: 
+.. literalinclude:: _include/traffic_tapping_ssl.yaml
+   :language: yaml
+   :lines: 44-60
+   :linenos:
+   :lineno-start: 44
+   :caption: :download:`traffic_tapping_ssl.yaml <_include/traffic_tapping_ssl.yaml>`
 
 where the TLS context configuration replaces any existing :ref:`downstream
 ` or :ref:`upstream

From 3e29621531e42c4f2aa8562f35b03d74db160bd7 Mon Sep 17 00:00:00 2001
From: Kevin Baichoo 
Date: Wed, 28 Jun 2023 13:51:11 -0400
Subject: [PATCH 672/740] Efficiency: Compact HostSource. (#28165)

* Squeeze Host Source -> goes from 12B -> 8B.

Signed-off-by: Kevin Baichoo 
---
 source/common/upstream/load_balancer_impl.h | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h
index 4f632905ae80..1c20752b1afe 100644
--- a/source/common/upstream/load_balancer_impl.h
+++ b/source/common/upstream/load_balancer_impl.h
@@ -266,7 +266,7 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase {
   // RoundRobinLoadBalancer, to index into auxiliary data structures specific to the LB for
   // a given host set selection.
   struct HostsSource {
-    enum class SourceType {
+    enum class SourceType : uint8_t {
       // All hosts in the host set.
       AllHosts,
       // All healthy hosts in the host set.
@@ -281,20 +281,27 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase {
 
     HostsSource() = default;
 
+    // TODO(kbaichoo): plumb the priority parameter as uint8_t all the way from
+    // the config.
     HostsSource(uint32_t priority, SourceType source_type)
-        : priority_(priority), source_type_(source_type) {
+        : priority_(static_cast(priority)), source_type_(source_type) {
+      ASSERT(priority <= 128, "Priority out of bounds.");
       ASSERT(source_type == SourceType::AllHosts || source_type == SourceType::HealthyHosts ||
              source_type == SourceType::DegradedHosts);
     }
 
+    // TODO(kbaichoo): plumb the priority parameter as uint8_t all the way from
+    // the config.
     HostsSource(uint32_t priority, SourceType source_type, uint32_t locality_index)
-        : priority_(priority), source_type_(source_type), locality_index_(locality_index) {
+        : priority_(static_cast(priority)), source_type_(source_type),
+          locality_index_(locality_index) {
+      ASSERT(priority <= 128, "Priority out of bounds.");
       ASSERT(source_type == SourceType::LocalityHealthyHosts ||
              source_type == SourceType::LocalityDegradedHosts);
     }
 
     // Priority in PrioritySet.
-    uint32_t priority_{};
+    uint8_t priority_{};
 
     // How to index into HostSet for a given priority.
     SourceType source_type_{};

From edebad172551bb5029a5027ae5fb927510ca2696 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Wed, 28 Jun 2023 19:11:47 +0100
Subject: [PATCH 673/740] docs/config: Exclude tap config from server test
 (#28171)

Signed-off-by: Ryan Northey 
---
 docs/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/BUILD b/docs/BUILD
index 782352cd70f3..5426a19734c9 100644
--- a/docs/BUILD
+++ b/docs/BUILD
@@ -25,6 +25,7 @@ filegroup(
             "root/**/envoy-dynamic-cds-demo.yaml",
             "root/**/envoy-dynamic-lds-demo.yaml",
             "root/**/envoy-dynamic-filesystem-demo.yaml",
+            "root/operations/_include/traffic_tapping_*.yaml",
             # TODO(phlax/windows-dev): figure out how to get this working on windows
             #      "Error: unable to read file: /etc/ssl/certs/ca-certificates.crt"
             "root/configuration/http/http_filters/_include/dns-cache-circuit-breaker.yaml",

From d266941360e811eb34fc632d914211437e13f962 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Wed, 28 Jun 2023 22:13:52 +0100
Subject: [PATCH 674/740] ci/tests: Disable flakey resource monitor test
 (#28174)

Signed-off-by: Ryan Northey 
---
 .../injected_resource_monitor_integration_test.cc              | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
index 9d41d6e6b899..d25a03a00dc8 100644
--- a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
+++ b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
@@ -184,7 +184,8 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ListenerMaxConnectionPerSocketEventTest,
                          testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
                          TestUtility::ipTestParamsToString);
 
-TEST_P(ListenerMaxConnectionPerSocketEventTest, AcceptsConnectionsUpToTheMaximumPerSocketEvent) {
+TEST_P(ListenerMaxConnectionPerSocketEventTest,
+       DISABLED_AcceptsConnectionsUpToTheMaximumPerSocketEvent) {
   auto set_max_connections_per_socket_event_to_two =
       [](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
         for (auto& listener_config : *bootstrap.mutable_static_resources()->mutable_listeners()) {

From c1cae43bed0cd91b423dafa388a370a27cb163e7 Mon Sep 17 00:00:00 2001
From: Haiyue Wang 
Date: Thu, 29 Jun 2023 17:53:36 +0800
Subject: [PATCH 675/740] tap: Fix the failure of protobuf to PCAP generation
 (#28180)

tap: Fix the protobuf to PCAP generation failure

When run 'bazel run @envoy_api//tools:tap2pcap path_0.pb path_0.pcap':
  ...
  Traceback (most recent call last):
  File "..../tools/tap2pcap.runfiles/envoy_api/tools/tap2pcap.py", line 88, in 
    tap2pcap(sys.argv[1], sys.argv[2])
  File "..../tools/tap2pcap.runfiles/envoy_api/tools/tap2pcap.py", line 53, in tap2pcap
    wrapper.ParseFromString(f.read())
                            ^^^^^^^^
  File "", line 322, in decode
  UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb8 in position 1: invalid start byte
  ...

The protobuf file is in binary format, opening this file in binary mode
will help to generate the PCAP file successfully.

Signed-off-by: Haiyue Wang 
---
 api/tools/tap2pcap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/tools/tap2pcap.py b/api/tools/tap2pcap.py
index bcb13fdf9a09..1f77325684c1 100644
--- a/api/tools/tap2pcap.py
+++ b/api/tools/tap2pcap.py
@@ -49,7 +49,7 @@ def tap2pcap(tap_path, pcap_path):
         with open(tap_path, 'r') as f:
             text_format.Merge(f.read(), wrapper)
     else:
-        with open(tap_path, 'r') as f:
+        with open(tap_path, 'rb') as f:
             wrapper.ParseFromString(f.read())
 
     trace = wrapper.socket_buffered_trace

From cc5d3d057fbca302a3ee2b524e4f8948d9bf88ea Mon Sep 17 00:00:00 2001
From: Dmitri Dolguikh 
Date: Thu, 29 Jun 2023 04:49:54 -0700
Subject: [PATCH 676/740] Log uncaught exception (#28147)

Currently when the terminate handler handles an uncaught exception, it provides limited information about what happened and what exception was thrown. This change attempts to log additional details about the exception.

Commit Message: Log uncaught exceptions when possible
Additional Description: Provide more details when possible when an uncaught exception is handled in terminate handler.
Risk Level: low

Log uncaught exceptions when possible

Signed-off-by: Dmitri Dolguikh 
---
 source/exe/terminate_handler.cc    | 23 ++++++++++++++++++-
 source/exe/terminate_handler.h     |  7 ++++++
 test/exe/terminate_handler_test.cc | 36 ++++++++++++++++++++++++++++++
 tools/code_format/config.yaml      |  1 +
 4 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/source/exe/terminate_handler.cc b/source/exe/terminate_handler.cc
index 74c68e2076f2..40b987a4065d 100644
--- a/source/exe/terminate_handler.cc
+++ b/source/exe/terminate_handler.cc
@@ -2,17 +2,38 @@
 
 #include 
 
+#include "envoy/common/exception.h"
+
 #include "source/common/common/logger.h"
 #include "source/server/backtrace.h"
 
+#include "absl/strings/str_format.h"
+
 namespace Envoy {
 
 std::terminate_handler TerminateHandler::logOnTerminate() const {
   return std::set_terminate([]() {
-    ENVOY_LOG(critical, "std::terminate called! (possible uncaught exception, see trace)");
+    logException(std::current_exception());
     BACKTRACE_LOG();
     std::abort();
   });
 }
 
+void TerminateHandler::logException(const std::exception_ptr current_exception) {
+  if (current_exception != nullptr) {
+    try {
+      std::rethrow_exception(current_exception);
+    } catch (const EnvoyException& e) {
+      ENVOY_LOG(critical, "std::terminate called! Uncaught EnvoyException '{}', see trace.",
+                e.what());
+    } catch (const std::exception& e) {
+      ENVOY_LOG(critical, "std::terminate called! Uncaught exception '{}', see trace.", e.what());
+    } catch (...) {
+      ENVOY_LOG(critical, "std::terminate called! Uncaught unknown exception, see trace.");
+    }
+  } else {
+    ENVOY_LOG(critical, "std::terminate called! See trace.");
+  }
+}
+
 } // namespace Envoy
diff --git a/source/exe/terminate_handler.h b/source/exe/terminate_handler.h
index 62ddfe587673..ab9155c3abaa 100644
--- a/source/exe/terminate_handler.h
+++ b/source/exe/terminate_handler.h
@@ -13,12 +13,19 @@ class TerminateHandler : NonCopyable, protected Logger::Loggable void { std::terminate(); }(), ".*std::terminate called!.*");
 }
 
+TEST_F(TerminateHandlerTest, LogsEnvoyException) {
+  EXPECT_LOG_CONTAINS("critical",
+                      "std::terminate called! Uncaught EnvoyException 'boom', see trace.",
+                      logException(std::make_exception_ptr(EnvoyException("boom"))));
+}
+
+#ifdef WIN32
+TEST_F(TerminateHandlerTest, LogsStdException) {
+  EXPECT_LOG_CONTAINS("critical",
+                      "std::terminate called! Uncaught exception 'Unknown exception', see trace.",
+                      logException(std::make_exception_ptr(std::exception())));
+}
+#else
+TEST_F(TerminateHandlerTest, LogsStdException) {
+  EXPECT_LOG_CONTAINS("critical",
+                      "std::terminate called! Uncaught exception 'std::exception', see trace.",
+                      logException(std::make_exception_ptr(std::exception())));
+}
+#endif
+
+TEST_F(TerminateHandlerTest, LogsUnknownException) {
+  EXPECT_LOG_CONTAINS("critical", "std::terminate called! Uncaught unknown exception, see trace.",
+                      logException(std::make_exception_ptr("Boom")));
+}
+
+TEST_F(TerminateHandlerTest, HandlesNullptrCurrentException) {
+  EXPECT_LOG_CONTAINS("critical", "std::terminate called! See trace.", logException(nullptr));
+}
+
 } // namespace Envoy
diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml
index 7a3747fc1fcb..d3e8a9a15a60 100644
--- a/tools/code_format/config.yaml
+++ b/tools/code_format/config.yaml
@@ -246,6 +246,7 @@ paths:
   raw_try:
     include:
     - source/common/common/thread.h
+    - source/exe/terminate_handler.cc
 
   # Files matching these exact names can reference real-world time. These include the class
   # definitions for real-world time, the construction of them in main(), and perf annotation.

From eb662c2df4ffe5382056482d7a4d351bc5556b94 Mon Sep 17 00:00:00 2001
From: ohadvano <49730675+ohadvano@users.noreply.github.com>
Date: Thu, 29 Jun 2023 14:17:22 +0200
Subject: [PATCH 677/740] tcp_proxy: fix allowing multiple readDisable calls
 when tunneling  (#28014)

Signed-off-by: ohadvano 
---
 changelogs/current.yaml                       |   6 +
 envoy/network/connection.h                    |  10 +-
 source/common/network/connection_impl.cc      |  15 +-
 source/common/network/connection_impl.h       |   2 +-
 .../network/multi_connection_base_impl.cc     |  18 +-
 .../network/multi_connection_base_impl.h      |   2 +-
 .../quic_filter_manager_connection_impl.h     |   5 +-
 source/common/tcp_proxy/tcp_proxy.cc          |  15 +-
 source/server/api_listener_impl.h             |   2 +-
 test/common/network/connection_impl_test.cc   | 177 +++++++++++-------
 .../multi_connection_base_impl_test.cc        |  22 ++-
 test/integration/BUILD                        |   2 +
 .../buffer_accounting_integration_test.cc     |  66 +++++++
 test/mocks/network/connection.h               |   2 +-
 14 files changed, 250 insertions(+), 94 deletions(-)

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
index 03e2fcfdcb38..b65becf19699 100644
--- a/changelogs/current.yaml
+++ b/changelogs/current.yaml
@@ -182,6 +182,12 @@ bug_fixes:
     - `CVE-2023-31147 `_.
     - `CVE-2023-31124 `_.
     - `CVE-2023-32067 `_.
+- area: tcp_proxy
+  change: |
+    fixed assert crash when multiple ``readDisable`` are called for TCP tunneling
+    scenarios, by allowing multiple calls. This will also cause stats that indicate
+    disable or enable of downstream read to be flushed only once per actual disabling
+    or enabling.
 - area: redis_proxy
   change: |
     Fixes a bug where route properties such as key_formatter,
diff --git a/envoy/network/connection.h b/envoy/network/connection.h
index 301366c4e54f..8c48af9f0928 100644
--- a/envoy/network/connection.h
+++ b/envoy/network/connection.h
@@ -84,6 +84,13 @@ class Connection : public Event::DeferredDeletable,
 public:
   enum class State { Open, Closing, Closed };
 
+  enum class ReadDisableStatus {
+    NoTransition,
+    StillReadDisabled,
+    TransitionedToReadEnabled,
+    TransitionedToReadDisabled
+  };
+
   /**
    * Callback function for when bytes have been sent by a connection.
    * @param bytes_sent supplies the number of bytes written to the connection.
@@ -179,6 +186,7 @@ class Connection : public Event::DeferredDeletable,
    * enabled again if there is data still in the input buffer it will be re-dispatched through
    * the filter chain.
    * @param disable supplies TRUE is reads should be disabled, FALSE if they should be enabled.
+   * @return status enum indicating the outcome of calling readDisable on the underlying socket.
    *
    * Note that this function reference counts calls. For example
    * readDisable(true);  // Disables data
@@ -186,7 +194,7 @@ class Connection : public Event::DeferredDeletable,
    * readDisable(false);  // Notes the connection is blocked by one source
    * readDisable(false);  // Marks the connection as unblocked, so resumes reading.
    */
-  virtual void readDisable(bool disable) PURE;
+  virtual ReadDisableStatus readDisable(bool disable) PURE;
 
   /**
    * Set if Envoy should detect TCP connection close when readDisable(true) is called.
diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc
index c61ca149d3ae..8ed55494fe07 100644
--- a/source/common/network/connection_impl.cc
+++ b/source/common/network/connection_impl.cc
@@ -354,7 +354,7 @@ void ConnectionImpl::enableHalfClose(bool enabled) {
   enable_half_close_ = enabled;
 }
 
-void ConnectionImpl::readDisable(bool disable) {
+Connection::ReadDisableStatus ConnectionImpl::readDisable(bool disable) {
   // Calls to readEnabled on a closed socket are considered to be an error.
   ASSERT(state() == State::Open);
 
@@ -373,11 +373,12 @@ void ConnectionImpl::readDisable(bool disable) {
 
     if (state() != State::Open) {
       // If readDisable is called on a closed connection, do not crash.
-      return;
+      return ReadDisableStatus::NoTransition;
     }
+
     if (read_disable_count_ > 1) {
       // The socket has already been read disabled.
-      return;
+      return ReadDisableStatus::StillReadDisabled;
     }
 
     // If half-close semantics are enabled, we never want early close notifications; we
@@ -387,18 +388,22 @@ void ConnectionImpl::readDisable(bool disable) {
     } else {
       ioHandle().enableFileEvents(Event::FileReadyType::Write);
     }
+
+    return ReadDisableStatus::TransitionedToReadDisabled;
   } else {
     ASSERT(read_disable_count_ != 0);
     --read_disable_count_;
     if (state() != State::Open) {
       // If readDisable is called on a closed connection, do not crash.
-      return;
+      return ReadDisableStatus::NoTransition;
     }
 
+    auto read_disable_status = ReadDisableStatus::StillReadDisabled;
     if (read_disable_count_ == 0) {
       // We never ask for both early close and read at the same time. If we are reading, we want to
       // consume all available data.
       ioHandle().enableFileEvents(Event::FileReadyType::Read | Event::FileReadyType::Write);
+      read_disable_status = ReadDisableStatus::TransitionedToReadEnabled;
     }
 
     if (filterChainWantsData() && (read_buffer_->length() > 0 || transport_wants_read_)) {
@@ -416,6 +421,8 @@ void ConnectionImpl::readDisable(bool disable) {
       dispatch_buffered_data_ = true;
       ioHandle().activateFileEvents(Event::FileReadyType::Read);
     }
+
+    return read_disable_status;
   }
 }
 
diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h
index a8b7012a37a5..44a2a4efb74c 100644
--- a/source/common/network/connection_impl.h
+++ b/source/common/network/connection_impl.h
@@ -75,7 +75,7 @@ class ConnectionImpl : public ConnectionImplBase, public TransportSocketCallback
   }
   std::string nextProtocol() const override { return transport_socket_->protocol(); }
   void noDelay(bool enable) override;
-  void readDisable(bool disable) override;
+  ReadDisableStatus readDisable(bool disable) override;
   void detectEarlyCloseWhenReadDisabled(bool value) override { detect_early_close_ = value; }
   bool readEnabled() const override;
   ConnectionInfoSetter& connectionInfoSetter() override {
diff --git a/source/common/network/multi_connection_base_impl.cc b/source/common/network/multi_connection_base_impl.cc
index 839ad3af4cad..dfd149f2b918 100644
--- a/source/common/network/multi_connection_base_impl.cc
+++ b/source/common/network/multi_connection_base_impl.cc
@@ -119,21 +119,33 @@ void MultiConnectionBaseImpl::noDelay(bool enable) {
   }
 }
 
-void MultiConnectionBaseImpl::readDisable(bool disable) {
+Connection::ReadDisableStatus MultiConnectionBaseImpl::readDisable(bool disable) {
   if (connect_finished_) {
-    connections_[0]->readDisable(disable);
-    return;
+    return connections_[0]->readDisable(disable);
   }
+
   if (!post_connect_state_.read_disable_count_.has_value()) {
     post_connect_state_.read_disable_count_ = 0;
   }
 
+  auto read_disable_state = ReadDisableStatus::StillReadDisabled;
+
   if (disable) {
+    if (post_connect_state_.read_disable_count_ == 0) {
+      read_disable_state = ReadDisableStatus::TransitionedToReadDisabled;
+    }
+
     post_connect_state_.read_disable_count_.value()++;
   } else {
     ASSERT(post_connect_state_.read_disable_count_ != 0);
     post_connect_state_.read_disable_count_.value()--;
+
+    if (post_connect_state_.read_disable_count_ == 0) {
+      read_disable_state = ReadDisableStatus::TransitionedToReadEnabled;
+    }
   }
+
+  return read_disable_state;
 }
 
 void MultiConnectionBaseImpl::detectEarlyCloseWhenReadDisabled(bool value) {
diff --git a/source/common/network/multi_connection_base_impl.h b/source/common/network/multi_connection_base_impl.h
index a6521d0a0254..c88bc9924609 100644
--- a/source/common/network/multi_connection_base_impl.h
+++ b/source/common/network/multi_connection_base_impl.h
@@ -89,7 +89,7 @@ class MultiConnectionBaseImpl : public ClientConnection,
   // Methods which are applied to each connection attempt.
   void enableHalfClose(bool enabled) override;
   void noDelay(bool enable) override;
-  void readDisable(bool disable) override;
+  ReadDisableStatus readDisable(bool disable) override;
   void detectEarlyCloseWhenReadDisabled(bool value) override;
   void setConnectionStats(const ConnectionStats& stats) override;
   void setDelayedCloseTimeout(std::chrono::milliseconds timeout) override;
diff --git a/source/common/quic/quic_filter_manager_connection_impl.h b/source/common/quic/quic_filter_manager_connection_impl.h
index 1520e01400e4..154123b64160 100644
--- a/source/common/quic/quic_filter_manager_connection_impl.h
+++ b/source/common/quic/quic_filter_manager_connection_impl.h
@@ -65,7 +65,10 @@ class QuicFilterManagerConnectionImpl : public Network::ConnectionImplBase,
   void noDelay(bool /*enable*/) override {}
   // Neither readDisable nor detectEarlyCloseWhenReadDisabled are supported for QUIC.
   // Crash in debug mode if they are called.
-  void readDisable(bool /*disable*/) override { IS_ENVOY_BUG("Unexpected call to readDisable"); }
+  ReadDisableStatus readDisable(bool /*disable*/) override {
+    IS_ENVOY_BUG("Unexpected call to readDisable");
+    return ReadDisableStatus::NoTransition;
+  }
   void detectEarlyCloseWhenReadDisabled(bool /*value*/) override {
     IS_ENVOY_BUG("Unexpected call to detectEarlyCloseWhenReadDisabled");
   }
diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc
index fd5b12dbd144..4d611627a32b 100644
--- a/source/common/tcp_proxy/tcp_proxy.cc
+++ b/source/common/tcp_proxy/tcp_proxy.cc
@@ -293,11 +293,14 @@ void Filter::readDisableDownstream(bool disable) {
     // despite the downstream connection being closed.
     return;
   }
-  read_callbacks_->connection().readDisable(disable);
 
-  if (disable) {
+  const Network::Connection::ReadDisableStatus read_disable_status =
+      read_callbacks_->connection().readDisable(disable);
+
+  if (read_disable_status == Network::Connection::ReadDisableStatus::TransitionedToReadDisabled) {
     config_->stats().downstream_flow_control_paused_reading_total_.inc();
-  } else {
+  } else if (read_disable_status ==
+             Network::Connection::ReadDisableStatus::TransitionedToReadEnabled) {
     config_->stats().downstream_flow_control_resumed_reading_total_.inc();
   }
 }
@@ -333,7 +336,8 @@ void Filter::UpstreamCallbacks::onEvent(Network::ConnectionEvent event) {
 }
 
 void Filter::UpstreamCallbacks::onAboveWriteBufferHighWatermark() {
-  ASSERT(!on_high_watermark_called_);
+  // TCP Tunneling may call on high/low watermark multiple times.
+  ASSERT(parent_->config_->tunnelingConfigHelper() || !on_high_watermark_called_);
   on_high_watermark_called_ = true;
 
   if (parent_ != nullptr) {
@@ -343,7 +347,8 @@ void Filter::UpstreamCallbacks::onAboveWriteBufferHighWatermark() {
 }
 
 void Filter::UpstreamCallbacks::onBelowWriteBufferLowWatermark() {
-  ASSERT(on_high_watermark_called_);
+  // TCP Tunneling may call on high/low watermark multiple times.
+  ASSERT(parent_->config_->tunnelingConfigHelper() || on_high_watermark_called_);
   on_high_watermark_called_ = false;
 
   if (parent_ != nullptr) {
diff --git a/source/server/api_listener_impl.h b/source/server/api_listener_impl.h
index 8508c852e7b1..fb787f8172ef 100644
--- a/source/server/api_listener_impl.h
+++ b/source/server/api_listener_impl.h
@@ -127,7 +127,7 @@ class ApiListenerImplBase : public ApiListener,
       void hashKey(std::vector&) const override {}
       std::string nextProtocol() const override { return EMPTY_STRING; }
       void noDelay(bool) override { IS_ENVOY_BUG("Unexpected function call"); }
-      void readDisable(bool) override {}
+      ReadDisableStatus readDisable(bool) override { return ReadDisableStatus::NoTransition; }
       void detectEarlyCloseWhenReadDisabled(bool) override {
         IS_ENVOY_BUG("Unexpected function call");
       }
diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc
index 12eeef31f26e..27f558208237 100644
--- a/test/common/network/connection_impl_test.cc
+++ b/test/common/network/connection_impl_test.cc
@@ -758,31 +758,37 @@ TEST_P(ConnectionImplTest, ReadDisable) {
       std::move(mocks.transport_socket_), stream_info_, true);
 
   EXPECT_CALL(*mocks.file_event_, setEnabled(_));
-  connection->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            connection->readDisable(true));
   EXPECT_CALL(*mocks.file_event_, setEnabled(_));
-  connection->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            connection->readDisable(false));
 
   EXPECT_CALL(*mocks.file_event_, setEnabled(_));
-  connection->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            connection->readDisable(true));
   EXPECT_CALL(*mocks.file_event_, setEnabled(_)).Times(0);
-  connection->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection->readDisable(true));
   EXPECT_CALL(*mocks.file_event_, setEnabled(_)).Times(0);
-  connection->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection->readDisable(false));
   EXPECT_CALL(*mocks.file_event_, setEnabled(_));
-  connection->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            connection->readDisable(false));
 
   EXPECT_CALL(*mocks.file_event_, setEnabled(_));
-  connection->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            connection->readDisable(true));
   EXPECT_CALL(*mocks.file_event_, setEnabled(_)).Times(0);
-  connection->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection->readDisable(true));
   EXPECT_CALL(*mocks.file_event_, setEnabled(_)).Times(0);
-  connection->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection->readDisable(false));
   EXPECT_CALL(*mocks.file_event_, setEnabled(_)).Times(0);
-  connection->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection->readDisable(true));
   EXPECT_CALL(*mocks.file_event_, setEnabled(_)).Times(0);
-  connection->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection->readDisable(false));
   EXPECT_CALL(*mocks.file_event_, setEnabled(_));
-  connection->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            connection->readDisable(false));
 
   connection->close(ConnectionCloseType::NoFlush);
 }
@@ -808,14 +814,16 @@ TEST_P(ConnectionImplTest, ReadEnableDispatches) {
   }
 
   {
-    client_connection_->readDisable(true);
+    EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+              client_connection_->readDisable(true));
     EXPECT_CALL(*client_read_filter, onData(BufferStringEqual("data"), false))
         .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) -> FilterStatus {
           buffer.drain(buffer.length());
           dispatcher_->exit();
           return FilterStatus::StopIteration;
         }));
-    client_connection_->readDisable(false);
+    EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+              client_connection_->readDisable(false));
     dispatcher_->run(Event::Dispatcher::RunType::Block);
   }
 
@@ -848,9 +856,12 @@ TEST_P(ConnectionImplTest, KickUndone) {
     // Like ReadEnableDispatches above, read disable and read enable to kick off
     // an extra read. But then readDisable again and make sure the kick doesn't
     // happen.
-    client_connection_->readDisable(true);
-    client_connection_->readDisable(false); // Sets dispatch_buffered_data_
-    client_connection_->readDisable(true);
+    EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+              client_connection_->readDisable(true));
+    EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+              client_connection_->readDisable(false)); // Sets dispatch_buffered_data_
+    EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+              client_connection_->readDisable(true));
     EXPECT_CALL(*client_read_filter, onData(_, _)).Times(0);
     dispatcher_->run(Event::Dispatcher::RunType::NonBlock);
   }
@@ -859,7 +870,8 @@ TEST_P(ConnectionImplTest, KickUndone) {
   // pass up the stack (no data is read)
   {
     connection_buffer->drain(connection_buffer->length());
-    client_connection_->readDisable(false);
+    EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+              client_connection_->readDisable(false));
     EXPECT_CALL(*client_read_filter, onData(_, _)).Times(0);
     // Data no longer buffered - even if dispatch_buffered_data_ lingered it should have no effect.
     dispatcher_->run(Event::Dispatcher::RunType::NonBlock);
@@ -874,16 +886,24 @@ TEST_P(ConnectionImplTest, KickUndone) {
 TEST_P(ConnectionImplTest, ReadDisableAfterCloseHandledGracefully) {
   setUpBasicConnection();
 
-  client_connection_->readDisable(true);
-  client_connection_->readDisable(false);
-
-  client_connection_->readDisable(true);
-  client_connection_->readDisable(true);
-  client_connection_->readDisable(false);
-  client_connection_->readDisable(false);
-
-  client_connection_->readDisable(true);
-  client_connection_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            client_connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            client_connection_->readDisable(false));
+
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            client_connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+            client_connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+            client_connection_->readDisable(false));
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            client_connection_->readDisable(false));
+
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            client_connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+            client_connection_->readDisable(true));
   disconnect(false);
 #ifndef NDEBUG
   // When running in debug mode, verify that calls to readDisable and readEnabled on a closed socket
@@ -893,11 +913,11 @@ TEST_P(ConnectionImplTest, ReadDisableAfterCloseHandledGracefully) {
   EXPECT_DEBUG_DEATH(client_connection_->readDisable(false), "");
 #else
   // When running in release mode, verify that calls to readDisable change the readEnabled state.
-  client_connection_->readDisable(false);
-  client_connection_->readDisable(true);
-  client_connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::NoTransition, client_connection_->readDisable(false));
+  EXPECT_EQ(Connection::ReadDisableStatus::NoTransition, client_connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::NoTransition, client_connection_->readDisable(false));
   EXPECT_FALSE(client_connection_->readEnabled());
-  client_connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::NoTransition, client_connection_->readDisable(false));
   EXPECT_TRUE(client_connection_->readEnabled());
 #endif
 }
@@ -911,7 +931,8 @@ TEST_P(ConnectionImplTest, EarlyCloseOnReadDisabledConnection) {
   setUpBasicConnection();
   connect();
 
-  client_connection_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            client_connection_->readDisable(true));
 
   EXPECT_CALL(client_callbacks_, onEvent(ConnectionEvent::RemoteClose))
       .WillOnce(InvokeWithoutArgs([&]() -> void { dispatcher_->exit(); }));
@@ -926,7 +947,8 @@ TEST_P(ConnectionImplTest, CloseOnReadDisableWithoutCloseDetection) {
   connect();
 
   client_connection_->detectEarlyCloseWhenReadDisabled(false);
-  client_connection_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            client_connection_->readDisable(true));
 
   EXPECT_CALL(client_callbacks_, onEvent(ConnectionEvent::RemoteClose)).Times(0);
   EXPECT_CALL(server_callbacks_, onEvent(ConnectionEvent::LocalClose))
@@ -934,7 +956,8 @@ TEST_P(ConnectionImplTest, CloseOnReadDisableWithoutCloseDetection) {
   server_connection_->close(ConnectionCloseType::FlushWrite);
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 
-  client_connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            client_connection_->readDisable(false));
   EXPECT_CALL(client_callbacks_, onEvent(ConnectionEvent::RemoteClose))
       .WillOnce(InvokeWithoutArgs([&]() -> void { dispatcher_->exit(); }));
   dispatcher_->run(Event::Dispatcher::RunType::Block);
@@ -986,7 +1009,8 @@ TEST_P(ConnectionImplTest, HalfCloseNoEarlyCloseDetection) {
   connect();
 
   server_connection_->enableHalfClose(true);
-  server_connection_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            server_connection_->readDisable(true));
 
   EXPECT_CALL(server_callbacks_, onEvent(ConnectionEvent::RemoteClose)).Times(0);
   EXPECT_CALL(*read_filter_, onData(_, _)).Times(0);
@@ -995,7 +1019,8 @@ TEST_P(ConnectionImplTest, HalfCloseNoEarlyCloseDetection) {
   client_connection_->close(ConnectionCloseType::FlushWrite);
   dispatcher_->run(Event::Dispatcher::RunType::Block);
 
-  server_connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            server_connection_->readDisable(false));
   EXPECT_CALL(*read_filter_, onData(_, _)).WillOnce(InvokeWithoutArgs([&]() -> FilterStatus {
     dispatcher_->exit();
     return FilterStatus::StopIteration;
@@ -1130,7 +1155,8 @@ TEST_P(ConnectionImplTest, ReadWatermarks) {
   // This time when the buffer is drained, there will be no kick as the consumer
   // does not want to read.
   {
-    client_connection_->readDisable(true);
+    EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+              client_connection_->readDisable(true));
     testClientConnection()->readBuffer().drain(2);
     EXPECT_FALSE(testClientConnection()->readBuffer().highWatermarkTriggered());
     EXPECT_FALSE(testClientConnection()->shouldDrainReadBuffer());
@@ -1144,12 +1170,15 @@ TEST_P(ConnectionImplTest, ReadWatermarks) {
   // Inside the onData call, readDisable and readEnable. This should trigger
   // another kick on the next dispatcher loop, so onData gets called twice.
   {
-    client_connection_->readDisable(false);
+    EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+              client_connection_->readDisable(false));
     EXPECT_CALL(*client_read_filter, onData(_, false))
         .Times(2)
         .WillOnce(Invoke([&](Buffer::Instance&, bool) -> FilterStatus {
-          client_connection_->readDisable(true);
-          client_connection_->readDisable(false);
+          EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+                    client_connection_->readDisable(true));
+          EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+                    client_connection_->readDisable(false));
           return FilterStatus::StopIteration;
         }))
         .WillRepeatedly(Invoke(on_filter_data_exit));
@@ -1173,16 +1202,20 @@ TEST_P(ConnectionImplTest, ReadWatermarks) {
     EXPECT_FALSE(client_connection_->readEnabled());
 
     // Read disable and read enable, to set dispatch_buffered_data_ true.
-    client_connection_->readDisable(true);
-    client_connection_->readDisable(false);
+    EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+              client_connection_->readDisable(true));
+    EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+              client_connection_->readDisable(false));
     // Now event loop. This hits the early on-Read path. As above, read
     // disable and read enable from inside the stack of onData, to ensure that
     // dispatch_buffered_data_ works correctly.
     EXPECT_CALL(*client_read_filter, onData(_, false))
         .Times(2)
         .WillOnce(Invoke([&](Buffer::Instance&, bool) -> FilterStatus {
-          client_connection_->readDisable(true);
-          client_connection_->readDisable(false);
+          EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+                    client_connection_->readDisable(true));
+          EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+                    client_connection_->readDisable(false));
           return FilterStatus::StopIteration;
         }))
         .WillRepeatedly(Invoke(on_filter_data_exit));
@@ -2159,38 +2192,44 @@ TEST_F(MockTransportConnectionImplTest, ReadBufferReadyResumeAfterReadDisable) {
   // When processing a sequence of read disable/read enable, changes to the enabled event mask
   // happen only when the disable count transitions to/from 0.
   EXPECT_CALL(*file_event_, setEnabled(Event::FileReadyType::Write));
-  connection_->readDisable(true);
-  connection_->readDisable(true);
-  connection_->readDisable(true);
-  connection_->readDisable(false);
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(false));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(false));
   EXPECT_CALL(*file_event_, setEnabled(Event::FileReadyType::Read | Event::FileReadyType::Write));
   // Expect a read activation since there have been no transport doRead calls since the call to
   // setTransportSocketIsReadable.
   EXPECT_CALL(*file_event_, activate(Event::FileReadyType::Read));
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            connection_->readDisable(false));
 
   // No calls to doRead when file_ready_cb is invoked while read disabled.
   EXPECT_CALL(*file_event_, setEnabled(_));
-  connection_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            connection_->readDisable(true));
   EXPECT_CALL(*transport_socket_, doRead(_)).Times(0);
   file_ready_cb_(Event::FileReadyType::Read);
 
   // Expect a read activate when re-enabling since the file ready cb has not done a read.
   EXPECT_CALL(*file_event_, setEnabled(_));
   EXPECT_CALL(*file_event_, activate(Event::FileReadyType::Read));
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            connection_->readDisable(false));
 
   // Do a read to clear the transport_wants_read_ flag, verify that no read activation is scheduled.
   EXPECT_CALL(*transport_socket_, doRead(_))
       .WillOnce(Return(IoResult{PostIoAction::KeepOpen, 0, false}));
   file_ready_cb_(Event::FileReadyType::Read);
   EXPECT_CALL(*file_event_, setEnabled(_));
-  connection_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            connection_->readDisable(true));
   EXPECT_CALL(*file_event_, setEnabled(_));
   // No read activate call.
   EXPECT_CALL(*file_event_, activate(_)).Times(0);
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            connection_->readDisable(false));
 }
 
 // Verify that read resumption is scheduled when read is re-enabled while the read buffer is
@@ -2241,14 +2280,14 @@ TEST_F(MockTransportConnectionImplTest, ReadBufferResumeAfterReadDisable) {
 
   // Already read disabled, expect no changes to enabled events mask.
   EXPECT_CALL(*file_event_, setEnabled(_)).Times(0);
-  connection_->readDisable(true);
-  connection_->readDisable(true);
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(false));
   // Read buffer is at the high watermark so read_disable_count should be == 1. Expect a read
   // activate but no call to setEnable to change the registration mask.
   EXPECT_CALL(*file_event_, setEnabled(_)).Times(0);
   EXPECT_CALL(*file_event_, activate(Event::FileReadyType::Read));
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(false));
 
   // Invoke the file event cb while read_disable_count_ == 1 to partially drain the read buffer.
   // Expect no transport reads.
@@ -2279,11 +2318,13 @@ TEST_F(MockTransportConnectionImplTest, ReadBufferResumeAfterReadDisable) {
   EXPECT_FALSE(connection_->shouldDrainReadBuffer());
 
   EXPECT_CALL(*file_event_, setEnabled(_));
-  connection_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            connection_->readDisable(true));
   EXPECT_CALL(*file_event_, setEnabled(_));
   // read buffer is empty, no read activate call.
   EXPECT_CALL(*file_event_, activate(_)).Times(0);
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            connection_->readDisable(false));
 }
 
 // Verify that transport_wants_read_ read resumption is not lost when processing read buffer
@@ -2315,14 +2356,14 @@ TEST_F(MockTransportConnectionImplTest, ResumeWhileAndAfterReadDisable) {
 
   // Already read disabled, expect no changes to enabled events mask.
   EXPECT_CALL(*file_event_, setEnabled(_)).Times(0);
-  connection_->readDisable(true);
-  connection_->readDisable(true);
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(false));
   // Read buffer is at the high watermark so read_disable_count should be == 1. Expect a read
   // activate but no call to setEnable to change the registration mask.
   EXPECT_CALL(*file_event_, setEnabled(_)).Times(0);
   EXPECT_CALL(*file_event_, activate(Event::FileReadyType::Read));
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, connection_->readDisable(false));
 
   // Invoke the file event cb while read_disable_count_ == 1 and fully drain the read buffer.
   // Expect no transport reads. Expect a read resumption due to transport_wants_read_ being true
@@ -2347,10 +2388,12 @@ TEST_F(MockTransportConnectionImplTest, ResumeWhileAndAfterReadDisable) {
   // Verify there are no read activate calls the event callback does a transport read and clears the
   // transport_wants_read_ state.
   EXPECT_CALL(*file_event_, setEnabled(_));
-  connection_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            connection_->readDisable(true));
   EXPECT_CALL(*file_event_, setEnabled(_));
   EXPECT_CALL(*file_event_, activate(_)).Times(0);
-  connection_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            connection_->readDisable(false));
 }
 
 // Test that BytesSentCb is invoked at the correct times
diff --git a/test/common/network/multi_connection_base_impl_test.cc b/test/common/network/multi_connection_base_impl_test.cc
index 59be1155f469..a5cf8cccab50 100644
--- a/test/common/network/multi_connection_base_impl_test.cc
+++ b/test/common/network/multi_connection_base_impl_test.cc
@@ -966,17 +966,17 @@ TEST_F(MultiConnectionBaseImplTest, ReadDisable) {
   setupMultiConnectionImpl(3);
 
   // The disables will be captured by the impl and not passed to the connection until it completes.
-  impl_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled, impl_->readDisable(true));
 
   startConnect();
 
   timeOutAndStartNextAttempt();
 
   // The disables will be captured by the impl and not passed to the connection until it completes.
-  impl_->readDisable(true);
-  impl_->readDisable(true);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, impl_->readDisable(true));
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, impl_->readDisable(true));
   // Read disable count should now be 2.
-  impl_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled, impl_->readDisable(false));
 
   // readDisable() should be applied to the now final connection.
   EXPECT_CALL(*createdConnections()[1], readDisable(true)).Times(2);
@@ -984,23 +984,27 @@ TEST_F(MultiConnectionBaseImplTest, ReadDisable) {
 
   // Verify that addBytesSentCallback() calls are delegated to the remaining connection.
   EXPECT_CALL(*createdConnections()[1], readDisable(false));
-  impl_->readDisable(false);
+  EXPECT_EQ(Connection::ReadDisableStatus::NoTransition, impl_->readDisable(false));
 }
 
 TEST_F(MultiConnectionBaseImplTest, ReadEnabled) {
   setupMultiConnectionImpl(2);
 
   EXPECT_TRUE(impl_->readEnabled());
-  impl_->readDisable(true); // Disable count 1.
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadDisabled,
+            impl_->readDisable(true)); // Disable count 1.
   EXPECT_FALSE(impl_->readEnabled());
 
   startConnect();
 
-  impl_->readDisable(true); // Disable count 2
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+            impl_->readDisable(true)); // Disable count 2
   EXPECT_FALSE(impl_->readEnabled());
-  impl_->readDisable(false); // Disable count 1
+  EXPECT_EQ(Connection::ReadDisableStatus::StillReadDisabled,
+            impl_->readDisable(false)); // Disable count 1
   EXPECT_FALSE(impl_->readEnabled());
-  impl_->readDisable(false); // Disable count 0
+  EXPECT_EQ(Connection::ReadDisableStatus::TransitionedToReadEnabled,
+            impl_->readDisable(false)); // Disable count 0
   EXPECT_TRUE(impl_->readEnabled());
 }
 
diff --git a/test/integration/BUILD b/test/integration/BUILD
index 3f60a1f16ed9..c6eb768d6f25 100644
--- a/test/integration/BUILD
+++ b/test/integration/BUILD
@@ -580,6 +580,7 @@ envoy_cc_test(
         ":http_protocol_integration_lib",
         ":socket_interface_swap_lib",
         ":tracked_watermark_buffer_lib",
+        "//source/extensions/filters/network/tcp_proxy:config",
         "//test/integration/filters:tee_filter_lib",
         "//test/mocks/http:http_mocks",
         "//test/test_common:test_runtime_lib",
@@ -587,6 +588,7 @@ envoy_cc_test(
         "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto",
     ],
 )
 
diff --git a/test/integration/buffer_accounting_integration_test.cc b/test/integration/buffer_accounting_integration_test.cc
index f9fe4ff3b8f6..05bb8c1ae3ea 100644
--- a/test/integration/buffer_accounting_integration_test.cc
+++ b/test/integration/buffer_accounting_integration_test.cc
@@ -4,6 +4,7 @@
 #include "envoy/config/bootstrap/v3/bootstrap.pb.h"
 #include "envoy/config/cluster/v3/cluster.pb.h"
 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h"
+#include "envoy/extensions/filters/network/tcp_proxy/v3/tcp_proxy.pb.h"
 #include "envoy/network/address.h"
 
 #include "source/common/buffer/buffer_impl.h"
@@ -606,6 +607,71 @@ TEST_P(ProtocolsBufferWatermarksTest, ResettingStreamUnregistersAccount) {
   }
 }
 
+class TcpTunnelingWatermarkIntegrationTest : public Http2BufferWatermarksTest {
+public:
+  void SetUp() override {
+    enableHalfClose(true);
+
+    setDownstreamProtocol(std::get<0>(GetParam()).downstream_protocol);
+    setUpstreamProtocol(std::get<0>(GetParam()).upstream_protocol);
+
+    config_helper_.addConfigModifier(
+        [&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void {
+          envoy::extensions::filters::network::tcp_proxy::v3::TcpProxy proxy_config;
+          proxy_config.set_stat_prefix("tcp_stats");
+          proxy_config.set_cluster("cluster_0");
+          proxy_config.mutable_tunneling_config()->set_hostname("foo.lyft.com:80");
+
+          auto* listener = bootstrap.mutable_static_resources()->add_listeners();
+          listener->set_name("tcp_proxy");
+          auto* socket_address = listener->mutable_address()->mutable_socket_address();
+          socket_address->set_address(Network::Test::getLoopbackAddressString(version_));
+          socket_address->set_port_value(0);
+
+          auto* filter_chain = listener->add_filter_chains();
+          auto* filter = filter_chain->add_filters();
+          filter->mutable_typed_config()->PackFrom(proxy_config);
+          filter->set_name("envoy.filters.network.tcp_proxy");
+
+          RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() >= 1, "");
+          ConfigHelper::HttpProtocolOptions protocol_options;
+          auto* options =
+              protocol_options.mutable_explicit_http_config()->mutable_http2_protocol_options();
+          options->mutable_initial_stream_window_size()->set_value(65536);
+          ConfigHelper::setProtocolOptions(
+              *bootstrap.mutable_static_resources()->mutable_clusters(0), protocol_options);
+        });
+  }
+
+protected:
+  IntegrationTcpClientPtr tcp_client_;
+};
+
+INSTANTIATE_TEST_SUITE_P(
+    IpVersions, TcpTunnelingWatermarkIntegrationTest,
+    testing::Combine(testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams(
+                         {Http::CodecType::HTTP2}, {FakeHttpConnection::Type::HTTP2})),
+                     testing::Bool()),
+    protocolTestParamsAndBoolToString);
+
+TEST_P(TcpTunnelingWatermarkIntegrationTest, MultipleReadDisableCallsIncrementsStatsOnce) {
+  config_helper_.setBufferLimits(16384, 131072);
+  initialize();
+
+  write_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port());
+
+  tcp_client_ = makeTcpConnection(lookupPort("tcp_proxy"));
+  ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_));
+  ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_));
+  ASSERT_TRUE(upstream_request_->waitForHeadersComplete());
+  upstream_request_->encodeHeaders(default_response_headers_, false);
+
+  write_matcher_->setWriteReturnsEgain();
+  ASSERT_TRUE(tcp_client_->write(std::string(524288, 'a'), false));
+  test_server_->waitForCounterEq("tcp.tcp_stats.downstream_flow_control_paused_reading_total", 1);
+  tcp_client_->close();
+}
+
 class Http2OverloadManagerIntegrationTest : public Http2BufferWatermarksTest,
                                             public Envoy::BaseOverloadIntegrationTest {
 protected:
diff --git a/test/mocks/network/connection.h b/test/mocks/network/connection.h
index 99ecdd17a207..2bba420f051c 100644
--- a/test/mocks/network/connection.h
+++ b/test/mocks/network/connection.h
@@ -64,7 +64,7 @@ class MockConnectionBase {
   MOCK_METHOD(bool, initializeReadFilters, ());                                                    \
   MOCK_METHOD(std::string, nextProtocol, (), (const));                                             \
   MOCK_METHOD(void, noDelay, (bool enable));                                                       \
-  MOCK_METHOD(void, readDisable, (bool disable));                                                  \
+  MOCK_METHOD(ReadDisableStatus, readDisable, (bool disable));                                     \
   MOCK_METHOD(void, detectEarlyCloseWhenReadDisabled, (bool));                                     \
   MOCK_METHOD(bool, readEnabled, (), (const));                                                     \
   MOCK_METHOD(ConnectionInfoSetter&, connectionInfoSetter, ());                                    \

From 6ccd0a71caeece9f751a15ca72cd642366b9c868 Mon Sep 17 00:00:00 2001
From: Tianyu <72890320+tyxia@users.noreply.github.com>
Date: Thu, 29 Jun 2023 08:18:25 -0400
Subject: [PATCH 678/740] utility: handle the failure of dynamic_cast
 gracefully (#28145)

In getMergedPerFilterConfig(which is used by ext_authz, ext_proc), the failure of dynamic_cast should be handled gracefully. We should not crash (deference the nullptr) if resolving per route config failed (i.e., dynamic_cast failed) since it is on data plane path.

Signed-off-by: tyxia 
---
 source/common/http/utility.h     | 22 +++++++++++++---------
 test/common/http/utility_test.cc | 26 ++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/source/common/http/utility.h b/source/common/http/utility.h
index 87ec1cd02398..673d31b59212 100644
--- a/source/common/http/utility.h
+++ b/source/common/http/utility.h
@@ -659,15 +659,19 @@ getMergedPerFilterConfig(const Http::StreamFilterCallbacks* callbacks,
 
   absl::optional merged;
 
-  callbacks->traversePerFilterConfig(
-      [&reduce, &merged](const Router::RouteSpecificFilterConfig& cfg) {
-        const ConfigType* typed_cfg = dynamic_cast(&cfg);
-        if (!merged) {
-          merged.emplace(*typed_cfg);
-        } else {
-          reduce(merged.value(), *typed_cfg);
-        }
-      });
+  callbacks->traversePerFilterConfig([&reduce,
+                                      &merged](const Router::RouteSpecificFilterConfig& cfg) {
+    const ConfigType* typed_cfg = dynamic_cast(&cfg);
+    if (typed_cfg == nullptr) {
+      ENVOY_LOG_MISC(debug, "Failed to retrieve the correct type of route specific filter config");
+      return;
+    }
+    if (!merged) {
+      merged.emplace(*typed_cfg);
+    } else {
+      reduce(merged.value(), *typed_cfg);
+    }
+  });
 
   return merged;
 }
diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc
index 63beeea63e0e..4670fc32017f 100644
--- a/test/common/http/utility_test.cc
+++ b/test/common/http/utility_test.cc
@@ -1239,6 +1239,32 @@ TEST(HttpUtility, GetMergedPerFilterConfig) {
   EXPECT_EQ(2, merged_cfg.value().state_);
 }
 
+class BadConfig {
+public:
+  int state_;
+  void merge(const BadConfig& other) { state_ += other.state_; }
+};
+
+// Verify that merging result is empty as expected when the bad config is provided.
+TEST(HttpUtility, GetMergedPerFilterBadConfig) {
+  TestConfig testConfig;
+  NiceMock filter_callbacks;
+
+  EXPECT_CALL(*filter_callbacks.route_, traversePerFilterConfig(_, _))
+      .WillOnce(Invoke([&](const std::string&,
+                           std::function cb) {
+        cb(testConfig);
+      }));
+
+  EXPECT_LOG_CONTAINS(
+      "debug", "Failed to retrieve the correct type of route specific filter config",
+      auto merged_cfg = Utility::getMergedPerFilterConfig(
+          &filter_callbacks,
+          [&](BadConfig& base_cfg, const BadConfig& route_cfg) { base_cfg.merge(route_cfg); });
+      // Dynamic_cast failed, so merged_cfg is not set.
+      ASSERT_FALSE(merged_cfg.has_value()););
+}
+
 TEST(HttpUtility, CheckIsIpAddress) {
   std::array>, 15> patterns{
       std::make_tuple(true, "1.2.3.4", "1.2.3.4", absl::nullopt),

From 27b1082df679b6b7f95a4f524911cb8c5545d4fc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 29 Jun 2023 13:43:11 +0100
Subject: [PATCH 679/740] build(deps): bump node from `c92280d` to `1154591` in
 /examples/shared/node (#28182)

build(deps): bump node in /examples/shared/node

Bumps node from `c92280d` to `1154591`.

---
updated-dependencies:
- dependency-name: node
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/node/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile
index 8f530552c0f1..2fb07cbb6fa8 100644
--- a/examples/shared/node/Dockerfile
+++ b/examples/shared/node/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:20.3-bullseye-slim@sha256:c92280d8fb6e7ca07f258c45e9f18cb643ea798a5441855a05e982cfd2b90789 as node-base
+FROM node:20.3-bullseye-slim@sha256:115459129ed17d1c8c4a7911e7a3756c8e49b9d89e3eac48f34249578c9971ef as node-base
 
 
 FROM node-base as node-http-auth

From 8bd12cec755a3538d0280180035eb20cb5800096 Mon Sep 17 00:00:00 2001
From: Joshua Marantz 
Date: Thu, 29 Jun 2023 09:06:20 -0400
Subject: [PATCH 680/740] buffer: Disable flaky test on windows. (#28176)

Signed-off-by: Joshua Marantz 
---
 test/common/buffer/owned_impl_test.cc    | 4 ++++
 test/integration/base_integration_test.h | 9 +--------
 test/test_common/utility.h               | 8 ++++++++
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/test/common/buffer/owned_impl_test.cc b/test/common/buffer/owned_impl_test.cc
index 18884e3dc957..2f6c73df536a 100644
--- a/test/common/buffer/owned_impl_test.cc
+++ b/test/common/buffer/owned_impl_test.cc
@@ -1385,6 +1385,10 @@ TYPED_TEST(OwnedImplTypedTest, ReadReserveAndCommit) {
   std::string data = "e";
   const ssize_t rc = os_sys_calls.write(fds[1], data.data(), data.size()).return_value_;
   ASSERT_GT(rc, 0);
+
+  // The remainder of this test flakes under Windows.
+  // See https://github.com/envoyproxy/envoy/issues/28177.
+  DISABLE_UNDER_WINDOWS;
   Api::IoCallUint64Result result = io_handle.read(buf, read_length);
   ASSERT_EQ(result.return_value_, static_cast(rc));
   ASSERT_EQ(os_sys_calls.close(fds[1]).return_value_, 0);
diff --git a/test/integration/base_integration_test.h b/test/integration/base_integration_test.h
index 0f750483b4a6..d0e6c6572c99 100644
--- a/test/integration/base_integration_test.h
+++ b/test/integration/base_integration_test.h
@@ -22,6 +22,7 @@
 #include "test/mocks/server/transport_socket_factory_context.h"
 #include "test/test_common/environment.h"
 #include "test/test_common/test_time.h"
+#include "test/test_common/utility.h"
 
 #include "absl/strings/str_format.h"
 #include "absl/types/optional.h"
@@ -34,14 +35,6 @@
   } while (0)
 #endif
 
-#ifdef WIN32
-#define DISABLE_UNDER_WINDOWS return
-#else
-#define DISABLE_UNDER_WINDOWS                                                                      \
-  do {                                                                                             \
-  } while (0)
-#endif
-
 #ifndef ENVOY_ADMIN_FUNCTIONALITY
 #define DISABLE_IF_ADMIN_DISABLED return
 #else
diff --git a/test/test_common/utility.h b/test/test_common/utility.h
index e5cc0c5cfcd5..b4aed8a24080 100644
--- a/test/test_common/utility.h
+++ b/test/test_common/utility.h
@@ -1321,4 +1321,12 @@ MATCHER_P(JsonStringEq, expected, "") {
 }
 #endif
 
+#ifdef WIN32
+#define DISABLE_UNDER_WINDOWS return
+#else
+#define DISABLE_UNDER_WINDOWS                                                                      \
+  do {                                                                                             \
+  } while (0)
+#endif
+
 } // namespace Envoy

From 7e13226389f64d59b50770d6cdf93c6fc8ee8ba7 Mon Sep 17 00:00:00 2001
From: Kuo-Chung Hsu 
Date: Thu, 29 Jun 2023 06:47:50 -0700
Subject: [PATCH 681/740] Make json value getter no throw for data plane usage
 (#28178)

* Make json value getter no throw for data plane usage

Signed-off-by: kuochunghsu 
---
 envoy/json/json_object.h             |  4 ++-
 source/common/json/json_internal.cc  | 12 ++++-----
 test/common/json/BUILD               |  1 +
 test/common/json/json_loader_test.cc | 38 ++++++++++++++++++++--------
 4 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/envoy/json/json_object.h b/envoy/json/json_object.h
index 486954a1d3b8..ab4e7e463b36 100644
--- a/envoy/json/json_object.h
+++ b/envoy/json/json_object.h
@@ -9,6 +9,8 @@
 #include "envoy/common/exception.h"
 #include "envoy/common/pure.h"
 
+#include "source/common/common/statusor.h"
+
 #include "absl/types/variant.h"
 
 namespace Envoy {
@@ -49,7 +51,7 @@ class Object {
    * @param name supplies the key name.
    * @return bool the value.
    */
-  virtual ValueType getValue(const std::string& name) const PURE;
+  virtual absl::StatusOr getValue(const std::string& name) const PURE;
 
   /**
    * Get a boolean value by name.
diff --git a/source/common/json/json_internal.cc b/source/common/json/json_internal.cc
index ad775d2adf35..4706f30c4426 100644
--- a/source/common/json/json_internal.cc
+++ b/source/common/json/json_internal.cc
@@ -60,7 +60,7 @@ class Field : public Object {
 
   uint64_t hash() const override;
 
-  ValueType getValue(const std::string& name) const override;
+  absl::StatusOr getValue(const std::string& name) const override;
   bool getBoolean(const std::string& name) const override;
   bool getBoolean(const std::string& name, bool default_value) const override;
   double getDouble(const std::string& name) const override;
@@ -384,11 +384,11 @@ nlohmann::json Field::asJsonDocument() const {
 
 uint64_t Field::hash() const { return HashUtil::xxHash64(asJsonString()); }
 
-ValueType Field::getValue(const std::string& name) const {
+absl::StatusOr Field::getValue(const std::string& name) const {
   auto value_itr = value_.object_value_.find(name);
   if (value_itr == value_.object_value_.end()) {
-    throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_,
-                                line_number_end_));
+    return absl::NotFoundError(fmt::format("key '{}' missing from lines {}-{}", name,
+                                           line_number_start_, line_number_end_));
   }
   switch (value_itr->second->type_) {
   case Type::Boolean:
@@ -400,8 +400,8 @@ ValueType Field::getValue(const std::string& name) const {
   case Type::String:
     return value_itr->second->stringValue();
   default:
-    throw Exception(fmt::format("key '{}' not a value type from lines {}-{}", name,
-                                line_number_start_, line_number_end_));
+    return absl::InternalError(fmt::format("key '{}' not a value type from lines {}-{}", name,
+                                           line_number_start_, line_number_end_));
   }
 }
 
diff --git a/test/common/json/BUILD b/test/common/json/BUILD
index 03e59b250d41..614ffb0e93d2 100644
--- a/test/common/json/BUILD
+++ b/test/common/json/BUILD
@@ -36,6 +36,7 @@ envoy_cc_test(
     deps = [
         "//source/common/json:json_loader_lib",
         "//source/common/stats:isolated_store_lib",
+        "//test/test_common:status_utility_lib",
         "//test/test_common:utility_lib",
     ],
 )
diff --git a/test/common/json/json_loader_test.cc b/test/common/json/json_loader_test.cc
index 552c01916396..aff37e24b510 100644
--- a/test/common/json/json_loader_test.cc
+++ b/test/common/json/json_loader_test.cc
@@ -4,6 +4,7 @@
 #include "source/common/json/json_loader.h"
 #include "source/common/stats/isolated_store_impl.h"
 
+#include "test/test_common/status_utility.h"
 #include "test/test_common/utility.h"
 
 #include "gtest/gtest.h"
@@ -12,10 +13,25 @@ namespace Envoy {
 namespace Json {
 namespace {
 
+using ::Envoy::StatusHelpers::StatusIs;
+
 class JsonLoaderTest : public testing::Test {
 protected:
   JsonLoaderTest() : api_(Api::createApiForTest()) {}
 
+  ValueType getValidValue(const Json::Object& json, const std::string& key) {
+    auto result = json.getValue(key);
+    EXPECT_TRUE(result.ok());
+    return result.value();
+  }
+
+  void expectErrorValue(const Json::Object& json, const std::string& key,
+                        absl::StatusCode status_code, const std::string& message) {
+    auto result = json.getValue(key);
+    EXPECT_FALSE(result.ok());
+    EXPECT_THAT(result, StatusIs(status_code));
+    EXPECT_EQ(result.status().message(), message);
+  }
   Api::ApiPtr api_;
 };
 
@@ -49,8 +65,9 @@ TEST_F(JsonLoaderTest, Basic) {
     EXPECT_TRUE(json->getBoolean("hello", false));
     EXPECT_FALSE(json->getBoolean("world", false));
 
-    EXPECT_TRUE(absl::get(json->getValue("hello")));
-    EXPECT_THROW(json->getValue("world"), Exception);
+    EXPECT_TRUE(absl::get(getValidValue(*json, "hello")));
+    expectErrorValue(*json, "world", absl::StatusCode::kNotFound,
+                     "key 'world' missing from lines 1-1");
   }
 
   {
@@ -64,7 +81,7 @@ TEST_F(JsonLoaderTest, Basic) {
     EXPECT_EQ(123, json->getInteger("hello", 456));
     EXPECT_EQ(456, json->getInteger("world", 456));
 
-    EXPECT_EQ(123, absl::get(json->getValue("hello")));
+    EXPECT_EQ(123, absl::get(getValidValue(*json, "hello")));
   }
 
   {
@@ -96,15 +113,15 @@ TEST_F(JsonLoaderTest, Basic) {
     ObjectSharedPtr json =
         Factory::loadFromString("{\"1\":{\"11\":\"111\"},\"2\":{\"22\":\"222\"}}");
     int pos = 0;
-    json->iterate([&pos](const std::string& key, const Json::Object& value) {
+    json->iterate([this, &pos](const std::string& key, const Json::Object& value) {
       EXPECT_TRUE(key == "1" || key == "2");
 
       if (key == "1") {
         EXPECT_EQ("111", value.getString("11"));
-        EXPECT_EQ("111", absl::get(value.getValue("11")));
+        EXPECT_EQ("111", absl::get(getValidValue(value, "11")));
       } else {
         EXPECT_EQ("222", value.getString("22"));
-        EXPECT_EQ("222", absl::get(value.getValue("22")));
+        EXPECT_EQ("222", absl::get(getValidValue(value, "22")));
       }
 
       pos++;
@@ -161,7 +178,8 @@ TEST_F(JsonLoaderTest, Basic) {
     EXPECT_THROW(array[0]->asObjectArray(), Exception);
 
     // Object Array is not supported as an value.
-    EXPECT_THROW(config->getValue("descriptors"), Exception);
+    expectErrorValue(*config, "descriptors", absl::StatusCode::kInternal,
+                     "key 'descriptors' not a value type from lines 2-4");
   }
 
   {
@@ -279,15 +297,15 @@ TEST_F(JsonLoaderTest, Double) {
     EXPECT_EQ(10.5, json->getDouble("value1"));
     EXPECT_EQ(-12.3, json->getDouble("value2"));
 
-    EXPECT_EQ(10.5, absl::get(json->getValue("value1")));
-    EXPECT_EQ(-12.3, absl::get(json->getValue("value2")));
+    EXPECT_EQ(10.5, absl::get(getValidValue(*json, "value1")));
+    EXPECT_EQ(-12.3, absl::get(getValidValue(*json, "value2")));
   }
   {
     ObjectSharedPtr json = Factory::loadFromString("{\"foo\": 13.22}");
     EXPECT_EQ(13.22, json->getDouble("foo", 0));
     EXPECT_EQ(0, json->getDouble("bar", 0));
 
-    EXPECT_EQ(13.22, absl::get(json->getValue("foo")));
+    EXPECT_EQ(13.22, absl::get(getValidValue(*json, "foo")));
   }
   {
     ObjectSharedPtr json = Factory::loadFromString("{\"foo\": \"bar\"}");

From d96fd3622b30e629a00fac4b2445210ed3173720 Mon Sep 17 00:00:00 2001
From: "Adi (Suissa) Peleg" 
Date: Thu, 29 Jun 2023 10:03:20 -0400
Subject: [PATCH 682/740] sub-formatter: reject NUL and CR characters in
 formatted header keys (#28172)

* sub-formatter: reject NUL and CR characters in formatted header keys

Signed-off-by: Adi Suissa-Peleg 
---
 source/common/formatter/substitution_formatter.cc          | 7 +++----
 test/common/formatter/substitution_formatter_test.cc       | 4 ++++
 .../header_parser_corpus/sub_formatter_header_newline      | 6 ++++++
 3 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100644 test/common/router/header_parser_corpus/sub_formatter_header_newline

diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc
index 6d2d50cdf922..700fe242c965 100644
--- a/source/common/formatter/substitution_formatter.cc
+++ b/source/common/formatter/substitution_formatter.cc
@@ -56,7 +56,6 @@ void truncate(std::string& str, absl::optional max_length) {
 const std::regex& getSystemTimeFormatNewlinePattern() {
   CONSTRUCT_ON_FIRST_USE(std::regex, "%[-_0^#]*[1-9]*(E|O)?n");
 }
-const std::regex& getNewlinePattern() { CONSTRUCT_ON_FIRST_USE(std::regex, "\n"); }
 
 } // namespace
 
@@ -348,9 +347,9 @@ void SubstitutionFormatParser::parseSubcommandHeaders(const std::string& subcomm
   }
 
   // The main and alternative header should not contain invalid characters {NUL, LR, CF}.
-  if (std::regex_search(main_header, getNewlinePattern()) ||
-      std::regex_search(alternative_header, getNewlinePattern())) {
-    throw EnvoyException("Invalid header configuration. Format string contains newline.");
+  if (!Envoy::Http::validHeaderString(main_header) ||
+      !Envoy::Http::validHeaderString(alternative_header)) {
+    throw EnvoyException("Invalid header configuration. Format string contains null or newline.");
   }
 }
 
diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc
index 94ce4f716bce..8aa40485189f 100644
--- a/test/common/formatter/substitution_formatter_test.cc
+++ b/test/common/formatter/substitution_formatter_test.cc
@@ -4460,6 +4460,10 @@ TEST(SubstitutionFormatterTest, ParserFailures) {
       "%REQ(TEST):-3%",
       "%REQ(\n)%",
       "%REQ(?\n)%",
+      "%REQ(\0)%",
+      "%REQ(?\0)%",
+      "%REQ(\r)%",
+      "%REQ(?\r)%",
       "%RESP(TEST):%",
       "%RESP(X?Y):%",
       "%RESP(X?Y):343o24%",
diff --git a/test/common/router/header_parser_corpus/sub_formatter_header_newline b/test/common/router/header_parser_corpus/sub_formatter_header_newline
new file mode 100644
index 000000000000..7e59cd1f2893
--- /dev/null
+++ b/test/common/router/header_parser_corpus/sub_formatter_header_newline
@@ -0,0 +1,6 @@
+headers_to_add {
+  header {
+    key: "_"
+    value: "%UPSTREAM_METADATA([\"~nTE( )%%REQ( \\ream_rq_retry_limit_exceededT_TIME(%)%\"])%"
+  }
+}

From c4d918647c6746193a598a2c1c570929f5e7dff1 Mon Sep 17 00:00:00 2001
From: Kuat 
Date: Thu, 29 Jun 2023 07:25:22 -0700
Subject: [PATCH 683/740] cel: use singleton builder in rate limit (#28080)

Commit Message: Follow up to https://github.com/envoyproxy/envoy/pull/27861, for CEL rate limit descriptor.
Additional Description: Requires signature change for descriptor factory:
```c++
   virtual DescriptorProducerPtr
   createDescriptorProducerFromProto(const Protobuf::Message& config,
-                                    ProtobufMessage::ValidationVisitor& validator) PURE;
+                                    Server::Configuration::CommonFactoryContext& context) PURE;
```
Risk Level: low
Testing: yes
Docs Changes: none
Release Notes: none


Signed-off-by: Kuat Yessenov 
---
 envoy/ratelimit/BUILD                         |  1 +
 envoy/ratelimit/ratelimit.h                   |  5 ++--
 envoy/server/BUILD                            |  1 -
 source/common/router/config_impl.cc           |  4 +--
 source/common/router/router_ratelimit.cc      |  9 ++++---
 source/common/router/router_ratelimit.h       |  4 +--
 .../rate_limit_descriptors/expr/config.cc     | 27 +++++++++----------
 .../rate_limit_descriptors/expr/config.h      |  6 +----
 test/common/router/router_ratelimit_test.cc   | 10 ++++---
 .../rate_limit_descriptors/expr/BUILD         |  1 +
 .../expr/config_test.cc                       |  5 ++--
 test/per_file_coverage.sh                     |  4 +--
 12 files changed, 38 insertions(+), 39 deletions(-)

diff --git a/envoy/ratelimit/BUILD b/envoy/ratelimit/BUILD
index 42e2cd2b6b23..a9f2344752a4 100644
--- a/envoy/ratelimit/BUILD
+++ b/envoy/ratelimit/BUILD
@@ -15,6 +15,7 @@ envoy_cc_library(
         "//envoy/config:typed_config_interface",
         "//envoy/http:header_map_interface",
         "//envoy/protobuf:message_validator_interface",
+        "//envoy/server:factory_context_interface",
         "//envoy/stream_info:stream_info_interface",
         "@envoy_api//envoy/type/v3:pkg_cc_proto",
     ],
diff --git a/envoy/ratelimit/ratelimit.h b/envoy/ratelimit/ratelimit.h
index 00646eda753e..2839cfa86c65 100644
--- a/envoy/ratelimit/ratelimit.h
+++ b/envoy/ratelimit/ratelimit.h
@@ -6,6 +6,7 @@
 #include "envoy/config/typed_config.h"
 #include "envoy/http/header_map.h"
 #include "envoy/protobuf/message_validator.h"
+#include "envoy/server/factory_context.h"
 #include "envoy/stream_info/stream_info.h"
 #include "envoy/type/v3/ratelimit_unit.pb.h"
 
@@ -102,13 +103,13 @@ class DescriptorProducerFactory : public Config::TypedFactory {
    * Creates a particular DescriptorProducer implementation.
    *
    * @param config supplies the configuration for the descriptor extension.
-   * @param validator configuration validation visitor.
+   * @param context supplies the factory context.
    * @return DescriptorProducerPtr the rate limit descriptor producer which will be used to
    * populate rate limit descriptors.
    */
   virtual DescriptorProducerPtr
   createDescriptorProducerFromProto(const Protobuf::Message& config,
-                                    ProtobufMessage::ValidationVisitor& validator) PURE;
+                                    Server::Configuration::CommonFactoryContext& context) PURE;
 
   std::string category() const override { return "envoy.rate_limit_descriptors"; }
 };
diff --git a/envoy/server/BUILD b/envoy/server/BUILD
index 7649e4a728fb..6361b6958a9f 100644
--- a/envoy/server/BUILD
+++ b/envoy/server/BUILD
@@ -43,7 +43,6 @@ envoy_cc_library(
     external_deps = ["abseil_optional"],
     deps = [
         "//envoy/http:context_interface",
-        "//envoy/ratelimit:ratelimit_interface",
         "//envoy/upstream:cluster_manager_interface",
         "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
     ],
diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc
index 891af09a8e37..6205ebe2ce33 100644
--- a/source/common/router/config_impl.cc
+++ b/source/common/router/config_impl.cc
@@ -626,7 +626,7 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost,
 
   if (!route.route().rate_limits().empty()) {
     rate_limit_policy_ =
-        std::make_unique(route.route().rate_limits(), validator);
+        std::make_unique(route.route().rate_limits(), factory_context);
   }
 
   // Returns true if include_vh_rate_limits is explicitly set to true otherwise it defaults to false
@@ -1686,7 +1686,7 @@ CommonVirtualHostImpl::CommonVirtualHostImpl(
 
   if (!virtual_host.rate_limits().empty()) {
     rate_limit_policy_ =
-        std::make_unique(virtual_host.rate_limits(), validator);
+        std::make_unique(virtual_host.rate_limits(), factory_context);
   }
 
   shadow_policies_.reserve(virtual_host.request_mirror_policies().size());
diff --git a/source/common/router/router_ratelimit.cc b/source/common/router/router_ratelimit.cc
index 24af81716d68..0d75d283fe9d 100644
--- a/source/common/router/router_ratelimit.cc
+++ b/source/common/router/router_ratelimit.cc
@@ -280,7 +280,7 @@ QueryParameterValueMatchAction::buildQueryParameterMatcherVector(
 
 RateLimitPolicyEntryImpl::RateLimitPolicyEntryImpl(
     const envoy::config::route::v3::RateLimit& config,
-    ProtobufMessage::ValidationVisitor& validator)
+    Server::Configuration::CommonFactoryContext& context)
     : disable_key_(config.disable_key()),
       stage_(static_cast(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, stage, 0))) {
   for (const auto& action : config.actions()) {
@@ -310,6 +310,7 @@ RateLimitPolicyEntryImpl::RateLimitPolicyEntryImpl(
       actions_.emplace_back(new HeaderValueMatchAction(action.header_value_match()));
       break;
     case envoy::config::route::v3::RateLimit::Action::ActionSpecifierCase::kExtension: {
+      ProtobufMessage::ValidationVisitor& validator = context.messageValidationVisitor();
       auto* factory = Envoy::Config::Utility::getFactory(
           action.extension());
       if (!factory) {
@@ -329,7 +330,7 @@ RateLimitPolicyEntryImpl::RateLimitPolicyEntryImpl(
       auto message = Envoy::Config::Utility::translateAnyToFactoryConfig(
           action.extension().typed_config(), validator, *factory);
       RateLimit::DescriptorProducerPtr producer =
-          factory->createDescriptorProducerFromProto(*message, validator);
+          factory->createDescriptorProducerFromProto(*message, context);
       if (producer) {
         actions_.emplace_back(std::move(producer));
       } else {
@@ -397,11 +398,11 @@ RateLimitPolicyImpl::RateLimitPolicyImpl()
 
 RateLimitPolicyImpl::RateLimitPolicyImpl(
     const Protobuf::RepeatedPtrField& rate_limits,
-    ProtobufMessage::ValidationVisitor& validator)
+    Server::Configuration::CommonFactoryContext& context)
     : RateLimitPolicyImpl() {
   for (const auto& rate_limit : rate_limits) {
     std::unique_ptr rate_limit_policy_entry(
-        new RateLimitPolicyEntryImpl(rate_limit, validator));
+        new RateLimitPolicyEntryImpl(rate_limit, context));
     uint64_t stage = rate_limit_policy_entry->stage();
     ASSERT(stage < rate_limit_entries_reference_.size());
     rate_limit_entries_reference_[stage].emplace_back(*rate_limit_policy_entry);
diff --git a/source/common/router/router_ratelimit.h b/source/common/router/router_ratelimit.h
index c4a1da54e84c..9f790270be28 100644
--- a/source/common/router/router_ratelimit.h
+++ b/source/common/router/router_ratelimit.h
@@ -211,7 +211,7 @@ class QueryParameterValueMatchAction : public RateLimit::DescriptorProducer {
 class RateLimitPolicyEntryImpl : public RateLimitPolicyEntry {
 public:
   RateLimitPolicyEntryImpl(const envoy::config::route::v3::RateLimit& config,
-                           ProtobufMessage::ValidationVisitor& validator);
+                           Server::Configuration::CommonFactoryContext& context);
 
   // Router::RateLimitPolicyEntry
   uint64_t stage() const override { return stage_; }
@@ -239,7 +239,7 @@ class RateLimitPolicyImpl : public RateLimitPolicy {
   RateLimitPolicyImpl();
   RateLimitPolicyImpl(
       const Protobuf::RepeatedPtrField& rate_limits,
-      ProtobufMessage::ValidationVisitor& validator);
+      Server::Configuration::CommonFactoryContext& context);
 
   // Router::RateLimitPolicy
   const std::vector>&
diff --git a/source/extensions/rate_limit_descriptors/expr/config.cc b/source/extensions/rate_limit_descriptors/expr/config.cc
index e780574ec813..a50dec82f0f2 100644
--- a/source/extensions/rate_limit_descriptors/expr/config.cc
+++ b/source/extensions/rate_limit_descriptors/expr/config.cc
@@ -23,10 +23,12 @@ class ExpressionDescriptor : public RateLimit::DescriptorProducer {
 public:
   ExpressionDescriptor(
       const envoy::extensions::rate_limit_descriptors::expr::v3::Descriptor& config,
-      Filters::Common::Expr::Builder& builder, const google::api::expr::v1alpha1::Expr& input_expr)
-      : input_expr_(input_expr), descriptor_key_(config.descriptor_key()),
+      Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr& builder,
+      const google::api::expr::v1alpha1::Expr& input_expr)
+      : builder_(builder), input_expr_(input_expr), descriptor_key_(config.descriptor_key()),
         skip_if_error_(config.skip_if_error()) {
-    compiled_expr_ = Extensions::Filters::Common::Expr::createExpression(builder, input_expr_);
+    compiled_expr_ =
+        Extensions::Filters::Common::Expr::createExpression(builder_->builder(), input_expr_);
   }
 
   // Ratelimit::DescriptorProducer
@@ -47,6 +49,7 @@ class ExpressionDescriptor : public RateLimit::DescriptorProducer {
   }
 
 private:
+  Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr builder_;
   const google::api::expr::v1alpha1::Expr input_expr_;
   const std::string descriptor_key_;
   const bool skip_if_error_;
@@ -62,9 +65,11 @@ ProtobufTypes::MessagePtr ExprDescriptorFactory::createEmptyConfigProto() {
 }
 
 RateLimit::DescriptorProducerPtr ExprDescriptorFactory::createDescriptorProducerFromProto(
-    const Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validator) {
+    const Protobuf::Message& message, Server::Configuration::CommonFactoryContext& context) {
   const auto& config = MessageUtil::downcastAndValidate<
-      const envoy::extensions::rate_limit_descriptors::expr::v3::Descriptor&>(message, validator);
+      const envoy::extensions::rate_limit_descriptors::expr::v3::Descriptor&>(
+      message, context.messageValidationVisitor());
+  auto builder = Extensions::Filters::Common::Expr::getBuilder(context);
   switch (config.expr_specifier_case()) {
 #if defined(USE_CEL_PARSER)
   case envoy::extensions::rate_limit_descriptors::expr::v3::Descriptor::kText: {
@@ -73,24 +78,16 @@ RateLimit::DescriptorProducerPtr ExprDescriptorFactory::createDescriptorProducer
       throw EnvoyException("Unable to parse descriptor expression: " +
                            parse_status.status().ToString());
     }
-    return std::make_unique(config, getOrCreateBuilder(),
-                                                  parse_status.value().expr());
+    return std::make_unique(config, builder, parse_status.value().expr());
   }
 #endif
   case envoy::extensions::rate_limit_descriptors::expr::v3::Descriptor::kParsed:
-    return std::make_unique(config, getOrCreateBuilder(), config.parsed());
+    return std::make_unique(config, builder, config.parsed());
   default:
     return nullptr;
   }
 }
 
-Filters::Common::Expr::Builder& ExprDescriptorFactory::getOrCreateBuilder() {
-  if (expr_builder_ == nullptr) {
-    expr_builder_ = Filters::Common::Expr::createBuilder(nullptr);
-  }
-  return *expr_builder_;
-}
-
 REGISTER_FACTORY(ExprDescriptorFactory, RateLimit::DescriptorProducerFactory);
 
 } // namespace Expr
diff --git a/source/extensions/rate_limit_descriptors/expr/config.h b/source/extensions/rate_limit_descriptors/expr/config.h
index da50da1ab14b..400ce6c4c1ad 100644
--- a/source/extensions/rate_limit_descriptors/expr/config.h
+++ b/source/extensions/rate_limit_descriptors/expr/config.h
@@ -20,11 +20,7 @@ class ExprDescriptorFactory : public RateLimit::DescriptorProducerFactory {
   ProtobufTypes::MessagePtr createEmptyConfigProto() override;
   RateLimit::DescriptorProducerPtr
   createDescriptorProducerFromProto(const Protobuf::Message& message,
-                                    ProtobufMessage::ValidationVisitor& validator) override;
-
-private:
-  Filters::Common::Expr::Builder& getOrCreateBuilder();
-  Filters::Common::Expr::BuilderPtr expr_builder_;
+                                    Server::Configuration::CommonFactoryContext& context) override;
 };
 
 } // namespace Expr
diff --git a/test/common/router/router_ratelimit_test.cc b/test/common/router/router_ratelimit_test.cc
index eec6b1ef105e..236b674966b7 100644
--- a/test/common/router/router_ratelimit_test.cc
+++ b/test/common/router/router_ratelimit_test.cc
@@ -292,14 +292,15 @@ TEST_F(RateLimitConfiguration, Stages) {
 class RateLimitPolicyEntryTest : public testing::Test {
 public:
   void setupTest(const std::string& yaml) {
-    rate_limit_entry_ = std::make_unique(
-        parseRateLimitFromV3Yaml(yaml), ProtobufMessage::getStrictValidationVisitor());
+    rate_limit_entry_ = std::make_unique(parseRateLimitFromV3Yaml(yaml),
+                                                                   factory_context_);
     descriptors_.clear();
     local_descriptors_.clear();
     stream_info_.downstream_connection_info_provider_->setRemoteAddress(default_remote_address_);
     ON_CALL(Const(stream_info_), route()).WillByDefault(testing::Return(route_));
   }
 
+  NiceMock factory_context_;
   std::unique_ptr rate_limit_entry_;
   Http::TestRequestHeaderMapImpl header_;
   std::shared_ptr route_{new NiceMock()};
@@ -313,14 +314,15 @@ class RateLimitPolicyEntryTest : public testing::Test {
 class RateLimitPolicyEntryIpv6Test : public testing::Test {
 public:
   void setupTest(const std::string& yaml) {
-    rate_limit_entry_ = std::make_unique(
-        parseRateLimitFromV3Yaml(yaml), ProtobufMessage::getStrictValidationVisitor());
+    rate_limit_entry_ = std::make_unique(parseRateLimitFromV3Yaml(yaml),
+                                                                   factory_context_);
     descriptors_.clear();
     local_descriptors_.clear();
     stream_info_.downstream_connection_info_provider_->setRemoteAddress(default_remote_address_);
     ON_CALL(Const(stream_info_), route()).WillByDefault(testing::Return(route_));
   }
 
+  NiceMock factory_context_;
   std::unique_ptr rate_limit_entry_;
   Http::TestRequestHeaderMapImpl header_;
   std::shared_ptr route_{new NiceMock()};
diff --git a/test/extensions/rate_limit_descriptors/expr/BUILD b/test/extensions/rate_limit_descriptors/expr/BUILD
index 39851c79c20c..2107c282c88b 100644
--- a/test/extensions/rate_limit_descriptors/expr/BUILD
+++ b/test/extensions/rate_limit_descriptors/expr/BUILD
@@ -27,6 +27,7 @@ envoy_extension_cc_test(
         "//source/extensions/rate_limit_descriptors/expr:config",
         "//test/mocks/http:http_mocks",
         "//test/mocks/ratelimit:ratelimit_mocks",
+        "//test/mocks/server:server_mocks",
         "//test/test_common:utility_lib",
         "@envoy_api//envoy/config/route/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/rate_limit_descriptors/expr/v3:pkg_cc_proto",
diff --git a/test/extensions/rate_limit_descriptors/expr/config_test.cc b/test/extensions/rate_limit_descriptors/expr/config_test.cc
index b90bb357a127..fcfd6c3cb24c 100644
--- a/test/extensions/rate_limit_descriptors/expr/config_test.cc
+++ b/test/extensions/rate_limit_descriptors/expr/config_test.cc
@@ -9,6 +9,7 @@
 
 #include "test/mocks/http/mocks.h"
 #include "test/mocks/ratelimit/mocks.h"
+#include "test/mocks/server/factory_context.h"
 #include "test/test_common/utility.h"
 
 #include "gmock/gmock.h"
@@ -28,10 +29,10 @@ class RateLimitPolicyEntryTest : public testing::Test {
     envoy::config::route::v3::RateLimit rate_limit;
     TestUtility::loadFromYaml(yaml, rate_limit);
     TestUtility::validate(rate_limit);
-    rate_limit_entry_ = std::make_unique(
-        rate_limit, ProtobufMessage::getStrictValidationVisitor());
+    rate_limit_entry_ = std::make_unique(rate_limit, context_);
   }
 
+  NiceMock context_;
   std::unique_ptr rate_limit_entry_;
   Http::TestRequestHeaderMapImpl header_;
   std::vector descriptors_;
diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh
index da38e06f8c44..67441cc3c1c4 100755
--- a/test/per_file_coverage.sh
+++ b/test/per_file_coverage.sh
@@ -49,8 +49,8 @@ declare -a KNOWN_LOW_COVERAGE=(
 "source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata:96.3"
 "source/extensions/filters/network/wasm:76.9"
 "source/extensions/http/cache/simple_http_cache:95.9"
-"source/extensions/rate_limit_descriptors:95.5"
-"source/extensions/rate_limit_descriptors/expr:95.5"
+"source/extensions/rate_limit_descriptors:95.0"
+"source/extensions/rate_limit_descriptors/expr:95.0"
 "source/extensions/stat_sinks/graphite_statsd:82.1"
 "source/extensions/stat_sinks/statsd:84.6"
 "source/extensions/tracers:95.8"

From bb8aa93c53c808065ba4207224d1a301d9713439 Mon Sep 17 00:00:00 2001
From: Kevin Baichoo 
Date: Thu, 29 Jun 2023 17:43:07 -0400
Subject: [PATCH 684/740] Fix flake in
 ListenerMaxConnectionPerSocketEventTest.AcceptsConnectionsUpToTheMaximumPerSocketEvent
 (#28192)

Signed-off-by: Kevin Baichoo 
---
 ...ected_resource_monitor_integration_test.cc | 32 ++++++++++---------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
index d25a03a00dc8..88cbd4b8a5fd 100644
--- a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
+++ b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc
@@ -184,8 +184,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ListenerMaxConnectionPerSocketEventTest,
                          testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
                          TestUtility::ipTestParamsToString);
 
-TEST_P(ListenerMaxConnectionPerSocketEventTest,
-       DISABLED_AcceptsConnectionsUpToTheMaximumPerSocketEvent) {
+TEST_P(ListenerMaxConnectionPerSocketEventTest, AcceptsConnectionsUpToTheMaximumPerSocketEvent) {
   auto set_max_connections_per_socket_event_to_two =
       [](envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
         for (auto& listener_config : *bootstrap.mutable_static_resources()->mutable_listeners()) {
@@ -219,23 +218,26 @@ TEST_P(ListenerMaxConnectionPerSocketEventTest,
 
     // As we are using level trigger for listeners, all new connections get recognized.
     test_server_->waitForGaugeEq(downstream_cx_active, 10);
+
+    // Wait for the histogram to be updated as that occurs after the logs we are
+    // expecting.
+    const std::string connections_accepted_per_socket_event =
+        (version_ == Network::Address::IpVersion::v4)
+            ? "listener.127.0.0.1_0.connections_accepted_per_socket_event"
+            : "listener.[__1]_0.connections_accepted_per_socket_event";
+    test_server_->waitUntilHistogramHasSamples(connections_accepted_per_socket_event);
+    auto connections_accepted_histogram =
+        test_server_->histogram(connections_accepted_per_socket_event);
+    EXPECT_EQ(TestUtility::readSampleCount(test_server_->server().dispatcher(),
+                                           *connections_accepted_histogram),
+              5);
+    EXPECT_EQ(static_cast(TestUtility::readSampleSum(test_server_->server().dispatcher(),
+                                                          *connections_accepted_histogram)),
+              10);
   });
 
   std::for_each(client_codecs.begin(), client_codecs.end(),
                 [](IntegrationCodecClientPtr& client_codec) { client_codec->close(); });
-  const std::string connections_accepted_per_socket_event =
-      (version_ == Network::Address::IpVersion::v4)
-          ? "listener.127.0.0.1_0.connections_accepted_per_socket_event"
-          : "listener.[__1]_0.connections_accepted_per_socket_event";
-  test_server_->waitUntilHistogramHasSamples(connections_accepted_per_socket_event);
-  auto connections_accepted_histogram =
-      test_server_->histogram(connections_accepted_per_socket_event);
-  EXPECT_EQ(TestUtility::readSampleCount(test_server_->server().dispatcher(),
-                                         *connections_accepted_histogram),
-            5);
-  EXPECT_EQ(static_cast(TestUtility::readSampleSum(test_server_->server().dispatcher(),
-                                                        *connections_accepted_histogram)),
-            10);
 }
 
 } // namespace Envoy

From 0966609c24afdf36efe7aaa7dad0e92407085ecf Mon Sep 17 00:00:00 2001
From: StarryNight 
Date: Fri, 30 Jun 2023 10:50:42 +0800
Subject: [PATCH 685/740] go get metadata api (#28057)

* Golang filter: Add suport for get dynamic metadata

Follow up from #26622 is the other side of the api to get metadta from a go filter.

Fixes #25371

Signed-off-by: Sotiris Nanopoulos 

* add mock for fuzz test

Signed-off-by: Sotiris Nanopoulos 

* add envoyGoRequestSemaDec to simple.so

Signed-off-by: Sotiris Nanopoulos 

* fix clang

Signed-off-by: Sotiris Nanopoulos 

* fix clang tidy vol 2

Signed-off-by: Sotiris Nanopoulos 

* fix review comments

Signed-off-by: wangkai19 

* fix review comments

Signed-off-by: wangkai19 

* lock sema

Signed-off-by: wangkai19 

---------

Signed-off-by: Sotiris Nanopoulos 
Signed-off-by: wangkai19 
Co-authored-by: Sotiris Nanopoulos 
---
 contrib/golang/common/go/api/api.h            |  2 +-
 contrib/golang/common/go/api/capi.go          |  2 +-
 contrib/golang/common/go/api/filter.go        |  2 +-
 contrib/golang/filters/http/source/cgo.cc     |  7 +++
 .../http/source/go/pkg/http/capi_impl.go      | 22 +++++++++
 .../filters/http/source/go/pkg/http/filter.go |  4 ++
 .../filters/http/source/golang_filter.cc      | 46 +++++++++++++++++++
 .../filters/http/source/golang_filter.h       |  5 ++
 .../http/test/test_data/basic/filter.go       | 17 +++++++
 9 files changed, 104 insertions(+), 3 deletions(-)

diff --git a/contrib/golang/common/go/api/api.h b/contrib/golang/common/go/api/api.h
index 85857ef06dc4..481e86475809 100644
--- a/contrib/golang/common/go/api/api.h
+++ b/contrib/golang/common/go/api/api.h
@@ -62,7 +62,7 @@ CAPIStatus envoyGoFilterHttpRemoveTrailer(void* r, void* key);
 CAPIStatus envoyGoFilterHttpGetStringValue(void* r, int id, void* value);
 CAPIStatus envoyGoFilterHttpGetIntegerValue(void* r, int id, void* value);
 
-// TODO: implement get dynamic metadata
+CAPIStatus envoyGoFilterHttpGetDynamicMetadata(void* r, void* name, void* hand);
 CAPIStatus envoyGoFilterHttpSetDynamicMetadata(void* r, void* name, void* key, void* buf);
 
 void envoyGoFilterHttpLog(uint32_t level, void* message);
diff --git a/contrib/golang/common/go/api/capi.go b/contrib/golang/common/go/api/capi.go
index eb9c6db5ab21..a482aa88524c 100644
--- a/contrib/golang/common/go/api/capi.go
+++ b/contrib/golang/common/go/api/capi.go
@@ -42,7 +42,7 @@ type HttpCAPI interface {
 	HttpGetStringValue(r unsafe.Pointer, id int) (string, bool)
 	HttpGetIntegerValue(r unsafe.Pointer, id int) (uint64, bool)
 
-	// TODO: HttpGetDynamicMetadata(r unsafe.Pointer, filterName string) map[string]interface{}
+	HttpGetDynamicMetadata(r unsafe.Pointer, filterName string) map[string]interface{}
 	HttpSetDynamicMetadata(r unsafe.Pointer, filterName string, key string, value interface{})
 
 	HttpLog(level LogType, message string)
diff --git a/contrib/golang/common/go/api/filter.go b/contrib/golang/common/go/api/filter.go
index 850227730654..99c17e09fa31 100644
--- a/contrib/golang/common/go/api/filter.go
+++ b/contrib/golang/common/go/api/filter.go
@@ -99,7 +99,7 @@ type FilterCallbackHandler interface {
 }
 
 type DynamicMetadata interface {
-	// TODO: Get(filterName string) map[string]interface{}
+	Get(filterName string) map[string]interface{}
 	Set(filterName string, key string, value interface{})
 }
 
diff --git a/contrib/golang/filters/http/source/cgo.cc b/contrib/golang/filters/http/source/cgo.cc
index 4a593dbe2016..c0d7bc8616ad 100644
--- a/contrib/golang/filters/http/source/cgo.cc
+++ b/contrib/golang/filters/http/source/cgo.cc
@@ -175,6 +175,13 @@ void envoyGoFilterHttpLog(uint32_t level, void* message) {
   getFilterLogger().log(level, mesg);
 }
 
+CAPIStatus envoyGoFilterHttpGetDynamicMetadata(void* r, void* name, void* buf) {
+  return envoyGoFilterHandlerWrapper(r, [name, buf](std::shared_ptr& filter) -> CAPIStatus {
+    auto name_str = copyGoString(name);
+    auto buf_slice = reinterpret_cast(buf);
+    return filter->getDynamicMetadata(name_str, buf_slice);
+  });
+}
 uint32_t envoyGoFilterHttpLogLevel() { return getFilterLogger().level(); }
 
 CAPIStatus envoyGoFilterHttpSetDynamicMetadata(void* r, void* name, void* key, void* buf) {
diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
index 4b9d58592671..673fd9e73399 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go
@@ -243,6 +243,26 @@ func (c *httpCApiImpl) HttpGetIntegerValue(r unsafe.Pointer, id int) (uint64, bo
 	return value, true
 }
 
+func (c *httpCApiImpl) HttpGetDynamicMetadata(rr unsafe.Pointer, filterName string) map[string]interface{} {
+	r := (*httpRequest)(rr)
+	var buf []byte
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	r.sema.Add(1)
+	res := C.envoyGoFilterHttpGetDynamicMetadata(unsafe.Pointer(r.req), unsafe.Pointer(&filterName), unsafe.Pointer(&buf))
+	if res == C.CAPIYield {
+		atomic.AddInt32(&r.waitingOnEnvoy, 1)
+		r.sema.Wait()
+	} else {
+		r.sema.Done()
+		handleCApiStatus(res)
+	}
+	// copy the memory from c to Go.
+	var meta structpb.Struct
+	proto.Unmarshal(buf, &meta)
+	return meta.AsMap()
+}
+
 func (c *httpCApiImpl) HttpSetDynamicMetadata(r unsafe.Pointer, filterName string, key string, value interface{}) {
 	v, err := structpb.NewValue(value)
 	if err != nil {
@@ -283,6 +303,8 @@ func (c *httpCApiImpl) HttpSetStringFilterState(r unsafe.Pointer, key string, va
 func (c *httpCApiImpl) HttpGetStringFilterState(rr unsafe.Pointer, key string) string {
 	r := (*httpRequest)(rr)
 	var value string
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
 	r.sema.Add(1)
 	res := C.envoyGoFilterHttpGetStringFilterState(unsafe.Pointer(r.req), unsafe.Pointer(&key), unsafe.Pointer(&value))
 	if res == C.CAPIYield {
diff --git a/contrib/golang/filters/http/source/go/pkg/http/filter.go b/contrib/golang/filters/http/source/go/pkg/http/filter.go
index afb8d26fa751..a348d3f014a8 100644
--- a/contrib/golang/filters/http/source/go/pkg/http/filter.go
+++ b/contrib/golang/filters/http/source/go/pkg/http/filter.go
@@ -183,6 +183,10 @@ func (s *streamInfo) DynamicMetadata() api.DynamicMetadata {
 	}
 }
 
+func (d *dynamicMetadata) Get(filterName string) map[string]interface{} {
+	return cAPI.HttpGetDynamicMetadata(unsafe.Pointer(d.request), filterName)
+}
+
 func (d *dynamicMetadata) Set(filterName string, key string, value interface{}) {
 	cAPI.HttpSetDynamicMetadata(unsafe.Pointer(d.request.req), filterName, key, value)
 }
diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc
index a3fda0e9d4fe..0cbef587f65f 100644
--- a/contrib/golang/filters/http/source/golang_filter.cc
+++ b/contrib/golang/filters/http/source/golang_filter.cc
@@ -1018,6 +1018,52 @@ CAPIStatus Filter::getStringValue(int id, GoString* value_str) {
   return CAPIStatus::CAPIOK;
 }
 
+CAPIStatus Filter::getDynamicMetadata(const std::string& filter_name, GoSlice* buf_slice) {
+  Thread::LockGuard lock(mutex_);
+  if (has_destroyed_) {
+    ENVOY_LOG(debug, "golang filter has been destroyed");
+    return CAPIStatus::CAPIFilterIsDestroy;
+  }
+
+  auto& state = getProcessorState();
+  if (!state.isProcessingInGo()) {
+    ENVOY_LOG(debug, "golang filter is not processing Go");
+    return CAPIStatus::CAPINotInGo;
+  }
+
+  if (!state.isThreadSafe()) {
+    auto weak_ptr = weak_from_this();
+    ENVOY_LOG(debug, "golang filter getDynamicMetadata posting request to dispatcher");
+    state.getDispatcher().post([this, &state, weak_ptr, filter_name, buf_slice] {
+      ENVOY_LOG(debug, "golang filter getDynamicMetadata request in worker thread");
+      if (!weak_ptr.expired() && !hasDestroyed()) {
+        populateSliceWithMetadata(state, filter_name, buf_slice);
+        dynamic_lib_->envoyGoRequestSemaDec(req_);
+      } else {
+        ENVOY_LOG(info, "golang filter has gone or destroyed in getDynamicMetadata");
+      }
+    });
+    return CAPIStatus::CAPIYield;
+  } else {
+    ENVOY_LOG(debug, "golang filter getDynamicMetadata replying directly");
+    populateSliceWithMetadata(state, filter_name, buf_slice);
+  }
+
+  return CAPIStatus::CAPIOK;
+}
+
+void Filter::populateSliceWithMetadata(ProcessorState& state, const std::string& filter_name,
+                                       GoSlice* buf_slice) {
+  const auto& metadata = state.streamInfo().dynamicMetadata().filter_metadata();
+  const auto filter_it = metadata.find(filter_name);
+  if (filter_it != metadata.end()) {
+    filter_it->second.SerializeToString(&req_->strValue);
+    buf_slice->data = req_->strValue.data();
+    buf_slice->len = req_->strValue.length();
+    buf_slice->cap = req_->strValue.length();
+  }
+}
+
 CAPIStatus Filter::setDynamicMetadata(std::string filter_name, std::string key,
                                       absl::string_view buf) {
   // lock until this function return since it may running in a Go thread.
diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h
index f959a60f08c5..cc122f39b714 100644
--- a/contrib/golang/filters/http/source/golang_filter.h
+++ b/contrib/golang/filters/http/source/golang_filter.h
@@ -181,6 +181,8 @@ class Filter : public Http::StreamFilter,
   CAPIStatus removeTrailer(absl::string_view key);
   CAPIStatus getStringValue(int id, GoString* value_str);
   CAPIStatus getIntegerValue(int id, uint64_t* value);
+
+  CAPIStatus getDynamicMetadata(const std::string& filter_name, GoSlice* buf_slice);
   CAPIStatus setDynamicMetadata(std::string filter_name, std::string key, absl::string_view buf);
   CAPIStatus setStringFilterState(absl::string_view key, absl::string_view value, int state_type,
                                   int life_span, int stream_sharing);
@@ -216,6 +218,9 @@ class Filter : public Http::StreamFilter,
   void setDynamicMetadataInternal(ProcessorState& state, std::string filter_name, std::string key,
                                   const absl::string_view& buf);
 
+  void populateSliceWithMetadata(ProcessorState& state, const std::string& filter_name,
+                                 GoSlice* buf_slice);
+
   const FilterConfigSharedPtr config_;
   Dso::HttpFilterDsoPtr dynamic_lib_;
 
diff --git a/contrib/golang/filters/http/test/test_data/basic/filter.go b/contrib/golang/filters/http/test/test_data/basic/filter.go
index 4774078ebf67..8984bc3597eb 100644
--- a/contrib/golang/filters/http/test/test_data/basic/filter.go
+++ b/contrib/golang/filters/http/test/test_data/basic/filter.go
@@ -108,7 +108,24 @@ func (f *filter) decodeHeaders(header api.RequestHeaderMap, endStream bool) api.
 	_, found := header.Get("x-set-metadata")
 	if found {
 		md := f.callbacks.StreamInfo().DynamicMetadata()
+		empty_metadata := md.Get("filter.go")
+		if len(empty_metadata) != 0 {
+			return f.fail("Metadata should be empty")
+		}
 		md.Set("filter.go", "foo", "bar")
+		metadata := md.Get("filter.go")
+		if len(metadata) == 0 {
+			return f.fail("Metadata should not be empty")
+		}
+
+		k, ok := metadata["foo"]
+		if !ok {
+			return f.fail("Metadata foo should be found")
+		}
+
+		if fmt.Sprint(k) != "bar" {
+			return f.fail("Metadata foo has unexpected value %v", k)
+		}
 	}
 
 	fs := f.callbacks.StreamInfo().FilterState()

From 59beb4a1c23c8467508114b3a2bb34e82af18d70 Mon Sep 17 00:00:00 2001
From: code 
Date: Fri, 30 Jun 2023 11:50:24 +0800
Subject: [PATCH 686/740] upstream: only recreate worker local lb for thread
 aware lb (#28132)

Signed-off-by: wbpcode 
---
 envoy/upstream/load_balancer.h                               | 5 +++++
 source/common/upstream/cluster_manager_impl.cc               | 2 +-
 .../extensions/load_balancing_policies/common/factory_base.h | 2 ++
 source/extensions/load_balancing_policies/subset/config.cc   | 1 +
 .../extensions/load_balancing_policies/subset/subset_lb.cc   | 2 +-
 5 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/envoy/upstream/load_balancer.h b/envoy/upstream/load_balancer.h
index 8c1bf0c42a6a..0cf04779bb13 100644
--- a/envoy/upstream/load_balancer.h
+++ b/envoy/upstream/load_balancer.h
@@ -178,6 +178,11 @@ class LoadBalancerFactory {
    * @return LoadBalancerPtr a new worker local load balancer.
    */
   virtual LoadBalancerPtr create(LoadBalancerParams params) PURE;
+
+  /**
+   * @return bool whether the load balancer should be recreated when the host set changes.
+   */
+  virtual bool recreateOnHostChange() const { return true; }
 };
 
 using LoadBalancerFactorySharedPtr = std::shared_ptr;
diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc
index 12dbd970cd20..d6a3c1c9e429 100644
--- a/source/common/upstream/cluster_manager_impl.cc
+++ b/source/common/upstream/cluster_manager_impl.cc
@@ -1225,7 +1225,7 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::updateHost
                             hosts_added, hosts_removed, weighted_priority_health,
                             overprovisioning_factor, std::move(cross_priority_host_map));
   // If an LB is thread aware, create a new worker local LB on membership changes.
-  if (lb_factory_ != nullptr) {
+  if (lb_factory_ != nullptr && lb_factory_->recreateOnHostChange()) {
     ENVOY_LOG(debug, "re-creating local LB for TLS cluster {}", name);
     lb_ = lb_factory_->create({priority_set_, parent_.local_priority_set_});
   }
diff --git a/source/extensions/load_balancing_policies/common/factory_base.h b/source/extensions/load_balancing_policies/common/factory_base.h
index 04797d5ab953..c200d600e506 100644
--- a/source/extensions/load_balancing_policies/common/factory_base.h
+++ b/source/extensions/load_balancing_policies/common/factory_base.h
@@ -47,6 +47,8 @@ class FactoryBase : public Upstream::TypedLoadBalancerFactoryBase {
                     time_source_);
     }
 
+    bool recreateOnHostChange() const override { return false; }
+
   public:
     OptRef proto_config_;
     const Upstream::ClusterInfo& cluster_info_;
diff --git a/source/extensions/load_balancing_policies/subset/config.cc b/source/extensions/load_balancing_policies/subset/config.cc
index f7e6facc2d78..c370e3f24e58 100644
--- a/source/extensions/load_balancing_policies/subset/config.cc
+++ b/source/extensions/load_balancing_policies/subset/config.cc
@@ -116,6 +116,7 @@ class LbFactory : public Upstream::LoadBalancerFactory {
         params.local_priority_set, cluster_info_.lbStats(), cluster_info_.statsScope(), runtime_,
         random_, time_source_);
   }
+  bool recreateOnHostChange() const override { return false; }
 
 private:
   const SubsetLoadBalancerConfig& subset_config_;
diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.cc b/source/extensions/load_balancing_policies/subset/subset_lb.cc
index ead745e73020..8d901df87385 100644
--- a/source/extensions/load_balancing_policies/subset/subset_lb.cc
+++ b/source/extensions/load_balancing_policies/subset/subset_lb.cc
@@ -903,7 +903,7 @@ void SubsetLoadBalancer::PrioritySubsetImpl::update(uint32_t priority,
   // Create a new worker local LB if needed.
   // TODO(mattklein123): See the PrioritySubsetImpl constructor for additional comments on how
   // we can do better here.
-  if (thread_aware_lb_ != nullptr) {
+  if (thread_aware_lb_ != nullptr && thread_aware_lb_->factory()->recreateOnHostChange()) {
     lb_ = thread_aware_lb_->factory()->create({*this, original_local_priority_set_});
   }
 }

From 3b5169386da47fd75acd5427b7c79a39e42979a2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 30 Jun 2023 11:08:33 +0100
Subject: [PATCH 687/740] build(deps): bump openpolicyagent/opa from
 0.53.1-istio to 0.54.0-istio in /examples/ext_authz (#28199)

build(deps): bump openpolicyagent/opa in /examples/ext_authz

Bumps openpolicyagent/opa from 0.53.1-istio to 0.54.0-istio.

---
updated-dependencies:
- dependency-name: openpolicyagent/opa
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/ext_authz/Dockerfile-opa | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/ext_authz/Dockerfile-opa b/examples/ext_authz/Dockerfile-opa
index f3ed85f6e594..2180789b0083 100644
--- a/examples/ext_authz/Dockerfile-opa
+++ b/examples/ext_authz/Dockerfile-opa
@@ -1 +1 @@
-FROM openpolicyagent/opa:0.53.1-istio@sha256:b4821d33fdbca4fe3d2ff85e3f4a18d292f2fb32fe7db995b9239d7242702bab
+FROM openpolicyagent/opa:0.54.0-istio@sha256:02186a95654106d1bda57e8e676ce5453c2dca8dfd9ba67a59d5098f42bb471e

From 701aa7858e78b3bcad7f1fb1ac5ea880603c92a2 Mon Sep 17 00:00:00 2001
From: phlax 
Date: Fri, 30 Jun 2023 14:05:52 +0100
Subject: [PATCH 688/740] examples/ci: Fix compose by adding explicit context
 (#28186)

Signed-off-by: Ryan Northey 
---
 examples/brotli/docker-compose.yaml                  | 1 +
 examples/cache/docker-compose.yaml                   | 1 +
 examples/cors/backend/docker-compose.yaml            | 1 +
 examples/cors/frontend/docker-compose.yaml           | 1 +
 examples/csrf/crosssite/docker-compose.yml           | 1 +
 examples/csrf/samesite/docker-compose.yml            | 1 +
 examples/double-proxy/docker-compose.yaml            | 3 +++
 examples/dynamic-config-cp/docker-compose.yaml       | 2 ++
 examples/dynamic-config-fs/docker-compose.yaml       | 1 +
 examples/ext_authz/docker-compose.yaml               | 2 ++
 examples/fault-injection/docker-compose.yaml         | 3 +++
 examples/front-proxy/docker-compose.yaml             | 1 +
 examples/golang/docker-compose.yaml                  | 1 +
 examples/grpc-bridge/docker-compose-protos.yaml      | 2 ++
 examples/grpc-bridge/docker-compose.yaml             | 2 ++
 examples/gzip/docker-compose.yaml                    | 1 +
 examples/jaeger-native-tracing/docker-compose.yaml   | 2 ++
 examples/jaeger-tracing/docker-compose.yaml          | 2 ++
 examples/kafka/docker-compose.yaml                   | 4 ++++
 examples/load-reporting-service/docker-compose.yaml  | 2 ++
 examples/local_ratelimit/docker-compose.yaml         | 3 +++
 examples/locality-load-balancing/docker-compose.yaml | 1 +
 examples/lua/docker-compose.yaml                     | 1 +
 examples/mysql/docker-compose.yaml                   | 2 ++
 examples/opentelemetry/docker-compose.yaml           | 4 ++++
 examples/postgres/docker-compose.yaml                | 1 +
 examples/rbac/docker-compose.yaml                    | 1 +
 examples/redis/docker-compose.yaml                   | 2 ++
 examples/route-mirror/docker-compose.yaml            | 1 +
 examples/skywalking/docker-compose.yaml              | 6 ++++++
 examples/tls-inspector/docker-compose.yaml           | 4 ++++
 examples/tls-sni/docker-compose.yaml                 | 5 +++++
 examples/tls/docker-compose.yaml                     | 6 ++++++
 examples/udp/docker-compose.yaml                     | 2 ++
 examples/wasm-cc/docker-compose.yaml                 | 1 +
 examples/websocket/docker-compose.yaml               | 6 ++++++
 examples/zipkin/docker-compose.yaml                  | 4 ++++
 examples/zstd/docker-compose.yaml                    | 1 +
 38 files changed, 85 insertions(+)

diff --git a/examples/brotli/docker-compose.yaml b/examples/brotli/docker-compose.yaml
index 93a8c4624d89..a6241e2c18ed 100644
--- a/examples/brotli/docker-compose.yaml
+++ b/examples/brotli/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   envoy-stats:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     depends_on:
       service:
diff --git a/examples/cache/docker-compose.yaml b/examples/cache/docker-compose.yaml
index 9337a7830ae6..674c73d193b3 100644
--- a/examples/cache/docker-compose.yaml
+++ b/examples/cache/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   front-envoy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     depends_on:
       service1:
diff --git a/examples/cors/backend/docker-compose.yaml b/examples/cors/backend/docker-compose.yaml
index b689a6c652cd..262f09fa74d0 100644
--- a/examples/cors/backend/docker-compose.yaml
+++ b/examples/cors/backend/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   front-envoy:
     build:
+      context: .
       dockerfile: ../../shared/envoy/Dockerfile
     depends_on:
       backend-service:
diff --git a/examples/cors/frontend/docker-compose.yaml b/examples/cors/frontend/docker-compose.yaml
index 893534930791..6546e87369ee 100644
--- a/examples/cors/frontend/docker-compose.yaml
+++ b/examples/cors/frontend/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   front-envoy:
     build:
+      context: .
       dockerfile: ../../shared/envoy/Dockerfile
     depends_on:
       frontend-service:
diff --git a/examples/csrf/crosssite/docker-compose.yml b/examples/csrf/crosssite/docker-compose.yml
index af9981f52a13..96a5586e76a8 100644
--- a/examples/csrf/crosssite/docker-compose.yml
+++ b/examples/csrf/crosssite/docker-compose.yml
@@ -2,6 +2,7 @@ services:
 
   front-envoy:
     build:
+      context: .
       dockerfile: ../../shared/envoy/Dockerfile
     depends_on:
       service:
diff --git a/examples/csrf/samesite/docker-compose.yml b/examples/csrf/samesite/docker-compose.yml
index fb00aed10f12..feabf37afc92 100644
--- a/examples/csrf/samesite/docker-compose.yml
+++ b/examples/csrf/samesite/docker-compose.yml
@@ -2,6 +2,7 @@ services:
 
   front-envoy:
     build:
+      context: .
       dockerfile: ../../shared/envoy/Dockerfile
     depends_on:
       service:
diff --git a/examples/double-proxy/docker-compose.yaml b/examples/double-proxy/docker-compose.yaml
index e90c2e20d214..23e3a7c0a482 100644
--- a/examples/double-proxy/docker-compose.yaml
+++ b/examples/double-proxy/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy-frontend:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     networks:
       edge:
@@ -26,6 +27,7 @@ services:
 
   proxy-postgres-frontend:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-double-proxy-frontend
       args:
@@ -41,6 +43,7 @@ services:
 
   proxy-postgres-backend:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-double-proxy-backend
       args:
diff --git a/examples/dynamic-config-cp/docker-compose.yaml b/examples/dynamic-config-cp/docker-compose.yaml
index 5640800b0bf5..b83883f42b21 100644
--- a/examples/dynamic-config-cp/docker-compose.yaml
+++ b/examples/dynamic-config-cp/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     depends_on:
     - service1
@@ -22,6 +23,7 @@ services:
 
   go-control-plane:
     build:
+      context: .
       dockerfile: ../shared/golang/Dockerfile
       target: golang-control-plane
     command: /usr/local/bin/example
diff --git a/examples/dynamic-config-fs/docker-compose.yaml b/examples/dynamic-config-fs/docker-compose.yaml
index d239f8c2f710..9734368cbf37 100644
--- a/examples/dynamic-config-fs/docker-compose.yaml
+++ b/examples/dynamic-config-fs/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-dynamic-fs
     depends_on:
diff --git a/examples/ext_authz/docker-compose.yaml b/examples/ext_authz/docker-compose.yaml
index 53272e89d680..3bf74828b5b8 100644
--- a/examples/ext_authz/docker-compose.yaml
+++ b/examples/ext_authz/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   front-envoy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-ext_authz
       args:
@@ -34,6 +35,7 @@ services:
 
   ext_authz-opa-service:
     build:
+      context: .
       dockerfile: Dockerfile-opa
     volumes:
     - ./config/opa-service/policy.rego:/etc/policy.rego
diff --git a/examples/fault-injection/docker-compose.yaml b/examples/fault-injection/docker-compose.yaml
index d56011a867f7..f31ad11fcf6d 100644
--- a/examples/fault-injection/docker-compose.yaml
+++ b/examples/fault-injection/docker-compose.yaml
@@ -1,14 +1,17 @@
 services:
   envoy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-fault-injection
     volumes:
     - ./runtime:/srv/runtime
     ports:
     - 9211:9211
+
   backend:
     build:
+      context: .
       dockerfile: Dockerfile-backend
     ports:
     - ${PORT_PROXY:-8080}:80
diff --git a/examples/front-proxy/docker-compose.yaml b/examples/front-proxy/docker-compose.yaml
index 7c8bb3ec0a9e..faa4dd2843cd 100644
--- a/examples/front-proxy/docker-compose.yaml
+++ b/examples/front-proxy/docker-compose.yaml
@@ -3,6 +3,7 @@ services:
   # front-proxy
   front-envoy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     depends_on:
       service1:
diff --git a/examples/golang/docker-compose.yaml b/examples/golang/docker-compose.yaml
index d340f57e48f8..2b05743c1cd0 100644
--- a/examples/golang/docker-compose.yaml
+++ b/examples/golang/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-go
       args:
diff --git a/examples/grpc-bridge/docker-compose-protos.yaml b/examples/grpc-bridge/docker-compose-protos.yaml
index 339ed4e21430..ee12b691c5d8 100644
--- a/examples/grpc-bridge/docker-compose-protos.yaml
+++ b/examples/grpc-bridge/docker-compose-protos.yaml
@@ -3,6 +3,7 @@ services:
   # $ docker run -ti -v $(pwd):/protos -v $(pwd)/stubs:/stubs grpc/go protoc --go_out=plugins=grpc:/stubs -I/protos /protos/kv.proto
   stubs_go:
     build:
+      context: .
       dockerfile: Dockerfile-grpc-go
     command: protoc --go_out=plugins=grpc:/stubs -I/protos /protos/kv.proto
     volumes:
@@ -12,6 +13,7 @@ services:
   # $ docker run -ti -v $(pwd):/protos -v $(pwd)/stubs:/stubs grpc/python python -m grpc.tools.protoc --python_out=/stubs --grpc_python_out=/stubs -I/protos /protos/kv.proto
   stubs_python:
     build:
+      context: .
       dockerfile: Dockerfile-grpc-python
     command: python -m grpc.tools.protoc --python_out=/stubs --grpc_python_out=/stubs -I/protos /protos/kv.proto
     volumes:
diff --git a/examples/grpc-bridge/docker-compose.yaml b/examples/grpc-bridge/docker-compose.yaml
index 9740d95fd471..75f27fdee359 100644
--- a/examples/grpc-bridge/docker-compose.yaml
+++ b/examples/grpc-bridge/docker-compose.yaml
@@ -9,6 +9,7 @@ services:
 
   grpc-server-proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: ./server/envoy-proxy.yaml
@@ -27,6 +28,7 @@ services:
 
   grpc-client-proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: ./client/envoy-proxy.yaml
diff --git a/examples/gzip/docker-compose.yaml b/examples/gzip/docker-compose.yaml
index 93a8c4624d89..a6241e2c18ed 100644
--- a/examples/gzip/docker-compose.yaml
+++ b/examples/gzip/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   envoy-stats:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     depends_on:
       service:
diff --git a/examples/jaeger-native-tracing/docker-compose.yaml b/examples/jaeger-native-tracing/docker-compose.yaml
index b317573cff30..4279e0339ed7 100644
--- a/examples/jaeger-native-tracing/docker-compose.yaml
+++ b/examples/jaeger-native-tracing/docker-compose.yaml
@@ -3,6 +3,7 @@ services:
   # jaeger
   front-envoy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-jaeger-native
     depends_on:
@@ -35,6 +36,7 @@ services:
 
   jaeger:
     build:
+      context: .
       dockerfile: ../shared/jaeger/Dockerfile
     environment:
     - COLLECTOR_ZIPKIN_HOST_PORT=9411
diff --git a/examples/jaeger-tracing/docker-compose.yaml b/examples/jaeger-tracing/docker-compose.yaml
index 6bfacbff5aba..64e461fb8bb5 100644
--- a/examples/jaeger-tracing/docker-compose.yaml
+++ b/examples/jaeger-tracing/docker-compose.yaml
@@ -3,6 +3,7 @@ services:
   # jaeger
   front-envoy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     depends_on:
       service1:
@@ -34,6 +35,7 @@ services:
 
   jaeger:
     build:
+      context: .
       dockerfile: ../shared/jaeger/Dockerfile
     environment:
     - COLLECTOR_ZIPKIN_HOST_PORT=9411
diff --git a/examples/kafka/docker-compose.yaml b/examples/kafka/docker-compose.yaml
index a3410defafc1..c07ead7cd307 100644
--- a/examples/kafka/docker-compose.yaml
+++ b/examples/kafka/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   kafka-client:
     build:
+      context: .
       dockerfile: Dockerfile-kafka
     restart: "no"
     deploy:
@@ -9,6 +10,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_VARIANT: contrib-dev
@@ -18,6 +20,7 @@ services:
 
   kafka-server:
     build:
+      context: .
       dockerfile: Dockerfile-kafka
     depends_on:
       zookeeper:
@@ -38,6 +41,7 @@ services:
 
   zookeeper:
     build:
+      context: .
       dockerfile: Dockerfile-zookeeper
     healthcheck:
       test: ["CMD", "sh", "-c", "echo ruok | nc 127.0.0.1 2181 || exit -1"]
diff --git a/examples/load-reporting-service/docker-compose.yaml b/examples/load-reporting-service/docker-compose.yaml
index d11e4d8f7686..b18d2c0264e6 100644
--- a/examples/load-reporting-service/docker-compose.yaml
+++ b/examples/load-reporting-service/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   envoy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     ports:
     - "${PORT_PROXY0:-80}-${PORT_PROXY1:-81}:80"
@@ -19,6 +20,7 @@ services:
 
   lrs_server:
     build:
+      context: .
       dockerfile: ../shared/golang/Dockerfile
       target: golang-lrs
     volumes:
diff --git a/examples/local_ratelimit/docker-compose.yaml b/examples/local_ratelimit/docker-compose.yaml
index e2718912b328..307dc5f1ebbf 100644
--- a/examples/local_ratelimit/docker-compose.yaml
+++ b/examples/local_ratelimit/docker-compose.yaml
@@ -1,11 +1,14 @@
 services:
   envoy-stat:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     ports:
     - "${PORT_PROXY:-10000}:10000"
     - "${PORT_STATS0:-9901}:9901"
     - "${PORT_STATS1:-9902}:9902"
+
   service:
     build:
+      context: .
       dockerfile: Dockerfile-nginx
diff --git a/examples/locality-load-balancing/docker-compose.yaml b/examples/locality-load-balancing/docker-compose.yaml
index d3a063d5c296..7366425bc8e5 100644
--- a/examples/locality-load-balancing/docker-compose.yaml
+++ b/examples/locality-load-balancing/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   client-envoy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-load-balancing
     depends_on:
diff --git a/examples/lua/docker-compose.yaml b/examples/lua/docker-compose.yaml
index 0cbc5f07fa96..b503ac37193b 100644
--- a/examples/lua/docker-compose.yaml
+++ b/examples/lua/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-lua
     ports:
diff --git a/examples/mysql/docker-compose.yaml b/examples/mysql/docker-compose.yaml
index 85cdee2d9c4a..891358dd3127 100644
--- a/examples/mysql/docker-compose.yaml
+++ b/examples/mysql/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_VARIANT: contrib-dev
@@ -13,6 +14,7 @@ services:
 
   mysql:
     build:
+      context: .
       dockerfile: Dockerfile-mysql
     environment:
     - MYSQL_ALLOW_EMPTY_PASSWORD=yes
diff --git a/examples/opentelemetry/docker-compose.yaml b/examples/opentelemetry/docker-compose.yaml
index 3a6dac21c247..153da43ee9ba 100644
--- a/examples/opentelemetry/docker-compose.yaml
+++ b/examples/opentelemetry/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   envoy-front-proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: envoy-front-proxy.yaml
@@ -17,6 +18,7 @@ services:
 
   envoy-1:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: envoy-1.yaml
@@ -30,6 +32,7 @@ services:
 
   envoy-2:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: envoy-2.yaml
@@ -55,6 +58,7 @@ services:
 
   opentelemetry:
     build:
+      context: .
       dockerfile: Dockerfile-opentelemetry
     healthcheck:
       test: ["CMD-SHELL", "curl -sf http://localhost:13133 || exit 1"]
diff --git a/examples/postgres/docker-compose.yaml b/examples/postgres/docker-compose.yaml
index 86b5273f7f26..5a9f026c0eba 100644
--- a/examples/postgres/docker-compose.yaml
+++ b/examples/postgres/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_VARIANT: contrib-dev
diff --git a/examples/rbac/docker-compose.yaml b/examples/rbac/docker-compose.yaml
index 39a8fa33e7c1..6b51b21c40f9 100644
--- a/examples/rbac/docker-compose.yaml
+++ b/examples/rbac/docker-compose.yaml
@@ -1,6 +1,7 @@
 services:
   envoy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     ports:
     - "${PORT_PROXY:-10000}:10000"
diff --git a/examples/redis/docker-compose.yaml b/examples/redis/docker-compose.yaml
index be5acc666477..b1559e6c82bb 100644
--- a/examples/redis/docker-compose.yaml
+++ b/examples/redis/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     ports:
     - "${PORT_PROXY:-1999}:1999"
@@ -9,4 +10,5 @@ services:
 
   redis:
     build:
+      context: .
       dockerfile: Dockerfile-redis
diff --git a/examples/route-mirror/docker-compose.yaml b/examples/route-mirror/docker-compose.yaml
index 056a2dac2950..a002e4d2ec08 100644
--- a/examples/route-mirror/docker-compose.yaml
+++ b/examples/route-mirror/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   envoy-front-proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     ports:
     - "${PORT_PROXY:-10000}:10000"
diff --git a/examples/skywalking/docker-compose.yaml b/examples/skywalking/docker-compose.yaml
index 9581eecf3022..535171928459 100644
--- a/examples/skywalking/docker-compose.yaml
+++ b/examples/skywalking/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   envoy-front-proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: envoy-front-proxy.yaml
@@ -17,6 +18,7 @@ services:
 
   envoy-1:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: envoy-1.yaml
@@ -30,6 +32,7 @@ services:
 
   envoy-2:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: envoy-2.yaml
@@ -56,6 +59,7 @@ services:
   # Skywalking components.
   elasticsearch:
     build:
+      context: .
       dockerfile: Dockerfile-elasticsearch
     healthcheck:
       test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
@@ -83,6 +87,7 @@ services:
 
   skywalking-oap:
     build:
+      context: .
       dockerfile: Dockerfile-skywalking-oap
     depends_on:
       elasticsearch:
@@ -101,6 +106,7 @@ services:
 
   skywalking-ui:
     build:
+      context: .
       dockerfile: Dockerfile-skywalking-ui
     healthcheck:
       test: ["CMD-SHELL", 'curl --silent --fail http://localhost:8080/graphql -X POST -H "Content-Type:application/json" -d "{ \"query\": \"query version { version }\"}" || exit 1']
diff --git a/examples/tls-inspector/docker-compose.yaml b/examples/tls-inspector/docker-compose.yaml
index 2e7fa95ed63d..162f846321bb 100644
--- a/examples/tls-inspector/docker-compose.yaml
+++ b/examples/tls-inspector/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   tls-inspector:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     depends_on:
       service-https-http2:
@@ -16,6 +17,7 @@ services:
 
   service-https-http2:
     build:
+      context: .
       dockerfile: ../shared/echo2/Dockerfile
     hostname: service-https-http2
     environment:
@@ -23,6 +25,7 @@ services:
 
   service-https-http1.1:
     build:
+      context: .
       dockerfile: ../shared/echo2/Dockerfile
     hostname: service-https-http1.1
     environment:
@@ -30,6 +33,7 @@ services:
 
   service-http:
     build:
+      context: .
       dockerfile: ../shared/echo2/Dockerfile
     hostname: service-http
     environment:
diff --git a/examples/tls-sni/docker-compose.yaml b/examples/tls-sni/docker-compose.yaml
index 977c3d357361..25fdfc74e9a9 100644
--- a/examples/tls-sni/docker-compose.yaml
+++ b/examples/tls-sni/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-certs
     ports:
@@ -9,6 +10,7 @@ services:
 
   proxy-client:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       target: envoy-certs
       args:
@@ -18,6 +20,7 @@ services:
 
   http-upstream1:
     build:
+      context: .
       dockerfile: ../shared/echo2/Dockerfile
     hostname: http-upstream1
     environment:
@@ -25,6 +28,7 @@ services:
 
   http-upstream2:
     build:
+      context: .
       dockerfile: ../shared/echo2/Dockerfile
     hostname: http-upstream2
     environment:
@@ -32,6 +36,7 @@ services:
 
   https-upstream3:
     build:
+      context: .
       dockerfile: ../shared/echo2/Dockerfile
     hostname: https-upstream3
     environment:
diff --git a/examples/tls/docker-compose.yaml b/examples/tls/docker-compose.yaml
index ca105f0dd082..3b58da76c8b4 100644
--- a/examples/tls/docker-compose.yaml
+++ b/examples/tls/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy-https-to-http:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: ./envoy-https-http.yaml
@@ -10,6 +11,7 @@ services:
 
   proxy-https-to-https:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: ./envoy-https-https.yaml
@@ -18,6 +20,7 @@ services:
 
   proxy-http-to-https:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: ./envoy-http-https.yaml
@@ -26,6 +29,7 @@ services:
 
   proxy-https-passthrough:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: ./envoy-https-passthrough.yaml
@@ -34,6 +38,7 @@ services:
 
   service-http:
     build:
+      context: .
       dockerfile: ../shared/echo2/Dockerfile
     hostname: service-http
     environment:
@@ -41,6 +46,7 @@ services:
 
   service-https:
     build:
+      context: .
       dockerfile: ../shared/echo2/Dockerfile
     hostname: service-https
     environment:
diff --git a/examples/udp/docker-compose.yaml b/examples/udp/docker-compose.yaml
index 79c92ff90859..5189bd79f51f 100644
--- a/examples/udp/docker-compose.yaml
+++ b/examples/udp/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   testing:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     ports:
     - "${PORT_PROXY:-10000}:10000/udp"
@@ -9,4 +10,5 @@ services:
 
   service-udp:
     build:
+      context: .
       dockerfile: Dockerfile-udp
diff --git a/examples/wasm-cc/docker-compose.yaml b/examples/wasm-cc/docker-compose.yaml
index e4ac4a93fc54..bf57942182cc 100644
--- a/examples/wasm-cc/docker-compose.yaml
+++ b/examples/wasm-cc/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   proxy:
     build:
+      context: .
       dockerfile: Dockerfile-proxy
     depends_on:
     - web_service
diff --git a/examples/websocket/docker-compose.yaml b/examples/websocket/docker-compose.yaml
index 9e012bff888e..1de791969e77 100644
--- a/examples/websocket/docker-compose.yaml
+++ b/examples/websocket/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   client-ws:
     build:
+      context: .
       dockerfile: ../shared/websocket/Dockerfile
       target: websocket-client
     network_mode: host
@@ -11,6 +12,7 @@ services:
 
   proxy-ws:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: ./envoy-ws.yaml
@@ -19,6 +21,7 @@ services:
 
   proxy-wss-wss:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: ./envoy-wss.yaml
@@ -27,6 +30,7 @@ services:
 
   proxy-wss-passthrough:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: ./envoy-wss-passthrough.yaml
@@ -35,12 +39,14 @@ services:
 
   service-ws:
     build:
+      context: .
       dockerfile: ../shared/websocket/Dockerfile
     hostname: service-ws
     command: -E ws-listen:0.0.0.0:80 literalreply:'[ws] HELO'
 
   service-wss:
     build:
+      context: .
       dockerfile: ../shared/websocket/Dockerfile
     hostname: service-wss
     command: wss-listen:0.0.0.0:443 literalreply:"[wss] HELO" --pkcs12-der /certs/output.pkcs12
diff --git a/examples/zipkin/docker-compose.yaml b/examples/zipkin/docker-compose.yaml
index 1f8532e5bd81..79c353c0adff 100644
--- a/examples/zipkin/docker-compose.yaml
+++ b/examples/zipkin/docker-compose.yaml
@@ -2,6 +2,7 @@ services:
 
   envoy-front-proxy:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: envoy-front-proxy.yaml
@@ -17,6 +18,7 @@ services:
 
   envoy-1:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: envoy-1.yaml
@@ -30,6 +32,7 @@ services:
 
   envoy-2:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
       args:
         ENVOY_CONFIG: envoy-2.yaml
@@ -55,6 +58,7 @@ services:
 
   zipkin:
     build:
+      context: .
       dockerfile: Dockerfile-zipkin
     ports:
     - "${PORT_UI:-9411}:9411"
diff --git a/examples/zstd/docker-compose.yaml b/examples/zstd/docker-compose.yaml
index 507fe47db65c..60aa34e395b4 100644
--- a/examples/zstd/docker-compose.yaml
+++ b/examples/zstd/docker-compose.yaml
@@ -2,6 +2,7 @@ version: "3.7"
 services:
   envoy-stats:
     build:
+      context: .
       dockerfile: ../shared/envoy/Dockerfile
     ports:
     - "${PORT_PROXY:-10000}:10000"

From b4e61389f7db842a7bcee89d0fc080b23b27a9d2 Mon Sep 17 00:00:00 2001
From: yanavlasov 
Date: Fri, 30 Jun 2023 10:58:49 -0400
Subject: [PATCH 689/740] Add TSAN timeout scale to both timers (#28191)

Signed-off-by: Yan Avlasov 
---
 test/integration/http2_flood_integration_test.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/integration/http2_flood_integration_test.cc b/test/integration/http2_flood_integration_test.cc
index b53e6f8cdacd..1b845e80ca9c 100644
--- a/test/integration/http2_flood_integration_test.cc
+++ b/test/integration/http2_flood_integration_test.cc
@@ -956,13 +956,13 @@ TEST_P(Http2FloodMitigationTest, RstStreamOnStreamIdleTimeoutAfterResponseHeader
 // Verify detection of overflowing outbound frame queue with the PING frames sent by the keep alive
 // timer. The test verifies protocol constraint violation handling in the
 // Http2::ConnectionImpl::sendKeepalive() method.
-TEST_P(Http2FloodMitigationTest, KeepAliveTimeeTriggersFloodProtection) {
+TEST_P(Http2FloodMitigationTest, KeepAliveTimerTriggersFloodProtection) {
   config_helper_.addConfigModifier(
       [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager&
              hcm) {
         auto* keep_alive = hcm.mutable_http2_protocol_options()->mutable_connection_keepalive();
-        keep_alive->mutable_interval()->set_nanos(500 * 1000 * 1000);
-        keep_alive->mutable_timeout()->set_seconds(1 * TSAN_TIMEOUT_FACTOR);
+        keep_alive->mutable_interval()->set_seconds(1 * TSAN_TIMEOUT_FACTOR);
+        keep_alive->mutable_timeout()->set_seconds(2 * TSAN_TIMEOUT_FACTOR);
       });
 
   prefillOutboundDownstreamQueue(AllFrameFloodLimit - 1);

From ee5ec2b8291eb464bd9058062a28d4b41e552858 Mon Sep 17 00:00:00 2001
From: botengyao 
Date: Fri, 30 Jun 2023 15:23:23 -0400
Subject: [PATCH 690/740] tcp: add metrics to async client (#28204)

* add metrics to async client

Signed-off-by: Boteng Yao 
---
 source/common/tcp/BUILD                       |  2 +
 source/common/tcp/async_tcp_client_impl.cc    | 67 ++++++++++++++++-
 source/common/tcp/async_tcp_client_impl.h     | 17 ++++-
 test/common/tcp/async_tcp_client_impl_test.cc | 73 ++++++++++++++++++-
 .../tcp_async_client_integration_test.cc      | 12 ++-
 5 files changed, 161 insertions(+), 10 deletions(-)

diff --git a/source/common/tcp/BUILD b/source/common/tcp/BUILD
index cf110dd25897..cbe05d4b1575 100644
--- a/source/common/tcp/BUILD
+++ b/source/common/tcp/BUILD
@@ -48,8 +48,10 @@ envoy_cc_library(
         "//envoy/buffer:buffer_interface",
         "//envoy/event:dispatcher_interface",
         "//envoy/network:connection_interface",
+        "//envoy/stats:timespan_interface",
         "//envoy/tcp:async_tcp_client_interface",
         "//envoy/upstream:thread_local_cluster_interface",
         "//source/common/network:filter_lib",
+        "//source/common/stats:timespan_lib",
     ],
 )
diff --git a/source/common/tcp/async_tcp_client_impl.cc b/source/common/tcp/async_tcp_client_impl.cc
index 3f8c3ba5d9b7..49f6bba350d8 100644
--- a/source/common/tcp/async_tcp_client_impl.cc
+++ b/source/common/tcp/async_tcp_client_impl.cc
@@ -10,6 +10,7 @@
 #include "envoy/upstream/upstream.h"
 
 #include "source/common/common/assert.h"
+#include "source/common/stats/timespan_impl.h"
 
 namespace Envoy {
 namespace Tcp {
@@ -18,8 +19,18 @@ AsyncTcpClientImpl::AsyncTcpClientImpl(Event::Dispatcher& dispatcher,
                                        Upstream::ThreadLocalCluster& thread_local_cluster,
                                        Upstream::LoadBalancerContext* context,
                                        bool enable_half_close)
-    : dispatcher_(dispatcher), thread_local_cluster_(thread_local_cluster), context_(context),
-      enable_half_close_(enable_half_close) {}
+    : dispatcher_(dispatcher), thread_local_cluster_(thread_local_cluster),
+      cluster_info_(thread_local_cluster_.info()), context_(context),
+      connect_timer_(dispatcher.createTimer([this]() { onConnectTimeout(); })),
+      enable_half_close_(enable_half_close) {
+  connect_timer_->enableTimer(cluster_info_->connectTimeout());
+  cluster_info_->trafficStats()->upstream_cx_active_.inc();
+  cluster_info_->trafficStats()->upstream_cx_total_.inc();
+}
+
+AsyncTcpClientImpl::~AsyncTcpClientImpl() {
+  cluster_info_->trafficStats()->upstream_cx_active_.dec();
+}
 
 bool AsyncTcpClientImpl::connect() {
   connection_ = std::move(thread_local_cluster_.tcpConn(context_).connection_);
@@ -29,11 +40,26 @@ bool AsyncTcpClientImpl::connect() {
   connection_->enableHalfClose(enable_half_close_);
   connection_->addConnectionCallbacks(*this);
   connection_->addReadFilter(std::make_shared(*this));
+  conn_connect_ms_ = std::make_unique(
+      cluster_info_->trafficStats()->upstream_cx_connect_ms_, dispatcher_.timeSource());
+
+  connect_timer_->enableTimer(cluster_info_->connectTimeout());
+  connection_->setConnectionStats({cluster_info_->trafficStats()->upstream_cx_rx_bytes_total_,
+                                   cluster_info_->trafficStats()->upstream_cx_rx_bytes_buffered_,
+                                   cluster_info_->trafficStats()->upstream_cx_tx_bytes_total_,
+                                   cluster_info_->trafficStats()->upstream_cx_tx_bytes_buffered_,
+                                   &cluster_info_->trafficStats()->bind_errors_, nullptr});
   connection_->noDelay(true);
   connection_->connect();
   return true;
 }
 
+void AsyncTcpClientImpl::onConnectTimeout() {
+  ENVOY_CONN_LOG(debug, "async tcp connection timed out", *connection_);
+  cluster_info_->trafficStats()->upstream_cx_connect_timeout_.inc();
+  close(Network::ConnectionCloseType::NoFlush);
+}
+
 void AsyncTcpClientImpl::close(Network::ConnectionCloseType type) {
   if (connection_) {
     connection_->close(type);
@@ -55,10 +81,39 @@ void AsyncTcpClientImpl::onData(Buffer::Instance& data, bool end_stream) {
   }
 }
 
+void AsyncTcpClientImpl::disableConnectTimeout() {
+  if (connect_timer_) {
+    connect_timer_->disableTimer();
+    connect_timer_.reset();
+  }
+}
+
+void AsyncTcpClientImpl::reportConnectionDestroy(Network::ConnectionEvent event) {
+  auto& stats = cluster_info_->trafficStats();
+  stats->upstream_cx_destroy_.inc();
+  if (event == Network::ConnectionEvent::RemoteClose) {
+    stats->upstream_cx_destroy_remote_.inc();
+  } else {
+    stats->upstream_cx_destroy_local_.inc();
+  }
+}
+
 void AsyncTcpClientImpl::onEvent(Network::ConnectionEvent event) {
   if (event == Network::ConnectionEvent::RemoteClose ||
       event == Network::ConnectionEvent::LocalClose) {
+    if (disconnected_) {
+      cluster_info_->trafficStats()->upstream_cx_connect_fail_.inc();
+    }
+
+    if (!disconnected_ && conn_length_ms_ != nullptr) {
+      conn_length_ms_->complete();
+      conn_length_ms_.reset();
+    }
+
+    disableConnectTimeout();
+    reportConnectionDestroy(event);
     disconnected_ = true;
+
     dispatcher_.deferredDelete(std::move(connection_));
     if (callbacks_) {
       callbacks_->onEvent(event);
@@ -66,6 +121,14 @@ void AsyncTcpClientImpl::onEvent(Network::ConnectionEvent event) {
     }
   } else {
     disconnected_ = false;
+    conn_connect_ms_->complete();
+    conn_connect_ms_.reset();
+    disableConnectTimeout();
+
+    if (!conn_length_ms_) {
+      conn_length_ms_ = std::make_unique(
+          cluster_info_->trafficStats()->upstream_cx_length_ms_, dispatcher_.timeSource());
+    }
     if (callbacks_) {
       callbacks_->onEvent(event);
     }
diff --git a/source/common/tcp/async_tcp_client_impl.h b/source/common/tcp/async_tcp_client_impl.h
index d7bf8beb2fbd..9494ae47a1db 100644
--- a/source/common/tcp/async_tcp_client_impl.h
+++ b/source/common/tcp/async_tcp_client_impl.h
@@ -7,6 +7,8 @@
 #include "envoy/event/dispatcher.h"
 #include "envoy/network/connection.h"
 #include "envoy/network/filter.h"
+#include "envoy/stats/timespan.h"
+#include "envoy/stream_info/stream_info.h"
 #include "envoy/tcp/async_tcp_client.h"
 #include "envoy/upstream/thread_local_cluster.h"
 
@@ -17,12 +19,16 @@
 namespace Envoy {
 namespace Tcp {
 
-class AsyncTcpClientImpl : public AsyncTcpClient, public Network::ConnectionCallbacks {
+class AsyncTcpClientImpl : public AsyncTcpClient,
+                           public Network::ConnectionCallbacks,
+                           public Logger::Loggable {
 public:
   AsyncTcpClientImpl(Event::Dispatcher& dispatcher,
                      Upstream::ThreadLocalCluster& thread_local_cluster,
                      Upstream::LoadBalancerContext* context, bool enable_half_close);
 
+  ~AsyncTcpClientImpl() override;
+
   void close(Network::ConnectionCloseType type) override;
 
   /**
@@ -31,6 +37,8 @@ class AsyncTcpClientImpl : public AsyncTcpClient, public Network::ConnectionCall
    */
   bool connect() override;
 
+  void onConnectTimeout();
+
   void setAsyncTcpClientCallbacks(AsyncTcpClientCallbacks& callbacks) override;
 
   void write(Buffer::Instance& data, bool end_stream) override;
@@ -75,10 +83,17 @@ class AsyncTcpClientImpl : public AsyncTcpClient, public Network::ConnectionCall
     }
   }
 
+  void disableConnectTimeout();
+  void reportConnectionDestroy(Network::ConnectionEvent event);
+
   Event::Dispatcher& dispatcher_;
   Network::ClientConnectionPtr connection_;
   Upstream::ThreadLocalCluster& thread_local_cluster_;
+  Upstream::ClusterInfoConstSharedPtr cluster_info_;
   Upstream::LoadBalancerContext* context_;
+  Stats::TimespanPtr conn_connect_ms_;
+  Stats::TimespanPtr conn_length_ms_;
+  Event::TimerPtr connect_timer_;
   AsyncTcpClientCallbacks* callbacks_{};
   bool disconnected_{true};
   bool enable_half_close_{false};
diff --git a/test/common/tcp/async_tcp_client_impl_test.cc b/test/common/tcp/async_tcp_client_impl_test.cc
index 16eca0ca69ae..26298161919a 100644
--- a/test/common/tcp/async_tcp_client_impl_test.cc
+++ b/test/common/tcp/async_tcp_client_impl_test.cc
@@ -22,28 +22,37 @@ class AsyncTcpClientImplTest : public Event::TestUsingSimulatedTime, public test
   AsyncTcpClientImplTest() {
     cluster_manager_.initializeClusters({"fake_cluster"}, {});
     cluster_manager_.initializeThreadLocalClusters({"fake_cluster"});
+    connect_timer_ = new NiceMock(&dispatcher_);
     client_ = std::make_unique(
         dispatcher_, cluster_manager_.thread_local_cluster_, nullptr, false);
     client_->setAsyncTcpClientCallbacks(callbacks_);
   }
 
-  void expectCreateConnection() {
+  void expectCreateConnection(bool trigger_connected = true) {
     connection_ = new NiceMock();
     Upstream::MockHost::MockCreateConnectionData conn_info;
     conn_info.connection_ = connection_;
+
     conn_info.host_description_ = Upstream::makeTestHost(
         std::make_unique>(), "tcp://127.0.0.1:80", simTime());
 
     EXPECT_CALL(cluster_manager_.thread_local_cluster_, tcpConn_(_)).WillOnce(Return(conn_info));
     EXPECT_CALL(*connection_, connect());
     EXPECT_CALL(*connection_, addReadFilter(_));
-    EXPECT_CALL(callbacks_, onEvent(Network::ConnectionEvent::Connected));
+    if (trigger_connected) {
+      EXPECT_CALL(callbacks_, onEvent(Network::ConnectionEvent::Connected));
+    }
+
     ASSERT_TRUE(client_->connect());
-    connection_->raiseEvent(Network::ConnectionEvent::Connected);
-    ASSERT_TRUE(client_->connected());
+
+    if (trigger_connected) {
+      connection_->raiseEvent(Network::ConnectionEvent::Connected);
+      ASSERT_TRUE(client_->connected());
+    }
   }
 
   std::unique_ptr client_;
+  NiceMock* connect_timer_;
   NiceMock dispatcher_;
   NiceMock cluster_manager_;
   Network::MockClientConnection* connection_{};
@@ -108,5 +117,61 @@ TEST_F(AsyncTcpClientImplTest, TestCloseType) {
   ASSERT_FALSE(client_->connected());
 }
 
+TEST_F(AsyncTcpClientImplTest, TestTimingStats) {
+  expectCreateConnection();
+  EXPECT_CALL(callbacks_, onEvent(Network::ConnectionEvent::LocalClose));
+  EXPECT_CALL(
+      cluster_manager_.thread_local_cluster_.cluster_.info_->stats_store_,
+      deliverHistogramToSinks(testing::Property(&Stats::Metric::name, "upstream_cx_length_ms"), _));
+  client_->close(Network::ConnectionCloseType::NoFlush);
+  ASSERT_FALSE(client_->connected());
+}
+
+TEST_F(AsyncTcpClientImplTest, TestCounterStats) {
+  expectCreateConnection();
+  EXPECT_CALL(callbacks_, onEvent(Network::ConnectionEvent::LocalClose));
+  Buffer::OwnedImpl buff("test data");
+  client_->write(buff, false);
+  EXPECT_EQ(buff.length(), cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                               ->upstream_cx_tx_bytes_total_.value());
+  client_->close(Network::ConnectionCloseType::NoFlush);
+  EXPECT_EQ(1UL, cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                     ->upstream_cx_total_.value());
+  EXPECT_EQ(1UL, cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                     ->upstream_cx_destroy_.value());
+  EXPECT_EQ(1UL, cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                     ->upstream_cx_destroy_local_.value());
+  ASSERT_FALSE(client_->connected());
+}
+
+TEST_F(AsyncTcpClientImplTest, TestFailStats) {
+  expectCreateConnection(false);
+  connect_timer_->invokeCallback();
+  EXPECT_EQ(1UL, cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                     ->upstream_cx_connect_timeout_.value());
+  EXPECT_EQ(1UL, cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                     ->upstream_cx_connect_fail_.value());
+}
+
+TEST_F(AsyncTcpClientImplTest, TestCxDestroyRemoteClose) {
+  expectCreateConnection();
+  EXPECT_CALL(callbacks_, onEvent(Network::ConnectionEvent::RemoteClose));
+  connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
+  EXPECT_EQ(1UL, cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                     ->upstream_cx_destroy_.value());
+  EXPECT_EQ(1UL, cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                     ->upstream_cx_destroy_remote_.value());
+  ASSERT_FALSE(client_->connected());
+}
+
+TEST_F(AsyncTcpClientImplTest, TestActiveCx) {
+  expectCreateConnection();
+  EXPECT_EQ(1UL, cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                     ->upstream_cx_active_.value());
+  client_.reset();
+  EXPECT_EQ(0UL, cluster_manager_.thread_local_cluster_.cluster_.info_->traffic_stats_
+                     ->upstream_cx_active_.value());
+}
+
 } // namespace Tcp
 } // namespace Envoy
diff --git a/test/integration/tcp_async_client_integration_test.cc b/test/integration/tcp_async_client_integration_test.cc
index 0525e2432e60..4384589fbbd1 100644
--- a/test/integration/tcp_async_client_integration_test.cc
+++ b/test/integration/tcp_async_client_integration_test.cc
@@ -31,17 +31,23 @@ TEST_P(TcpAsyncClientIntegrationTest, SingleRequest) {
 
   IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0"));
   test_server_->waitForCounterEq("test_network_async_tcp_filter.on_new_connection", 1);
-
+  test_server_->waitForGaugeEq("cluster.cluster_0.upstream_cx_active", 1);
+  test_server_->waitForNumHistogramSamplesGe("cluster.cluster_0.upstream_cx_connect_ms", 1);
   ASSERT_TRUE(tcp_client->write(request, true));
-
+  test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_tx_bytes_total", request.size());
   FakeRawConnectionPtr fake_upstream_connection;
   ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection));
   ASSERT_TRUE(fake_upstream_connection->waitForData(request.size()));
   ASSERT_TRUE(fake_upstream_connection->write(response, true));
   test_server_->waitForCounterGe("test_network_async_tcp_filter.on_receive_async_data", 1);
-
+  test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_rx_bytes_total", response.size());
   ASSERT_TRUE(tcp_client->waitForData(response.size()));
   tcp_client->close();
+  test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_destroy_local", 1);
+  test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_destroy", 1);
+  test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_total", 1);
+  test_server_->waitForGaugeEq("cluster.cluster_0.upstream_cx_active", 0);
+  test_server_->waitForNumHistogramSamplesGe("cluster.cluster_0.upstream_cx_length_ms", 1);
 }
 
 TEST_P(TcpAsyncClientIntegrationTest, MultipleRequestFrames) {

From 8fc16edcfc578d07874cdce064302d9241217a77 Mon Sep 17 00:00:00 2001
From: yanjunxiang-google
 <78807980+yanjunxiang-google@users.noreply.github.com>
Date: Fri, 30 Jun 2023 15:24:13 -0400
Subject: [PATCH 691/740] Plumbing HCM max header size and count limits into
 HeaderMap. (#28131)

* Plumbing HCM max header size and count limits into HeaderMap.

Signed-off-by: Yanjun Xiang 
---
 envoy/http/header_map.h                       | 10 +++
 source/common/http/header_map_impl.h          | 70 ++++++++++++++-----
 source/common/http/http1/codec_impl.h         | 20 +++---
 source/common/http/http2/codec_impl.h         | 16 +++--
 .../common/quic/envoy_quic_client_stream.cc   | 10 +--
 .../common/quic/envoy_quic_server_stream.cc   |  9 ++-
 source/common/quic/envoy_quic_utils.h         | 10 +--
 test/common/http/header_map_impl_test.cc      | 35 ++++++++++
 test/common/http/http1/codec_impl_test.cc     | 66 +++++++++++++++++
 test/common/http/http2/codec_impl_test.cc     | 46 ++++++++++++
 test/common/quic/envoy_quic_utils_test.cc     | 64 ++++++++++++++---
 test/mocks/http/mocks.h                       | 21 ++++++
 test/test_common/utility.h                    | 15 ++++
 13 files changed, 340 insertions(+), 52 deletions(-)

diff --git a/envoy/http/header_map.h b/envoy/http/header_map.h
index b40f25433428..44dddc49d52f 100644
--- a/envoy/http/header_map.h
+++ b/envoy/http/header_map.h
@@ -449,6 +449,16 @@ class HeaderMap {
    */
   virtual uint64_t byteSize() const PURE;
 
+  /**
+   * @return uint32_t the max size of the header map in kilobyte.
+   */
+  virtual uint32_t maxHeadersKb() const PURE;
+
+  /**
+   * @return uint32_t the max count of headers in a header map.
+   */
+  virtual uint32_t maxHeadersCount() const PURE;
+
   /**
    * This is a wrapper for the return result from get(). It avoids a copy when translating from
    * non-const HeaderEntry to const HeaderEntry and only provides const access to the result.
diff --git a/source/common/http/header_map_impl.h b/source/common/http/header_map_impl.h
index 4d252ff22bd5..df1d76431bb7 100644
--- a/source/common/http/header_map_impl.h
+++ b/source/common/http/header_map_impl.h
@@ -52,6 +52,9 @@ public:
  */
 class HeaderMapImpl : NonCopyable {
 public:
+  HeaderMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
+                const uint32_t max_headers_count = UINT32_MAX)
+      : max_headers_kb_(max_headers_kb), max_headers_count_(max_headers_count) {}
   virtual ~HeaderMapImpl() = default;
 
   // The following "constructors" call virtual functions during construction and must use the
@@ -91,6 +94,8 @@ class HeaderMapImpl : NonCopyable {
   void setReferenceKey(const LowerCaseString& key, absl::string_view value);
   void setCopy(const LowerCaseString& key, absl::string_view value);
   uint64_t byteSize() const;
+  uint32_t maxHeadersKb() const { return max_headers_kb_; }
+  uint32_t maxHeadersCount() const { return max_headers_count_; }
   HeaderMap::GetResult get(const LowerCaseString& key) const;
   void iterate(HeaderMap::ConstIterateCb cb) const;
   void iterateReverse(HeaderMap::ConstIterateCb cb) const;
@@ -335,6 +340,10 @@ class HeaderMapImpl : NonCopyable {
   StatefulHeaderKeyFormatterPtr formatter_;
   // This holds the internal byte size of the HeaderMap.
   uint64_t cached_byte_size_ = 0;
+  // This holds the max size of the headers in kilobyte in the HeaderMap.
+  const uint32_t max_headers_kb_ = UINT32_MAX;
+  // This holds the max count of the headers in the HeaderMap.
+  const uint32_t max_headers_count_ = UINT32_MAX;
 };
 
 /**
@@ -344,6 +353,9 @@ class HeaderMapImpl : NonCopyable {
  */
 template  class TypedHeaderMapImpl : public HeaderMapImpl, public Interface {
 public:
+  TypedHeaderMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
+                     const uint32_t max_headers_count = UINT32_MAX)
+      : HeaderMapImpl(max_headers_kb, max_headers_count) {}
   void setFormatter(StatefulHeaderKeyFormatterPtr&& formatter) {
     formatter_ = std::move(formatter);
   }
@@ -382,6 +394,8 @@ template  class TypedHeaderMapImpl : public HeaderMapImpl, publ
     HeaderMapImpl::setCopy(key, value);
   }
   uint64_t byteSize() const override { return HeaderMapImpl::byteSize(); }
+  uint32_t maxHeadersKb() const override { return HeaderMapImpl::maxHeadersKb(); }
+  uint32_t maxHeadersCount() const override { return HeaderMapImpl::maxHeadersCount(); }
   HeaderMap::GetResult get(const LowerCaseString& key) const override {
     return HeaderMapImpl::get(key);
   }
@@ -465,8 +479,11 @@ template  class TypedHeaderMapImpl : public HeaderMapImpl, publ
 class RequestHeaderMapImpl final : public TypedHeaderMapImpl,
                                    public InlineStorage {
 public:
-  static std::unique_ptr create() {
-    return std::unique_ptr(new (inlineHeadersSize()) RequestHeaderMapImpl());
+  static std::unique_ptr
+  create(const uint32_t max_headers_kb = UINT32_MAX,
+         const uint32_t max_headers_count = UINT32_MAX) {
+    return std::unique_ptr(
+        new (inlineHeadersSize()) RequestHeaderMapImpl(max_headers_kb, max_headers_count));
   }
 
   INLINE_REQ_STRING_HEADERS(DEFINE_INLINE_HEADER_STRING_FUNCS)
@@ -504,7 +521,11 @@ class RequestHeaderMapImpl final : public TypedHeaderMapImpl,
 
   using HeaderHandles = ConstSingleton;
 
-  RequestHeaderMapImpl() { clearInline(); }
+  RequestHeaderMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
+                       const uint32_t max_headers_count = UINT32_MAX)
+      : TypedHeaderMapImpl(max_headers_kb, max_headers_count) {
+    clearInline();
+  }
 
   HeaderEntryImpl* inline_headers_[];
 };
@@ -516,9 +537,11 @@ class RequestHeaderMapImpl final : public TypedHeaderMapImpl,
 class RequestTrailerMapImpl final : public TypedHeaderMapImpl,
                                     public InlineStorage {
 public:
-  static std::unique_ptr create() {
-    return std::unique_ptr(new (inlineHeadersSize())
-                                                      RequestTrailerMapImpl());
+  static std::unique_ptr
+  create(const uint32_t max_headers_kb = UINT32_MAX,
+         const uint32_t max_headers_count = UINT32_MAX) {
+    return std::unique_ptr(
+        new (inlineHeadersSize()) RequestTrailerMapImpl(max_headers_kb, max_headers_count));
   }
 
 protected:
@@ -528,7 +551,11 @@ class RequestTrailerMapImpl final : public TypedHeaderMapImpl
   HeaderEntryImpl** inlineHeaders() override { return inline_headers_; }
 
 private:
-  RequestTrailerMapImpl() { clearInline(); }
+  RequestTrailerMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
+                        const uint32_t max_headers_count = UINT32_MAX)
+      : TypedHeaderMapImpl(max_headers_kb, max_headers_count) {
+    clearInline();
+  }
 
   HeaderEntryImpl* inline_headers_[];
 };
@@ -540,9 +567,11 @@ class RequestTrailerMapImpl final : public TypedHeaderMapImpl
 class ResponseHeaderMapImpl final : public TypedHeaderMapImpl,
                                     public InlineStorage {
 public:
-  static std::unique_ptr create() {
-    return std::unique_ptr(new (inlineHeadersSize())
-                                                      ResponseHeaderMapImpl());
+  static std::unique_ptr
+  create(const uint32_t max_headers_kb = UINT32_MAX,
+         const uint32_t max_headers_count = UINT32_MAX) {
+    return std::unique_ptr(
+        new (inlineHeadersSize()) ResponseHeaderMapImpl(max_headers_kb, max_headers_count));
   }
 
   INLINE_RESP_STRING_HEADERS(DEFINE_INLINE_HEADER_STRING_FUNCS)
@@ -570,8 +599,11 @@ class ResponseHeaderMapImpl final : public TypedHeaderMapImpl
 
   using HeaderHandles = ConstSingleton;
 
-  ResponseHeaderMapImpl() { clearInline(); }
-
+  ResponseHeaderMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
+                        const uint32_t max_headers_count = UINT32_MAX)
+      : TypedHeaderMapImpl(max_headers_kb, max_headers_count) {
+    clearInline();
+  }
   HeaderEntryImpl* inline_headers_[];
 };
 
@@ -582,9 +614,11 @@ class ResponseHeaderMapImpl final : public TypedHeaderMapImpl
 class ResponseTrailerMapImpl final : public TypedHeaderMapImpl,
                                      public InlineStorage {
 public:
-  static std::unique_ptr create() {
-    return std::unique_ptr(new (inlineHeadersSize())
-                                                       ResponseTrailerMapImpl());
+  static std::unique_ptr
+  create(const uint32_t max_headers_kb = UINT32_MAX,
+         const uint32_t max_headers_count = UINT32_MAX) {
+    return std::unique_ptr(
+        new (inlineHeadersSize()) ResponseTrailerMapImpl(max_headers_kb, max_headers_count));
   }
 
   INLINE_RESP_STRING_HEADERS_TRAILERS(DEFINE_INLINE_HEADER_STRING_FUNCS)
@@ -604,7 +638,11 @@ class ResponseTrailerMapImpl final : public TypedHeaderMapImpl;
 
-  ResponseTrailerMapImpl() { clearInline(); }
+  ResponseTrailerMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
+                         const uint32_t max_headers_count = UINT32_MAX)
+      : TypedHeaderMapImpl(max_headers_kb, max_headers_count) {
+    clearInline();
+  }
 
   HeaderEntryImpl* inline_headers_[];
 };
diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h
index 2238f750fe7b..b59e1931a785 100644
--- a/source/common/http/http1/codec_impl.h
+++ b/source/common/http/http1/codec_impl.h
@@ -28,6 +28,9 @@ namespace Envoy {
 namespace Http {
 namespace Http1 {
 
+// The default limit of 80 KiB is the vanilla http_parser behaviour.
+inline constexpr uint32_t MAX_RESPONSE_HEADERS_KB = 80;
+
 class ConnectionImpl;
 
 /**
@@ -314,6 +317,8 @@ class ConnectionImpl : public virtual Connection,
   bool dispatching_ : 1;
   bool dispatching_slice_already_drained_ : 1;
   StreamInfo::BytesMeterSharedPtr bytes_meter_before_stream_;
+  const uint32_t max_headers_kb_;
+  const uint32_t max_headers_count_;
 
 private:
   enum class HeaderParsingState { Field, Value, Done };
@@ -451,8 +456,6 @@ class ConnectionImpl : public virtual Connection,
   // the last byte of the body is processed (whichever happens first).
   Buffer::OwnedImpl buffered_body_;
   Protocol protocol_{Protocol::Http11};
-  const uint32_t max_headers_kb_;
-  const uint32_t max_headers_count_;
 };
 
 /**
@@ -540,14 +543,15 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl {
   void allocHeaders(StatefulHeaderKeyFormatterPtr&& formatter) override {
     ASSERT(nullptr == absl::get(headers_or_trailers_));
     ASSERT(!processing_trailers_);
-    auto headers = RequestHeaderMapImpl::create();
+    auto headers = RequestHeaderMapImpl::create(max_headers_kb_, max_headers_count_);
     headers->setFormatter(std::move(formatter));
     headers_or_trailers_.emplace(std::move(headers));
   }
   void allocTrailers() override {
     ASSERT(processing_trailers_);
     if (!absl::holds_alternative(headers_or_trailers_)) {
-      headers_or_trailers_.emplace(RequestTrailerMapImpl::create());
+      headers_or_trailers_.emplace(
+          RequestTrailerMapImpl::create(max_headers_kb_, max_headers_count_));
     }
   }
   void dumpAdditionalState(std::ostream& os, int indent_level) const override;
@@ -645,14 +649,15 @@ class ClientConnectionImpl : public ClientConnection, public ConnectionImpl {
   void allocHeaders(StatefulHeaderKeyFormatterPtr&& formatter) override {
     ASSERT(nullptr == absl::get(headers_or_trailers_));
     ASSERT(!processing_trailers_);
-    auto headers = ResponseHeaderMapImpl::create();
+    auto headers = ResponseHeaderMapImpl::create(max_headers_kb_, max_headers_count_);
     headers->setFormatter(std::move(formatter));
     headers_or_trailers_.emplace(std::move(headers));
   }
   void allocTrailers() override {
     ASSERT(processing_trailers_);
     if (!absl::holds_alternative(headers_or_trailers_)) {
-      headers_or_trailers_.emplace(ResponseTrailerMapImpl::create());
+      headers_or_trailers_.emplace(
+          ResponseTrailerMapImpl::create(max_headers_kb_, max_headers_count_));
     }
   }
   void dumpAdditionalState(std::ostream& os, int indent_level) const override;
@@ -677,9 +682,6 @@ class ClientConnectionImpl : public ClientConnection, public ConnectionImpl {
   // to null headers on message complete for assertion purposes.
   absl::variant headers_or_trailers_;
 
-  // The default limit of 80 KiB is the vanilla http_parser behaviour.
-  static constexpr uint32_t MAX_RESPONSE_HEADERS_KB = 80;
-
   // True if the upstream connection is pointed at an HTTP/1.1 proxy, and
   // plaintext HTTP should be sent with fully qualified URLs.
   bool passing_through_proxy_ = false;
diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h
index 5d4a6033d34f..2cbb5d28814e 100644
--- a/source/common/http/http2/codec_impl.h
+++ b/source/common/http/http2/codec_impl.h
@@ -438,7 +438,8 @@ class ConnectionImpl : public virtual Connection,
     ClientStreamImpl(ConnectionImpl& parent, uint32_t buffer_limit,
                      ResponseDecoder& response_decoder)
         : StreamImpl(parent, buffer_limit), response_decoder_(response_decoder),
-          headers_or_trailers_(ResponseHeaderMapImpl::create()) {}
+          headers_or_trailers_(
+              ResponseHeaderMapImpl::create(parent_.max_headers_kb_, parent_.max_headers_count_)) {}
 
     // Http::MultiplexedStreamImplBase
     // Client streams do not need a flush timer because we currently assume that any failure
@@ -466,9 +467,11 @@ class ConnectionImpl : public virtual Connection,
       // If we are waiting for informational headers, make a new response header map, otherwise
       // we are about to receive trailers. The codec makes sure this is the only valid sequence.
       if (received_noninformational_headers_) {
-        headers_or_trailers_.emplace(ResponseTrailerMapImpl::create());
+        headers_or_trailers_.emplace(
+            ResponseTrailerMapImpl::create(parent_.max_headers_kb_, parent_.max_headers_count_));
       } else {
-        headers_or_trailers_.emplace(ResponseHeaderMapImpl::create());
+        headers_or_trailers_.emplace(
+            ResponseHeaderMapImpl::create(parent_.max_headers_kb_, parent_.max_headers_count_));
       }
     }
     HeaderMapPtr cloneTrailers(const HeaderMap& trailers) override {
@@ -497,7 +500,9 @@ class ConnectionImpl : public virtual Connection,
    */
   struct ServerStreamImpl : public StreamImpl, public ResponseEncoder {
     ServerStreamImpl(ConnectionImpl& parent, uint32_t buffer_limit)
-        : StreamImpl(parent, buffer_limit), headers_or_trailers_(RequestHeaderMapImpl::create()) {}
+        : StreamImpl(parent, buffer_limit),
+          headers_or_trailers_(
+              RequestHeaderMapImpl::create(parent_.max_headers_kb_, parent_.max_headers_count_)) {}
 
     // StreamImpl
     void destroy() override;
@@ -517,7 +522,8 @@ class ConnectionImpl : public virtual Connection,
       }
     }
     void allocTrailers() override {
-      headers_or_trailers_.emplace(RequestTrailerMapImpl::create());
+      headers_or_trailers_.emplace(
+          RequestTrailerMapImpl::create(parent_.max_headers_kb_, parent_.max_headers_count_));
     }
     HeaderMapPtr cloneTrailers(const HeaderMap& trailers) override {
       return createHeaderMap(trailers);
diff --git a/source/common/quic/envoy_quic_client_stream.cc b/source/common/quic/envoy_quic_client_stream.cc
index dca38b6518cc..63439c1c4fa3 100644
--- a/source/common/quic/envoy_quic_client_stream.cc
+++ b/source/common/quic/envoy_quic_client_stream.cc
@@ -234,10 +234,11 @@ void EnvoyQuicClientStream::OnInitialHeadersComplete(bool fin, size_t frame_len,
   }
   saw_regular_headers_ = false;
   quic::QuicRstStreamErrorCode transform_rst = quic::QUIC_STREAM_NO_ERROR;
+  auto client_session = static_cast(session());
   std::unique_ptr headers =
       quicHeadersToEnvoyHeaders(
-          header_list, *this, filterManagerConnection()->maxIncomingHeadersCount(), details_,
-          transform_rst);
+          header_list, *this, client_session->max_inbound_header_list_size(),
+          filterManagerConnection()->maxIncomingHeadersCount(), details_, transform_rst);
   if (headers == nullptr) {
     onStreamError(close_connection_upon_invalid_header_, transform_rst);
     return;
@@ -383,9 +384,10 @@ void EnvoyQuicClientStream::maybeDecodeTrailers() {
       return;
     }
     quic::QuicRstStreamErrorCode transform_rst = quic::QUIC_STREAM_NO_ERROR;
+    auto client_session = static_cast(session());
     auto trailers = http2HeaderBlockToEnvoyTrailers(
-        received_trailers(), filterManagerConnection()->maxIncomingHeadersCount(), *this, details_,
-        transform_rst);
+        received_trailers(), client_session->max_inbound_header_list_size(),
+        filterManagerConnection()->maxIncomingHeadersCount(), *this, details_, transform_rst);
     if (trailers == nullptr) {
       onStreamError(close_connection_upon_invalid_header_, transform_rst);
       return;
diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc
index 7a12c93e7b08..003ff29c5b77 100644
--- a/source/common/quic/envoy_quic_server_stream.cc
+++ b/source/common/quic/envoy_quic_server_stream.cc
@@ -233,9 +233,11 @@ void EnvoyQuicServerStream::OnInitialHeadersComplete(bool fin, size_t frame_len,
   }
   saw_regular_headers_ = false;
   quic::QuicRstStreamErrorCode rst = quic::QUIC_STREAM_NO_ERROR;
+  auto server_session = static_cast(session());
   std::unique_ptr headers =
       quicHeadersToEnvoyHeaders(
-          header_list, *this, filterManagerConnection()->maxIncomingHeadersCount(), details_, rst);
+          header_list, *this, server_session->max_inbound_header_list_size(),
+          filterManagerConnection()->maxIncomingHeadersCount(), details_, rst);
   if (headers == nullptr) {
     onStreamError(close_connection_upon_invalid_header_, rst);
     return;
@@ -366,9 +368,10 @@ void EnvoyQuicServerStream::maybeDecodeTrailers() {
       return;
     }
     quic::QuicRstStreamErrorCode rst = quic::QUIC_STREAM_NO_ERROR;
+    auto server_session = static_cast(session());
     auto trailers = http2HeaderBlockToEnvoyTrailers(
-        received_trailers(), filterManagerConnection()->maxIncomingHeadersCount(), *this, details_,
-        rst);
+        received_trailers(), server_session->max_inbound_header_list_size(),
+        filterManagerConnection()->maxIncomingHeadersCount(), *this, details_, rst);
     if (trailers == nullptr) {
       onStreamError(close_connection_upon_invalid_header_, rst);
       return;
diff --git a/source/common/quic/envoy_quic_utils.h b/source/common/quic/envoy_quic_utils.h
index 5e3333b00bc2..e5fcc2532a7b 100644
--- a/source/common/quic/envoy_quic_utils.h
+++ b/source/common/quic/envoy_quic_utils.h
@@ -65,9 +65,9 @@ class HeaderValidator {
 template 
 std::unique_ptr
 quicHeadersToEnvoyHeaders(const quic::QuicHeaderList& header_list, HeaderValidator& validator,
-                          uint32_t max_headers_allowed, absl::string_view& details,
-                          quic::QuicRstStreamErrorCode& rst) {
-  auto headers = T::create();
+                          uint32_t max_headers_kb, uint32_t max_headers_allowed,
+                          absl::string_view& details, quic::QuicRstStreamErrorCode& rst) {
+  auto headers = T::create(max_headers_kb, max_headers_allowed);
   for (const auto& entry : header_list) {
     if (max_headers_allowed == 0) {
       details = Http3ResponseCodeDetailValues::too_many_headers;
@@ -101,10 +101,10 @@ quicHeadersToEnvoyHeaders(const quic::QuicHeaderList& header_list, HeaderValidat
 
 template 
 std::unique_ptr
-http2HeaderBlockToEnvoyTrailers(const spdy::Http2HeaderBlock& header_block,
+http2HeaderBlockToEnvoyTrailers(const spdy::Http2HeaderBlock& header_block, uint32_t max_headers_kb,
                                 uint32_t max_headers_allowed, HeaderValidator& validator,
                                 absl::string_view& details, quic::QuicRstStreamErrorCode& rst) {
-  auto headers = T::create();
+  auto headers = T::create(max_headers_kb, max_headers_allowed);
   if (header_block.size() > max_headers_allowed) {
     details = Http3ResponseCodeDetailValues::too_many_trailers;
     rst = quic::QUIC_STREAM_EXCESSIVE_LOAD;
diff --git a/test/common/http/header_map_impl_test.cc b/test/common/http/header_map_impl_test.cc
index 50fa5d1600a8..8cee2dd72fe2 100644
--- a/test/common/http/header_map_impl_test.cc
+++ b/test/common/http/header_map_impl_test.cc
@@ -1112,5 +1112,40 @@ TEST(HeaderMapImplTest, HttpTraceContextTest) {
   }
 }
 
+// Test the header map max limits are setup correctly.
+TEST(HeaderMapImplTest, HeaderMapMaxLimits) {
+  auto request_header_default = RequestHeaderMapImpl::create();
+  EXPECT_EQ(request_header_default->maxHeadersKb(), UINT32_MAX);
+  EXPECT_EQ(request_header_default->maxHeadersCount(), UINT32_MAX);
+
+  auto request_header = RequestHeaderMapImpl::create(5, 10);
+  EXPECT_EQ(request_header->maxHeadersKb(), 5);
+  EXPECT_EQ(request_header->maxHeadersCount(), 10);
+
+  auto request_trailer_default = RequestTrailerMapImpl::create();
+  EXPECT_EQ(request_trailer_default->maxHeadersKb(), UINT32_MAX);
+  EXPECT_EQ(request_trailer_default->maxHeadersCount(), UINT32_MAX);
+
+  auto request_trailer = RequestTrailerMapImpl::create(10, 20);
+  EXPECT_EQ(request_trailer->maxHeadersKb(), 10);
+  EXPECT_EQ(request_trailer->maxHeadersCount(), 20);
+
+  auto response_header_default = ResponseHeaderMapImpl::create();
+  EXPECT_EQ(response_header_default->maxHeadersKb(), UINT32_MAX);
+  EXPECT_EQ(response_header_default->maxHeadersCount(), UINT32_MAX);
+
+  auto response_header = ResponseHeaderMapImpl::create(50, 100);
+  EXPECT_EQ(response_header->maxHeadersKb(), 50);
+  EXPECT_EQ(response_header->maxHeadersCount(), 100);
+
+  auto response_trailer_default = ResponseTrailerMapImpl::create();
+  EXPECT_EQ(response_trailer_default->maxHeadersKb(), UINT32_MAX);
+  EXPECT_EQ(response_trailer_default->maxHeadersCount(), UINT32_MAX);
+
+  auto response_trailer = ResponseTrailerMapImpl::create(2, 3);
+  EXPECT_EQ(response_trailer->maxHeadersKb(), 2);
+  EXPECT_EQ(response_trailer->maxHeadersCount(), 3);
+}
+
 } // namespace Http
 } // namespace Envoy
diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc
index fd2e365b78ed..a58a590a55f9 100644
--- a/test/common/http/http1/codec_impl_test.cc
+++ b/test/common/http/http1/codec_impl_test.cc
@@ -1978,6 +1978,40 @@ TEST_P(Http1ServerConnectionImplTest, ChunkedResponse) {
             output);
 }
 
+TEST_P(Http1ServerConnectionImplTest, VerifyRequestHeaderTrailerMapMaxLimits) {
+  initialize();
+  InSequence sequence;
+
+  codec_settings_.allow_absolute_url_ = true;
+  codec_settings_.enable_trailers_ = true;
+  codec_ = std::make_unique(
+      connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_,
+      max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW,
+      overload_manager_);
+
+  MockRequestDecoder decoder;
+  EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder));
+  TestRequestHeaderMapImpl expected_headers{
+      {{":path", "/"}, {":method", "POST"}, {"transfer-encoding", "chunked"}},
+      max_request_headers_kb_,
+      max_request_headers_count_};
+  EXPECT_CALL(decoder, decodeHeaders_(HeaderMapEqualWithMaxSize(&expected_headers), false));
+  Buffer::OwnedImpl expected_data("Hello World");
+  EXPECT_CALL(decoder, decodeData(BufferEqual(&expected_data), false));
+
+  TestRequestTrailerMapImpl expected_trailers{{{"hello", "world"}, {"second", "header"}},
+                                              max_request_headers_kb_,
+                                              max_request_headers_count_};
+  EXPECT_CALL(decoder, decodeTrailers_(HeaderMapEqualWithMaxSize(&expected_trailers)));
+
+  Buffer::OwnedImpl buffer("POST / HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n"
+                           "6\r\nHello \r\n"
+                           "5\r\nWorld\r\n"
+                           "0\r\nhello: world\r\nsecond: header\r\n\r\n");
+  auto status = codec_->dispatch(buffer);
+  EXPECT_TRUE(status.ok());
+}
+
 TEST_P(Http1ServerConnectionImplTest, ChunkedResponseWithTrailers) {
   codec_settings_.enable_trailers_ = true;
   initialize();
@@ -3786,6 +3820,38 @@ TEST_P(Http1ClientConnectionImplTest, TestResponseSplitAllowChunkedLength100) {
   testClientAllowChunkedContentLength(100, true);
 }
 
+TEST_P(Http1ClientConnectionImplTest, VerifyResponseHeaderTrailerMapMaxLimits) {
+  codec_settings_.allow_chunked_length_ = true;
+  codec_settings_.enable_trailers_ = true;
+  codec_ = std::make_unique(
+      connection_, http1CodecStats(), callbacks_, codec_settings_, max_response_headers_count_);
+
+  NiceMock response_decoder;
+  Http::RequestEncoder& request_encoder = codec_->newStream(response_decoder);
+  TestRequestHeaderMapImpl headers{{":method", "GET"}, {":path", "/"}, {":authority", "host"}};
+  EXPECT_TRUE(request_encoder.encodeHeaders(headers, true).ok());
+
+  TestResponseHeaderMapImpl expected_headers{{{":status", "200"}, {"transfer-encoding", "chunked"}},
+                                             Http1::MAX_RESPONSE_HEADERS_KB,
+                                             max_response_headers_count_};
+  Buffer::OwnedImpl expected_data("Hello World");
+
+  EXPECT_CALL(response_decoder,
+              decodeHeaders_(HeaderMapEqualWithMaxSize(&expected_headers), false));
+  EXPECT_CALL(response_decoder, decodeData(BufferEqual(&expected_data), false));
+  TestResponseTrailerMapImpl expected_trailers{{{"hello", "world"}, {"second", "header"}},
+                                               Http1::MAX_RESPONSE_HEADERS_KB,
+                                               max_response_headers_count_};
+  EXPECT_CALL(response_decoder, decodeTrailers_(HeaderMapEqualWithMaxSize(&expected_trailers)));
+
+  Buffer::OwnedImpl buffer("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n"
+                           "6\r\nHello \r\n"
+                           "5\r\nWorld\r\n"
+                           "0\r\nhello: world\r\nsecond: header\r\n\r\n");
+  auto status = codec_->dispatch(buffer);
+  EXPECT_TRUE(status.ok());
+}
+
 TEST_P(Http1ClientConnectionImplTest,
        ShouldDumpParsedAndPartialHeadersWithoutAllocatingMemoryIfProcessingHeaders) {
   if (parser_impl_ == Http1ParserImpl::BalsaParser) {
diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc
index f790f3e7a2b0..104032980065 100644
--- a/test/common/http/http2/codec_impl_test.cc
+++ b/test/common/http/http2/codec_impl_test.cc
@@ -982,6 +982,52 @@ TEST_P(Http2CodecImplTest, InvalidHeadersFrameMissing) {
   EXPECT_THAT(status.message(), testing::HasSubstr("missing required"));
 }
 
+TEST_P(Http2CodecImplTest, VerifyHeaderMapMaxSizeLimits) {
+  initialize();
+
+  TestRequestHeaderMapImpl request_headers;
+  HttpTestUtility::addDefaultHeaders(request_headers);
+  TestRequestHeaderMapImpl expected_request_headers{
+      {}, max_request_headers_kb_, max_request_headers_count_};
+  HttpTestUtility::addDefaultHeaders(expected_request_headers);
+  EXPECT_CALL(request_decoder_,
+              decodeHeaders_(HeaderMapEqualWithMaxSize(&expected_request_headers), false));
+  EXPECT_TRUE(request_encoder_->encodeHeaders(request_headers, false).ok());
+  driveToCompletion();
+  EXPECT_CALL(request_decoder_, decodeData(_, false));
+  Buffer::OwnedImpl hello("hello");
+  request_encoder_->encodeData(hello, false);
+  driveToCompletion();
+
+  TestRequestTrailerMapImpl request_trailers{{"trailing", "header"}};
+  TestRequestTrailerMapImpl expected_request_trailers{
+      {{"trailing", "header"}}, max_request_headers_kb_, max_request_headers_count_};
+  EXPECT_CALL(request_decoder_,
+              decodeTrailers_(HeaderMapEqualWithMaxSize(&expected_request_trailers)));
+  request_encoder_->encodeTrailers(request_trailers);
+  driveToCompletion();
+
+  TestResponseHeaderMapImpl response_headers{{":status", "200"}};
+  TestResponseHeaderMapImpl expected_response_headers{
+      {{":status", "200"}}, max_request_headers_kb_, max_request_headers_count_};
+  EXPECT_CALL(response_decoder_,
+              decodeHeaders_(HeaderMapEqualWithMaxSize(&expected_response_headers), false));
+  response_encoder_->encodeHeaders(response_headers, false);
+  driveToCompletion();
+  EXPECT_CALL(response_decoder_, decodeData(_, false));
+  Buffer::OwnedImpl world("world");
+  response_encoder_->encodeData(world, false);
+  driveToCompletion();
+
+  TestResponseTrailerMapImpl response_trailers{{"trailing", "header"}};
+  TestResponseTrailerMapImpl expected_response_trailers{
+      {{"trailing", "header"}}, max_request_headers_kb_, max_request_headers_count_};
+  EXPECT_CALL(response_decoder_,
+              decodeTrailers_(HeaderMapEqualWithMaxSize(&expected_response_trailers)));
+  response_encoder_->encodeTrailers(response_trailers);
+  driveToCompletion();
+}
+
 TEST_P(Http2CodecImplTest, TrailingHeaders) {
   initialize();
 
diff --git a/test/common/quic/envoy_quic_utils_test.cc b/test/common/quic/envoy_quic_utils_test.cc
index ea634b27a489..c7f57b64f12a 100644
--- a/test/common/quic/envoy_quic_utils_test.cc
+++ b/test/common/quic/envoy_quic_utils_test.cc
@@ -59,7 +59,7 @@ TEST(EnvoyQuicUtilsTest, HeadersConversion) {
   absl::string_view details;
   quic::QuicRstStreamErrorCode rst = quic::QUIC_REFUSED_STREAM;
   auto envoy_headers = http2HeaderBlockToEnvoyTrailers(
-      headers_block, 100, validator, details, rst);
+      headers_block, 60, 100, validator, details, rst);
   // Envoy header block is 3 headers larger because QUICHE header block does coalescing.
   EXPECT_EQ(headers_block.size() + 3u, envoy_headers->size());
   EXPECT_EQ("www.google.com", envoy_headers->getHostValue());
@@ -95,7 +95,7 @@ TEST(EnvoyQuicUtilsTest, HeadersConversion) {
         return Http::HeaderUtility::HeaderValidationResult::ACCEPT;
       });
   auto envoy_headers2 = quicHeadersToEnvoyHeaders(
-      quic_headers, validator, 100, details, rst);
+      quic_headers, validator, 60, 100, details, rst);
   EXPECT_EQ(*envoy_headers, *envoy_headers2);
   EXPECT_EQ(rst, quic::QUIC_REFUSED_STREAM); // With no error it will be untouched.
 
@@ -114,7 +114,7 @@ TEST(EnvoyQuicUtilsTest, HeadersConversion) {
         return Http::HeaderUtility::HeaderValidationResult::ACCEPT;
       });
   EXPECT_EQ(nullptr, quicHeadersToEnvoyHeaders(quic_headers2, validator,
-                                                                           100, details, rst));
+                                                                           60, 100, details, rst));
   EXPECT_EQ(rst, quic::QUIC_BAD_APPLICATION_PAYLOAD);
 }
 
@@ -129,13 +129,13 @@ TEST(EnvoyQuicUtilsTest, HeadersSizeBounds) {
   NiceMock validator;
   quic::QuicRstStreamErrorCode rst = quic::QUIC_REFUSED_STREAM;
   EXPECT_NE(nullptr, http2HeaderBlockToEnvoyTrailers(
-                         headers_block, 6, validator, details, rst));
+                         headers_block, 60, 6, validator, details, rst));
   // Given the cap is 6, make sure anything lower, exact or otherwise, is rejected.
   EXPECT_EQ(nullptr, http2HeaderBlockToEnvoyTrailers(
-                         headers_block, 5, validator, details, rst));
+                         headers_block, 60, 5, validator, details, rst));
   EXPECT_EQ("http3.too_many_trailers", details);
   EXPECT_EQ(nullptr, http2HeaderBlockToEnvoyTrailers(
-                         headers_block, 4, validator, details, rst));
+                         headers_block, 60, 4, validator, details, rst));
   EXPECT_EQ(rst, quic::QUIC_STREAM_EXCESSIVE_LOAD);
 }
 
@@ -149,12 +149,12 @@ TEST(EnvoyQuicUtilsTest, TrailersSizeBounds) {
   NiceMock validator;
   quic::QuicRstStreamErrorCode rst = quic::QUIC_REFUSED_STREAM;
   EXPECT_NE(nullptr, http2HeaderBlockToEnvoyTrailers(
-                         headers_block, 6, validator, details, rst));
+                         headers_block, 60, 6, validator, details, rst));
   EXPECT_EQ(nullptr, http2HeaderBlockToEnvoyTrailers(
-                         headers_block, 2, validator, details, rst));
+                         headers_block, 60, 2, validator, details, rst));
   EXPECT_EQ("http3.too_many_trailers", details);
   EXPECT_EQ(nullptr, http2HeaderBlockToEnvoyTrailers(
-                         headers_block, 2, validator, details, rst));
+                         headers_block, 60, 2, validator, details, rst));
   EXPECT_EQ(rst, quic::QUIC_STREAM_EXCESSIVE_LOAD);
 }
 
@@ -169,7 +169,7 @@ TEST(EnvoyQuicUtilsTest, TrailerCharacters) {
       .WillRepeatedly(Return(Http::HeaderUtility::HeaderValidationResult::REJECT));
   quic::QuicRstStreamErrorCode rst = quic::QUIC_REFUSED_STREAM;
   EXPECT_EQ(nullptr, http2HeaderBlockToEnvoyTrailers(
-                         headers_block, 5, validator, details, rst));
+                         headers_block, 60, 5, validator, details, rst));
   EXPECT_EQ(rst, quic::QUIC_BAD_APPLICATION_PAYLOAD);
 }
 
@@ -200,5 +200,49 @@ TEST(EnvoyQuicUtilsTest, ConvertQuicConfig) {
   EXPECT_EQ(3, quic_config.GetInitialMaxStreamDataBytesIncomingBidirectionalToSend());
 }
 
+TEST(EnvoyQuicUtilsTest, HeaderMapMaxSizeLimit) {
+  NiceMock validator;
+  absl::string_view details;
+  quic::QuicRstStreamErrorCode rst = quic::QUIC_REFUSED_STREAM;
+  quic::QuicHeaderList quic_headers;
+  quic_headers.OnHeaderBlockStart();
+  quic_headers.OnHeader(":authority", "www.google.com");
+  quic_headers.OnHeader(":path", "/index.hml");
+  quic_headers.OnHeader(":scheme", "https");
+  quic_headers.OnHeaderBlockEnd(0, 0);
+  EXPECT_CALL(validator, validateHeader(_, _))
+      .WillRepeatedly([](absl::string_view, absl::string_view) {
+        return Http::HeaderUtility::HeaderValidationResult::ACCEPT;
+      });
+  // Request header map test.
+  auto request_header = quicHeadersToEnvoyHeaders(
+      quic_headers, validator, 60, 100, details, rst);
+  EXPECT_EQ(request_header->maxHeadersCount(), 100);
+  EXPECT_EQ(request_header->maxHeadersKb(), 60);
+
+  // Response header map test.
+  auto response_header = quicHeadersToEnvoyHeaders(
+      quic_headers, validator, 60, 100, details, rst);
+  EXPECT_EQ(response_header->maxHeadersCount(), 100);
+  EXPECT_EQ(response_header->maxHeadersKb(), 60);
+
+  spdy::Http2HeaderBlock headers_block;
+  headers_block[":authority"] = "www.google.com";
+  headers_block[":path"] = "/index.hml";
+  headers_block[":scheme"] = "https";
+
+  // Request trailer map test.
+  auto request_trailer = http2HeaderBlockToEnvoyTrailers(
+      headers_block, 60, 100, validator, details, rst);
+  EXPECT_EQ(request_trailer->maxHeadersCount(), 100);
+  EXPECT_EQ(request_trailer->maxHeadersKb(), 60);
+
+  // Response trailer map test.
+  auto response_trailer = http2HeaderBlockToEnvoyTrailers(
+      headers_block, 60, 100, validator, details, rst);
+  EXPECT_EQ(response_trailer->maxHeadersCount(), 100);
+  EXPECT_EQ(response_trailer->maxHeadersKb(), 60);
+}
+
 } // namespace Quic
 } // namespace Envoy
diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h
index adac56934c96..846d20a8f494 100644
--- a/test/mocks/http/mocks.h
+++ b/test/mocks/http/mocks.h
@@ -907,6 +907,27 @@ MATCHER_P(HeaderMapEqual, rhs, "") {
   return equal;
 }
 
+MATCHER_P(HeaderMapEqualWithMaxSize, rhs, "") {
+  bool equal = (*arg == *rhs);
+
+  // Check the max header count and size of the HeaderMap also equal.
+  if (equal) {
+    if (arg->maxHeadersCount() != rhs->maxHeadersCount() ||
+        arg->maxHeadersKb() != rhs->maxHeadersKb()) {
+      equal = false;
+    }
+  }
+
+  if (!equal) {
+    *result_listener << "\n"
+                     << TestUtility::addLeftAndRightPadding("header map:") << "\n"
+                     << *rhs << TestUtility::addLeftAndRightPadding("is not equal to:") << "\n"
+                     << *arg << TestUtility::addLeftAndRightPadding("") // line full of padding
+                     << "\n";
+  }
+  return equal;
+}
+
 MATCHER_P(HeaderMapEqualRef, rhs, "") {
   const bool equal = (arg == *rhs);
   if (!equal) {
diff --git a/test/test_common/utility.h b/test/test_common/utility.h
index b4aed8a24080..acaa1b68c508 100644
--- a/test/test_common/utility.h
+++ b/test/test_common/utility.h
@@ -948,6 +948,19 @@ template  class TestHeaderMapImplBase : public Inte
     }
     header_map_->verifyByteSizeInternalForTest();
   }
+  TestHeaderMapImplBase(const std::initializer_list>& values,
+                        const uint32_t max_headers_kb, const uint32_t max_headers_count) {
+    if (header_map_) {
+      header_map_.reset();
+    }
+    header_map_ = Impl::create(max_headers_kb, max_headers_count);
+
+    for (auto& value : values) {
+      header_map_->addCopy(LowerCaseString(value.first), value.second);
+    }
+    header_map_->verifyByteSizeInternalForTest();
+  }
+
   TestHeaderMapImplBase(const TestHeaderMapImplBase& rhs)
       : TestHeaderMapImplBase(*rhs.header_map_) {}
   TestHeaderMapImplBase(const HeaderMap& rhs) {
@@ -1030,6 +1043,8 @@ template  class TestHeaderMapImplBase : public Inte
     header_map_->verifyByteSizeInternalForTest();
   }
   uint64_t byteSize() const override { return header_map_->byteSize(); }
+  uint32_t maxHeadersKb() const override { return header_map_->maxHeadersKb(); }
+  uint32_t maxHeadersCount() const override { return header_map_->maxHeadersCount(); }
   HeaderMap::GetResult get(const LowerCaseString& key) const override {
     return header_map_->get(key);
   }

From cacd32bc9e3285f6eee7efbc3fcb3e7291975055 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 1 Jul 2023 16:22:19 +0100
Subject: [PATCH 692/740] build(deps): bump jaegertracing/all-in-one from
 `9c75fc2` to `0bfdc7a` in /examples/shared/jaeger (#28198)

build(deps): bump jaegertracing/all-in-one in /examples/shared/jaeger

Bumps jaegertracing/all-in-one from `9c75fc2` to `0bfdc7a`.

---
updated-dependencies:
- dependency-name: jaegertracing/all-in-one
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/shared/jaeger/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile
index 315c0bc1fff1..02112e6ee7fe 100644
--- a/examples/shared/jaeger/Dockerfile
+++ b/examples/shared/jaeger/Dockerfile
@@ -1,4 +1,4 @@
-FROM jaegertracing/all-in-one@sha256:9c75fc21ad3691e5bb882672e38c3bd3eacd9b97b8ad8239e60a6fc578bef66a
+FROM jaegertracing/all-in-one@sha256:0bfdc7a447c44330c97088ff634daaa718368de80b08fab0eea5a19e8f3a043d
 HEALTHCHECK \
     --interval=1s \
     --timeout=1s \

From 4e51ba92fe9effd943f68568762c3b213ec818bd Mon Sep 17 00:00:00 2001
From: Joshua Marantz 
Date: Mon, 3 Jul 2023 16:40:58 -0400
Subject: [PATCH 693/740] admin: Histogram html use summary-buckets mode by
 default (#28203)

The detailed histograms may be too detailed for the default HTML view. Instead, respect the buckets-mode, and only use detailed mode if this was requested by the user.

Also, fix an accidental script-driven change (#27987) to a JS file that made it invalid JS, breaking active_html mode.

Signed-off-by: Joshua Marantz 
---
 source/server/admin/html/active_stats.js    |  4 +--
 source/server/admin/stats_html_render.cc    | 20 ++++++++++--
 source/server/admin/stats_html_render.h     |  9 +-----
 test/server/admin/stats_html_render_test.cc | 34 ++++++++++++---------
 4 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/source/server/admin/html/active_stats.js b/source/server/admin/html/active_stats.js
index 0e05bebfef1e..ca6bf6723981 100644
--- a/source/server/admin/html/active_stats.js
+++ b/source/server/admin/html/active_stats.js
@@ -97,11 +97,11 @@ async function loadStats() {
   const url = prefix + '/stats?format=json&usedonly&histogram_buckets=detailed&' +
         params.map(makeQueryParam).join('&');
 
-  TRY_NEEDS_AUDIT {
+  try {
     const response = await fetch(url);
     const data = await response.json();
     renderStats(data);
-  } END_TRY catch (e) {
+  } catch (e) {
     statusDiv.textContent = 'Error fetching ' + url + ': ' + e;
   }
 
diff --git a/source/server/admin/stats_html_render.cc b/source/server/admin/stats_html_render.cc
index 52bbbbfa1108..15e79e6b8443 100644
--- a/source/server/admin/stats_html_render.cc
+++ b/source/server/admin/stats_html_render.cc
@@ -23,7 +23,7 @@ StatsHtmlRender::StatsHtmlRender(Http::ResponseHeaderMap& response_headers,
                                  Buffer::Instance& response, const StatsParams& params)
     : StatsTextRender(params), active_(params.format_ == StatsFormat::ActiveHtml) {
   AdminHtmlUtil::renderHead(response_headers, response);
-  if (!active_) {
+  if (!active_ && params.histogram_buckets_mode_ == Utility::HistogramBucketsMode::Detailed) {
     StatsParams json_params(params);
     json_params.histogram_buckets_mode_ = Utility::HistogramBucketsMode::Detailed;
     json_response_headers_ = Http::ResponseHeaderMapImpl::create();
@@ -35,7 +35,7 @@ StatsHtmlRender::StatsHtmlRender(Http::ResponseHeaderMap& response_headers,
 void StatsHtmlRender::finalize(Buffer::Instance& response) {
   // Render all the histograms here using the JSON data we've accumulated
   // for them.
-  if (!active_) {
+  if (histogram_json_render_ != nullptr) {
     histogram_json_render_->finalize(json_data_);
     response.add("
\n
\n